diff --git a/common/lz4_armv8_acceleration.patch b/common/lz4_armv8_acceleration.patch new file mode 100644 index 00000000..600cda57 --- /dev/null +++ b/common/lz4_armv8_acceleration.patch @@ -0,0 +1,11238 @@ +commit 6e136e8d10039fd02b20515413b95bb547ae8ff3 +Author: DogEZ +Date: Sat Oct 11 23:55:42 2025 +0800 + + lib/lz4: introduce lz4 1.10.0 with armv8 acceleration + +diff --git a/crypto/lz4.c b/crypto/lz4.c +index 0606f8862e78..650c2cefcbb9 100644 +--- a/crypto/lz4.c ++++ b/crypto/lz4.c +@@ -81,7 +81,13 @@ static int lz4_compress_crypto(struct crypto_tfm *tfm, const u8 *src, + static int __lz4_decompress_crypto(const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *ctx) + { +- int out_len = LZ4_decompress_safe(src, dst, slen, *dlen); ++ int out_len; ++ ++#if defined(CONFIG_ARM64) && defined(CONFIG_KERNEL_MODE_NEON) ++ out_len = LZ4_arm64_decompress_safe(src, dst, slen, *dlen, false); ++#else ++ out_len = LZ4_decompress_safe(src, dst, slen, *dlen); ++#endif + + if (out_len < 0) + return -EINVAL; +diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c +index d7cc94aa2fcf..e562d642feb5 100644 +--- a/crypto/lz4hc.c ++++ b/crypto/lz4hc.c +@@ -82,7 +82,13 @@ static int lz4hc_compress_crypto(struct crypto_tfm *tfm, const u8 *src, + static int __lz4hc_decompress_crypto(const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *ctx) + { +- int out_len = LZ4_decompress_safe(src, dst, slen, *dlen); ++ int out_len; ++ ++#if defined(CONFIG_ARM64) && defined(CONFIG_KERNEL_MODE_NEON) ++ out_len = LZ4_arm64_decompress_safe(src, dst, slen, *dlen, false); ++#else ++ out_len = LZ4_decompress_safe(src, dst, slen, *dlen); ++#endif + + if (out_len < 0) + return -EINVAL; +diff --git a/fs/f2fs/Makefile b/fs/f2fs/Makefile +index 183c96303f58..8a7322d229e4 100644 +--- a/fs/f2fs/Makefile ++++ b/fs/f2fs/Makefile +@@ -10,6 +10,3 @@ f2fs-$(CONFIG_F2FS_FS_POSIX_ACL) += acl.o + f2fs-$(CONFIG_FS_VERITY) += verity.o + f2fs-$(CONFIG_F2FS_FS_COMPRESSION) += compress.o + f2fs-$(CONFIG_F2FS_IOSTAT) += iostat.o +-ifeq ($(CONFIG_F2FS_FS_COMPRESSION_FIXED_OUTPUT),y) +-f2fs-$(CONFIG_ARM64) += $(addprefix lz4armv8/, lz4accel.o lz4armv8.o) +-endif +diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c +index 8106b6167abd..2ddc1cea78ae 100644 +--- a/fs/f2fs/compress.c ++++ b/fs/f2fs/compress.c +@@ -19,7 +19,6 @@ + #include "segment.h" + #include + #if defined(CONFIG_F2FS_FS_COMPRESSION_FIXED_OUTPUT) || defined(__ARCH_HAS_LZ4_ACCELERATOR) +-#include "lz4armv8/lz4accel.h" + #include "f2fs_lz4.h" + #endif + +@@ -355,8 +354,13 @@ static int lz4_decompress_pages(struct decompress_io_ctx *dic) + + if (f2fs_compress_layout(dic->inode) == COMPRESS_FIXED_INPUT) { + expected = PAGE_SIZE << dic->log_cluster_size; ++#if defined(CONFIG_ARM64) && defined(CONFIG_KERNEL_MODE_NEON) ++ ret = LZ4_arm64_decompress_safe(dic->cbuf->cdata, dic->rbuf, ++ dic->clen, dic->rlen, false); ++#else + ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf, +- dic->clen, dic->rlen); ++ dic->clen, dic->rlen, false); ++#endif + #ifdef CONFIG_F2FS_FS_COMPRESSION_FIXED_OUTPUT + } else { + uint8_t *dst = (uint8_t *)dic->rbuf + dic->rofs; +diff --git a/fs/incfs/data_mgmt.c b/fs/incfs/data_mgmt.c +index 5abb20a1098c..613e7919ba37 100644 +--- a/fs/incfs/data_mgmt.c ++++ b/fs/incfs/data_mgmt.c +@@ -472,8 +472,11 @@ static ssize_t decompress(struct mount_info *mi, + + switch (alg) { + case INCFS_BLOCK_COMPRESSED_LZ4: +- result = LZ4_decompress_safe(src.data, dst.data, src.len, +- dst.len); ++#if defined(CONFIG_ARM64) && defined(CONFIG_KERNEL_MODE_NEON) ++ result = LZ4_arm64_decompress_safe(src.data, dst.data, src.len, dst.len, false); ++#else ++ result = LZ4_decompress_safe(src.data, dst.data, src.len, dst.len); ++#endif + if (result < 0) + return -EBADMSG; + return result; +diff --git a/include/linux/lz4.h b/include/linux/lz4.h +index b16e15b9587a..025b2a0cc125 100644 +--- a/include/linux/lz4.h ++++ b/include/linux/lz4.h +@@ -1,648 +1,17 @@ +-/* LZ4 Kernel Interface +- * +- * Copyright (C) 2013, LG Electronics, Kyungsik Lee +- * Copyright (C) 2016, Sven Schmidt <4sschmid@informatik.uni-hamburg.de> +- * +- * This program is free software; you can redistribute it and/or modify +- * it under the terms of the GNU General Public License version 2 as +- * published by the Free Software Foundation. +- * +- * This file is based on the original header file +- * for LZ4 - Fast LZ compression algorithm. +- * +- * LZ4 - Fast LZ compression algorithm +- * Copyright (C) 2011-2016, Yann Collet. +- * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) +- * Redistribution and use in source and binary forms, with or without +- * modification, are permitted provided that the following conditions are +- * met: +- * * Redistributions of source code must retain the above copyright +- * notice, this list of conditions and the following disclaimer. +- * * Redistributions in binary form must reproduce the above +- * copyright notice, this list of conditions and the following disclaimer +- * in the documentation and/or other materials provided with the +- * distribution. +- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +- * You can contact the author at : +- * - LZ4 homepage : http://www.lz4.org +- * - LZ4 source repository : https://github.com/lz4/lz4 +- */ ++/* SPDX-License-Identifier: BSD-2-Clause */ ++// LZ4 compatibility wrapper for Linux kernel + +-#ifndef __LZ4_H__ +-#define __LZ4_H__ ++#ifndef __LINUX_LZ4_H__ ++#define __LINUX_LZ4_H__ + +-#include +-#include /* memset, memcpy */ ++#include "../../lib/lz4/lz4.h" ++#include "../../lib/lz4/lz4hc.h" + +-/*-************************************************************************ +- * CONSTANTS +- **************************************************************************/ +-/* +- * LZ4_MEMORY_USAGE : +- * Memory usage formula : N->2^N Bytes +- * (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +- * Increasing memory usage improves compression ratio +- * Reduced memory usage can improve speed, due to cache effect +- * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache +- */ +-#define LZ4_MEMORY_USAGE 14 ++#define LZ4_MEM_COMPRESS LZ4_STREAM_MINSIZE ++#define LZ4HC_MEM_COMPRESS LZ4_STREAMHC_MINSIZE + +-#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +-#define LZ4_COMPRESSBOUND(isize) (\ +- (unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE \ +- ? 0 \ +- : (isize) + ((isize)/255) + 16) +- +-#define LZ4_ACCELERATION_DEFAULT 1 +-#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) +-#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) +-#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) +- +-#define LZ4HC_MIN_CLEVEL 3 +-#define LZ4HC_DEFAULT_CLEVEL 9 +-#define LZ4HC_MAX_CLEVEL 16 +- +-#define LZ4HC_DICTIONARY_LOGSIZE 16 +-#define LZ4HC_MAXD (1<= LZ4_compressBound(inputSize). +- * It also runs faster, so it's a recommended setting. +- * If the function cannot compress 'source' into a more limited 'dest' budget, +- * compression stops *immediately*, and the function result is zero. +- * As a consequence, 'dest' content is not valid. +- * +- * Return: Number of bytes written into buffer 'dest' +- * (necessarily <= maxOutputSize) or 0 if compression fails +- */ +-int LZ4_compress_default(const char *source, char *dest, int inputSize, +- int maxOutputSize, void *wrkmem); +- +-/** +- * LZ4_compress_fast() - As LZ4_compress_default providing an acceleration param +- * @source: source address of the original data +- * @dest: output buffer address of the compressed data +- * @inputSize: size of the input data. Max supported value is LZ4_MAX_INPUT_SIZE +- * @maxOutputSize: full or partial size of buffer 'dest' +- * which must be already allocated +- * @acceleration: acceleration factor +- * @wrkmem: address of the working memory. +- * This requires 'workmem' of LZ4_MEM_COMPRESS. +- * +- * Same as LZ4_compress_default(), but allows to select an "acceleration" +- * factor. The larger the acceleration value, the faster the algorithm, +- * but also the lesser the compression. It's a trade-off. It can be fine tuned, +- * with each successive value providing roughly +~3% to speed. +- * An acceleration value of "1" is the same as regular LZ4_compress_default() +- * Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT, which is 1. +- * +- * Return: Number of bytes written into buffer 'dest' +- * (necessarily <= maxOutputSize) or 0 if compression fails +- */ +-int LZ4_compress_fast(const char *source, char *dest, int inputSize, +- int maxOutputSize, int acceleration, void *wrkmem); +- +-/** +- * LZ4_compress_destSize() - Compress as much data as possible +- * from source to dest +- * @source: source address of the original data +- * @dest: output buffer address of the compressed data +- * @sourceSizePtr: will be modified to indicate how many bytes where read +- * from 'source' to fill 'dest'. New value is necessarily <= old value. +- * @targetDestSize: Size of buffer 'dest' which must be already allocated +- * @wrkmem: address of the working memory. +- * This requires 'workmem' of LZ4_MEM_COMPRESS. +- * +- * Reverse the logic, by compressing as much data as possible +- * from 'source' buffer into already allocated buffer 'dest' +- * of size 'targetDestSize'. +- * This function either compresses the entire 'source' content into 'dest' +- * if it's large enough, or fill 'dest' buffer completely with as much data as +- * possible from 'source'. +- * +- * Return: Number of bytes written into 'dest' (necessarily <= targetDestSize) +- * or 0 if compression fails +- */ +-int LZ4_compress_destSize(const char *source, char *dest, int *sourceSizePtr, +- int targetDestSize, void *wrkmem); +- +-/*-************************************************************************ +- * Decompression Functions +- **************************************************************************/ +- +-/** +- * LZ4_decompress_fast() - Decompresses data from 'source' into 'dest' +- * @source: source address of the compressed data +- * @dest: output buffer address of the uncompressed data +- * which must be already allocated with 'originalSize' bytes +- * @originalSize: is the original and therefore uncompressed size +- * +- * Decompresses data from 'source' into 'dest'. +- * This function fully respect memory boundaries for properly formed +- * compressed data. +- * It is a bit faster than LZ4_decompress_safe(). +- * However, it does not provide any protection against intentionally +- * modified data stream (malicious input). +- * Use this function in trusted environment only +- * (data to decode comes from a trusted source). +- * +- * Return: number of bytes read from the source buffer +- * or a negative result if decompression fails. +- */ +-int LZ4_decompress_fast(const char *source, char *dest, int originalSize); +- +-/** +- * LZ4_decompress_safe() - Decompression protected against buffer overflow +- * @source: source address of the compressed data +- * @dest: output buffer address of the uncompressed data +- * which must be already allocated +- * @compressedSize: is the precise full size of the compressed block +- * @maxDecompressedSize: is the size of 'dest' buffer +- * +- * Decompresses data from 'source' into 'dest'. +- * If the source stream is detected malformed, the function will +- * stop decoding and return a negative result. +- * This function is protected against buffer overflow exploits, +- * including malicious data packets. It never writes outside output buffer, +- * nor reads outside input buffer. +- * +- * Return: number of bytes decompressed into destination buffer +- * (necessarily <= maxDecompressedSize) +- * or a negative result in case of error +- */ +-int LZ4_decompress_safe(const char *source, char *dest, int compressedSize, +- int maxDecompressedSize); +- +-/** +- * LZ4_decompress_safe_partial() - Decompress a block of size 'compressedSize' +- * at position 'source' into buffer 'dest' +- * @source: source address of the compressed data +- * @dest: output buffer address of the decompressed data which must be +- * already allocated +- * @compressedSize: is the precise full size of the compressed block. +- * @targetOutputSize: the decompression operation will try +- * to stop as soon as 'targetOutputSize' has been reached +- * @maxDecompressedSize: is the size of destination buffer +- * +- * This function decompresses a compressed block of size 'compressedSize' +- * at position 'source' into destination buffer 'dest' +- * of size 'maxDecompressedSize'. +- * The function tries to stop decompressing operation as soon as +- * 'targetOutputSize' has been reached, reducing decompression time. +- * This function never writes outside of output buffer, +- * and never reads outside of input buffer. +- * It is therefore protected against malicious data packets. +- * +- * Return: the number of bytes decoded in the destination buffer +- * (necessarily <= maxDecompressedSize) +- * or a negative result in case of error +- * +- */ +-int LZ4_decompress_safe_partial(const char *source, char *dest, +- int compressedSize, int targetOutputSize, int maxDecompressedSize); +- +-/*-************************************************************************ +- * LZ4 HC Compression +- **************************************************************************/ +- +-/** +- * LZ4_compress_HC() - Compress data from `src` into `dst`, using HC algorithm +- * @src: source address of the original data +- * @dst: output buffer address of the compressed data +- * @srcSize: size of the input data. Max supported value is LZ4_MAX_INPUT_SIZE +- * @dstCapacity: full or partial size of buffer 'dst', +- * which must be already allocated +- * @compressionLevel: Recommended values are between 4 and 9, although any +- * value between 1 and LZ4HC_MAX_CLEVEL will work. +- * Values >LZ4HC_MAX_CLEVEL behave the same as 16. +- * @wrkmem: address of the working memory. +- * This requires 'wrkmem' of size LZ4HC_MEM_COMPRESS. +- * +- * Compress data from 'src' into 'dst', using the more powerful +- * but slower "HC" algorithm. Compression is guaranteed to succeed if +- * `dstCapacity >= LZ4_compressBound(srcSize) +- * +- * Return : the number of bytes written into 'dst' or 0 if compression fails. +- */ +-int LZ4_compress_HC(const char *src, char *dst, int srcSize, int dstCapacity, +- int compressionLevel, void *wrkmem); +- +-/** +- * LZ4_resetStreamHC() - Init an allocated 'LZ4_streamHC_t' structure +- * @streamHCPtr: pointer to the 'LZ4_streamHC_t' structure +- * @compressionLevel: Recommended values are between 4 and 9, although any +- * value between 1 and LZ4HC_MAX_CLEVEL will work. +- * Values >LZ4HC_MAX_CLEVEL behave the same as 16. +- * +- * An LZ4_streamHC_t structure can be allocated once +- * and re-used multiple times. +- * Use this function to init an allocated `LZ4_streamHC_t` structure +- * and start a new compression. +- */ +-void LZ4_resetStreamHC(LZ4_streamHC_t *streamHCPtr, int compressionLevel); +- +-/** +- * LZ4_loadDictHC() - Load a static dictionary into LZ4_streamHC +- * @streamHCPtr: pointer to the LZ4HC_stream_t +- * @dictionary: dictionary to load +- * @dictSize: size of dictionary +- * +- * Use this function to load a static dictionary into LZ4HC_stream. +- * Any previous data will be forgotten, only 'dictionary' +- * will remain in memory. +- * Loading a size of 0 is allowed. +- * +- * Return : dictionary size, in bytes (necessarily <= 64 KB) +- */ +-int LZ4_loadDictHC(LZ4_streamHC_t *streamHCPtr, const char *dictionary, +- int dictSize); +- +-/** +- * LZ4_compress_HC_continue() - Compress 'src' using data from previously +- * compressed blocks as a dictionary using the HC algorithm +- * @streamHCPtr: Pointer to the previous 'LZ4_streamHC_t' structure +- * @src: source address of the original data +- * @dst: output buffer address of the compressed data, +- * which must be already allocated +- * @srcSize: size of the input data. Max supported value is LZ4_MAX_INPUT_SIZE +- * @maxDstSize: full or partial size of buffer 'dest' +- * which must be already allocated +- * +- * These functions compress data in successive blocks of any size, using +- * previous blocks as dictionary. One key assumption is that previous +- * blocks (up to 64 KB) remain read-accessible while +- * compressing next blocks. There is an exception for ring buffers, +- * which can be smaller than 64 KB. +- * Ring buffers scenario is automatically detected and handled by +- * LZ4_compress_HC_continue(). +- * Before starting compression, state must be properly initialized, +- * using LZ4_resetStreamHC(). +- * A first "fictional block" can then be designated as +- * initial dictionary, using LZ4_loadDictHC() (Optional). +- * Then, use LZ4_compress_HC_continue() +- * to compress each successive block. Previous memory blocks +- * (including initial dictionary when present) must remain accessible +- * and unmodified during compression. +- * 'dst' buffer should be sized to handle worst case scenarios, using +- * LZ4_compressBound(), to ensure operation success. +- * If, for any reason, previous data blocks can't be preserved unmodified +- * in memory during next compression block, +- * you must save it to a safer memory space, using LZ4_saveDictHC(). +- * Return value of LZ4_saveDictHC() is the size of dictionary +- * effectively saved into 'safeBuffer'. +- * +- * Return: Number of bytes written into buffer 'dst' or 0 if compression fails +- */ +-int LZ4_compress_HC_continue(LZ4_streamHC_t *streamHCPtr, const char *src, +- char *dst, int srcSize, int maxDstSize); +- +-/** +- * LZ4_saveDictHC() - Save static dictionary from LZ4HC_stream +- * @streamHCPtr: pointer to the 'LZ4HC_stream_t' structure +- * @safeBuffer: buffer to save dictionary to, must be already allocated +- * @maxDictSize: size of 'safeBuffer' +- * +- * If previously compressed data block is not guaranteed +- * to remain available at its memory location, +- * save it into a safer place (char *safeBuffer). +- * Note : you don't need to call LZ4_loadDictHC() afterwards, +- * dictionary is immediately usable, you can therefore call +- * LZ4_compress_HC_continue(). +- * +- * Return : saved dictionary size in bytes (necessarily <= maxDictSize), +- * or 0 if error. +- */ +-int LZ4_saveDictHC(LZ4_streamHC_t *streamHCPtr, char *safeBuffer, +- int maxDictSize); +- +-/*-********************************************* +- * Streaming Compression Functions +- ***********************************************/ +- +-/** +- * LZ4_resetStream() - Init an allocated 'LZ4_stream_t' structure +- * @LZ4_stream: pointer to the 'LZ4_stream_t' structure +- * +- * An LZ4_stream_t structure can be allocated once +- * and re-used multiple times. +- * Use this function to init an allocated `LZ4_stream_t` structure +- * and start a new compression. +- */ +-void LZ4_resetStream(LZ4_stream_t *LZ4_stream); +- +-/** +- * LZ4_loadDict() - Load a static dictionary into LZ4_stream +- * @streamPtr: pointer to the LZ4_stream_t +- * @dictionary: dictionary to load +- * @dictSize: size of dictionary +- * +- * Use this function to load a static dictionary into LZ4_stream. +- * Any previous data will be forgotten, only 'dictionary' +- * will remain in memory. +- * Loading a size of 0 is allowed. +- * +- * Return : dictionary size, in bytes (necessarily <= 64 KB) +- */ +-int LZ4_loadDict(LZ4_stream_t *streamPtr, const char *dictionary, +- int dictSize); +- +-/** +- * LZ4_saveDict() - Save static dictionary from LZ4_stream +- * @streamPtr: pointer to the 'LZ4_stream_t' structure +- * @safeBuffer: buffer to save dictionary to, must be already allocated +- * @dictSize: size of 'safeBuffer' +- * +- * If previously compressed data block is not guaranteed +- * to remain available at its memory location, +- * save it into a safer place (char *safeBuffer). +- * Note : you don't need to call LZ4_loadDict() afterwards, +- * dictionary is immediately usable, you can therefore call +- * LZ4_compress_fast_continue(). +- * +- * Return : saved dictionary size in bytes (necessarily <= dictSize), +- * or 0 if error. +- */ +-int LZ4_saveDict(LZ4_stream_t *streamPtr, char *safeBuffer, int dictSize); +- +-/** +- * LZ4_compress_fast_continue() - Compress 'src' using data from previously +- * compressed blocks as a dictionary +- * @streamPtr: Pointer to the previous 'LZ4_stream_t' structure +- * @src: source address of the original data +- * @dst: output buffer address of the compressed data, +- * which must be already allocated +- * @srcSize: size of the input data. Max supported value is LZ4_MAX_INPUT_SIZE +- * @maxDstSize: full or partial size of buffer 'dest' +- * which must be already allocated +- * @acceleration: acceleration factor +- * +- * Compress buffer content 'src', using data from previously compressed blocks +- * as dictionary to improve compression ratio. +- * Important : Previous data blocks are assumed to still +- * be present and unmodified ! +- * If maxDstSize >= LZ4_compressBound(srcSize), +- * compression is guaranteed to succeed, and runs faster. +- * +- * Return: Number of bytes written into buffer 'dst' or 0 if compression fails +- */ +-int LZ4_compress_fast_continue(LZ4_stream_t *streamPtr, const char *src, +- char *dst, int srcSize, int maxDstSize, int acceleration); +- +-/** +- * LZ4_setStreamDecode() - Instruct where to find dictionary +- * @LZ4_streamDecode: the 'LZ4_streamDecode_t' structure +- * @dictionary: dictionary to use +- * @dictSize: size of dictionary +- * +- * Use this function to instruct where to find the dictionary. +- * Setting a size of 0 is allowed (same effect as reset). +- * +- * Return: 1 if OK, 0 if error +- */ +-int LZ4_setStreamDecode(LZ4_streamDecode_t *LZ4_streamDecode, +- const char *dictionary, int dictSize); +- +-/** +- * LZ4_decompress_safe_continue() - Decompress blocks in streaming mode +- * @LZ4_streamDecode: the 'LZ4_streamDecode_t' structure +- * @source: source address of the compressed data +- * @dest: output buffer address of the uncompressed data +- * which must be already allocated +- * @compressedSize: is the precise full size of the compressed block +- * @maxDecompressedSize: is the size of 'dest' buffer +- * +- * This decoding function allows decompression of multiple blocks +- * in "streaming" mode. +- * Previously decoded blocks *must* remain available at the memory position +- * where they were decoded (up to 64 KB) +- * In the case of a ring buffers, decoding buffer must be either : +- * - Exactly same size as encoding buffer, with same update rule +- * (block boundaries at same positions) In which case, +- * the decoding & encoding ring buffer can have any size, +- * including very small ones ( < 64 KB). +- * - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. +- * maxBlockSize is implementation dependent. +- * It's the maximum size you intend to compress into a single block. +- * In which case, encoding and decoding buffers do not need +- * to be synchronized, and encoding ring buffer can have any size, +- * including small ones ( < 64 KB). +- * - _At least_ 64 KB + 8 bytes + maxBlockSize. +- * In which case, encoding and decoding buffers do not need to be +- * synchronized, and encoding ring buffer can have any size, +- * including larger than decoding buffer. W +- * Whenever these conditions are not possible, save the last 64KB of decoded +- * data into a safe buffer, and indicate where it is saved +- * using LZ4_setStreamDecode() +- * +- * Return: number of bytes decompressed into destination buffer +- * (necessarily <= maxDecompressedSize) +- * or a negative result in case of error +- */ +-int LZ4_decompress_safe_continue(LZ4_streamDecode_t *LZ4_streamDecode, +- const char *source, char *dest, int compressedSize, +- int maxDecompressedSize); +- +-/** +- * LZ4_decompress_fast_continue() - Decompress blocks in streaming mode +- * @LZ4_streamDecode: the 'LZ4_streamDecode_t' structure +- * @source: source address of the compressed data +- * @dest: output buffer address of the uncompressed data +- * which must be already allocated with 'originalSize' bytes +- * @originalSize: is the original and therefore uncompressed size +- * +- * This decoding function allows decompression of multiple blocks +- * in "streaming" mode. +- * Previously decoded blocks *must* remain available at the memory position +- * where they were decoded (up to 64 KB) +- * In the case of a ring buffers, decoding buffer must be either : +- * - Exactly same size as encoding buffer, with same update rule +- * (block boundaries at same positions) In which case, +- * the decoding & encoding ring buffer can have any size, +- * including very small ones ( < 64 KB). +- * - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. +- * maxBlockSize is implementation dependent. +- * It's the maximum size you intend to compress into a single block. +- * In which case, encoding and decoding buffers do not need +- * to be synchronized, and encoding ring buffer can have any size, +- * including small ones ( < 64 KB). +- * - _At least_ 64 KB + 8 bytes + maxBlockSize. +- * In which case, encoding and decoding buffers do not need to be +- * synchronized, and encoding ring buffer can have any size, +- * including larger than decoding buffer. W +- * Whenever these conditions are not possible, save the last 64KB of decoded +- * data into a safe buffer, and indicate where it is saved +- * using LZ4_setStreamDecode() +- * +- * Return: number of bytes decompressed into destination buffer +- * (necessarily <= maxDecompressedSize) +- * or a negative result in case of error +- */ +-int LZ4_decompress_fast_continue(LZ4_streamDecode_t *LZ4_streamDecode, +- const char *source, char *dest, int originalSize); +- +-/** +- * LZ4_decompress_safe_usingDict() - Same as LZ4_setStreamDecode() +- * followed by LZ4_decompress_safe_continue() +- * @source: source address of the compressed data +- * @dest: output buffer address of the uncompressed data +- * which must be already allocated +- * @compressedSize: is the precise full size of the compressed block +- * @maxDecompressedSize: is the size of 'dest' buffer +- * @dictStart: pointer to the start of the dictionary in memory +- * @dictSize: size of dictionary +- * +- * This decoding function works the same as +- * a combination of LZ4_setStreamDecode() followed by +- * LZ4_decompress_safe_continue() +- * It is stand-alone, and doesn't need an LZ4_streamDecode_t structure. +- * +- * Return: number of bytes decompressed into destination buffer +- * (necessarily <= maxDecompressedSize) +- * or a negative result in case of error +- */ +-int LZ4_decompress_safe_usingDict(const char *source, char *dest, +- int compressedSize, int maxDecompressedSize, const char *dictStart, +- int dictSize); +- +-/** +- * LZ4_decompress_fast_usingDict() - Same as LZ4_setStreamDecode() +- * followed by LZ4_decompress_fast_continue() +- * @source: source address of the compressed data +- * @dest: output buffer address of the uncompressed data +- * which must be already allocated with 'originalSize' bytes +- * @originalSize: is the original and therefore uncompressed size +- * @dictStart: pointer to the start of the dictionary in memory +- * @dictSize: size of dictionary +- * +- * This decoding function works the same as +- * a combination of LZ4_setStreamDecode() followed by +- * LZ4_decompress_fast_continue() +- * It is stand-alone, and doesn't need an LZ4_streamDecode_t structure. +- * +- * Return: number of bytes decompressed into destination buffer +- * (necessarily <= maxDecompressedSize) +- * or a negative result in case of error +- */ +-int LZ4_decompress_fast_usingDict(const char *source, char *dest, +- int originalSize, const char *dictStart, int dictSize); ++#define LZ4HC_MIN_CLEVEL LZ4HC_CLEVEL_MIN ++#define LZ4HC_DEFAULT_CLEVEL LZ4HC_CLEVEL_DEFAULT ++#define LZ4HC_MAX_CLEVEL LZ4HC_CLEVEL_MAX + + #endif +diff --git a/lib/lz4/Makefile b/lib/lz4/Makefile +index 5b42242afaa2..dc6445d23080 100644 +--- a/lib/lz4/Makefile ++++ b/lib/lz4/Makefile +@@ -1,6 +1,8 @@ + # SPDX-License-Identifier: GPL-2.0-only +-ccflags-y += -O3 ++ccflags-y += -O3 \ ++ -DLZ4_FREESTANDING=1 \ ++ -DLZ4_FAST_DEC_LOOP=1 + +-obj-$(CONFIG_LZ4_COMPRESS) += lz4_compress.o +-obj-$(CONFIG_LZ4HC_COMPRESS) += lz4hc_compress.o +-obj-$(CONFIG_LZ4_DECOMPRESS) += lz4_decompress.o ++obj-y += lz4.o lz4hc.o ++ ++obj-$(CONFIG_ARM64) += $(addprefix lz4armv8/, lz4accel.o lz4armv8.o) +diff --git a/lib/lz4/lz4.c b/lib/lz4/lz4.c +new file mode 100644 +index 000000000000..112edb700469 +--- /dev/null ++++ b/lib/lz4/lz4.c +@@ -0,0 +1,3484 @@ ++/* ++ LZ4 - Fast LZ compression algorithm ++ Copyright (C) 2011-2023, Yann Collet. ++ ++ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) ++ ++ Redistribution and use in source and binary forms, with or without ++ modification, are permitted provided that the following conditions are ++ met: ++ ++ * Redistributions of source code must retain the above copyright ++ notice, this list of conditions and the following disclaimer. ++ * Redistributions in binary form must reproduce the above ++ copyright notice, this list of conditions and the following disclaimer ++ in the documentation and/or other materials provided with the ++ distribution. ++ ++ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ++ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ++ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ++ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ++ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ++ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ++ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ++ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ ++ You can contact the author at : ++ - LZ4 homepage : http://www.lz4.org ++ - LZ4 source repository : https://github.com/lz4/lz4 ++*/ ++ ++/*-************************************ ++* Tuning parameters ++**************************************/ ++/* ++ * LZ4_HEAPMODE : ++ * Select how stateless compression functions like `LZ4_compress_default()` ++ * allocate memory for their hash table, ++ * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). ++ */ ++#ifndef LZ4_HEAPMODE ++#define LZ4_HEAPMODE 1 ++#endif ++ ++/*-************************************ ++* CPU Feature Detection ++**************************************/ ++/* LZ4_FORCE_MEMORY_ACCESS ++ * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. ++ * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. ++ * The below switch allow to select different access method for improved performance. ++ * Method 0 (default) : use `memcpy()`. Safe and portable. ++ * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). ++ * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. ++ * Method 2 : direct access. This method is portable but violate C standard. ++ * It can generate buggy code on targets which assembly generation depends on alignment. ++ * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) ++ * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. ++ * Prefer these methods in priority order (0 > 1 > 2) ++ */ ++#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ ++#if defined(__GNUC__) && \ ++ (defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || \ ++ defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || \ ++ defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__)) ++#define LZ4_FORCE_MEMORY_ACCESS 2 ++#elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__) || \ ++ defined(_MSC_VER) ++#define LZ4_FORCE_MEMORY_ACCESS 1 ++#endif ++#endif ++ ++/* ++ * LZ4_FORCE_SW_BITCOUNT ++ * Define this parameter if your target system or compiler does not support hardware bit count ++ */ ++#if defined(_MSC_VER) && \ ++ defined(_WIN32_WCE) /* Visual Studio for WinCE doesn't support Hardware bit count */ ++#undef LZ4_FORCE_SW_BITCOUNT /* avoid double def */ ++#define LZ4_FORCE_SW_BITCOUNT ++#endif ++ ++/*-************************************ ++* Dependency ++**************************************/ ++/* ++ * LZ4_SRC_INCLUDED: ++ * Amalgamation flag, whether lz4.c is included ++ */ ++#ifndef LZ4_SRC_INCLUDED ++#define LZ4_SRC_INCLUDED 1 ++#endif ++ ++#ifndef LZ4_DISABLE_DEPRECATE_WARNINGS ++#define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */ ++#endif ++ ++#ifndef LZ4_STATIC_LINKING_ONLY ++#define LZ4_STATIC_LINKING_ONLY ++#endif ++#include "lz4.h" ++/* see also "memory routines" below */ ++ ++/*-************************************ ++* Compiler Options ++**************************************/ ++#if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Visual Studio 2005+ */ ++#include /* only present in VS2005+ */ ++#pragma warning( \ ++ disable : 4127) /* disable: C4127: conditional expression is constant */ ++#pragma warning( \ ++ disable : 6237) /* disable: C6237: conditional expression is always 0 */ ++#pragma warning( \ ++ disable : 6239) /* disable: C6239: ( && ) always evaluates to the result of */ ++#pragma warning( \ ++ disable : 6240) /* disable: C6240: ( && ) always evaluates to the result of */ ++#pragma warning( \ ++ disable : 6326) /* disable: C6326: Potential comparison of a constant with another constant */ ++#endif /* _MSC_VER */ ++ ++#ifndef LZ4_FORCE_INLINE ++#if defined(_MSC_VER) && !defined(__clang__) /* MSVC */ ++#define LZ4_FORCE_INLINE static __forceinline ++#else ++#if defined(__cplusplus) || \ ++ defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ ++#if defined(__GNUC__) || defined(__clang__) ++#define LZ4_FORCE_INLINE static inline __attribute__((always_inline)) ++#else ++#define LZ4_FORCE_INLINE static inline ++#endif ++#else ++#define LZ4_FORCE_INLINE static ++#endif /* __STDC_VERSION__ */ ++#endif /* _MSC_VER */ ++#endif /* LZ4_FORCE_INLINE */ ++ ++/* LZ4_FORCE_O2 and LZ4_FORCE_INLINE ++ * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8, ++ * together with a simple 8-byte copy loop as a fall-back path. ++ * However, this optimization hurts the decompression speed by >30%, ++ * because the execution does not go to the optimized loop ++ * for typical compressible data, and all of the preamble checks ++ * before going to the fall-back path become useless overhead. ++ * This optimization happens only with the -O3 flag, and -O2 generates ++ * a simple 8-byte copy loop. ++ * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy8 ++ * functions are annotated with __attribute__((optimize("O2"))), ++ * and also LZ4_wildCopy8 is forcibly inlined, so that the O2 attribute ++ * of LZ4_wildCopy8 does not affect the compression speed. ++ */ ++#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && \ ++ !defined(__clang__) ++#define LZ4_FORCE_O2 __attribute__((optimize("O2"))) ++#undef LZ4_FORCE_INLINE ++#define LZ4_FORCE_INLINE \ ++ static __inline __attribute__((optimize("O2"), always_inline)) ++#else ++#define LZ4_FORCE_O2 ++#endif ++ ++#if (defined(__GNUC__) && (__GNUC__ >= 3)) || \ ++ (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || \ ++ defined(__clang__) ++#define expect(expr, value) (__builtin_expect((expr), (value))) ++#else ++#define expect(expr, value) (expr) ++#endif ++ ++/* Should the alignment test prove unreliable, for some reason, ++ * it can be disabled by setting LZ4_ALIGN_TEST to 0 */ ++#ifndef LZ4_ALIGN_TEST /* can be externally provided */ ++#define LZ4_ALIGN_TEST 1 ++#endif ++ ++/*-************************************ ++* Memory routines ++**************************************/ ++ ++#if !LZ4_FREESTANDING ++#include /* memset, memcpy */ ++#endif ++#if !defined(LZ4_memset) ++#define LZ4_memset(p, v, s) memset((p), (v), (s)) ++#endif ++#define MEM_INIT(p, v, s) LZ4_memset((p), (v), (s)) ++ ++/*-************************************ ++* Common Constants ++**************************************/ ++#define MINMATCH 4 ++ ++#define WILDCOPYLENGTH 8 ++#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ ++#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ ++#define MATCH_SAFEGUARD_DISTANCE \ ++ ((2 * WILDCOPYLENGTH) - \ ++ MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */ ++#define FASTLOOP_SAFE_DISTANCE 64 ++static const int LZ4_minLength = (MFLIMIT + 1); ++ ++#define KB *(1 << 10) ++#define MB *(1 << 20) ++#define GB *(1U << 30) ++ ++#define LZ4_DISTANCE_ABSOLUTE_MAX 65535 ++#if (LZ4_DISTANCE_MAX > \ ++ LZ4_DISTANCE_ABSOLUTE_MAX) /* max supported by LZ4 format */ ++#error "LZ4_DISTANCE_MAX is too big : must be <= 65535" ++#endif ++ ++#define ML_BITS 4 ++#define ML_MASK ((1U << ML_BITS) - 1) ++#define RUN_BITS (8 - ML_BITS) ++#define RUN_MASK ((1U << RUN_BITS) - 1) ++ ++/*-************************************ ++* Error detection ++**************************************/ ++#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 1) ++#include ++#else ++#ifndef assert ++#define assert(condition) ((void)0) ++#endif ++#endif ++ ++#define LZ4_STATIC_ASSERT(c) \ ++ { \ ++ enum { LZ4_static_assert = 1 / (int)(!!(c)) }; \ ++ } /* use after variable declarations */ ++ ++#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 2) ++#include ++static int g_debuglog_enable = 1; ++#define DEBUGLOG(l, ...) \ ++ { \ ++ if ((g_debuglog_enable) && (l <= LZ4_DEBUG)) { \ ++ fprintf(stderr, __FILE__ " %i: ", __LINE__); \ ++ fprintf(stderr, __VA_ARGS__); \ ++ fprintf(stderr, " \n"); \ ++ } \ ++ } ++#else ++#define DEBUGLOG(l, ...) \ ++ { \ ++ } /* disabled */ ++#endif ++ ++static int LZ4_isAligned(const void *ptr, size_t alignment) ++{ ++ return ((size_t)ptr & (alignment - 1)) == 0; ++} ++ ++/*-************************************ ++* Types ++**************************************/ ++#include ++typedef uint8_t BYTE; ++typedef uint16_t U16; ++typedef uint32_t U32; ++typedef int32_t S32; ++typedef uint64_t U64; ++typedef uintptr_t uptrval; ++ ++#if defined(__x86_64__) ++typedef U64 reg_t; /* 64-bits in x32 mode */ ++#else ++typedef size_t reg_t; /* 32-bits in x32 mode */ ++#endif ++ ++typedef enum { ++ notLimited = 0, ++ limitedOutput = 1, ++ fillOutput = 2 ++} limitedOutput_directive; ++ ++static unsigned LZ4_isLittleEndian(void) ++{ ++ const union { ++ U32 u; ++ BYTE c[4]; ++ } one = { 1 }; /* don't use static : performance detrimental */ ++ return one.c[0]; ++} ++ ++#if defined(__GNUC__) || defined(__INTEL_COMPILER) ++#define LZ4_PACK(__Declaration__) __Declaration__ __attribute__((__packed__)) ++#elif defined(_MSC_VER) ++#define LZ4_PACK(__Declaration__) \ ++ __pragma(pack(push, 1)) __Declaration__ __pragma(pack(pop)) ++#endif ++ ++#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS == 2) ++/* lie to the compiler about data alignment; use with caution */ ++ ++static U16 LZ4_read16(const void *memPtr) ++{ ++ return *(const U16 *)memPtr; ++} ++static U32 LZ4_read32(const void *memPtr) ++{ ++ return *(const U32 *)memPtr; ++} ++static reg_t LZ4_read_ARCH(const void *memPtr) ++{ ++ return *(const reg_t *)memPtr; ++} ++ ++static void LZ4_write16(void *memPtr, U16 value) ++{ ++ *(U16 *)memPtr = value; ++} ++static void LZ4_write32(void *memPtr, U32 value) ++{ ++ *(U32 *)memPtr = value; ++} ++ ++#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS == 1) ++ ++/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ ++/* currently only defined for gcc and icc */ ++LZ4_PACK(typedef struct { U16 u16; }) LZ4_unalign16; ++LZ4_PACK(typedef struct { U32 u32; }) LZ4_unalign32; ++LZ4_PACK(typedef struct { reg_t uArch; }) LZ4_unalignST; ++ ++static U16 LZ4_read16(const void *ptr) ++{ ++ return ((const LZ4_unalign16 *)ptr)->u16; ++} ++static U32 LZ4_read32(const void *ptr) ++{ ++ return ((const LZ4_unalign32 *)ptr)->u32; ++} ++static reg_t LZ4_read_ARCH(const void *ptr) ++{ ++ return ((const LZ4_unalignST *)ptr)->uArch; ++} ++ ++static void LZ4_write16(void *memPtr, U16 value) ++{ ++ ((LZ4_unalign16 *)memPtr)->u16 = value; ++} ++static void LZ4_write32(void *memPtr, U32 value) ++{ ++ ((LZ4_unalign32 *)memPtr)->u32 = value; ++} ++ ++#else /* safe and portable access using memcpy() */ ++ ++static U16 LZ4_read16(const void *memPtr) ++{ ++ U16 val; ++ LZ4_memcpy(&val, memPtr, sizeof(val)); ++ return val; ++} ++ ++static U32 LZ4_read32(const void *memPtr) ++{ ++ U32 val; ++ LZ4_memcpy(&val, memPtr, sizeof(val)); ++ return val; ++} ++ ++static reg_t LZ4_read_ARCH(const void *memPtr) ++{ ++ reg_t val; ++ LZ4_memcpy(&val, memPtr, sizeof(val)); ++ return val; ++} ++ ++static void LZ4_write16(void *memPtr, U16 value) ++{ ++ LZ4_memcpy(memPtr, &value, sizeof(value)); ++} ++ ++static void LZ4_write32(void *memPtr, U32 value) ++{ ++ LZ4_memcpy(memPtr, &value, sizeof(value)); ++} ++ ++#endif /* LZ4_FORCE_MEMORY_ACCESS */ ++ ++static U16 LZ4_readLE16(const void *memPtr) ++{ ++ if (LZ4_isLittleEndian()) { ++ return LZ4_read16(memPtr); ++ } else { ++ const BYTE *p = (const BYTE *)memPtr; ++ return (U16)((U16)p[0] | (p[1] << 8)); ++ } ++} ++ ++#ifdef LZ4_STATIC_LINKING_ONLY_ENDIANNESS_INDEPENDENT_OUTPUT ++static U32 LZ4_readLE32(const void *memPtr) ++{ ++ if (LZ4_isLittleEndian()) { ++ return LZ4_read32(memPtr); ++ } else { ++ const BYTE *p = (const BYTE *)memPtr; ++ return (U32)p[0] | (p[1] << 8) | (p[2] << 16) | (p[3] << 24); ++ } ++} ++#endif ++ ++static void LZ4_writeLE16(void *memPtr, U16 value) ++{ ++ if (LZ4_isLittleEndian()) { ++ LZ4_write16(memPtr, value); ++ } else { ++ BYTE *p = (BYTE *)memPtr; ++ p[0] = (BYTE)value; ++ p[1] = (BYTE)(value >> 8); ++ } ++} ++ ++/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ ++LZ4_FORCE_INLINE ++void LZ4_wildCopy8(void *dstPtr, const void *srcPtr, void *dstEnd) ++{ ++ BYTE *d = (BYTE *)dstPtr; ++ const BYTE *s = (const BYTE *)srcPtr; ++ BYTE *const e = (BYTE *)dstEnd; ++ ++ do { ++ LZ4_memcpy(d, s, 8); ++ d += 8; ++ s += 8; ++ } while (d < e); ++} ++ ++static const unsigned inc32table[8] = { 0, 1, 2, 1, 0, 4, 4, 4 }; ++static const int dec64table[8] = { 0, 0, 0, -1, -4, 1, 2, 3 }; ++ ++#ifndef LZ4_FAST_DEC_LOOP ++#if defined __i386__ || defined _M_IX86 || defined __x86_64__ || defined _M_X64 ++#define LZ4_FAST_DEC_LOOP 1 ++#elif defined(__aarch64__) && defined(__APPLE__) ++#define LZ4_FAST_DEC_LOOP 1 ++#elif defined(__aarch64__) && !defined(__clang__) ++/* On non-Apple aarch64, we disable this optimization for clang because ++ * on certain mobile chipsets, performance is reduced with clang. For ++ * more information refer to https://github.com/lz4/lz4/pull/707 */ ++#define LZ4_FAST_DEC_LOOP 1 ++#else ++#define LZ4_FAST_DEC_LOOP 0 ++#endif ++#endif ++ ++#if LZ4_FAST_DEC_LOOP ++ ++LZ4_FORCE_INLINE void LZ4_memcpy_using_offset_base(BYTE *dstPtr, ++ const BYTE *srcPtr, ++ BYTE *dstEnd, ++ const size_t offset) ++{ ++ assert(srcPtr + offset == dstPtr); ++ if (offset < 8) { ++ LZ4_write32(dstPtr, ++ 0); /* silence an msan warning when offset==0 */ ++ dstPtr[0] = srcPtr[0]; ++ dstPtr[1] = srcPtr[1]; ++ dstPtr[2] = srcPtr[2]; ++ dstPtr[3] = srcPtr[3]; ++ srcPtr += inc32table[offset]; ++ LZ4_memcpy(dstPtr + 4, srcPtr, 4); ++ srcPtr -= dec64table[offset]; ++ dstPtr += 8; ++ } else { ++ LZ4_memcpy(dstPtr, srcPtr, 8); ++ dstPtr += 8; ++ srcPtr += 8; ++ } ++ ++ LZ4_wildCopy8(dstPtr, srcPtr, dstEnd); ++} ++ ++/* customized variant of memcpy, which can overwrite up to 32 bytes beyond dstEnd ++ * this version copies two times 16 bytes (instead of one time 32 bytes) ++ * because it must be compatible with offsets >= 16. */ ++LZ4_FORCE_INLINE void LZ4_wildCopy32(void *dstPtr, const void *srcPtr, ++ void *dstEnd) ++{ ++ BYTE *d = (BYTE *)dstPtr; ++ const BYTE *s = (const BYTE *)srcPtr; ++ BYTE *const e = (BYTE *)dstEnd; ++ ++ do { ++ LZ4_memcpy(d, s, 16); ++ LZ4_memcpy(d + 16, s + 16, 16); ++ d += 32; ++ s += 32; ++ } while (d < e); ++} ++ ++/* LZ4_memcpy_using_offset() presumes : ++ * - dstEnd >= dstPtr + MINMATCH ++ * - there is at least 12 bytes available to write after dstEnd */ ++LZ4_FORCE_INLINE void LZ4_memcpy_using_offset(BYTE *dstPtr, const BYTE *srcPtr, ++ BYTE *dstEnd, const size_t offset) ++{ ++ BYTE v[8]; ++ ++ assert(dstEnd >= dstPtr + MINMATCH); ++ ++ switch (offset) { ++ case 1: ++ MEM_INIT(v, *srcPtr, 8); ++ break; ++ case 2: ++ LZ4_memcpy(v, srcPtr, 2); ++ LZ4_memcpy(&v[2], srcPtr, 2); ++#if defined(_MSC_VER) && (_MSC_VER <= 1937) /* MSVC 2022 ver 17.7 or earlier */ ++#pragma warning(push) ++#pragma warning( \ ++ disable : 6385) /* warning C6385: Reading invalid data from 'v'. */ ++#endif ++ LZ4_memcpy(&v[4], v, 4); ++#if defined(_MSC_VER) && (_MSC_VER <= 1937) /* MSVC 2022 ver 17.7 or earlier */ ++#pragma warning(pop) ++#endif ++ break; ++ case 4: ++ LZ4_memcpy(v, srcPtr, 4); ++ LZ4_memcpy(&v[4], srcPtr, 4); ++ break; ++ default: ++ LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset); ++ return; ++ } ++ ++ LZ4_memcpy(dstPtr, v, 8); ++ dstPtr += 8; ++ while (dstPtr < dstEnd) { ++ LZ4_memcpy(dstPtr, v, 8); ++ dstPtr += 8; ++ } ++} ++#endif ++ ++/*-************************************ ++* Common functions ++**************************************/ ++static unsigned LZ4_NbCommonBytes(reg_t val) ++{ ++ assert(val != 0); ++ if (LZ4_isLittleEndian()) { ++ if (sizeof(val) == 8) { ++#if defined(_MSC_VER) && (_MSC_VER >= 1800) && \ ++ (defined(_M_AMD64) && !defined(_M_ARM64EC)) && \ ++ !defined(LZ4_FORCE_SW_BITCOUNT) ++/*-************************************************************************************************* ++* ARM64EC is a Microsoft-designed ARM64 ABI compatible with AMD64 applications on ARM64 Windows 11. ++* The ARM64EC ABI does not support AVX/AVX2/AVX512 instructions, nor their relevant intrinsics ++* including _tzcnt_u64. Therefore, we need to neuter the _tzcnt_u64 code path for ARM64EC. ++****************************************************************************************************/ ++#if defined(__clang__) && (__clang_major__ < 10) ++ /* Avoid undefined clang-cl intrinsics issue. ++ * See https://github.com/lz4/lz4/pull/1017 for details. */ ++ return (unsigned)__builtin_ia32_tzcnt_u64(val) >> 3; ++#else ++ /* x64 CPUS without BMI support interpret `TZCNT` as `REP BSF` */ ++ return (unsigned)_tzcnt_u64(val) >> 3; ++#endif ++#elif defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) ++ unsigned long r = 0; ++ _BitScanForward64(&r, (U64)val); ++ return (unsigned)r >> 3; ++#elif (defined(__clang__) || \ ++ (defined(__GNUC__) && \ ++ ((__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ ++ !defined(LZ4_FORCE_SW_BITCOUNT) ++ return (unsigned)__builtin_ctzll((U64)val) >> 3; ++#else ++ const U64 m = 0x0101010101010101ULL; ++ val ^= val - 1; ++ return (unsigned)(((U64)((val & (m - 1)) * m)) >> 56); ++#endif ++ } else /* 32 bits */ { ++#if defined(_MSC_VER) && (_MSC_VER >= 1400) && !defined(LZ4_FORCE_SW_BITCOUNT) ++ unsigned long r; ++ _BitScanForward(&r, (U32)val); ++ return (unsigned)r >> 3; ++#elif (defined(__clang__) || \ ++ (defined(__GNUC__) && \ ++ ((__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ ++ !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) ++ return (unsigned)__builtin_ctz((U32)val) >> 3; ++#else ++ const U32 m = 0x01010101; ++ return (unsigned)((((val - 1) ^ val) & (m - 1)) * m) >> ++ 24; ++#endif ++ } ++ } else /* Big Endian CPU */ { ++ if (sizeof(val) == 8) { ++#if (defined(__clang__) || \ ++ (defined(__GNUC__) && \ ++ ((__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ ++ !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) ++ return (unsigned)__builtin_clzll((U64)val) >> 3; ++#else ++#if 1 ++ /* this method is probably faster, ++ * but adds a 128 bytes lookup table */ ++ static const unsigned char ctz7_tab[128] = { ++ 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ }; ++ U64 const mask = 0x0101010101010101ULL; ++ U64 const t = (((val >> 8) - mask) | val) & mask; ++ return ctz7_tab[(t * 0x0080402010080402ULL) >> 57]; ++#else ++ /* this method doesn't consume memory space like the previous one, ++ * but it contains several branches, ++ * that may end up slowing execution */ ++ static const U32 by32 = ++ sizeof(val) * ++ 4; /* 32 on 64 bits (goal), 16 on 32 bits. ++ Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. ++ Note that this code path is never triggered in 32-bits mode. */ ++ unsigned r; ++ if (!(val >> by32)) { ++ r = 4; ++ } else { ++ r = 0; ++ val >>= by32; ++ } ++ if (!(val >> 16)) { ++ r += 2; ++ val >>= 8; ++ } else { ++ val >>= 24; ++ } ++ r += (!val); ++ return r; ++#endif ++#endif ++ } else /* 32 bits */ { ++#if (defined(__clang__) || \ ++ (defined(__GNUC__) && \ ++ ((__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ ++ !defined(LZ4_FORCE_SW_BITCOUNT) ++ return (unsigned)__builtin_clz((U32)val) >> 3; ++#else ++ val >>= 8; ++ val = ((((val + 0x00FFFF00) | 0x00FFFFFF) + val) | ++ (val + 0x00FF0000)) >> ++ 24; ++ return (unsigned)val ^ 3; ++#endif ++ } ++ } ++} ++ ++#define STEPSIZE sizeof(reg_t) ++LZ4_FORCE_INLINE ++unsigned LZ4_count(const BYTE *pIn, const BYTE *pMatch, const BYTE *pInLimit) ++{ ++ const BYTE *const pStart = pIn; ++ ++ if (likely(pIn < pInLimit - (STEPSIZE - 1))) { ++ reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); ++ if (!diff) { ++ pIn += STEPSIZE; ++ pMatch += STEPSIZE; ++ } else { ++ return LZ4_NbCommonBytes(diff); ++ } ++ } ++ ++ while (likely(pIn < pInLimit - (STEPSIZE - 1))) { ++ reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); ++ if (!diff) { ++ pIn += STEPSIZE; ++ pMatch += STEPSIZE; ++ continue; ++ } ++ pIn += LZ4_NbCommonBytes(diff); ++ return (unsigned)(pIn - pStart); ++ } ++ ++ if ((STEPSIZE == 8) && (pIn < (pInLimit - 3)) && ++ (LZ4_read32(pMatch) == LZ4_read32(pIn))) { ++ pIn += 4; ++ pMatch += 4; ++ } ++ if ((pIn < (pInLimit - 1)) && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { ++ pIn += 2; ++ pMatch += 2; ++ } ++ if ((pIn < pInLimit) && (*pMatch == *pIn)) ++ pIn++; ++ return (unsigned)(pIn - pStart); ++} ++ ++#ifndef LZ4_COMMONDEFS_ONLY ++/*-************************************ ++* Local Constants ++**************************************/ ++static const int LZ4_64Klimit = ((64 KB) + (MFLIMIT - 1)); ++static const U32 LZ4_skipTrigger = ++ 6; /* Increase this value ==> compression run slower on incompressible data */ ++ ++/*-************************************ ++* Local Structures and types ++**************************************/ ++typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; ++ ++/** ++ * This enum distinguishes several different modes of accessing previous ++ * content in the stream. ++ * ++ * - noDict : There is no preceding content. ++ * - withPrefix64k : Table entries up to ctx->dictSize before the current blob ++ * blob being compressed are valid and refer to the preceding ++ * content (of length ctx->dictSize), which is available ++ * contiguously preceding in memory the content currently ++ * being compressed. ++ * - usingExtDict : Like withPrefix64k, but the preceding content is somewhere ++ * else in memory, starting at ctx->dictionary with length ++ * ctx->dictSize. ++ * - usingDictCtx : Everything concerning the preceding content is ++ * in a separate context, pointed to by ctx->dictCtx. ++ * ctx->dictionary, ctx->dictSize, and table entries ++ * in the current context that refer to positions ++ * preceding the beginning of the current compression are ++ * ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx ++ * ->dictSize describe the location and size of the preceding ++ * content, and matches are found by looking in the ctx ++ * ->dictCtx->hashTable. ++ */ ++typedef enum { ++ noDict = 0, ++ withPrefix64k, ++ usingExtDict, ++ usingDictCtx ++} dict_directive; ++typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; ++ ++/*-************************************ ++* Local Utils ++**************************************/ ++int LZ4_versionNumber(void) ++{ ++ return LZ4_VERSION_NUMBER; ++} ++const char *LZ4_versionString(void) ++{ ++ return LZ4_VERSION_STRING; ++} ++int LZ4_compressBound(int isize) ++{ ++ return LZ4_COMPRESSBOUND(isize); ++} ++int LZ4_sizeofState(void) ++{ ++ return sizeof(LZ4_stream_t); ++} ++ ++/*-**************************************** ++* Internal Definitions, used only in Tests ++*******************************************/ ++#if defined(__cplusplus) ++extern "C" { ++#endif ++ ++int LZ4_compress_forceExtDict(LZ4_stream_t *LZ4_dict, const char *source, ++ char *dest, int srcSize); ++ ++int LZ4_decompress_safe_forceExtDict(const char *source, char *dest, ++ int compressedSize, int maxOutputSize, ++ const void *dictStart, size_t dictSize); ++int LZ4_decompress_safe_partial_forceExtDict(const char *source, char *dest, ++ int compressedSize, ++ int targetOutputSize, ++ int dstCapacity, ++ const void *dictStart, ++ size_t dictSize); ++#if defined(__cplusplus) ++} ++#endif ++ ++/*-****************************** ++* Compression functions ++********************************/ ++LZ4_FORCE_INLINE U32 LZ4_hash4(U32 sequence, tableType_t const tableType) ++{ ++ if (tableType == byU16) ++ return ((sequence * 2654435761U) >> ++ ((MINMATCH * 8) - (LZ4_HASHLOG + 1))); ++ else ++ return ((sequence * 2654435761U) >> ++ ((MINMATCH * 8) - LZ4_HASHLOG)); ++} ++ ++LZ4_FORCE_INLINE U32 LZ4_hash5(U64 sequence, tableType_t const tableType) ++{ ++ const U32 hashLog = ++ (tableType == byU16) ? LZ4_HASHLOG + 1 : LZ4_HASHLOG; ++ if (LZ4_isLittleEndian()) { ++ const U64 prime5bytes = 889523592379ULL; ++ return (U32)(((sequence << 24) * prime5bytes) >> ++ (64 - hashLog)); ++ } else { ++ const U64 prime8bytes = 11400714785074694791ULL; ++ return (U32)(((sequence >> 24) * prime8bytes) >> ++ (64 - hashLog)); ++ } ++} ++ ++LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void *const p, ++ tableType_t const tableType) ++{ ++ if ((sizeof(reg_t) == 8) && (tableType != byU16)) ++ return LZ4_hash5(LZ4_read_ARCH(p), tableType); ++ ++#ifdef LZ4_STATIC_LINKING_ONLY_ENDIANNESS_INDEPENDENT_OUTPUT ++ return LZ4_hash4(LZ4_readLE32(p), tableType); ++#else ++ return LZ4_hash4(LZ4_read32(p), tableType); ++#endif ++} ++ ++LZ4_FORCE_INLINE void LZ4_clearHash(U32 h, void *tableBase, ++ tableType_t const tableType) ++{ ++ switch (tableType) { ++ default: /* fallthrough */ ++ case clearedTable: { /* illegal! */ ++ assert(0); ++ return; ++ } ++ case byPtr: { ++ const BYTE **hashTable = (const BYTE **)tableBase; ++ hashTable[h] = NULL; ++ return; ++ } ++ case byU32: { ++ U32 *hashTable = (U32 *)tableBase; ++ hashTable[h] = 0; ++ return; ++ } ++ case byU16: { ++ U16 *hashTable = (U16 *)tableBase; ++ hashTable[h] = 0; ++ return; ++ } ++ } ++} ++ ++LZ4_FORCE_INLINE void LZ4_putIndexOnHash(U32 idx, U32 h, void *tableBase, ++ tableType_t const tableType) ++{ ++ switch (tableType) { ++ default: /* fallthrough */ ++ case clearedTable: /* fallthrough */ ++ case byPtr: { /* illegal! */ ++ assert(0); ++ return; ++ } ++ case byU32: { ++ U32 *hashTable = (U32 *)tableBase; ++ hashTable[h] = idx; ++ return; ++ } ++ case byU16: { ++ U16 *hashTable = (U16 *)tableBase; ++ assert(idx < 65536); ++ hashTable[h] = (U16)idx; ++ return; ++ } ++ } ++} ++ ++/* LZ4_putPosition*() : only used in byPtr mode */ ++LZ4_FORCE_INLINE void LZ4_putPositionOnHash(const BYTE *p, U32 h, ++ void *tableBase, ++ tableType_t const tableType) ++{ ++ const BYTE **const hashTable = (const BYTE **)tableBase; ++ assert(tableType == byPtr); ++ (void)tableType; ++ hashTable[h] = p; ++} ++ ++LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE *p, void *tableBase, ++ tableType_t tableType) ++{ ++ U32 const h = LZ4_hashPosition(p, tableType); ++ LZ4_putPositionOnHash(p, h, tableBase, tableType); ++} ++ ++/* LZ4_getIndexOnHash() : ++ * Index of match position registered in hash table. ++ * hash position must be calculated by using base+index, or dictBase+index. ++ * Assumption 1 : only valid if tableType == byU32 or byU16. ++ * Assumption 2 : h is presumed valid (within limits of hash table) ++ */ ++LZ4_FORCE_INLINE U32 LZ4_getIndexOnHash(U32 h, const void *tableBase, ++ tableType_t tableType) ++{ ++ LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2); ++ if (tableType == byU32) { ++ const U32 *const hashTable = (const U32 *)tableBase; ++ assert(h < (1U << (LZ4_MEMORY_USAGE - 2))); ++ return hashTable[h]; ++ } ++ if (tableType == byU16) { ++ const U16 *const hashTable = (const U16 *)tableBase; ++ assert(h < (1U << (LZ4_MEMORY_USAGE - 1))); ++ return hashTable[h]; ++ } ++ assert(0); ++ return 0; /* forbidden case */ ++} ++ ++static const BYTE *LZ4_getPositionOnHash(U32 h, const void *tableBase, ++ tableType_t tableType) ++{ ++ assert(tableType == byPtr); ++ (void)tableType; ++ { ++ const BYTE *const *hashTable = (const BYTE *const *)tableBase; ++ return hashTable[h]; ++ } ++} ++ ++LZ4_FORCE_INLINE const BYTE * ++LZ4_getPosition(const BYTE *p, const void *tableBase, tableType_t tableType) ++{ ++ U32 const h = LZ4_hashPosition(p, tableType); ++ return LZ4_getPositionOnHash(h, tableBase, tableType); ++} ++ ++LZ4_FORCE_INLINE void LZ4_prepareTable(LZ4_stream_t_internal *const cctx, ++ const int inputSize, ++ const tableType_t tableType) ++{ ++ /* If the table hasn't been used, it's guaranteed to be zeroed out, and is ++ * therefore safe to use no matter what mode we're in. Otherwise, we figure ++ * out if it's safe to leave as is or whether it needs to be reset. ++ */ ++ if ((tableType_t)cctx->tableType != clearedTable) { ++ assert(inputSize >= 0); ++ if ((tableType_t)cctx->tableType != tableType || ++ ((tableType == byU16) && ++ cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU) || ++ ((tableType == byU32) && cctx->currentOffset > 1 GB) || ++ tableType == byPtr || inputSize >= 4 KB) { ++ DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", ++ cctx); ++ MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE); ++ cctx->currentOffset = 0; ++ cctx->tableType = (U32)clearedTable; ++ } else { ++ DEBUGLOG( ++ 4, ++ "LZ4_prepareTable: Re-use hash table (no reset)"); ++ } ++ } ++ ++ /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, ++ * is faster than compressing without a gap. ++ * However, compressing with currentOffset == 0 is faster still, ++ * so we preserve that case. ++ */ ++ if (cctx->currentOffset != 0 && tableType == byU32) { ++ DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset"); ++ cctx->currentOffset += 64 KB; ++ } ++ ++ /* Finally, clear history */ ++ cctx->dictCtx = NULL; ++ cctx->dictionary = NULL; ++ cctx->dictSize = 0; ++} ++ ++/** LZ4_compress_generic_validated() : ++ * inlined, to ensure branches are decided at compilation time. ++ * The following conditions are presumed already validated: ++ * - source != NULL ++ * - inputSize > 0 ++ */ ++LZ4_FORCE_INLINE int LZ4_compress_generic_validated( ++ LZ4_stream_t_internal *const cctx, const char *const source, ++ char *const dest, const int inputSize, ++ int *inputConsumed, /* only written when outputDirective == fillOutput */ ++ const int maxOutputSize, const limitedOutput_directive outputDirective, ++ const tableType_t tableType, const dict_directive dictDirective, ++ const dictIssue_directive dictIssue, const int acceleration) ++{ ++ int result; ++ const BYTE *ip = (const BYTE *)source; ++ ++ U32 const startIndex = cctx->currentOffset; ++ const BYTE *base = (const BYTE *)source - startIndex; ++ const BYTE *lowLimit; ++ ++ const LZ4_stream_t_internal *dictCtx = ++ (const LZ4_stream_t_internal *)cctx->dictCtx; ++ const BYTE *const dictionary = dictDirective == usingDictCtx ? ++ dictCtx->dictionary : ++ cctx->dictionary; ++ const U32 dictSize = dictDirective == usingDictCtx ? dictCtx->dictSize : ++ cctx->dictSize; ++ const U32 dictDelta = ++ (dictDirective == usingDictCtx) ? ++ startIndex - dictCtx->currentOffset : ++ 0; /* make indexes in dictCtx comparable with indexes in current context */ ++ ++ int const maybe_extMem = (dictDirective == usingExtDict) || ++ (dictDirective == usingDictCtx); ++ U32 const prefixIdxLimit = ++ startIndex - ++ dictSize; /* used when dictDirective == dictSmall */ ++ const BYTE *const dictEnd = ++ dictionary ? dictionary + dictSize : dictionary; ++ const BYTE *anchor = (const BYTE *)source; ++ const BYTE *const iend = ip + inputSize; ++ const BYTE *const mflimitPlusOne = iend - MFLIMIT + 1; ++ const BYTE *const matchlimit = iend - LASTLITERALS; ++ ++ /* the dictCtx currentOffset is indexed on the start of the dictionary, ++ * while a dictionary in the current context precedes the currentOffset */ ++ const BYTE *dictBase = ++ (dictionary == NULL) ? ++ NULL : ++ (dictDirective == usingDictCtx) ? ++ dictionary + dictSize - dictCtx->currentOffset : ++ dictionary + dictSize - startIndex; ++ ++ BYTE *op = (BYTE *)dest; ++ BYTE *const olimit = op + maxOutputSize; ++ ++ U32 offset = 0; ++ U32 forwardH; ++ ++ DEBUGLOG(5, "LZ4_compress_generic_validated: srcSize=%i, tableType=%u", ++ inputSize, tableType); ++ assert(ip != NULL); ++ if (tableType == byU16) ++ assert(inputSize < ++ LZ4_64Klimit); /* Size too large (not within 64K limit) */ ++ if (tableType == byPtr) ++ assert(dictDirective == ++ noDict); /* only supported use case with byPtr */ ++ /* If init conditions are not met, we don't have to mark stream ++ * as having dirty context, since no action was taken yet */ ++ if (outputDirective == fillOutput && maxOutputSize < 1) { ++ return 0; ++ } /* Impossible to store anything */ ++ assert(acceleration >= 1); ++ ++ lowLimit = (const BYTE *)source - ++ (dictDirective == withPrefix64k ? dictSize : 0); ++ ++ /* Update context state */ ++ if (dictDirective == usingDictCtx) { ++ /* Subsequent linked blocks can't use the dictionary. */ ++ /* Instead, they use the block we just compressed. */ ++ cctx->dictCtx = NULL; ++ cctx->dictSize = (U32)inputSize; ++ } else { ++ cctx->dictSize += (U32)inputSize; ++ } ++ cctx->currentOffset += (U32)inputSize; ++ cctx->tableType = (U32)tableType; ++ ++ if (inputSize < LZ4_minLength) ++ goto _last_literals; /* Input too small, no compression (all literals) */ ++ ++ /* First Byte */ ++ { ++ U32 const h = LZ4_hashPosition(ip, tableType); ++ if (tableType == byPtr) { ++ LZ4_putPositionOnHash(ip, h, cctx->hashTable, byPtr); ++ } else { ++ LZ4_putIndexOnHash(startIndex, h, cctx->hashTable, ++ tableType); ++ } ++ } ++ ip++; ++ forwardH = LZ4_hashPosition(ip, tableType); ++ ++ /* Main Loop */ ++ for (;;) { ++ const BYTE *match; ++ BYTE *token; ++ const BYTE *filledIp; ++ ++ /* Find a match */ ++ if (tableType == byPtr) { ++ const BYTE *forwardIp = ip; ++ int step = 1; ++ int searchMatchNb = acceleration << LZ4_skipTrigger; ++ do { ++ U32 const h = forwardH; ++ ip = forwardIp; ++ forwardIp += step; ++ step = (searchMatchNb++ >> LZ4_skipTrigger); ++ ++ if (unlikely(forwardIp > mflimitPlusOne)) ++ goto _last_literals; ++ assert(ip < mflimitPlusOne); ++ ++ match = LZ4_getPositionOnHash( ++ h, cctx->hashTable, tableType); ++ forwardH = ++ LZ4_hashPosition(forwardIp, tableType); ++ LZ4_putPositionOnHash(ip, h, cctx->hashTable, ++ tableType); ++ ++ } while ((match + LZ4_DISTANCE_MAX < ip) || ++ (LZ4_read32(match) != LZ4_read32(ip))); ++ ++ } else { /* byU32, byU16 */ ++ ++ const BYTE *forwardIp = ip; ++ int step = 1; ++ int searchMatchNb = acceleration << LZ4_skipTrigger; ++ do { ++ U32 const h = forwardH; ++ U32 const currentPos = (U32)(forwardIp - base); ++ U32 matchIndex = LZ4_getIndexOnHash( ++ h, cctx->hashTable, tableType); ++ assert(matchIndex <= currentPos); ++ assert(forwardIp - base < ++ (ptrdiff_t)(2 GB - 1)); ++ ip = forwardIp; ++ forwardIp += step; ++ step = (searchMatchNb++ >> LZ4_skipTrigger); ++ ++ if (unlikely(forwardIp > mflimitPlusOne)) ++ goto _last_literals; ++ assert(ip < mflimitPlusOne); ++ ++ if (dictDirective == usingDictCtx) { ++ if (matchIndex < startIndex) { ++ /* there was no match, try the dictionary */ ++ assert(tableType == byU32); ++ matchIndex = LZ4_getIndexOnHash( ++ h, dictCtx->hashTable, ++ byU32); ++ match = dictBase + matchIndex; ++ matchIndex += ++ dictDelta; /* make dictCtx index comparable with current context */ ++ lowLimit = dictionary; ++ } else { ++ match = base + matchIndex; ++ lowLimit = (const BYTE *)source; ++ } ++ } else if (dictDirective == usingExtDict) { ++ if (matchIndex < startIndex) { ++ DEBUGLOG( ++ 7, ++ "extDict candidate: matchIndex=%5u < startIndex=%5u", ++ matchIndex, startIndex); ++ assert(startIndex - ++ matchIndex >= ++ MINMATCH); ++ assert(dictBase); ++ match = dictBase + matchIndex; ++ lowLimit = dictionary; ++ } else { ++ match = base + matchIndex; ++ lowLimit = (const BYTE *)source; ++ } ++ } else { /* single continuous memory segment */ ++ match = base + matchIndex; ++ } ++ forwardH = ++ LZ4_hashPosition(forwardIp, tableType); ++ LZ4_putIndexOnHash(currentPos, h, ++ cctx->hashTable, tableType); ++ ++ DEBUGLOG(7, ++ "candidate at pos=%u (offset=%u \n", ++ matchIndex, currentPos - matchIndex); ++ if ((dictIssue == dictSmall) && ++ (matchIndex < prefixIdxLimit)) { ++ continue; ++ } /* match outside of valid area */ ++ assert(matchIndex < currentPos); ++ if (((tableType != byU16) || ++ (LZ4_DISTANCE_MAX < ++ LZ4_DISTANCE_ABSOLUTE_MAX)) && ++ (matchIndex + LZ4_DISTANCE_MAX < ++ currentPos)) { ++ continue; ++ } /* too far */ ++ assert((currentPos - matchIndex) <= ++ LZ4_DISTANCE_MAX); /* match now expected within distance */ ++ ++ if (LZ4_read32(match) == LZ4_read32(ip)) { ++ if (maybe_extMem) ++ offset = ++ currentPos - matchIndex; ++ break; /* match found */ ++ } ++ ++ } while (1); ++ } ++ ++ /* Catch up */ ++ filledIp = ip; ++ assert(ip > ++ anchor); /* this is always true as ip has been advanced before entering the main loop */ ++ if ((match > lowLimit) && unlikely(ip[-1] == match[-1])) { ++ do { ++ ip--; ++ match--; ++ } while (((ip > anchor) & (match > lowLimit)) && ++ (unlikely(ip[-1] == match[-1]))); ++ } ++ ++ /* Encode Literals */ ++ { ++ unsigned const litLength = (unsigned)(ip - anchor); ++ token = op++; ++ if ((outputDirective == ++ limitedOutput) && /* Check output buffer overflow */ ++ (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + ++ (litLength / 255) > ++ olimit))) { ++ return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ ++ } ++ if ((outputDirective == fillOutput) && ++ (unlikely( ++ op + (litLength + 240) / 255 /* litlen */ + ++ litLength /* literals */ + ++ 2 /* offset */ + 1 /* token */ + ++ MFLIMIT - ++ MINMATCH /* min last literals so last match is <= end - MFLIMIT */ ++ > olimit))) { ++ op--; ++ goto _last_literals; ++ } ++ if (litLength >= RUN_MASK) { ++ unsigned len = litLength - RUN_MASK; ++ *token = (RUN_MASK << ML_BITS); ++ for (; len >= 255; len -= 255) ++ *op++ = 255; ++ *op++ = (BYTE)len; ++ } else ++ *token = (BYTE)(litLength << ML_BITS); ++ ++ /* Copy Literals */ ++ LZ4_wildCopy8(op, anchor, op + litLength); ++ op += litLength; ++ DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", ++ (int)(anchor - (const BYTE *)source), ++ litLength, (int)(ip - (const BYTE *)source)); ++ } ++ ++ _next_match: ++ /* at this stage, the following variables must be correctly set : ++ * - ip : at start of LZ operation ++ * - match : at start of previous pattern occurrence; can be within current prefix, or within extDict ++ * - offset : if maybe_ext_memSegment==1 (constant) ++ * - lowLimit : must be == dictionary to mean "match is within extDict"; must be == source otherwise ++ * - token and *token : position to write 4-bits for match length; higher 4-bits for literal length supposed already written ++ */ ++ ++ if ((outputDirective == fillOutput) && ++ (op + 2 /* offset */ + 1 /* token */ + MFLIMIT - ++ MINMATCH /* min last literals so last match is <= end - MFLIMIT */ ++ > olimit)) { ++ /* the match was too close to the end, rewind and go to last literals */ ++ op = token; ++ goto _last_literals; ++ } ++ ++ /* Encode Offset */ ++ if (maybe_extMem) { /* static test */ ++ DEBUGLOG(6, ++ " with offset=%u (ext if > %i)", ++ offset, (int)(ip - (const BYTE *)source)); ++ assert(offset <= LZ4_DISTANCE_MAX && offset > 0); ++ LZ4_writeLE16(op, (U16)offset); ++ op += 2; ++ } else { ++ DEBUGLOG(6, ++ " with offset=%u (same segment)", ++ (U32)(ip - match)); ++ assert(ip - match <= LZ4_DISTANCE_MAX); ++ LZ4_writeLE16(op, (U16)(ip - match)); ++ op += 2; ++ } ++ ++ /* Encode MatchLength */ ++ { ++ unsigned matchCode; ++ ++ if ((dictDirective == usingExtDict || ++ dictDirective == usingDictCtx) && ++ (lowLimit == ++ dictionary) /* match within extDict */) { ++ const BYTE *limit = ip + (dictEnd - match); ++ assert(dictEnd > match); ++ if (limit > matchlimit) ++ limit = matchlimit; ++ matchCode = LZ4_count(ip + MINMATCH, ++ match + MINMATCH, limit); ++ ip += (size_t)matchCode + MINMATCH; ++ if (ip == limit) { ++ unsigned const more = ++ LZ4_count(limit, ++ (const BYTE *)source, ++ matchlimit); ++ matchCode += more; ++ ip += more; ++ } ++ DEBUGLOG( ++ 6, ++ " with matchLength=%u starting in extDict", ++ matchCode + MINMATCH); ++ } else { ++ matchCode = ++ LZ4_count(ip + MINMATCH, ++ match + MINMATCH, matchlimit); ++ ip += (size_t)matchCode + MINMATCH; ++ DEBUGLOG(6, " with matchLength=%u", ++ matchCode + MINMATCH); ++ } ++ ++ if ((outputDirective) && /* Check output buffer overflow */ ++ (unlikely(op + (1 + LASTLITERALS) + ++ (matchCode + 240) / 255 > ++ olimit))) { ++ if (outputDirective == fillOutput) { ++ /* Match description too long : reduce it */ ++ U32 newMatchCode = ++ 15 /* in token */ - ++ 1 /* to avoid needing a zero byte */ + ++ ((U32)(olimit - op) - 1 - ++ LASTLITERALS) * ++ 255; ++ ip -= matchCode - newMatchCode; ++ assert(newMatchCode < matchCode); ++ matchCode = newMatchCode; ++ if (unlikely(ip <= filledIp)) { ++ /* We have already filled up to filledIp so if ip ends up less than filledIp ++ * we have positions in the hash table beyond the current position. This is ++ * a problem if we reuse the hash table. So we have to remove these positions ++ * from the hash table. ++ */ ++ const BYTE *ptr; ++ DEBUGLOG( ++ 5, ++ "Clearing %u positions", ++ (U32)(filledIp - ip)); ++ for (ptr = ip; ptr <= filledIp; ++ ++ptr) { ++ U32 const h = ++ LZ4_hashPosition( ++ ptr, ++ tableType); ++ LZ4_clearHash( ++ h, ++ cctx->hashTable, ++ tableType); ++ } ++ } ++ } else { ++ assert(outputDirective == ++ limitedOutput); ++ return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ ++ } ++ } ++ if (matchCode >= ML_MASK) { ++ *token += ML_MASK; ++ matchCode -= ML_MASK; ++ LZ4_write32(op, 0xFFFFFFFF); ++ while (matchCode >= 4 * 255) { ++ op += 4; ++ LZ4_write32(op, 0xFFFFFFFF); ++ matchCode -= 4 * 255; ++ } ++ op += matchCode / 255; ++ *op++ = (BYTE)(matchCode % 255); ++ } else ++ *token += (BYTE)(matchCode); ++ } ++ /* Ensure we have enough space for the last literals. */ ++ assert(!(outputDirective == fillOutput && ++ op + 1 + LASTLITERALS > olimit)); ++ ++ anchor = ip; ++ ++ /* Test end of chunk */ ++ if (ip >= mflimitPlusOne) ++ break; ++ ++ /* Fill table */ ++ { ++ U32 const h = LZ4_hashPosition(ip - 2, tableType); ++ if (tableType == byPtr) { ++ LZ4_putPositionOnHash(ip - 2, h, ++ cctx->hashTable, byPtr); ++ } else { ++ U32 const idx = (U32)((ip - 2) - base); ++ LZ4_putIndexOnHash(idx, h, cctx->hashTable, ++ tableType); ++ } ++ } ++ ++ /* Test next position */ ++ if (tableType == byPtr) { ++ match = LZ4_getPosition(ip, cctx->hashTable, tableType); ++ LZ4_putPosition(ip, cctx->hashTable, tableType); ++ if ((match + LZ4_DISTANCE_MAX >= ip) && ++ (LZ4_read32(match) == LZ4_read32(ip))) { ++ token = op++; ++ *token = 0; ++ goto _next_match; ++ } ++ ++ } else { /* byU32, byU16 */ ++ ++ U32 const h = LZ4_hashPosition(ip, tableType); ++ U32 const currentPos = (U32)(ip - base); ++ U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, ++ tableType); ++ assert(matchIndex < currentPos); ++ if (dictDirective == usingDictCtx) { ++ if (matchIndex < startIndex) { ++ /* there was no match, try the dictionary */ ++ assert(tableType == byU32); ++ matchIndex = LZ4_getIndexOnHash( ++ h, dictCtx->hashTable, byU32); ++ match = dictBase + matchIndex; ++ lowLimit = ++ dictionary; /* required for match length counter */ ++ matchIndex += dictDelta; ++ } else { ++ match = base + matchIndex; ++ lowLimit = (const BYTE *) ++ source; /* required for match length counter */ ++ } ++ } else if (dictDirective == usingExtDict) { ++ if (matchIndex < startIndex) { ++ assert(dictBase); ++ match = dictBase + matchIndex; ++ lowLimit = ++ dictionary; /* required for match length counter */ ++ } else { ++ match = base + matchIndex; ++ lowLimit = (const BYTE *) ++ source; /* required for match length counter */ ++ } ++ } else { /* single memory segment */ ++ match = base + matchIndex; ++ } ++ LZ4_putIndexOnHash(currentPos, h, cctx->hashTable, ++ tableType); ++ assert(matchIndex < currentPos); ++ if (((dictIssue == dictSmall) ? ++ (matchIndex >= prefixIdxLimit) : ++ 1) && ++ (((tableType == byU16) && ++ (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? ++ 1 : ++ (matchIndex + LZ4_DISTANCE_MAX >= ++ currentPos)) && ++ (LZ4_read32(match) == LZ4_read32(ip))) { ++ token = op++; ++ *token = 0; ++ if (maybe_extMem) ++ offset = currentPos - matchIndex; ++ DEBUGLOG( ++ 6, ++ "seq.start:%i, literals=%u, match.start:%i", ++ (int)(anchor - (const BYTE *)source), 0, ++ (int)(ip - (const BYTE *)source)); ++ goto _next_match; ++ } ++ } ++ ++ /* Prepare next loop */ ++ forwardH = LZ4_hashPosition(++ip, tableType); ++ } ++ ++_last_literals: ++ /* Encode Last Literals */ ++ { ++ size_t lastRun = (size_t)(iend - anchor); ++ if ((outputDirective) && /* Check output buffer overflow */ ++ (op + lastRun + 1 + ((lastRun + 255 - RUN_MASK) / 255) > ++ olimit)) { ++ if (outputDirective == fillOutput) { ++ /* adapt lastRun to fill 'dst' */ ++ assert(olimit >= op); ++ lastRun = (size_t)(olimit - op) - 1 /*token*/; ++ lastRun -= (lastRun + 256 - RUN_MASK) / ++ 256; /*additional length tokens*/ ++ } else { ++ assert(outputDirective == limitedOutput); ++ return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ ++ } ++ } ++ DEBUGLOG(6, "Final literal run : %i literals", (int)lastRun); ++ if (lastRun >= RUN_MASK) { ++ size_t accumulator = lastRun - RUN_MASK; ++ *op++ = RUN_MASK << ML_BITS; ++ for (; accumulator >= 255; accumulator -= 255) ++ *op++ = 255; ++ *op++ = (BYTE)accumulator; ++ } else { ++ *op++ = (BYTE)(lastRun << ML_BITS); ++ } ++ LZ4_memcpy(op, anchor, lastRun); ++ ip = anchor + lastRun; ++ op += lastRun; ++ } ++ ++ if (outputDirective == fillOutput) { ++ *inputConsumed = (int)(((const char *)ip) - source); ++ } ++ result = (int)(((char *)op) - dest); ++ assert(result > 0); ++ DEBUGLOG(5, "LZ4_compress_generic: compressed %i bytes into %i bytes", ++ inputSize, result); ++ return result; ++} ++ ++/** LZ4_compress_generic() : ++ * inlined, to ensure branches are decided at compilation time; ++ * takes care of src == (NULL, 0) ++ * and forward the rest to LZ4_compress_generic_validated */ ++LZ4_FORCE_INLINE int LZ4_compress_generic( ++ LZ4_stream_t_internal *const cctx, const char *const src, ++ char *const dst, const int srcSize, ++ int *inputConsumed, /* only written when outputDirective == fillOutput */ ++ const int dstCapacity, const limitedOutput_directive outputDirective, ++ const tableType_t tableType, const dict_directive dictDirective, ++ const dictIssue_directive dictIssue, const int acceleration) ++{ ++ DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, dstCapacity=%i", srcSize, ++ dstCapacity); ++ ++ if ((U32)srcSize > (U32)LZ4_MAX_INPUT_SIZE) { ++ return 0; ++ } /* Unsupported srcSize, too large (or negative) */ ++ if (srcSize == 0) { /* src == NULL supported if srcSize == 0 */ ++ if (outputDirective != notLimited && dstCapacity <= 0) ++ return 0; /* no output, can't write anything */ ++ DEBUGLOG(5, "Generating an empty block"); ++ assert(outputDirective == notLimited || dstCapacity >= 1); ++ assert(dst != NULL); ++ dst[0] = 0; ++ if (outputDirective == fillOutput) { ++ assert(inputConsumed != NULL); ++ *inputConsumed = 0; ++ } ++ return 1; ++ } ++ assert(src != NULL); ++ ++ return LZ4_compress_generic_validated( ++ cctx, src, dst, srcSize, ++ inputConsumed, /* only written into if outputDirective == fillOutput */ ++ dstCapacity, outputDirective, tableType, dictDirective, ++ dictIssue, acceleration); ++} ++ ++int LZ4_compress_fast_extState(void *state, const char *source, char *dest, ++ int inputSize, int maxOutputSize, ++ int acceleration) ++{ ++ LZ4_stream_t_internal *const ctx = ++ &LZ4_initStream(state, sizeof(LZ4_stream_t))->internal_donotuse; ++ assert(ctx != NULL); ++ if (acceleration < 1) ++ acceleration = LZ4_ACCELERATION_DEFAULT; ++ if (acceleration > LZ4_ACCELERATION_MAX) ++ acceleration = LZ4_ACCELERATION_MAX; ++ if (maxOutputSize >= LZ4_compressBound(inputSize)) { ++ if (inputSize < LZ4_64Klimit) { ++ return LZ4_compress_generic(ctx, source, dest, ++ inputSize, NULL, 0, ++ notLimited, byU16, noDict, ++ noDictIssue, acceleration); ++ } else { ++ const tableType_t tableType = ++ ((sizeof(void *) == 4) && ++ ((uptrval)source > LZ4_DISTANCE_MAX)) ? ++ byPtr : ++ byU32; ++ return LZ4_compress_generic(ctx, source, dest, ++ inputSize, NULL, 0, ++ notLimited, tableType, ++ noDict, noDictIssue, ++ acceleration); ++ } ++ } else { ++ if (inputSize < LZ4_64Klimit) { ++ return LZ4_compress_generic( ++ ctx, source, dest, inputSize, NULL, ++ maxOutputSize, limitedOutput, byU16, noDict, ++ noDictIssue, acceleration); ++ } else { ++ const tableType_t tableType = ++ ((sizeof(void *) == 4) && ++ ((uptrval)source > LZ4_DISTANCE_MAX)) ? ++ byPtr : ++ byU32; ++ return LZ4_compress_generic( ++ ctx, source, dest, inputSize, NULL, ++ maxOutputSize, limitedOutput, tableType, noDict, ++ noDictIssue, acceleration); ++ } ++ } ++} ++ ++/** ++ * LZ4_compress_fast_extState_fastReset() : ++ * A variant of LZ4_compress_fast_extState(). ++ * ++ * Using this variant avoids an expensive initialization step. It is only safe ++ * to call if the state buffer is known to be correctly initialized already ++ * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of ++ * "correctly initialized"). ++ */ ++int LZ4_compress_fast_extState_fastReset(void *state, const char *src, ++ char *dst, int srcSize, ++ int dstCapacity, int acceleration) ++{ ++ LZ4_stream_t_internal *const ctx = ++ &((LZ4_stream_t *)state)->internal_donotuse; ++ if (acceleration < 1) ++ acceleration = LZ4_ACCELERATION_DEFAULT; ++ if (acceleration > LZ4_ACCELERATION_MAX) ++ acceleration = LZ4_ACCELERATION_MAX; ++ assert(ctx != NULL); ++ ++ if (dstCapacity >= LZ4_compressBound(srcSize)) { ++ if (srcSize < LZ4_64Klimit) { ++ const tableType_t tableType = byU16; ++ LZ4_prepareTable(ctx, srcSize, tableType); ++ if (ctx->currentOffset) { ++ return LZ4_compress_generic( ++ ctx, src, dst, srcSize, NULL, 0, ++ notLimited, tableType, noDict, ++ dictSmall, acceleration); ++ } else { ++ return LZ4_compress_generic( ++ ctx, src, dst, srcSize, NULL, 0, ++ notLimited, tableType, noDict, ++ noDictIssue, acceleration); ++ } ++ } else { ++ const tableType_t tableType = ++ ((sizeof(void *) == 4) && ++ ((uptrval)src > LZ4_DISTANCE_MAX)) ? ++ byPtr : ++ byU32; ++ LZ4_prepareTable(ctx, srcSize, tableType); ++ return LZ4_compress_generic(ctx, src, dst, srcSize, ++ NULL, 0, notLimited, ++ tableType, noDict, ++ noDictIssue, acceleration); ++ } ++ } else { ++ if (srcSize < LZ4_64Klimit) { ++ const tableType_t tableType = byU16; ++ LZ4_prepareTable(ctx, srcSize, tableType); ++ if (ctx->currentOffset) { ++ return LZ4_compress_generic( ++ ctx, src, dst, srcSize, NULL, ++ dstCapacity, limitedOutput, tableType, ++ noDict, dictSmall, acceleration); ++ } else { ++ return LZ4_compress_generic( ++ ctx, src, dst, srcSize, NULL, ++ dstCapacity, limitedOutput, tableType, ++ noDict, noDictIssue, acceleration); ++ } ++ } else { ++ const tableType_t tableType = ++ ((sizeof(void *) == 4) && ++ ((uptrval)src > LZ4_DISTANCE_MAX)) ? ++ byPtr : ++ byU32; ++ LZ4_prepareTable(ctx, srcSize, tableType); ++ return LZ4_compress_generic(ctx, src, dst, srcSize, ++ NULL, dstCapacity, ++ limitedOutput, tableType, ++ noDict, noDictIssue, ++ acceleration); ++ } ++ } ++} ++ ++int LZ4_compress_fast(const char* src, char* dest, int srcSize, int dstCapacity, ++ int acceleration, void *wrkmem) ++{ ++ return LZ4_compress_fast_extState(wrkmem, src, dest, srcSize, dstCapacity, acceleration); ++} ++EXPORT_SYMBOL(LZ4_compress_fast); ++ ++int LZ4_compress_default(const char *src, char *dst, int srcSize, ++ int dstCapacity, void *wrkmem) ++{ ++ return LZ4_compress_fast_extState(wrkmem, src, dst, srcSize, ++ dstCapacity, 1); ++} ++EXPORT_SYMBOL(LZ4_compress_default); ++ ++/* Note!: This function leaves the stream in an unclean/broken state! ++ * It is not safe to subsequently use the same state with a _fastReset() or ++ * _continue() call without resetting it. */ ++static int LZ4_compress_destSize_extState_internal(LZ4_stream_t *state, ++ const char *src, char *dst, ++ int *srcSizePtr, ++ int targetDstSize, ++ int acceleration) ++{ ++ void *const s = LZ4_initStream(state, sizeof(*state)); ++ assert(s != NULL); ++ (void)s; ++ ++ if (targetDstSize >= ++ LZ4_compressBound( ++ *srcSizePtr)) { /* compression success is guaranteed */ ++ return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, ++ targetDstSize, acceleration); ++ } else { ++ if (*srcSizePtr < LZ4_64Klimit) { ++ return LZ4_compress_generic(&state->internal_donotuse, ++ src, dst, *srcSizePtr, ++ srcSizePtr, targetDstSize, ++ fillOutput, byU16, noDict, ++ noDictIssue, acceleration); ++ } else { ++ tableType_t const addrMode = ++ ((sizeof(void *) == 4) && ++ ((uptrval)src > LZ4_DISTANCE_MAX)) ? ++ byPtr : ++ byU32; ++ return LZ4_compress_generic(&state->internal_donotuse, ++ src, dst, *srcSizePtr, ++ srcSizePtr, targetDstSize, ++ fillOutput, addrMode, ++ noDict, noDictIssue, ++ acceleration); ++ } ++ } ++} ++ ++int LZ4_compress_destSize_extState(void *state, const char *src, char *dst, ++ int *srcSizePtr, int targetDstSize, ++ int acceleration) ++{ ++ int const r = LZ4_compress_destSize_extState_internal( ++ (LZ4_stream_t *)state, src, dst, srcSizePtr, targetDstSize, ++ acceleration); ++ /* clean the state on exit */ ++ LZ4_initStream(state, sizeof(LZ4_stream_t)); ++ return r; ++} ++ ++int LZ4_compress_destSize(const char *src, char *dst, int *srcSizePtr, ++ int targetDstSize, void *wrkmem) ++{ ++ return LZ4_compress_destSize_extState_internal( ++ wrkmem, src, dst, srcSizePtr, targetDstSize, 1); ++} ++EXPORT_SYMBOL(LZ4_compress_destSize); ++ ++/*-****************************** ++* Streaming functions ++********************************/ ++ ++#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) ++LZ4_stream_t *LZ4_createStream(void) ++{ ++ LZ4_stream_t *const lz4s = (LZ4_stream_t *)ALLOC(sizeof(LZ4_stream_t)); ++ LZ4_STATIC_ASSERT(sizeof(LZ4_stream_t) >= ++ sizeof(LZ4_stream_t_internal)); ++ DEBUGLOG(4, "LZ4_createStream %p", lz4s); ++ if (lz4s == NULL) ++ return NULL; ++ LZ4_initStream(lz4s, sizeof(*lz4s)); ++ return lz4s; ++} ++#endif ++ ++static size_t LZ4_stream_t_alignment(void) ++{ ++#if LZ4_ALIGN_TEST ++ typedef struct { ++ char c; ++ LZ4_stream_t t; ++ } t_a; ++ return sizeof(t_a) - sizeof(LZ4_stream_t); ++#else ++ return 1; /* effectively disabled */ ++#endif ++} ++ ++LZ4_stream_t *LZ4_initStream(void *buffer, size_t size) ++{ ++ DEBUGLOG(5, "LZ4_initStream"); ++ if (buffer == NULL) { ++ return NULL; ++ } ++ if (size < sizeof(LZ4_stream_t)) { ++ return NULL; ++ } ++ if (!LZ4_isAligned(buffer, LZ4_stream_t_alignment())) ++ return NULL; ++ MEM_INIT(buffer, 0, sizeof(LZ4_stream_t_internal)); ++ return (LZ4_stream_t *)buffer; ++} ++ ++/* resetStream is now deprecated, ++ * prefer initStream() which is more general */ ++void LZ4_resetStream(LZ4_stream_t *LZ4_stream) ++{ ++ DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream); ++ MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t_internal)); ++} ++ ++void LZ4_resetStream_fast(LZ4_stream_t *ctx) ++{ ++ LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32); ++} ++ ++#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) ++int LZ4_freeStream(LZ4_stream_t *LZ4_stream) ++{ ++ if (!LZ4_stream) ++ return 0; /* support free on NULL */ ++ DEBUGLOG(5, "LZ4_freeStream %p", LZ4_stream); ++ FREEMEM(LZ4_stream); ++ return (0); ++} ++#endif ++ ++typedef enum { _ld_fast, _ld_slow } LoadDict_mode_e; ++#define HASH_UNIT sizeof(reg_t) ++int LZ4_loadDict_internal(LZ4_stream_t *LZ4_dict, const char *dictionary, ++ int dictSize, LoadDict_mode_e _ld) ++{ ++ LZ4_stream_t_internal *const dict = &LZ4_dict->internal_donotuse; ++ const tableType_t tableType = byU32; ++ const BYTE *p = (const BYTE *)dictionary; ++ const BYTE *const dictEnd = p + dictSize; ++ U32 idx32; ++ ++ DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, ++ dictionary, LZ4_dict); ++ ++ /* It's necessary to reset the context, ++ * and not just continue it with prepareTable() ++ * to avoid any risk of generating overflowing matchIndex ++ * when compressing using this dictionary */ ++ LZ4_resetStream(LZ4_dict); ++ ++ /* We always increment the offset by 64 KB, since, if the dict is longer, ++ * we truncate it to the last 64k, and if it's shorter, we still want to ++ * advance by a whole window length so we can provide the guarantee that ++ * there are only valid offsets in the window, which allows an optimization ++ * in LZ4_compress_fast_continue() where it uses noDictIssue even when the ++ * dictionary isn't a full 64k. */ ++ dict->currentOffset += 64 KB; ++ ++ if (dictSize < (int)HASH_UNIT) { ++ return 0; ++ } ++ ++ if ((dictEnd - p) > 64 KB) ++ p = dictEnd - 64 KB; ++ dict->dictionary = p; ++ dict->dictSize = (U32)(dictEnd - p); ++ dict->tableType = (U32)tableType; ++ idx32 = dict->currentOffset - dict->dictSize; ++ ++ while (p <= dictEnd - HASH_UNIT) { ++ U32 const h = LZ4_hashPosition(p, tableType); ++ /* Note: overwriting => favors positions end of dictionary */ ++ LZ4_putIndexOnHash(idx32, h, dict->hashTable, tableType); ++ p += 3; ++ idx32 += 3; ++ } ++ ++ if (_ld == _ld_slow) { ++ /* Fill hash table with additional references, to improve compression capability */ ++ p = dict->dictionary; ++ idx32 = dict->currentOffset - dict->dictSize; ++ while (p <= dictEnd - HASH_UNIT) { ++ U32 const h = LZ4_hashPosition(p, tableType); ++ U32 const limit = dict->currentOffset - 64 KB; ++ if (LZ4_getIndexOnHash(h, dict->hashTable, tableType) <= ++ limit) { ++ /* Note: not overwriting => favors positions beginning of dictionary */ ++ LZ4_putIndexOnHash(idx32, h, dict->hashTable, ++ tableType); ++ } ++ p++; ++ idx32++; ++ } ++ } ++ ++ return (int)dict->dictSize; ++} ++ ++int LZ4_loadDict(LZ4_stream_t *LZ4_dict, const char *dictionary, int dictSize) ++{ ++ return LZ4_loadDict_internal(LZ4_dict, dictionary, dictSize, _ld_fast); ++} ++EXPORT_SYMBOL(LZ4_loadDict); ++ ++int LZ4_loadDictSlow(LZ4_stream_t *LZ4_dict, const char *dictionary, ++ int dictSize) ++{ ++ return LZ4_loadDict_internal(LZ4_dict, dictionary, dictSize, _ld_slow); ++} ++ ++void LZ4_attach_dictionary(LZ4_stream_t *workingStream, ++ const LZ4_stream_t *dictionaryStream) ++{ ++ const LZ4_stream_t_internal *dictCtx = ++ (dictionaryStream == NULL) ? ++ NULL : ++ &(dictionaryStream->internal_donotuse); ++ ++ DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)", workingStream, ++ dictionaryStream, dictCtx != NULL ? dictCtx->dictSize : 0); ++ ++ if (dictCtx != NULL) { ++ /* If the current offset is zero, we will never look in the ++ * external dictionary context, since there is no value a table ++ * entry can take that indicate a miss. In that case, we need ++ * to bump the offset to something non-zero. ++ */ ++ if (workingStream->internal_donotuse.currentOffset == 0) { ++ workingStream->internal_donotuse.currentOffset = 64 KB; ++ } ++ ++ /* Don't actually attach an empty dictionary. ++ */ ++ if (dictCtx->dictSize == 0) { ++ dictCtx = NULL; ++ } ++ } ++ workingStream->internal_donotuse.dictCtx = dictCtx; ++} ++ ++static void LZ4_renormDictT(LZ4_stream_t_internal *LZ4_dict, int nextSize) ++{ ++ assert(nextSize >= 0); ++ if (LZ4_dict->currentOffset + (unsigned)nextSize > ++ 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */ ++ /* rescale hash table */ ++ U32 const delta = LZ4_dict->currentOffset - 64 KB; ++ const BYTE *dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; ++ int i; ++ DEBUGLOG(4, "LZ4_renormDictT"); ++ for (i = 0; i < LZ4_HASH_SIZE_U32; i++) { ++ if (LZ4_dict->hashTable[i] < delta) ++ LZ4_dict->hashTable[i] = 0; ++ else ++ LZ4_dict->hashTable[i] -= delta; ++ } ++ LZ4_dict->currentOffset = 64 KB; ++ if (LZ4_dict->dictSize > 64 KB) ++ LZ4_dict->dictSize = 64 KB; ++ LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; ++ } ++} ++ ++int LZ4_compress_fast_continue(LZ4_stream_t *LZ4_stream, const char *source, ++ char *dest, int inputSize, int maxOutputSize, ++ int acceleration) ++{ ++ const tableType_t tableType = byU32; ++ LZ4_stream_t_internal *const streamPtr = &LZ4_stream->internal_donotuse; ++ const char *dictEnd = streamPtr->dictSize ? ++ (const char *)streamPtr->dictionary + ++ streamPtr->dictSize : ++ NULL; ++ ++ DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i, dictSize=%u)", ++ inputSize, streamPtr->dictSize); ++ ++ LZ4_renormDictT(streamPtr, inputSize); /* fix index overflow */ ++ if (acceleration < 1) ++ acceleration = LZ4_ACCELERATION_DEFAULT; ++ if (acceleration > LZ4_ACCELERATION_MAX) ++ acceleration = LZ4_ACCELERATION_MAX; ++ ++ /* invalidate tiny dictionaries */ ++ if ((streamPtr->dictSize < ++ 4) /* tiny dictionary : not enough for a hash */ ++ && (dictEnd != source) /* prefix mode */ ++ && ++ (inputSize > ++ 0) /* tolerance : don't lose history, in case next invocation would use prefix mode */ ++ && (streamPtr->dictCtx == NULL) /* usingDictCtx */ ++ ) { ++ DEBUGLOG( ++ 5, ++ "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", ++ streamPtr->dictSize, streamPtr->dictionary); ++ /* remove dictionary existence from history, to employ faster prefix mode */ ++ streamPtr->dictSize = 0; ++ streamPtr->dictionary = (const BYTE *)source; ++ dictEnd = source; ++ } ++ ++ /* Check overlapping input/dictionary space */ ++ { ++ const char *const sourceEnd = source + inputSize; ++ if ((sourceEnd > (const char *)streamPtr->dictionary) && ++ (sourceEnd < dictEnd)) { ++ streamPtr->dictSize = (U32)(dictEnd - sourceEnd); ++ if (streamPtr->dictSize > 64 KB) ++ streamPtr->dictSize = 64 KB; ++ if (streamPtr->dictSize < 4) ++ streamPtr->dictSize = 0; ++ streamPtr->dictionary = ++ (const BYTE *)dictEnd - streamPtr->dictSize; ++ } ++ } ++ ++ /* prefix mode : source data follows dictionary */ ++ if (dictEnd == source) { ++ if ((streamPtr->dictSize < 64 KB) && ++ (streamPtr->dictSize < streamPtr->currentOffset)) ++ return LZ4_compress_generic( ++ streamPtr, source, dest, inputSize, NULL, ++ maxOutputSize, limitedOutput, tableType, ++ withPrefix64k, dictSmall, acceleration); ++ else ++ return LZ4_compress_generic( ++ streamPtr, source, dest, inputSize, NULL, ++ maxOutputSize, limitedOutput, tableType, ++ withPrefix64k, noDictIssue, acceleration); ++ } ++ ++ /* external dictionary mode */ ++ { ++ int result; ++ if (streamPtr->dictCtx) { ++ /* We depend here on the fact that dictCtx'es (produced by ++ * LZ4_loadDict) guarantee that their tables contain no references ++ * to offsets between dictCtx->currentOffset - 64 KB and ++ * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe ++ * to use noDictIssue even when the dict isn't a full 64 KB. ++ */ ++ if (inputSize > 4 KB) { ++ /* For compressing large blobs, it is faster to pay the setup ++ * cost to copy the dictionary's tables into the active context, ++ * so that the compression loop is only looking into one table. ++ */ ++ LZ4_memcpy(streamPtr, streamPtr->dictCtx, ++ sizeof(*streamPtr)); ++ result = LZ4_compress_generic( ++ streamPtr, source, dest, inputSize, ++ NULL, maxOutputSize, limitedOutput, ++ tableType, usingExtDict, noDictIssue, ++ acceleration); ++ } else { ++ result = LZ4_compress_generic( ++ streamPtr, source, dest, inputSize, ++ NULL, maxOutputSize, limitedOutput, ++ tableType, usingDictCtx, noDictIssue, ++ acceleration); ++ } ++ } else { /* small data <= 4 KB */ ++ if ((streamPtr->dictSize < 64 KB) && ++ (streamPtr->dictSize < streamPtr->currentOffset)) { ++ result = LZ4_compress_generic( ++ streamPtr, source, dest, inputSize, ++ NULL, maxOutputSize, limitedOutput, ++ tableType, usingExtDict, dictSmall, ++ acceleration); ++ } else { ++ result = LZ4_compress_generic( ++ streamPtr, source, dest, inputSize, ++ NULL, maxOutputSize, limitedOutput, ++ tableType, usingExtDict, noDictIssue, ++ acceleration); ++ } ++ } ++ streamPtr->dictionary = (const BYTE *)source; ++ streamPtr->dictSize = (U32)inputSize; ++ return result; ++ } ++} ++ ++/* Hidden debug function, to force-test external dictionary mode */ ++int LZ4_compress_forceExtDict(LZ4_stream_t *LZ4_dict, const char *source, ++ char *dest, int srcSize) ++{ ++ LZ4_stream_t_internal *const streamPtr = &LZ4_dict->internal_donotuse; ++ int result; ++ ++ LZ4_renormDictT(streamPtr, srcSize); ++ ++ if ((streamPtr->dictSize < 64 KB) && ++ (streamPtr->dictSize < streamPtr->currentOffset)) { ++ result = LZ4_compress_generic(streamPtr, source, dest, srcSize, ++ NULL, 0, notLimited, byU32, ++ usingExtDict, dictSmall, 1); ++ } else { ++ result = LZ4_compress_generic(streamPtr, source, dest, srcSize, ++ NULL, 0, notLimited, byU32, ++ usingExtDict, noDictIssue, 1); ++ } ++ ++ streamPtr->dictionary = (const BYTE *)source; ++ streamPtr->dictSize = (U32)srcSize; ++ ++ return result; ++} ++ ++/*! LZ4_saveDict() : ++ * If previously compressed data block is not guaranteed to remain available at its memory location, ++ * save it into a safer place (char* safeBuffer). ++ * Note : no need to call LZ4_loadDict() afterwards, dictionary is immediately usable, ++ * one can therefore call LZ4_compress_fast_continue() right after. ++ * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. ++ */ ++int LZ4_saveDict(LZ4_stream_t *LZ4_dict, char *safeBuffer, int dictSize) ++{ ++ LZ4_stream_t_internal *const dict = &LZ4_dict->internal_donotuse; ++ ++ DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, ++ safeBuffer); ++ ++ if ((U32)dictSize > 64 KB) { ++ dictSize = 64 KB; ++ } /* useless to define a dictionary > 64 KB */ ++ if ((U32)dictSize > dict->dictSize) { ++ dictSize = (int)dict->dictSize; ++ } ++ ++ if (safeBuffer == NULL) ++ assert(dictSize == 0); ++ if (dictSize > 0) { ++ const BYTE *const previousDictEnd = ++ dict->dictionary + dict->dictSize; ++ assert(dict->dictionary); ++ LZ4_memmove(safeBuffer, previousDictEnd - dictSize, ++ (size_t)dictSize); ++ } ++ ++ dict->dictionary = (const BYTE *)safeBuffer; ++ dict->dictSize = (U32)dictSize; ++ ++ return dictSize; ++} ++EXPORT_SYMBOL(LZ4_saveDict); ++ ++/*-******************************* ++ * Decompression functions ++ ********************************/ ++ ++typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; ++ ++#undef MIN ++#define MIN(a, b) ((a) < (b) ? (a) : (b)) ++ ++/* variant for decompress_unsafe() ++ * does not know end of input ++ * presumes input is well formed ++ * note : will consume at least one byte */ ++static size_t read_long_length_no_check(const BYTE **pp) ++{ ++ size_t b, l = 0; ++ do { ++ b = **pp; ++ (*pp)++; ++ l += b; ++ } while (b == 255); ++ DEBUGLOG(6, ++ "read_long_length_no_check: +length=%zu using %zu input bytes", ++ l, l / 255 + 1) ++ return l; ++} ++ ++/* core decoder variant for LZ4_decompress_fast*() ++ * for legacy support only : these entry points are deprecated. ++ * - Presumes input is correctly formed (no defense vs malformed inputs) ++ * - Does not know input size (presume input buffer is "large enough") ++ * - Decompress a full block (only) ++ * @return : nb of bytes read from input. ++ * Note : this variant is not optimized for speed, just for maintenance. ++ * the goal is to remove support of decompress_fast*() variants by v2.0 ++**/ ++LZ4_FORCE_INLINE int LZ4_decompress_unsafe_generic( ++ const BYTE *const istart, BYTE *const ostart, int decompressedSize, ++ ++ size_t prefixSize, ++ const BYTE *const dictStart, /* only if dict==usingExtDict */ ++ const size_t dictSize /* note: =0 if dictStart==NULL */ ++) ++{ ++ const BYTE *ip = istart; ++ BYTE *op = (BYTE *)ostart; ++ BYTE *const oend = ostart + decompressedSize; ++ const BYTE *const prefixStart = ostart - prefixSize; ++ ++ DEBUGLOG(5, "LZ4_decompress_unsafe_generic"); ++ if (dictStart == NULL) ++ assert(dictSize == 0); ++ ++ while (1) { ++ /* start new sequence */ ++ unsigned token = *ip++; ++ ++ /* literals */ ++ { ++ size_t ll = token >> ML_BITS; ++ if (ll == 15) { ++ /* long literal length */ ++ ll += read_long_length_no_check(&ip); ++ } ++ if ((size_t)(oend - op) < ll) ++ return -1; /* output buffer overflow */ ++ LZ4_memmove(op, ip, ++ ll); /* support in-place decompression */ ++ op += ll; ++ ip += ll; ++ if ((size_t)(oend - op) < MFLIMIT) { ++ if (op == oend) ++ break; /* end of block */ ++ DEBUGLOG( ++ 5, ++ "invalid: literals end at distance %zi from end of block", ++ oend - op); ++ /* incorrect end of block : ++ * last match must start at least MFLIMIT==12 bytes before end of output block */ ++ return -1; ++ } ++ } ++ ++ /* match */ ++ { ++ size_t ml = token & 15; ++ size_t const offset = LZ4_readLE16(ip); ++ ip += 2; ++ ++ if (ml == 15) { ++ /* long literal length */ ++ ml += read_long_length_no_check(&ip); ++ } ++ ml += MINMATCH; ++ ++ if ((size_t)(oend - op) < ml) ++ return -1; /* output buffer overflow */ ++ ++ { ++ const BYTE *match = op - offset; ++ ++ /* out of range */ ++ if (offset > ++ (size_t)(op - prefixStart) + dictSize) { ++ DEBUGLOG(6, "offset out of range"); ++ return -1; ++ } ++ ++ /* check special case : extDict */ ++ if (offset > (size_t)(op - prefixStart)) { ++ /* extDict scenario */ ++ const BYTE *const dictEnd = ++ dictStart + dictSize; ++ const BYTE *extMatch = ++ dictEnd - ++ (offset - ++ (size_t)(op - prefixStart)); ++ size_t const extml = ++ (size_t)(dictEnd - extMatch); ++ if (extml > ml) { ++ /* match entirely within extDict */ ++ LZ4_memmove(op, extMatch, ml); ++ op += ml; ++ ml = 0; ++ } else { ++ /* match split between extDict & prefix */ ++ LZ4_memmove(op, extMatch, ++ extml); ++ op += extml; ++ ml -= extml; ++ } ++ match = prefixStart; ++ } ++ ++ /* match copy - slow variant, supporting overlap copy */ ++ { ++ size_t u; ++ for (u = 0; u < ml; u++) { ++ op[u] = match[u]; ++ } ++ } ++ } ++ op += ml; ++ if ((size_t)(oend - op) < LASTLITERALS) { ++ DEBUGLOG( ++ 5, ++ "invalid: match ends at distance %zi from end of block", ++ oend - op); ++ /* incorrect end of block : ++ * last match must stop at least LASTLITERALS==5 bytes before end of output block */ ++ return -1; ++ } ++ } /* match */ ++ } /* main loop */ ++ return (int)(ip - istart); ++} ++ ++/* Read the variable-length literal or match length. ++ * ++ * @ip : input pointer ++ * @ilimit : position after which if length is not decoded, the input is necessarily corrupted. ++ * @initial_check - check ip >= ipmax before start of loop. Returns initial_error if so. ++ * @error (output) - error code. Must be set to 0 before call. ++**/ ++typedef size_t Rvl_t; ++static const Rvl_t rvl_error = (Rvl_t)(-1); ++LZ4_FORCE_INLINE Rvl_t read_variable_length(const BYTE **ip, const BYTE *ilimit, ++ int initial_check) ++{ ++ Rvl_t s, length = 0; ++ assert(ip != NULL); ++ assert(*ip != NULL); ++ assert(ilimit != NULL); ++ if (initial_check && ++ unlikely((*ip) >= ilimit)) { /* read limit reached */ ++ return rvl_error; ++ } ++ s = **ip; ++ (*ip)++; ++ length += s; ++ if (unlikely((*ip) > ilimit)) { /* read limit reached */ ++ return rvl_error; ++ } ++ /* accumulator overflow detection (32-bit mode only) */ ++ if ((sizeof(length) < 8) && unlikely(length > ((Rvl_t)(-1) / 2))) { ++ return rvl_error; ++ } ++ if (likely(s != 255)) ++ return length; ++ do { ++ s = **ip; ++ (*ip)++; ++ length += s; ++ if (unlikely((*ip) > ilimit)) { /* read limit reached */ ++ return rvl_error; ++ } ++ /* accumulator overflow detection (32-bit mode only) */ ++ if ((sizeof(length) < 8) && ++ unlikely(length > ((Rvl_t)(-1) / 2))) { ++ return rvl_error; ++ } ++ } while (s == 255); ++ ++ return length; ++} ++ ++/*! LZ4_decompress_generic() : ++ * This generic decompression function covers all use cases. ++ * It shall be instantiated several times, using different sets of directives. ++ * Note that it is important for performance that this function really get inlined, ++ * in order to remove useless branches during compilation optimization. ++ */ ++LZ4_FORCE_INLINE int __LZ4_decompress_generic( ++ const char *const src, char *const dst, const BYTE *ip, BYTE *op, ++ int srcSize, ++ int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ ++ ++ earlyEnd_directive partialDecoding, /* full, partial */ ++ dict_directive dict, /* noDict, withPrefix64k, usingExtDict */ ++ const BYTE *const lowPrefix, /* always <= dst, == dst when no prefix */ ++ const BYTE *const dictStart, /* only if dict==usingExtDict */ ++ const size_t dictSize /* note : = 0 if noDict */ ++) ++{ ++ if ((src == NULL) || (outputSize < 0)) { ++ return -1; ++ } ++ ++ { ++ const BYTE *const iend = src + srcSize; ++ ++ BYTE *const oend = dst + outputSize; ++ BYTE *cpy; ++ ++ const BYTE *const dictEnd = ++ (dictStart == NULL) ? NULL : dictStart + dictSize; ++ ++ const int checkOffset = (dictSize < (int)(64 KB)); ++ ++ /* Set up the "end" pointers for the shortcut. */ ++ const BYTE *const shortiend = ++ iend - 14 /*maxLL*/ - 2 /*offset*/; ++ const BYTE *const shortoend = ++ oend - 14 /*maxLL*/ - 18 /*maxML*/; ++ ++ const BYTE *match; ++ size_t offset; ++ unsigned token; ++ size_t length; ++ ++ DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", ++ srcSize, outputSize); ++ ++ /* Special cases */ ++ assert(lowPrefix <= op); ++ if (unlikely(outputSize == 0)) { ++ /* Empty output buffer */ ++ if (partialDecoding) ++ return 0; ++ return ((srcSize == 1) && (*ip == 0)) ? 0 : -1; ++ } ++ if (unlikely(srcSize == 0)) { ++ return -1; ++ } ++ ++ /* LZ4_FAST_DEC_LOOP: ++ * designed for modern OoO performance cpus, ++ * where copying reliably 32-bytes is preferable to an unpredictable branch. ++ * note : fast loop may show a regression for some client arm chips. */ ++#if LZ4_FAST_DEC_LOOP ++ if ((oend - op) < FASTLOOP_SAFE_DISTANCE) { ++ DEBUGLOG(6, "move to safe decode loop"); ++ goto safe_decode; ++ } ++ ++ /* Fast loop : decode sequences as long as output < oend-FASTLOOP_SAFE_DISTANCE */ ++ DEBUGLOG(6, "using fast decode loop"); ++ while (1) { ++ /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ ++ assert(oend - op >= FASTLOOP_SAFE_DISTANCE); ++ assert(ip < iend); ++ token = *ip++; ++ length = token >> ML_BITS; /* literal length */ ++ DEBUGLOG(7, "blockPos%6u: litLength token = %u", ++ (unsigned)(op - (BYTE *)dst), ++ (unsigned)length); ++ ++ /* decode literal length */ ++ if (length == RUN_MASK) { ++ size_t const addl = read_variable_length( ++ &ip, iend - RUN_MASK, 1); ++ if (addl == rvl_error) { ++ DEBUGLOG( ++ 6, ++ "error reading long literal length"); ++ goto _output_error; ++ } ++ length += addl; ++ if (unlikely((uptrval)(op) + length < ++ (uptrval)(op))) { ++ goto _output_error; ++ } /* overflow detection */ ++ if (unlikely((uptrval)(ip) + length < ++ (uptrval)(ip))) { ++ goto _output_error; ++ } /* overflow detection */ ++ ++ /* copy literals */ ++ LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); ++ if ((op + length > oend - 32) || ++ (ip + length > iend - 32)) { ++ goto safe_literal_copy; ++ } ++ LZ4_wildCopy32(op, ip, op + length); ++ ip += length; ++ op += length; ++ } else if (ip <= ++ iend - (16 + ++ 1 /*max lit + offset + nextToken*/)) { ++ /* We don't need to check oend, since we check it once for each loop below */ ++ DEBUGLOG(7, ++ "copy %u bytes in a 16-bytes stripe", ++ (unsigned)length); ++ /* Literals can only be <= 14, but hope compilers optimize better when copy by a register size */ ++ LZ4_memcpy(op, ip, 16); ++ ip += length; ++ op += length; ++ } else { ++ goto safe_literal_copy; ++ } ++ ++ /* get offset */ ++ offset = LZ4_readLE16(ip); ++ ip += 2; ++ DEBUGLOG(6, "blockPos%6u: offset = %u", ++ (unsigned)(op - (BYTE *)dst), ++ (unsigned)offset); ++ match = op - offset; ++ assert(match <= op); /* overflow check */ ++ ++ /* get matchlength */ ++ length = token & ML_MASK; ++ DEBUGLOG(7, " match length token = %u (len==%u)", ++ (unsigned)length, (unsigned)length + MINMATCH); ++ ++ if (length == ML_MASK) { ++ size_t const addl = read_variable_length( ++ &ip, iend - LASTLITERALS + 1, 0); ++ if (addl == rvl_error) { ++ DEBUGLOG( ++ 5, ++ "error reading long match length"); ++ goto _output_error; ++ } ++ length += addl; ++ length += MINMATCH; ++ DEBUGLOG(7, " long match length == %u", ++ (unsigned)length); ++ if (unlikely((uptrval)(op) + length < ++ (uptrval)op)) { ++ goto _output_error; ++ } /* overflow detection */ ++ if (op + length >= ++ oend - FASTLOOP_SAFE_DISTANCE) { ++ goto safe_match_copy; ++ } ++ } else { ++ length += MINMATCH; ++ if (op + length >= ++ oend - FASTLOOP_SAFE_DISTANCE) { ++ DEBUGLOG( ++ 7, ++ "moving to safe_match_copy (ml==%u)", ++ (unsigned)length); ++ goto safe_match_copy; ++ } ++ ++ /* Fastpath check: skip LZ4_wildCopy32 when true */ ++ if ((dict == withPrefix64k) || ++ (match >= lowPrefix)) { ++ if (offset >= 8) { ++ assert(match >= lowPrefix); ++ assert(match <= op); ++ assert(op + 18 <= oend); ++ ++ LZ4_memcpy(op, match, 8); ++ LZ4_memcpy(op + 8, match + 8, ++ 8); ++ LZ4_memcpy(op + 16, match + 16, ++ 2); ++ op += length; ++ continue; ++ } ++ } ++ } ++ ++ if (checkOffset && ++ (unlikely(match + dictSize < lowPrefix))) { ++ DEBUGLOG( ++ 5, ++ "Error : pos=%zi, offset=%zi => outside buffers", ++ op - lowPrefix, op - match); ++ goto _output_error; ++ } ++ /* match starting within external dictionary */ ++ if ((dict == usingExtDict) && (match < lowPrefix)) { ++ assert(dictEnd != NULL); ++ if (unlikely(op + length > ++ oend - LASTLITERALS)) { ++ if (partialDecoding) { ++ DEBUGLOG( ++ 7, ++ "partialDecoding: dictionary match, close to dstEnd"); ++ length = MIN( ++ length, ++ (size_t)(oend - op)); ++ } else { ++ DEBUGLOG( ++ 6, ++ "end-of-block condition violated") ++ goto _output_error; ++ } ++ } ++ ++ if (length <= (size_t)(lowPrefix - match)) { ++ /* match fits entirely within external dictionary : just copy */ ++ LZ4_memmove(op, ++ dictEnd - ++ (lowPrefix - match), ++ length); ++ op += length; ++ } else { ++ /* match stretches into both external dictionary and current block */ ++ size_t const copySize = ++ (size_t)(lowPrefix - match); ++ size_t const restSize = ++ length - copySize; ++ LZ4_memcpy(op, dictEnd - copySize, ++ copySize); ++ op += copySize; ++ if (restSize > ++ (size_t)(op - ++ lowPrefix)) { /* overlap copy */ ++ BYTE *const endOfMatch = ++ op + restSize; ++ const BYTE *copyFrom = ++ lowPrefix; ++ while (op < endOfMatch) { ++ *op++ = *copyFrom++; ++ } ++ } else { ++ LZ4_memcpy(op, lowPrefix, ++ restSize); ++ op += restSize; ++ } ++ } ++ continue; ++ } ++ ++ /* copy match within block */ ++ cpy = op + length; ++ ++ assert((op <= oend) && (oend - op >= 32)); ++ if (unlikely(offset < 16)) { ++ LZ4_memcpy_using_offset(op, match, cpy, offset); ++ } else { ++ LZ4_wildCopy32(op, match, cpy); ++ } ++ ++ op = cpy; /* wildcopy correction */ ++ } ++ safe_decode: ++#endif ++ ++ /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */ ++ DEBUGLOG(6, "using safe decode loop"); ++ while (1) { ++ assert(ip < iend); ++ token = *ip++; ++ length = token >> ML_BITS; /* literal length */ ++ DEBUGLOG(7, "blockPos%6u: litLength token = %u", ++ (unsigned)(op - (BYTE *)dst), ++ (unsigned)length); ++ ++ /* A two-stage shortcut for the most common case: ++ * 1) If the literal length is 0..14, and there is enough space, ++ * enter the shortcut and copy 16 bytes on behalf of the literals ++ * (in the fast mode, only 8 bytes can be safely copied this way). ++ * 2) Further if the match length is 4..18, copy 18 bytes in a similar ++ * manner; but we ensure that there's enough space in the output for ++ * those 18 bytes earlier, upon entering the shortcut (in other words, ++ * there is a combined check for both stages). ++ */ ++ if ((length != RUN_MASK) ++ /* strictly "less than" on input, to re-enter the loop with at least one byte */ ++ && likely((ip < shortiend) & (op <= shortoend))) { ++ /* Copy the literals */ ++ LZ4_memcpy(op, ip, 16); ++ op += length; ++ ip += length; ++ ++ /* The second stage: prepare for match copying, decode full info. ++ * If it doesn't work out, the info won't be wasted. */ ++ length = token & ML_MASK; /* match length */ ++ DEBUGLOG( ++ 7, ++ "blockPos%6u: matchLength token = %u (len=%u)", ++ (unsigned)(op - (BYTE *)dst), ++ (unsigned)length, (unsigned)length + 4); ++ offset = LZ4_readLE16(ip); ++ ip += 2; ++ match = op - offset; ++ assert(match <= op); /* check overflow */ ++ ++ /* Do not deal with overlapping matches. */ ++ if ((length != ML_MASK) && (offset >= 8) && ++ (dict == withPrefix64k || ++ match >= lowPrefix)) { ++ /* Copy the match. */ ++ LZ4_memcpy(op + 0, match + 0, 8); ++ LZ4_memcpy(op + 8, match + 8, 8); ++ LZ4_memcpy(op + 16, match + 16, 2); ++ op += length + MINMATCH; ++ /* Both stages worked, load the next token. */ ++ continue; ++ } ++ ++ /* The second stage didn't work out, but the info is ready. ++ * Propel it right to the point of match copying. */ ++ goto _copy_match; ++ } ++ ++ /* decode literal length */ ++ if (length == RUN_MASK) { ++ size_t const addl = read_variable_length( ++ &ip, iend - RUN_MASK, 1); ++ if (addl == rvl_error) { ++ goto _output_error; ++ } ++ length += addl; ++ if (unlikely((uptrval)(op) + length < ++ (uptrval)(op))) { ++ goto _output_error; ++ } /* overflow detection */ ++ if (unlikely((uptrval)(ip) + length < ++ (uptrval)(ip))) { ++ goto _output_error; ++ } /* overflow detection */ ++ } ++ ++#if LZ4_FAST_DEC_LOOP ++ safe_literal_copy: ++#endif ++ /* copy literals */ ++ cpy = op + length; ++ ++ LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); ++ if ((cpy > oend - MFLIMIT) || ++ (ip + length > iend - (2 + 1 + LASTLITERALS))) { ++ /* We've either hit the input parsing restriction or the output parsing restriction. ++ * In the normal scenario, decoding a full block, it must be the last sequence, ++ * otherwise it's an error (invalid input or dimensions). ++ * In partialDecoding scenario, it's necessary to ensure there is no buffer overflow. ++ */ ++ if (partialDecoding) { ++ /* Since we are partial decoding we may be in this block because of the output parsing ++ * restriction, which is not valid since the output buffer is allowed to be undersized. ++ */ ++ DEBUGLOG( ++ 7, ++ "partialDecoding: copying literals, close to input or output end") ++ DEBUGLOG( ++ 7, ++ "partialDecoding: literal length = %u", ++ (unsigned)length); ++ DEBUGLOG( ++ 7, ++ "partialDecoding: remaining space in dstBuffer : %i", ++ (int)(oend - op)); ++ DEBUGLOG( ++ 7, ++ "partialDecoding: remaining space in srcBuffer : %i", ++ (int)(iend - ip)); ++ /* Finishing in the middle of a literals segment, ++ * due to lack of input. ++ */ ++ if (ip + length > iend) { ++ length = (size_t)(iend - ip); ++ cpy = op + length; ++ } ++ /* Finishing in the middle of a literals segment, ++ * due to lack of output space. ++ */ ++ if (cpy > oend) { ++ cpy = oend; ++ assert(op <= oend); ++ length = (size_t)(oend - op); ++ } ++ } else { ++ /* We must be on the last sequence (or invalid) because of the parsing limitations ++ * so check that we exactly consume the input and don't overrun the output buffer. ++ */ ++ if ((ip + length != iend) || ++ (cpy > oend)) { ++ DEBUGLOG( ++ 5, ++ "should have been last run of literals") ++ DEBUGLOG( ++ 5, ++ "ip(%p) + length(%i) = %p != iend (%p)", ++ ip, (int)length, ++ ip + length, iend); ++ DEBUGLOG( ++ 5, ++ "or cpy(%p) > (oend-MFLIMIT)(%p)", ++ cpy, oend - MFLIMIT); ++ DEBUGLOG( ++ 5, ++ "after writing %u bytes / %i bytes available", ++ (unsigned)(op - ++ (BYTE *)dst), ++ outputSize); ++ goto _output_error; ++ } ++ } ++ LZ4_memmove( ++ op, ip, ++ length); /* supports overlapping memory regions, for in-place decompression scenarios */ ++ ip += length; ++ op += length; ++ /* Necessarily EOF when !partialDecoding. ++ * When partialDecoding, it is EOF if we've either ++ * filled the output buffer or ++ * can't proceed with reading an offset for following match. ++ */ ++ if (!partialDecoding || (cpy == oend) || ++ (ip >= (iend - 2))) { ++ break; ++ } ++ } else { ++ LZ4_wildCopy8( ++ op, ip, ++ cpy); /* can overwrite up to 8 bytes beyond cpy */ ++ ip += length; ++ op = cpy; ++ } ++ ++ /* get offset */ ++ offset = LZ4_readLE16(ip); ++ ip += 2; ++ match = op - offset; ++ ++ /* get matchlength */ ++ length = token & ML_MASK; ++ DEBUGLOG(7, "blockPos%6u: matchLength token = %u", ++ (unsigned)(op - (BYTE *)dst), ++ (unsigned)length); ++ ++ _copy_match: ++ if (length == ML_MASK) { ++ size_t const addl = read_variable_length( ++ &ip, iend - LASTLITERALS + 1, 0); ++ if (addl == rvl_error) { ++ goto _output_error; ++ } ++ length += addl; ++ if (unlikely((uptrval)(op) + length < ++ (uptrval)op)) ++ goto _output_error; /* overflow detection */ ++ } ++ length += MINMATCH; ++ ++#if LZ4_FAST_DEC_LOOP ++ safe_match_copy: ++#endif ++ if ((checkOffset) && ++ (unlikely(match + dictSize < lowPrefix))) ++ goto _output_error; /* Error : offset outside buffers */ ++ /* match starting within external dictionary */ ++ if ((dict == usingExtDict) && (match < lowPrefix)) { ++ assert(dictEnd != NULL); ++ if (unlikely(op + length > ++ oend - LASTLITERALS)) { ++ if (partialDecoding) ++ length = MIN( ++ length, ++ (size_t)(oend - op)); ++ else ++ goto _output_error; /* doesn't respect parsing restriction */ ++ } ++ ++ if (length <= (size_t)(lowPrefix - match)) { ++ /* match fits entirely within external dictionary : just copy */ ++ LZ4_memmove(op, ++ dictEnd - ++ (lowPrefix - match), ++ length); ++ op += length; ++ } else { ++ /* match stretches into both external dictionary and current block */ ++ size_t const copySize = ++ (size_t)(lowPrefix - match); ++ size_t const restSize = ++ length - copySize; ++ LZ4_memcpy(op, dictEnd - copySize, ++ copySize); ++ op += copySize; ++ if (restSize > ++ (size_t)(op - ++ lowPrefix)) { /* overlap copy */ ++ BYTE *const endOfMatch = ++ op + restSize; ++ const BYTE *copyFrom = ++ lowPrefix; ++ while (op < endOfMatch) ++ *op++ = *copyFrom++; ++ } else { ++ LZ4_memcpy(op, lowPrefix, ++ restSize); ++ op += restSize; ++ } ++ } ++ continue; ++ } ++ assert(match >= lowPrefix); ++ ++ /* copy match within block */ ++ cpy = op + length; ++ ++ /* partialDecoding : may end anywhere within the block */ ++ assert(op <= oend); ++ if (partialDecoding && ++ (cpy > oend - MATCH_SAFEGUARD_DISTANCE)) { ++ size_t const mlen = ++ MIN(length, (size_t)(oend - op)); ++ const BYTE *const matchEnd = match + mlen; ++ BYTE *const copyEnd = op + mlen; ++ if (matchEnd > op) { /* overlap copy */ ++ while (op < copyEnd) { ++ *op++ = *match++; ++ } ++ } else { ++ LZ4_memcpy(op, match, mlen); ++ } ++ op = copyEnd; ++ if (op == oend) { ++ break; ++ } ++ continue; ++ } ++ ++ if (unlikely(offset < 8)) { ++ LZ4_write32( ++ op, ++ 0); /* silence msan warning when offset==0 */ ++ op[0] = match[0]; ++ op[1] = match[1]; ++ op[2] = match[2]; ++ op[3] = match[3]; ++ match += inc32table[offset]; ++ LZ4_memcpy(op + 4, match, 4); ++ match -= dec64table[offset]; ++ } else { ++ LZ4_memcpy(op, match, 8); ++ match += 8; ++ } ++ op += 8; ++ ++ if (unlikely(cpy > oend - MATCH_SAFEGUARD_DISTANCE)) { ++ BYTE *const oCopyLimit = ++ oend - (WILDCOPYLENGTH - 1); ++ if (cpy > oend - LASTLITERALS) { ++ goto _output_error; ++ } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ ++ if (op < oCopyLimit) { ++ LZ4_wildCopy8(op, match, oCopyLimit); ++ match += oCopyLimit - op; ++ op = oCopyLimit; ++ } ++ while (op < cpy) { ++ *op++ = *match++; ++ } ++ } else { ++ LZ4_memcpy(op, match, 8); ++ if (length > 16) { ++ LZ4_wildCopy8(op + 8, match + 8, cpy); ++ } ++ } ++ op = cpy; /* wildcopy correction */ ++ } ++ ++ /* end of decoding */ ++ DEBUGLOG(5, "decoded %i bytes", (int)(((char *)op) - dst)); ++ return (int)(((char *)op) - ++ dst); /* Nb of output bytes decoded */ ++ ++ /* Overflow error detected */ ++ _output_error: ++ return (int)(-(((const char *)ip) - src)) - 1; ++ } ++} ++ ++/*===== Instantiate the API decoding functions. =====*/ ++ ++LZ4_FORCE_INLINE int ++LZ4_decompress_generic(const char *const src, char *const dst, int srcSize, ++ /* ++ * If endOnInput == endOnInputSize, ++ * this value is `dstCapacity` ++ */ ++ int outputSize, ++ /* full, partial */ ++ earlyEnd_directive partialDecoding, ++ /* noDict, withPrefix64k, usingExtDict */ ++ dict_directive dict, ++ /* always <= dst, == dst when no prefix */ ++ const BYTE *const lowPrefix, ++ /* only if dict == usingExtDict */ ++ const BYTE *const dictStart, ++ /* note : = 0 if noDict */ ++ const size_t dictSize) ++{ ++ return __LZ4_decompress_generic(src, dst, (const BYTE *)src, ++ (BYTE *)dst, srcSize, outputSize, ++ partialDecoding, dict, lowPrefix, ++ dictStart, dictSize); ++} ++ ++LZ4_FORCE_O2 ++int LZ4_decompress_safe(const char *source, char *dest, int compressedSize, ++ int maxDecompressedSize) ++{ ++ return LZ4_decompress_generic(source, dest, compressedSize, ++ maxDecompressedSize, decode_full_block, ++ noDict, (BYTE *)dest, NULL, 0); ++} ++EXPORT_SYMBOL(LZ4_decompress_safe); ++ ++LZ4_FORCE_O2 ++int LZ4_decompress_safe_partial(const char *src, char *dst, int compressedSize, ++ int targetOutputSize, int dstCapacity) ++{ ++ dstCapacity = MIN(targetOutputSize, dstCapacity); ++ return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity, ++ partial_decode, noDict, (BYTE *)dst, NULL, ++ 0); ++} ++EXPORT_SYMBOL(LZ4_decompress_safe_partial); ++ ++LZ4_FORCE_O2 ++int LZ4_decompress_fast(const char *source, char *dest, int originalSize) ++{ ++ DEBUGLOG(5, "LZ4_decompress_fast"); ++ return LZ4_decompress_unsafe_generic((const BYTE *)source, (BYTE *)dest, ++ originalSize, 0, NULL, 0); ++} ++EXPORT_SYMBOL(LZ4_decompress_fast); ++ ++/*===== Instantiate a few more decoding cases, used more than once. =====*/ ++ ++LZ4_FORCE_O2 /* Exported, an obsolete API function. */ ++ int ++ LZ4_decompress_safe_withPrefix64k(const char *source, char *dest, ++ int compressedSize, int maxOutputSize) ++{ ++ return LZ4_decompress_generic(source, dest, compressedSize, ++ maxOutputSize, decode_full_block, ++ withPrefix64k, (BYTE *)dest - 64 KB, NULL, ++ 0); ++} ++ ++LZ4_FORCE_O2 ++static int LZ4_decompress_safe_partial_withPrefix64k(const char *source, ++ char *dest, ++ int compressedSize, ++ int targetOutputSize, ++ int dstCapacity) ++{ ++ dstCapacity = MIN(targetOutputSize, dstCapacity); ++ return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, ++ partial_decode, withPrefix64k, ++ (BYTE *)dest - 64 KB, NULL, 0); ++} ++ ++/* Another obsolete API function, paired with the previous one. */ ++int LZ4_decompress_fast_withPrefix64k(const char *source, char *dest, ++ int originalSize) ++{ ++ return LZ4_decompress_unsafe_generic((const BYTE *)source, (BYTE *)dest, ++ originalSize, 64 KB, NULL, 0); ++} ++ ++LZ4_FORCE_O2 ++static int LZ4_decompress_safe_withSmallPrefix(const char *source, char *dest, ++ int compressedSize, ++ int maxOutputSize, ++ size_t prefixSize) ++{ ++ return LZ4_decompress_generic(source, dest, compressedSize, ++ maxOutputSize, decode_full_block, noDict, ++ (BYTE *)dest - prefixSize, NULL, 0); ++} ++ ++LZ4_FORCE_O2 ++static int LZ4_decompress_safe_partial_withSmallPrefix( ++ const char *source, char *dest, int compressedSize, ++ int targetOutputSize, int dstCapacity, size_t prefixSize) ++{ ++ dstCapacity = MIN(targetOutputSize, dstCapacity); ++ return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, ++ partial_decode, noDict, ++ (BYTE *)dest - prefixSize, NULL, 0); ++} ++ ++LZ4_FORCE_O2 ++int LZ4_decompress_safe_forceExtDict(const char *source, char *dest, ++ int compressedSize, int maxOutputSize, ++ const void *dictStart, size_t dictSize) ++{ ++ DEBUGLOG(5, "LZ4_decompress_safe_forceExtDict"); ++ return LZ4_decompress_generic(source, dest, compressedSize, ++ maxOutputSize, decode_full_block, ++ usingExtDict, (BYTE *)dest, ++ (const BYTE *)dictStart, dictSize); ++} ++ ++LZ4_FORCE_O2 ++int LZ4_decompress_safe_partial_forceExtDict(const char *source, char *dest, ++ int compressedSize, ++ int targetOutputSize, ++ int dstCapacity, ++ const void *dictStart, ++ size_t dictSize) ++{ ++ dstCapacity = MIN(targetOutputSize, dstCapacity); ++ return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, ++ partial_decode, usingExtDict, ++ (BYTE *)dest, (const BYTE *)dictStart, ++ dictSize); ++} ++ ++LZ4_FORCE_O2 ++static int LZ4_decompress_fast_extDict(const char *source, char *dest, ++ int originalSize, const void *dictStart, ++ size_t dictSize) ++{ ++ return LZ4_decompress_unsafe_generic((const BYTE *)source, (BYTE *)dest, ++ originalSize, 0, ++ (const BYTE *)dictStart, dictSize); ++} ++ ++/* The "double dictionary" mode, for use with e.g. ring buffers: the first part ++ * of the dictionary is passed as prefix, and the second via dictStart + dictSize. ++ * These routines are used only once, in LZ4_decompress_*_continue(). ++ */ ++LZ4_FORCE_INLINE ++int LZ4_decompress_safe_doubleDict(const char *source, char *dest, ++ int compressedSize, int maxOutputSize, ++ size_t prefixSize, const void *dictStart, ++ size_t dictSize) ++{ ++ return LZ4_decompress_generic(source, dest, compressedSize, ++ maxOutputSize, decode_full_block, ++ usingExtDict, (BYTE *)dest - prefixSize, ++ (const BYTE *)dictStart, dictSize); ++} ++ ++/*===== streaming decompression functions =====*/ ++ ++#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) ++LZ4_streamDecode_t *LZ4_createStreamDecode(void) ++{ ++ LZ4_STATIC_ASSERT(sizeof(LZ4_streamDecode_t) >= ++ sizeof(LZ4_streamDecode_t_internal)); ++ return (LZ4_streamDecode_t *)ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t)); ++} ++ ++int LZ4_freeStreamDecode(LZ4_streamDecode_t *LZ4_stream) ++{ ++ if (LZ4_stream == NULL) { ++ return 0; ++ } /* support free on NULL */ ++ FREEMEM(LZ4_stream); ++ return 0; ++} ++#endif ++ ++/*! LZ4_setStreamDecode() : ++ * Use this function to instruct where to find the dictionary. ++ * This function is not necessary if previous data is still available where it was decoded. ++ * Loading a size of 0 is allowed (same effect as no dictionary). ++ * @return : 1 if OK, 0 if error ++ */ ++int LZ4_setStreamDecode(LZ4_streamDecode_t *LZ4_streamDecode, ++ const char *dictionary, int dictSize) ++{ ++ LZ4_streamDecode_t_internal *lz4sd = ++ &LZ4_streamDecode->internal_donotuse; ++ lz4sd->prefixSize = (size_t)dictSize; ++ if (dictSize) { ++ assert(dictionary != NULL); ++ lz4sd->prefixEnd = (const BYTE *)dictionary + dictSize; ++ } else { ++ lz4sd->prefixEnd = (const BYTE *)dictionary; ++ } ++ lz4sd->externalDict = NULL; ++ lz4sd->extDictSize = 0; ++ return 1; ++} ++EXPORT_SYMBOL(LZ4_setStreamDecode); ++ ++/*! LZ4_decoderRingBufferSize() : ++ * when setting a ring buffer for streaming decompression (optional scenario), ++ * provides the minimum size of this ring buffer ++ * to be compatible with any source respecting maxBlockSize condition. ++ * Note : in a ring buffer scenario, ++ * blocks are presumed decompressed next to each other. ++ * When not enough space remains for next block (remainingSize < maxBlockSize), ++ * decoding resumes from beginning of ring buffer. ++ * @return : minimum ring buffer size, ++ * or 0 if there is an error (invalid maxBlockSize). ++ */ ++int LZ4_decoderRingBufferSize(int maxBlockSize) ++{ ++ if (maxBlockSize < 0) ++ return 0; ++ if (maxBlockSize > LZ4_MAX_INPUT_SIZE) ++ return 0; ++ if (maxBlockSize < 16) ++ maxBlockSize = 16; ++ return LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize); ++} ++ ++/* ++*_continue() : ++ These decoding functions allow decompression of multiple blocks in "streaming" mode. ++ Previously decoded blocks must still be available at the memory position where they were decoded. ++ If it's not possible, save the relevant part of decoded data into a safe buffer, ++ and indicate where it stands using LZ4_setStreamDecode() ++*/ ++LZ4_FORCE_O2 ++int LZ4_decompress_safe_continue(LZ4_streamDecode_t *LZ4_streamDecode, ++ const char *source, char *dest, ++ int compressedSize, int maxOutputSize) ++{ ++ LZ4_streamDecode_t_internal *lz4sd = ++ &LZ4_streamDecode->internal_donotuse; ++ int result; ++ ++ if (lz4sd->prefixSize == 0) { ++ /* The first call, no dictionary yet. */ ++ assert(lz4sd->extDictSize == 0); ++#if defined(CONFIG_ARM64) && defined(CONFIG_KERNEL_MODE_NEON) ++ result = LZ4_arm64_decompress_safe(source, dest, compressedSize, ++ maxOutputSize, false); ++#else ++ result = LZ4_decompress_safe(source, dest, compressedSize, ++ maxOutputSize); ++#endif ++ if (result <= 0) ++ return result; ++ lz4sd->prefixSize = (size_t)result; ++ lz4sd->prefixEnd = (BYTE *)dest + result; ++ } else if (lz4sd->prefixEnd == (BYTE *)dest) { ++ /* They're rolling the current segment. */ ++ if (lz4sd->prefixSize >= 64 KB - 1) ++ result = LZ4_decompress_safe_withPrefix64k( ++ source, dest, compressedSize, maxOutputSize); ++ else if (lz4sd->extDictSize == 0) ++ result = LZ4_decompress_safe_withSmallPrefix( ++ source, dest, compressedSize, maxOutputSize, ++ lz4sd->prefixSize); ++ else ++ result = LZ4_decompress_safe_doubleDict( ++ source, dest, compressedSize, maxOutputSize, ++ lz4sd->prefixSize, lz4sd->externalDict, ++ lz4sd->extDictSize); ++ if (result <= 0) ++ return result; ++ lz4sd->prefixSize += (size_t)result; ++ lz4sd->prefixEnd += result; ++ } else { ++ /* The buffer wraps around, or they're switching to another buffer. */ ++ lz4sd->extDictSize = lz4sd->prefixSize; ++ lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; ++ result = LZ4_decompress_safe_forceExtDict( ++ source, dest, compressedSize, maxOutputSize, ++ lz4sd->externalDict, lz4sd->extDictSize); ++ if (result <= 0) ++ return result; ++ lz4sd->prefixSize = (size_t)result; ++ lz4sd->prefixEnd = (BYTE *)dest + result; ++ } ++ ++ return result; ++} ++EXPORT_SYMBOL(LZ4_decompress_safe_continue); ++ ++LZ4_FORCE_O2 ssize_t LZ4_arm64_decompress_safe_partial(const void *source, ++ void *dest, ++ size_t inputSize, ++ size_t outputSize, ++ bool dip) ++{ ++ uint8_t *dstPtr = dest; ++ const uint8_t *srcPtr = source; ++ ssize_t ret; ++ ++#ifdef __ARCH_HAS_LZ4_ACCELERATOR ++ /* Go fast if we can, keeping away from the end of buffers */ ++ if (outputSize > LZ4_FAST_MARGIN && inputSize > LZ4_FAST_MARGIN && ++ lz4_decompress_accel_enable()) { ++ ret = lz4_decompress_asm( ++ &dstPtr, dest, dest + outputSize - LZ4_FAST_MARGIN, ++ &srcPtr, source + inputSize - LZ4_FAST_MARGIN, dip); ++ if (ret) ++ return -EIO; ++ } ++#endif ++ /* Finish in safe */ ++ return __LZ4_decompress_generic(source, dest, srcPtr, dstPtr, inputSize, ++ outputSize, partial_decode, noDict, ++ (BYTE *)dest, NULL, 0); ++} ++EXPORT_SYMBOL(LZ4_arm64_decompress_safe_partial); ++ ++LZ4_FORCE_O2 ssize_t LZ4_arm64_decompress_safe(const void *source, void *dest, ++ size_t inputSize, ++ size_t outputSize, bool dip) ++{ ++ uint8_t *dstPtr = dest; ++ const uint8_t *srcPtr = source; ++ ssize_t ret; ++ ++#ifdef __ARCH_HAS_LZ4_ACCELERATOR ++ /* Go fast if we can, keeping away from the end of buffers */ ++ if (outputSize > LZ4_FAST_MARGIN && inputSize > LZ4_FAST_MARGIN && ++ lz4_decompress_accel_enable()) { ++ ret = lz4_decompress_asm( ++ &dstPtr, dest, dest + outputSize - LZ4_FAST_MARGIN, ++ &srcPtr, source + inputSize - LZ4_FAST_MARGIN, dip); ++ if (ret) ++ return -EIO; ++ } ++#endif ++ /* Finish in safe */ ++ return __LZ4_decompress_generic(source, dest, srcPtr, dstPtr, inputSize, ++ outputSize, decode_full_block, noDict, ++ (BYTE *)dest, NULL, 0); ++} ++EXPORT_SYMBOL(LZ4_arm64_decompress_safe); ++ ++LZ4_FORCE_O2 int ++LZ4_decompress_fast_continue(LZ4_streamDecode_t *LZ4_streamDecode, ++ const char *source, char *dest, int originalSize) ++{ ++ LZ4_streamDecode_t_internal *const lz4sd = ++ (assert(LZ4_streamDecode != NULL), ++ &LZ4_streamDecode->internal_donotuse); ++ int result; ++ ++ DEBUGLOG(5, "LZ4_decompress_fast_continue (toDecodeSize=%i)", ++ originalSize); ++ assert(originalSize >= 0); ++ ++ if (lz4sd->prefixSize == 0) { ++ DEBUGLOG(5, "first invocation : no prefix nor extDict"); ++ assert(lz4sd->extDictSize == 0); ++ result = LZ4_decompress_fast(source, dest, originalSize); ++ if (result <= 0) ++ return result; ++ lz4sd->prefixSize = (size_t)originalSize; ++ lz4sd->prefixEnd = (BYTE *)dest + originalSize; ++ } else if (lz4sd->prefixEnd == (BYTE *)dest) { ++ DEBUGLOG(5, "continue using existing prefix"); ++ result = LZ4_decompress_unsafe_generic( ++ (const BYTE *)source, (BYTE *)dest, originalSize, ++ lz4sd->prefixSize, lz4sd->externalDict, ++ lz4sd->extDictSize); ++ if (result <= 0) ++ return result; ++ lz4sd->prefixSize += (size_t)originalSize; ++ lz4sd->prefixEnd += originalSize; ++ } else { ++ DEBUGLOG(5, "prefix becomes extDict"); ++ lz4sd->extDictSize = lz4sd->prefixSize; ++ lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; ++ result = LZ4_decompress_fast_extDict(source, dest, originalSize, ++ lz4sd->externalDict, ++ lz4sd->extDictSize); ++ if (result <= 0) ++ return result; ++ lz4sd->prefixSize = (size_t)originalSize; ++ lz4sd->prefixEnd = (BYTE *)dest + originalSize; ++ } ++ ++ return result; ++} ++EXPORT_SYMBOL(LZ4_decompress_fast_continue); ++ ++/* ++Advanced decoding functions : ++*_usingDict() : ++ These decoding functions work the same as "_continue" ones, ++ the dictionary must be explicitly provided within parameters ++*/ ++ ++int LZ4_decompress_safe_usingDict(const char *source, char *dest, ++ int compressedSize, int maxOutputSize, ++ const char *dictStart, int dictSize) ++{ ++ if (dictSize == 0) ++ return LZ4_decompress_safe(source, dest, compressedSize, ++ maxOutputSize); ++ if (dictStart + dictSize == dest) { ++ if (dictSize >= 64 KB - 1) { ++ return LZ4_decompress_safe_withPrefix64k( ++ source, dest, compressedSize, maxOutputSize); ++ } ++ assert(dictSize >= 0); ++ return LZ4_decompress_safe_withSmallPrefix(source, dest, ++ compressedSize, ++ maxOutputSize, ++ (size_t)dictSize); ++ } ++ assert(dictSize >= 0); ++ return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, ++ maxOutputSize, dictStart, ++ (size_t)dictSize); ++} ++EXPORT_SYMBOL(LZ4_decompress_safe_usingDict); ++ ++int LZ4_decompress_safe_partial_usingDict(const char *source, char *dest, ++ int compressedSize, ++ int targetOutputSize, int dstCapacity, ++ const char *dictStart, int dictSize) ++{ ++ if (dictSize == 0) ++ return LZ4_decompress_safe_partial(source, dest, compressedSize, ++ targetOutputSize, ++ dstCapacity); ++ if (dictStart + dictSize == dest) { ++ if (dictSize >= 64 KB - 1) { ++ return LZ4_decompress_safe_partial_withPrefix64k( ++ source, dest, compressedSize, targetOutputSize, ++ dstCapacity); ++ } ++ assert(dictSize >= 0); ++ return LZ4_decompress_safe_partial_withSmallPrefix( ++ source, dest, compressedSize, targetOutputSize, ++ dstCapacity, (size_t)dictSize); ++ } ++ assert(dictSize >= 0); ++ return LZ4_decompress_safe_partial_forceExtDict( ++ source, dest, compressedSize, targetOutputSize, dstCapacity, ++ dictStart, (size_t)dictSize); ++} ++ ++int LZ4_decompress_fast_usingDict(const char *source, char *dest, ++ int originalSize, const char *dictStart, ++ int dictSize) ++{ ++ if (dictSize == 0 || dictStart + dictSize == dest) ++ return LZ4_decompress_unsafe_generic((const BYTE *)source, ++ (BYTE *)dest, originalSize, ++ (size_t)dictSize, NULL, 0); ++ assert(dictSize >= 0); ++ return LZ4_decompress_fast_extDict(source, dest, originalSize, ++ dictStart, (size_t)dictSize); ++} ++EXPORT_SYMBOL(LZ4_decompress_fast_usingDict); ++ ++/* ++These decompression functions are deprecated and should no longer be used. ++They are only provided here for compatibility with older user programs. ++- LZ4_uncompress is totally equivalent to LZ4_decompress_fast ++- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe ++*/ ++int LZ4_uncompress(const char *source, char *dest, int outputSize) ++{ ++ return LZ4_decompress_fast(source, dest, outputSize); ++} ++int LZ4_uncompress_unknownOutputSize(const char *source, char *dest, int isize, ++ int maxOutputSize) ++{ ++ return LZ4_decompress_safe(source, dest, isize, maxOutputSize); ++} ++ ++/* Obsolete Streaming functions */ ++ ++int LZ4_sizeofStreamState(void) ++{ ++ return sizeof(LZ4_stream_t); ++} ++ ++int LZ4_resetStreamState(void *state, char *inputBuffer) ++{ ++ (void)inputBuffer; ++ LZ4_resetStream((LZ4_stream_t *)state); ++ return 0; ++} ++ ++#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) ++void *LZ4_create(char *inputBuffer) ++{ ++ (void)inputBuffer; ++ return LZ4_createStream(); ++} ++#endif ++ ++char *LZ4_slideInputBuffer(void *state) ++{ ++ /* avoid const char * -> char * conversion warning */ ++ return (char *)(uptrval)((LZ4_stream_t *)state) ++ ->internal_donotuse.dictionary; ++} ++ ++#endif /* LZ4_COMMONDEFS_ONLY */ +diff --git a/lib/lz4/lz4.h b/lib/lz4/lz4.h +new file mode 100644 +index 000000000000..580a2e3f6f37 +--- /dev/null ++++ b/lib/lz4/lz4.h +@@ -0,0 +1,984 @@ ++/* ++ * LZ4 - Fast LZ compression algorithm ++ * Header File ++ * Copyright (C) 2011-2023, Yann Collet. ++ ++ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) ++ ++ Redistribution and use in source and binary forms, with or without ++ modification, are permitted provided that the following conditions are ++ met: ++ ++ * Redistributions of source code must retain the above copyright ++ notice, this list of conditions and the following disclaimer. ++ * Redistributions in binary form must reproduce the above ++ copyright notice, this list of conditions and the following disclaimer ++ in the documentation and/or other materials provided with the ++ distribution. ++ ++ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ++ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ++ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ++ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ++ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ++ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ++ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ++ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ ++ You can contact the author at : ++ - LZ4 homepage : http://www.lz4.org ++ - LZ4 source repository : https://github.com/lz4/lz4 ++*/ ++#if defined(__cplusplus) ++extern "C" { ++#endif ++ ++#ifndef LZ4_H_2983827168210 ++#define LZ4_H_2983827168210 ++ ++/** ++ Introduction ++ ++ LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core, ++ scalable with multi-cores CPU. It features an extremely fast decoder, with speed in ++ multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. ++ ++ The LZ4 compression library provides in-memory compression and decompression functions. ++ It gives full buffer control to user. ++ Compression can be done in: ++ - a single step (described as Simple Functions) ++ - a single step, reusing a context (described in Advanced Functions) ++ - unbounded multiple steps (described as Streaming compression) ++ ++ lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md). ++ Decompressing such a compressed block requires additional metadata. ++ Exact metadata depends on exact decompression function. ++ For the typical case of LZ4_decompress_safe(), ++ metadata includes block's compressed size, and maximum bound of decompressed size. ++ Each application is free to encode and pass such metadata in whichever way it wants. ++ ++ lz4.h only handle blocks, it can not generate Frames. ++ ++ Blocks are different from Frames (doc/lz4_Frame_format.md). ++ Frames bundle both blocks and metadata in a specified manner. ++ Embedding metadata is required for compressed data to be self-contained and portable. ++ Frame format is delivered through a companion API, declared in lz4frame.h. ++ The `lz4` CLI can only manage frames. ++*/ ++ ++#include ++#include ++#include ++ ++#include "lz4armv8/lz4accel.h" ++ ++#define LZ4_FORCE_INLINE static inline __attribute__((always_inline)) ++ ++/*^*************************************************************** ++* Export parameters ++*****************************************************************/ ++/* ++* LZ4_DLL_EXPORT : ++* Enable exporting of functions when building a Windows DLL ++* LZ4LIB_VISIBILITY : ++* Control library symbols visibility. ++*/ ++#ifndef LZ4LIB_VISIBILITY ++#if defined(__GNUC__) && (__GNUC__ >= 4) ++#define LZ4LIB_VISIBILITY __attribute__((visibility("default"))) ++#else ++#define LZ4LIB_VISIBILITY ++#endif ++#endif ++#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT == 1) ++#define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY ++#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT == 1) ++#define LZ4LIB_API \ ++ __declspec(dllimport) \ ++ LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ ++#else ++#define LZ4LIB_API LZ4LIB_VISIBILITY ++#endif ++ ++/*-************************************ ++* Reading and writing into memory ++**************************************/ ++ ++/** ++ * LZ4 relies on memcpy with a constant size being inlined. In freestanding ++ * environments, the compiler can't assume the implementation of memcpy() is ++ * standard compliant, so it can't apply its specialized memcpy() inlining ++ * logic. When possible, use __builtin_memcpy() to tell the compiler to analyze ++ * memcpy() as if it were standard compliant, so it can inline it in freestanding ++ * environments. This is needed when decompressing the Linux Kernel, for example. ++ */ ++#define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size) ++#define LZ4_memset(dst, src, size) __builtin_memset(dst, src, size) ++#define LZ4_memmove(dst, src, size) __builtin_memmove(dst, src, size) ++ ++/*! LZ4_FREESTANDING : ++ * When this macro is set to 1, it enables "freestanding mode" that is ++ * suitable for typical freestanding environment which doesn't support ++ * standard C library. ++ * ++ * - LZ4_FREESTANDING is a compile-time switch. ++ * - It requires the following macros to be defined: ++ * LZ4_memcpy, LZ4_memmove, LZ4_memset. ++ * - It only enables LZ4/HC functions which don't use heap. ++ * All LZ4F_* functions are not supported. ++ * - See tests/freestanding.c to check its basic setup. ++ */ ++#if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1) ++#define LZ4_HEAPMODE 1 ++#define LZ4HC_HEAPMODE 1 ++#define LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION 1 ++#if !defined(LZ4_memcpy) ++#error "LZ4_FREESTANDING requires macro 'LZ4_memcpy'." ++#endif ++#if !defined(LZ4_memset) ++#error "LZ4_FREESTANDING requires macro 'LZ4_memset'." ++#endif ++#if !defined(LZ4_memmove) ++#error "LZ4_FREESTANDING requires macro 'LZ4_memmove'." ++#endif ++#elif !defined(LZ4_FREESTANDING) ++#define LZ4_FREESTANDING 0 ++#endif ++ ++/*------ Version ------*/ ++#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ ++#define LZ4_VERSION_MINOR 10 /* for new (non-breaking) interface capabilities */ ++#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ ++ ++#define LZ4_VERSION_NUMBER \ ++ (LZ4_VERSION_MAJOR * 100 * 100 + LZ4_VERSION_MINOR * 100 + \ ++ LZ4_VERSION_RELEASE) ++ ++#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE ++#define LZ4_QUOTE(str) #str ++#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) ++#define LZ4_VERSION_STRING \ ++ LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) /* requires v1.7.3+ */ ++ ++LZ4LIB_API int LZ4_versionNumber( ++ void); /**< library version number; useful to check dll version; requires v1.3.0+ */ ++LZ4LIB_API const char *LZ4_versionString( ++ void); /**< library version string; useful to check dll version; requires v1.7.5+ */ ++ ++/*-************************************ ++* Tuning memory usage ++**************************************/ ++/*! ++ * LZ4_MEMORY_USAGE : ++ * Can be selected at compile time, by setting LZ4_MEMORY_USAGE. ++ * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB) ++ * Increasing memory usage improves compression ratio, generally at the cost of speed. ++ * Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality. ++ * Default value is 14, for 16KB, which nicely fits into most L1 caches. ++ */ ++#ifndef LZ4_MEMORY_USAGE ++#define LZ4_MEMORY_USAGE LZ4_MEMORY_USAGE_DEFAULT ++#endif ++ ++/* These are absolute limits, they should not be changed by users */ ++#define LZ4_MEMORY_USAGE_MIN 10 ++#define LZ4_MEMORY_USAGE_DEFAULT 14 ++#define LZ4_MEMORY_USAGE_MAX 20 ++ ++#if (LZ4_MEMORY_USAGE < LZ4_MEMORY_USAGE_MIN) ++#error "LZ4_MEMORY_USAGE is too small !" ++#endif ++ ++#if (LZ4_MEMORY_USAGE > LZ4_MEMORY_USAGE_MAX) ++#error "LZ4_MEMORY_USAGE is too large !" ++#endif ++ ++/* ++ * LZ4_ACCELERATION_DEFAULT : ++ * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 ++ */ ++#define LZ4_ACCELERATION_DEFAULT 1 ++/* ++ * LZ4_ACCELERATION_MAX : ++ * Any "acceleration" value higher than this threshold ++ * get treated as LZ4_ACCELERATION_MAX instead (fix #876) ++ */ ++#define LZ4_ACCELERATION_MAX 65537 ++ ++/*-************************************ ++* Simple Functions ++**************************************/ ++/*! LZ4_compress_default() : ++ * Compresses 'srcSize' bytes from buffer 'src' ++ * into already allocated 'dst' buffer of size 'dstCapacity'. ++ * Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). ++ * It also runs faster, so it's a recommended setting. ++ * If the function cannot compress 'src' into a more limited 'dst' budget, ++ * compression stops *immediately*, and the function result is zero. ++ * In which case, 'dst' content is undefined (invalid). ++ * srcSize : max supported value is LZ4_MAX_INPUT_SIZE. ++ * dstCapacity : size of buffer 'dst' (which must be already allocated) ++ * @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) ++ * or 0 if compression fails ++ * Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). ++ */ ++LZ4LIB_API int LZ4_compress_default(const char *src, char *dst, int srcSize, ++ int dstCapacity, void *wrkmem); ++ ++/*! LZ4_decompress_safe() : ++ * @compressedSize : is the exact complete size of the compressed block. ++ * @dstCapacity : is the size of destination buffer (which must be already allocated), ++ * presumed an upper bound of decompressed size. ++ * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) ++ * If destination buffer is not large enough, decoding will stop and output an error code (negative value). ++ * If the source stream is detected malformed, the function will stop decoding and return a negative result. ++ * Note 1 : This function is protected against malicious data packets : ++ * it will never writes outside 'dst' buffer, nor read outside 'source' buffer, ++ * even if the compressed block is maliciously modified to order the decoder to do these actions. ++ * In such case, the decoder stops immediately, and considers the compressed block malformed. ++ * Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them. ++ * The implementation is free to send / store / derive this information in whichever way is most beneficial. ++ * If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead. ++ */ ++LZ4LIB_API int LZ4_decompress_safe(const char *src, char *dst, ++ int compressedSize, int dstCapacity); ++ ++/*-************************************ ++* Advanced Functions ++**************************************/ ++#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ ++#define LZ4_COMPRESSBOUND(isize) \ ++ ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? \ ++ 0 : \ ++ (isize) + ((isize) / 255) + 16) ++ ++/*! LZ4_compressBound() : ++ Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) ++ This function is primarily useful for memory allocation purposes (destination buffer size). ++ Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). ++ Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize) ++ inputSize : max supported value is LZ4_MAX_INPUT_SIZE ++ return : maximum output size in a "worst case" scenario ++ or 0, if input size is incorrect (too large or negative) ++*/ ++LZ4LIB_API int LZ4_compressBound(int inputSize); ++ ++/*! LZ4_compress_fast() : ++ Same as LZ4_compress_default(), but allows selection of "acceleration" factor. ++ The larger the acceleration value, the faster the algorithm, but also the lesser the compression. ++ It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. ++ An acceleration value of "1" is the same as regular LZ4_compress_default() ++ Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT (currently == 1, see lz4.c). ++ Values > LZ4_ACCELERATION_MAX will be replaced by LZ4_ACCELERATION_MAX (currently == 65537, see lz4.c). ++*/ ++LZ4LIB_API int LZ4_compress_fast(const char *src, char *dst, int srcSize, ++ int dstCapacity, int acceleration, void *wrkmem); ++ ++/*! LZ4_compress_fast_extState() : ++ * Same as LZ4_compress_fast(), using an externally allocated memory space for its state. ++ * Use LZ4_sizeofState() to know how much memory must be allocated, ++ * and allocate it on 8-bytes boundaries (using `malloc()` typically). ++ * Then, provide this buffer as `void* state` to compression function. ++ */ ++LZ4LIB_API int LZ4_sizeofState(void); ++LZ4LIB_API int LZ4_compress_fast_extState(void *state, const char *src, ++ char *dst, int srcSize, ++ int dstCapacity, int acceleration); ++ ++/*! LZ4_compress_destSize() : ++ * Reverse the logic : compresses as much data as possible from 'src' buffer ++ * into already allocated buffer 'dst', of size >= 'dstCapacity'. ++ * This function either compresses the entire 'src' content into 'dst' if it's large enough, ++ * or fill 'dst' buffer completely with as much data as possible from 'src'. ++ * note: acceleration parameter is fixed to "default". ++ * ++ * *srcSizePtr : in+out parameter. Initially contains size of input. ++ * Will be modified to indicate how many bytes where read from 'src' to fill 'dst'. ++ * New value is necessarily <= input value. ++ * @return : Nb bytes written into 'dst' (necessarily <= dstCapacity) ++ * or 0 if compression fails. ++ * ++ * Note : from v1.8.2 to v1.9.1, this function had a bug (fixed in v1.9.2+): ++ * the produced compressed content could, in specific circumstances, ++ * require to be decompressed into a destination buffer larger ++ * by at least 1 byte than the content to decompress. ++ * If an application uses `LZ4_compress_destSize()`, ++ * it's highly recommended to update liblz4 to v1.9.2 or better. ++ * If this can't be done or ensured, ++ * the receiving decompression function should provide ++ * a dstCapacity which is > decompressedSize, by at least 1 byte. ++ * See https://github.com/lz4/lz4/issues/859 for details ++ */ ++LZ4LIB_API int LZ4_compress_destSize(const char *src, char *dst, ++ int *srcSizePtr, int targetDstSize, void *wrkmem); ++ ++/*! LZ4_decompress_safe_partial() : ++ * Decompress an LZ4 compressed block, of size 'srcSize' at position 'src', ++ * into destination buffer 'dst' of size 'dstCapacity'. ++ * Up to 'targetOutputSize' bytes will be decoded. ++ * The function stops decoding on reaching this objective. ++ * This can be useful to boost performance ++ * whenever only the beginning of a block is required. ++ * ++ * @return : the number of bytes decoded in `dst` (necessarily <= targetOutputSize) ++ * If source stream is detected malformed, function returns a negative result. ++ * ++ * Note 1 : @return can be < targetOutputSize, if compressed block contains less data. ++ * ++ * Note 2 : targetOutputSize must be <= dstCapacity ++ * ++ * Note 3 : this function effectively stops decoding on reaching targetOutputSize, ++ * so dstCapacity is kind of redundant. ++ * This is because in older versions of this function, ++ * decoding operation would still write complete sequences. ++ * Therefore, there was no guarantee that it would stop writing at exactly targetOutputSize, ++ * it could write more bytes, though only up to dstCapacity. ++ * Some "margin" used to be required for this operation to work properly. ++ * Thankfully, this is no longer necessary. ++ * The function nonetheless keeps the same signature, in an effort to preserve API compatibility. ++ * ++ * Note 4 : If srcSize is the exact size of the block, ++ * then targetOutputSize can be any value, ++ * including larger than the block's decompressed size. ++ * The function will, at most, generate block's decompressed size. ++ * ++ * Note 5 : If srcSize is _larger_ than block's compressed size, ++ * then targetOutputSize **MUST** be <= block's decompressed size. ++ * Otherwise, *silent corruption will occur*. ++ */ ++LZ4LIB_API int LZ4_decompress_safe_partial(const char *src, char *dst, ++ int srcSize, int targetOutputSize, ++ int dstCapacity); ++ ++/*-********************************************* ++* Streaming Compression Functions ++***********************************************/ ++typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ ++ ++/*! ++ Note about RC_INVOKED ++ ++ - RC_INVOKED is predefined symbol of rc.exe (the resource compiler which is part of MSVC/Visual Studio). ++ https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros ++ ++ - Since rc.exe is a legacy compiler, it truncates long symbol (> 30 chars) ++ and reports warning "RC4011: identifier truncated". ++ ++ - To eliminate the warning, we surround long preprocessor symbol with ++ "#if !defined(RC_INVOKED) ... #endif" block that means ++ "skip this block when rc.exe is trying to read it". ++*/ ++#if !defined( \ ++ RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */ ++#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) ++LZ4LIB_API LZ4_stream_t *LZ4_createStream(void); ++LZ4LIB_API int LZ4_freeStream(LZ4_stream_t *streamPtr); ++#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */ ++#endif ++ ++/*! LZ4_resetStream_fast() : v1.9.0+ ++ * Use this to prepare an LZ4_stream_t for a new chain of dependent blocks ++ * (e.g., LZ4_compress_fast_continue()). ++ * ++ * An LZ4_stream_t must be initialized once before usage. ++ * This is automatically done when created by LZ4_createStream(). ++ * However, should the LZ4_stream_t be simply declared on stack (for example), ++ * it's necessary to initialize it first, using LZ4_initStream(). ++ * ++ * After init, start any new stream with LZ4_resetStream_fast(). ++ * A same LZ4_stream_t can be re-used multiple times consecutively ++ * and compress multiple streams, ++ * provided that it starts each new stream with LZ4_resetStream_fast(). ++ * ++ * LZ4_resetStream_fast() is much faster than LZ4_initStream(), ++ * but is not compatible with memory regions containing garbage data. ++ * ++ * Note: it's only useful to call LZ4_resetStream_fast() ++ * in the context of streaming compression. ++ * The *extState* functions perform their own resets. ++ * Invoking LZ4_resetStream_fast() before is redundant, and even counterproductive. ++ */ ++LZ4LIB_API void LZ4_resetStream_fast(LZ4_stream_t *streamPtr); ++ ++/*! LZ4_loadDict() : ++ * Use this function to reference a static dictionary into LZ4_stream_t. ++ * The dictionary must remain available during compression. ++ * LZ4_loadDict() triggers a reset, so any previous data will be forgotten. ++ * The same dictionary will have to be loaded on decompression side for successful decoding. ++ * Dictionary are useful for better compression of small data (KB range). ++ * While LZ4 itself accepts any input as dictionary, dictionary efficiency is also a topic. ++ * When in doubt, employ the Zstandard's Dictionary Builder. ++ * Loading a size of 0 is allowed, and is the same as reset. ++ * @return : loaded dictionary size, in bytes (note: only the last 64 KB are loaded) ++ */ ++LZ4LIB_API int LZ4_loadDict(LZ4_stream_t *streamPtr, const char *dictionary, ++ int dictSize); ++ ++/*! LZ4_loadDictSlow() : v1.10.0+ ++ * Same as LZ4_loadDict(), ++ * but uses a bit more cpu to reference the dictionary content more thoroughly. ++ * This is expected to slightly improve compression ratio. ++ * The extra-cpu cost is likely worth it if the dictionary is re-used across multiple sessions. ++ * @return : loaded dictionary size, in bytes (note: only the last 64 KB are loaded) ++ */ ++LZ4LIB_API int LZ4_loadDictSlow(LZ4_stream_t *streamPtr, const char *dictionary, ++ int dictSize); ++ ++/*! LZ4_attach_dictionary() : stable since v1.10.0 ++ * ++ * This allows efficient re-use of a static dictionary multiple times. ++ * ++ * Rather than re-loading the dictionary buffer into a working context before ++ * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a ++ * working LZ4_stream_t, this function introduces a no-copy setup mechanism, ++ * in which the working stream references @dictionaryStream in-place. ++ * ++ * Several assumptions are made about the state of @dictionaryStream. ++ * Currently, only states which have been prepared by LZ4_loadDict() or ++ * LZ4_loadDictSlow() should be expected to work. ++ * ++ * Alternatively, the provided @dictionaryStream may be NULL, ++ * in which case any existing dictionary stream is unset. ++ * ++ * If a dictionary is provided, it replaces any pre-existing stream history. ++ * The dictionary contents are the only history that can be referenced and ++ * logically immediately precede the data compressed in the first subsequent ++ * compression call. ++ * ++ * The dictionary will only remain attached to the working stream through the ++ * first compression call, at the end of which it is cleared. ++ * @dictionaryStream stream (and source buffer) must remain in-place / accessible / unchanged ++ * through the completion of the compression session. ++ * ++ * Note: there is no equivalent LZ4_attach_*() method on the decompression side ++ * because there is no initialization cost, hence no need to share the cost across multiple sessions. ++ * To decompress LZ4 blocks using dictionary, attached or not, ++ * just employ the regular LZ4_setStreamDecode() for streaming, ++ * or the stateless LZ4_decompress_safe_usingDict() for one-shot decompression. ++ */ ++LZ4LIB_API void LZ4_attach_dictionary(LZ4_stream_t *workingStream, ++ const LZ4_stream_t *dictionaryStream); ++ ++/*! LZ4_compress_fast_continue() : ++ * Compress 'src' content using data from previously compressed blocks, for better compression ratio. ++ * 'dst' buffer must be already allocated. ++ * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. ++ * ++ * @return : size of compressed block ++ * or 0 if there is an error (typically, cannot fit into 'dst'). ++ * ++ * Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block. ++ * Each block has precise boundaries. ++ * Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata. ++ * It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together. ++ * ++ * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory ! ++ * ++ * Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB. ++ * Make sure that buffers are separated, by at least one byte. ++ * This construction ensures that each block only depends on previous block. ++ * ++ * Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. ++ * ++ * Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed. ++ */ ++LZ4LIB_API int LZ4_compress_fast_continue(LZ4_stream_t *streamPtr, ++ const char *src, char *dst, ++ int srcSize, int dstCapacity, ++ int acceleration); ++ ++/*! LZ4_saveDict() : ++ * If last 64KB data cannot be guaranteed to remain available at its current memory location, ++ * save it into a safer place (char* safeBuffer). ++ * This is schematically equivalent to a memcpy() followed by LZ4_loadDict(), ++ * but is much faster, because LZ4_saveDict() doesn't need to rebuild tables. ++ * @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error. ++ */ ++LZ4LIB_API int LZ4_saveDict(LZ4_stream_t *streamPtr, char *safeBuffer, ++ int maxDictSize); ++ ++/*-********************************************** ++* Streaming Decompression Functions ++* Bufferless synchronous API ++************************************************/ ++typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */ ++ ++/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : ++ * creation / destruction of streaming decompression tracking context. ++ * A tracking context can be re-used multiple times. ++ */ ++#if !defined( \ ++ RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */ ++#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) ++LZ4LIB_API LZ4_streamDecode_t *LZ4_createStreamDecode(void); ++LZ4LIB_API int LZ4_freeStreamDecode(LZ4_streamDecode_t *LZ4_stream); ++#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */ ++#endif ++ ++/*! LZ4_setStreamDecode() : ++ * An LZ4_streamDecode_t context can be allocated once and re-used multiple times. ++ * Use this function to start decompression of a new stream of blocks. ++ * A dictionary can optionally be set. Use NULL or size 0 for a reset order. ++ * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. ++ * @return : 1 if OK, 0 if error ++ */ ++LZ4LIB_API int LZ4_setStreamDecode(LZ4_streamDecode_t *LZ4_streamDecode, ++ const char *dictionary, int dictSize); ++ ++/*! LZ4_decoderRingBufferSize() : v1.8.2+ ++ * Note : in a ring buffer scenario (optional), ++ * blocks are presumed decompressed next to each other ++ * up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize), ++ * at which stage it resumes from beginning of ring buffer. ++ * When setting such a ring buffer for streaming decompression, ++ * provides the minimum size of this ring buffer ++ * to be compatible with any source respecting maxBlockSize condition. ++ * @return : minimum ring buffer size, ++ * or 0 if there is an error (invalid maxBlockSize). ++ */ ++LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); ++#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) \ ++ (65536 + 14 + \ ++ (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */ ++ ++/*! LZ4_decompress_safe_continue() : ++ * This decoding function allows decompression of consecutive blocks in "streaming" mode. ++ * The difference with the usual independent blocks is that ++ * new blocks are allowed to find references into former blocks. ++ * A block is an unsplittable entity, and must be presented entirely to the decompression function. ++ * LZ4_decompress_safe_continue() only accepts one block at a time. ++ * It's modeled after `LZ4_decompress_safe()` and behaves similarly. ++ * ++ * @LZ4_streamDecode : decompression state, tracking the position in memory of past data ++ * @compressedSize : exact complete size of one compressed block. ++ * @dstCapacity : size of destination buffer (which must be already allocated), ++ * must be an upper bound of decompressed size. ++ * @return : number of bytes decompressed into destination buffer (necessarily <= dstCapacity) ++ * If destination buffer is not large enough, decoding will stop and output an error code (negative value). ++ * If the source stream is detected malformed, the function will stop decoding and return a negative result. ++ * ++ * The last 64KB of previously decoded data *must* remain available and unmodified ++ * at the memory position where they were previously decoded. ++ * If less than 64KB of data has been decoded, all the data must be present. ++ * ++ * Special : if decompression side sets a ring buffer, it must respect one of the following conditions : ++ * - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize). ++ * maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes. ++ * In which case, encoding and decoding buffers do not need to be synchronized. ++ * Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize. ++ * - Synchronized mode : ++ * Decompression buffer size is _exactly_ the same as compression buffer size, ++ * and follows exactly same update rule (block boundaries at same positions), ++ * and decoding function is provided with exact decompressed size of each block (exception for last block of the stream), ++ * _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB). ++ * - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes. ++ * In which case, encoding and decoding buffers do not need to be synchronized, ++ * and encoding ring buffer can have any size, including small ones ( < 64 KB). ++ * ++ * Whenever these conditions are not possible, ++ * save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression, ++ * then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block. ++*/ ++LZ4LIB_API int ++LZ4_decompress_safe_continue(LZ4_streamDecode_t *LZ4_streamDecode, ++ const char *src, char *dst, int srcSize, ++ int dstCapacity); ++ ++LZ4LIB_API ssize_t LZ4_arm64_decompress_safe_partial(const void *source, ++ void *dest, ++ size_t inputSize, ++ size_t outputSize, ++ bool dip); ++ ++LZ4LIB_API ssize_t LZ4_arm64_decompress_safe(const void *source, void *dest, ++ size_t inputSize, ++ size_t outputSize, bool dip); ++ ++/*! LZ4_decompress_safe_usingDict() : ++ * Works the same as ++ * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_safe_continue() ++ * However, it's stateless: it doesn't need any LZ4_streamDecode_t state. ++ * Dictionary is presumed stable : it must remain accessible and unmodified during decompression. ++ * Performance tip : Decompression speed can be substantially increased ++ * when dst == dictStart + dictSize. ++ */ ++LZ4LIB_API int LZ4_decompress_safe_usingDict(const char *src, char *dst, ++ int srcSize, int dstCapacity, ++ const char *dictStart, ++ int dictSize); ++ ++/*! LZ4_decompress_safe_partial_usingDict() : ++ * Behaves the same as LZ4_decompress_safe_partial() ++ * with the added ability to specify a memory segment for past data. ++ * Performance tip : Decompression speed can be substantially increased ++ * when dst == dictStart + dictSize. ++ */ ++LZ4LIB_API int LZ4_decompress_safe_partial_usingDict( ++ const char *src, char *dst, int compressedSize, int targetOutputSize, ++ int maxOutputSize, const char *dictStart, int dictSize); ++ ++#endif /* LZ4_H_2983827168210 */ ++ ++/*^************************************* ++ * !!!!!! STATIC LINKING ONLY !!!!!! ++ ***************************************/ ++ ++/*-**************************************************************************** ++ * Experimental section ++ * ++ * Symbols declared in this section must be considered unstable. Their ++ * signatures or semantics may change, or they may be removed altogether in the ++ * future. They are therefore only safe to depend on when the caller is ++ * statically linked against the library. ++ * ++ * To protect against unsafe usage, not only are the declarations guarded, ++ * the definitions are hidden by default ++ * when building LZ4 as a shared/dynamic library. ++ * ++ * In order to access these declarations, ++ * define LZ4_STATIC_LINKING_ONLY in your application ++ * before including LZ4's headers. ++ * ++ * In order to make their implementations accessible dynamically, you must ++ * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library. ++ ******************************************************************************/ ++ ++#ifdef LZ4_STATIC_LINKING_ONLY ++ ++#ifndef LZ4_STATIC_3504398509 ++#define LZ4_STATIC_3504398509 ++ ++#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS ++#define LZ4LIB_STATIC_API LZ4LIB_API ++#else ++#define LZ4LIB_STATIC_API ++#endif ++ ++/*! LZ4_compress_fast_extState_fastReset() : ++ * A variant of LZ4_compress_fast_extState(). ++ * ++ * Using this variant avoids an expensive initialization step. ++ * It is only safe to call if the state buffer is known to be correctly initialized already ++ * (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized"). ++ * From a high level, the difference is that ++ * this function initializes the provided state with a call to something like LZ4_resetStream_fast() ++ * while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream(). ++ */ ++LZ4LIB_STATIC_API int ++LZ4_compress_fast_extState_fastReset(void *state, const char *src, char *dst, ++ int srcSize, int dstCapacity, ++ int acceleration); ++ ++/*! LZ4_compress_destSize_extState() : introduced in v1.10.0 ++ * Same as LZ4_compress_destSize(), but using an externally allocated state. ++ * Also: exposes @acceleration ++ */ ++int LZ4_compress_destSize_extState(void *state, const char *src, char *dst, ++ int *srcSizePtr, int targetDstSize, ++ int acceleration); ++ ++/*! In-place compression and decompression ++ * ++ * It's possible to have input and output sharing the same buffer, ++ * for highly constrained memory environments. ++ * In both cases, it requires input to lay at the end of the buffer, ++ * and decompression to start at beginning of the buffer. ++ * Buffer size must feature some margin, hence be larger than final size. ++ * ++ * |<------------------------buffer--------------------------------->| ++ * |<-----------compressed data--------->| ++ * |<-----------decompressed size------------------>| ++ * |<----margin---->| ++ * ++ * This technique is more useful for decompression, ++ * since decompressed size is typically larger, ++ * and margin is short. ++ * ++ * In-place decompression will work inside any buffer ++ * which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize). ++ * This presumes that decompressedSize > compressedSize. ++ * Otherwise, it means compression actually expanded data, ++ * and it would be more efficient to store such data with a flag indicating it's not compressed. ++ * This can happen when data is not compressible (already compressed, or encrypted). ++ * ++ * For in-place compression, margin is larger, as it must be able to cope with both ++ * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX, ++ * and data expansion, which can happen when input is not compressible. ++ * As a consequence, buffer size requirements are much higher, ++ * and memory savings offered by in-place compression are more limited. ++ * ++ * There are ways to limit this cost for compression : ++ * - Reduce history size, by modifying LZ4_DISTANCE_MAX. ++ * Note that it is a compile-time constant, so all compressions will apply this limit. ++ * Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX, ++ * so it's a reasonable trick when inputs are known to be small. ++ * - Require the compressor to deliver a "maximum compressed size". ++ * This is the `dstCapacity` parameter in `LZ4_compress*()`. ++ * When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail, ++ * in which case, the return code will be 0 (zero). ++ * The caller must be ready for these cases to happen, ++ * and typically design a backup scheme to send data uncompressed. ++ * The combination of both techniques can significantly reduce ++ * the amount of margin required for in-place compression. ++ * ++ * In-place compression can work in any buffer ++ * which size is >= (maxCompressedSize) ++ * with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success. ++ * LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX, ++ * so it's possible to reduce memory requirements by playing with them. ++ */ ++ ++#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) \ ++ (((compressedSize) >> 8) + 32) ++#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) \ ++ ((decompressedSize) + \ ++ LZ4_DECOMPRESS_INPLACE_MARGIN( \ ++ decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */ ++ ++#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */ ++#define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ ++#endif ++ ++#define LZ4_COMPRESS_INPLACE_MARGIN \ ++ (LZ4_DISTANCE_MAX + \ ++ 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */ ++#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) \ ++ ((maxCompressedSize) + \ ++ LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */ ++ ++#endif /* LZ4_STATIC_3504398509 */ ++#endif /* LZ4_STATIC_LINKING_ONLY */ ++ ++#ifndef LZ4_H_98237428734687 ++#define LZ4_H_98237428734687 ++ ++/*-************************************************************ ++ * Private Definitions ++ ************************************************************** ++ * Do not use these definitions directly. ++ * They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. ++ * Accessing members will expose user code to API and/or ABI break in future versions of the library. ++ **************************************************************/ ++#define LZ4_HASHLOG (LZ4_MEMORY_USAGE - 2) ++#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) ++#define LZ4_HASH_SIZE_U32 \ ++ (1 << LZ4_HASHLOG) /* required as macro for static allocation */ ++ ++#include ++#include ++typedef int8_t LZ4_i8; ++typedef uint8_t LZ4_byte; ++typedef uint16_t LZ4_u16; ++typedef uint32_t LZ4_u32; ++ ++/*! LZ4_stream_t : ++ * Never ever use below internal definitions directly ! ++ * These definitions are not API/ABI safe, and may change in future versions. ++ * If you need static allocation, declare or allocate an LZ4_stream_t object. ++**/ ++ ++typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; ++struct LZ4_stream_t_internal { ++ LZ4_u32 hashTable[LZ4_HASH_SIZE_U32]; ++ const LZ4_byte *dictionary; ++ const LZ4_stream_t_internal *dictCtx; ++ LZ4_u32 currentOffset; ++ LZ4_u32 tableType; ++ LZ4_u32 dictSize; ++ /* Implicit padding to ensure structure is aligned */ ++}; ++ ++#define LZ4_STREAM_MINSIZE \ ++ ((1UL << (LZ4_MEMORY_USAGE)) + \ ++ 32) /* static size, for inter-version compatibility */ ++union LZ4_stream_u { ++ char minStateSize[LZ4_STREAM_MINSIZE]; ++ LZ4_stream_t_internal internal_donotuse; ++}; /* previously typedef'd to LZ4_stream_t */ ++ ++/*! LZ4_initStream() : v1.9.0+ ++ * An LZ4_stream_t structure must be initialized at least once. ++ * This is automatically done when invoking LZ4_createStream(), ++ * but it's not when the structure is simply declared on stack (for example). ++ * ++ * Use LZ4_initStream() to properly initialize a newly declared LZ4_stream_t. ++ * It can also initialize any arbitrary buffer of sufficient size, ++ * and will @return a pointer of proper type upon initialization. ++ * ++ * Note : initialization fails if size and alignment conditions are not respected. ++ * In which case, the function will @return NULL. ++ * Note2: An LZ4_stream_t structure guarantees correct alignment and size. ++ * Note3: Before v1.9.0, use LZ4_resetStream() instead ++**/ ++LZ4LIB_API LZ4_stream_t *LZ4_initStream(void *stateBuffer, size_t size); ++ ++/*! LZ4_streamDecode_t : ++ * Never ever use below internal definitions directly ! ++ * These definitions are not API/ABI safe, and may change in future versions. ++ * If you need static allocation, declare or allocate an LZ4_streamDecode_t object. ++**/ ++typedef struct { ++ const LZ4_byte *externalDict; ++ const LZ4_byte *prefixEnd; ++ size_t extDictSize; ++ size_t prefixSize; ++} LZ4_streamDecode_t_internal; ++ ++#define LZ4_STREAMDECODE_MINSIZE 32 ++union LZ4_streamDecode_u { ++ char minStateSize[LZ4_STREAMDECODE_MINSIZE]; ++ LZ4_streamDecode_t_internal internal_donotuse; ++}; /* previously typedef'd to LZ4_streamDecode_t */ ++ ++/*-************************************ ++* Obsolete Functions ++**************************************/ ++ ++/*! Deprecation warnings ++ * ++ * Deprecated functions make the compiler generate a warning when invoked. ++ * This is meant to invite users to update their source code. ++ * Should deprecation warnings be a problem, it is generally possible to disable them, ++ * typically with -Wno-deprecated-declarations for gcc ++ * or _CRT_SECURE_NO_WARNINGS in Visual. ++ * ++ * Another method is to define LZ4_DISABLE_DEPRECATE_WARNINGS ++ * before including the header file. ++ */ ++#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS ++#define LZ4_DEPRECATED(message) /* disable deprecation warnings */ ++#else ++#if defined(__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ ++#define LZ4_DEPRECATED(message) [[deprecated(message)]] ++#elif defined(_MSC_VER) ++#define LZ4_DEPRECATED(message) __declspec(deprecated(message)) ++#elif defined(__clang__) || \ ++ (defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 45)) ++#define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) ++#elif defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 31) ++#define LZ4_DEPRECATED(message) __attribute__((deprecated)) ++#else ++#pragma message( \ ++ "WARNING: LZ4_DEPRECATED needs custom implementation for this compiler") ++#define LZ4_DEPRECATED(message) /* disabled */ ++#endif ++#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ ++ ++/*! Obsolete compression functions (since v1.7.3) */ ++LZ4_DEPRECATED("use LZ4_compress_default() instead") ++LZ4LIB_API int LZ4_compress(const char *src, char *dest, int srcSize); ++LZ4_DEPRECATED("use LZ4_compress_default() instead") ++LZ4LIB_API int LZ4_compress_limitedOutput(const char *src, char *dest, ++ int srcSize, int maxOutputSize); ++LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") ++LZ4LIB_API int LZ4_compress_withState(void *state, const char *source, ++ char *dest, int inputSize); ++LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") ++LZ4LIB_API int LZ4_compress_limitedOutput_withState(void *state, ++ const char *source, ++ char *dest, int inputSize, ++ int maxOutputSize); ++LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") ++LZ4LIB_API int LZ4_compress_continue(LZ4_stream_t *LZ4_streamPtr, ++ const char *source, char *dest, ++ int inputSize); ++LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") ++LZ4LIB_API int LZ4_compress_limitedOutput_continue(LZ4_stream_t *LZ4_streamPtr, ++ const char *source, ++ char *dest, int inputSize, ++ int maxOutputSize); ++ ++/*! Obsolete decompression functions (since v1.8.0) */ ++LZ4_DEPRECATED("use LZ4_decompress_fast() instead") ++LZ4LIB_API int LZ4_uncompress(const char *source, char *dest, int outputSize); ++LZ4_DEPRECATED("use LZ4_decompress_safe() instead") ++LZ4LIB_API int LZ4_uncompress_unknownOutputSize(const char *source, char *dest, ++ int isize, int maxOutputSize); ++ ++/* Obsolete streaming functions (since v1.7.0) ++ * degraded functionality; do not use! ++ * ++ * In order to perform streaming compression, these functions depended on data ++ * that is no longer tracked in the state. They have been preserved as well as ++ * possible: using them will still produce a correct output. However, they don't ++ * actually retain any history between compression calls. The compression ratio ++ * achieved will therefore be no better than compressing each chunk ++ * independently. ++ */ ++LZ4_DEPRECATED("Use LZ4_createStream() instead") ++LZ4LIB_API void *LZ4_create(char *inputBuffer); ++LZ4_DEPRECATED("Use LZ4_createStream() instead") ++LZ4LIB_API int LZ4_sizeofStreamState(void); ++LZ4_DEPRECATED("Use LZ4_resetStream() instead") ++LZ4LIB_API int LZ4_resetStreamState(void *state, char *inputBuffer); ++LZ4_DEPRECATED("Use LZ4_saveDict() instead") ++LZ4LIB_API char *LZ4_slideInputBuffer(void *state); ++ ++/*! Obsolete streaming decoding functions (since v1.7.0) */ ++LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") ++LZ4LIB_API int LZ4_decompress_safe_withPrefix64k(const char *src, char *dst, ++ int compressedSize, ++ int maxDstSize); ++LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") ++LZ4LIB_API int LZ4_decompress_fast_withPrefix64k(const char *src, char *dst, ++ int originalSize); ++ ++/*! Obsolete LZ4_decompress_fast variants (since v1.9.0) : ++ * These functions used to be faster than LZ4_decompress_safe(), ++ * but this is no longer the case. They are now slower. ++ * This is because LZ4_decompress_fast() doesn't know the input size, ++ * and therefore must progress more cautiously into the input buffer to not read beyond the end of block. ++ * On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability. ++ * As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated. ++ * ++ * The last remaining LZ4_decompress_fast() specificity is that ++ * it can decompress a block without knowing its compressed size. ++ * Such functionality can be achieved in a more secure manner ++ * by employing LZ4_decompress_safe_partial(). ++ * ++ * Parameters: ++ * originalSize : is the uncompressed size to regenerate. ++ * `dst` must be already allocated, its size must be >= 'originalSize' bytes. ++ * @return : number of bytes read from source buffer (== compressed size). ++ * The function expects to finish at block's end exactly. ++ * If the source stream is detected malformed, the function stops decoding and returns a negative result. ++ * note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer. ++ * However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds. ++ * Also, since match offsets are not validated, match reads from 'src' may underflow too. ++ * These issues never happen if input (compressed) data is correct. ++ * But they may happen if input data is invalid (error or intentional tampering). ++ * As a consequence, use these functions in trusted environments with trusted data **only**. ++ */ ++LZ4_DEPRECATED( ++ "This function is deprecated and unsafe. Consider using LZ4_decompress_safe_partial() instead") ++LZ4LIB_API int LZ4_decompress_fast(const char *src, char *dst, ++ int originalSize); ++LZ4_DEPRECATED( ++ "This function is deprecated and unsafe. Consider migrating towards LZ4_decompress_safe_continue() instead. " ++ "Note that the contract will change (requires block's compressed size, instead of decompressed size)") ++LZ4LIB_API int ++LZ4_decompress_fast_continue(LZ4_streamDecode_t *LZ4_streamDecode, ++ const char *src, char *dst, int originalSize); ++LZ4_DEPRECATED( ++ "This function is deprecated and unsafe. Consider using LZ4_decompress_safe_partial_usingDict() instead") ++LZ4LIB_API int LZ4_decompress_fast_usingDict(const char *src, char *dst, ++ int originalSize, ++ const char *dictStart, ++ int dictSize); ++ ++/*! LZ4_resetStream() : ++ * An LZ4_stream_t structure must be initialized at least once. ++ * This is done with LZ4_initStream(), or LZ4_resetStream(). ++ * Consider switching to LZ4_initStream(), ++ * invoking LZ4_resetStream() will trigger deprecation warnings in the future. ++ */ ++LZ4LIB_API void LZ4_resetStream(LZ4_stream_t *streamPtr); ++ ++#endif /* LZ4_H_98237428734687 */ ++ ++#if defined(__cplusplus) ++} ++#endif +diff --git a/lib/lz4/lz4_compress.c b/lib/lz4/lz4_compress.c +deleted file mode 100644 +index 90bb67994688..000000000000 +--- a/lib/lz4/lz4_compress.c ++++ /dev/null +@@ -1,940 +0,0 @@ +-/* +- * LZ4 - Fast LZ compression algorithm +- * Copyright (C) 2011 - 2016, Yann Collet. +- * BSD 2 - Clause License (http://www.opensource.org/licenses/bsd - license.php) +- * Redistribution and use in source and binary forms, with or without +- * modification, are permitted provided that the following conditions are +- * met: +- * * Redistributions of source code must retain the above copyright +- * notice, this list of conditions and the following disclaimer. +- * * Redistributions in binary form must reproduce the above +- * copyright notice, this list of conditions and the following disclaimer +- * in the documentation and/or other materials provided with the +- * distribution. +- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +- * You can contact the author at : +- * - LZ4 homepage : http://www.lz4.org +- * - LZ4 source repository : https://github.com/lz4/lz4 +- * +- * Changed for kernel usage by: +- * Sven Schmidt <4sschmid@informatik.uni-hamburg.de> +- */ +- +-/*-************************************ +- * Dependencies +- **************************************/ +-#include +-#include "lz4defs.h" +-#include +-#include +-#include +- +-static const int LZ4_minLength = (MFLIMIT + 1); +-static const int LZ4_64Klimit = ((64 * KB) + (MFLIMIT - 1)); +- +-/*-****************************** +- * Compression functions +- ********************************/ +-static FORCE_INLINE U32 LZ4_hash4( +- U32 sequence, +- tableType_t const tableType) +-{ +- if (tableType == byU16) +- return ((sequence * 2654435761U) +- >> ((MINMATCH * 8) - (LZ4_HASHLOG + 1))); +- else +- return ((sequence * 2654435761U) +- >> ((MINMATCH * 8) - LZ4_HASHLOG)); +-} +- +-static FORCE_INLINE U32 LZ4_hash5( +- U64 sequence, +- tableType_t const tableType) +-{ +- const U32 hashLog = (tableType == byU16) +- ? LZ4_HASHLOG + 1 +- : LZ4_HASHLOG; +- +-#if LZ4_LITTLE_ENDIAN +- static const U64 prime5bytes = 889523592379ULL; +- +- return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); +-#else +- static const U64 prime8bytes = 11400714785074694791ULL; +- +- return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); +-#endif +-} +- +-static FORCE_INLINE U32 LZ4_hashPosition( +- const void *p, +- tableType_t const tableType) +-{ +-#if LZ4_ARCH64 +- if (tableType == byU32) +- return LZ4_hash5(LZ4_read_ARCH(p), tableType); +-#endif +- +- return LZ4_hash4(LZ4_read32(p), tableType); +-} +- +-static void LZ4_putPositionOnHash( +- const BYTE *p, +- U32 h, +- void *tableBase, +- tableType_t const tableType, +- const BYTE *srcBase) +-{ +- switch (tableType) { +- case byPtr: +- { +- const BYTE **hashTable = (const BYTE **)tableBase; +- +- hashTable[h] = p; +- return; +- } +- case byU32: +- { +- U32 *hashTable = (U32 *) tableBase; +- +- hashTable[h] = (U32)(p - srcBase); +- return; +- } +- case byU16: +- { +- U16 *hashTable = (U16 *) tableBase; +- +- hashTable[h] = (U16)(p - srcBase); +- return; +- } +- } +-} +- +-static FORCE_INLINE void LZ4_putPosition( +- const BYTE *p, +- void *tableBase, +- tableType_t tableType, +- const BYTE *srcBase) +-{ +- U32 const h = LZ4_hashPosition(p, tableType); +- +- LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); +-} +- +-static const BYTE *LZ4_getPositionOnHash( +- U32 h, +- void *tableBase, +- tableType_t tableType, +- const BYTE *srcBase) +-{ +- if (tableType == byPtr) { +- const BYTE **hashTable = (const BYTE **) tableBase; +- +- return hashTable[h]; +- } +- +- if (tableType == byU32) { +- const U32 * const hashTable = (U32 *) tableBase; +- +- return hashTable[h] + srcBase; +- } +- +- { +- /* default, to ensure a return */ +- const U16 * const hashTable = (U16 *) tableBase; +- +- return hashTable[h] + srcBase; +- } +-} +- +-static FORCE_INLINE const BYTE *LZ4_getPosition( +- const BYTE *p, +- void *tableBase, +- tableType_t tableType, +- const BYTE *srcBase) +-{ +- U32 const h = LZ4_hashPosition(p, tableType); +- +- return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); +-} +- +- +-/* +- * LZ4_compress_generic() : +- * inlined, to ensure branches are decided at compilation time +- */ +-static FORCE_INLINE int LZ4_compress_generic( +- LZ4_stream_t_internal * const dictPtr, +- const char * const source, +- char * const dest, +- const int inputSize, +- const int maxOutputSize, +- const limitedOutput_directive outputLimited, +- const tableType_t tableType, +- const dict_directive dict, +- const dictIssue_directive dictIssue, +- const U32 acceleration) +-{ +- const BYTE *ip = (const BYTE *) source; +- const BYTE *base; +- const BYTE *lowLimit; +- const BYTE * const lowRefLimit = ip - dictPtr->dictSize; +- const BYTE * const dictionary = dictPtr->dictionary; +- const BYTE * const dictEnd = dictionary + dictPtr->dictSize; +- const size_t dictDelta = dictEnd - (const BYTE *)source; +- const BYTE *anchor = (const BYTE *) source; +- const BYTE * const iend = ip + inputSize; +- const BYTE * const mflimit = iend - MFLIMIT; +- const BYTE * const matchlimit = iend - LASTLITERALS; +- +- BYTE *op = (BYTE *) dest; +- BYTE * const olimit = op + maxOutputSize; +- +- U32 forwardH; +- size_t refDelta = 0; +- +- /* Init conditions */ +- if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) { +- /* Unsupported inputSize, too large (or negative) */ +- return 0; +- } +- +- switch (dict) { +- case noDict: +- default: +- base = (const BYTE *)source; +- lowLimit = (const BYTE *)source; +- break; +- case withPrefix64k: +- base = (const BYTE *)source - dictPtr->currentOffset; +- lowLimit = (const BYTE *)source - dictPtr->dictSize; +- break; +- case usingExtDict: +- base = (const BYTE *)source - dictPtr->currentOffset; +- lowLimit = (const BYTE *)source; +- break; +- } +- +- if ((tableType == byU16) +- && (inputSize >= LZ4_64Klimit)) { +- /* Size too large (not within 64K limit) */ +- return 0; +- } +- +- if (inputSize < LZ4_minLength) { +- /* Input too small, no compression (all literals) */ +- goto _last_literals; +- } +- +- /* First Byte */ +- LZ4_putPosition(ip, dictPtr->hashTable, tableType, base); +- ip++; +- forwardH = LZ4_hashPosition(ip, tableType); +- +- /* Main Loop */ +- for ( ; ; ) { +- const BYTE *match; +- BYTE *token; +- +- /* Find a match */ +- { +- const BYTE *forwardIp = ip; +- unsigned int step = 1; +- unsigned int searchMatchNb = acceleration << LZ4_SKIPTRIGGER; +- +- do { +- U32 const h = forwardH; +- +- ip = forwardIp; +- forwardIp += step; +- step = (searchMatchNb++ >> LZ4_SKIPTRIGGER); +- +- if (unlikely(forwardIp > mflimit)) +- goto _last_literals; +- +- match = LZ4_getPositionOnHash(h, +- dictPtr->hashTable, +- tableType, base); +- +- if (dict == usingExtDict) { +- if (match < (const BYTE *)source) { +- refDelta = dictDelta; +- lowLimit = dictionary; +- } else { +- refDelta = 0; +- lowLimit = (const BYTE *)source; +- } } +- +- forwardH = LZ4_hashPosition(forwardIp, +- tableType); +- +- LZ4_putPositionOnHash(ip, h, dictPtr->hashTable, +- tableType, base); +- } while (((dictIssue == dictSmall) +- ? (match < lowRefLimit) +- : 0) +- || ((tableType == byU16) +- ? 0 +- : (match + MAX_DISTANCE < ip)) +- || (LZ4_read32(match + refDelta) +- != LZ4_read32(ip))); +- } +- +- /* Catch up */ +- while (((ip > anchor) & (match + refDelta > lowLimit)) +- && (unlikely(ip[-1] == match[refDelta - 1]))) { +- ip--; +- match--; +- } +- +- /* Encode Literals */ +- { +- unsigned const int litLength = (unsigned int)(ip - anchor); +- +- token = op++; +- +- if ((outputLimited) && +- /* Check output buffer overflow */ +- (unlikely(op + litLength + +- (2 + 1 + LASTLITERALS) + +- (litLength / 255) > olimit))) +- return 0; +- +- if (litLength >= RUN_MASK) { +- int len = (int)litLength - RUN_MASK; +- +- *token = (RUN_MASK << ML_BITS); +- +- for (; len >= 255; len -= 255) +- *op++ = 255; +- *op++ = (BYTE)len; +- } else +- *token = (BYTE)(litLength << ML_BITS); +- +- /* Copy Literals */ +- LZ4_wildCopy(op, anchor, op + litLength); +- op += litLength; +- } +- +-_next_match: +- /* Encode Offset */ +- LZ4_writeLE16(op, (U16)(ip - match)); +- op += 2; +- +- /* Encode MatchLength */ +- { +- unsigned int matchCode; +- +- if ((dict == usingExtDict) +- && (lowLimit == dictionary)) { +- const BYTE *limit; +- +- match += refDelta; +- limit = ip + (dictEnd - match); +- +- if (limit > matchlimit) +- limit = matchlimit; +- +- matchCode = LZ4_count(ip + MINMATCH, +- match + MINMATCH, limit); +- +- ip += MINMATCH + matchCode; +- +- if (ip == limit) { +- unsigned const int more = LZ4_count(ip, +- (const BYTE *)source, +- matchlimit); +- +- matchCode += more; +- ip += more; +- } +- } else { +- matchCode = LZ4_count(ip + MINMATCH, +- match + MINMATCH, matchlimit); +- ip += MINMATCH + matchCode; +- } +- +- if (outputLimited && +- /* Check output buffer overflow */ +- (unlikely(op + +- (1 + LASTLITERALS) + +- (matchCode >> 8) > olimit))) +- return 0; +- +- if (matchCode >= ML_MASK) { +- *token += ML_MASK; +- matchCode -= ML_MASK; +- LZ4_write32(op, 0xFFFFFFFF); +- +- while (matchCode >= 4 * 255) { +- op += 4; +- LZ4_write32(op, 0xFFFFFFFF); +- matchCode -= 4 * 255; +- } +- +- op += matchCode / 255; +- *op++ = (BYTE)(matchCode % 255); +- } else +- *token += (BYTE)(matchCode); +- } +- +- anchor = ip; +- +- /* Test end of chunk */ +- if (ip > mflimit) +- break; +- +- /* Fill table */ +- LZ4_putPosition(ip - 2, dictPtr->hashTable, tableType, base); +- +- /* Test next position */ +- match = LZ4_getPosition(ip, dictPtr->hashTable, +- tableType, base); +- +- if (dict == usingExtDict) { +- if (match < (const BYTE *)source) { +- refDelta = dictDelta; +- lowLimit = dictionary; +- } else { +- refDelta = 0; +- lowLimit = (const BYTE *)source; +- } +- } +- +- LZ4_putPosition(ip, dictPtr->hashTable, tableType, base); +- +- if (((dictIssue == dictSmall) ? (match >= lowRefLimit) : 1) +- && (match + MAX_DISTANCE >= ip) +- && (LZ4_read32(match + refDelta) == LZ4_read32(ip))) { +- token = op++; +- *token = 0; +- goto _next_match; +- } +- +- /* Prepare next loop */ +- forwardH = LZ4_hashPosition(++ip, tableType); +- } +- +-_last_literals: +- /* Encode Last Literals */ +- { +- size_t const lastRun = (size_t)(iend - anchor); +- +- if ((outputLimited) && +- /* Check output buffer overflow */ +- ((op - (BYTE *)dest) + lastRun + 1 + +- ((lastRun + 255 - RUN_MASK) / 255) > (U32)maxOutputSize)) +- return 0; +- +- if (lastRun >= RUN_MASK) { +- size_t accumulator = lastRun - RUN_MASK; +- *op++ = RUN_MASK << ML_BITS; +- for (; accumulator >= 255; accumulator -= 255) +- *op++ = 255; +- *op++ = (BYTE) accumulator; +- } else { +- *op++ = (BYTE)(lastRun << ML_BITS); +- } +- +- LZ4_memcpy(op, anchor, lastRun); +- +- op += lastRun; +- } +- +- /* End */ +- return (int) (((char *)op) - dest); +-} +- +-static int LZ4_compress_fast_extState( +- void *state, +- const char *source, +- char *dest, +- int inputSize, +- int maxOutputSize, +- int acceleration) +-{ +- LZ4_stream_t_internal *ctx = &((LZ4_stream_t *)state)->internal_donotuse; +-#if LZ4_ARCH64 +- const tableType_t tableType = byU32; +-#else +- const tableType_t tableType = byPtr; +-#endif +- +- LZ4_resetStream((LZ4_stream_t *)state); +- +- if (acceleration < 1) +- acceleration = LZ4_ACCELERATION_DEFAULT; +- +- if (maxOutputSize >= LZ4_COMPRESSBOUND(inputSize)) { +- if (inputSize < LZ4_64Klimit) +- return LZ4_compress_generic(ctx, source, +- dest, inputSize, 0, +- noLimit, byU16, noDict, +- noDictIssue, acceleration); +- else +- return LZ4_compress_generic(ctx, source, +- dest, inputSize, 0, +- noLimit, tableType, noDict, +- noDictIssue, acceleration); +- } else { +- if (inputSize < LZ4_64Klimit) +- return LZ4_compress_generic(ctx, source, +- dest, inputSize, +- maxOutputSize, limitedOutput, byU16, noDict, +- noDictIssue, acceleration); +- else +- return LZ4_compress_generic(ctx, source, +- dest, inputSize, +- maxOutputSize, limitedOutput, tableType, noDict, +- noDictIssue, acceleration); +- } +-} +- +-int LZ4_compress_fast(const char *source, char *dest, int inputSize, +- int maxOutputSize, int acceleration, void *wrkmem) +-{ +- return LZ4_compress_fast_extState(wrkmem, source, dest, inputSize, +- maxOutputSize, acceleration); +-} +-EXPORT_SYMBOL(LZ4_compress_fast); +- +-int LZ4_compress_default(const char *source, char *dest, int inputSize, +- int maxOutputSize, void *wrkmem) +-{ +- return LZ4_compress_fast(source, dest, inputSize, +- maxOutputSize, LZ4_ACCELERATION_DEFAULT, wrkmem); +-} +-EXPORT_SYMBOL(LZ4_compress_default); +- +-/*-****************************** +- * *_destSize() variant +- ********************************/ +-static int LZ4_compress_destSize_generic( +- LZ4_stream_t_internal * const ctx, +- const char * const src, +- char * const dst, +- int * const srcSizePtr, +- const int targetDstSize, +- const tableType_t tableType) +-{ +- const BYTE *ip = (const BYTE *) src; +- const BYTE *base = (const BYTE *) src; +- const BYTE *lowLimit = (const BYTE *) src; +- const BYTE *anchor = ip; +- const BYTE * const iend = ip + *srcSizePtr; +- const BYTE * const mflimit = iend - MFLIMIT; +- const BYTE * const matchlimit = iend - LASTLITERALS; +- +- BYTE *op = (BYTE *) dst; +- BYTE * const oend = op + targetDstSize; +- BYTE * const oMaxLit = op + targetDstSize - 2 /* offset */ +- - 8 /* because 8 + MINMATCH == MFLIMIT */ - 1 /* token */; +- BYTE * const oMaxMatch = op + targetDstSize +- - (LASTLITERALS + 1 /* token */); +- BYTE * const oMaxSeq = oMaxLit - 1 /* token */; +- +- U32 forwardH; +- +- /* Init conditions */ +- /* Impossible to store anything */ +- if (targetDstSize < 1) +- return 0; +- /* Unsupported input size, too large (or negative) */ +- if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) +- return 0; +- /* Size too large (not within 64K limit) */ +- if ((tableType == byU16) && (*srcSizePtr >= LZ4_64Klimit)) +- return 0; +- /* Input too small, no compression (all literals) */ +- if (*srcSizePtr < LZ4_minLength) +- goto _last_literals; +- +- /* First Byte */ +- *srcSizePtr = 0; +- LZ4_putPosition(ip, ctx->hashTable, tableType, base); +- ip++; forwardH = LZ4_hashPosition(ip, tableType); +- +- /* Main Loop */ +- for ( ; ; ) { +- const BYTE *match; +- BYTE *token; +- +- /* Find a match */ +- { +- const BYTE *forwardIp = ip; +- unsigned int step = 1; +- unsigned int searchMatchNb = 1 << LZ4_SKIPTRIGGER; +- +- do { +- U32 h = forwardH; +- +- ip = forwardIp; +- forwardIp += step; +- step = (searchMatchNb++ >> LZ4_SKIPTRIGGER); +- +- if (unlikely(forwardIp > mflimit)) +- goto _last_literals; +- +- match = LZ4_getPositionOnHash(h, ctx->hashTable, +- tableType, base); +- forwardH = LZ4_hashPosition(forwardIp, +- tableType); +- LZ4_putPositionOnHash(ip, h, +- ctx->hashTable, tableType, +- base); +- +- } while (((tableType == byU16) +- ? 0 +- : (match + MAX_DISTANCE < ip)) +- || (LZ4_read32(match) != LZ4_read32(ip))); +- } +- +- /* Catch up */ +- while ((ip > anchor) +- && (match > lowLimit) +- && (unlikely(ip[-1] == match[-1]))) { +- ip--; +- match--; +- } +- +- /* Encode Literal length */ +- { +- unsigned int litLength = (unsigned int)(ip - anchor); +- +- token = op++; +- if (op + ((litLength + 240) / 255) +- + litLength > oMaxLit) { +- /* Not enough space for a last match */ +- op--; +- goto _last_literals; +- } +- if (litLength >= RUN_MASK) { +- unsigned int len = litLength - RUN_MASK; +- *token = (RUN_MASK<= 255; len -= 255) +- *op++ = 255; +- *op++ = (BYTE)len; +- } else +- *token = (BYTE)(litLength << ML_BITS); +- +- /* Copy Literals */ +- LZ4_wildCopy(op, anchor, op + litLength); +- op += litLength; +- } +- +-_next_match: +- /* Encode Offset */ +- LZ4_writeLE16(op, (U16)(ip - match)); op += 2; +- +- /* Encode MatchLength */ +- { +- size_t matchLength = LZ4_count(ip + MINMATCH, +- match + MINMATCH, matchlimit); +- +- if (op + ((matchLength + 240)/255) > oMaxMatch) { +- /* Match description too long : reduce it */ +- matchLength = (15 - 1) + (oMaxMatch - op) * 255; +- } +- ip += MINMATCH + matchLength; +- +- if (matchLength >= ML_MASK) { +- *token += ML_MASK; +- matchLength -= ML_MASK; +- while (matchLength >= 255) { +- matchLength -= 255; +- *op++ = 255; +- } +- *op++ = (BYTE)matchLength; +- } else +- *token += (BYTE)(matchLength); +- } +- +- anchor = ip; +- +- /* Test end of block */ +- if (ip > mflimit) +- break; +- if (op > oMaxSeq) +- break; +- +- /* Fill table */ +- LZ4_putPosition(ip - 2, ctx->hashTable, tableType, base); +- +- /* Test next position */ +- match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); +- LZ4_putPosition(ip, ctx->hashTable, tableType, base); +- +- if ((match + MAX_DISTANCE >= ip) +- && (LZ4_read32(match) == LZ4_read32(ip))) { +- token = op++; *token = 0; +- goto _next_match; +- } +- +- /* Prepare next loop */ +- forwardH = LZ4_hashPosition(++ip, tableType); +- } +- +-_last_literals: +- /* Encode Last Literals */ +- { +- size_t lastRunSize = (size_t)(iend - anchor); +- +- if (op + 1 /* token */ +- + ((lastRunSize + 240) / 255) /* litLength */ +- + lastRunSize /* literals */ > oend) { +- /* adapt lastRunSize to fill 'dst' */ +- lastRunSize = (oend - op) - 1; +- lastRunSize -= (lastRunSize + 240) / 255; +- } +- ip = anchor + lastRunSize; +- +- if (lastRunSize >= RUN_MASK) { +- size_t accumulator = lastRunSize - RUN_MASK; +- +- *op++ = RUN_MASK << ML_BITS; +- for (; accumulator >= 255; accumulator -= 255) +- *op++ = 255; +- *op++ = (BYTE) accumulator; +- } else { +- *op++ = (BYTE)(lastRunSize<= LZ4_COMPRESSBOUND(*srcSizePtr)) { +- /* compression success is guaranteed */ +- return LZ4_compress_fast_extState( +- state, src, dst, *srcSizePtr, +- targetDstSize, 1); +- } else { +- if (*srcSizePtr < LZ4_64Klimit) +- return LZ4_compress_destSize_generic( +- &state->internal_donotuse, +- src, dst, srcSizePtr, +- targetDstSize, byU16); +- else +- return LZ4_compress_destSize_generic( +- &state->internal_donotuse, +- src, dst, srcSizePtr, +- targetDstSize, tableType); +- } +-} +- +- +-int LZ4_compress_destSize( +- const char *src, +- char *dst, +- int *srcSizePtr, +- int targetDstSize, +- void *wrkmem) +-{ +- return LZ4_compress_destSize_extState(wrkmem, src, dst, srcSizePtr, +- targetDstSize); +-} +-EXPORT_SYMBOL(LZ4_compress_destSize); +- +-/*-****************************** +- * Streaming functions +- ********************************/ +-void LZ4_resetStream(LZ4_stream_t *LZ4_stream) +-{ +- memset(LZ4_stream, 0, sizeof(LZ4_stream_t)); +-} +- +-int LZ4_loadDict(LZ4_stream_t *LZ4_dict, +- const char *dictionary, int dictSize) +-{ +- LZ4_stream_t_internal *dict = &LZ4_dict->internal_donotuse; +- const BYTE *p = (const BYTE *)dictionary; +- const BYTE * const dictEnd = p + dictSize; +- const BYTE *base; +- +- if ((dict->initCheck) +- || (dict->currentOffset > 1 * GB)) { +- /* Uninitialized structure, or reuse overflow */ +- LZ4_resetStream(LZ4_dict); +- } +- +- if (dictSize < (int)HASH_UNIT) { +- dict->dictionary = NULL; +- dict->dictSize = 0; +- return 0; +- } +- +- if ((dictEnd - p) > 64 * KB) +- p = dictEnd - 64 * KB; +- dict->currentOffset += 64 * KB; +- base = p - dict->currentOffset; +- dict->dictionary = p; +- dict->dictSize = (U32)(dictEnd - p); +- dict->currentOffset += dict->dictSize; +- +- while (p <= dictEnd - HASH_UNIT) { +- LZ4_putPosition(p, dict->hashTable, byU32, base); +- p += 3; +- } +- +- return dict->dictSize; +-} +-EXPORT_SYMBOL(LZ4_loadDict); +- +-static void LZ4_renormDictT(LZ4_stream_t_internal *LZ4_dict, +- const BYTE *src) +-{ +- if ((LZ4_dict->currentOffset > 0x80000000) || +- ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { +- /* address space overflow */ +- /* rescale hash table */ +- U32 const delta = LZ4_dict->currentOffset - 64 * KB; +- const BYTE *dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; +- int i; +- +- for (i = 0; i < LZ4_HASH_SIZE_U32; i++) { +- if (LZ4_dict->hashTable[i] < delta) +- LZ4_dict->hashTable[i] = 0; +- else +- LZ4_dict->hashTable[i] -= delta; +- } +- LZ4_dict->currentOffset = 64 * KB; +- if (LZ4_dict->dictSize > 64 * KB) +- LZ4_dict->dictSize = 64 * KB; +- LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; +- } +-} +- +-int LZ4_saveDict(LZ4_stream_t *LZ4_dict, char *safeBuffer, int dictSize) +-{ +- LZ4_stream_t_internal * const dict = &LZ4_dict->internal_donotuse; +- const BYTE * const previousDictEnd = dict->dictionary + dict->dictSize; +- +- if ((U32)dictSize > 64 * KB) { +- /* useless to define a dictionary > 64 * KB */ +- dictSize = 64 * KB; +- } +- if ((U32)dictSize > dict->dictSize) +- dictSize = dict->dictSize; +- +- memmove(safeBuffer, previousDictEnd - dictSize, dictSize); +- +- dict->dictionary = (const BYTE *)safeBuffer; +- dict->dictSize = (U32)dictSize; +- +- return dictSize; +-} +-EXPORT_SYMBOL(LZ4_saveDict); +- +-int LZ4_compress_fast_continue(LZ4_stream_t *LZ4_stream, const char *source, +- char *dest, int inputSize, int maxOutputSize, int acceleration) +-{ +- LZ4_stream_t_internal *streamPtr = &LZ4_stream->internal_donotuse; +- const BYTE * const dictEnd = streamPtr->dictionary +- + streamPtr->dictSize; +- +- const BYTE *smallest = (const BYTE *) source; +- +- if (streamPtr->initCheck) { +- /* Uninitialized structure detected */ +- return 0; +- } +- +- if ((streamPtr->dictSize > 0) && (smallest > dictEnd)) +- smallest = dictEnd; +- +- LZ4_renormDictT(streamPtr, smallest); +- +- if (acceleration < 1) +- acceleration = LZ4_ACCELERATION_DEFAULT; +- +- /* Check overlapping input/dictionary space */ +- { +- const BYTE *sourceEnd = (const BYTE *) source + inputSize; +- +- if ((sourceEnd > streamPtr->dictionary) +- && (sourceEnd < dictEnd)) { +- streamPtr->dictSize = (U32)(dictEnd - sourceEnd); +- if (streamPtr->dictSize > 64 * KB) +- streamPtr->dictSize = 64 * KB; +- if (streamPtr->dictSize < 4) +- streamPtr->dictSize = 0; +- streamPtr->dictionary = dictEnd - streamPtr->dictSize; +- } +- } +- +- /* prefix mode : source data follows dictionary */ +- if (dictEnd == (const BYTE *)source) { +- int result; +- +- if ((streamPtr->dictSize < 64 * KB) && +- (streamPtr->dictSize < streamPtr->currentOffset)) { +- result = LZ4_compress_generic( +- streamPtr, source, dest, inputSize, +- maxOutputSize, limitedOutput, byU32, +- withPrefix64k, dictSmall, acceleration); +- } else { +- result = LZ4_compress_generic( +- streamPtr, source, dest, inputSize, +- maxOutputSize, limitedOutput, byU32, +- withPrefix64k, noDictIssue, acceleration); +- } +- streamPtr->dictSize += (U32)inputSize; +- streamPtr->currentOffset += (U32)inputSize; +- return result; +- } +- +- /* external dictionary mode */ +- { +- int result; +- +- if ((streamPtr->dictSize < 64 * KB) && +- (streamPtr->dictSize < streamPtr->currentOffset)) { +- result = LZ4_compress_generic( +- streamPtr, source, dest, inputSize, +- maxOutputSize, limitedOutput, byU32, +- usingExtDict, dictSmall, acceleration); +- } else { +- result = LZ4_compress_generic( +- streamPtr, source, dest, inputSize, +- maxOutputSize, limitedOutput, byU32, +- usingExtDict, noDictIssue, acceleration); +- } +- streamPtr->dictionary = (const BYTE *)source; +- streamPtr->dictSize = (U32)inputSize; +- streamPtr->currentOffset += (U32)inputSize; +- return result; +- } +-} +-EXPORT_SYMBOL(LZ4_compress_fast_continue); +- +-MODULE_LICENSE("Dual BSD/GPL"); +-MODULE_DESCRIPTION("LZ4 compressor"); +diff --git a/lib/lz4/lz4_decompress.c b/lib/lz4/lz4_decompress.c +deleted file mode 100644 +index fd1728d94bab..000000000000 +--- a/lib/lz4/lz4_decompress.c ++++ /dev/null +@@ -1,720 +0,0 @@ +-/* +- * LZ4 - Fast LZ compression algorithm +- * Copyright (C) 2011 - 2016, Yann Collet. +- * BSD 2 - Clause License (http://www.opensource.org/licenses/bsd - license.php) +- * Redistribution and use in source and binary forms, with or without +- * modification, are permitted provided that the following conditions are +- * met: +- * * Redistributions of source code must retain the above copyright +- * notice, this list of conditions and the following disclaimer. +- * * Redistributions in binary form must reproduce the above +- * copyright notice, this list of conditions and the following disclaimer +- * in the documentation and/or other materials provided with the +- * distribution. +- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +- * You can contact the author at : +- * - LZ4 homepage : http://www.lz4.org +- * - LZ4 source repository : https://github.com/lz4/lz4 +- * +- * Changed for kernel usage by: +- * Sven Schmidt <4sschmid@informatik.uni-hamburg.de> +- */ +- +-/*-************************************ +- * Dependencies +- **************************************/ +-#include +-#include "lz4defs.h" +-#include +-#include +-#include +-#include +- +-/*-***************************** +- * Decompression functions +- *******************************/ +- +-#define DEBUGLOG(l, ...) {} /* disabled */ +- +-#ifndef assert +-#define assert(condition) ((void)0) +-#endif +- +-/* +- * LZ4_decompress_generic() : +- * This generic decompression function covers all use cases. +- * It shall be instantiated several times, using different sets of directives. +- * Note that it is important for performance that this function really get inlined, +- * in order to remove useless branches during compilation optimization. +- */ +-static FORCE_INLINE int LZ4_decompress_generic( +- const char * const src, +- char * const dst, +- int srcSize, +- /* +- * If endOnInput == endOnInputSize, +- * this value is `dstCapacity` +- */ +- int outputSize, +- /* endOnOutputSize, endOnInputSize */ +- endCondition_directive endOnInput, +- /* full, partial */ +- earlyEnd_directive partialDecoding, +- /* noDict, withPrefix64k, usingExtDict */ +- dict_directive dict, +- /* always <= dst, == dst when no prefix */ +- const BYTE * const lowPrefix, +- /* only if dict == usingExtDict */ +- const BYTE * const dictStart, +- /* note : = 0 if noDict */ +- const size_t dictSize +- ) +-{ +- const BYTE *ip = (const BYTE *) src; +- const BYTE * const iend = ip + srcSize; +- +- BYTE *op = (BYTE *) dst; +- BYTE * const oend = op + outputSize; +- BYTE *cpy; +- +- const BYTE * const dictEnd = (const BYTE *)dictStart + dictSize; +- static const unsigned int inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4}; +- static const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; +- +- const int safeDecode = (endOnInput == endOnInputSize); +- const int checkOffset = ((safeDecode) && (dictSize < (int)(64 * KB))); +- +- /* Set up the "end" pointers for the shortcut. */ +- const BYTE *const shortiend = iend - +- (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; +- const BYTE *const shortoend = oend - +- (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; +- +- DEBUGLOG(5, "%s (srcSize:%i, dstSize:%i)", __func__, +- srcSize, outputSize); +- +- /* Special cases */ +- assert(lowPrefix <= op); +- assert(src != NULL); +- +- /* Empty output buffer */ +- if ((endOnInput) && (unlikely(outputSize == 0))) +- return ((srcSize == 1) && (*ip == 0)) ? 0 : -1; +- +- if ((!endOnInput) && (unlikely(outputSize == 0))) +- return (*ip == 0 ? 1 : -1); +- +- if ((endOnInput) && unlikely(srcSize == 0)) +- return -1; +- +- /* Main Loop : decode sequences */ +- while (1) { +- size_t length; +- const BYTE *match; +- size_t offset; +- +- /* get literal length */ +- unsigned int const token = *ip++; +- length = token>>ML_BITS; +- +- /* ip < iend before the increment */ +- assert(!endOnInput || ip <= iend); +- +- /* +- * A two-stage shortcut for the most common case: +- * 1) If the literal length is 0..14, and there is enough +- * space, enter the shortcut and copy 16 bytes on behalf +- * of the literals (in the fast mode, only 8 bytes can be +- * safely copied this way). +- * 2) Further if the match length is 4..18, copy 18 bytes +- * in a similar manner; but we ensure that there's enough +- * space in the output for those 18 bytes earlier, upon +- * entering the shortcut (in other words, there is a +- * combined check for both stages). +- * +- * The & in the likely() below is intentionally not && so that +- * some compilers can produce better parallelized runtime code +- */ +- if ((endOnInput ? length != RUN_MASK : length <= 8) +- /* +- * strictly "less than" on input, to re-enter +- * the loop with at least one byte +- */ +- && likely((endOnInput ? ip < shortiend : 1) & +- (op <= shortoend))) { +- /* Copy the literals */ +- LZ4_memcpy(op, ip, endOnInput ? 16 : 8); +- op += length; ip += length; +- +- /* +- * The second stage: +- * prepare for match copying, decode full info. +- * If it doesn't work out, the info won't be wasted. +- */ +- length = token & ML_MASK; /* match length */ +- offset = LZ4_readLE16(ip); +- ip += 2; +- match = op - offset; +- assert(match <= op); /* check overflow */ +- +- /* Do not deal with overlapping matches. */ +- if ((length != ML_MASK) && +- (offset >= 8) && +- (dict == withPrefix64k || match >= lowPrefix)) { +- /* Copy the match. */ +- LZ4_memcpy(op + 0, match + 0, 8); +- LZ4_memcpy(op + 8, match + 8, 8); +- LZ4_memcpy(op + 16, match + 16, 2); +- op += length + MINMATCH; +- /* Both stages worked, load the next token. */ +- continue; +- } +- +- /* +- * The second stage didn't work out, but the info +- * is ready. Propel it right to the point of match +- * copying. +- */ +- goto _copy_match; +- } +- +- /* decode literal length */ +- if (length == RUN_MASK) { +- unsigned int s; +- +- if (unlikely(endOnInput ? ip >= iend - RUN_MASK : 0)) { +- /* overflow detection */ +- goto _output_error; +- } +- do { +- s = *ip++; +- length += s; +- } while (likely(endOnInput +- ? ip < iend - RUN_MASK +- : 1) & (s == 255)); +- +- if ((safeDecode) +- && unlikely((uptrval)(op) + +- length < (uptrval)(op))) { +- /* overflow detection */ +- goto _output_error; +- } +- if ((safeDecode) +- && unlikely((uptrval)(ip) + +- length < (uptrval)(ip))) { +- /* overflow detection */ +- goto _output_error; +- } +- } +- +- /* copy literals */ +- cpy = op + length; +- LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); +- +- if (((endOnInput) && ((cpy > oend - MFLIMIT) +- || (ip + length > iend - (2 + 1 + LASTLITERALS)))) +- || ((!endOnInput) && (cpy > oend - WILDCOPYLENGTH))) { +- if (partialDecoding) { +- if (cpy > oend) { +- /* +- * Partial decoding : +- * stop in the middle of literal segment +- */ +- cpy = oend; +- length = oend - op; +- } +- if ((endOnInput) +- && (ip + length > iend)) { +- /* +- * Error : +- * read attempt beyond +- * end of input buffer +- */ +- goto _output_error; +- } +- } else { +- if ((!endOnInput) +- && (cpy != oend)) { +- /* +- * Error : +- * block decoding must +- * stop exactly there +- */ +- goto _output_error; +- } +- if ((endOnInput) +- && ((ip + length != iend) +- || (cpy > oend))) { +- /* +- * Error : +- * input must be consumed +- */ +- goto _output_error; +- } +- } +- +- /* +- * supports overlapping memory regions; only matters +- * for in-place decompression scenarios +- */ +- LZ4_memmove(op, ip, length); +- ip += length; +- op += length; +- +- /* Necessarily EOF when !partialDecoding. +- * When partialDecoding, it is EOF if we've either +- * filled the output buffer or +- * can't proceed with reading an offset for following match. +- */ +- if (!partialDecoding || (cpy == oend) || (ip >= (iend - 2))) +- break; +- } else { +- /* may overwrite up to WILDCOPYLENGTH beyond cpy */ +- LZ4_wildCopy(op, ip, cpy); +- ip += length; +- op = cpy; +- } +- +- /* get offset */ +- offset = LZ4_readLE16(ip); +- ip += 2; +- match = op - offset; +- +- /* get matchlength */ +- length = token & ML_MASK; +- +-_copy_match: +- if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { +- /* Error : offset outside buffers */ +- goto _output_error; +- } +- +- /* costs ~1%; silence an msan warning when offset == 0 */ +- /* +- * note : when partialDecoding, there is no guarantee that +- * at least 4 bytes remain available in output buffer +- */ +- if (!partialDecoding) { +- assert(oend > op); +- assert(oend - op >= 4); +- +- LZ4_write32(op, (U32)offset); +- } +- +- if (length == ML_MASK) { +- unsigned int s; +- +- do { +- s = *ip++; +- +- if ((endOnInput) && (ip > iend - LASTLITERALS)) +- goto _output_error; +- +- length += s; +- } while (s == 255); +- +- if ((safeDecode) +- && unlikely( +- (uptrval)(op) + length < (uptrval)op)) { +- /* overflow detection */ +- goto _output_error; +- } +- } +- +- length += MINMATCH; +- +- /* match starting within external dictionary */ +- if ((dict == usingExtDict) && (match < lowPrefix)) { +- if (unlikely(op + length > oend - LASTLITERALS)) { +- /* doesn't respect parsing restriction */ +- if (!partialDecoding) +- goto _output_error; +- length = min(length, (size_t)(oend - op)); +- } +- +- if (length <= (size_t)(lowPrefix - match)) { +- /* +- * match fits entirely within external +- * dictionary : just copy +- */ +- memmove(op, dictEnd - (lowPrefix - match), +- length); +- op += length; +- } else { +- /* +- * match stretches into both external +- * dictionary and current block +- */ +- size_t const copySize = (size_t)(lowPrefix - match); +- size_t const restSize = length - copySize; +- +- LZ4_memcpy(op, dictEnd - copySize, copySize); +- op += copySize; +- if (restSize > (size_t)(op - lowPrefix)) { +- /* overlap copy */ +- BYTE * const endOfMatch = op + restSize; +- const BYTE *copyFrom = lowPrefix; +- +- while (op < endOfMatch) +- *op++ = *copyFrom++; +- } else { +- LZ4_memcpy(op, lowPrefix, restSize); +- op += restSize; +- } +- } +- continue; +- } +- +- /* copy match within block */ +- cpy = op + length; +- +- /* +- * partialDecoding : +- * may not respect endBlock parsing restrictions +- */ +- assert(op <= oend); +- if (partialDecoding && +- (cpy > oend - MATCH_SAFEGUARD_DISTANCE)) { +- size_t const mlen = min(length, (size_t)(oend - op)); +- const BYTE * const matchEnd = match + mlen; +- BYTE * const copyEnd = op + mlen; +- +- if (matchEnd > op) { +- /* overlap copy */ +- while (op < copyEnd) +- *op++ = *match++; +- } else { +- LZ4_memcpy(op, match, mlen); +- } +- op = copyEnd; +- if (op == oend) +- break; +- continue; +- } +- +- if (unlikely(offset < 8)) { +- op[0] = match[0]; +- op[1] = match[1]; +- op[2] = match[2]; +- op[3] = match[3]; +- match += inc32table[offset]; +- LZ4_memcpy(op + 4, match, 4); +- match -= dec64table[offset]; +- } else { +- LZ4_copy8(op, match); +- match += 8; +- } +- +- op += 8; +- +- if (unlikely(cpy > oend - MATCH_SAFEGUARD_DISTANCE)) { +- BYTE * const oCopyLimit = oend - (WILDCOPYLENGTH - 1); +- +- if (cpy > oend - LASTLITERALS) { +- /* +- * Error : last LASTLITERALS bytes +- * must be literals (uncompressed) +- */ +- goto _output_error; +- } +- +- if (op < oCopyLimit) { +- LZ4_wildCopy(op, match, oCopyLimit); +- match += oCopyLimit - op; +- op = oCopyLimit; +- } +- while (op < cpy) +- *op++ = *match++; +- } else { +- LZ4_copy8(op, match); +- if (length > 16) +- LZ4_wildCopy(op + 8, match + 8, cpy); +- } +- op = cpy; /* wildcopy correction */ +- } +- +- /* end of decoding */ +- if (endOnInput) { +- /* Nb of output bytes decoded */ +- return (int) (((char *)op) - dst); +- } else { +- /* Nb of input bytes read */ +- return (int) (((const char *)ip) - src); +- } +- +- /* Overflow error detected */ +-_output_error: +- return (int) (-(((const char *)ip) - src)) - 1; +-} +- +-int LZ4_decompress_safe(const char *source, char *dest, +- int compressedSize, int maxDecompressedSize) +-{ +- return LZ4_decompress_generic(source, dest, +- compressedSize, maxDecompressedSize, +- endOnInputSize, decode_full_block, +- noDict, (BYTE *)dest, NULL, 0); +-} +- +-int LZ4_decompress_safe_partial(const char *src, char *dst, +- int compressedSize, int targetOutputSize, int dstCapacity) +-{ +- dstCapacity = min(targetOutputSize, dstCapacity); +- return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity, +- endOnInputSize, partial_decode, +- noDict, (BYTE *)dst, NULL, 0); +-} +- +-int LZ4_decompress_fast(const char *source, char *dest, int originalSize) +-{ +- return LZ4_decompress_generic(source, dest, 0, originalSize, +- endOnOutputSize, decode_full_block, +- withPrefix64k, +- (BYTE *)dest - 64 * KB, NULL, 0); +-} +- +-/* ===== Instantiate a few more decoding cases, used more than once. ===== */ +- +-static int LZ4_decompress_safe_withPrefix64k(const char *source, char *dest, +- int compressedSize, int maxOutputSize) +-{ +- return LZ4_decompress_generic(source, dest, +- compressedSize, maxOutputSize, +- endOnInputSize, decode_full_block, +- withPrefix64k, +- (BYTE *)dest - 64 * KB, NULL, 0); +-} +- +-static int LZ4_decompress_safe_withSmallPrefix(const char *source, char *dest, +- int compressedSize, +- int maxOutputSize, +- size_t prefixSize) +-{ +- return LZ4_decompress_generic(source, dest, +- compressedSize, maxOutputSize, +- endOnInputSize, decode_full_block, +- noDict, +- (BYTE *)dest - prefixSize, NULL, 0); +-} +- +-int LZ4_decompress_safe_forceExtDict(const char *source, char *dest, +- int compressedSize, int maxOutputSize, +- const void *dictStart, size_t dictSize) +-{ +- return LZ4_decompress_generic(source, dest, +- compressedSize, maxOutputSize, +- endOnInputSize, decode_full_block, +- usingExtDict, (BYTE *)dest, +- (const BYTE *)dictStart, dictSize); +-} +- +-static int LZ4_decompress_fast_extDict(const char *source, char *dest, +- int originalSize, +- const void *dictStart, size_t dictSize) +-{ +- return LZ4_decompress_generic(source, dest, +- 0, originalSize, +- endOnOutputSize, decode_full_block, +- usingExtDict, (BYTE *)dest, +- (const BYTE *)dictStart, dictSize); +-} +- +-/* +- * The "double dictionary" mode, for use with e.g. ring buffers: the first part +- * of the dictionary is passed as prefix, and the second via dictStart + dictSize. +- * These routines are used only once, in LZ4_decompress_*_continue(). +- */ +-static FORCE_INLINE +-int LZ4_decompress_safe_doubleDict(const char *source, char *dest, +- int compressedSize, int maxOutputSize, +- size_t prefixSize, +- const void *dictStart, size_t dictSize) +-{ +- return LZ4_decompress_generic(source, dest, +- compressedSize, maxOutputSize, +- endOnInputSize, decode_full_block, +- usingExtDict, (BYTE *)dest - prefixSize, +- (const BYTE *)dictStart, dictSize); +-} +- +-static FORCE_INLINE +-int LZ4_decompress_fast_doubleDict(const char *source, char *dest, +- int originalSize, size_t prefixSize, +- const void *dictStart, size_t dictSize) +-{ +- return LZ4_decompress_generic(source, dest, +- 0, originalSize, +- endOnOutputSize, decode_full_block, +- usingExtDict, (BYTE *)dest - prefixSize, +- (const BYTE *)dictStart, dictSize); +-} +- +-/* ===== streaming decompression functions ===== */ +- +-int LZ4_setStreamDecode(LZ4_streamDecode_t *LZ4_streamDecode, +- const char *dictionary, int dictSize) +-{ +- LZ4_streamDecode_t_internal *lz4sd = +- &LZ4_streamDecode->internal_donotuse; +- +- lz4sd->prefixSize = (size_t) dictSize; +- lz4sd->prefixEnd = (const BYTE *) dictionary + dictSize; +- lz4sd->externalDict = NULL; +- lz4sd->extDictSize = 0; +- return 1; +-} +- +-/* +- * *_continue() : +- * These decoding functions allow decompression of multiple blocks +- * in "streaming" mode. +- * Previously decoded blocks must still be available at the memory +- * position where they were decoded. +- * If it's not possible, save the relevant part of +- * decoded data into a safe buffer, +- * and indicate where it stands using LZ4_setStreamDecode() +- */ +-int LZ4_decompress_safe_continue(LZ4_streamDecode_t *LZ4_streamDecode, +- const char *source, char *dest, int compressedSize, int maxOutputSize) +-{ +- LZ4_streamDecode_t_internal *lz4sd = +- &LZ4_streamDecode->internal_donotuse; +- int result; +- +- if (lz4sd->prefixSize == 0) { +- /* The first call, no dictionary yet. */ +- assert(lz4sd->extDictSize == 0); +- result = LZ4_decompress_safe(source, dest, +- compressedSize, maxOutputSize); +- if (result <= 0) +- return result; +- lz4sd->prefixSize = result; +- lz4sd->prefixEnd = (BYTE *)dest + result; +- } else if (lz4sd->prefixEnd == (BYTE *)dest) { +- /* They're rolling the current segment. */ +- if (lz4sd->prefixSize >= 64 * KB - 1) +- result = LZ4_decompress_safe_withPrefix64k(source, dest, +- compressedSize, maxOutputSize); +- else if (lz4sd->extDictSize == 0) +- result = LZ4_decompress_safe_withSmallPrefix(source, +- dest, compressedSize, maxOutputSize, +- lz4sd->prefixSize); +- else +- result = LZ4_decompress_safe_doubleDict(source, dest, +- compressedSize, maxOutputSize, +- lz4sd->prefixSize, +- lz4sd->externalDict, lz4sd->extDictSize); +- if (result <= 0) +- return result; +- lz4sd->prefixSize += result; +- lz4sd->prefixEnd += result; +- } else { +- /* +- * The buffer wraps around, or they're +- * switching to another buffer. +- */ +- lz4sd->extDictSize = lz4sd->prefixSize; +- lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; +- result = LZ4_decompress_safe_forceExtDict(source, dest, +- compressedSize, maxOutputSize, +- lz4sd->externalDict, lz4sd->extDictSize); +- if (result <= 0) +- return result; +- lz4sd->prefixSize = result; +- lz4sd->prefixEnd = (BYTE *)dest + result; +- } +- +- return result; +-} +- +-int LZ4_decompress_fast_continue(LZ4_streamDecode_t *LZ4_streamDecode, +- const char *source, char *dest, int originalSize) +-{ +- LZ4_streamDecode_t_internal *lz4sd = &LZ4_streamDecode->internal_donotuse; +- int result; +- +- if (lz4sd->prefixSize == 0) { +- assert(lz4sd->extDictSize == 0); +- result = LZ4_decompress_fast(source, dest, originalSize); +- if (result <= 0) +- return result; +- lz4sd->prefixSize = originalSize; +- lz4sd->prefixEnd = (BYTE *)dest + originalSize; +- } else if (lz4sd->prefixEnd == (BYTE *)dest) { +- if (lz4sd->prefixSize >= 64 * KB - 1 || +- lz4sd->extDictSize == 0) +- result = LZ4_decompress_fast(source, dest, +- originalSize); +- else +- result = LZ4_decompress_fast_doubleDict(source, dest, +- originalSize, lz4sd->prefixSize, +- lz4sd->externalDict, lz4sd->extDictSize); +- if (result <= 0) +- return result; +- lz4sd->prefixSize += originalSize; +- lz4sd->prefixEnd += originalSize; +- } else { +- lz4sd->extDictSize = lz4sd->prefixSize; +- lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; +- result = LZ4_decompress_fast_extDict(source, dest, +- originalSize, lz4sd->externalDict, lz4sd->extDictSize); +- if (result <= 0) +- return result; +- lz4sd->prefixSize = originalSize; +- lz4sd->prefixEnd = (BYTE *)dest + originalSize; +- } +- return result; +-} +- +-int LZ4_decompress_safe_usingDict(const char *source, char *dest, +- int compressedSize, int maxOutputSize, +- const char *dictStart, int dictSize) +-{ +- if (dictSize == 0) +- return LZ4_decompress_safe(source, dest, +- compressedSize, maxOutputSize); +- if (dictStart+dictSize == dest) { +- if (dictSize >= 64 * KB - 1) +- return LZ4_decompress_safe_withPrefix64k(source, dest, +- compressedSize, maxOutputSize); +- return LZ4_decompress_safe_withSmallPrefix(source, dest, +- compressedSize, maxOutputSize, dictSize); +- } +- return LZ4_decompress_safe_forceExtDict(source, dest, +- compressedSize, maxOutputSize, dictStart, dictSize); +-} +- +-int LZ4_decompress_fast_usingDict(const char *source, char *dest, +- int originalSize, +- const char *dictStart, int dictSize) +-{ +- if (dictSize == 0 || dictStart + dictSize == dest) +- return LZ4_decompress_fast(source, dest, originalSize); +- +- return LZ4_decompress_fast_extDict(source, dest, originalSize, +- dictStart, dictSize); +-} +- +-#ifndef STATIC +-EXPORT_SYMBOL(LZ4_decompress_safe); +-EXPORT_SYMBOL(LZ4_decompress_safe_partial); +-EXPORT_SYMBOL(LZ4_decompress_fast); +-EXPORT_SYMBOL(LZ4_setStreamDecode); +-EXPORT_SYMBOL(LZ4_decompress_safe_continue); +-EXPORT_SYMBOL(LZ4_decompress_fast_continue); +-EXPORT_SYMBOL(LZ4_decompress_safe_usingDict); +-EXPORT_SYMBOL(LZ4_decompress_fast_usingDict); +- +-MODULE_LICENSE("Dual BSD/GPL"); +-MODULE_DESCRIPTION("LZ4 decompressor"); +-#endif +diff --git a/fs/f2fs/lz4armv8/lz4accel.c b/lib/lz4/lz4armv8/lz4accel.c +similarity index 100% +rename from fs/f2fs/lz4armv8/lz4accel.c +rename to lib/lz4/lz4armv8/lz4accel.c +diff --git a/fs/f2fs/lz4armv8/lz4accel.h b/lib/lz4/lz4armv8/lz4accel.h +similarity index 100% +rename from fs/f2fs/lz4armv8/lz4accel.h +rename to lib/lz4/lz4armv8/lz4accel.h +diff --git a/fs/f2fs/lz4armv8/lz4armv8.S b/lib/lz4/lz4armv8/lz4armv8.S +similarity index 100% +rename from fs/f2fs/lz4armv8/lz4armv8.S +rename to lib/lz4/lz4armv8/lz4armv8.S +diff --git a/lib/lz4/lz4defs.h b/lib/lz4/lz4defs.h +deleted file mode 100644 +index 673bd206aa98..000000000000 +--- a/lib/lz4/lz4defs.h ++++ /dev/null +@@ -1,245 +0,0 @@ +-#ifndef __LZ4DEFS_H__ +-#define __LZ4DEFS_H__ +- +-/* +- * lz4defs.h -- common and architecture specific defines for the kernel usage +- +- * LZ4 - Fast LZ compression algorithm +- * Copyright (C) 2011-2016, Yann Collet. +- * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) +- * Redistribution and use in source and binary forms, with or without +- * modification, are permitted provided that the following conditions are +- * met: +- * * Redistributions of source code must retain the above copyright +- * notice, this list of conditions and the following disclaimer. +- * * Redistributions in binary form must reproduce the above +- * copyright notice, this list of conditions and the following disclaimer +- * in the documentation and/or other materials provided with the +- * distribution. +- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +- * You can contact the author at : +- * - LZ4 homepage : http://www.lz4.org +- * - LZ4 source repository : https://github.com/lz4/lz4 +- * +- * Changed for kernel usage by: +- * Sven Schmidt <4sschmid@informatik.uni-hamburg.de> +- */ +- +-#include +-#include /* memset, memcpy */ +- +-#define FORCE_INLINE __always_inline +- +-/*-************************************ +- * Basic Types +- **************************************/ +-#include +- +-typedef uint8_t BYTE; +-typedef uint16_t U16; +-typedef uint32_t U32; +-typedef int32_t S32; +-typedef uint64_t U64; +-typedef uintptr_t uptrval; +- +-/*-************************************ +- * Architecture specifics +- **************************************/ +-#if defined(CONFIG_64BIT) +-#define LZ4_ARCH64 1 +-#else +-#define LZ4_ARCH64 0 +-#endif +- +-#if defined(__LITTLE_ENDIAN) +-#define LZ4_LITTLE_ENDIAN 1 +-#else +-#define LZ4_LITTLE_ENDIAN 0 +-#endif +- +-/*-************************************ +- * Constants +- **************************************/ +-#define MINMATCH 4 +- +-#define WILDCOPYLENGTH 8 +-#define LASTLITERALS 5 +-#define MFLIMIT (WILDCOPYLENGTH + MINMATCH) +-/* +- * ensure it's possible to write 2 x wildcopyLength +- * without overflowing output buffer +- */ +-#define MATCH_SAFEGUARD_DISTANCE ((2 * WILDCOPYLENGTH) - MINMATCH) +- +-/* Increase this value ==> compression run slower on incompressible data */ +-#define LZ4_SKIPTRIGGER 6 +- +-#define HASH_UNIT sizeof(size_t) +- +-#define KB (1 << 10) +-#define MB (1 << 20) +-#define GB (1U << 30) +- +-#define MAXD_LOG 16 +-#define MAX_DISTANCE ((1 << MAXD_LOG) - 1) +-#define STEPSIZE sizeof(size_t) +- +-#define ML_BITS 4 +-#define ML_MASK ((1U << ML_BITS) - 1) +-#define RUN_BITS (8 - ML_BITS) +-#define RUN_MASK ((1U << RUN_BITS) - 1) +- +-/*-************************************ +- * Reading and writing into memory +- **************************************/ +-static FORCE_INLINE U16 LZ4_read16(const void *ptr) +-{ +- return get_unaligned((const U16 *)ptr); +-} +- +-static FORCE_INLINE U32 LZ4_read32(const void *ptr) +-{ +- return get_unaligned((const U32 *)ptr); +-} +- +-static FORCE_INLINE size_t LZ4_read_ARCH(const void *ptr) +-{ +- return get_unaligned((const size_t *)ptr); +-} +- +-static FORCE_INLINE void LZ4_write16(void *memPtr, U16 value) +-{ +- put_unaligned(value, (U16 *)memPtr); +-} +- +-static FORCE_INLINE void LZ4_write32(void *memPtr, U32 value) +-{ +- put_unaligned(value, (U32 *)memPtr); +-} +- +-static FORCE_INLINE U16 LZ4_readLE16(const void *memPtr) +-{ +- return get_unaligned_le16(memPtr); +-} +- +-static FORCE_INLINE void LZ4_writeLE16(void *memPtr, U16 value) +-{ +- return put_unaligned_le16(value, memPtr); +-} +- +-/* +- * LZ4 relies on memcpy with a constant size being inlined. In freestanding +- * environments, the compiler can't assume the implementation of memcpy() is +- * standard compliant, so apply its specialized memcpy() inlining logic. When +- * possible, use __builtin_memcpy() to tell the compiler to analyze memcpy() +- * as-if it were standard compliant, so it can inline it in freestanding +- * environments. This is needed when decompressing the Linux Kernel, for example. +- */ +-#define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size) +-#define LZ4_memmove(dst, src, size) __builtin_memmove(dst, src, size) +- +-static FORCE_INLINE void LZ4_copy8(void *dst, const void *src) +-{ +-#if LZ4_ARCH64 +- U64 a = get_unaligned((const U64 *)src); +- +- put_unaligned(a, (U64 *)dst); +-#else +- U32 a = get_unaligned((const U32 *)src); +- U32 b = get_unaligned((const U32 *)src + 1); +- +- put_unaligned(a, (U32 *)dst); +- put_unaligned(b, (U32 *)dst + 1); +-#endif +-} +- +-/* +- * customized variant of memcpy, +- * which can overwrite up to 7 bytes beyond dstEnd +- */ +-static FORCE_INLINE void LZ4_wildCopy(void *dstPtr, +- const void *srcPtr, void *dstEnd) +-{ +- BYTE *d = (BYTE *)dstPtr; +- const BYTE *s = (const BYTE *)srcPtr; +- BYTE *const e = (BYTE *)dstEnd; +- +- do { +- LZ4_copy8(d, s); +- d += 8; +- s += 8; +- } while (d < e); +-} +- +-static FORCE_INLINE unsigned int LZ4_NbCommonBytes(register size_t val) +-{ +-#if LZ4_LITTLE_ENDIAN +- return __ffs(val) >> 3; +-#else +- return (BITS_PER_LONG - 1 - __fls(val)) >> 3; +-#endif +-} +- +-static FORCE_INLINE unsigned int LZ4_count( +- const BYTE *pIn, +- const BYTE *pMatch, +- const BYTE *pInLimit) +-{ +- const BYTE *const pStart = pIn; +- +- while (likely(pIn < pInLimit - (STEPSIZE - 1))) { +- size_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); +- +- if (!diff) { +- pIn += STEPSIZE; +- pMatch += STEPSIZE; +- continue; +- } +- +- pIn += LZ4_NbCommonBytes(diff); +- +- return (unsigned int)(pIn - pStart); +- } +- +-#if LZ4_ARCH64 +- if ((pIn < (pInLimit - 3)) +- && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { +- pIn += 4; +- pMatch += 4; +- } +-#endif +- +- if ((pIn < (pInLimit - 1)) +- && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { +- pIn += 2; +- pMatch += 2; +- } +- +- if ((pIn < pInLimit) && (*pMatch == *pIn)) +- pIn++; +- +- return (unsigned int)(pIn - pStart); +-} +- +-typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive; +-typedef enum { byPtr, byU32, byU16 } tableType_t; +- +-typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; +-typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; +- +-typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; +-typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; +- +-#define LZ4_STATIC_ASSERT(c) BUILD_BUG_ON(!(c)) +- +-#endif +diff --git a/lib/lz4/lz4hc.c b/lib/lz4/lz4hc.c +new file mode 100644 +index 000000000000..cccd7b7d0ab9 +--- /dev/null ++++ b/lib/lz4/lz4hc.c +@@ -0,0 +1,2805 @@ ++/* ++ LZ4 HC - High Compression Mode of LZ4 ++ Copyright (C) 2011-2020, Yann Collet. ++ ++ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) ++ ++ Redistribution and use in source and binary forms, with or without ++ modification, are permitted provided that the following conditions are ++ met: ++ ++ * Redistributions of source code must retain the above copyright ++ notice, this list of conditions and the following disclaimer. ++ * Redistributions in binary form must reproduce the above ++ copyright notice, this list of conditions and the following disclaimer ++ in the documentation and/or other materials provided with the ++ distribution. ++ ++ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ++ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ++ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ++ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ++ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ++ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ++ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ++ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ ++ You can contact the author at : ++ - LZ4 source repository : https://github.com/lz4/lz4 ++ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c ++*/ ++/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */ ++ ++/* ************************************* ++* Tuning Parameter ++***************************************/ ++ ++#include ++#define ALLOC(size) kzalloc(size, GFP_KERNEL) ++#define FREEMEM(ptr) kfree(ptr) ++ ++/*=== Dependency ===*/ ++#define LZ4_HC_STATIC_LINKING_ONLY ++#include "lz4hc.h" ++ ++/*! HEAPMODE : ++ * Select how stateless HC compression functions like `LZ4_compress_HC()` ++ * allocate memory for their workspace: ++ * in stack (0:fastest), or in heap (1:default, requires malloc()). ++ * Since workspace is rather large, heap mode is recommended. ++**/ ++#ifndef LZ4HC_HEAPMODE ++#define LZ4HC_HEAPMODE 1 ++#endif ++ ++/*=== Shared lz4.c code ===*/ ++#ifndef LZ4_SRC_INCLUDED ++#if defined(__GNUC__) ++#pragma GCC diagnostic ignored "-Wunused-function" ++#endif ++#if defined(__clang__) ++#pragma clang diagnostic ignored "-Wunused-function" ++#endif ++#define LZ4_COMMONDEFS_ONLY ++#include "lz4.c" /* LZ4_count, constants, mem */ ++#endif ++ ++/*=== Enums ===*/ ++typedef enum { noDictCtx, usingDictCtxHc } dictCtx_directive; ++ ++/*=== Constants ===*/ ++#define OPTIMAL_ML (int)((ML_MASK - 1) + MINMATCH) ++#define LZ4_OPT_NUM (1 << 12) ++ ++/*=== Macros ===*/ ++#define MIN(a, b) ((a) < (b) ? (a) : (b)) ++#define MAX(a, b) ((a) > (b) ? (a) : (b)) ++ ++/*=== Levels definition ===*/ ++typedef enum { lz4mid, lz4hc, lz4opt } lz4hc_strat_e; ++typedef struct { ++ lz4hc_strat_e strat; ++ int nbSearches; ++ U32 targetLength; ++} cParams_t; ++static const cParams_t k_clTable[LZ4HC_CLEVEL_MAX + 1] = { ++ { lz4mid, 2, 16 }, /* 0, unused */ ++ { lz4mid, 2, 16 }, /* 1, unused */ ++ { lz4mid, 2, 16 }, /* 2 */ ++ { lz4hc, 4, 16 }, /* 3 */ ++ { lz4hc, 8, 16 }, /* 4 */ ++ { lz4hc, 16, 16 }, /* 5 */ ++ { lz4hc, 32, 16 }, /* 6 */ ++ { lz4hc, 64, 16 }, /* 7 */ ++ { lz4hc, 128, 16 }, /* 8 */ ++ { lz4hc, 256, 16 }, /* 9 */ ++ { lz4opt, 96, 64 }, /*10==LZ4HC_CLEVEL_OPT_MIN*/ ++ { lz4opt, 512, 128 }, /*11 */ ++ { lz4opt, 16384, LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */ ++}; ++ ++static cParams_t LZ4HC_getCLevelParams(int cLevel) ++{ ++ /* note : clevel convention is a bit different from lz4frame, ++ * possibly something worth revisiting for consistency */ ++ if (cLevel < 1) ++ cLevel = LZ4HC_CLEVEL_DEFAULT; ++ cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel); ++ return k_clTable[cLevel]; ++} ++ ++/*=== Hashing ===*/ ++#define LZ4HC_HASHSIZE 4 ++#define HASH_FUNCTION(i) \ ++ (((i) * 2654435761U) >> ((MINMATCH * 8) - LZ4HC_HASH_LOG)) ++static U32 LZ4HC_hashPtr(const void *ptr) ++{ ++ return HASH_FUNCTION(LZ4_read32(ptr)); ++} ++ ++#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS == 2) ++/* lie to the compiler about data alignment; use with caution */ ++static U64 LZ4_read64(const void *memPtr) ++{ ++ return *(const U64 *)memPtr; ++} ++ ++#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS == 1) ++/* __pack instructions are safer, but compiler specific */ ++LZ4_PACK(typedef struct { U64 u64; }) LZ4_unalign64; ++static U64 LZ4_read64(const void *ptr) ++{ ++ return ((const LZ4_unalign64 *)ptr)->u64; ++} ++ ++#else /* safe and portable access using memcpy() */ ++static U64 LZ4_read64(const void *memPtr) ++{ ++ U64 val; ++ LZ4_memcpy(&val, memPtr, sizeof(val)); ++ return val; ++} ++ ++#endif /* LZ4_FORCE_MEMORY_ACCESS */ ++ ++#define LZ4MID_HASHSIZE 8 ++#define LZ4MID_HASHLOG (LZ4HC_HASH_LOG - 1) ++#define LZ4MID_HASHTABLESIZE (1 << LZ4MID_HASHLOG) ++ ++static U32 LZ4MID_hash4(U32 v) ++{ ++ return (v * 2654435761U) >> (32 - LZ4MID_HASHLOG); ++} ++static U32 LZ4MID_hash4Ptr(const void *ptr) ++{ ++ return LZ4MID_hash4(LZ4_read32(ptr)); ++} ++/* note: hash7 hashes the lower 56-bits. ++ * It presumes input was read using little endian.*/ ++static U32 LZ4MID_hash7(U64 v) ++{ ++ return (U32)(((v << (64 - 56)) * 58295818150454627ULL) >> ++ (64 - LZ4MID_HASHLOG)); ++} ++static U64 LZ4_readLE64(const void *memPtr); ++static U32 LZ4MID_hash8Ptr(const void *ptr) ++{ ++ return LZ4MID_hash7(LZ4_readLE64(ptr)); ++} ++ ++static U64 LZ4_readLE64(const void *memPtr) ++{ ++ if (LZ4_isLittleEndian()) { ++ return LZ4_read64(memPtr); ++ } else { ++ const BYTE *p = (const BYTE *)memPtr; ++ /* note: relies on the compiler to simplify this expression */ ++ return (U64)p[0] | ((U64)p[1] << 8) | ((U64)p[2] << 16) | ++ ((U64)p[3] << 24) | ((U64)p[4] << 32) | ++ ((U64)p[5] << 40) | ((U64)p[6] << 48) | ++ ((U64)p[7] << 56); ++ } ++} ++ ++/*=== Count match length ===*/ ++LZ4_FORCE_INLINE ++unsigned LZ4HC_NbCommonBytes32(U32 val) ++{ ++ assert(val != 0); ++ if (LZ4_isLittleEndian()) { ++#if defined(_MSC_VER) && (_MSC_VER >= 1400) && !defined(LZ4_FORCE_SW_BITCOUNT) ++ unsigned long r; ++ _BitScanReverse(&r, val); ++ return (unsigned)((31 - r) >> 3); ++#elif (defined(__clang__) || \ ++ (defined(__GNUC__) && \ ++ ((__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ ++ !defined(LZ4_FORCE_SW_BITCOUNT) ++ return (unsigned)__builtin_clz(val) >> 3; ++#else ++ val >>= 8; ++ val = ((((val + 0x00FFFF00) | 0x00FFFFFF) + val) | ++ (val + 0x00FF0000)) >> ++ 24; ++ return (unsigned)val ^ 3; ++#endif ++ } else { ++#if defined(_MSC_VER) && (_MSC_VER >= 1400) && !defined(LZ4_FORCE_SW_BITCOUNT) ++ unsigned long r; ++ _BitScanForward(&r, val); ++ return (unsigned)(r >> 3); ++#elif (defined(__clang__) || \ ++ (defined(__GNUC__) && \ ++ ((__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ ++ !defined(LZ4_FORCE_SW_BITCOUNT) ++ return (unsigned)__builtin_ctz(val) >> 3; ++#else ++ const U32 m = 0x01010101; ++ return (unsigned)((((val - 1) ^ val) & (m - 1)) * m) >> 24; ++#endif ++ } ++} ++ ++/** LZ4HC_countBack() : ++ * @return : negative value, nb of common bytes before ip/match */ ++LZ4_FORCE_INLINE ++int LZ4HC_countBack(const BYTE *const ip, const BYTE *const match, ++ const BYTE *const iMin, const BYTE *const mMin) ++{ ++ int back = 0; ++ int const min = (int)MAX(iMin - ip, mMin - match); ++ assert(min <= 0); ++ assert(ip >= iMin); ++ assert((size_t)(ip - iMin) < (1U << 31)); ++ assert(match >= mMin); ++ assert((size_t)(match - mMin) < (1U << 31)); ++ ++ while ((back - min) > 3) { ++ U32 const v = LZ4_read32(ip + back - 4) ^ ++ LZ4_read32(match + back - 4); ++ if (v) { ++ return (back - (int)LZ4HC_NbCommonBytes32(v)); ++ } else ++ back -= 4; /* 4-byte step */ ++ } ++ /* check remainder if any */ ++ while ((back > min) && (ip[back - 1] == match[back - 1])) ++ back--; ++ return back; ++} ++ ++/*=== Chain table updates ===*/ ++#define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */ ++/* Make fields passed to, and updated by LZ4HC_encodeSequence explicit */ ++#define UPDATABLE(ip, op, anchor) &ip, &op, &anchor ++ ++/************************************** ++* Init ++**************************************/ ++static void LZ4HC_clearTables(LZ4HC_CCtx_internal *hc4) ++{ ++ MEM_INIT(hc4->hashTable, 0, sizeof(hc4->hashTable)); ++ MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); ++} ++ ++static void LZ4HC_init_internal(LZ4HC_CCtx_internal *hc4, const BYTE *start) ++{ ++ size_t const bufferSize = (size_t)(hc4->end - hc4->prefixStart); ++ size_t newStartingOffset = bufferSize + hc4->dictLimit; ++ DEBUGLOG(5, "LZ4HC_init_internal"); ++ assert(newStartingOffset >= bufferSize); /* check overflow */ ++ if (newStartingOffset > 1 GB) { ++ LZ4HC_clearTables(hc4); ++ newStartingOffset = 0; ++ } ++ newStartingOffset += 64 KB; ++ hc4->nextToUpdate = (U32)newStartingOffset; ++ hc4->prefixStart = start; ++ hc4->end = start; ++ hc4->dictStart = start; ++ hc4->dictLimit = (U32)newStartingOffset; ++ hc4->lowLimit = (U32)newStartingOffset; ++} ++ ++/************************************** ++* Encode ++**************************************/ ++/* LZ4HC_encodeSequence() : ++ * @return : 0 if ok, ++ * 1 if buffer issue detected */ ++LZ4_FORCE_INLINE int LZ4HC_encodeSequence(const BYTE **_ip, BYTE **_op, ++ const BYTE **_anchor, int matchLength, ++ int offset, ++ limitedOutput_directive limit, ++ BYTE *oend) ++{ ++#define ip (*_ip) ++#define op (*_op) ++#define anchor (*_anchor) ++ ++ size_t length; ++ BYTE *const token = op++; ++ ++#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 6) ++ static const BYTE *start = NULL; ++ static U32 totalCost = 0; ++ U32 const pos = (start == NULL) ? 0 : (U32)(anchor - start); ++ U32 const ll = (U32)(ip - anchor); ++ U32 const llAdd = (ll >= 15) ? ((ll - 15) / 255) + 1 : 0; ++ U32 const mlAdd = ++ (matchLength >= 19) ? ((matchLength - 19) / 255) + 1 : 0; ++ U32 const cost = 1 + llAdd + ll + 2 + mlAdd; ++ if (start == NULL) ++ start = anchor; /* only works for single segment */ ++ /* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */ ++ DEBUGLOG( ++ 6, ++ "pos:%7u -- literals:%4u, match:%4i, offset:%5i, cost:%4u + %5u", ++ pos, (U32)(ip - anchor), matchLength, offset, cost, totalCost); ++ totalCost += cost; ++#endif ++ ++ /* Encode Literal length */ ++ length = (size_t)(ip - anchor); ++ LZ4_STATIC_ASSERT(notLimited == 0); ++ /* Check output limit */ ++ if (limit && ++ ((op + (length / 255) + length + (2 + 1 + LASTLITERALS)) > oend)) { ++ DEBUGLOG( ++ 6, ++ "Not enough room to write %i literals (%i bytes remaining)", ++ (int)length, (int)(oend - op)); ++ return 1; ++ } ++ if (length >= RUN_MASK) { ++ size_t len = length - RUN_MASK; ++ *token = (RUN_MASK << ML_BITS); ++ for (; len >= 255; len -= 255) ++ *op++ = 255; ++ *op++ = (BYTE)len; ++ } else { ++ *token = (BYTE)(length << ML_BITS); ++ } ++ ++ /* Copy Literals */ ++ LZ4_wildCopy8(op, anchor, op + length); ++ op += length; ++ ++ /* Encode Offset */ ++ assert(offset <= LZ4_DISTANCE_MAX); ++ assert(offset > 0); ++ LZ4_writeLE16(op, (U16)(offset)); ++ op += 2; ++ ++ /* Encode MatchLength */ ++ assert(matchLength >= MINMATCH); ++ length = (size_t)matchLength - MINMATCH; ++ if (limit && (op + (length / 255) + (1 + LASTLITERALS) > oend)) { ++ DEBUGLOG(6, "Not enough room to write match length"); ++ return 1; /* Check output limit */ ++ } ++ if (length >= ML_MASK) { ++ *token += ML_MASK; ++ length -= ML_MASK; ++ for (; length >= 510; length -= 510) { ++ *op++ = 255; ++ *op++ = 255; ++ } ++ if (length >= 255) { ++ length -= 255; ++ *op++ = 255; ++ } ++ *op++ = (BYTE)length; ++ } else { ++ *token += (BYTE)(length); ++ } ++ ++ /* Prepare next loop */ ++ ip += matchLength; ++ anchor = ip; ++ ++ return 0; ++ ++#undef ip ++#undef op ++#undef anchor ++} ++ ++typedef struct { ++ int off; ++ int len; ++ int back; /* negative value */ ++} LZ4HC_match_t; ++ ++LZ4HC_match_t LZ4HC_searchExtDict(const BYTE *ip, U32 ipIndex, ++ const BYTE *const iLowLimit, ++ const BYTE *const iHighLimit, ++ const LZ4HC_CCtx_internal *dictCtx, ++ U32 gDictEndIndex, int currentBestML, ++ int nbAttempts) ++{ ++ size_t const lDictEndIndex = ++ (size_t)(dictCtx->end - dictCtx->prefixStart) + ++ dictCtx->dictLimit; ++ U32 lDictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; ++ U32 matchIndex = lDictMatchIndex + gDictEndIndex - (U32)lDictEndIndex; ++ int offset = 0, sBack = 0; ++ assert(lDictEndIndex <= 1 GB); ++ if (lDictMatchIndex > 0) ++ DEBUGLOG(7, "lDictEndIndex = %zu, lDictMatchIndex = %u", ++ lDictEndIndex, lDictMatchIndex); ++ while (ipIndex - matchIndex <= LZ4_DISTANCE_MAX && nbAttempts--) { ++ const BYTE *const matchPtr = dictCtx->prefixStart - ++ dictCtx->dictLimit + ++ lDictMatchIndex; ++ ++ if (LZ4_read32(matchPtr) == LZ4_read32(ip)) { ++ int mlt; ++ int back = 0; ++ const BYTE *vLimit = ++ ip + (lDictEndIndex - lDictMatchIndex); ++ if (vLimit > iHighLimit) ++ vLimit = iHighLimit; ++ mlt = (int)LZ4_count(ip + MINMATCH, matchPtr + MINMATCH, ++ vLimit) + ++ MINMATCH; ++ back = (ip > iLowLimit) ? ++ LZ4HC_countBack(ip, matchPtr, iLowLimit, ++ dictCtx->prefixStart) : ++ 0; ++ mlt -= back; ++ if (mlt > currentBestML) { ++ currentBestML = mlt; ++ offset = (int)(ipIndex - matchIndex); ++ sBack = back; ++ DEBUGLOG( ++ 7, ++ "found match of length %i within extDictCtx", ++ currentBestML); ++ } ++ } ++ ++ { ++ U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, ++ lDictMatchIndex); ++ lDictMatchIndex -= nextOffset; ++ matchIndex -= nextOffset; ++ } ++ } ++ ++ { ++ LZ4HC_match_t md; ++ md.len = currentBestML; ++ md.off = offset; ++ md.back = sBack; ++ return md; ++ } ++} ++ ++typedef LZ4HC_match_t (*LZ4MID_searchIntoDict_f)( ++ const BYTE *ip, U32 ipIndex, const BYTE *const iHighLimit, ++ const LZ4HC_CCtx_internal *dictCtx, U32 gDictEndIndex); ++ ++static LZ4HC_match_t LZ4MID_searchHCDict(const BYTE *ip, U32 ipIndex, ++ const BYTE *const iHighLimit, ++ const LZ4HC_CCtx_internal *dictCtx, ++ U32 gDictEndIndex) ++{ ++ return LZ4HC_searchExtDict(ip, ipIndex, ip, iHighLimit, dictCtx, ++ gDictEndIndex, MINMATCH - 1, 2); ++} ++ ++static LZ4HC_match_t LZ4MID_searchExtDict(const BYTE *ip, U32 ipIndex, ++ const BYTE *const iHighLimit, ++ const LZ4HC_CCtx_internal *dictCtx, ++ U32 gDictEndIndex) ++{ ++ size_t const lDictEndIndex = ++ (size_t)(dictCtx->end - dictCtx->prefixStart) + ++ dictCtx->dictLimit; ++ const U32 *const hash4Table = dictCtx->hashTable; ++ const U32 *const hash8Table = hash4Table + LZ4MID_HASHTABLESIZE; ++ DEBUGLOG(7, "LZ4MID_searchExtDict (ipIdx=%u)", ipIndex); ++ ++ /* search long match first */ ++ { ++ U32 l8DictMatchIndex = hash8Table[LZ4MID_hash8Ptr(ip)]; ++ U32 m8Index = ++ l8DictMatchIndex + gDictEndIndex - (U32)lDictEndIndex; ++ assert(lDictEndIndex <= 1 GB); ++ if (ipIndex - m8Index <= LZ4_DISTANCE_MAX) { ++ const BYTE *const matchPtr = dictCtx->prefixStart - ++ dictCtx->dictLimit + ++ l8DictMatchIndex; ++ const size_t safeLen = ++ MIN(lDictEndIndex - l8DictMatchIndex, ++ (size_t)(iHighLimit - ip)); ++ int mlt = (int)LZ4_count(ip, matchPtr, ip + safeLen); ++ if (mlt >= MINMATCH) { ++ LZ4HC_match_t md; ++ DEBUGLOG(7, ++ "Found long ExtDict match of len=%u", ++ mlt); ++ md.len = mlt; ++ md.off = (int)(ipIndex - m8Index); ++ md.back = 0; ++ return md; ++ } ++ } ++ } ++ ++ /* search for short match second */ ++ { ++ U32 l4DictMatchIndex = hash4Table[LZ4MID_hash4Ptr(ip)]; ++ U32 m4Index = ++ l4DictMatchIndex + gDictEndIndex - (U32)lDictEndIndex; ++ if (ipIndex - m4Index <= LZ4_DISTANCE_MAX) { ++ const BYTE *const matchPtr = dictCtx->prefixStart - ++ dictCtx->dictLimit + ++ l4DictMatchIndex; ++ const size_t safeLen = ++ MIN(lDictEndIndex - l4DictMatchIndex, ++ (size_t)(iHighLimit - ip)); ++ int mlt = (int)LZ4_count(ip, matchPtr, ip + safeLen); ++ if (mlt >= MINMATCH) { ++ LZ4HC_match_t md; ++ DEBUGLOG(7, ++ "Found short ExtDict match of len=%u", ++ mlt); ++ md.len = mlt; ++ md.off = (int)(ipIndex - m4Index); ++ md.back = 0; ++ return md; ++ } ++ } ++ } ++ ++ /* nothing found */ ++ { ++ LZ4HC_match_t const md = { 0, 0, 0 }; ++ return md; ++ } ++} ++ ++/************************************** ++* Mid Compression (level 2) ++**************************************/ ++ ++LZ4_FORCE_INLINE void LZ4MID_addPosition(U32 *hTable, U32 hValue, U32 index) ++{ ++ hTable[hValue] = index; ++} ++ ++#define ADDPOS8(_p, _idx) \ ++ LZ4MID_addPosition(hash8Table, LZ4MID_hash8Ptr(_p), _idx) ++#define ADDPOS4(_p, _idx) \ ++ LZ4MID_addPosition(hash4Table, LZ4MID_hash4Ptr(_p), _idx) ++ ++/* Fill hash tables with references into dictionary. ++ * The resulting table is only exploitable by LZ4MID (level 2) */ ++static void LZ4MID_fillHTable(LZ4HC_CCtx_internal *cctx, const void *dict, ++ size_t size) ++{ ++ U32 *const hash4Table = cctx->hashTable; ++ U32 *const hash8Table = hash4Table + LZ4MID_HASHTABLESIZE; ++ const BYTE *const prefixPtr = (const BYTE *)dict; ++ U32 const prefixIdx = cctx->dictLimit; ++ U32 const target = prefixIdx + (U32)size - LZ4MID_HASHSIZE; ++ U32 idx = cctx->nextToUpdate; ++ assert(dict == cctx->prefixStart); ++ DEBUGLOG(4, "LZ4MID_fillHTable (size:%zu)", size); ++ if (size <= LZ4MID_HASHSIZE) ++ return; ++ ++ for (; idx < target; idx += 3) { ++ ADDPOS4(prefixPtr + idx - prefixIdx, idx); ++ ADDPOS8(prefixPtr + idx + 1 - prefixIdx, idx + 1); ++ } ++ ++ idx = (size > 32 KB + LZ4MID_HASHSIZE) ? target - 32 KB : ++ cctx->nextToUpdate; ++ for (; idx < target; idx += 1) { ++ ADDPOS8(prefixPtr + idx - prefixIdx, idx); ++ } ++ ++ cctx->nextToUpdate = target; ++} ++ ++static LZ4MID_searchIntoDict_f ++select_searchDict_function(const LZ4HC_CCtx_internal *dictCtx) ++{ ++ if (dictCtx == NULL) ++ return NULL; ++ if (LZ4HC_getCLevelParams(dictCtx->compressionLevel).strat == lz4mid) ++ return LZ4MID_searchExtDict; ++ return LZ4MID_searchHCDict; ++} ++ ++static int LZ4MID_compress(LZ4HC_CCtx_internal *const ctx, ++ const char *const src, char *const dst, ++ int *srcSizePtr, int const maxOutputSize, ++ const limitedOutput_directive limit, ++ const dictCtx_directive dict) ++{ ++ U32 *const hash4Table = ctx->hashTable; ++ U32 *const hash8Table = hash4Table + LZ4MID_HASHTABLESIZE; ++ const BYTE *ip = (const BYTE *)src; ++ const BYTE *anchor = ip; ++ const BYTE *const iend = ip + *srcSizePtr; ++ const BYTE *const mflimit = iend - MFLIMIT; ++ const BYTE *const matchlimit = (iend - LASTLITERALS); ++ const BYTE *const ilimit = (iend - LZ4MID_HASHSIZE); ++ BYTE *op = (BYTE *)dst; ++ BYTE *oend = op + maxOutputSize; ++ ++ const BYTE *const prefixPtr = ctx->prefixStart; ++ const U32 prefixIdx = ctx->dictLimit; ++ const U32 ilimitIdx = (U32)(ilimit - prefixPtr) + prefixIdx; ++ const BYTE *const dictStart = ctx->dictStart; ++ const U32 dictIdx = ctx->lowLimit; ++ const U32 gDictEndIndex = ctx->lowLimit; ++ const LZ4MID_searchIntoDict_f searchIntoDict = ++ (dict == usingDictCtxHc) ? ++ select_searchDict_function(ctx->dictCtx) : ++ NULL; ++ unsigned matchLength; ++ unsigned matchDistance; ++ ++ /* input sanitization */ ++ DEBUGLOG(5, "LZ4MID_compress (%i bytes)", *srcSizePtr); ++ if (dict == usingDictCtxHc) ++ DEBUGLOG(5, "usingDictCtxHc"); ++ assert(*srcSizePtr >= 0); ++ if (*srcSizePtr) ++ assert(src != NULL); ++ if (maxOutputSize) ++ assert(dst != NULL); ++ if (*srcSizePtr < 0) ++ return 0; /* invalid */ ++ if (maxOutputSize < 0) ++ return 0; /* invalid */ ++ if (*srcSizePtr > LZ4_MAX_INPUT_SIZE) { ++ /* forbidden: no input is allowed to be that large */ ++ return 0; ++ } ++ if (limit == fillOutput) ++ oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ ++ if (*srcSizePtr < LZ4_minLength) ++ goto _lz4mid_last_literals; /* Input too small, no compression (all literals) */ ++ ++ /* main loop */ ++ while (ip <= mflimit) { ++ const U32 ipIndex = (U32)(ip - prefixPtr) + prefixIdx; ++ /* search long match */ ++ { ++ U32 const h8 = LZ4MID_hash8Ptr(ip); ++ U32 const pos8 = hash8Table[h8]; ++ assert(h8 < LZ4MID_HASHTABLESIZE); ++ assert(pos8 < ipIndex); ++ LZ4MID_addPosition(hash8Table, h8, ipIndex); ++ if (ipIndex - pos8 <= LZ4_DISTANCE_MAX) { ++ /* match candidate found */ ++ if (pos8 >= prefixIdx) { ++ const BYTE *const matchPtr = ++ prefixPtr + pos8 - prefixIdx; ++ assert(matchPtr < ip); ++ matchLength = LZ4_count(ip, matchPtr, ++ matchlimit); ++ if (matchLength >= MINMATCH) { ++ DEBUGLOG( ++ 7, ++ "found long match at pos %u (len=%u)", ++ pos8, matchLength); ++ matchDistance = ipIndex - pos8; ++ goto _lz4mid_encode_sequence; ++ } ++ } else { ++ if (pos8 >= dictIdx) { ++ /* extDict match candidate */ ++ const BYTE *const matchPtr = ++ dictStart + ++ (pos8 - dictIdx); ++ const size_t safeLen = MIN( ++ prefixIdx - pos8, ++ (size_t)(matchlimit - ++ ip)); ++ matchLength = ++ LZ4_count(ip, matchPtr, ++ ip + safeLen); ++ if (matchLength >= MINMATCH) { ++ DEBUGLOG( ++ 7, ++ "found long match at ExtDict pos %u (len=%u)", ++ pos8, ++ matchLength); ++ matchDistance = ++ ipIndex - pos8; ++ goto _lz4mid_encode_sequence; ++ } ++ } ++ } ++ } ++ } ++ /* search short match */ ++ { ++ U32 const h4 = LZ4MID_hash4Ptr(ip); ++ U32 const pos4 = hash4Table[h4]; ++ assert(h4 < LZ4MID_HASHTABLESIZE); ++ assert(pos4 < ipIndex); ++ LZ4MID_addPosition(hash4Table, h4, ipIndex); ++ if (ipIndex - pos4 <= LZ4_DISTANCE_MAX) { ++ /* match candidate found */ ++ if (pos4 >= prefixIdx) { ++ /* only search within prefix */ ++ const BYTE *const matchPtr = ++ prefixPtr + (pos4 - prefixIdx); ++ assert(matchPtr < ip); ++ assert(matchPtr >= prefixPtr); ++ matchLength = LZ4_count(ip, matchPtr, ++ matchlimit); ++ if (matchLength >= MINMATCH) { ++ /* short match found, let's just check ip+1 for longer */ ++ U32 const h8 = ++ LZ4MID_hash8Ptr(ip + 1); ++ U32 const pos8 = hash8Table[h8]; ++ U32 const m2Distance = ++ ipIndex + 1 - pos8; ++ matchDistance = ipIndex - pos4; ++ if (m2Distance <= ++ LZ4_DISTANCE_MAX && ++ pos8 >= prefixIdx /* only search within prefix */ ++ && likely(ip < mflimit)) { ++ const BYTE *const m2Ptr = ++ prefixPtr + ++ (pos8 - ++ prefixIdx); ++ unsigned ml2 = LZ4_count( ++ ip + 1, m2Ptr, ++ matchlimit); ++ if (ml2 > matchLength) { ++ LZ4MID_addPosition( ++ hash8Table, ++ h8, ++ ipIndex + ++ 1); ++ ip++; ++ matchLength = ++ ml2; ++ matchDistance = ++ m2Distance; ++ } ++ } ++ goto _lz4mid_encode_sequence; ++ } ++ } else { ++ if (pos4 >= dictIdx) { ++ /* extDict match candidate */ ++ const BYTE *const matchPtr = ++ dictStart + ++ (pos4 - dictIdx); ++ const size_t safeLen = MIN( ++ prefixIdx - pos4, ++ (size_t)(matchlimit - ++ ip)); ++ matchLength = ++ LZ4_count(ip, matchPtr, ++ ip + safeLen); ++ if (matchLength >= MINMATCH) { ++ DEBUGLOG( ++ 7, ++ "found match at ExtDict pos %u (len=%u)", ++ pos4, ++ matchLength); ++ matchDistance = ++ ipIndex - pos4; ++ goto _lz4mid_encode_sequence; ++ } ++ } ++ } ++ } ++ } ++ /* no match found in prefix */ ++ if ((dict == usingDictCtxHc) && ++ (ipIndex - gDictEndIndex < LZ4_DISTANCE_MAX - 8)) { ++ /* search a match into external dictionary */ ++ LZ4HC_match_t dMatch = ++ searchIntoDict(ip, ipIndex, matchlimit, ++ ctx->dictCtx, gDictEndIndex); ++ if (dMatch.len >= MINMATCH) { ++ DEBUGLOG(7, ++ "found Dictionary match (offset=%i)", ++ dMatch.off); ++ assert(dMatch.back == 0); ++ matchLength = (unsigned)dMatch.len; ++ matchDistance = (unsigned)dMatch.off; ++ goto _lz4mid_encode_sequence; ++ } ++ } ++ /* no match found */ ++ ip += 1 + ((ip - anchor) >> ++ 9); /* skip faster over incompressible data */ ++ continue; ++ ++ _lz4mid_encode_sequence: ++ /* catch back */ ++ while (((ip > anchor) & ++ ((U32)(ip - prefixPtr) > matchDistance)) && ++ (unlikely(ip[-1] == ip[-(int)matchDistance - 1]))) { ++ ip--; ++ matchLength++; ++ }; ++ ++ /* fill table with beginning of match */ ++ ADDPOS8(ip + 1, ipIndex + 1); ++ ADDPOS8(ip + 2, ipIndex + 2); ++ ADDPOS4(ip + 1, ipIndex + 1); ++ ++ /* encode */ ++ { ++ BYTE *const saved_op = op; ++ /* LZ4HC_encodeSequence always updates @op; on success, it updates @ip and @anchor */ ++ if (LZ4HC_encodeSequence( ++ UPDATABLE(ip, op, anchor), (int)matchLength, ++ (int)matchDistance, limit, oend)) { ++ op = saved_op; /* restore @op value before failed LZ4HC_encodeSequence */ ++ goto _lz4mid_dest_overflow; ++ } ++ } ++ ++ /* fill table with end of match */ ++ { ++ U32 endMatchIdx = (U32)(ip - prefixPtr) + prefixIdx; ++ U32 pos_m2 = endMatchIdx - 2; ++ if (pos_m2 < ilimitIdx) { ++ if (likely(ip - prefixPtr > 5)) { ++ ADDPOS8(ip - 5, endMatchIdx - 5); ++ } ++ ADDPOS8(ip - 3, endMatchIdx - 3); ++ ADDPOS8(ip - 2, endMatchIdx - 2); ++ ADDPOS4(ip - 2, endMatchIdx - 2); ++ ADDPOS4(ip - 1, endMatchIdx - 1); ++ } ++ } ++ } ++ ++_lz4mid_last_literals: ++ /* Encode Last Literals */ ++ { ++ size_t lastRunSize = (size_t)(iend - anchor); /* literals */ ++ size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255; ++ size_t const totalSize = 1 + llAdd + lastRunSize; ++ if (limit == fillOutput) ++ oend += LASTLITERALS; /* restore correct value */ ++ if (limit && (op + totalSize > oend)) { ++ if (limit == limitedOutput) ++ return 0; /* not enough space in @dst */ ++ /* adapt lastRunSize to fill 'dest' */ ++ lastRunSize = (size_t)(oend - op) - 1 /*token*/; ++ llAdd = (lastRunSize + 256 - RUN_MASK) / 256; ++ lastRunSize -= llAdd; ++ } ++ DEBUGLOG(6, "Final literal run : %i literals", ++ (int)lastRunSize); ++ ip = anchor + ++ lastRunSize; /* can be != iend if limit==fillOutput */ ++ ++ if (lastRunSize >= RUN_MASK) { ++ size_t accumulator = lastRunSize - RUN_MASK; ++ *op++ = (RUN_MASK << ML_BITS); ++ for (; accumulator >= 255; accumulator -= 255) ++ *op++ = 255; ++ *op++ = (BYTE)accumulator; ++ } else { ++ *op++ = (BYTE)(lastRunSize << ML_BITS); ++ } ++ assert(lastRunSize <= (size_t)(oend - op)); ++ LZ4_memcpy(op, anchor, lastRunSize); ++ op += lastRunSize; ++ } ++ ++ /* End */ ++ DEBUGLOG(5, "compressed %i bytes into %i bytes", *srcSizePtr, ++ (int)((char *)op - dst)); ++ assert(ip >= (const BYTE *)src); ++ assert(ip <= iend); ++ *srcSizePtr = (int)(ip - (const BYTE *)src); ++ assert((char *)op >= dst); ++ assert(op <= oend); ++ assert((char *)op - dst < INT_MAX); ++ return (int)((char *)op - dst); ++ ++_lz4mid_dest_overflow: ++ if (limit == fillOutput) { ++ /* Assumption : @ip, @anchor, @optr and @matchLength must be set correctly */ ++ size_t const ll = (size_t)(ip - anchor); ++ size_t const ll_addbytes = (ll + 240) / 255; ++ size_t const ll_totalCost = 1 + ll_addbytes + ll; ++ BYTE *const maxLitPos = ++ oend - 3; /* 2 for offset, 1 for token */ ++ DEBUGLOG( ++ 6, ++ "Last sequence is overflowing : %u literals, %u remaining space", ++ (unsigned)ll, (unsigned)(oend - op)); ++ if (op + ll_totalCost <= maxLitPos) { ++ /* ll validated; now adjust match length */ ++ size_t const bytesLeftForMl = ++ (size_t)(maxLitPos - (op + ll_totalCost)); ++ size_t const maxMlSize = MINMATCH + (ML_MASK - 1) + ++ (bytesLeftForMl * 255); ++ assert(maxMlSize < INT_MAX); ++ if ((size_t)matchLength > maxMlSize) ++ matchLength = (unsigned)maxMlSize; ++ if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - ++ 1 + matchLength >= ++ MFLIMIT) { ++ DEBUGLOG( ++ 6, ++ "Let's encode a last sequence (ll=%u, ml=%u)", ++ (unsigned)ll, matchLength); ++ LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ++ (int)matchLength, ++ (int)matchDistance, ++ notLimited, oend); ++ } ++ } ++ DEBUGLOG(6, ++ "Let's finish with a run of literals (%u bytes left)", ++ (unsigned)(oend - op)); ++ goto _lz4mid_last_literals; ++ } ++ /* compression failed */ ++ return 0; ++} ++ ++/************************************** ++* HC Compression - Search ++**************************************/ ++ ++/* Update chains up to ip (excluded) */ ++LZ4_FORCE_INLINE void LZ4HC_Insert(LZ4HC_CCtx_internal *hc4, const BYTE *ip) ++{ ++ U16 *const chainTable = hc4->chainTable; ++ U32 *const hashTable = hc4->hashTable; ++ const BYTE *const prefixPtr = hc4->prefixStart; ++ U32 const prefixIdx = hc4->dictLimit; ++ U32 const target = (U32)(ip - prefixPtr) + prefixIdx; ++ U32 idx = hc4->nextToUpdate; ++ assert(ip >= prefixPtr); ++ assert(target >= prefixIdx); ++ ++ while (idx < target) { ++ U32 const h = LZ4HC_hashPtr(prefixPtr + idx - prefixIdx); ++ size_t delta = idx - hashTable[h]; ++ if (delta > LZ4_DISTANCE_MAX) ++ delta = LZ4_DISTANCE_MAX; ++ DELTANEXTU16(chainTable, idx) = (U16)delta; ++ hashTable[h] = idx; ++ idx++; ++ } ++ ++ hc4->nextToUpdate = target; ++} ++ ++#if defined(_MSC_VER) ++#define LZ4HC_rotl32(x, r) _rotl(x, r) ++#else ++#define LZ4HC_rotl32(x, r) ((x << r) | (x >> (32 - r))) ++#endif ++ ++static U32 LZ4HC_rotatePattern(size_t const rotate, U32 const pattern) ++{ ++ size_t const bitsToRotate = (rotate & (sizeof(pattern) - 1)) << 3; ++ if (bitsToRotate == 0) ++ return pattern; ++ return LZ4HC_rotl32(pattern, (int)bitsToRotate); ++} ++ ++/* LZ4HC_countPattern() : ++ * pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */ ++static unsigned LZ4HC_countPattern(const BYTE *ip, const BYTE *const iEnd, ++ U32 const pattern32) ++{ ++ const BYTE *const iStart = ip; ++ reg_t const pattern = ++ (sizeof(pattern) == 8) ? ++ (reg_t)pattern32 + ++ (((reg_t)pattern32) << (sizeof(pattern) * 4)) : ++ pattern32; ++ ++ while (likely(ip < iEnd - (sizeof(pattern) - 1))) { ++ reg_t const diff = LZ4_read_ARCH(ip) ^ pattern; ++ if (!diff) { ++ ip += sizeof(pattern); ++ continue; ++ } ++ ip += LZ4_NbCommonBytes(diff); ++ return (unsigned)(ip - iStart); ++ } ++ ++ if (LZ4_isLittleEndian()) { ++ reg_t patternByte = pattern; ++ while ((ip < iEnd) && (*ip == (BYTE)patternByte)) { ++ ip++; ++ patternByte >>= 8; ++ } ++ } else { /* big endian */ ++ U32 bitOffset = (sizeof(pattern) * 8) - 8; ++ while (ip < iEnd) { ++ BYTE const byte = (BYTE)(pattern >> bitOffset); ++ if (*ip != byte) ++ break; ++ ip++; ++ bitOffset -= 8; ++ } ++ } ++ ++ return (unsigned)(ip - iStart); ++} ++ ++/* LZ4HC_reverseCountPattern() : ++ * pattern must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) ++ * read using natural platform endianness */ ++static unsigned LZ4HC_reverseCountPattern(const BYTE *ip, ++ const BYTE *const iLow, U32 pattern) ++{ ++ const BYTE *const iStart = ip; ++ ++ while (likely(ip >= iLow + 4)) { ++ if (LZ4_read32(ip - 4) != pattern) ++ break; ++ ip -= 4; ++ } ++ { ++ const BYTE *bytePtr = (const BYTE *)(&pattern) + ++ 3; /* works for any endianness */ ++ while (likely(ip > iLow)) { ++ if (ip[-1] != *bytePtr) ++ break; ++ ip--; ++ bytePtr--; ++ } ++ } ++ return (unsigned)(iStart - ip); ++} ++ ++/* LZ4HC_protectDictEnd() : ++ * Checks if the match is in the last 3 bytes of the dictionary, so reading the ++ * 4 byte MINMATCH would overflow. ++ * @returns true if the match index is okay. ++ */ ++static int LZ4HC_protectDictEnd(U32 const dictLimit, U32 const matchIndex) ++{ ++ return ((U32)((dictLimit - 1) - matchIndex) >= 3); ++} ++ ++typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e; ++typedef enum { favorCompressionRatio = 0, favorDecompressionSpeed } HCfavor_e; ++ ++LZ4_FORCE_INLINE LZ4HC_match_t LZ4HC_InsertAndGetWiderMatch( ++ LZ4HC_CCtx_internal *const hc4, const BYTE *const ip, ++ const BYTE *const iLowLimit, const BYTE *const iHighLimit, int longest, ++ const int maxNbAttempts, const int patternAnalysis, const int chainSwap, ++ const dictCtx_directive dict, const HCfavor_e favorDecSpeed) ++{ ++ U16 *const chainTable = hc4->chainTable; ++ U32 *const hashTable = hc4->hashTable; ++ const LZ4HC_CCtx_internal *const dictCtx = hc4->dictCtx; ++ const BYTE *const prefixPtr = hc4->prefixStart; ++ const U32 prefixIdx = hc4->dictLimit; ++ const U32 ipIndex = (U32)(ip - prefixPtr) + prefixIdx; ++ const int withinStartDistance = ++ (hc4->lowLimit + (LZ4_DISTANCE_MAX + 1) > ipIndex); ++ const U32 lowestMatchIndex = (withinStartDistance) ? ++ hc4->lowLimit : ++ ipIndex - LZ4_DISTANCE_MAX; ++ const BYTE *const dictStart = hc4->dictStart; ++ const U32 dictIdx = hc4->lowLimit; ++ const BYTE *const dictEnd = dictStart + prefixIdx - dictIdx; ++ int const lookBackLength = (int)(ip - iLowLimit); ++ int nbAttempts = maxNbAttempts; ++ U32 matchChainPos = 0; ++ U32 const pattern = LZ4_read32(ip); ++ U32 matchIndex; ++ repeat_state_e repeat = rep_untested; ++ size_t srcPatternLength = 0; ++ int offset = 0, sBack = 0; ++ ++ DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch"); ++ /* First Match */ ++ LZ4HC_Insert(hc4, ++ ip); /* insert all prior positions up to ip (excluded) */ ++ matchIndex = hashTable[LZ4HC_hashPtr(ip)]; ++ DEBUGLOG( ++ 7, ++ "First candidate match for pos %u found at index %u / %u (lowestMatchIndex)", ++ ipIndex, matchIndex, lowestMatchIndex); ++ ++ while ((matchIndex >= lowestMatchIndex) && (nbAttempts > 0)) { ++ int matchLength = 0; ++ nbAttempts--; ++ assert(matchIndex < ipIndex); ++ if (favorDecSpeed && (ipIndex - matchIndex < 8)) { ++ /* do nothing: ++ * favorDecSpeed intentionally skips matches with offset < 8 */ ++ } else if (matchIndex >= ++ prefixIdx) { /* within current Prefix */ ++ const BYTE *const matchPtr = ++ prefixPtr + (matchIndex - prefixIdx); ++ assert(matchPtr < ip); ++ assert(longest >= 1); ++ if (LZ4_read16(iLowLimit + longest - 1) == ++ LZ4_read16(matchPtr - lookBackLength + longest - ++ 1)) { ++ if (LZ4_read32(matchPtr) == pattern) { ++ int const back = ++ lookBackLength ? ++ LZ4HC_countBack( ++ ip, matchPtr, ++ iLowLimit, ++ prefixPtr) : ++ 0; ++ matchLength = ++ MINMATCH + ++ (int)LZ4_count(ip + MINMATCH, ++ matchPtr + ++ MINMATCH, ++ iHighLimit); ++ matchLength -= back; ++ if (matchLength > longest) { ++ longest = matchLength; ++ offset = (int)(ipIndex - ++ matchIndex); ++ sBack = back; ++ DEBUGLOG( ++ 7, ++ "Found match of len=%i within prefix, offset=%i, back=%i", ++ longest, offset, -back); ++ } ++ } ++ } ++ } else { /* lowestMatchIndex <= matchIndex < dictLimit : within Ext Dict */ ++ const BYTE *const matchPtr = ++ dictStart + (matchIndex - dictIdx); ++ assert(matchIndex >= dictIdx); ++ if (likely(matchIndex <= prefixIdx - 4) && ++ (LZ4_read32(matchPtr) == pattern)) { ++ int back = 0; ++ const BYTE *vLimit = ++ ip + (prefixIdx - matchIndex); ++ if (vLimit > iHighLimit) ++ vLimit = iHighLimit; ++ matchLength = ++ (int)LZ4_count(ip + MINMATCH, ++ matchPtr + MINMATCH, ++ vLimit) + ++ MINMATCH; ++ if ((ip + matchLength == vLimit) && ++ (vLimit < iHighLimit)) ++ matchLength += ++ LZ4_count(ip + matchLength, ++ prefixPtr, ++ iHighLimit); ++ back = lookBackLength ? ++ LZ4HC_countBack(ip, matchPtr, ++ iLowLimit, ++ dictStart) : ++ 0; ++ matchLength -= back; ++ if (matchLength > longest) { ++ longest = matchLength; ++ offset = (int)(ipIndex - matchIndex); ++ sBack = back; ++ DEBUGLOG( ++ 7, ++ "Found match of len=%i within dict, offset=%i, back=%i", ++ longest, offset, -back); ++ } ++ } ++ } ++ ++ if (chainSwap && ++ matchLength == ++ longest) { /* better match => select a better chain */ ++ assert(lookBackLength == 0); /* search forward only */ ++ if (matchIndex + (U32)longest <= ipIndex) { ++ int const kTrigger = 4; ++ U32 distanceToNextMatch = 1; ++ int const end = longest - MINMATCH + 1; ++ int step = 1; ++ int accel = 1 << kTrigger; ++ int pos; ++ for (pos = 0; pos < end; pos += step) { ++ U32 const candidateDist = DELTANEXTU16( ++ chainTable, ++ matchIndex + (U32)pos); ++ step = (accel++ >> kTrigger); ++ if (candidateDist > ++ distanceToNextMatch) { ++ distanceToNextMatch = ++ candidateDist; ++ matchChainPos = (U32)pos; ++ accel = 1 << kTrigger; ++ } ++ } ++ if (distanceToNextMatch > 1) { ++ if (distanceToNextMatch > matchIndex) ++ break; /* avoid overflow */ ++ matchIndex -= distanceToNextMatch; ++ continue; ++ } ++ } ++ } ++ ++ { ++ U32 const distNextMatch = ++ DELTANEXTU16(chainTable, matchIndex); ++ if (patternAnalysis && distNextMatch == 1 && ++ matchChainPos == 0) { ++ U32 const matchCandidateIdx = matchIndex - 1; ++ /* may be a repeated pattern */ ++ if (repeat == rep_untested) { ++ if (((pattern & 0xFFFF) == ++ (pattern >> 16)) & ++ ((pattern & 0xFF) == ++ (pattern >> 24))) { ++ DEBUGLOG( ++ 7, ++ "Repeat pattern detected, char %02X", ++ pattern >> 24); ++ repeat = rep_confirmed; ++ srcPatternLength = ++ LZ4HC_countPattern( ++ ip + sizeof(pattern), ++ iHighLimit, ++ pattern) + ++ sizeof(pattern); ++ } else { ++ repeat = rep_not; ++ } ++ } ++ if ((repeat == rep_confirmed) && ++ (matchCandidateIdx >= lowestMatchIndex) && ++ LZ4HC_protectDictEnd(prefixIdx, ++ matchCandidateIdx)) { ++ const int extDict = ++ matchCandidateIdx < prefixIdx; ++ const BYTE *const matchPtr = ++ extDict ? ++ dictStart + ++ (matchCandidateIdx - ++ dictIdx) : ++ prefixPtr + ++ (matchCandidateIdx - ++ prefixIdx); ++ if (LZ4_read32(matchPtr) == ++ pattern) { /* good candidate */ ++ const BYTE *const iLimit = ++ extDict ? dictEnd : ++ iHighLimit; ++ size_t forwardPatternLength = ++ LZ4HC_countPattern( ++ matchPtr + ++ sizeof(pattern), ++ iLimit, ++ pattern) + ++ sizeof(pattern); ++ if (extDict && ++ matchPtr + forwardPatternLength == ++ iLimit) { ++ U32 const rotatedPattern = ++ LZ4HC_rotatePattern( ++ forwardPatternLength, ++ pattern); ++ forwardPatternLength += ++ LZ4HC_countPattern( ++ prefixPtr, ++ iHighLimit, ++ rotatedPattern); ++ } ++ { ++ const BYTE *const lowestMatchPtr = ++ extDict ? ++ dictStart : ++ prefixPtr; ++ size_t backLength = ++ LZ4HC_reverseCountPattern( ++ matchPtr, ++ lowestMatchPtr, ++ pattern); ++ size_t currentSegmentLength; ++ if (!extDict && ++ matchPtr - backLength == ++ prefixPtr && ++ dictIdx < ++ prefixIdx) { ++ U32 const rotatedPattern = LZ4HC_rotatePattern( ++ (U32)(-(int)backLength), ++ pattern); ++ backLength += LZ4HC_reverseCountPattern( ++ dictEnd, ++ dictStart, ++ rotatedPattern); ++ } ++ /* Limit backLength not go further than lowestMatchIndex */ ++ backLength = ++ matchCandidateIdx - ++ MAX(matchCandidateIdx - ++ (U32)backLength, ++ lowestMatchIndex); ++ assert(matchCandidateIdx - ++ backLength >= ++ lowestMatchIndex); ++ currentSegmentLength = ++ backLength + ++ forwardPatternLength; ++ /* Adjust to end of pattern if the source pattern fits, otherwise the beginning of the pattern */ ++ if ((currentSegmentLength >= ++ srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */ ++ && ++ (forwardPatternLength <= ++ srcPatternLength)) { /* haven't reached this position yet */ ++ U32 const newMatchIndex = ++ matchCandidateIdx + ++ (U32)forwardPatternLength - ++ (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */ ++ if (LZ4HC_protectDictEnd( ++ prefixIdx, ++ newMatchIndex)) ++ matchIndex = ++ newMatchIndex; ++ else { ++ /* Can only happen if started in the prefix */ ++ assert(newMatchIndex >= ++ prefixIdx - ++ 3 && ++ newMatchIndex < ++ prefixIdx && ++ !extDict); ++ matchIndex = ++ prefixIdx; ++ } ++ } else { ++ U32 const newMatchIndex = ++ matchCandidateIdx - ++ (U32)backLength; /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */ ++ if (!LZ4HC_protectDictEnd( ++ prefixIdx, ++ newMatchIndex)) { ++ assert(newMatchIndex >= ++ prefixIdx - ++ 3 && ++ newMatchIndex < ++ prefixIdx && ++ !extDict); ++ matchIndex = ++ prefixIdx; ++ } else { ++ matchIndex = ++ newMatchIndex; ++ if (lookBackLength == ++ 0) { /* no back possible */ ++ size_t const maxML = ++ MIN(currentSegmentLength, ++ srcPatternLength); ++ if ((size_t)longest < ++ maxML) { ++ assert(prefixPtr - ++ prefixIdx + ++ matchIndex != ++ ip); ++ if ((size_t)(ip - ++ prefixPtr) + ++ prefixIdx - ++ matchIndex > ++ LZ4_DISTANCE_MAX) ++ break; ++ assert(maxML < ++ 2 GB); ++ longest = (int) ++ maxML; ++ offset = ++ (int)(ipIndex - ++ matchIndex); ++ assert(sBack == ++ 0); ++ DEBUGLOG( ++ 7, ++ "Found repeat pattern match of len=%i, offset=%i", ++ longest, ++ offset); ++ } ++ { ++ U32 const distToNextPattern = ++ DELTANEXTU16( ++ chainTable, ++ matchIndex); ++ if (distToNextPattern > ++ matchIndex) ++ break; /* avoid overflow */ ++ matchIndex -= ++ distToNextPattern; ++ } ++ } ++ } ++ } ++ } ++ continue; ++ } ++ } ++ } ++ } /* PA optimization */ ++ ++ /* follow current chain */ ++ matchIndex -= ++ DELTANEXTU16(chainTable, matchIndex + matchChainPos); ++ ++ } /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */ ++ ++ if (dict == usingDictCtxHc && nbAttempts > 0 && withinStartDistance) { ++ size_t const dictEndOffset = ++ (size_t)(dictCtx->end - dictCtx->prefixStart) + ++ dictCtx->dictLimit; ++ U32 dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; ++ assert(dictEndOffset <= 1 GB); ++ matchIndex = ++ dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset; ++ if (dictMatchIndex > 0) ++ DEBUGLOG( ++ 7, ++ "dictEndOffset = %zu, dictMatchIndex = %u => relative matchIndex = %i", ++ dictEndOffset, dictMatchIndex, ++ (int)dictMatchIndex - (int)dictEndOffset); ++ while (ipIndex - matchIndex <= LZ4_DISTANCE_MAX && ++ nbAttempts--) { ++ const BYTE *const matchPtr = dictCtx->prefixStart - ++ dictCtx->dictLimit + ++ dictMatchIndex; ++ ++ if (LZ4_read32(matchPtr) == pattern) { ++ int mlt; ++ int back = 0; ++ const BYTE *vLimit = ++ ip + (dictEndOffset - dictMatchIndex); ++ if (vLimit > iHighLimit) ++ vLimit = iHighLimit; ++ mlt = (int)LZ4_count(ip + MINMATCH, ++ matchPtr + MINMATCH, ++ vLimit) + ++ MINMATCH; ++ back = lookBackLength ? ++ LZ4HC_countBack( ++ ip, matchPtr, iLowLimit, ++ dictCtx->prefixStart) : ++ 0; ++ mlt -= back; ++ if (mlt > longest) { ++ longest = mlt; ++ offset = (int)(ipIndex - matchIndex); ++ sBack = back; ++ DEBUGLOG( ++ 7, ++ "found match of length %i within extDictCtx", ++ longest); ++ } ++ } ++ ++ { ++ U32 const nextOffset = DELTANEXTU16( ++ dictCtx->chainTable, dictMatchIndex); ++ dictMatchIndex -= nextOffset; ++ matchIndex -= nextOffset; ++ } ++ } ++ } ++ ++ { ++ LZ4HC_match_t md; ++ assert(longest >= 0); ++ md.len = longest; ++ md.off = offset; ++ md.back = sBack; ++ return md; ++ } ++} ++ ++LZ4_FORCE_INLINE LZ4HC_match_t LZ4HC_InsertAndFindBestMatch( ++ LZ4HC_CCtx_internal *const hc4, /* Index table will be updated */ ++ const BYTE *const ip, const BYTE *const iLimit, const int maxNbAttempts, ++ const int patternAnalysis, const dictCtx_directive dict) ++{ ++ DEBUGLOG(7, "LZ4HC_InsertAndFindBestMatch"); ++ /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), ++ * but this won't be the case here, as we define iLowLimit==ip, ++ * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ ++ return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH - 1, ++ maxNbAttempts, patternAnalysis, ++ 0 /*chainSwap*/, dict, ++ favorCompressionRatio); ++} ++ ++LZ4_FORCE_INLINE int ++LZ4HC_compress_hashChain(LZ4HC_CCtx_internal *const ctx, ++ const char *const source, char *const dest, ++ int *srcSizePtr, int const maxOutputSize, ++ int maxNbAttempts, const limitedOutput_directive limit, ++ const dictCtx_directive dict) ++{ ++ const int inputSize = *srcSizePtr; ++ const int patternAnalysis = (maxNbAttempts > 128); /* levels 9+ */ ++ ++ const BYTE *ip = (const BYTE *)source; ++ const BYTE *anchor = ip; ++ const BYTE *const iend = ip + inputSize; ++ const BYTE *const mflimit = iend - MFLIMIT; ++ const BYTE *const matchlimit = (iend - LASTLITERALS); ++ ++ BYTE *optr = (BYTE *)dest; ++ BYTE *op = (BYTE *)dest; ++ BYTE *oend = op + maxOutputSize; ++ ++ const BYTE *start0; ++ const BYTE *start2 = NULL; ++ const BYTE *start3 = NULL; ++ LZ4HC_match_t m0, m1, m2, m3; ++ const LZ4HC_match_t nomatch = { 0, 0, 0 }; ++ ++ /* init */ ++ DEBUGLOG(5, "LZ4HC_compress_hashChain (dict?=>%i)", dict); ++ *srcSizePtr = 0; ++ if (limit == fillOutput) ++ oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ ++ if (inputSize < LZ4_minLength) ++ goto _last_literals; /* Input too small, no compression (all literals) */ ++ ++ /* Main Loop */ ++ while (ip <= mflimit) { ++ m1 = LZ4HC_InsertAndFindBestMatch(ctx, ip, matchlimit, ++ maxNbAttempts, ++ patternAnalysis, dict); ++ if (m1.len < MINMATCH) { ++ ip++; ++ continue; ++ } ++ ++ /* saved, in case we would skip too much */ ++ start0 = ip; ++ m0 = m1; ++ ++ _Search2: ++ DEBUGLOG(7, "_Search2 (currently found match of size %i)", ++ m1.len); ++ if (ip + m1.len <= mflimit) { ++ start2 = ip + m1.len - 2; ++ m2 = LZ4HC_InsertAndGetWiderMatch( ++ ctx, start2, ip + 0, matchlimit, m1.len, ++ maxNbAttempts, patternAnalysis, 0, dict, ++ favorCompressionRatio); ++ start2 += m2.back; ++ } else { ++ m2 = nomatch; /* do not search further */ ++ } ++ ++ if (m2.len <= ++ m1.len) { /* No better match => encode ML1 immediately */ ++ optr = op; ++ if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ++ m1.len, m1.off, limit, oend)) ++ goto _dest_overflow; ++ continue; ++ } ++ ++ if (start0 < ip) { /* first match was skipped at least once */ ++ if (start2 < ++ ip + m0.len) { /* squeezing ML1 between ML0(original ML1) and ML2 */ ++ ip = start0; ++ m1 = m0; /* restore initial Match1 */ ++ } ++ } ++ ++ /* Here, start0==ip */ ++ if ((start2 - ip) < 3) { /* First Match too small : removed */ ++ ip = start2; ++ m1 = m2; ++ goto _Search2; ++ } ++ ++ _Search3: ++ if ((start2 - ip) < OPTIMAL_ML) { ++ int correction; ++ int new_ml = m1.len; ++ if (new_ml > OPTIMAL_ML) ++ new_ml = OPTIMAL_ML; ++ if (ip + new_ml > start2 + m2.len - MINMATCH) ++ new_ml = (int)(start2 - ip) + m2.len - MINMATCH; ++ correction = new_ml - (int)(start2 - ip); ++ if (correction > 0) { ++ start2 += correction; ++ m2.len -= correction; ++ } ++ } ++ ++ if (start2 + m2.len <= mflimit) { ++ start3 = start2 + m2.len - 3; ++ m3 = LZ4HC_InsertAndGetWiderMatch( ++ ctx, start3, start2, matchlimit, m2.len, ++ maxNbAttempts, patternAnalysis, 0, dict, ++ favorCompressionRatio); ++ start3 += m3.back; ++ } else { ++ m3 = nomatch; /* do not search further */ ++ } ++ ++ if (m3.len <= ++ m2.len) { /* No better match => encode ML1 and ML2 */ ++ /* ip & ref are known; Now for ml */ ++ if (start2 < ip + m1.len) ++ m1.len = (int)(start2 - ip); ++ /* Now, encode 2 sequences */ ++ optr = op; ++ if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ++ m1.len, m1.off, limit, oend)) ++ goto _dest_overflow; ++ ip = start2; ++ optr = op; ++ if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ++ m2.len, m2.off, limit, oend)) { ++ m1 = m2; ++ goto _dest_overflow; ++ } ++ continue; ++ } ++ ++ if (start3 < ++ ip + m1.len + ++ 3) { /* Not enough space for match 2 : remove it */ ++ if (start3 >= ++ (ip + ++ m1.len)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ ++ if (start2 < ip + m1.len) { ++ int correction = ++ (int)(ip + m1.len - start2); ++ start2 += correction; ++ m2.len -= correction; ++ if (m2.len < MINMATCH) { ++ start2 = start3; ++ m2 = m3; ++ } ++ } ++ ++ optr = op; ++ if (LZ4HC_encodeSequence( ++ UPDATABLE(ip, op, anchor), m1.len, ++ m1.off, limit, oend)) ++ goto _dest_overflow; ++ ip = start3; ++ m1 = m3; ++ ++ start0 = start2; ++ m0 = m2; ++ goto _Search2; ++ } ++ ++ start2 = start3; ++ m2 = m3; ++ goto _Search3; ++ } ++ ++ /* ++ * OK, now we have 3 ascending matches; ++ * let's write the first one ML1. ++ * ip & ref are known; Now decide ml. ++ */ ++ if (start2 < ip + m1.len) { ++ if ((start2 - ip) < OPTIMAL_ML) { ++ int correction; ++ if (m1.len > OPTIMAL_ML) ++ m1.len = OPTIMAL_ML; ++ if (ip + m1.len > start2 + m2.len - MINMATCH) ++ m1.len = (int)(start2 - ip) + m2.len - ++ MINMATCH; ++ correction = m1.len - (int)(start2 - ip); ++ if (correction > 0) { ++ start2 += correction; ++ m2.len -= correction; ++ } ++ } else { ++ m1.len = (int)(start2 - ip); ++ } ++ } ++ optr = op; ++ if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), m1.len, ++ m1.off, limit, oend)) ++ goto _dest_overflow; ++ ++ /* ML2 becomes ML1 */ ++ ip = start2; ++ m1 = m2; ++ ++ /* ML3 becomes ML2 */ ++ start2 = start3; ++ m2 = m3; ++ ++ /* let's find a new ML3 */ ++ goto _Search3; ++ } ++ ++_last_literals: ++ /* Encode Last Literals */ ++ { ++ size_t lastRunSize = (size_t)(iend - anchor); /* literals */ ++ size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255; ++ size_t const totalSize = 1 + llAdd + lastRunSize; ++ if (limit == fillOutput) ++ oend += LASTLITERALS; /* restore correct value */ ++ if (limit && (op + totalSize > oend)) { ++ if (limit == limitedOutput) ++ return 0; ++ /* adapt lastRunSize to fill 'dest' */ ++ lastRunSize = (size_t)(oend - op) - 1 /*token*/; ++ llAdd = (lastRunSize + 256 - RUN_MASK) / 256; ++ lastRunSize -= llAdd; ++ } ++ DEBUGLOG(6, "Final literal run : %i literals", ++ (int)lastRunSize); ++ ip = anchor + ++ lastRunSize; /* can be != iend if limit==fillOutput */ ++ ++ if (lastRunSize >= RUN_MASK) { ++ size_t accumulator = lastRunSize - RUN_MASK; ++ *op++ = (RUN_MASK << ML_BITS); ++ for (; accumulator >= 255; accumulator -= 255) ++ *op++ = 255; ++ *op++ = (BYTE)accumulator; ++ } else { ++ *op++ = (BYTE)(lastRunSize << ML_BITS); ++ } ++ LZ4_memcpy(op, anchor, lastRunSize); ++ op += lastRunSize; ++ } ++ ++ /* End */ ++ *srcSizePtr = (int)(((const char *)ip) - source); ++ return (int)(((char *)op) - dest); ++ ++_dest_overflow: ++ if (limit == fillOutput) { ++ /* Assumption : @ip, @anchor, @optr and @m1 must be set correctly */ ++ size_t const ll = (size_t)(ip - anchor); ++ size_t const ll_addbytes = (ll + 240) / 255; ++ size_t const ll_totalCost = 1 + ll_addbytes + ll; ++ BYTE *const maxLitPos = ++ oend - 3; /* 2 for offset, 1 for token */ ++ DEBUGLOG(6, "Last sequence overflowing"); ++ op = optr; /* restore correct out pointer */ ++ if (op + ll_totalCost <= maxLitPos) { ++ /* ll validated; now adjust match length */ ++ size_t const bytesLeftForMl = ++ (size_t)(maxLitPos - (op + ll_totalCost)); ++ size_t const maxMlSize = MINMATCH + (ML_MASK - 1) + ++ (bytesLeftForMl * 255); ++ assert(maxMlSize < INT_MAX); ++ assert(m1.len >= 0); ++ if ((size_t)m1.len > maxMlSize) ++ m1.len = (int)maxMlSize; ++ if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - ++ 1 + m1.len >= ++ MFLIMIT) { ++ LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ++ m1.len, m1.off, notLimited, ++ oend); ++ } ++ } ++ goto _last_literals; ++ } ++ /* compression failed */ ++ return 0; ++} ++ ++static int LZ4HC_compress_optimal(LZ4HC_CCtx_internal *ctx, ++ const char *const source, char *dst, ++ int *srcSizePtr, int dstCapacity, ++ int const nbSearches, size_t sufficient_len, ++ const limitedOutput_directive limit, ++ int const fullUpdate, ++ const dictCtx_directive dict, ++ const HCfavor_e favorDecSpeed); ++ ++LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal( ++ LZ4HC_CCtx_internal *const ctx, const char *const src, char *const dst, ++ int *const srcSizePtr, int const dstCapacity, int cLevel, ++ const limitedOutput_directive limit, const dictCtx_directive dict) ++{ ++ DEBUGLOG(5, "LZ4HC_compress_generic_internal(src=%p, srcSize=%d)", src, ++ *srcSizePtr); ++ ++ if (limit == fillOutput && dstCapacity < 1) ++ return 0; /* Impossible to store anything */ ++ if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) ++ return 0; /* Unsupported input size (too large or negative) */ ++ ++ ctx->end += *srcSizePtr; ++ { ++ cParams_t const cParam = LZ4HC_getCLevelParams(cLevel); ++ HCfavor_e const favor = ctx->favorDecSpeed ? ++ favorDecompressionSpeed : ++ favorCompressionRatio; ++ int result; ++ ++ if (cParam.strat == lz4mid) { ++ result = LZ4MID_compress(ctx, src, dst, srcSizePtr, ++ dstCapacity, limit, dict); ++ } else if (cParam.strat == lz4hc) { ++ result = LZ4HC_compress_hashChain( ++ ctx, src, dst, srcSizePtr, dstCapacity, ++ cParam.nbSearches, limit, dict); ++ } else { ++ assert(cParam.strat == lz4opt); ++ result = LZ4HC_compress_optimal( ++ ctx, src, dst, srcSizePtr, dstCapacity, ++ cParam.nbSearches, cParam.targetLength, limit, ++ cLevel >= LZ4HC_CLEVEL_MAX, /* ultra mode */ ++ dict, favor); ++ } ++ if (result <= 0) ++ ctx->dirty = 1; ++ return result; ++ } ++} ++ ++static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal *ctxPtr, ++ const BYTE *newBlock); ++ ++static int LZ4HC_compress_generic_noDictCtx(LZ4HC_CCtx_internal *const ctx, ++ const char *const src, ++ char *const dst, ++ int *const srcSizePtr, ++ int const dstCapacity, int cLevel, ++ limitedOutput_directive limit) ++{ ++ assert(ctx->dictCtx == NULL); ++ return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, ++ dstCapacity, cLevel, limit, ++ noDictCtx); ++} ++ ++static int isStateCompatible(const LZ4HC_CCtx_internal *ctx1, ++ const LZ4HC_CCtx_internal *ctx2) ++{ ++ int const isMid1 = ++ LZ4HC_getCLevelParams(ctx1->compressionLevel).strat == lz4mid; ++ int const isMid2 = ++ LZ4HC_getCLevelParams(ctx2->compressionLevel).strat == lz4mid; ++ return !(isMid1 ^ isMid2); ++} ++ ++static int LZ4HC_compress_generic_dictCtx(LZ4HC_CCtx_internal *const ctx, ++ const char *const src, ++ char *const dst, ++ int *const srcSizePtr, ++ int const dstCapacity, int cLevel, ++ limitedOutput_directive limit) ++{ ++ const size_t position = (size_t)(ctx->end - ctx->prefixStart) + ++ (ctx->dictLimit - ctx->lowLimit); ++ assert(ctx->dictCtx != NULL); ++ if (position >= 64 KB) { ++ ctx->dictCtx = NULL; ++ return LZ4HC_compress_generic_noDictCtx( ++ ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); ++ } else if (position == 0 && *srcSizePtr > 4 KB && ++ isStateCompatible(ctx, ctx->dictCtx)) { ++ LZ4_memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal)); ++ LZ4HC_setExternalDict(ctx, (const BYTE *)src); ++ ctx->compressionLevel = (short)cLevel; ++ return LZ4HC_compress_generic_noDictCtx( ++ ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); ++ } else { ++ return LZ4HC_compress_generic_internal(ctx, src, dst, ++ srcSizePtr, dstCapacity, ++ cLevel, limit, ++ usingDictCtxHc); ++ } ++} ++ ++static int LZ4HC_compress_generic(LZ4HC_CCtx_internal *const ctx, ++ const char *const src, char *const dst, ++ int *const srcSizePtr, int const dstCapacity, ++ int cLevel, limitedOutput_directive limit) ++{ ++ if (ctx->dictCtx == NULL) { ++ return LZ4HC_compress_generic_noDictCtx( ++ ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); ++ } else { ++ return LZ4HC_compress_generic_dictCtx( ++ ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); ++ } ++} ++ ++int LZ4_sizeofStateHC(void) ++{ ++ return (int)sizeof(LZ4_streamHC_t); ++} ++ ++static size_t LZ4_streamHC_t_alignment(void) ++{ ++#if LZ4_ALIGN_TEST ++ typedef struct { ++ char c; ++ LZ4_streamHC_t t; ++ } t_a; ++ return sizeof(t_a) - sizeof(LZ4_streamHC_t); ++#else ++ return 1; /* effectively disabled */ ++#endif ++} ++ ++/* state is presumed correctly initialized, ++ * in which case its size and alignment have already been validate */ ++int LZ4_compress_HC_extStateHC_fastReset(void *state, const char *src, ++ char *dst, int srcSize, ++ int dstCapacity, int compressionLevel) ++{ ++ LZ4HC_CCtx_internal *const ctx = ++ &((LZ4_streamHC_t *)state)->internal_donotuse; ++ if (!LZ4_isAligned(state, LZ4_streamHC_t_alignment())) ++ return 0; ++ LZ4_resetStreamHC_fast((LZ4_streamHC_t *)state, compressionLevel); ++ LZ4HC_init_internal(ctx, (const BYTE *)src); ++ if (dstCapacity < LZ4_compressBound(srcSize)) ++ return LZ4HC_compress_generic(ctx, src, dst, &srcSize, ++ dstCapacity, compressionLevel, ++ limitedOutput); ++ else ++ return LZ4HC_compress_generic(ctx, src, dst, &srcSize, ++ dstCapacity, compressionLevel, ++ notLimited); ++} ++ ++int LZ4_compress_HC_extStateHC(void *state, const char *src, char *dst, ++ int srcSize, int dstCapacity, ++ int compressionLevel) ++{ ++ LZ4_streamHC_t *const ctx = LZ4_initStreamHC(state, sizeof(*ctx)); ++ if (ctx == NULL) ++ return 0; /* init failure */ ++ return LZ4_compress_HC_extStateHC_fastReset( ++ state, src, dst, srcSize, dstCapacity, compressionLevel); ++} ++ ++int LZ4_compress_HC(const char *src, char *dst, int srcSize, int dstCapacity, ++ int compressionLevel, void *wrkmem) ++{ ++ DEBUGLOG(5, "LZ4_compress_HC") ++ return LZ4_compress_HC_extStateHC(wrkmem, src, dst, srcSize, ++ dstCapacity, compressionLevel); ++} ++EXPORT_SYMBOL(LZ4_compress_HC); ++ ++/* state is presumed sized correctly (>= sizeof(LZ4_streamHC_t)) */ ++int LZ4_compress_HC_destSize(void *state, const char *source, char *dest, ++ int *sourceSizePtr, int targetDestSize, int cLevel) ++{ ++ LZ4_streamHC_t *const ctx = LZ4_initStreamHC(state, sizeof(*ctx)); ++ if (ctx == NULL) ++ return 0; /* init failure */ ++ LZ4HC_init_internal(&ctx->internal_donotuse, (const BYTE *)source); ++ LZ4_setCompressionLevel(ctx, cLevel); ++ return LZ4HC_compress_generic(&ctx->internal_donotuse, source, dest, ++ sourceSizePtr, targetDestSize, cLevel, ++ fillOutput); ++} ++ ++/************************************** ++* Streaming Functions ++**************************************/ ++/* allocation */ ++#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) ++LZ4_streamHC_t *LZ4_createStreamHC(void) ++{ ++ LZ4_streamHC_t *const state = ++ (LZ4_streamHC_t *)ALLOC_AND_ZERO(sizeof(LZ4_streamHC_t)); ++ if (state == NULL) ++ return NULL; ++ LZ4_setCompressionLevel(state, LZ4HC_CLEVEL_DEFAULT); ++ return state; ++} ++ ++int LZ4_freeStreamHC(LZ4_streamHC_t *LZ4_streamHCPtr) ++{ ++ DEBUGLOG(4, "LZ4_freeStreamHC(%p)", LZ4_streamHCPtr); ++ if (!LZ4_streamHCPtr) ++ return 0; /* support free on NULL */ ++ FREEMEM(LZ4_streamHCPtr); ++ return 0; ++} ++#endif ++ ++LZ4_streamHC_t *LZ4_initStreamHC(void *buffer, size_t size) ++{ ++ LZ4_streamHC_t *const LZ4_streamHCPtr = (LZ4_streamHC_t *)buffer; ++ DEBUGLOG(4, "LZ4_initStreamHC(%p, %u)", buffer, (unsigned)size); ++ /* check conditions */ ++ if (buffer == NULL) ++ return NULL; ++ if (size < sizeof(LZ4_streamHC_t)) ++ return NULL; ++ if (!LZ4_isAligned(buffer, LZ4_streamHC_t_alignment())) ++ return NULL; ++ /* init */ ++ { ++ LZ4HC_CCtx_internal *const hcstate = ++ &(LZ4_streamHCPtr->internal_donotuse); ++ MEM_INIT(hcstate, 0, sizeof(*hcstate)); ++ } ++ LZ4_setCompressionLevel(LZ4_streamHCPtr, LZ4HC_CLEVEL_DEFAULT); ++ return LZ4_streamHCPtr; ++} ++ ++/* just a stub */ ++void LZ4_resetStreamHC(LZ4_streamHC_t *LZ4_streamHCPtr, int compressionLevel) ++{ ++ LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); ++ LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); ++} ++ ++void LZ4_resetStreamHC_fast(LZ4_streamHC_t *LZ4_streamHCPtr, ++ int compressionLevel) ++{ ++ LZ4HC_CCtx_internal *const s = &LZ4_streamHCPtr->internal_donotuse; ++ DEBUGLOG(5, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, ++ compressionLevel); ++ if (s->dirty) { ++ LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); ++ } else { ++ assert(s->end >= s->prefixStart); ++ s->dictLimit += (U32)(s->end - s->prefixStart); ++ s->prefixStart = NULL; ++ s->end = NULL; ++ s->dictCtx = NULL; ++ } ++ LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); ++} ++ ++void LZ4_setCompressionLevel(LZ4_streamHC_t *LZ4_streamHCPtr, ++ int compressionLevel) ++{ ++ DEBUGLOG(5, "LZ4_setCompressionLevel(%p, %d)", LZ4_streamHCPtr, ++ compressionLevel); ++ if (compressionLevel < 1) ++ compressionLevel = LZ4HC_CLEVEL_DEFAULT; ++ if (compressionLevel > LZ4HC_CLEVEL_MAX) ++ compressionLevel = LZ4HC_CLEVEL_MAX; ++ LZ4_streamHCPtr->internal_donotuse.compressionLevel = ++ (short)compressionLevel; ++} ++ ++void LZ4_favorDecompressionSpeed(LZ4_streamHC_t *LZ4_streamHCPtr, int favor) ++{ ++ LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = (favor != 0); ++} ++ ++/* LZ4_loadDictHC() : ++ * LZ4_streamHCPtr is presumed properly initialized */ ++int LZ4_loadDictHC(LZ4_streamHC_t *LZ4_streamHCPtr, const char *dictionary, ++ int dictSize) ++{ ++ LZ4HC_CCtx_internal *const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; ++ cParams_t cp; ++ DEBUGLOG(4, "LZ4_loadDictHC(ctx:%p, dict:%p, dictSize:%d, clevel=%d)", ++ LZ4_streamHCPtr, dictionary, dictSize, ++ ctxPtr->compressionLevel); ++ assert(dictSize >= 0); ++ assert(LZ4_streamHCPtr != NULL); ++ if (dictSize > 64 KB) { ++ dictionary += (size_t)dictSize - 64 KB; ++ dictSize = 64 KB; ++ } ++ /* need a full initialization, there are bad side-effects when using resetFast() */ ++ { ++ int const cLevel = ctxPtr->compressionLevel; ++ LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); ++ LZ4_setCompressionLevel(LZ4_streamHCPtr, cLevel); ++ cp = LZ4HC_getCLevelParams(cLevel); ++ } ++ LZ4HC_init_internal(ctxPtr, (const BYTE *)dictionary); ++ ctxPtr->end = (const BYTE *)dictionary + dictSize; ++ if (cp.strat == lz4mid) { ++ LZ4MID_fillHTable(ctxPtr, dictionary, (size_t)dictSize); ++ } else { ++ if (dictSize >= LZ4HC_HASHSIZE) ++ LZ4HC_Insert(ctxPtr, ctxPtr->end - 3); ++ } ++ return dictSize; ++} ++EXPORT_SYMBOL(LZ4_loadDictHC); ++ ++void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, ++ const LZ4_streamHC_t *dictionary_stream) ++{ ++ working_stream->internal_donotuse.dictCtx = ++ dictionary_stream != NULL ? ++ &(dictionary_stream->internal_donotuse) : ++ NULL; ++} ++ ++/* compression */ ++ ++static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal *ctxPtr, ++ const BYTE *newBlock) ++{ ++ DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock); ++ if ((ctxPtr->end >= ctxPtr->prefixStart + 4) && ++ (LZ4HC_getCLevelParams(ctxPtr->compressionLevel).strat != lz4mid)) { ++ LZ4HC_Insert( ++ ctxPtr, ++ ctxPtr->end - ++ 3); /* Referencing remaining dictionary content */ ++ } ++ ++ /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ ++ ctxPtr->lowLimit = ctxPtr->dictLimit; ++ ctxPtr->dictStart = ctxPtr->prefixStart; ++ ctxPtr->dictLimit += (U32)(ctxPtr->end - ctxPtr->prefixStart); ++ ctxPtr->prefixStart = newBlock; ++ ctxPtr->end = newBlock; ++ ctxPtr->nextToUpdate = ++ ctxPtr->dictLimit; /* match referencing will resume from there */ ++ ++ /* cannot reference an extDict and a dictCtx at the same time */ ++ ctxPtr->dictCtx = NULL; ++} ++ ++static int LZ4_compressHC_continue_generic(LZ4_streamHC_t *LZ4_streamHCPtr, ++ const char *src, char *dst, ++ int *srcSizePtr, int dstCapacity, ++ limitedOutput_directive limit) ++{ ++ LZ4HC_CCtx_internal *const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; ++ DEBUGLOG( ++ 5, ++ "LZ4_compressHC_continue_generic(ctx=%p, src=%p, srcSize=%d, limit=%d)", ++ LZ4_streamHCPtr, src, *srcSizePtr, limit); ++ assert(ctxPtr != NULL); ++ /* auto-init if forgotten */ ++ if (ctxPtr->prefixStart == NULL) ++ LZ4HC_init_internal(ctxPtr, (const BYTE *)src); ++ ++ /* Check overflow */ ++ if ((size_t)(ctxPtr->end - ctxPtr->prefixStart) + ctxPtr->dictLimit > ++ 2 GB) { ++ size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->prefixStart); ++ if (dictSize > 64 KB) ++ dictSize = 64 KB; ++ LZ4_loadDictHC(LZ4_streamHCPtr, ++ (const char *)(ctxPtr->end) - dictSize, ++ (int)dictSize); ++ } ++ ++ /* Check if blocks follow each other */ ++ if ((const BYTE *)src != ctxPtr->end) ++ LZ4HC_setExternalDict(ctxPtr, (const BYTE *)src); ++ ++ /* Check overlapping input/dictionary space */ ++ { ++ const BYTE *sourceEnd = (const BYTE *)src + *srcSizePtr; ++ const BYTE *const dictBegin = ctxPtr->dictStart; ++ const BYTE *const dictEnd = ++ ctxPtr->dictStart + ++ (ctxPtr->dictLimit - ctxPtr->lowLimit); ++ if ((sourceEnd > dictBegin) && ((const BYTE *)src < dictEnd)) { ++ if (sourceEnd > dictEnd) ++ sourceEnd = dictEnd; ++ ctxPtr->lowLimit += ++ (U32)(sourceEnd - ctxPtr->dictStart); ++ ctxPtr->dictStart += ++ (U32)(sourceEnd - ctxPtr->dictStart); ++ /* invalidate dictionary is it's too small */ ++ if (ctxPtr->dictLimit - ctxPtr->lowLimit < ++ LZ4HC_HASHSIZE) { ++ ctxPtr->lowLimit = ctxPtr->dictLimit; ++ ctxPtr->dictStart = ctxPtr->prefixStart; ++ } ++ } ++ } ++ ++ return LZ4HC_compress_generic(ctxPtr, src, dst, srcSizePtr, dstCapacity, ++ ctxPtr->compressionLevel, limit); ++} ++ ++int LZ4_compress_HC_continue(LZ4_streamHC_t *LZ4_streamHCPtr, const char *src, ++ char *dst, int srcSize, int dstCapacity) ++{ ++ DEBUGLOG(5, "LZ4_compress_HC_continue"); ++ if (dstCapacity < LZ4_compressBound(srcSize)) ++ return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, ++ dst, &srcSize, ++ dstCapacity, ++ limitedOutput); ++ else ++ return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, ++ dst, &srcSize, ++ dstCapacity, notLimited); ++} ++EXPORT_SYMBOL(LZ4_compress_HC_continue); ++ ++int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t *LZ4_streamHCPtr, ++ const char *src, char *dst, ++ int *srcSizePtr, int targetDestSize) ++{ ++ return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, ++ srcSizePtr, targetDestSize, ++ fillOutput); ++} ++ ++/* LZ4_saveDictHC : ++ * save history content ++ * into a user-provided buffer ++ * which is then used to continue compression ++ */ ++int LZ4_saveDictHC(LZ4_streamHC_t *LZ4_streamHCPtr, char *safeBuffer, ++ int dictSize) ++{ ++ LZ4HC_CCtx_internal *const streamPtr = ++ &LZ4_streamHCPtr->internal_donotuse; ++ int const prefixSize = (int)(streamPtr->end - streamPtr->prefixStart); ++ DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, ++ dictSize); ++ assert(prefixSize >= 0); ++ if (dictSize > 64 KB) ++ dictSize = 64 KB; ++ if (dictSize < 4) ++ dictSize = 0; ++ if (dictSize > prefixSize) ++ dictSize = prefixSize; ++ if (safeBuffer == NULL) ++ assert(dictSize == 0); ++ if (dictSize > 0) ++ LZ4_memmove(safeBuffer, streamPtr->end - dictSize, ++ (size_t)dictSize); ++ { ++ U32 const endIndex = ++ (U32)(streamPtr->end - streamPtr->prefixStart) + ++ streamPtr->dictLimit; ++ streamPtr->end = (safeBuffer == NULL) ? ++ NULL : ++ (const BYTE *)safeBuffer + dictSize; ++ streamPtr->prefixStart = (const BYTE *)safeBuffer; ++ streamPtr->dictLimit = endIndex - (U32)dictSize; ++ streamPtr->lowLimit = endIndex - (U32)dictSize; ++ streamPtr->dictStart = streamPtr->prefixStart; ++ if (streamPtr->nextToUpdate < streamPtr->dictLimit) ++ streamPtr->nextToUpdate = streamPtr->dictLimit; ++ } ++ return dictSize; ++} ++EXPORT_SYMBOL(LZ4_saveDictHC); ++ ++/* ================================================ ++ * LZ4 Optimal parser (levels [LZ4HC_CLEVEL_OPT_MIN - LZ4HC_CLEVEL_MAX]) ++ * ===============================================*/ ++typedef struct { ++ int price; ++ int off; ++ int mlen; ++ int litlen; ++} LZ4HC_optimal_t; ++ ++/* price in bytes */ ++LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen) ++{ ++ int price = litlen; ++ assert(litlen >= 0); ++ if (litlen >= (int)RUN_MASK) ++ price += 1 + ((litlen - (int)RUN_MASK) / 255); ++ return price; ++} ++ ++/* requires mlen >= MINMATCH */ ++LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen) ++{ ++ int price = 1 + 2; /* token + 16-bit offset */ ++ assert(litlen >= 0); ++ assert(mlen >= MINMATCH); ++ ++ price += LZ4HC_literalsPrice(litlen); ++ ++ if (mlen >= (int)(ML_MASK + MINMATCH)) ++ price += 1 + ((mlen - (int)(ML_MASK + MINMATCH)) / 255); ++ ++ return price; ++} ++ ++LZ4_FORCE_INLINE LZ4HC_match_t LZ4HC_FindLongerMatch( ++ LZ4HC_CCtx_internal *const ctx, const BYTE *ip, ++ const BYTE *const iHighLimit, int minLen, int nbSearches, ++ const dictCtx_directive dict, const HCfavor_e favorDecSpeed) ++{ ++ LZ4HC_match_t const match0 = { 0, 0, 0 }; ++ /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), ++ * but this won't be the case here, as we define iLowLimit==ip, ++ ** so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ ++ LZ4HC_match_t md = LZ4HC_InsertAndGetWiderMatch( ++ ctx, ip, ip, iHighLimit, minLen, nbSearches, ++ 1 /*patternAnalysis*/, 1 /*chainSwap*/, dict, favorDecSpeed); ++ assert(md.back == 0); ++ if (md.len <= minLen) ++ return match0; ++ if (favorDecSpeed) { ++ if ((md.len > 18) & (md.len <= 36)) ++ md.len = 18; /* favor dec.speed (shortcut) */ ++ } ++ return md; ++} ++ ++static int LZ4HC_compress_optimal(LZ4HC_CCtx_internal *ctx, ++ const char *const source, char *dst, ++ int *srcSizePtr, int dstCapacity, ++ int const nbSearches, size_t sufficient_len, ++ const limitedOutput_directive limit, ++ int const fullUpdate, ++ const dictCtx_directive dict, ++ const HCfavor_e favorDecSpeed) ++{ ++ int retval = 0; ++#define TRAILING_LITERALS 3 ++#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE == 1 ++ LZ4HC_optimal_t *const opt = (LZ4HC_optimal_t *)ALLOC( ++ sizeof(LZ4HC_optimal_t) * (LZ4_OPT_NUM + TRAILING_LITERALS)); ++#else ++ LZ4HC_optimal_t ++ opt[LZ4_OPT_NUM + ++ TRAILING_LITERALS]; /* ~64 KB, which is a bit large for stack... */ ++#endif ++ ++ const BYTE *ip = (const BYTE *)source; ++ const BYTE *anchor = ip; ++ const BYTE *const iend = ip + *srcSizePtr; ++ const BYTE *const mflimit = iend - MFLIMIT; ++ const BYTE *const matchlimit = iend - LASTLITERALS; ++ BYTE *op = (BYTE *)dst; ++ BYTE *opSaved = (BYTE *)dst; ++ BYTE *oend = op + dstCapacity; ++ int ovml = MINMATCH; /* overflow - last sequence */ ++ int ovoff = 0; ++ ++ /* init */ ++#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE == 1 ++ if (opt == NULL) ++ goto _return_label; ++#endif ++ DEBUGLOG(5, "LZ4HC_compress_optimal(dst=%p, dstCapa=%u)", dst, ++ (unsigned)dstCapacity); ++ *srcSizePtr = 0; ++ if (limit == fillOutput) ++ oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ ++ if (sufficient_len >= LZ4_OPT_NUM) ++ sufficient_len = LZ4_OPT_NUM - 1; ++ ++ /* Main Loop */ ++ while (ip <= mflimit) { ++ int const llen = (int)(ip - anchor); ++ int best_mlen, best_off; ++ int cur, last_match_pos = 0; ++ ++ LZ4HC_match_t const firstMatch = ++ LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH - 1, ++ nbSearches, dict, favorDecSpeed); ++ if (firstMatch.len == 0) { ++ ip++; ++ continue; ++ } ++ ++ if ((size_t)firstMatch.len > sufficient_len) { ++ /* good enough solution : immediate encoding */ ++ int const firstML = firstMatch.len; ++ opSaved = op; ++ if (LZ4HC_encodeSequence( ++ UPDATABLE(ip, op, anchor), firstML, ++ firstMatch.off, limit, ++ oend)) { /* updates ip, op and anchor */ ++ ovml = firstML; ++ ovoff = firstMatch.off; ++ goto _dest_overflow; ++ } ++ continue; ++ } ++ ++ /* set prices for first positions (literals) */ ++ { ++ int rPos; ++ for (rPos = 0; rPos < MINMATCH; rPos++) { ++ int const cost = ++ LZ4HC_literalsPrice(llen + rPos); ++ opt[rPos].mlen = 1; ++ opt[rPos].off = 0; ++ opt[rPos].litlen = llen + rPos; ++ opt[rPos].price = cost; ++ DEBUGLOG( ++ 7, ++ "rPos:%3i => price:%3i (litlen=%i) -- initial setup", ++ rPos, cost, opt[rPos].litlen); ++ } ++ } ++ /* set prices using initial match */ ++ { ++ int const matchML = ++ firstMatch ++ .len; /* necessarily < sufficient_len < LZ4_OPT_NUM */ ++ int const offset = firstMatch.off; ++ int mlen; ++ assert(matchML < LZ4_OPT_NUM); ++ for (mlen = MINMATCH; mlen <= matchML; mlen++) { ++ int const cost = ++ LZ4HC_sequencePrice(llen, mlen); ++ opt[mlen].mlen = mlen; ++ opt[mlen].off = offset; ++ opt[mlen].litlen = llen; ++ opt[mlen].price = cost; ++ DEBUGLOG( ++ 7, ++ "rPos:%3i => price:%3i (matchlen=%i) -- initial setup", ++ mlen, cost, mlen); ++ } ++ } ++ last_match_pos = firstMatch.len; ++ { ++ int addLit; ++ for (addLit = 1; addLit <= TRAILING_LITERALS; ++ addLit++) { ++ opt[last_match_pos + addLit].mlen = ++ 1; /* literal */ ++ opt[last_match_pos + addLit].off = 0; ++ opt[last_match_pos + addLit].litlen = addLit; ++ opt[last_match_pos + addLit].price = ++ opt[last_match_pos].price + ++ LZ4HC_literalsPrice(addLit); ++ DEBUGLOG( ++ 7, ++ "rPos:%3i => price:%3i (litlen=%i) -- initial setup", ++ last_match_pos + addLit, ++ opt[last_match_pos + addLit].price, ++ addLit); ++ } ++ } ++ ++ /* check further positions */ ++ for (cur = 1; cur < last_match_pos; cur++) { ++ const BYTE *const curPtr = ip + cur; ++ LZ4HC_match_t newMatch; ++ ++ if (curPtr > mflimit) ++ break; ++ DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u", cur, ++ opt[cur].price, opt[cur + 1].price, cur + 1); ++ if (fullUpdate) { ++ /* not useful to search here if next position has same (or lower) cost */ ++ if ((opt[cur + 1].price <= opt[cur].price) ++ /* in some cases, next position has same cost, but cost rises sharply after, so a small match would still be beneficial */ ++ && (opt[cur + MINMATCH].price < ++ opt[cur].price + 3 /*min seq price*/)) ++ continue; ++ } else { ++ /* not useful to search here if next position has same (or lower) cost */ ++ if (opt[cur + 1].price <= opt[cur].price) ++ continue; ++ } ++ ++ DEBUGLOG(7, "search at rPos:%u", cur); ++ if (fullUpdate) ++ newMatch = LZ4HC_FindLongerMatch( ++ ctx, curPtr, matchlimit, MINMATCH - 1, ++ nbSearches, dict, favorDecSpeed); ++ else ++ /* only test matches of minimum length; slightly faster, but misses a few bytes */ ++ newMatch = LZ4HC_FindLongerMatch( ++ ctx, curPtr, matchlimit, ++ last_match_pos - cur, nbSearches, dict, ++ favorDecSpeed); ++ if (!newMatch.len) ++ continue; ++ ++ if (((size_t)newMatch.len > sufficient_len) || ++ (newMatch.len + cur >= LZ4_OPT_NUM)) { ++ /* immediate encoding */ ++ best_mlen = newMatch.len; ++ best_off = newMatch.off; ++ last_match_pos = cur + 1; ++ goto encode; ++ } ++ ++ /* before match : set price with literals at beginning */ ++ { ++ int const baseLitlen = opt[cur].litlen; ++ int litlen; ++ for (litlen = 1; litlen < MINMATCH; litlen++) { ++ int const price = ++ opt[cur].price - ++ LZ4HC_literalsPrice( ++ baseLitlen) + ++ LZ4HC_literalsPrice(baseLitlen + ++ litlen); ++ int const pos = cur + litlen; ++ if (price < opt[pos].price) { ++ opt[pos].mlen = 1; /* literal */ ++ opt[pos].off = 0; ++ opt[pos].litlen = ++ baseLitlen + litlen; ++ opt[pos].price = price; ++ DEBUGLOG( ++ 7, ++ "rPos:%3i => price:%3i (litlen=%i)", ++ pos, price, ++ opt[pos].litlen); ++ } ++ } ++ } ++ ++ /* set prices using match at position = cur */ ++ { ++ int const matchML = newMatch.len; ++ int ml = MINMATCH; ++ ++ assert(cur + newMatch.len < LZ4_OPT_NUM); ++ for (; ml <= matchML; ml++) { ++ int const pos = cur + ml; ++ int const offset = newMatch.off; ++ int price; ++ int ll; ++ DEBUGLOG( ++ 7, ++ "testing price rPos %i (last_match_pos=%i)", ++ pos, last_match_pos); ++ if (opt[cur].mlen == 1) { ++ ll = opt[cur].litlen; ++ price = ((cur > ll) ? ++ opt[cur - ll] ++ .price : ++ 0) + ++ LZ4HC_sequencePrice(ll, ++ ml); ++ } else { ++ ll = 0; ++ price = opt[cur].price + ++ LZ4HC_sequencePrice(0, ++ ml); ++ } ++ ++ assert((U32)favorDecSpeed <= 1); ++ if (pos > last_match_pos + ++ TRAILING_LITERALS || ++ price <= ++ opt[pos].price - ++ (int)favorDecSpeed) { ++ DEBUGLOG( ++ 7, ++ "rPos:%3i => price:%3i (matchlen=%i)", ++ pos, price, ml); ++ assert(pos < LZ4_OPT_NUM); ++ if ((ml == ++ matchML) /* last pos of last match */ ++ && (last_match_pos < pos)) ++ last_match_pos = pos; ++ opt[pos].mlen = ml; ++ opt[pos].off = offset; ++ opt[pos].litlen = ll; ++ opt[pos].price = price; ++ } ++ } ++ } ++ /* complete following positions with literals */ ++ { ++ int addLit; ++ for (addLit = 1; addLit <= TRAILING_LITERALS; ++ addLit++) { ++ opt[last_match_pos + addLit].mlen = ++ 1; /* literal */ ++ opt[last_match_pos + addLit].off = 0; ++ opt[last_match_pos + addLit].litlen = ++ addLit; ++ opt[last_match_pos + addLit].price = ++ opt[last_match_pos].price + ++ LZ4HC_literalsPrice(addLit); ++ DEBUGLOG( ++ 7, ++ "rPos:%3i => price:%3i (litlen=%i)", ++ last_match_pos + addLit, ++ opt[last_match_pos + addLit] ++ .price, ++ addLit); ++ } ++ } ++ } /* for (cur = 1; cur <= last_match_pos; cur++) */ ++ ++ assert(last_match_pos < LZ4_OPT_NUM + TRAILING_LITERALS); ++ best_mlen = opt[last_match_pos].mlen; ++ best_off = opt[last_match_pos].off; ++ cur = last_match_pos - best_mlen; ++ ++ encode: /* cur, last_match_pos, best_mlen, best_off must be set */ ++ assert(cur < LZ4_OPT_NUM); ++ assert(last_match_pos >= 1); /* == 1 when only one candidate */ ++ DEBUGLOG( ++ 6, ++ "reverse traversal, looking for shortest path (last_match_pos=%i)", ++ last_match_pos); ++ { ++ int candidate_pos = cur; ++ int selected_matchLength = best_mlen; ++ int selected_offset = best_off; ++ while (1) { /* from end to beginning */ ++ int const next_matchLength = ++ opt[candidate_pos] ++ .mlen; /* can be 1, means literal */ ++ int const next_offset = opt[candidate_pos].off; ++ DEBUGLOG(7, "pos %i: sequence length %i", ++ candidate_pos, selected_matchLength); ++ opt[candidate_pos].mlen = selected_matchLength; ++ opt[candidate_pos].off = selected_offset; ++ selected_matchLength = next_matchLength; ++ selected_offset = next_offset; ++ if (next_matchLength > candidate_pos) ++ break; /* last match elected, first match to encode */ ++ assert(next_matchLength > ++ 0); /* can be 1, means literal */ ++ candidate_pos -= next_matchLength; ++ } ++ } ++ ++ /* encode all recorded sequences in order */ ++ { ++ int rPos = 0; /* relative position (to ip) */ ++ while (rPos < last_match_pos) { ++ int const ml = opt[rPos].mlen; ++ int const offset = opt[rPos].off; ++ if (ml == 1) { ++ ip++; ++ rPos++; ++ continue; ++ } /* literal; note: can end up with several literals, in which case, skip them */ ++ rPos += ml; ++ assert(ml >= MINMATCH); ++ assert((offset >= 1) && ++ (offset <= LZ4_DISTANCE_MAX)); ++ opSaved = op; ++ if (LZ4HC_encodeSequence( ++ UPDATABLE(ip, op, anchor), ml, ++ offset, limit, ++ oend)) { /* updates ip, op and anchor */ ++ ovml = ml; ++ ovoff = offset; ++ goto _dest_overflow; ++ } ++ } ++ } ++ } /* while (ip <= mflimit) */ ++ ++_last_literals: ++ /* Encode Last Literals */ ++ { ++ size_t lastRunSize = (size_t)(iend - anchor); /* literals */ ++ size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255; ++ size_t const totalSize = 1 + llAdd + lastRunSize; ++ if (limit == fillOutput) ++ oend += LASTLITERALS; /* restore correct value */ ++ if (limit && (op + totalSize > oend)) { ++ if (limit == limitedOutput) { /* Check output limit */ ++ retval = 0; ++ goto _return_label; ++ } ++ /* adapt lastRunSize to fill 'dst' */ ++ lastRunSize = (size_t)(oend - op) - 1 /*token*/; ++ llAdd = (lastRunSize + 256 - RUN_MASK) / 256; ++ lastRunSize -= llAdd; ++ } ++ DEBUGLOG(6, "Final literal run : %i literals", ++ (int)lastRunSize); ++ ip = anchor + ++ lastRunSize; /* can be != iend if limit==fillOutput */ ++ ++ if (lastRunSize >= RUN_MASK) { ++ size_t accumulator = lastRunSize - RUN_MASK; ++ *op++ = (RUN_MASK << ML_BITS); ++ for (; accumulator >= 255; accumulator -= 255) ++ *op++ = 255; ++ *op++ = (BYTE)accumulator; ++ } else { ++ *op++ = (BYTE)(lastRunSize << ML_BITS); ++ } ++ LZ4_memcpy(op, anchor, lastRunSize); ++ op += lastRunSize; ++ } ++ ++ /* End */ ++ *srcSizePtr = (int)(((const char *)ip) - source); ++ retval = (int)((char *)op - dst); ++ goto _return_label; ++ ++_dest_overflow: ++ if (limit == fillOutput) { ++ /* Assumption : ip, anchor, ovml and ovref must be set correctly */ ++ size_t const ll = (size_t)(ip - anchor); ++ size_t const ll_addbytes = (ll + 240) / 255; ++ size_t const ll_totalCost = 1 + ll_addbytes + ll; ++ BYTE *const maxLitPos = ++ oend - 3; /* 2 for offset, 1 for token */ ++ DEBUGLOG(6, ++ "Last sequence overflowing (only %i bytes remaining)", ++ (int)(oend - 1 - opSaved)); ++ op = opSaved; /* restore correct out pointer */ ++ if (op + ll_totalCost <= maxLitPos) { ++ /* ll validated; now adjust match length */ ++ size_t const bytesLeftForMl = ++ (size_t)(maxLitPos - (op + ll_totalCost)); ++ size_t const maxMlSize = MINMATCH + (ML_MASK - 1) + ++ (bytesLeftForMl * 255); ++ assert(maxMlSize < INT_MAX); ++ assert(ovml >= 0); ++ if ((size_t)ovml > maxMlSize) ++ ovml = (int)maxMlSize; ++ if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - ++ 1 + ovml >= ++ MFLIMIT) { ++ DEBUGLOG(6, "Space to end : %i + ml (%i)", ++ (int)((oend + LASTLITERALS) - ++ (op + ll_totalCost + 2) - 1), ++ ovml); ++ DEBUGLOG(6, "Before : ip = %p, anchor = %p", ip, ++ anchor); ++ LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ++ ovml, ovoff, notLimited, ++ oend); ++ DEBUGLOG(6, "After : ip = %p, anchor = %p", ip, ++ anchor); ++ } ++ } ++ goto _last_literals; ++ } ++_return_label: ++#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE == 1 ++ if (opt) ++ FREEMEM(opt); ++#endif ++ return retval; ++} ++ ++/* state is presumed correctly sized, aka >= sizeof(LZ4_streamHC_t) ++ * @return : 0 on success, !=0 if error */ ++int LZ4_resetStreamStateHC(void *state, char *inputBuffer) ++{ ++ LZ4_streamHC_t *const hc4 = LZ4_initStreamHC(state, sizeof(*hc4)); ++ if (hc4 == NULL) ++ return 1; /* init failed */ ++ LZ4HC_init_internal(&hc4->internal_donotuse, (const BYTE *)inputBuffer); ++ return 0; ++} ++ ++#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) ++void *LZ4_createHC(const char *inputBuffer) ++{ ++ LZ4_streamHC_t *const hc4 = LZ4_createStreamHC(); ++ if (hc4 == NULL) ++ return NULL; /* not enough memory */ ++ LZ4HC_init_internal(&hc4->internal_donotuse, (const BYTE *)inputBuffer); ++ return hc4; ++} ++ ++int LZ4_freeHC(void *LZ4HC_Data) ++{ ++ if (!LZ4HC_Data) ++ return 0; /* support free on NULL */ ++ FREEMEM(LZ4HC_Data); ++ return 0; ++} ++#endif ++ ++int LZ4_compressHC2_continue(void *LZ4HC_Data, const char *src, char *dst, ++ int srcSize, int cLevel) ++{ ++ return LZ4HC_compress_generic( ++ &((LZ4_streamHC_t *)LZ4HC_Data)->internal_donotuse, src, dst, ++ &srcSize, 0, cLevel, notLimited); ++} ++ ++int LZ4_compressHC2_limitedOutput_continue(void *LZ4HC_Data, const char *src, ++ char *dst, int srcSize, ++ int dstCapacity, int cLevel) ++{ ++ return LZ4HC_compress_generic( ++ &((LZ4_streamHC_t *)LZ4HC_Data)->internal_donotuse, src, dst, ++ &srcSize, dstCapacity, cLevel, limitedOutput); ++} ++ ++char *LZ4_slideInputBufferHC(void *LZ4HC_Data) ++{ ++ LZ4HC_CCtx_internal *const s = ++ &((LZ4_streamHC_t *)LZ4HC_Data)->internal_donotuse; ++ const BYTE *const bufferStart = ++ s->prefixStart - s->dictLimit + s->lowLimit; ++ LZ4_resetStreamHC_fast((LZ4_streamHC_t *)LZ4HC_Data, ++ s->compressionLevel); ++ /* ugly conversion trick, required to evade (const char*) -> (char*) cast-qual warning :( */ ++ return (char *)(uptrval)bufferStart; ++} +diff --git a/lib/lz4/lz4hc.h b/lib/lz4/lz4hc.h +new file mode 100644 +index 000000000000..92d2aef17b6f +--- /dev/null ++++ b/lib/lz4/lz4hc.h +@@ -0,0 +1,451 @@ ++/* ++ LZ4 HC - High Compression Mode of LZ4 ++ Header File ++ Copyright (C) 2011-2020, Yann Collet. ++ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) ++ ++ Redistribution and use in source and binary forms, with or without ++ modification, are permitted provided that the following conditions are ++ met: ++ ++ * Redistributions of source code must retain the above copyright ++ notice, this list of conditions and the following disclaimer. ++ * Redistributions in binary form must reproduce the above ++ copyright notice, this list of conditions and the following disclaimer ++ in the documentation and/or other materials provided with the ++ distribution. ++ ++ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ++ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ++ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ++ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ++ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ++ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ++ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ++ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ ++ You can contact the author at : ++ - LZ4 source repository : https://github.com/lz4/lz4 ++ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c ++*/ ++#ifndef LZ4_HC_H_19834876238432 ++#define LZ4_HC_H_19834876238432 ++ ++#if defined(__cplusplus) ++extern "C" { ++#endif ++ ++/* --- Dependency --- */ ++/* note : lz4hc requires lz4.h/lz4.c for compilation */ ++#include "lz4.h" /* stddef, LZ4LIB_API, LZ4_DEPRECATED */ ++ ++/* --- Useful constants --- */ ++#define LZ4HC_CLEVEL_MIN 2 ++#define LZ4HC_CLEVEL_DEFAULT 9 ++#define LZ4HC_CLEVEL_OPT_MIN 10 ++#define LZ4HC_CLEVEL_MAX 12 ++ ++/*-************************************ ++ * Block Compression ++ **************************************/ ++/*! LZ4_compress_HC() : ++ * Compress data from `src` into `dst`, using the powerful but slower "HC" algorithm. ++ * `dst` must be already allocated. ++ * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") ++ * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") ++ * `compressionLevel` : any value between 1 and LZ4HC_CLEVEL_MAX will work. ++ * Values > LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX. ++ * @return : the number of bytes written into 'dst' ++ * or 0 if compression fails. ++ */ ++LZ4LIB_API int LZ4_compress_HC(const char *src, char *dst, int srcSize, ++ int dstCapacity, int compressionLevel, ++ void *wrkmem); ++ ++/* Note : ++ * Decompression functions are provided within "lz4.h" (BSD license) ++ */ ++ ++/*! LZ4_compress_HC_extStateHC() : ++ * Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`. ++ * `state` size is provided by LZ4_sizeofStateHC(). ++ * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() should do properly). ++ */ ++LZ4LIB_API int LZ4_sizeofStateHC(void); ++LZ4LIB_API int LZ4_compress_HC_extStateHC(void *stateHC, const char *src, ++ char *dst, int srcSize, ++ int maxDstSize, int compressionLevel); ++ ++/*! LZ4_compress_HC_destSize() : v1.9.0+ ++ * Will compress as much data as possible from `src` ++ * to fit into `targetDstSize` budget. ++ * Result is provided in 2 parts : ++ * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize) ++ * or 0 if compression fails. ++ * `srcSizePtr` : on success, *srcSizePtr is updated to indicate how much bytes were read from `src` ++ */ ++LZ4LIB_API int LZ4_compress_HC_destSize(void *stateHC, const char *src, ++ char *dst, int *srcSizePtr, ++ int targetDstSize, ++ int compressionLevel); ++ ++/*-************************************ ++ * Streaming Compression ++ * Bufferless synchronous API ++ **************************************/ ++typedef union LZ4_streamHC_u LZ4_streamHC_t; /* incomplete type (defined later) */ ++ ++/*! LZ4_createStreamHC() and LZ4_freeStreamHC() : ++ * These functions create and release memory for LZ4 HC streaming state. ++ * Newly created states are automatically initialized. ++ * A same state can be used multiple times consecutively, ++ * starting with LZ4_resetStreamHC_fast() to start a new stream of blocks. ++ */ ++LZ4LIB_API LZ4_streamHC_t *LZ4_createStreamHC(void); ++LZ4LIB_API int LZ4_freeStreamHC(LZ4_streamHC_t *streamHCPtr); ++ ++/* ++ These functions compress data in successive blocks of any size, ++ using previous blocks as dictionary, to improve compression ratio. ++ One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks. ++ There is an exception for ring buffers, which can be smaller than 64 KB. ++ Ring-buffer scenario is automatically detected and handled within LZ4_compress_HC_continue(). ++ ++ Before starting compression, state must be allocated and properly initialized. ++ LZ4_createStreamHC() does both, though compression level is set to LZ4HC_CLEVEL_DEFAULT. ++ ++ Selecting the compression level can be done with LZ4_resetStreamHC_fast() (starts a new stream) ++ or LZ4_setCompressionLevel() (anytime, between blocks in the same stream) (experimental). ++ LZ4_resetStreamHC_fast() only works on states which have been properly initialized at least once, ++ which is automatically the case when state is created using LZ4_createStreamHC(). ++ ++ After reset, a first "fictional block" can be designated as initial dictionary, ++ using LZ4_loadDictHC() (Optional). ++ Note: In order for LZ4_loadDictHC() to create the correct data structure, ++ it is essential to set the compression level _before_ loading the dictionary. ++ ++ Invoke LZ4_compress_HC_continue() to compress each successive block. ++ The number of blocks is unlimited. ++ Previous input blocks, including initial dictionary when present, ++ must remain accessible and unmodified during compression. ++ ++ It's allowed to update compression level anytime between blocks, ++ using LZ4_setCompressionLevel() (experimental). ++ ++ @dst buffer should be sized to handle worst case scenarios ++ (see LZ4_compressBound(), it ensures compression success). ++ In case of failure, the API does not guarantee recovery, ++ so the state _must_ be reset. ++ To ensure compression success ++ whenever @dst buffer size cannot be made >= LZ4_compressBound(), ++ consider using LZ4_compress_HC_continue_destSize(). ++ ++ Whenever previous input blocks can't be preserved unmodified in-place during compression of next blocks, ++ it's possible to copy the last blocks into a more stable memory space, using LZ4_saveDictHC(). ++ Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer' (<= 64 KB) ++ ++ After completing a streaming compression, ++ it's possible to start a new stream of blocks, using the same LZ4_streamHC_t state, ++ just by resetting it, using LZ4_resetStreamHC_fast(). ++*/ ++ ++LZ4LIB_API void LZ4_resetStreamHC_fast(LZ4_streamHC_t *streamHCPtr, ++ int compressionLevel); /* v1.9.0+ */ ++LZ4LIB_API int LZ4_loadDictHC(LZ4_streamHC_t *streamHCPtr, ++ const char *dictionary, int dictSize); ++ ++LZ4LIB_API int LZ4_compress_HC_continue(LZ4_streamHC_t *streamHCPtr, ++ const char *src, char *dst, int srcSize, ++ int maxDstSize); ++ ++/*! LZ4_compress_HC_continue_destSize() : v1.9.0+ ++ * Similar to LZ4_compress_HC_continue(), ++ * but will read as much data as possible from `src` ++ * to fit into `targetDstSize` budget. ++ * Result is provided into 2 parts : ++ * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize) ++ * or 0 if compression fails. ++ * `srcSizePtr` : on success, *srcSizePtr will be updated to indicate how much bytes were read from `src`. ++ * Note that this function may not consume the entire input. ++ */ ++LZ4LIB_API int ++LZ4_compress_HC_continue_destSize(LZ4_streamHC_t *LZ4_streamHCPtr, ++ const char *src, char *dst, int *srcSizePtr, ++ int targetDstSize); ++ ++LZ4LIB_API int LZ4_saveDictHC(LZ4_streamHC_t *streamHCPtr, char *safeBuffer, ++ int maxDictSize); ++ ++/*! LZ4_attach_HC_dictionary() : stable since v1.10.0 ++ * This API allows for the efficient re-use of a static dictionary many times. ++ * ++ * Rather than re-loading the dictionary buffer into a working context before ++ * each compression, or copying a pre-loaded dictionary's LZ4_streamHC_t into a ++ * working LZ4_streamHC_t, this function introduces a no-copy setup mechanism, ++ * in which the working stream references the dictionary stream in-place. ++ * ++ * Several assumptions are made about the state of the dictionary stream. ++ * Currently, only streams which have been prepared by LZ4_loadDictHC() should ++ * be expected to work. ++ * ++ * Alternatively, the provided dictionary stream pointer may be NULL, in which ++ * case any existing dictionary stream is unset. ++ * ++ * A dictionary should only be attached to a stream without any history (i.e., ++ * a stream that has just been reset). ++ * ++ * The dictionary will remain attached to the working stream only for the ++ * current stream session. Calls to LZ4_resetStreamHC(_fast) will remove the ++ * dictionary context association from the working stream. The dictionary ++ * stream (and source buffer) must remain in-place / accessible / unchanged ++ * through the lifetime of the stream session. ++ */ ++LZ4LIB_API void ++LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, ++ const LZ4_streamHC_t *dictionary_stream); ++ ++/*^********************************************** ++ * !!!!!! STATIC LINKING ONLY !!!!!! ++ ***********************************************/ ++ ++/*-****************************************************************** ++ * PRIVATE DEFINITIONS : ++ * Do not use these definitions directly. ++ * They are merely exposed to allow static allocation of `LZ4_streamHC_t`. ++ * Declare an `LZ4_streamHC_t` directly, rather than any type below. ++ * Even then, only do so in the context of static linking, as definitions may change between versions. ++ ********************************************************************/ ++ ++#define LZ4HC_DICTIONARY_LOGSIZE 16 ++#define LZ4HC_MAXD (1 << LZ4HC_DICTIONARY_LOGSIZE) ++#define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1) ++ ++#define LZ4HC_HASH_LOG 15 ++#define LZ4HC_HASHTABLESIZE (1 << LZ4HC_HASH_LOG) ++#define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1) ++ ++/* Never ever use these definitions directly ! ++ * Declare or allocate an LZ4_streamHC_t instead. ++**/ ++typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal; ++struct LZ4HC_CCtx_internal { ++ LZ4_u32 hashTable[LZ4HC_HASHTABLESIZE]; ++ LZ4_u16 chainTable[LZ4HC_MAXD]; ++ const LZ4_byte *end; /* next block here to continue on current prefix */ ++ const LZ4_byte *prefixStart; /* Indexes relative to this position */ ++ const LZ4_byte *dictStart; /* alternate reference for extDict */ ++ LZ4_u32 dictLimit; /* below that point, need extDict */ ++ LZ4_u32 lowLimit; /* below that point, no more history */ ++ LZ4_u32 nextToUpdate; /* index from which to continue dictionary update */ ++ short compressionLevel; ++ LZ4_i8 favorDecSpeed; /* favor decompression speed if this flag set, ++ otherwise, favor compression ratio */ ++ LZ4_i8 dirty; /* stream has to be fully reset if this flag is set */ ++ const LZ4HC_CCtx_internal *dictCtx; ++}; ++ ++#define LZ4_STREAMHC_MINSIZE \ ++ 262200 /* static size, for inter-version compatibility */ ++union LZ4_streamHC_u { ++ char minStateSize[LZ4_STREAMHC_MINSIZE]; ++ LZ4HC_CCtx_internal internal_donotuse; ++}; /* previously typedef'd to LZ4_streamHC_t */ ++ ++/* LZ4_streamHC_t : ++ * This structure allows static allocation of LZ4 HC streaming state. ++ * This can be used to allocate statically on stack, or as part of a larger structure. ++ * ++ * Such state **must** be initialized using LZ4_initStreamHC() before first use. ++ * ++ * Note that invoking LZ4_initStreamHC() is not required when ++ * the state was created using LZ4_createStreamHC() (which is recommended). ++ * Using the normal builder, a newly created state is automatically initialized. ++ * ++ * Static allocation shall only be used in combination with static linking. ++ */ ++ ++/* LZ4_initStreamHC() : v1.9.0+ ++ * Required before first use of a statically allocated LZ4_streamHC_t. ++ * Before v1.9.0 : use LZ4_resetStreamHC() instead ++ */ ++LZ4LIB_API LZ4_streamHC_t *LZ4_initStreamHC(void *buffer, size_t size); ++ ++/*-************************************ ++* Deprecated Functions ++**************************************/ ++/* see lz4.h LZ4_DISABLE_DEPRECATE_WARNINGS to turn off deprecation warnings */ ++ ++/* deprecated compression functions */ ++LZ4_DEPRECATED("use LZ4_compress_HC() instead") ++LZ4LIB_API int LZ4_compressHC(const char *source, char *dest, int inputSize); ++LZ4_DEPRECATED("use LZ4_compress_HC() instead") ++LZ4LIB_API int LZ4_compressHC_limitedOutput(const char *source, char *dest, ++ int inputSize, int maxOutputSize); ++LZ4_DEPRECATED("use LZ4_compress_HC() instead") ++LZ4LIB_API int LZ4_compressHC2(const char *source, char *dest, int inputSize, ++ int compressionLevel); ++LZ4_DEPRECATED("use LZ4_compress_HC() instead") ++LZ4LIB_API int LZ4_compressHC2_limitedOutput(const char *source, char *dest, ++ int inputSize, int maxOutputSize, ++ int compressionLevel); ++LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") ++LZ4LIB_API int LZ4_compressHC_withStateHC(void *state, const char *source, ++ char *dest, int inputSize); ++LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") ++LZ4LIB_API ++int LZ4_compressHC_limitedOutput_withStateHC(void *state, const char *source, ++ char *dest, int inputSize, ++ int maxOutputSize); ++LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") ++LZ4LIB_API int LZ4_compressHC2_withStateHC(void *state, const char *source, ++ char *dest, int inputSize, ++ int compressionLevel); ++LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") ++LZ4LIB_API ++int LZ4_compressHC2_limitedOutput_withStateHC(void *state, const char *source, ++ char *dest, int inputSize, ++ int maxOutputSize, ++ int compressionLevel); ++LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") ++LZ4LIB_API int LZ4_compressHC_continue(LZ4_streamHC_t *LZ4_streamHCPtr, ++ const char *source, char *dest, ++ int inputSize); ++LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") ++LZ4LIB_API int ++LZ4_compressHC_limitedOutput_continue(LZ4_streamHC_t *LZ4_streamHCPtr, ++ const char *source, char *dest, ++ int inputSize, int maxOutputSize); ++ ++/* Obsolete streaming functions; degraded functionality; do not use! ++ * ++ * In order to perform streaming compression, these functions depended on data ++ * that is no longer tracked in the state. They have been preserved as well as ++ * possible: using them will still produce a correct output. However, use of ++ * LZ4_slideInputBufferHC() will truncate the history of the stream, rather ++ * than preserve a window-sized chunk of history. ++ */ ++#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) ++LZ4_DEPRECATED("use LZ4_createStreamHC() instead") ++LZ4LIB_API void *LZ4_createHC(const char *inputBuffer); ++LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") ++LZ4LIB_API int LZ4_freeHC(void *LZ4HC_Data); ++#endif ++LZ4_DEPRECATED("use LZ4_saveDictHC() instead") ++LZ4LIB_API char *LZ4_slideInputBufferHC(void *LZ4HC_Data); ++LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") ++LZ4LIB_API int LZ4_compressHC2_continue(void *LZ4HC_Data, const char *source, ++ char *dest, int inputSize, ++ int compressionLevel); ++LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") ++LZ4LIB_API int LZ4_compressHC2_limitedOutput_continue(void *LZ4HC_Data, ++ const char *source, ++ char *dest, int inputSize, ++ int maxOutputSize, ++ int compressionLevel); ++LZ4_DEPRECATED("use LZ4_createStreamHC() instead") ++LZ4LIB_API int LZ4_sizeofStreamStateHC(void); ++LZ4_DEPRECATED("use LZ4_initStreamHC() instead") ++LZ4LIB_API int LZ4_resetStreamStateHC(void *state, char *inputBuffer); ++ ++/* LZ4_resetStreamHC() is now replaced by LZ4_initStreamHC(). ++ * The intention is to emphasize the difference with LZ4_resetStreamHC_fast(), ++ * which is now the recommended function to start a new stream of blocks, ++ * but cannot be used to initialize a memory segment containing arbitrary garbage data. ++ * ++ * It is recommended to switch to LZ4_initStreamHC(). ++ * LZ4_resetStreamHC() will generate deprecation warnings in a future version. ++ */ ++LZ4LIB_API void LZ4_resetStreamHC(LZ4_streamHC_t *streamHCPtr, ++ int compressionLevel); ++ ++#if defined(__cplusplus) ++} ++#endif ++ ++#endif /* LZ4_HC_H_19834876238432 */ ++ ++/*-************************************************** ++ * !!!!! STATIC LINKING ONLY !!!!! ++ * Following definitions are considered experimental. ++ * They should not be linked from DLL, ++ * as there is no guarantee of API stability yet. ++ * Prototypes will be promoted to "stable" status ++ * after successful usage in real-life scenarios. ++ ***************************************************/ ++#ifdef LZ4_HC_STATIC_LINKING_ONLY /* protection macro */ ++#ifndef LZ4_HC_SLO_098092834 ++#define LZ4_HC_SLO_098092834 ++ ++#define LZ4_STATIC_LINKING_ONLY /* LZ4LIB_STATIC_API */ ++#include "lz4.h" ++ ++#if defined(__cplusplus) ++extern "C" { ++#endif ++ ++/*! LZ4_setCompressionLevel() : v1.8.0+ (experimental) ++ * It's possible to change compression level ++ * between successive invocations of LZ4_compress_HC_continue*() ++ * for dynamic adaptation. ++ */ ++LZ4LIB_STATIC_API void LZ4_setCompressionLevel(LZ4_streamHC_t *LZ4_streamHCPtr, ++ int compressionLevel); ++ ++/*! LZ4_favorDecompressionSpeed() : v1.8.2+ (experimental) ++ * Opt. Parser will favor decompression speed over compression ratio. ++ * Only applicable to levels >= LZ4HC_CLEVEL_OPT_MIN. ++ */ ++LZ4LIB_STATIC_API void ++LZ4_favorDecompressionSpeed(LZ4_streamHC_t *LZ4_streamHCPtr, int favor); ++ ++/*! LZ4_resetStreamHC_fast() : v1.9.0+ ++ * When an LZ4_streamHC_t is known to be in a internally coherent state, ++ * it can often be prepared for a new compression with almost no work, only ++ * sometimes falling back to the full, expensive reset that is always required ++ * when the stream is in an indeterminate state (i.e., the reset performed by ++ * LZ4_resetStreamHC()). ++ * ++ * LZ4_streamHCs are guaranteed to be in a valid state when: ++ * - returned from LZ4_createStreamHC() ++ * - reset by LZ4_resetStreamHC() ++ * - memset(stream, 0, sizeof(LZ4_streamHC_t)) ++ * - the stream was in a valid state and was reset by LZ4_resetStreamHC_fast() ++ * - the stream was in a valid state and was then used in any compression call ++ * that returned success ++ * - the stream was in an indeterminate state and was used in a compression ++ * call that fully reset the state (LZ4_compress_HC_extStateHC()) and that ++ * returned success ++ * ++ * Note: ++ * A stream that was last used in a compression call that returned an error ++ * may be passed to this function. However, it will be fully reset, which will ++ * clear any existing history and settings from the context. ++ */ ++LZ4LIB_STATIC_API void LZ4_resetStreamHC_fast(LZ4_streamHC_t *LZ4_streamHCPtr, ++ int compressionLevel); ++ ++/*! LZ4_compress_HC_extStateHC_fastReset() : ++ * A variant of LZ4_compress_HC_extStateHC(). ++ * ++ * Using this variant avoids an expensive initialization step. It is only safe ++ * to call if the state buffer is known to be correctly initialized already ++ * (see above comment on LZ4_resetStreamHC_fast() for a definition of ++ * "correctly initialized"). From a high level, the difference is that this ++ * function initializes the provided state with a call to ++ * LZ4_resetStreamHC_fast() while LZ4_compress_HC_extStateHC() starts with a ++ * call to LZ4_resetStreamHC(). ++ */ ++LZ4LIB_STATIC_API int ++LZ4_compress_HC_extStateHC_fastReset(void *state, const char *src, char *dst, ++ int srcSize, int dstCapacity, ++ int compressionLevel); ++ ++#if defined(__cplusplus) ++} ++#endif ++ ++#endif /* LZ4_HC_SLO_098092834 */ ++#endif /* LZ4_HC_STATIC_LINKING_ONLY */ +diff --git a/lib/lz4/lz4hc_compress.c b/lib/lz4/lz4hc_compress.c +deleted file mode 100644 +index e7ac8694b797..000000000000 +--- a/lib/lz4/lz4hc_compress.c ++++ /dev/null +@@ -1,768 +0,0 @@ +-/* +- * LZ4 HC - High Compression Mode of LZ4 +- * Copyright (C) 2011-2015, Yann Collet. +- * +- * BSD 2 - Clause License (http://www.opensource.org/licenses/bsd - license.php) +- * Redistribution and use in source and binary forms, with or without +- * modification, are permitted provided that the following conditions are +- * met: +- * * Redistributions of source code must retain the above copyright +- * notice, this list of conditions and the following disclaimer. +- * * Redistributions in binary form must reproduce the above +- * copyright notice, this list of conditions and the following disclaimer +- * in the documentation and/or other materials provided with the +- * distribution. +- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +- * You can contact the author at : +- * - LZ4 homepage : http://www.lz4.org +- * - LZ4 source repository : https://github.com/lz4/lz4 +- * +- * Changed for kernel usage by: +- * Sven Schmidt <4sschmid@informatik.uni-hamburg.de> +- */ +- +-/*-************************************ +- * Dependencies +- **************************************/ +-#include +-#include "lz4defs.h" +-#include +-#include +-#include /* memset */ +- +-/* ************************************* +- * Local Constants and types +- ***************************************/ +- +-#define OPTIMAL_ML (int)((ML_MASK - 1) + MINMATCH) +- +-#define HASH_FUNCTION(i) (((i) * 2654435761U) \ +- >> ((MINMATCH*8) - LZ4HC_HASH_LOG)) +-#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ +- +-static U32 LZ4HC_hashPtr(const void *ptr) +-{ +- return HASH_FUNCTION(LZ4_read32(ptr)); +-} +- +-/************************************** +- * HC Compression +- **************************************/ +-static void LZ4HC_init(LZ4HC_CCtx_internal *hc4, const BYTE *start) +-{ +- memset((void *)hc4->hashTable, 0, sizeof(hc4->hashTable)); +- memset(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); +- hc4->nextToUpdate = 64 * KB; +- hc4->base = start - 64 * KB; +- hc4->end = start; +- hc4->dictBase = start - 64 * KB; +- hc4->dictLimit = 64 * KB; +- hc4->lowLimit = 64 * KB; +-} +- +-/* Update chains up to ip (excluded) */ +-static FORCE_INLINE void LZ4HC_Insert(LZ4HC_CCtx_internal *hc4, +- const BYTE *ip) +-{ +- U16 * const chainTable = hc4->chainTable; +- U32 * const hashTable = hc4->hashTable; +- const BYTE * const base = hc4->base; +- U32 const target = (U32)(ip - base); +- U32 idx = hc4->nextToUpdate; +- +- while (idx < target) { +- U32 const h = LZ4HC_hashPtr(base + idx); +- size_t delta = idx - hashTable[h]; +- +- if (delta > MAX_DISTANCE) +- delta = MAX_DISTANCE; +- +- DELTANEXTU16(idx) = (U16)delta; +- +- hashTable[h] = idx; +- idx++; +- } +- +- hc4->nextToUpdate = target; +-} +- +-static FORCE_INLINE int LZ4HC_InsertAndFindBestMatch( +- LZ4HC_CCtx_internal *hc4, /* Index table will be updated */ +- const BYTE *ip, +- const BYTE * const iLimit, +- const BYTE **matchpos, +- const int maxNbAttempts) +-{ +- U16 * const chainTable = hc4->chainTable; +- U32 * const HashTable = hc4->hashTable; +- const BYTE * const base = hc4->base; +- const BYTE * const dictBase = hc4->dictBase; +- const U32 dictLimit = hc4->dictLimit; +- const U32 lowLimit = (hc4->lowLimit + 64 * KB > (U32)(ip - base)) +- ? hc4->lowLimit +- : (U32)(ip - base) - (64 * KB - 1); +- U32 matchIndex; +- int nbAttempts = maxNbAttempts; +- size_t ml = 0; +- +- /* HC4 match finder */ +- LZ4HC_Insert(hc4, ip); +- matchIndex = HashTable[LZ4HC_hashPtr(ip)]; +- +- while ((matchIndex >= lowLimit) +- && (nbAttempts)) { +- nbAttempts--; +- if (matchIndex >= dictLimit) { +- const BYTE * const match = base + matchIndex; +- +- if (*(match + ml) == *(ip + ml) +- && (LZ4_read32(match) == LZ4_read32(ip))) { +- size_t const mlt = LZ4_count(ip + MINMATCH, +- match + MINMATCH, iLimit) + MINMATCH; +- +- if (mlt > ml) { +- ml = mlt; +- *matchpos = match; +- } +- } +- } else { +- const BYTE * const match = dictBase + matchIndex; +- +- if (LZ4_read32(match) == LZ4_read32(ip)) { +- size_t mlt; +- const BYTE *vLimit = ip +- + (dictLimit - matchIndex); +- +- if (vLimit > iLimit) +- vLimit = iLimit; +- mlt = LZ4_count(ip + MINMATCH, +- match + MINMATCH, vLimit) + MINMATCH; +- if ((ip + mlt == vLimit) +- && (vLimit < iLimit)) +- mlt += LZ4_count(ip + mlt, +- base + dictLimit, +- iLimit); +- if (mlt > ml) { +- /* virtual matchpos */ +- ml = mlt; +- *matchpos = base + matchIndex; +- } +- } +- } +- matchIndex -= DELTANEXTU16(matchIndex); +- } +- +- return (int)ml; +-} +- +-static FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch( +- LZ4HC_CCtx_internal *hc4, +- const BYTE * const ip, +- const BYTE * const iLowLimit, +- const BYTE * const iHighLimit, +- int longest, +- const BYTE **matchpos, +- const BYTE **startpos, +- const int maxNbAttempts) +-{ +- U16 * const chainTable = hc4->chainTable; +- U32 * const HashTable = hc4->hashTable; +- const BYTE * const base = hc4->base; +- const U32 dictLimit = hc4->dictLimit; +- const BYTE * const lowPrefixPtr = base + dictLimit; +- const U32 lowLimit = (hc4->lowLimit + 64 * KB > (U32)(ip - base)) +- ? hc4->lowLimit +- : (U32)(ip - base) - (64 * KB - 1); +- const BYTE * const dictBase = hc4->dictBase; +- U32 matchIndex; +- int nbAttempts = maxNbAttempts; +- int delta = (int)(ip - iLowLimit); +- +- /* First Match */ +- LZ4HC_Insert(hc4, ip); +- matchIndex = HashTable[LZ4HC_hashPtr(ip)]; +- +- while ((matchIndex >= lowLimit) +- && (nbAttempts)) { +- nbAttempts--; +- if (matchIndex >= dictLimit) { +- const BYTE *matchPtr = base + matchIndex; +- +- if (*(iLowLimit + longest) +- == *(matchPtr - delta + longest)) { +- if (LZ4_read32(matchPtr) == LZ4_read32(ip)) { +- int mlt = MINMATCH + LZ4_count( +- ip + MINMATCH, +- matchPtr + MINMATCH, +- iHighLimit); +- int back = 0; +- +- while ((ip + back > iLowLimit) +- && (matchPtr + back > lowPrefixPtr) +- && (ip[back - 1] == matchPtr[back - 1])) +- back--; +- +- mlt -= back; +- +- if (mlt > longest) { +- longest = (int)mlt; +- *matchpos = matchPtr + back; +- *startpos = ip + back; +- } +- } +- } +- } else { +- const BYTE * const matchPtr = dictBase + matchIndex; +- +- if (LZ4_read32(matchPtr) == LZ4_read32(ip)) { +- size_t mlt; +- int back = 0; +- const BYTE *vLimit = ip + (dictLimit - matchIndex); +- +- if (vLimit > iHighLimit) +- vLimit = iHighLimit; +- +- mlt = LZ4_count(ip + MINMATCH, +- matchPtr + MINMATCH, vLimit) + MINMATCH; +- +- if ((ip + mlt == vLimit) && (vLimit < iHighLimit)) +- mlt += LZ4_count(ip + mlt, base + dictLimit, +- iHighLimit); +- while ((ip + back > iLowLimit) +- && (matchIndex + back > lowLimit) +- && (ip[back - 1] == matchPtr[back - 1])) +- back--; +- +- mlt -= back; +- +- if ((int)mlt > longest) { +- longest = (int)mlt; +- *matchpos = base + matchIndex + back; +- *startpos = ip + back; +- } +- } +- } +- +- matchIndex -= DELTANEXTU16(matchIndex); +- } +- +- return longest; +-} +- +-static FORCE_INLINE int LZ4HC_encodeSequence( +- const BYTE **ip, +- BYTE **op, +- const BYTE **anchor, +- int matchLength, +- const BYTE * const match, +- limitedOutput_directive limitedOutputBuffer, +- BYTE *oend) +-{ +- int length; +- BYTE *token; +- +- /* Encode Literal length */ +- length = (int)(*ip - *anchor); +- token = (*op)++; +- +- if ((limitedOutputBuffer) +- && ((*op + (length>>8) +- + length + (2 + 1 + LASTLITERALS)) > oend)) { +- /* Check output limit */ +- return 1; +- } +- if (length >= (int)RUN_MASK) { +- int len; +- +- *token = (RUN_MASK< 254 ; len -= 255) +- *(*op)++ = 255; +- *(*op)++ = (BYTE)len; +- } else +- *token = (BYTE)(length<>8) +- + (1 + LASTLITERALS) > oend)) { +- /* Check output limit */ +- return 1; +- } +- +- if (length >= (int)ML_MASK) { +- *token += ML_MASK; +- length -= ML_MASK; +- +- for (; length > 509 ; length -= 510) { +- *(*op)++ = 255; +- *(*op)++ = 255; +- } +- +- if (length > 254) { +- length -= 255; +- *(*op)++ = 255; +- } +- +- *(*op)++ = (BYTE)length; +- } else +- *token += (BYTE)(length); +- +- /* Prepare next loop */ +- *ip += matchLength; +- *anchor = *ip; +- +- return 0; +-} +- +-static int LZ4HC_compress_generic( +- LZ4HC_CCtx_internal *const ctx, +- const char * const source, +- char * const dest, +- int const inputSize, +- int const maxOutputSize, +- int compressionLevel, +- limitedOutput_directive limit +- ) +-{ +- const BYTE *ip = (const BYTE *) source; +- const BYTE *anchor = ip; +- const BYTE * const iend = ip + inputSize; +- const BYTE * const mflimit = iend - MFLIMIT; +- const BYTE * const matchlimit = (iend - LASTLITERALS); +- +- BYTE *op = (BYTE *) dest; +- BYTE * const oend = op + maxOutputSize; +- +- unsigned int maxNbAttempts; +- int ml, ml2, ml3, ml0; +- const BYTE *ref = NULL; +- const BYTE *start2 = NULL; +- const BYTE *ref2 = NULL; +- const BYTE *start3 = NULL; +- const BYTE *ref3 = NULL; +- const BYTE *start0; +- const BYTE *ref0; +- +- /* init */ +- if (compressionLevel > LZ4HC_MAX_CLEVEL) +- compressionLevel = LZ4HC_MAX_CLEVEL; +- if (compressionLevel < 1) +- compressionLevel = LZ4HC_DEFAULT_CLEVEL; +- maxNbAttempts = 1 << (compressionLevel - 1); +- ctx->end += inputSize; +- +- ip++; +- +- /* Main Loop */ +- while (ip < mflimit) { +- ml = LZ4HC_InsertAndFindBestMatch(ctx, ip, +- matchlimit, (&ref), maxNbAttempts); +- if (!ml) { +- ip++; +- continue; +- } +- +- /* saved, in case we would skip too much */ +- start0 = ip; +- ref0 = ref; +- ml0 = ml; +- +-_Search2: +- if (ip + ml < mflimit) +- ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, +- ip + ml - 2, ip + 0, +- matchlimit, ml, &ref2, +- &start2, maxNbAttempts); +- else +- ml2 = ml; +- +- if (ml2 == ml) { +- /* No better match */ +- if (LZ4HC_encodeSequence(&ip, &op, +- &anchor, ml, ref, limit, oend)) +- return 0; +- continue; +- } +- +- if (start0 < ip) { +- if (start2 < ip + ml0) { +- /* empirical */ +- ip = start0; +- ref = ref0; +- ml = ml0; +- } +- } +- +- /* Here, start0 == ip */ +- if ((start2 - ip) < 3) { +- /* First Match too small : removed */ +- ml = ml2; +- ip = start2; +- ref = ref2; +- goto _Search2; +- } +- +-_Search3: +- /* +- * Currently we have : +- * ml2 > ml1, and +- * ip1 + 3 <= ip2 (usually < ip1 + ml1) +- */ +- if ((start2 - ip) < OPTIMAL_ML) { +- int correction; +- int new_ml = ml; +- +- if (new_ml > OPTIMAL_ML) +- new_ml = OPTIMAL_ML; +- if (ip + new_ml > start2 + ml2 - MINMATCH) +- new_ml = (int)(start2 - ip) + ml2 - MINMATCH; +- +- correction = new_ml - (int)(start2 - ip); +- +- if (correction > 0) { +- start2 += correction; +- ref2 += correction; +- ml2 -= correction; +- } +- } +- /* +- * Now, we have start2 = ip + new_ml, +- * with new_ml = min(ml, OPTIMAL_ML = 18) +- */ +- +- if (start2 + ml2 < mflimit) +- ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, +- start2 + ml2 - 3, start2, +- matchlimit, ml2, &ref3, &start3, +- maxNbAttempts); +- else +- ml3 = ml2; +- +- if (ml3 == ml2) { +- /* No better match : 2 sequences to encode */ +- /* ip & ref are known; Now for ml */ +- if (start2 < ip + ml) +- ml = (int)(start2 - ip); +- /* Now, encode 2 sequences */ +- if (LZ4HC_encodeSequence(&ip, &op, &anchor, +- ml, ref, limit, oend)) +- return 0; +- ip = start2; +- if (LZ4HC_encodeSequence(&ip, &op, &anchor, +- ml2, ref2, limit, oend)) +- return 0; +- continue; +- } +- +- if (start3 < ip + ml + 3) { +- /* Not enough space for match 2 : remove it */ +- if (start3 >= (ip + ml)) { +- /* can write Seq1 immediately +- * ==> Seq2 is removed, +- * so Seq3 becomes Seq1 +- */ +- if (start2 < ip + ml) { +- int correction = (int)(ip + ml - start2); +- +- start2 += correction; +- ref2 += correction; +- ml2 -= correction; +- if (ml2 < MINMATCH) { +- start2 = start3; +- ref2 = ref3; +- ml2 = ml3; +- } +- } +- +- if (LZ4HC_encodeSequence(&ip, &op, &anchor, +- ml, ref, limit, oend)) +- return 0; +- ip = start3; +- ref = ref3; +- ml = ml3; +- +- start0 = start2; +- ref0 = ref2; +- ml0 = ml2; +- goto _Search2; +- } +- +- start2 = start3; +- ref2 = ref3; +- ml2 = ml3; +- goto _Search3; +- } +- +- /* +- * OK, now we have 3 ascending matches; +- * let's write at least the first one +- * ip & ref are known; Now for ml +- */ +- if (start2 < ip + ml) { +- if ((start2 - ip) < (int)ML_MASK) { +- int correction; +- +- if (ml > OPTIMAL_ML) +- ml = OPTIMAL_ML; +- if (ip + ml > start2 + ml2 - MINMATCH) +- ml = (int)(start2 - ip) + ml2 - MINMATCH; +- correction = ml - (int)(start2 - ip); +- if (correction > 0) { +- start2 += correction; +- ref2 += correction; +- ml2 -= correction; +- } +- } else +- ml = (int)(start2 - ip); +- } +- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, +- ref, limit, oend)) +- return 0; +- +- ip = start2; +- ref = ref2; +- ml = ml2; +- +- start2 = start3; +- ref2 = ref3; +- ml2 = ml3; +- +- goto _Search3; +- } +- +- /* Encode Last Literals */ +- { +- int lastRun = (int)(iend - anchor); +- +- if ((limit) +- && (((char *)op - dest) + lastRun + 1 +- + ((lastRun + 255 - RUN_MASK)/255) +- > (U32)maxOutputSize)) { +- /* Check output limit */ +- return 0; +- } +- if (lastRun >= (int)RUN_MASK) { +- *op++ = (RUN_MASK< 254 ; lastRun -= 255) +- *op++ = 255; +- *op++ = (BYTE) lastRun; +- } else +- *op++ = (BYTE)(lastRun<internal_donotuse; +- +- if (((size_t)(state)&(sizeof(void *) - 1)) != 0) { +- /* Error : state is not aligned +- * for pointers (32 or 64 bits) +- */ +- return 0; +- } +- +- LZ4HC_init(ctx, (const BYTE *)src); +- +- if (maxDstSize < LZ4_compressBound(srcSize)) +- return LZ4HC_compress_generic(ctx, src, dst, +- srcSize, maxDstSize, compressionLevel, limitedOutput); +- else +- return LZ4HC_compress_generic(ctx, src, dst, +- srcSize, maxDstSize, compressionLevel, noLimit); +-} +- +-int LZ4_compress_HC(const char *src, char *dst, int srcSize, +- int maxDstSize, int compressionLevel, void *wrkmem) +-{ +- return LZ4_compress_HC_extStateHC(wrkmem, src, dst, +- srcSize, maxDstSize, compressionLevel); +-} +-EXPORT_SYMBOL(LZ4_compress_HC); +- +-/************************************** +- * Streaming Functions +- **************************************/ +-void LZ4_resetStreamHC(LZ4_streamHC_t *LZ4_streamHCPtr, int compressionLevel) +-{ +- LZ4_streamHCPtr->internal_donotuse.base = NULL; +- LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned int)compressionLevel; +-} +- +-int LZ4_loadDictHC(LZ4_streamHC_t *LZ4_streamHCPtr, +- const char *dictionary, +- int dictSize) +-{ +- LZ4HC_CCtx_internal *ctxPtr = &LZ4_streamHCPtr->internal_donotuse; +- +- if (dictSize > 64 * KB) { +- dictionary += dictSize - 64 * KB; +- dictSize = 64 * KB; +- } +- LZ4HC_init(ctxPtr, (const BYTE *)dictionary); +- if (dictSize >= 4) +- LZ4HC_Insert(ctxPtr, (const BYTE *)dictionary + (dictSize - 3)); +- ctxPtr->end = (const BYTE *)dictionary + dictSize; +- return dictSize; +-} +-EXPORT_SYMBOL(LZ4_loadDictHC); +- +-/* compression */ +- +-static void LZ4HC_setExternalDict( +- LZ4HC_CCtx_internal *ctxPtr, +- const BYTE *newBlock) +-{ +- if (ctxPtr->end >= ctxPtr->base + 4) { +- /* Referencing remaining dictionary content */ +- LZ4HC_Insert(ctxPtr, ctxPtr->end - 3); +- } +- +- /* +- * Only one memory segment for extDict, +- * so any previous extDict is lost at this stage +- */ +- ctxPtr->lowLimit = ctxPtr->dictLimit; +- ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); +- ctxPtr->dictBase = ctxPtr->base; +- ctxPtr->base = newBlock - ctxPtr->dictLimit; +- ctxPtr->end = newBlock; +- /* match referencing will resume from there */ +- ctxPtr->nextToUpdate = ctxPtr->dictLimit; +-} +- +-static int LZ4_compressHC_continue_generic( +- LZ4_streamHC_t *LZ4_streamHCPtr, +- const char *source, +- char *dest, +- int inputSize, +- int maxOutputSize, +- limitedOutput_directive limit) +-{ +- LZ4HC_CCtx_internal *ctxPtr = &LZ4_streamHCPtr->internal_donotuse; +- +- /* auto - init if forgotten */ +- if (ctxPtr->base == NULL) +- LZ4HC_init(ctxPtr, (const BYTE *) source); +- +- /* Check overflow */ +- if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 * GB) { +- size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) +- - ctxPtr->dictLimit; +- if (dictSize > 64 * KB) +- dictSize = 64 * KB; +- LZ4_loadDictHC(LZ4_streamHCPtr, +- (const char *)(ctxPtr->end) - dictSize, (int)dictSize); +- } +- +- /* Check if blocks follow each other */ +- if ((const BYTE *)source != ctxPtr->end) +- LZ4HC_setExternalDict(ctxPtr, (const BYTE *)source); +- +- /* Check overlapping input/dictionary space */ +- { +- const BYTE *sourceEnd = (const BYTE *) source + inputSize; +- const BYTE * const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; +- const BYTE * const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; +- +- if ((sourceEnd > dictBegin) +- && ((const BYTE *)source < dictEnd)) { +- if (sourceEnd > dictEnd) +- sourceEnd = dictEnd; +- ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); +- +- if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) +- ctxPtr->lowLimit = ctxPtr->dictLimit; +- } +- } +- +- return LZ4HC_compress_generic(ctxPtr, source, dest, +- inputSize, maxOutputSize, ctxPtr->compressionLevel, limit); +-} +- +-int LZ4_compress_HC_continue( +- LZ4_streamHC_t *LZ4_streamHCPtr, +- const char *source, +- char *dest, +- int inputSize, +- int maxOutputSize) +-{ +- if (maxOutputSize < LZ4_compressBound(inputSize)) +- return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, +- source, dest, inputSize, maxOutputSize, limitedOutput); +- else +- return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, +- source, dest, inputSize, maxOutputSize, noLimit); +-} +-EXPORT_SYMBOL(LZ4_compress_HC_continue); +- +-/* dictionary saving */ +- +-int LZ4_saveDictHC( +- LZ4_streamHC_t *LZ4_streamHCPtr, +- char *safeBuffer, +- int dictSize) +-{ +- LZ4HC_CCtx_internal *const streamPtr = &LZ4_streamHCPtr->internal_donotuse; +- int const prefixSize = (int)(streamPtr->end +- - (streamPtr->base + streamPtr->dictLimit)); +- +- if (dictSize > 64 * KB) +- dictSize = 64 * KB; +- if (dictSize < 4) +- dictSize = 0; +- if (dictSize > prefixSize) +- dictSize = prefixSize; +- +- memmove(safeBuffer, streamPtr->end - dictSize, dictSize); +- +- { +- U32 const endIndex = (U32)(streamPtr->end - streamPtr->base); +- +- streamPtr->end = (const BYTE *)safeBuffer + dictSize; +- streamPtr->base = streamPtr->end - endIndex; +- streamPtr->dictLimit = endIndex - dictSize; +- streamPtr->lowLimit = endIndex - dictSize; +- +- if (streamPtr->nextToUpdate < streamPtr->dictLimit) +- streamPtr->nextToUpdate = streamPtr->dictLimit; +- } +- return dictSize; +-} +-EXPORT_SYMBOL(LZ4_saveDictHC); +- +-MODULE_LICENSE("Dual BSD/GPL"); +-MODULE_DESCRIPTION("LZ4 HC compressor"); diff --git a/common/lz4_optimized_decompress.patch b/common/lz4_optimized_decompress.patch new file mode 100644 index 00000000..62f57c6b --- /dev/null +++ b/common/lz4_optimized_decompress.patch @@ -0,0 +1,26 @@ +commit 2ed5bf3ce1cf9c7011802cd6cc10920920caff19 +Author: brokestar233 <3765589194@qq.com> +Date: Sat Oct 11 08:05:54 2025 +0800 + + lib/decompress_unlz4: Add NEON-optimized LZ4 decompression for ARM64 + + Add support for NEON-optimized LZ4 decompression on ARM64 platforms when CONFIG_KERNEL_MODE_NEON is enabled. This optimization is applied during the decompression of some partitions(e.g., kernel, init_boot), improving performance by leveraging ARM64's NEON SIMD instructions. Fallback to standard LZ4_decompress_safe() when NEON is not available. + + Signed-off-by: brokestar233 <3765589194@qq.com> + +diff --git a/lib/decompress_unlz4.c b/lib/decompress_unlz4.c +index e6327391b6b6..57bfe602b8c6 100644 +--- a/lib/decompress_unlz4.c ++++ b/lib/decompress_unlz4.c +@@ -162,7 +162,11 @@ STATIC inline int INIT unlz4(u8 *input, long in_len, + #else + dest_len = uncomp_chunksize; + ++#if defined(CONFIG_ARM64) && defined(CONFIG_KERNEL_MODE_NEON) ++ ret = LZ4_arm64_decompress_safe(inp, outp, chunksize, dest_len, false); ++#else + ret = LZ4_decompress_safe(inp, outp, chunksize, dest_len); ++#endif + dest_len = ret; + #endif + if (ret < 0) { diff --git a/common/reduce_pelt.patch b/common/reduce_pelt.patch new file mode 100644 index 00000000..c4b211fd --- /dev/null +++ b/common/reduce_pelt.patch @@ -0,0 +1,34 @@ +commit aa532f4d2e45c5237c25a99d2a02df8a8ec55b21 +Author: Sultan Alsawaf +Date: Sun Oct 22 14:12:46 2023 -0700 + + sched/fair: Reduce PELT half-life from 32 ms to 16 ms + + Ramping up faster should improve interactivity. Reduce the PELT half-life + from 32 ms to 16 ms to achieve this. + + Signed-off-by: Sultan Alsawaf + +diff --git a/kernel/sched/sched-pelt.h b/kernel/sched/sched-pelt.h +index c529706bed11..71d186e0c925 100644 +--- a/kernel/sched/sched-pelt.h ++++ b/kernel/sched/sched-pelt.h +@@ -2,13 +2,10 @@ + /* Generated by Documentation/scheduler/sched-pelt; do not modify. */ + + static const u32 runnable_avg_yN_inv[] __maybe_unused = { +- 0xffffffff, 0xfa83b2da, 0xf5257d14, 0xefe4b99a, 0xeac0c6e6, 0xe5b906e6, +- 0xe0ccdeeb, 0xdbfbb796, 0xd744fcc9, 0xd2a81d91, 0xce248c14, 0xc9b9bd85, +- 0xc5672a10, 0xc12c4cc9, 0xbd08a39e, 0xb8fbaf46, 0xb504f333, 0xb123f581, +- 0xad583ee9, 0xa9a15ab4, 0xa5fed6a9, 0xa2704302, 0x9ef5325f, 0x9b8d39b9, +- 0x9837f050, 0x94f4efa8, 0x91c3d373, 0x8ea4398a, 0x8b95c1e3, 0x88980e80, +- 0x85aac367, 0x82cd8698, ++ 0xffffffff, 0xf5257d14, 0xeac0c6e6, 0xe0ccdeeb, 0xd744fcc9, 0xce248c14, ++ 0xc5672a10, 0xbd08a39e, 0xb504f333, 0xad583ee9, 0xa5fed6a9, 0x9ef5325f, ++ 0x9837f050, 0x91c3d373, 0x8b95c1e3, 0x85aac367, + }; + +-#define LOAD_AVG_PERIOD 32 +-#define LOAD_AVG_MAX 47742 ++#define LOAD_AVG_PERIOD 16 ++#define LOAD_AVG_MAX 24130 diff --git a/common/workqueue.patch b/common/workqueue.patch new file mode 100644 index 00000000..8a35a9dd --- /dev/null +++ b/common/workqueue.patch @@ -0,0 +1,81 @@ +commit 14beddd592a8be36b904c3753c0cf5fe9a370b1c +Author: Wangyang Guo +Date: Fri Nov 15 13:49:36 2024 +0800 + + workqueue: Reduce expensive locks for unbound workqueue + + For unbound workqueue, pwqs usually map to just a few pools. Most of + the time, pwqs will be linked sequentially to wq->pwqs list by cpu + index. Usually, consecutive CPUs have the same workqueue attribute + (e.g. belong to the same NUMA node). This makes pwqs with the same + pool cluster together in the pwq list. + + Only do lock/unlock if the pool has changed in flush_workqueue_prep_pwqs(). + This reduces the number of expensive lock operations. + + The performance data shows this change boosts FIO by 65x in some cases + when multiple concurrent threads write to xfs mount points with fsync. + + FIO Benchmark Details + - FIO version: v3.35 + - FIO Options: ioengine=libaio,iodepth=64,norandommap=1,rw=write, + size=128M,bs=4k,fsync=1 + - FIO Job Configs: 64 jobs in total writing to 4 mount points (ramdisks + formatted as xfs file system). + - Kernel Codebase: v6.12-rc5 + - Test Platform: Xeon 8380 (2 sockets) + + Reviewed-by: Tim Chen + Signed-off-by: Wangyang Guo + Reviewed-by: Lai Jiangshan + Signed-off-by: Tejun Heo + +diff --git a/kernel/workqueue.c b/kernel/workqueue.c +index 43045d078d62..4756e2483946 100644 +--- a/kernel/workqueue.c ++++ b/kernel/workqueue.c +@@ -2811,16 +2811,28 @@ static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq, + { + bool wait = false; + struct pool_workqueue *pwq; ++ struct worker_pool *current_pool = NULL; + + if (flush_color >= 0) { + WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush)); + atomic_set(&wq->nr_pwqs_to_flush, 1); + } + ++ /* ++ * For unbound workqueue, pwqs will map to only a few pools. ++ * Most of the time, pwqs within the same pool will be linked ++ * sequentially to wq->pwqs by cpu index. So in the majority ++ * of pwq iters, the pool is the same, only doing lock/unlock ++ * if the pool has changed. This can largely reduce expensive ++ * lock operations. ++ */ + for_each_pwq(pwq, wq) { +- struct worker_pool *pool = pwq->pool; +- +- raw_spin_lock_irq(&pool->lock); ++ if (current_pool != pwq->pool) { ++ if (likely(current_pool)) ++ raw_spin_unlock_irq(¤t_pool->lock); ++ current_pool = pwq->pool; ++ raw_spin_lock_irq(¤t_pool->lock); ++ } + + if (flush_color >= 0) { + WARN_ON_ONCE(pwq->flush_color != -1); +@@ -2837,9 +2849,11 @@ static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq, + pwq->work_color = work_color; + } + +- raw_spin_unlock_irq(&pool->lock); + } + ++ if (current_pool) ++ raw_spin_unlock_irq(¤t_pool->lock); ++ + if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush)) + complete(&wq->first_flusher->done); + diff --git a/common/xiaomi_zram_optimization.patch b/common/xiaomi_zram_optimization.patch new file mode 100644 index 00000000..7db8ac37 --- /dev/null +++ b/common/xiaomi_zram_optimization.patch @@ -0,0 +1,24 @@ +commit e12cc73bad4c95c19f9491ffebc135e84998d62e +Author: ztc1997 +Date: Sat May 11 07:12:55 2024 +0800 + + drivers/block: Import Xiaomi's zram optimization changes + + - From branch: `bsp-vermeer-t-oss` + + Change-Id: I9ff6ebb63ad4e92e31d0c559ea0c554ec72c5118 + +diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c +index 0da494afd62b..49e54ad1a0a0 100644 +--- a/drivers/block/zram/zram_drv.c ++++ b/drivers/block/zram/zram_drv.c +@@ -599,7 +599,7 @@ + if (!parent) { + - bio->bi_opf = REQ_OP_READ; + + bio->bi_opf = REQ_OP_READ | REQ_PRIO; + bio->bi_end_io = zram_page_end_io; + } else { + - bio->bi_opf = parent->bi_opf; + + bio->bi_opf = parent->bi_opf | REQ_PRIO; + bio_chain(bio, parent); + } diff --git a/oneplus/hmbird/fengchi_OP-ACE-5-PRO_A15.patch b/oneplus/hmbird/fengchi_OP-ACE-5-PRO_A15.patch deleted file mode 100644 index b62b60b2..00000000 --- a/oneplus/hmbird/fengchi_OP-ACE-5-PRO_A15.patch +++ /dev/null @@ -1,21314 +0,0 @@ -diff --git a/Documentation/scheduler/index.rst b/Documentation/scheduler/index.rst -index 0b650bb550e6..3170747226f6 100644 ---- a/Documentation/scheduler/index.rst -+++ b/Documentation/scheduler/index.rst -@@ -19,7 +19,6 @@ Scheduler - sched-nice-design - sched-rt-group - sched-stats -- sched-ext - sched-debug - - text_files -diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst -deleted file mode 100644 -index 84c30b44f104..000000000000 ---- a/Documentation/scheduler/sched-ext.rst -+++ /dev/null -@@ -1,230 +0,0 @@ --========================== --Extensible Scheduler Class --========================== -- --sched_ext is a scheduler class whose behavior can be defined by a set of BPF --programs - the BPF scheduler. -- --* sched_ext exports a full scheduling interface so that any scheduling -- algorithm can be implemented on top. -- --* The BPF scheduler can group CPUs however it sees fit and schedule them -- together, as tasks aren't tied to specific CPUs at the time of wakeup. -- --* The BPF scheduler can be turned on and off dynamically anytime. -- --* The system integrity is maintained no matter what the BPF scheduler does. -- The default scheduling behavior is restored anytime an error is detected, -- a runnable task stalls, or on invoking the SysRq key sequence -- :kbd:`SysRq-S`. -- --Switching to and from sched_ext --=============================== -- --``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and --``tools/sched_ext`` contains the example schedulers. -- --sched_ext is used only when the BPF scheduler is loaded and running. -- --If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be --treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is --loaded. On load, such tasks will be switched to and scheduled by sched_ext. -- --The BPF scheduler can choose to schedule all normal and lower class tasks by --calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this --case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and --``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers, --this mode can be selected with the ``-a`` option. -- --Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or --detection of any internal error including stalled runnable tasks aborts the --BPF scheduler and reverts all tasks back to CFS. -- --.. code-block:: none -- -- # make -j16 -C tools/sched_ext -- # tools/sched_ext/scx_example_simple -- local=0 global=3 -- local=5 global=24 -- local=9 global=44 -- local=13 global=56 -- local=17 global=72 -- ^CEXIT: BPF scheduler unregistered -- --If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler --and whether a given task is on sched_ext can be determined as follows: -- --.. code-block:: none -- -- # cat /sys/kernel/debug/sched/ext -- ops : simple -- enabled : 1 -- switching_all : 1 -- switched_all : 1 -- enable_state : enabled -- -- # grep ext /proc/self/sched -- ext.enabled : 1 -- --The Basics --========== -- --Userspace can implement an arbitrary BPF scheduler by loading a set of BPF --programs that implement ``struct sched_ext_ops``. The only mandatory field --is ``ops.name`` which must be a valid BPF object name. All operations are --optional. The following modified excerpt is from --``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO --scheduler. -- --.. code-block:: c -- -- s32 BPF_STRUCT_OPS(simple_init) -- { -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; -- } -- -- void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) -- { -- if (enq_flags & SCX_ENQ_LOCAL) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags); -- } -- -- void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) -- { -- exit_type = ei->type; -- } -- -- SEC(".struct_ops") -- struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", -- }; -- --Dispatch Queues ----------------- -- --To match the impedance between the scheduler core and the BPF scheduler, --sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a --priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``), --and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage --an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and --``scx_bpf_destroy_dsq()``. -- --A CPU always executes a task from its local DSQ. A task is "dispatched" to a --DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's --local DSQ. -- --When a CPU is looking for the next task to run, if the local DSQ is not --empty, the first task is picked. Otherwise, the CPU tries to consume the --global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()`` --is invoked. -- --Scheduling Cycle ------------------ -- --The following briefly shows how a waking task is scheduled and executed. -- --1. When a task is waking up, ``ops.select_cpu()`` is the first operation -- invoked. This serves two purposes. First, CPU selection optimization -- hint. Second, waking up the selected CPU if idle. -- -- The CPU selected by ``ops.select_cpu()`` is an optimization hint and not -- binding. The actual decision is made at the last step of scheduling. -- However, there is a small performance gain if the CPU -- ``ops.select_cpu()`` returns matches the CPU the task eventually runs on. -- -- A side-effect of selecting a CPU is waking it up from idle. While a BPF -- scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper, -- using ``ops.select_cpu()`` judiciously can be simpler and more efficient. -- -- Note that the scheduler core will ignore an invalid CPU selection, for -- example, if it's outside the allowed cpumask of the task. -- --2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can -- make one of the following decisions: -- -- * Immediately dispatch the task to either the global or local DSQ by -- calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or -- ``SCX_DSQ_LOCAL``, respectively. -- -- * Immediately dispatch the task to a custom DSQ by calling -- ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63. -- -- * Queue the task on the BPF side. -- --3. When a CPU is ready to schedule, it first looks at its local DSQ. If -- empty, it then looks at the global DSQ. If there still isn't a task to -- run, ``ops.dispatch()`` is invoked which can use the following two -- functions to populate the local DSQ. -- -- * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can -- be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``, -- ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()`` -- currently can't be called with BPF locks held, this is being worked on -- and will be supported. ``scx_bpf_dispatch()`` schedules dispatching -- rather than performing them immediately. There can be up to -- ``ops.dispatch_max_batch`` pending tasks. -- -- * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ -- to the dispatching DSQ. This function cannot be called with any BPF -- locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks -- before trying to consume the specified DSQ. -- --4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ, -- the CPU runs the first one. If empty, the following steps are taken: -- -- * Try to consume the global DSQ. If successful, run the task. -- -- * If ``ops.dispatch()`` has dispatched any tasks, retry #3. -- -- * If the previous task is an SCX task and still runnable, keep executing -- it (see ``SCX_OPS_ENQ_LAST``). -- -- * Go idle. -- --Note that the BPF scheduler can always choose to dispatch tasks immediately --in ``ops.enqueue()`` as illustrated in the above simple example. If only the --built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as --a task is never queued on the BPF scheduler and both the local and global --DSQs are consumed automatically. -- --``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use --``scx_bpf_dispatch_vtime()`` for the priority queue. See the function --documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for --more information. -- --Where to Look --============= -- --* ``include/linux/sched/ext.h`` defines the core data structures, ops table -- and constants. -- --* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. -- The functions prefixed with ``scx_bpf_`` can be called from the BPF -- scheduler. -- --* ``tools/sched_ext/`` hosts example BPF scheduler implementations. -- -- * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example -- using a custom DSQ. -- -- * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting -- five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. -- --ABI Instability --=============== -- --The APIs provided by sched_ext to BPF schedulers programs have no stability --guarantees. This includes the ops table callbacks and constants defined in --``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in --``kernel/sched/ext.c``. -- --While we will attempt to provide a relatively stable API surface when --possible, they are subject to change without warning between kernel --versions. -diff --git a/MAINTAINERS b/MAINTAINERS -index 8d13669d7816..1b46ab23da8d 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -19117,8 +19117,6 @@ R: Ben Segall (CONFIG_CFS_BANDWIDTH) - R: Mel Gorman (CONFIG_NUMA_BALANCING) - R: Daniel Bristot de Oliveira (SCHED_DEADLINE) - R: Valentin Schneider (TOPOLOGY) --R: Tejun Heo (SCHED_EXT) --R: David Vernet (SCHED_EXT) - L: linux-kernel@vger.kernel.org - S: Maintained - T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core -@@ -19127,7 +19125,6 @@ F: include/linux/sched.h - F: include/linux/wait.h - F: include/uapi/linux/sched.h - F: kernel/sched/ --F: tools/sched_ext/ - - SCSI LIBSAS SUBSYSTEM - R: John Garry -diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig -index 520a566bea97..27472095b210 100644 ---- a/arch/arm64/configs/defconfig -+++ b/arch/arm64/configs/defconfig -@@ -6,8 +6,7 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_BPF_SYSCALL=y - CONFIG_BPF_JIT=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_BSD_PROCESS_ACCT=y - CONFIG_BSD_PROCESS_ACCT_V3=y -diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig -index dbd881a37733..13780048caca 100644 ---- a/arch/arm64/configs/gki_defconfig -+++ b/arch/arm64/configs/gki_defconfig -@@ -10,8 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y - # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set - CONFIG_BPF_LSM=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_TASKSTATS=y - CONFIG_TASK_DELAY_ACCT=y -diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c -index 4bd028fa7500..5825ce9d6d7b 100644 ---- a/drivers/gpu/drm/msm/msm_drv.c -+++ b/drivers/gpu/drm/msm/msm_drv.c -@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) - } - - sched_set_fifo(ev_thread->worker->task); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3); -+#endif - } - - ret = drm_vblank_init(ddev, priv->num_crtcs); -diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c -index bd001445815e..dcf2e1ebf9c5 100644 ---- a/drivers/tty/sysrq.c -+++ b/drivers/tty/sysrq.c -@@ -524,7 +524,6 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { - NULL, /* P */ - NULL, /* Q */ - NULL, /* R */ -- /* S: May be registered by sched_ext for resetting */ - NULL, /* S */ - NULL, /* T */ - NULL, /* U */ -diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h -index b4c51e798e25..1d24e20f0e35 100644 ---- a/include/asm-generic/vmlinux.lds.h -+++ b/include/asm-generic/vmlinux.lds.h -@@ -131,7 +131,7 @@ - *(__dl_sched_class) \ - *(__rt_sched_class) \ - *(__fair_sched_class) \ -- *(__ext_sched_class) \ -+ *(__hmbird_sched_class) \ - *(__idle_sched_class) \ - __sched_class_lowest = .; - -diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h -index 90f8bd1736a2..df03e0659df0 100644 ---- a/include/linux/cpufreq.h -+++ b/include/linux/cpufreq.h -@@ -44,6 +44,10 @@ enum cpufreq_table_sorting { - CPUFREQ_TABLE_SORTED_DESCENDING - }; - -+ssize_t store_scaling_governor(struct cpufreq_policy *policy, -+ const char *buf, size_t count); -+ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf); -+ - struct cpufreq_cpuinfo { - unsigned int max_freq; - unsigned int min_freq; -diff --git a/include/linux/sched.h b/include/linux/sched.h -index 464131256ddd..39c797c5cc75 100644 ---- a/include/linux/sched.h -+++ b/include/linux/sched.h -@@ -73,7 +73,9 @@ struct task_delay_info; - struct task_group; - struct user_event_mm; - --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - - /* - * Task state bitmask. NOTE! These bits are also -@@ -298,11 +300,6 @@ enum { - - extern void scheduler_tick(void); - --#ifdef CONFIG_SLIM_SCHED --extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer); --extern void stop_shadow_tick_timer(void); --#endif -- - #define MAX_SCHEDULE_TIMEOUT LONG_MAX - - extern long schedule_timeout(long timeout); -@@ -1523,13 +1520,8 @@ struct task_struct { - */ - struct callback_head l1d_flush_kill; - #endif --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, unsigned long sched_prop); -- ANDROID_KABI_USE(2, struct sched_ext_entity *scx); --#else - ANDROID_KABI_RESERVE(1); - ANDROID_KABI_RESERVE(2); --#endif - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); - ANDROID_KABI_RESERVE(5); -@@ -1932,6 +1924,92 @@ static inline int task_nice(const struct task_struct *p) - return PRIO_TO_NICE((p)->static_prio); - } - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline bool task_is_top_task(struct task_struct *p) -+{ -+ return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ ->top_task_prop & TOP_TASK_BITS_MASK); -+} -+ -+static inline int get_top_task_prop(struct task_struct *p) -+{ -+ return get_hmbird_ts(p)->top_task_prop; -+} -+ -+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear) -+{ -+ if (set) -+ get_hmbird_ts(p)->top_task_prop |= set; -+ if (clear) -+ get_hmbird_ts(p)->top_task_prop &= ~clear; -+ return 0; -+} -+ -+static inline void reset_top_task_prop(struct task_struct *p) -+{ -+ get_hmbird_ts(p)->top_task_prop = 0; -+} -+ -+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ entity->sched_prop = sp; -+ } -+ return 0; -+} -+ -+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->sched_prop; -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq) -+{ -+ unsigned long new_dsq; -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq; -+ entity->sched_prop = new_dsq; -+ } -+} -+ -+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK); -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ entity->dsq_sync_ux = val; -+} -+ -+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->dsq_sync_ux; -+ else -+ return 0; -+} -+#endif -+ - extern int can_nice(const struct task_struct *p, const int nice); - extern int task_curr(const struct task_struct *p); - extern int idle_cpu(int cpu); -diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h -deleted file mode 100755 -index b737a00c1373..000000000000 ---- a/include/linux/sched/ext.h -+++ /dev/null -@@ -1,647 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef _LINUX_SCHED_EXT_H --#define _LINUX_SCHED_EXT_H -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --#include -- --enum scx_consts { -- SCX_OPS_NAME_LEN = 128, -- SCX_EXIT_REASON_LEN = 128, -- SCX_EXIT_BT_LEN = 64, -- SCX_EXIT_MSG_LEN = 1024, -- -- SCX_SLICE_DFL = 20 * NSEC_PER_MSEC, -- SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */ --}; -- --/* -- * DSQ (dispatch queue) IDs are 64bit of the format: -- * -- * Bits: [63] [62 .. 0] -- * [ B] [ ID ] -- * -- * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -- * ID: 63 bit ID -- * -- * Built-in IDs: -- * -- * Bits: [63] [62] [61..32] [31 .. 0] -- * [ 1] [ L] [ R ] [ V ] -- * -- * 1: 1 for built-in DSQs. -- * L: 1 for LOCAL_ON DSQ IDs, 0 for others -- * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -- */ --enum scx_dsq_id_flags { -- SCX_DSQ_FLAG_BUILTIN = 1LLU << 63, -- SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -- -- SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0, -- SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, -- SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, -- SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, -- SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, --}; -- --enum scx_exit_type { -- SCX_EXIT_NONE, -- SCX_EXIT_DONE, -- -- SCX_EXIT_UNREG = 64, /* BPF unregistration */ -- SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -- SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ -- SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ --}; -- --/* -- * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is -- * being disabled. -- */ --struct scx_exit_info { -- /* %SCX_EXIT_* - broad category of the exit reason */ -- enum scx_exit_type type; -- /* textual representation of the above */ -- char reason[SCX_EXIT_REASON_LEN]; -- /* number of entries in the backtrace */ -- u32 bt_len; -- /* backtrace if exiting due to an error */ -- unsigned long bt[SCX_EXIT_BT_LEN]; -- /* extra message */ -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --/* sched_ext_ops.flags */ --enum scx_ops_flags { -- /* -- * Keep built-in idle tracking even if ops.update_idle() is implemented. -- */ -- SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, -- -- /* -- * By default, if there are no other task to run on the CPU, ext core -- * keeps running the current task even after its slice expires. If this -- * flag is specified, such tasks are passed to ops.enqueue() with -- * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. -- */ -- SCX_OPS_ENQ_LAST = 1LLU << 1, -- -- /* -- * An exiting task may schedule after PF_EXITING is set. In such cases, -- * bpf_task_from_pid() may not be able to find the task and if the BPF -- * scheduler depends on pid lookup for dispatching, the task will be -- * lost leading to various issues including RCU grace period stalls. -- * -- * To mask this problem, by default, unhashed tasks are automatically -- * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't -- * depend on pid lookups and wants to handle these tasks directly, the -- * following flag can be used. -- */ -- SCX_OPS_ENQ_EXITING = 1LLU << 2, -- -- /* -- * CPU cgroup knob enable flags -- */ -- SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16, /* cpu.weight */ -- -- SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | -- SCX_OPS_ENQ_LAST | -- SCX_OPS_ENQ_EXITING | -- SCX_OPS_CGROUP_KNOB_WEIGHT, --}; -- --/* argument container for ops.enable() and friends */ --struct scx_enable_args { --}; -- --/* argument container for ops->cgroup_init() */ --struct scx_cgroup_init_args { -- /* the weight of the cgroup [1..10000] */ -- u32 weight; --}; -- --enum scx_cpu_preempt_reason { -- /* next task is being scheduled by &sched_class_rt */ -- SCX_CPU_PREEMPT_RT, -- /* next task is being scheduled by &sched_class_dl */ -- SCX_CPU_PREEMPT_DL, -- /* next task is being scheduled by &sched_class_stop */ -- SCX_CPU_PREEMPT_STOP, -- /* unknown reason for SCX being preempted */ -- SCX_CPU_PREEMPT_UNKNOWN, --}; -- --/* -- * Argument container for ops->cpu_acquire(). Currently empty, but may be -- * expanded in the future. -- */ --struct scx_cpu_acquire_args {}; -- --/* argument container for ops->cpu_release() */ --struct scx_cpu_release_args { -- /* the reason the CPU was preempted */ -- enum scx_cpu_preempt_reason reason; -- -- /* the task that's going to be scheduled on the CPU */ -- struct task_struct *task; --}; -- --/** -- * struct sched_ext_ops - Operation table for BPF scheduler implementation -- * -- * Userland can implement an arbitrary scheduling policy by implementing and -- * loading operations in this table. -- */ --struct sched_ext_ops { -- /** -- * select_cpu - Pick the target CPU for a task which is being woken up -- * @p: task being woken up -- * @prev_cpu: the cpu @p was on before sleeping -- * @wake_flags: SCX_WAKE_* -- * -- * Decision made here isn't final. @p may be moved to any CPU while it -- * is getting dispatched for execution later. However, as @p is not on -- * the rq at this point, getting the eventual execution CPU right here -- * saves a small bit of overhead down the line. -- * -- * If an idle CPU is returned, the CPU is kicked and will try to -- * dispatch. While an explicit custom mechanism can be added, -- * select_cpu() serves as the default way to wake up idle CPUs. -- */ -- s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); -- -- /** -- * enqueue - Enqueue a task on the BPF scheduler -- * @p: task being enqueued -- * @enq_flags: %SCX_ENQ_* -- * -- * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() -- * or enqueue on the BPF scheduler. If not directly dispatched, the bpf -- * scheduler owns @p and if it fails to dispatch @p, the task will -- * stall. -- */ -- void (*enqueue)(struct task_struct *p, u64 enq_flags); -- -- /** -- * dequeue - Remove a task from the BPF scheduler -- * @p: task being dequeued -- * @deq_flags: %SCX_DEQ_* -- * -- * Remove @p from the BPF scheduler. This is usually called to isolate -- * the task while updating its scheduling properties (e.g. priority). -- * -- * The ext core keeps track of whether the BPF side owns a given task or -- * not and can gracefully ignore spurious dispatches from BPF side, -- * which makes it safe to not implement this method. However, depending -- * on the scheduling logic, this can lead to confusing behaviors - e.g. -- * scheduling position not being updated across a priority change. -- */ -- void (*dequeue)(struct task_struct *p, u64 deq_flags); -- -- /** -- * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs -- * @cpu: CPU to dispatch tasks for -- * @prev: previous task being switched out -- * -- * Called when a CPU's local dsq is empty. The operation should dispatch -- * one or more tasks from the BPF scheduler into the DSQs using -- * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using -- * scx_bpf_consume(). -- * -- * The maximum number of times scx_bpf_dispatch() can be called without -- * an intervening scx_bpf_consume() is specified by -- * ops.dispatch_max_batch. See the comments on top of the two functions -- * for more details. -- * -- * When not %NULL, @prev is an SCX task with its slice depleted. If -- * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in -- * @prev->scx.flags, it is not enqueued yet and will be enqueued after -- * ops.dispatch() returns. To keep executing @prev, return without -- * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. -- */ -- void (*dispatch)(s32 cpu, struct task_struct *prev); -- -- /** -- * runnable - A task is becoming runnable on its associated CPU -- * @p: task becoming runnable -- * @enq_flags: %SCX_ENQ_* -- * -- * This and the following three functions can be used to track a task's -- * execution state transitions. A task becomes ->runnable() on a CPU, -- * and then goes through one or more ->running() and ->stopping() pairs -- * as it runs on the CPU, and eventually becomes ->quiescent() when it's -- * done running on the CPU. -- * -- * @p is becoming runnable on the CPU because it's -- * -- * - waking up (%SCX_ENQ_WAKEUP) -- * - being moved from another CPU -- * - being restored after temporarily taken off the queue for an -- * attribute change. -- * -- * This and ->enqueue() are related but not coupled. This operation -- * notifies @p's state transition and may not be followed by ->enqueue() -- * e.g. when @p is being dispatched to a remote CPU. Likewise, a task -- * may be ->enqueue()'d without being preceded by this operation e.g. -- * after exhausting its slice. -- */ -- void (*runnable)(struct task_struct *p, u64 enq_flags); -- -- /** -- * running - A task is starting to run on its associated CPU -- * @p: task starting to run -- * -- * See ->runnable() for explanation on the task state notifiers. -- */ -- void (*running)(struct task_struct *p); -- -- /** -- * stopping - A task is stopping execution -- * @p: task stopping to run -- * @runnable: is task @p still runnable? -- * -- * See ->runnable() for explanation on the task state notifiers. If -- * !@runnable, ->quiescent() will be invoked after this operation -- * returns. -- */ -- void (*stopping)(struct task_struct *p, bool runnable); -- -- /** -- * quiescent - A task is becoming not runnable on its associated CPU -- * @p: task becoming not runnable -- * @deq_flags: %SCX_DEQ_* -- * -- * See ->runnable() for explanation on the task state notifiers. -- * -- * @p is becoming quiescent on the CPU because it's -- * -- * - sleeping (%SCX_DEQ_SLEEP) -- * - being moved to another CPU -- * - being temporarily taken off the queue for an attribute change -- * (%SCX_DEQ_SAVE) -- * -- * This and ->dequeue() are related but not coupled. This operation -- * notifies @p's state transition and may not be preceded by ->dequeue() -- * e.g. when @p is being dispatched to a remote CPU. -- */ -- void (*quiescent)(struct task_struct *p, u64 deq_flags); -- -- /** -- * yield - Yield CPU -- * @from: yielding task -- * @to: optional yield target task -- * -- * If @to is NULL, @from is yielding the CPU to other runnable tasks. -- * The BPF scheduler should ensure that other available tasks are -- * dispatched before the yielding task. Return value is ignored in this -- * case. -- * -- * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf -- * scheduler can implement the request, return %true; otherwise, %false. -- */ -- bool (*yield)(struct task_struct *from, struct task_struct *to); -- -- /** -- * core_sched_before - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Used by core-sched to determine the ordering between two tasks. See -- * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on -- * core-sched. -- * -- * Both @a and @b are runnable and may or may not currently be queued on -- * the BPF scheduler. Should return %true if @a should run before @b. -- * %false if there's no required ordering or @b should run before @a. -- * -- * If not specified, the default is ordering them according to when they -- * became runnable. -- */ -- bool (*core_sched_before)(struct task_struct *a,struct task_struct *b); -- -- /** -- * set_weight - Set task weight -- * @p: task to set weight for -- * @weight: new eight [1..10000] -- * -- * Update @p's weight to @weight. -- */ -- void (*set_weight)(struct task_struct *p, u32 weight); -- -- /** -- * set_cpumask - Set CPU affinity -- * @p: task to set CPU affinity for -- * @cpumask: cpumask of cpus that @p can run on -- * -- * Update @p's CPU affinity to @cpumask. -- */ -- void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask); -- -- /** -- * update_idle - Update the idle state of a CPU -- * @cpu: CPU to udpate the idle state for -- * @idle: whether entering or exiting the idle state -- * -- * This operation is called when @rq's CPU goes or leaves the idle -- * state. By default, implementing this operation disables the built-in -- * idle CPU tracking and the following helpers become unavailable: -- * -- * - scx_bpf_select_cpu_dfl() -- * - scx_bpf_test_and_clear_cpu_idle() -- * - scx_bpf_pick_idle_cpu() -- * - scx_bpf_any_idle_cpu() -- * -- * The user also must implement ops.select_cpu() as the default -- * implementation relies on scx_bpf_select_cpu_dfl(). -- * -- * If you keep the built-in idle tracking, specify the -- * %SCX_OPS_KEEP_BUILTIN_IDLE flag. -- */ -- void (*update_idle)(s32 cpu, bool idle); -- -- /** -- * cpu_acquire - A CPU is becoming available to the BPF scheduler -- * @cpu: The CPU being acquired by the BPF scheduler. -- * @args: Acquire arguments, see the struct definition. -- * -- * A CPU that was previously released from the BPF scheduler is now once -- * again under its control. -- */ -- void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); -- -- /** -- * cpu_release - A CPU is taken away from the BPF scheduler -- * @cpu: The CPU being released by the BPF scheduler. -- * @args: Release arguments, see the struct definition. -- * -- * The specified CPU is no longer under the control of the BPF -- * scheduler. This could be because it was preempted by a higher -- * priority sched_class, though there may be other reasons as well. The -- * caller should consult @args->reason to determine the cause. -- */ -- void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -- -- /** -- * cpu_online - A CPU became online -- * @cpu: CPU which just came up -- * -- * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs beforehand. -- */ -- void (*cpu_online)(s32 cpu); -- -- /** -- * cpu_offline - A CPU is going offline -- * @cpu: CPU which is going offline -- * -- * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs afterwards. -- */ -- void (*cpu_offline)(s32 cpu); -- -- /** -- * prep_enable - Prepare to enable BPF scheduling for a task -- * @p: task to prepare BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Either we're loading a BPF scheduler or a new task is being forked. -- * Prepare BPF scheduling for @p. This operation may block and can be -- * used for allocations. -- * -- * Return 0 for success, -errno for failure. An error return while -- * loading will abort loading of the BPF scheduler. During a fork, will -- * abort the specific fork. -- */ -- s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * enable - Enable BPF scheduling for a task -- * @p: task to enable BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Enable @p for BPF scheduling. @p is now in the cgroup specified for -- * the preceding prep_enable() and will start running soon. -- */ -- void (*enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * cancel_enable - Cancel prep_enable() -- * @p: task being canceled -- * @args: enable arguments, see the struct definition -- * -- * @p was prep_enable()'d but failed before reaching enable(). Undo the -- * preparation. -- */ -- void (*cancel_enable)(struct task_struct *p, -- struct scx_enable_args *args); -- -- /** -- * disable - Disable BPF scheduling for a task -- * @p: task to disable BPF scheduling for -- * -- * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. -- * Disable BPF scheduling for @p. -- */ -- void (*disable)(struct task_struct *p); -- -- /* -- * All online ops must come before ops.init(). -- */ -- -- /** -- * init - Initialize the BPF scheduler -- */ -- s32 (*init)(void); -- -- /** -- * exit - Clean up after the BPF scheduler -- * @info: Exit info -- */ -- void (*exit)(struct scx_exit_info *info); -- -- /** -- * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch -- */ -- u32 dispatch_max_batch; -- -- /** -- * flags - %SCX_OPS_* flags -- */ -- u64 flags; -- -- /** -- * timeout_ms - The maximum amount of time, in milliseconds, that a -- * runnable task should be able to wait before being scheduled. The -- * maximum timeout may not exceed the default timeout of 30 seconds. -- * -- * Defaults to the maximum allowed timeout value of 30 seconds. -- */ -- u32 timeout_ms; -- -- /** -- * name - BPF scheduler's name -- * -- * Must be a non-zero valid BPF object name including only isalnum(), -- * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the -- * BPF scheduler is enabled. -- */ -- char name[SCX_OPS_NAME_LEN]; --}; -- --/* -- * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -- * scheduler core and the BPF scheduler. See the documentation for more details. -- */ --struct scx_dispatch_q { -- raw_spinlock_t lock; -- struct list_head fifo; /* processed in dispatching order */ -- struct rb_root_cached priq; /* processed in p->scx.dsq_vtime order */ -- u32 nr; -- u64 id; -- struct llist_node free_node; -- struct rcu_head rcu; --}; -- --/* scx_entity.flags */ --enum scx_ent_flags { -- SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ -- SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -- SCX_TASK_ENQ_LOCAL = 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */ -- -- SCX_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -- SCX_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -- -- SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -- SCX_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -- -- SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ --}; -- --/* scx_entity.dsq_flags */ --enum scx_ent_dsq_flags { -- SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ --}; -- --/* -- * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from -- * everywhere and the following bits track which kfunc sets are currently -- * allowed for %current. This simple per-task tracking works because SCX ops -- * nest in a limited way. BPF will likely implement a way to allow and disallow -- * kfuncs depending on the calling context which will replace this manual -- * mechanism. See scx_kf_allow(). -- */ --enum scx_kf_mask { -- SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ -- /* all non-sleepables may be nested inside INIT and SLEEPABLE */ -- SCX_KF_INIT = 1 << 0, /* running ops.init() */ -- SCX_KF_SLEEPABLE = 1 << 1, /* other sleepable init operations */ -- /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ -- SCX_KF_CPU_RELEASE = 1 << 2, /* ops.cpu_release() */ -- /* ops.dequeue (in REST) may be nested inside DISPATCH */ -- SCX_KF_DISPATCH = 1 << 3, /* ops.dispatch() */ -- SCX_KF_ENQUEUE = 1 << 4, /* ops.enqueue() */ -- SCX_KF_REST = 1 << 5, /* other rq-locked operations */ -- -- __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | -- SCX_KF_ENQUEUE | SCX_KF_REST, -- __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_REST, --}; -- --#define RAVG_HIST_SIZE 5 --struct scx_sched_task_stats { -- u64 mark_start; -- u64 window_start; -- u32 sum; -- u32 sum_history[RAVG_HIST_SIZE]; -- int cidx; -- -- u32 demand; -- u16 demand_scaled; -- void *sdsq; --}; -- -- -- --/* -- * The following is embedded in task_struct and contains all fields necessary -- * for a task to be scheduled by SCX. -- */ --struct sched_ext_entity { -- struct scx_dispatch_q *dsq; -- struct { -- struct list_head fifo; /* dispatch order */ -- struct rb_node priq; /* p->scx.dsq_vtime order */ -- } dsq_node; -- struct list_head watchdog_node; -- u32 flags; /* protected by rq lock */ -- u32 dsq_flags; /* protected by dsq lock */ -- u32 weight; -- s32 sticky_cpu; -- s32 holding_cpu; -- u32 kf_mask; /* see scx_kf_mask above */ -- struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ -- atomic64_t ops_state; -- unsigned long runnable_at; --#ifdef CONFIG_SCHED_CORE -- u64 core_sched_at; /* see scx_prio_less() */ --#endif -- -- /* BPF scheduler modifiable fields */ -- -- /* -- * Runtime budget in nsecs. This is usually set through -- * scx_bpf_dispatch() but can also be modified directly by the BPF -- * scheduler. Automatically decreased by SCX as the task executes. On -- * depletion, a scheduling event is triggered. -- * -- * This value is cleared to zero if the task is preempted by -- * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the -- * task ran. Use p->se.sum_exec_runtime instead. -- */ -- u64 slice; -- -- /* -- * Used to order tasks when dispatching to the vtime-ordered priority -- * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime() -- * but can also be modified directly by the BPF scheduler. Modifying it -- * while a task is queued on a dsq may mangle the ordering and is not -- * recommended. -- */ -- u64 dsq_vtime; -- -- /* -- * If set, reject future sched_setscheduler(2) calls updating the policy -- * to %SCHED_EXT with -%EACCES. -- * -- * If set from ops.prep_enable() and the task's policy is already -- * %SCHED_EXT, which can happen while the BPF scheduler is being loaded -- * or by inhering the parent's policy during fork, the task's policy is -- * rejected and forcefully reverted to %SCHED_NORMAL. The number of such -- * events are reported through /sys/kernel/debug/sched_ext::nr_rejected. -- */ -- bool disallow; /* reject switching into SCX */ -- -- /* cold fields */ -- struct list_head tasks_node; -- struct task_struct *task; -- struct scx_sched_task_stats sts; --}; -- --void sched_ext_free(struct task_struct *p); -- --#else /* !CONFIG_SCHED_CLASS_EXT */ -- --static inline void sched_ext_free(struct task_struct *p) {} -- --#endif /* CONFIG_SCHED_CLASS_EXT */ --#endif /* _LINUX_SCHED_EXT_H */ -diff --git a/include/linux/sched/hmbird.h b/include/linux/sched/hmbird.h -new file mode 100755 -index 000000000000..3c7861e37ade ---- /dev/null -+++ b/include/linux/sched/hmbird.h -@@ -0,0 +1,381 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class: Documentation/scheduler/hmbird.rst -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef _LINUX_SCHED_HMBIRD_H -+#define _LINUX_SCHED_HMBIRD_H -+ -+#define HMBIRD_TS_IDX 1 -+#define HMBIRD_OPS_IDX 14 -+#define HMBIRD_RQ_IDX 15 -+ -+#define get_hmbird_ts(p) \ -+ ((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ -+#define get_hmbird_rq(rq) \ -+ ((struct hmbird_rq *)(rq->android_oem_data1[HMBIRD_RQ_IDX])) -+ -+#define get_hmbird_ops(rq) \ -+ ((struct hmbird_ops *)(rq->android_oem_data1[HMBIRD_OPS_IDX])) -+ -+#define SCHED_PROP_DEADLINE_MASK (0xFF) /* deadline for ext sched class */ -+/* -+ * Every task has a DEADLINE_LEVEL which stands for -+ * max schedule latency this task can afford. LEVEL1~5 -+ * for user-aware tasks, LEVEL6~9 for other tasks. -+ */ -+#define SCHED_PROP_DEADLINE_LEVEL0 (0) -+#define SCHED_PROP_DEADLINE_LEVEL1 (1) -+#define SCHED_PROP_DEADLINE_LEVEL2 (2) -+#define SCHED_PROP_DEADLINE_LEVEL3 (3) -+#define SCHED_PROP_DEADLINE_LEVEL4 (4) -+#define SCHED_PROP_DEADLINE_LEVEL5 (5) -+#define SCHED_PROP_DEADLINE_LEVEL6 (6) -+#define SCHED_PROP_DEADLINE_LEVEL7 (7) -+#define SCHED_PROP_DEADLINE_LEVEL8 (8) -+#define SCHED_PROP_DEADLINE_LEVEL9 (9) -+/* -+ * Distinguish tasks into periodical tasks which requires -+ * low schedule latency and non-periodical tasks which are -+ * not sensitive to schedule latency. -+ */ -+#define SCHED_HMBIRD_DSQ_TYPE_PERIOD (0) /* period dsq of hmbird */ -+#define SCHED_HMBIRD_DSQ_TYPE_NON_PERIOD (1) /* non period dsq of hmbird */ -+ -+#define TOP_TASK_BITS_MASK (0xFF) -+#define TOP_TASK_BITS (8) -+#include -+ -+extern atomic_t non_hmbird_task; -+extern atomic_t __hmbird_ops_enabled; -+#define hmbird_enabled() atomic_read(&__hmbird_ops_enabled) -+#define MAX_GLOBAL_DSQS (10) -+ -+enum hmbird_consts { -+ HMBIRD_SLICE_DFL = 1 * NSEC_PER_MSEC, -+ HMBIRD_SLICE_ISO = 8 * HMBIRD_SLICE_DFL, -+ HMBIRD_SLICE_INF = U64_MAX, /* infinite, implies nohz */ -+}; -+ -+/* -+ * DSQ (dispatch queue) IDs are 64bit of the format: -+ * -+ * Bits: [63] [62 .. 0] -+ * [ B] [ ID ] -+ * -+ * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -+ * ID: 63 bit ID -+ * -+ * Built-in IDs: -+ * -+ * Bits: [63] [62] [61..32] [31 .. 0] -+ * [ 1] [ L] [ R ] [ V ] -+ * -+ * 1: 1 for built-in DSQs. -+ * L: 1 for LOCAL_ON DSQ IDs, 0 for others -+ * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -+ */ -+enum hmbird_dsq_id_flags { -+ HMBIRD_DSQ_FLAG_BUILTIN = 1LLU << 63, -+ HMBIRD_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -+ -+ HMBIRD_DSQ_INVALID = HMBIRD_DSQ_FLAG_BUILTIN | 0, -+ HMBIRD_DSQ_GLOBAL = HMBIRD_DSQ_FLAG_BUILTIN | 1, -+ HMBIRD_DSQ_LOCAL = HMBIRD_DSQ_FLAG_BUILTIN | 2, -+ HMBIRD_DSQ_LOCAL_ON = HMBIRD_DSQ_FLAG_BUILTIN | HMBIRD_DSQ_FLAG_LOCAL_ON, -+ HMBIRD_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, -+}; -+ -+enum hmbird_switch_type { -+ HMBIRD_SWITCH_PROC, -+ HMBIRD_SWITCH_ERR_WDT, -+ HMBIRD_SWITCH_ERR_HB, -+ HMBIRD_SWITCH_ERR_DSQ, -+ HMBIRD_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ -+ HMBIRD_EXIT_ERROR_HEARTBEAT, /* heart beat has stopped */ -+}; -+ -+/* -+ * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -+ * scheduler core and the BPF scheduler. See the documentation for more details. -+ */ -+struct hmbird_dispatch_q { -+ raw_spinlock_t lock; -+ struct list_head fifo; /* processed in dispatching order */ -+ struct rb_root_cached priq; -+ u32 nr; -+ u64 id; -+ struct llist_node free_node; -+ struct rcu_head rcu; -+ u64 last_consume_at; -+ bool is_timeout; -+}; -+ -+/* hmbird_entity.flags */ -+enum hmbird_ent_flags { -+ HMBIRD_TASK_QUEUED = 1 << 0, /* on hmbird runqueue */ -+ HMBIRD_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -+ HMBIRD_TASK_ENQ_LOCAL = 1 << 2, /* used by hmbird_select_cpu_dfl, set HMBIRD_ENQ_LOCAL */ -+ -+ HMBIRD_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -+ HMBIRD_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -+ -+ HMBIRD_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -+ HMBIRD_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -+ -+ HMBIRD_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ -+}; -+ -+/* hmbird_entity.dsq_flags */ -+enum hmbird_ent_dsq_flags { -+ HMBIRD_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ -+}; -+ -+#define RAVG_HIST_SIZE 5 -+struct hmbird_sched_task_stats { -+ u64 mark_start; -+ u64 window_start; -+ u32 sum; -+ u32 sum_history[RAVG_HIST_SIZE]; -+ int cidx; -+ -+ u32 demand; -+ u16 demand_scaled; -+ void *sdsq; -+}; -+ -+struct hmbird_sched_rq_stats { -+ u64 window_start; -+ u64 latest_clock; -+ u32 prev_window_size; -+ u64 task_exec_scale; -+ u64 prev_runnable_sum; -+ u64 curr_runnable_sum; -+ int *sched_ravg_window_ptr; -+}; -+ -+/* -+ * The following is embedded in task_struct and contains all fields necessary -+ * for a task to be scheduled by HMBIRD. -+ */ -+struct hmbird_entity { -+ struct hmbird_dispatch_q *dsq; -+ struct { -+ struct list_head fifo; /* dispatch order */ -+ struct rb_node priq; -+ } dsq_node; -+ struct list_head watchdog_node; -+ u32 flags; /* protected by rq lock */ -+ u32 dsq_flags; /* protected by dsq lock */ -+ u32 weight; -+ s32 sticky_cpu; -+ s32 holding_cpu; -+ u32 kf_mask; /* see hmbird_kf_mask above */ -+ struct task_struct *kf_tasks[2]; /* see HMBIRD_CALL_OP_TASK() */ -+ atomic64_t ops_state; -+ unsigned long runnable_at; -+ u64 slice; -+ u64 dsq_vtime; -+ bool disallow; /* reject switching into HMBIRD */ -+ u16 demand_scaled; -+ -+ /* cold fields */ -+ struct list_head tasks_node; -+ struct task_struct *task; -+ const struct sched_class *sched_class; -+ unsigned long sched_prop; -+ unsigned long top_task_prop; -+ struct hmbird_sched_task_stats sts; -+ unsigned long running_at; -+ int gdsq_idx; -+ -+ s32 critical_affinity_cpu; -+ int dsq_sync_ux; -+}; -+ -+ -+/* -+ * All variables use 64bits width, -+ * Avoid parsing problems caused by automatic alignment(with padding) of structures. -+ */ -+ -+/* NOTING : Must align to 64bits. */ -+#define DESC_STR_LEN (32) -+/* Supporti up to three-dimensional arrays. */ -+#define PARSE_DIMENS (3) -+struct meta_desc_t { -+ char desc_str[DESC_STR_LEN]; -+ u64 len; -+ u64 parse[PARSE_DIMENS]; -+}; -+ -+#define MAX_SWITCHS (5) -+struct hmbird_switch_t { -+ u64 switch_at; -+ u64 is_success; -+ u64 end_state; -+ u64 switch_reason; -+}; -+#define SWITCH_ITEMS (sizeof(struct hmbird_switch_t) / sizeof(u64)) -+ -+#define MAX_EXCEPS (5) -+enum excep_id { -+ NO_CGROUP_L1, -+ MODULE_UNLOAD, -+ ALREADY_ENABLED, -+ ALREADY_DISABLED, -+ INIT_TASK_FAIL, -+ ALLOC_RQSCX_FAIL, -+ DSQ_ID_ERR, -+ CPU_NO_MASK, -+ SCAN_ENTITY_NULL, -+ ITER_RET_NULL, -+ DEQ_DEQING, -+ ENQ_EXIST1, -+ ENQ_EXIST2, -+ TASK_LINKED1, -+ TASK_UNLINKED, -+ TASK_LINKED2, -+ TASK_UNQUED, -+ TASK_UNWATCHED, -+ HMBIRD_OPN, -+ TASK_WATCHED, -+ RQ_NO_RUNNING, -+ EXTRA_FLAGS, -+ HOLDING_CPU1, -+ HOLDING_CPU2, -+ TASK_OPS_PREPPED, -+ TASK_OPS_UNPREPPED, -+ HMBIRD_OPS_ERR, -+ MAX_EXCEP_ID, -+}; -+ -+struct snap_misc_t { -+ u64 hmbird_enabled; -+ u64 curr_ss; -+ u64 hmbird_ops_enable_state_var; -+ u64 non_ext_task; -+ u64 parctrl_high_ratio; -+ u64 parctrl_low_ratio; -+ u64 parctrl_high_ratio_l; -+ u64 parctrl_low_ratio_l; -+ u64 isoctrl_high_ratio; -+ u64 isoctrl_low_ratio; -+ u64 misfit_ds; -+ u64 partial_enable; -+ u64 iso_free_rescue; -+ u64 isolate_ctrl; -+ u64 snap_jiffies; -+ u64 snap_time; -+}; -+#define SNAP_ITEMS (sizeof(struct snap_misc_t) / sizeof(u64)) -+ -+struct panic_snapshot_t { -+ /* what time dose the first task of dsq turn in runnable, check starvation */ -+ struct meta_desc_t runnable_at_meta; -+ u64 runnable_at[MAX_GLOBAL_DSQS]; -+ -+ struct meta_desc_t rq_nr_meta; -+ u64 rq_nr[NR_CPUS]; -+ -+ struct meta_desc_t scxrq_nr_meta; -+ u64 scxrq_nr[NR_CPUS]; -+ -+ struct meta_desc_t snap_misc_meta; -+ struct snap_misc_t snap_misc; -+}; -+ -+/* -+ * Do not record info in hot paths unless absolutely necessary, -+ * The impact on performance should be minimized. -+ */ -+struct kernel_info_t { -+ struct meta_desc_t sw_rec_meta; -+ struct hmbird_switch_t sw_rec[MAX_SWITCHS]; -+ -+ struct meta_desc_t sw_idx_meta; -+ u64 sw_idx; -+ -+ struct meta_desc_t excep_rec_meta; -+ u64 excep_rec[MAX_EXCEP_ID][MAX_EXCEPS]; -+ -+ struct meta_desc_t excep_idx_meta; -+ u64 excep_idx[MAX_EXCEP_ID]; -+ -+ /* snapshot while panic. */ -+ struct panic_snapshot_t snap; -+}; -+ -+struct ko_info_t {}; -+ -+struct md_meta_t { -+ u64 self_len; -+ u64 unit_size; -+ u64 desc_meta_len; -+ u64 desc_str_len; -+ u64 switches; -+ u64 exceps; -+ u64 global_dsqs; -+ u64 parse_dimens; -+ u64 nr_cpus; -+ u64 real_cpus; -+ u64 nr_meta_desc; -+ u64 dump_real_size; -+}; -+ -+struct md_info_t { -+ struct md_meta_t meta; -+ struct kernel_info_t kern_dump; -+ struct ko_info_t ko_dump; -+}; -+ -+static inline void exceps_update(struct md_info_t *rec, int id, unsigned long jiffies) -+{ -+ u64 *idx; -+ -+ if (!rec) -+ return; -+ -+ idx = &rec->kern_dump.excep_idx[id]; -+ rec->kern_dump.excep_rec[id][*idx] = jiffies; -+ *idx = ++(*idx) % MAX_EXCEPS; -+} -+ -+static inline void sw_update(struct md_info_t *rec, u64 switch_at, -+ u64 is_success, u64 end_state, u64 switch_reason) -+{ -+ u64 *idx; -+ -+ if (!rec) -+ return; -+ -+ idx = &rec->kern_dump.sw_idx; -+ rec->kern_dump.sw_rec[*idx].switch_at = switch_at; -+ rec->kern_dump.sw_rec[*idx].is_success = is_success; -+ rec->kern_dump.sw_rec[*idx].end_state = end_state; -+ rec->kern_dump.sw_rec[*idx].switch_reason = switch_reason; -+ *idx = ++(*idx) % MAX_SWITCHS; -+} -+ -+struct hmbird_ops { -+ bool (*scx_enable)(void); -+ bool (*check_non_task)(void); -+ void (*do_sched_yield_before)(long *skip); -+ void (*window_rollover_run_once)(struct rq *rq); -+ void (*hmbird_get_md_info)(unsigned long *vaddr, unsigned long *size); -+}; -+ -+void hmbird_free(struct task_struct *p); -+ -+enum DSQ_SYNC_UX_FLAG { -+ DSQ_SYNC_UX_NONE = 0, -+ DSQ_SYNC_STATIC_UX = 1, -+ DSQ_SYNC_INHERIT_UX = 1 << 1, -+}; -+ -+#endif /* _LINUX_SCHED_HMBIRD_H */ -diff --git a/include/linux/sched/hmbird_proc_val.h b/include/linux/sched/hmbird_proc_val.h -new file mode 100755 -index 000000000000..e59708b16ea3 ---- /dev/null -+++ b/include/linux/sched/hmbird_proc_val.h -@@ -0,0 +1,22 @@ -+#ifndef __HMBORD_PROC_VAL_H__ -+#define __HMBORD_PROC_VAL_H__ -+ -+extern int scx_enable; -+extern int partial_enable; -+extern int cpuctrl_high_ratio; -+extern int cpuctrl_low_ratio; -+extern int slim_stats; -+extern int hmbirdcore_debug; -+extern int slim_for_app; -+extern int misfit_ds; -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+extern int cpu7_tl; -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int slim_gov_debug; -+extern int scx_gov_ctrl; -+extern int sched_ravg_window_frame_per_sec; -+ -+#endif -\ No newline at end of file -diff --git a/include/linux/sched/hmbird_version.h b/include/linux/sched/hmbird_version.h -new file mode 100755 -index 000000000000..b2843881c26f ---- /dev/null -+++ b/include/linux/sched/hmbird_version.h -@@ -0,0 +1,52 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#ifndef _OPLUS_HMBIRD_VERSION_H_ -+#define _OPLUS_HMBIRD_VERSION_H_ -+#include -+#include -+#include -+ -+enum hmbird_version { -+ HMBIRD_UNINIT, -+ HMBIRD_GKI_VERSION, -+ HMBIRD_OGKI_VERSION, -+ HMBIRD_UNKNOW_VERSION, -+}; -+ -+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT; -+ -+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type" -+ -+static inline enum hmbird_version get_hmbird_version_type(void) -+{ -+ struct device_node *np = NULL; -+ const char *hmbird_version_str = NULL; -+ if (HMBIRD_UNINIT != hmbird_version_type) -+ return hmbird_version_type; -+ np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH); -+ if (np) { -+ of_property_read_string(np, "type", &hmbird_version_str); -+ if (NULL != hmbird_version_str) { -+ if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) { -+ hmbird_version_type = HMBIRD_OGKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi"); -+ } else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) { -+ hmbird_version_type = HMBIRD_GKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi"); -+ } else { -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi"); -+ } -+ return hmbird_version_type; -+ } -+ } -+ -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION"); -+ return hmbird_version_type; -+} -+ -+#endif /*_OPLUS_HMBIRD_VERSION_H_ */ -diff --git a/include/linux/sched/sa_common.h b/include/linux/sched/sa_common.h -new file mode 100755 -index 000000000000..6f76d7cfd7bf ---- /dev/null -+++ b/include/linux/sched/sa_common.h -@@ -0,0 +1,797 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_COMMON_H_ -+#define _OPLUS_SA_COMMON_H_ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+#include -+#endif -+#include -+#include -+#include -+#include -+#include -+ -+#include "sa_oemdata.h" -+#include "sa_common_struct.h" -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) -+#define OPLUS_UX_EEVDF_COMPATIBLE 1 -+#endif -+ -+#define SA_DEBUG_ON 0 -+ -+#if (SA_DEBUG_ON >= 1) -+#define DEBUG_BUG_ON(x) BUG_ON(x) -+#define DEBUG_WARN_ON(x) WARN_ON(x) -+#else -+#define DEBUG_BUG_ON(x) -+#define DEBUG_WARN_ON(x) -+#endif -+ -+#define ux_err(fmt, ...) \ -+ pr_err("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_warn(fmt, ...) \ -+ pr_warn("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_debug(fmt, ...) \ -+ pr_info("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+ -+#define UX_MSG_LEN 64 -+#define UX_DEPTH_MAX 5 -+ -+/* define for debug */ -+#define DEBUG_SYSTRACE (1 << 0) -+#define DEBUG_FTRACE (1 << 1) -+#define DEBUG_KMSG (1 << 2) /* used for frameboost */ -+#define DEBUG_PIPELINE (1 << 3) -+#define DEBUG_DYNAMIC_HZ (1 << 4) -+#define DEBUG_DYNAMIC_PREEMPT (1 << 5) -+#define DEBUG_AMU_INSTRUCTION (1 << 6) -+#define DEBUG_VERBOSE (1 << 10) /* used for frameboost */ -+#define DEBUG_SET_DSQ_ID (1 << 11) /* used for hmbird */ -+ -+/* define for sched assist feature */ -+#define FEATURE_COMMON (1 << 0) -+#define FEATURE_SPREAD (1 << 1) -+ -+#define UX_EXEC_SLICE (4000000U) -+ -+/* define for sched assist thread type, keep same as the define in java file */ -+#define SA_OPT_CLEAR (0) -+#define SA_TYPE_LIGHT (1 << 0) -+#define SA_TYPE_HEAVY (1 << 1) -+#define SA_TYPE_ANIMATOR (1 << 2) -+/* SA_TYPE_LISTPICK for camera */ -+#define SA_TYPE_LISTPICK (1 << 3) -+#define SA_TYPE_MQ_VIP (1 << 4) -+#define SA_OPT_SET (1 << 7) -+#define SA_OPT_RESET (1 << 8) -+#define SA_OPT_SET_PRIORITY (1 << 9) -+ -+/* The following ux value only used in kernel */ -+#define SA_TYPE_SWIFT (1 << 14) -+/* clear ux type when dequeue */ -+#define SA_TYPE_ONCE (1 << 15) -+#define SA_TYPE_INHERIT (1 << 16) -+/* SA_TYPE_COMBINED isn't marked in ux_state, it only exists in return value of function */ -+#define SA_TYPE_COMBINED (1 << 17) -+#define SA_TYPE_URGENT_MASK (SA_TYPE_LIGHT|SA_TYPE_ANIMATOR|SA_TYPE_SWIFT) -+#define SCHED_ASSIST_UX_MASK (SA_TYPE_LIGHT|SA_TYPE_HEAVY|SA_TYPE_ANIMATOR|SA_TYPE_LISTPICK|SA_TYPE_SWIFT) -+ -+/* load balance operation is performed on the following ux types */ -+#define SCHED_ASSIST_LB_UX (SA_TYPE_SWIFT | SA_TYPE_ANIMATOR | SA_TYPE_LIGHT | SA_TYPE_HEAVY) -+#define POSSIBLE_UX_MASK (SA_TYPE_SWIFT | \ -+ SA_TYPE_LIGHT | \ -+ SA_TYPE_HEAVY | \ -+ SA_TYPE_ANIMATOR | \ -+ SA_TYPE_LISTPICK | \ -+ SA_TYPE_ONCE | \ -+ SA_TYPE_INHERIT) -+ -+#define SCHED_ASSIST_UX_PRIORITY_MASK (0xFF000000) -+#define SCHED_ASSIST_UX_PRIORITY_SHIFT 24 -+ -+#define UX_PRIORITY_TOP_APP 0x0A000000 -+#define UX_PRIORITY_AUDIO 0x0A000000 -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+#define UX_PRIORITY_PIPELINE_UI 0x06000000 -+#define UX_PRIORITY_PIPELINE 0x05000000 -+#endif -+ -+/* define for sched assist scene type, keep same as the define in java file */ -+#define SA_SCENE_OPT_CLEAR (0) -+#define SA_LAUNCH (1 << 0) -+#define SA_SLIDE (1 << 1) -+#define SA_CAMERA (1 << 2) -+#define SA_ANIM_START (1 << 3) /* we care about both launcher and top app */ -+#define SA_ANIM (1 << 4) /* we only care about launcher */ -+#define SA_INPUT (1 << 5) -+#define SA_LAUNCHER_SI (1 << 6) -+#define SA_SCENE_OPT_SET (1 << 7) -+ -+#define ROOT_UID 0 -+#define SYSTEM_UID 1000 -+#define FIRST_APPLICATION_UID 10000 -+#define LAST_APPLICATION_UID 19999 -+#define PER_USER_RANGE 100000 -+/* define for clear IM_FLAG -+if im_flag is 70, it should clear im_flag_audio (70 - 64 = 6) -+eg: gerrit patchset "30438485" -+ */ -+#define IM_FLAG_CLEAR 64 -+ -+extern pid_t save_audio_tgid; -+extern pid_t save_top_app_tgid; -+extern unsigned int top_app_type; -+extern int global_lowend_plat_opt; -+ -+#ifdef CONFIG_OPLUS_SCHED_HALT_MASK_PRT -+/* This must be the same as the definition of pause_type in walt_halt.c */ -+enum oplus_pause_type { -+ OPLUS_HALT, -+ OPLUS_PARTIAL_HALT, -+ -+ OPLUS_MAX_PAUSE_TYPE -+}; -+extern cpumask_t cur_cpus_halt_mask; -+extern cpumask_t cur_cpus_phalt_mask; -+DECLARE_PER_CPU(int[OPLUS_MAX_PAUSE_TYPE], oplus_cur_pause_client); -+extern void sa_corectl_systrace_c(void); -+#endif /* CONFIG_OPLUS_SCHED_HALT_MASK_PRT */ -+ -+/* define for boost threshold unit */ -+#define BOOST_THRESHOLD_UNIT (51) -+ -+ -+enum UX_STATE_TYPE { -+ UX_STATE_INVALID = 0, -+ UX_STATE_NONE, -+ UX_STATE_STATIC, -+ UX_STATE_INHERIT, -+ UX_STATE_COMBINED, -+ MAX_UX_STATE_TYPE, -+}; -+ -+enum INHERIT_UX_TYPE { -+ INHERIT_UX_BINDER = 0, -+ INHERIT_UX_RWSEM, -+ INHERIT_UX_MUTEX, -+ INHERIT_UX_FUTEX, -+ INHERIT_UX_PIFUTEX, -+ INHERIT_UX_MAX, -+}; -+ -+/* -+ * WANNING: -+ * new flag should be add before MAX_IM_FLAG_TYPE, never change -+ * the value of those existed flag type. -+ */ -+enum IM_FLAG_TYPE { -+ IM_FLAG_NONE = 0, -+ IM_FLAG_SURFACEFLINGER, -+ IM_FLAG_HWC, /* Discarded */ -+ IM_FLAG_RENDERENGINE, -+ IM_FLAG_WEBVIEW, -+ IM_FLAG_CAMERA_HAL, -+ IM_FLAG_AUDIO, -+ IM_FLAG_HWBINDER, -+ IM_FLAG_LAUNCHER, -+ IM_FLAG_LAUNCHER_NON_UX_RENDER, -+ IM_FLAG_SS_LOCK_OWNER, -+ IM_FLAG_FORBID_SET_CPU_AFFINITY = 11, /* forbid setting cpu affinity from app */ -+ IM_FLAG_SYSTEMSERVER_PID, -+ IM_FLAG_MIDASD, -+ IM_FLAG_AUDIO_CAMERA_HAL, /* audio mode disable camera hal ux */ -+ IM_FLAG_AFFINITY_THREAD, -+ IM_FLAG_TPD_SET_CPU_AFFINITY = 16, -+ IM_FLAG_COMPRESS_THREAD = 17, /* compress thread skips locking protect */ -+ IM_FLAG_RENDER_THREAD = 18, -+ MAX_IM_FLAG_TYPE, -+}; -+ -+#define MAX_IM_FLAG_PRIO MAX_IM_FLAG_TYPE -+ -+enum ots_state { -+ OTS_STATE_SET_AFFINITY, -+ OTS_STATE_DDL_ACTIVE, -+ OTS_STATE_DDL_ACTIVE_PREEMPTED, -+ OTS_STATE_MAX, -+}; -+ -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+DECLARE_PER_CPU(u64, retired_instrs); -+DECLARE_PER_CPU(u64, nvcsw); -+DECLARE_PER_CPU(u64, nivcsw); -+#endif -+ -+struct ux_sched_cluster { -+ struct cpumask cpus; -+ unsigned long capacity; -+}; -+ -+#define OPLUS_NR_CPUS (8) -+#define OPLUS_MAX_CLS (5) -+struct ux_sched_cputopo { -+ int cls_nr; -+ struct ux_sched_cluster sched_cls[OPLUS_NR_CPUS]; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ cpumask_t oplus_cpu_array[2*OPLUS_MAX_CLS][OPLUS_MAX_CLS]; -+#endif -+}; -+ -+struct oplus_rq { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_root_cached ux_list; -+ /* a tree to track minimum exec vruntime */ -+ struct rb_root_cached exec_timeline; -+ /* malloc this spinlock_t instead of built-in to shrank size of oplus_rq */ -+ spinlock_t *ux_list_lock; -+ int nr_running; -+ u64 min_vruntime; -+ u64 load_weight; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) -+ struct rb_root_cached ddl_root; -+ spinlock_t *ddl_lock; -+#endif -+ -+#ifdef CONFIG_LOCKING_PROTECT -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ struct list_head locking_thread_list; -+ spinlock_t *locking_list_lock; -+ int rq_locking_task; -+#else -+ struct sched_entity *last_entity; -+#endif -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ struct oplus_lb lb; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+ struct hrtimer *resched_timer; -+ int cpu; -+#endif -+}; -+ -+extern int global_debug_enabled; -+extern int global_sched_assist_enabled; -+extern int global_sched_assist_scene; -+extern int global_silver_perf_core; -+extern int global_sched_group_enabled; -+ -+struct rq; -+ -+#ifdef CONFIG_LOCKING_PROTECT -+struct sched_assist_locking_ops { -+ void (*check_preempt_tick)(struct task_struct *p, -+ unsigned long *ideal_runtime, bool *skip_preempt, -+ unsigned long delta_exec, struct cfs_rq *cfs_rq, -+ struct sched_entity *curr, unsigned int granularity); -+ void (*check_preempt_wakeup)(struct rq *rq, struct task_struct *p, bool *preempt, bool *nopreempt); -+ void (*state_systrace_c)(unsigned int cpu, struct task_struct *p); -+ void (*locking_tick_hit)(struct task_struct *prev, struct task_struct *next); -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ void (*replace_next_task_fair)(struct rq *rq, -+ struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*enqueue_entity)(struct rq *rq, struct task_struct *p); -+ void (*dequeue_entity)(struct rq *rq, struct task_struct *p); -+#else -+ void (*set_last_entity)(struct task_struct *prev, struct task_struct *next, struct rq *rq); -+ void (*pick_last_entity)(struct rq *rq, struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*clear_last_entity)(struct rq *rq, struct task_struct *p); -+#endif -+ void (*opt_ss_lock_contention)(struct task_struct *p, int old_im, int new_im); -+}; -+ -+extern struct sched_assist_locking_ops *locking_ops; -+ -+ -+#define LOCKING_CALL_OP(op, args...) \ -+do { \ -+ if (locking_ops && locking_ops->op) { \ -+ locking_ops->op(args); \ -+ } \ -+} while (0) -+ -+#define LOCKING_CALL_OP_RET(op, args...) \ -+({ \ -+ __typeof__(locking_ops->op(args)) __ret = 0; \ -+ if (locking_ops && locking_ops->op) \ -+ __ret = locking_ops->op(args); \ -+ __ret; \ -+}) -+ -+void register_sched_assist_locking_ops(struct sched_assist_locking_ops *ops); -+#endif -+ -+/* attention: before insert .ko, task's list->prev/next is init with 0 */ -+static inline bool oplus_list_empty(struct list_head *list) -+{ -+ return list_empty(list) || (list->prev == 0 && list->next == 0); -+} -+ -+static inline bool oplus_rbtree_empty(struct rb_root_cached *list) -+{ -+ return (rb_first_cached(list) == NULL); -+} -+ -+/** -+ * Check if there are ux tasks waiting to run on the specified cpu -+ */ -+static inline bool orq_has_ux_tasks(struct oplus_rq *orq) -+{ -+ return !oplus_rbtree_empty(&orq->ux_list); -+} -+ -+/* attention: before insert .ko, task's node->__rb_parent_color is init with 0 */ -+static inline bool oplus_rbnode_empty(struct rb_node *node) -+{ -+ return RB_EMPTY_NODE(node) || (node->__rb_parent_color == 0); -+} -+ -+static inline struct oplus_task_struct *ux_list_first_entry(struct rb_root_cached *list) -+{ -+ struct rb_node *leftmost = rb_first_cached(list); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, ux_entry); -+} -+ -+static inline struct oplus_task_struct *exec_timeline_first_entry(struct rb_root_cached *tree) -+{ -+ struct rb_node *leftmost = rb_first_cached(tree); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, exec_time_node); -+} -+ -+typedef bool (*migrate_task_callback_t)(struct task_struct *tsk, int src_cpu, int dst_cpu); -+extern migrate_task_callback_t fbg_migrate_task_callback; -+typedef void (*android_rvh_schedule_handler_t)(struct task_struct *prev, -+ struct task_struct *next, struct rq *rq); -+extern android_rvh_schedule_handler_t fbg_android_rvh_schedule_callback; -+ -+extern struct kmem_cache *oplus_task_struct_cachep; -+ -+#define ots_to_ts(ots) (ots->task) -+#define OTS_IDX 0 -+#define ORQ_IDX 0 -+ -+static inline bool test_task_is_fair(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid CFS priority is MAX_RT_PRIO..MAX_PRIO-1 */ -+ if ((task->prio >= MAX_RT_PRIO) && (task->prio <= MAX_PRIO-1)) -+ return true; -+ return false; -+} -+ -+static inline bool test_task_is_rt(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid RT priority is 0..MAX_RT_PRIO-1 */ -+ if (rt_prio(task->prio)) -+ return true; -+ -+ return false; -+} -+ -+static inline struct oplus_task_struct *get_oplus_task_struct(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = NULL; -+ -+ /* not Skip idle thread */ -+ if (!t) -+ return NULL; -+ -+ ots = (struct oplus_task_struct *) READ_ONCE(t->android_oem_data1[OTS_IDX]); -+ if (IS_ERR_OR_NULL(ots)) -+ return NULL; -+ -+ return ots; -+} -+ -+static inline unsigned long oplus_get_im_flag(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return IM_FLAG_NONE; -+ -+ return ots->im_flag; -+} -+ -+static inline bool is_optimized_audio_thread(struct task_struct *t) -+{ -+ unsigned long im_flag; -+ -+ im_flag = oplus_get_im_flag(t); -+ if (test_bit(IM_FLAG_AUDIO, &im_flag)) -+ return true; -+ -+ return false; -+} -+ -+static inline void oplus_set_im_flag(struct task_struct *t, int im_flag) -+{ -+ struct oplus_task_struct *ots = NULL; -+ ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ set_bit(im_flag, &ots->im_flag); -+} -+ -+static inline int get_ots_ux_state(struct oplus_task_struct *ots) -+{ -+ int ux_state; -+ int sub_ux_state; -+ int ux_prio; -+ int ret; -+ -+ ux_prio = ots->ux_state & SCHED_ASSIST_UX_PRIORITY_MASK; -+ ux_state = ots->ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ sub_ux_state = ots->sub_ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ -+ if (sub_ux_state) { -+ ret = ux_state | (sub_ux_state & ~SA_TYPE_INHERIT) | SA_TYPE_COMBINED; -+ } else { -+ ret = ux_state; -+ } -+ -+ if (ret & SCHED_ASSIST_UX_MASK) { -+ return ux_prio | ret; -+ } -+ -+ return 0; -+} -+ -+bool is_multiple_ux(struct oplus_task_struct *ots); -+ -+static inline int oplus_get_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return get_ots_ux_state(ots); -+} -+ -+static inline int oplus_get_static_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return 0; -+ } -+ return ots->ux_state; -+} -+ -+static inline int oplus_get_sub_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->sub_ux_state; -+} -+ -+static inline int oplus_get_inherited_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return ots->ux_state; -+ } -+ return ots->sub_ux_state; -+} -+ -+void oplus_set_ux_state_lock(struct task_struct *t, int ux_state, int inherit_type, bool need_lock_rq); -+ -+static inline s64 oplus_get_inherit_ux(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return atomic64_read(&ots->inherit_ux); -+} -+ -+static inline void oplus_set_inherit_ux(struct task_struct *t, s64 inherit_ux) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ atomic64_set(&ots->inherit_ux, inherit_ux); -+} -+ -+static inline int oplus_get_ux_depth(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->ux_depth; -+} -+ -+static inline void oplus_set_ux_depth(struct task_struct *t, int ux_depth) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->ux_depth = ux_depth; -+} -+ -+static inline u64 oplus_get_enqueue_time(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->enqueue_time; -+} -+ -+static inline void oplus_set_enqueue_time(struct task_struct *t, u64 enqueue_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->enqueue_time = enqueue_time; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* -+ * Record the number of task context switches -+ * and scheduling delays when enqueueing a task. -+ */ -+ ots->snap_pcount = t->sched_info.pcount; -+ ots->snap_run_delay = t->sched_info.run_delay; -+#endif -+} -+ -+static inline void oplus_set_inherit_ux_start(struct task_struct *t, u64 start_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->inherit_ux_start = t->se.sum_exec_runtime; -+} -+ -+static inline void init_task_ux_info(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ RB_CLEAR_NODE(&ots->ux_entry); -+ RB_CLEAR_NODE(&ots->exec_time_node); -+ ots->ux_state = 0; -+ ots->sub_ux_state = 0; -+ atomic64_set(&ots->inherit_ux, 0); -+ ots->ux_depth = 0; -+ ots->enqueue_time = 0; -+ ots->inherit_ux_start = 0; -+ ots->ux_priority = -1; -+ ots->ux_nice = -1; -+ ots->vruntime = 0; -+ ots->preset_vruntime = 0; -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG) -+ ots->abnormal_flag = 0; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_SCHED_SPREAD -+ ots->lb_state = 0; -+ ots->ld_flag = 0; -+#endif -+ ots->exec_calc_runtime = 0; -+ ots->is_update_runtime = 0; -+ ots->target_process = -1; -+ ots->wake_tid = 0; -+ ots->running_start_time = 0; -+ ots->update_running_start_time = false; -+ ots->last_wake_ts = 0; -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ memset(&ots->lkinfo, 0, sizeof(struct locking_info)); -+ INIT_LIST_HEAD(&ots->lkinfo.node); -+/*#endif*/ -+ ots->block_start_time = 0; -+#ifdef CONFIG_LOCKING_PROTECT -+ INIT_LIST_HEAD(&ots->locking_entry); -+ ots->locking_start_time = 0; -+ ots->locking_depth = 0; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ ots->snap_pcount = 0; -+ ots->snap_run_delay = 0; -+ plist_node_init(&ots->rtb, MAX_IM_FLAG_PRIO); -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+ atomic_set(&ots->pipeline_cpu, -1); -+#endif -+ -+#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO) -+ ots->amu_cycle = 0; -+ ots->amu_instruct = 0; -+#endif -+}; -+ -+static inline bool test_ux_type(struct task_struct *task, int ux_type) -+{ -+ return oplus_get_ux_state(task) & ux_type; -+} -+ -+static inline bool is_heavy_ux_task(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return false; -+ -+ return get_ots_ux_state(ots) & SA_TYPE_HEAVY; -+} -+ -+static inline bool sched_assist_scene(unsigned int scene) -+{ -+ if (unlikely(!global_sched_assist_enabled)) -+ return false; -+ -+ return global_sched_assist_scene & scene; -+} -+ -+static inline unsigned long oplus_task_util(struct task_struct *p) -+{ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+ struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; -+ -+ return wts->demand_scaled; -+#else -+ return READ_ONCE(p->se.avg.util_avg); -+#endif -+} -+ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+static inline u32 task_wts_sum(struct task_struct *tsk) -+{ -+ struct walt_task_struct *wts = (struct walt_task_struct *) tsk->android_vendor_data1; -+ -+ return wts->sum; -+} -+#endif -+ -+bool is_min_cluster(int cpu); -+bool is_max_cluster(int cpu); -+bool is_mid_cluster(int cpu); -+bool im_mali(const char *comm); -+bool is_top(struct task_struct *p); -+bool task_is_runnable(struct task_struct *task); -+struct oplus_rq *get_oplus_rq(struct rq *rq); -+ -+typedef unsigned long (*kallsyms_lookup_name_t)(const char *name); -+extern kallsyms_lookup_name_t _kallsyms_lookup_name; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+int ux_mask_to_prio(int ux_mask); -+int ux_prio_to_mask(int prio); -+#endif -+ -+noinline int tracing_mark_write(const char *buf); -+void hwbinder_systrace_c(unsigned int cpu, int flag); -+void sched_assist_init_oplus_rq(void); -+void queue_ux_thread(struct rq *rq, struct task_struct *p, int enqueue); -+ -+void inherit_ux_inc(struct task_struct *task, int type); -+void inherit_ux_sub(struct task_struct *task, int type, int value); -+void set_inherit_ux(struct task_struct *task, int type, int depth, int inherit_val); -+void reset_inherit_ux(struct task_struct *inherit_task, struct task_struct *ux_task, int reset_type); -+void unset_inherit_ux(struct task_struct *task, int type); -+void unset_inherit_ux_value(struct task_struct *task, int type, int value); -+void inc_inherit_ux_refs(struct task_struct *task, int type); -+void clear_all_inherit_type(struct task_struct *p); -+int get_max_inherit_gran(struct task_struct *p); -+ -+bool is_heavy_load_top_task(struct task_struct *p); -+bool test_task_is_fair(struct task_struct *task); -+bool test_task_is_rt(struct task_struct *task); -+ -+bool test_task_ux(struct task_struct *task); -+bool test_task_ux_depth(int ux_depth); -+bool test_inherit_ux(struct task_struct *task, int type); -+bool test_set_inherit_ux(struct task_struct *task); -+int get_ux_state_type(struct task_struct *task); -+void sched_assist_target_comm(struct task_struct *task, const char *comm); -+unsigned int ux_task_exec_limit(struct task_struct *p); -+ -+void update_ux_sched_cputopo(void); -+bool is_task_util_over(struct task_struct *tsk, int threshold); -+bool oplus_task_misfit(struct task_struct *tsk, int cpu); -+ssize_t oplus_show_cpus(const struct cpumask *mask, char *buf); -+void adjust_rt_lowest_mask(struct task_struct *p, struct cpumask *local_cpu_mask, int ret, bool force_adjust); -+bool sa_skip_rt_sync(struct rq *rq, struct task_struct *p, bool *sync); -+void ux_state_systrace_c(unsigned int cpu, struct task_struct *p); -+bool sa_rt_skip_ux_cpu(int cpu); -+int is_vip_mvp(struct task_struct *p); -+ -+/* s64 account_ux_runtime(struct rq *rq, struct task_struct *curr); */ -+void opt_ss_lock_contention(struct task_struct *p, unsigned long old_im, int new_im); -+ -+/* register vender hook in kernel/sched/topology.c */ -+void android_vh_build_sched_domains_handler(void *unused, bool has_asym); -+ -+/* register vender hook in kernel/sched/rt.c */ -+void android_rvh_select_task_rq_rt_handler(void *unused, struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags, int *new_cpu); -+void android_rvh_find_lowest_rq_handler(void *unused, struct task_struct *p, struct cpumask *local_cpu_mask, int ret, int *best_cpu); -+ -+/* register vender hook in kernel/sched/core.c */ -+void android_rvh_sched_fork_handler(void *unused, struct task_struct *p); -+void android_rvh_after_enqueue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_dequeue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_schedule_handler(void *unused, struct task_struct *prev, struct task_struct *next, struct rq *rq); -+void android_vh_scheduler_tick_handler(void *unused, struct rq *rq); -+ -+void android_vh_account_process_tick_gran_handler(void *unused, struct task_struct *p, struct rq *rq, int user_tick, int *ticks); -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+void sa_sched_switch_handler(void *unused, bool preempt, struct task_struct *prev, struct task_struct *next, unsigned int prev_state); -+#endif -+ -+void set_im_flag_with_bit(int im_flag, struct task_struct *task); -+ -+/* register vendor hook in kernel/cgroup/cgroup-v1.c */ -+void android_vh_cgroup_set_task_handler(void *unused, int ret, struct task_struct *task); -+/* register vendor hook in kernel/signal.c */ -+void android_vh_exit_signal_handler(void *unused, struct task_struct *p); -+void android_rvh_set_cpus_allowed_by_task_handler(void *unused, const struct cpumask *cpu_valid_mask, -+ const struct cpumask *new_mask, struct task_struct *task, unsigned int *dest_cpu); -+void android_rvh_setscheduler_handler(void *unused, struct task_struct *p); -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_BAN_APP_SET_AFFINITY) -+void android_vh_sched_setaffinity_early_handler(void *unused, struct task_struct *task, const struct cpumask *new_mask, bool *skip); -+#endif -+ -+#ifdef CONFIG_OPLUS_SCHED_GROUP_OPT -+void android_vh_reweight_entity_handler(void *unused, struct sched_entity *se); -+#endif -+ -+#ifdef CONFIG_BLOCKIO_UX_OPT -+int sa_blockio_init(void); -+void sa_blockio_exit(void); -+#endif -+extern struct notifier_block process_exit_notifier_block; -+#endif /* _OPLUS_SA_COMMON_H_ */ -diff --git a/include/linux/sched/sa_common_struct.h b/include/linux/sched/sa_common_struct.h -new file mode 100755 -index 000000000000..876feff48b36 ---- /dev/null -+++ b/include/linux/sched/sa_common_struct.h -@@ -0,0 +1,277 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+/* -+this file is splited from the sa_common.h to adapt the OKI, -+IS_ENABLED is not allowed in here, because the macro will not work in OKI. -+*/ -+#ifndef _OPLUS_SA_COMMON_STRUCT_H_ -+#define _OPLUS_SA_COMMON_STRUCT_H_ -+ -+#define MAX_CLUSTER (4) -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+/* hot-thread */ -+struct task_record { -+#define RECOED_WINSIZE (1 << 8) -+#define RECOED_WINIDX_MASK (RECOED_WINSIZE - 1) -+ u8 winidx; -+ u8 count; -+}; -+ -+#define MAX_TASK_COMM_LEN 256 -+struct uid_struct { -+ uid_t uid; -+ u64 uid_total_cycle; -+ u64 uid_total_inst; -+ spinlock_t lock; -+ char leader_comm[TASK_COMM_LEN]; -+ char cmdline[MAX_TASK_COMM_LEN]; -+}; -+ -+struct amu_uid_entry { -+ uid_t uid; -+ struct uid_struct *uid_struct; -+ struct hlist_node node; -+}; -+ -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+struct locking_info { -+ u64 waittime_stamp; -+ u64 holdtime_stamp; -+ /* Used in torture acquire latency statistic.*/ -+ u64 acquire_stamp; -+ /* -+ * mutex or rwsem optimistic spin start time. Because a task -+ * can't spin both on mutex and rwsem at one time, use one common -+ * threshold time is OK. -+ */ -+ u64 opt_spin_start_time; -+ struct task_struct *holder; -+ u32 waittype; -+ bool ux_contrib; -+ /* -+ * Whether task is ux when it's going to be added to mutex or -+ * rwsem waiter list. It helps us check whether there is ux -+ * task on mutex or rwsem waiter list. Also, a task can't be -+ * added to both mutex and rwsem at one time, so use one common -+ * field is OK. -+ */ -+ bool is_block_ux; -+ u32 kill_flag; -+ /* for cfs enqueue smoothly.*/ -+ struct list_head node; -+ struct task_struct *owner; -+ struct list_head lock_head; -+ u64 clear_seq; -+ atomic_t lock_depth; -+}; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+#define RAVG_HIST_SIZE 5 -+#define SCX_SLICE_DFL (1 * NSEC_PER_MSEC) -+#define SCX_SLICE_INF U64_MAX -+#define DEFAULT_CGROUP_DL_IDX (8) -+#define EXT_FLAG_RT_CHANGED (1 << 0) -+#define EXT_FLAG_CFS_CHANGED (1 << 1) -+struct scx_task_stats { -+ u64 mark_start; -+ u64 window_start; -+ u32 sum; -+ u32 sum_history[RAVG_HIST_SIZE]; -+ int cidx; -+ u32 demand; -+ u16 demand_scaled; -+ void *sdsq; -+}; -+/* -+ * The following is embedded in task_struct and contains all fields necessary -+ * for a task to be scheduled by SCX. -+ */ -+struct scx_entity { -+ struct scx_dispatch_q *dsq; -+ struct { -+ struct list_head fifo; /* dispatch order */ -+ struct rb_node priq; /* p->scx.dsq_vtime order */ -+ } dsq_node; -+ u32 flags; /* protected by rq lock */ -+ u32 dsq_flags; /* protected by dsq lock */ -+ s32 sticky_cpu; -+ unsigned long runnable_at; -+ u64 slice; -+ u64 dsq_vtime; -+ int gdsq_idx; -+ int ext_flags; -+ int prio_backup; -+ unsigned long sched_prop; -+ struct scx_task_stats sts; -+}; -+/*#endif*/ -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ -+#define MAX_CPU_FREQ_STATE 32 -+#define MAX_CPU_CNT 8 -+ -+#define STATE_IN_POOL 0 -+#define STATE_ACTIVE 1 -+ -+struct powermodel_freq_task_state { -+ u8 powermodel_enqueued:1; -+ u8 powermodel_index:7; -+}; -+ -+struct powermodel_cpu_task_state { -+ struct oplus_task_struct *ots; -+ struct powermodel_freq_task_state powermodel_freq_task_states[MAX_CPU_FREQ_STATE]; -+ u64 powermodel_last_seq; -+ u64 last_seq; -+ struct list_head node; -+ int state; -+ int pool_id; -+}; -+ -+#endif -+ -+ -+/* Please add your own members of task_struct here :) */ -+struct oplus_task_struct { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_node ux_entry; -+ struct rb_node exec_time_node; -+ struct task_struct *task; -+ atomic64_t inherit_ux; -+ u64 enqueue_time; -+ u64 inherit_ux_start; -+ /* u64 sum_exec_baseline; */ -+ u64 total_exec; -+ u64 vruntime; -+ u64 preset_vruntime; -+ /* contains ux state -+ 1. if static and inherited ux both exist, static ux stores in ux_state, inherited ux in sub_ux_state. -+ 2. if only static ux exists, static ux stores in ux_state. -+ 2. if only inherited ux exists, inherited ux stores in ux_state */ -+ int ux_state; -+ int sub_ux_state; -+ u8 ux_depth; -+ s8 ux_priority; -+ s8 ux_nice; -+ pid_t affinity_pid; -+ pid_t affinity_tgid; -+ unsigned long state; -+ unsigned long im_flag; -+ atomic_t is_vip_mvp; -+ -+/* #if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) */ -+ u64 ddl; -+ u64 ddl_active_ts; -+ u64 runnable_ts; -+ struct rb_node ddl_node; -+/* #endif */ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG)*/ -+ int abnormal_flag; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_SCHED_SPREAD */ -+ int lb_state; -+ int ld_flag:1; -+ /* CONFIG_OPLUS_FEATURE_TASK_LOAD */ -+ int is_update_runtime:1; -+ int target_process; -+ u64 wake_tid; -+ u64 running_start_time; -+ bool update_running_start_time; -+ u64 exec_calc_runtime; -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct task_record record[MAX_CLUSTER]; /* 2*u64 */ -+ u64 block_start_time; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_FRAME_BOOST */ -+ struct list_head fbg_list; -+ raw_spinlock_t fbg_list_entry_lock; -+ bool fbg_running; /* task belongs to a group, and in running */ -+ u16 fbg_state; -+ s8 preferred_cluster_id; -+ s8 fbg_depth; -+ u64 last_wake_ts; -+ int fbg_cur_group; -+/*#ifdef CONFIG_LOCKING_PROTECT*/ -+ unsigned long locking_start_time; -+ unsigned long last_jiffies; -+ struct list_head locking_entry; -+ int locking_depth; -+ int lk_tick_hit; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ struct locking_info lkinfo; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+ struct scx_entity scx; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_FDLEAK_CHECK)*/ -+ u8 fdleak_flag; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+ /* for loadbalance */ -+ struct plist_node rtb; /* rt boost task */ -+ -+ /* -+ * The following variables are used to calculate the time -+ * a task spends in the running/runnable state. -+ */ -+ u64 snap_run_delay; -+ unsigned long snap_pcount; -+/*#endif*/ -+#if IS_ENABLED(CONFIG_OPLUS_SCHED_TUNE) -+ int stune_idx; -+#endif -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE)*/ -+ atomic_t pipeline_cpu; -+/*#endif*/ -+ -+ /* for oplus secure guard */ -+ int sg_flag; -+ int sg_scno; -+ uid_t sg_uid; -+ uid_t sg_euid; -+ gid_t sg_gid; -+ gid_t sg_egid; -+/*#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct uid_struct *uid_struct; -+ u64 amu_instruct; -+ u64 amu_cycle; -+/*#endif*/ -+ /* for binder ux */ -+ int binder_async_ux_enable; -+ bool binder_async_ux_sts; -+ int binder_thread_mode; -+ struct binder_node *binder_thread_node; -+ -+/* for powermodel */ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ struct powermodel_cpu_task_state *powermodel_cpu_task_states[MAX_CPU_CNT]; -+ u64 exec_runtime; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_CFBT) -+ int cfbt_cur_group; -+ bool cfbt_running; -+#endif /* CONFIG_OPLUS_FEATURE_SCHED_CFBT */ -+} ____cacheline_aligned; -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+#define INVALID_PID (-1) -+struct oplus_lb { -+ /* used for active_balance to record the running task. */ -+ pid_t pid; -+}; -+/*#endif*/ -+ -+#endif /* _OPLUS_SA_COMMON_STRUCT_H_ */ -+ -diff --git a/include/linux/sched/sa_oemdata.h b/include/linux/sched/sa_oemdata.h -new file mode 100755 -index 000000000000..ebbbc754e391 ---- /dev/null -+++ b/include/linux/sched/sa_oemdata.h -@@ -0,0 +1,13 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_OEMDATA_H_ -+#define _OPLUS_SA_OEMDATA_H_ -+ -+int sa_oemdata_init(void); -+void sa_oemdata_deinit(void); -+ -+#endif /* _OPLUS_SA_OEMDATA_H_ */ -diff --git a/include/linux/sched/sched_ext.h b/include/linux/sched/sched_ext.h -new file mode 100755 -index 000000000000..9f1afdb8a04b ---- /dev/null -+++ b/include/linux/sched/sched_ext.h -@@ -0,0 +1,57 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef _OPLUS_SCHED_EXT_H -+#define _OPLUS_SCHED_EXT_H -+#include "sa_common.h" -+ -+#define SCHED_PROP_TOP_THREAD_SHIFT (8) -+#define SCHED_PROP_TOP_THREAD_MASK (0xf << SCHED_PROP_TOP_THREAD_SHIFT) -+#define SCHED_PROP_DEADLINE_MASK (0xFF) /* deadline for ext sched class */ -+#define SCHED_PROP_DEADLINE_LEVEL1 (1) /* 1ms for user-aware audio tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL2 (2) /* 2ms for user-aware touch tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL3 (3) /* 4ms for user aware dispaly tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL4 (4) /* 6ms */ -+#define SCHED_PROP_DEADLINE_LEVEL5 (5) /* 8ms */ -+#define SCHED_PROP_DEADLINE_LEVEL6 (6) /* 16ms */ -+#define SCHED_PROP_DEADLINE_LEVEL7 (7) /* 32ms */ -+#define SCHED_PROP_DEADLINE_LEVEL8 (8) /* 64ms */ -+#define SCHED_PROP_DEADLINE_LEVEL9 (9) /* 128ms */ -+ -+static inline int sched_prop_get_top_thread_id(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ return -EPERM; -+ } -+ -+ return ((ots->scx.sched_prop & SCHED_PROP_TOP_THREAD_MASK) >> SCHED_PROP_TOP_THREAD_SHIFT); -+} -+ -+static inline int sched_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_set_sched_prop failed! fn=%s\n", __func__); -+ return -EPERM; -+ } -+ -+ ots->scx.sched_prop = sp; -+ return 0; -+} -+ -+static inline unsigned long sched_get_sched_prop(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_get_sched_prop failed! fn=%s\n", __func__); -+ return (unsigned long)-1; -+ } -+ return ots->scx.sched_prop; -+} -+ -+#endif /*_OPLUS_SCHED_EXT_H */ -diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h -index 03d35e3eda3c..6a5f0c0d74bd 100644 ---- a/include/linux/sched/task.h -+++ b/include/linux/sched/task.h -@@ -61,8 +61,10 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); - extern void init_idle(struct task_struct *idle, int cpu); - - extern int sched_fork(unsigned long clone_flags, struct task_struct *p); --extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); --extern void sched_cancel_fork(struct task_struct *p); -+extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); -+#ifdef CONFIG_HMBIRD_SCHED -+extern void hmbird_cancel_fork(struct task_struct *p); -+#endif - extern void sched_post_fork(struct task_struct *p); - extern void sched_dead(struct task_struct *p); - -diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h -index 359a14cc76a4..969716ab4320 100644 ---- a/include/uapi/linux/sched.h -+++ b/include/uapi/linux/sched.h -@@ -118,7 +118,7 @@ struct clone_args { - /* SCHED_ISO: reserved but not implemented yet */ - #define SCHED_IDLE 5 - #define SCHED_DEADLINE 6 --#define SCHED_EXT 7 -+#define SCHED_HMBIRD 7 - - /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ - #define SCHED_RESET_ON_FORK 0x40000000 -diff --git a/init/Kconfig b/init/Kconfig -index f1a81cc2e412..b5c87221addb 100644 ---- a/init/Kconfig -+++ b/init/Kconfig -@@ -1021,10 +1021,6 @@ config RT_GROUP_SCHED - realtime bandwidth for them. - See Documentation/scheduler/sched-rt-group.rst for more information. - --config EXT_GROUP_SCHED -- bool -- depends on SCHED_CLASS_EXT && CGROUP_SCHED -- default y - endif #CGROUP_SCHED - - config SCHED_MM_CID -diff --git a/init/init_task.c b/init/init_task.c -index 69a046388995..31ceb0e469f7 100644 ---- a/init/init_task.c -+++ b/init/init_task.c -@@ -6,7 +6,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -102,9 +101,6 @@ struct task_struct init_task - #endif - #ifdef CONFIG_CGROUP_SCHED - .sched_task_group = &root_task_group, --#endif --#ifdef CONFIG_SLIM_SCHED -- .scx = NULL, - #endif - .ptraced = LIST_HEAD_INIT(init_task.ptraced), - .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), -diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt -index cf22ef6107b8..b131d5662b48 100644 ---- a/kernel/Kconfig.preempt -+++ b/kernel/Kconfig.preempt -@@ -133,12 +133,8 @@ config SCHED_CORE - which is the likely usage by Linux distributions, there should - be no measurable impact on performance. - --config SLIM_SCHED -- bool "slim sched" -- default n --config SCHED_CLASS_EXT -- bool "Extensible Scheduling Class" -- depends on BPF_SYSCALL && BPF_JIT -+config HMBIRD_SCHED -+ bool "hmbird Scheduling Class(base on ext scheduler)" - help - This option enables a new scheduler class sched_ext (SCX), which - allows scheduling policies to be implemented as BPF programs to -@@ -159,3 +155,10 @@ config SCHED_CLASS_EXT - similar to struct sched_class. - - See Documentation/scheduler/sched-ext.rst for more details. -+ -+config DYNAMIC_WALT_SUPPORT -+ bool "Dynamic WALT Support for Extensible Scheduling Class" -+ depends on HMBIRD_SCHED -+ default n -+ help -+ Enable dynamic WALT support for Extensible Scheduling Class. -diff --git a/kernel/bpf/bpf_struct_ops_types.h b/kernel/bpf/bpf_struct_ops_types.h -index 3618769d853d..5678a9ddf817 100644 ---- a/kernel/bpf/bpf_struct_ops_types.h -+++ b/kernel/bpf/bpf_struct_ops_types.h -@@ -9,8 +9,4 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) - #include - BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) - #endif --#ifdef CONFIG_SCHED_CLASS_EXT --#include --BPF_STRUCT_OPS_TYPE(sched_ext_ops) --#endif - #endif -diff --git a/kernel/fork.c b/kernel/fork.c -index d2f5ae66a322..bc97804f3411 100644 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -23,7 +23,9 @@ - #include - #include - #include --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - #include - #include - #include -@@ -998,7 +1000,9 @@ void __put_task_struct(struct task_struct *tsk) - WARN_ON(refcount_read(&tsk->usage)); - WARN_ON(tsk == current); - -- sched_ext_free(tsk); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_free(tsk); -+#endif - io_uring_free(tsk); - cgroup_free(tsk); - task_numa_free(tsk, true); -@@ -2511,7 +2515,11 @@ __latent_entropy struct task_struct *copy_process( - - retval = perf_event_init_task(p, clone_flags); - if (retval) -- goto bad_fork_sched_cancel_fork; -+#ifdef CONFIG_HMBIRD_SCHED -+ goto bad_fork_hmbird_cancel_fork; -+#else -+ goto bad_fork_cleanup_policy; -+#endif - retval = audit_alloc(p); - if (retval) - goto bad_fork_cleanup_perf; -@@ -2643,9 +2651,7 @@ __latent_entropy struct task_struct *copy_process( - * cgroup specific, it unconditionally needs to place the task on a - * runqueue. - */ -- retval = sched_cgroup_fork(p, args); -- if (retval) -- goto bad_fork_cancel_cgroup; -+ sched_cgroup_fork(p, args); - - /* - * From this point on we must avoid any synchronous user-space -@@ -2691,13 +2697,13 @@ __latent_entropy struct task_struct *copy_process( - /* Don't start children in a dying pid namespace */ - if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { - retval = -ENOMEM; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* Let kill terminate clone/fork in the middle */ - if (fatal_signal_pending(current)) { - retval = -EINTR; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* No more failure paths after this point. */ -@@ -2774,11 +2780,10 @@ __latent_entropy struct task_struct *copy_process( - - return p; - --bad_fork_core_free: -+bad_fork_cancel_cgroup: - sched_core_free(p); - spin_unlock(¤t->sighand->siglock); - write_unlock_irq(&tasklist_lock); --bad_fork_cancel_cgroup: - cgroup_cancel_fork(p, args); - bad_fork_put_pidfd: - if (clone_flags & CLONE_PIDFD) { -@@ -2817,8 +2822,10 @@ __latent_entropy struct task_struct *copy_process( - audit_free(p); - bad_fork_cleanup_perf: - perf_event_free_task(p); --bad_fork_sched_cancel_fork: -- sched_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+bad_fork_hmbird_cancel_fork: -+ hmbird_cancel_fork(p); -+#endif - bad_fork_cleanup_policy: - lockdep_free_task(p); - #ifdef CONFIG_NUMA -diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c -index 005025f55bea..c091539a0f60 100644 ---- a/kernel/sched/build_policy.c -+++ b/kernel/sched/build_policy.c -@@ -28,8 +28,10 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED - #include - #include -+#endif - - #include - -@@ -54,6 +56,10 @@ - #include "cputime.c" - #include "deadline.c" - --#ifdef CONFIG_SCHED_CLASS_EXT --# include "ext.c" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/hmbird_util_track.c" -+#include "hmbird/hmbird_sched_proc.c" -+#include "hmbird/hmbird_shadow_tick.c" -+#include "hmbird/hmbird.c" -+#include "hmbird/hmbird_misc.c" - #endif -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index 2adea38f9f5c..15133c67d0c3 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -96,6 +96,12 @@ - #include "../../io_uring/io-wq.h" - #include "../smpboot.h" - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/slim.h" -+#include "hmbird/hmbird_shadow_tick.h" -+#include "hmbird.h" -+#endif -+ - #include - #include - #include -@@ -170,7 +176,6 @@ __read_mostly int scheduler_running; - - DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); - -- - /* kernel prio, less is more */ - static inline int __task_prio(const struct task_struct *p) - { -@@ -183,8 +188,8 @@ static inline int __task_prio(const struct task_struct *p) - if (p->sched_class == &idle_sched_class) - return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - --#ifdef CONFIG_SCHED_CLASS_EXT -- if (p->sched_class == &ext_sched_class) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (p->sched_class == &hmbird_sched_class) - return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ - #endif - -@@ -217,9 +222,9 @@ static inline bool prio_less(const struct task_struct *a, - if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ - return cfs_prio_less(a, b, in_fi); - --#ifdef CONFIG_SCHED_CLASS_EXT -+#ifdef CONFIG_HMBIRD_SCHED - if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ -- return scx_prio_less(a, b, in_fi); -+ return hmbird_prio_less(a, b, in_fi); - #endif - - return false; -@@ -1277,17 +1282,26 @@ bool sched_can_stop_tick(struct rq *rq) - if (fifo_nr_running) - return true; - -+#ifdef CONFIG_HMBIRD_SCHED - /* -- * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks -+ * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks - * left. For CFS, if there's more than one we need the tick for -- * involuntary preemption. For SCX, ask. -+ * involuntary preemption. For HMBIRD, ask. - */ -- if (!scx_switched_all() && rq->nr_running > 1) -+ if (!hmbird_enabled() && rq->nr_running > 1) - return false; - -- if (scx_enabled() && !scx_can_stop_tick(rq)) -+ if (hmbird_enabled() && !hmbird_can_stop_tick(rq)) - return false; -- -+#else -+ /* -+ * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; -+ * if there's more than one we need the tick for involuntary -+ * preemption. -+ */ -+ if (rq->nr_running > 1) -+ return false; -+#endif - /* - * If there is one task and it has CFS runtime bandwidth constraints - * and it's on the cpu now we don't want to stop the tick. -@@ -2212,10 +2226,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) - } - EXPORT_SYMBOL_GPL(deactivate_task); - --struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) -+#ifdef CONFIG_HMBIRD_SCHED -+struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - { -- struct sched_change_guard cg = { -+ struct hmbird_sched_change_guard cg = { - .rq = rq, - .p = p, - .queued = task_on_rq_queued(p), -@@ -2237,7 +2252,7 @@ sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - return cg; - } - --void sched_change_guard_fini(struct sched_change_guard *cg, int flags) -+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags) - { - if (cg->queued) - enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); -@@ -2245,6 +2260,7 @@ void sched_change_guard_fini(struct sched_change_guard *cg, int flags) - set_next_task(cg->rq, cg->p); - cg->done = true; - } -+#endif - - static inline int __normal_prio(int policy, int rt_prio, int nice) - { -@@ -2310,9 +2326,9 @@ inline int task_curr(const struct task_struct *p) - * this means any call to check_class_changed() must be followed by a call to - * balance_callback(). - */ --void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio) -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) - { - if (prev_class != p->sched_class) { - if (prev_class->switched_from) -@@ -2875,6 +2891,7 @@ static void - __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - { - struct rq *rq = task_rq(p); -+ bool queued, running; - - /* - * This here violates the locking rules for affinity, since we're only -@@ -2893,9 +2910,26 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - else - lockdep_assert_held(&p->pi_lock); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->sched_class->set_cpus_allowed(p, ctx); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ -+ if (queued) { -+ /* -+ * Because __kthread_bind() calls this on blocked tasks without -+ * holding rq->lock. -+ */ -+ lockdep_assert_rq_held(rq); -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); - } -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->sched_class->set_cpus_allowed(p, ctx); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - } - - /* -@@ -3762,7 +3796,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class)) -+#else - if (unlikely(!is_cpu_allowed(p, cpu))) -+#endif - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -@@ -4880,7 +4918,9 @@ late_initcall(sched_core_sysctl_init); - */ - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { -+#ifdef CONFIG_HMBIRD_SCHED - int ret; -+#endif - - trace_android_rvh_sched_fork(p); - -@@ -4921,21 +4961,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_reset_on_fork = 0; - } - -- scx_pre_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ ret = hmbird_pre_fork(p); -+ if (ret) -+ goto out_cancel; - - if (dl_prio(p->prio)) { - ret = -EAGAIN; - goto out_cancel; --#ifdef CONFIG_SCHED_CLASS_EXT -- } else if (task_on_scx(p)) { -- p->sched_class = &ext_sched_class; --#endif -+ } else if (task_on_hmbird(p)) { -+ p->sched_class = &hmbird_sched_class; - } else if (rt_prio(p->prio)) { - p->sched_class = &rt_sched_class; - } else { - p->sched_class = &fair_sched_class; - } -- -+#else -+ if (dl_prio(p->prio)) -+ return -EAGAIN; -+ else if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - init_entity_runnable_average(&p->se); - trace_android_rvh_finish_prio_fork(p); - -@@ -4954,12 +5002,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - #endif - return 0; - -+#ifdef CONFIG_HMBIRD_SCHED - out_cancel: -- scx_cancel_fork(p); -+ hmbird_cancel_fork(p); - return ret; -+#endif - } - --int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) -+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - { - unsigned long flags; - -@@ -4986,19 +5036,17 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -- -- return scx_fork(p); --} -- --void sched_cancel_fork(struct task_struct *p) --{ -- scx_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_fork(p); -+#endif - } - - void sched_post_fork(struct task_struct *p) - { - uclamp_post_fork(p); -- scx_post_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_post_fork(p); -+#endif - } - - unsigned long to_ratio(u64 period, u64 runtime) -@@ -5860,20 +5908,28 @@ void scheduler_tick(void) - if (sched_feat(LATENCY_WARN) && resched_latency) - resched_latency_warn(cpu, resched_latency); - -- scx_notify_sched_tick(); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_notify_sched_tick(); -+#endif - perf_event_task_tick(); - - if (curr->flags & PF_WQ_WORKER) - wq_worker_tick(curr); - - #ifdef CONFIG_SMP -- if (!scx_switched_all()) { -+#ifdef CONFIG_HMBIRD_SCHED -+ if (!hmbird_enabled()) { -+#endif - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); -+#ifdef CONFIG_HMBIRD_SCHED - } - #endif -- -+#endif - trace_android_vh_scheduler_tick(rq); -+#ifdef CONFIG_HMBIRD_SCHED -+ scheduler_tick_handler(NULL, NULL); -+#endif - } - - #ifdef CONFIG_NO_HZ_FULL -@@ -6175,7 +6231,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ -+#ifdef CONFIG_HMBIRD_SCHED - for_balance_class_range(class, prev->sched_class, &idle_sched_class) { -+#else -+ for_class_range(class, prev->sched_class, &idle_sched_class) { -+#endif - if (class->balance(rq, prev, rf)) - break; - } -@@ -6193,9 +6253,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - const struct sched_class *class; - struct task_struct *p; - -- if (scx_enabled()) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (hmbird_enabled()) - goto restart; -- -+#endif - /* - * Optimization: we know that if all tasks are in the fair class we can - * call that function directly, but only if the @prev task wasn't of a -@@ -6221,13 +6282,21 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - restart: - put_prev_task_balance(rq, prev, rf); - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { - p = class->pick_next_task(rq); - if (p) { -- scx_notify_pick_next_task(rq, p, class); -+ hmbird_notify_pick_next_task(rq, p, class); - return p; - } - } -+#else -+ for_each_class(class) { -+ p = class->pick_next_task(rq); -+ if (p) -+ return p; -+ } -+#endif - - BUG(); /* The idle class should always have a runnable task. */ - } -@@ -6256,7 +6325,11 @@ static inline struct task_struct *pick_task(struct rq *rq) - const struct sched_class *class; - struct task_struct *p; - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { -+#else -+ for_each_class(class) { -+#endif - p = class->pick_task(rq); - if (p) - return p; -@@ -6898,6 +6971,9 @@ static void __sched notrace __schedule(unsigned int sched_mode) - psi_sched_switch(prev, next, !task_on_rq_queued(prev)); - - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#ifdef CONFIG_HMBIRD_SCHED -+ sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#endif - - /* Also unlocks the rq: */ - rq = context_switch(rq, prev, next, &rf); -@@ -7226,24 +7302,31 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag - } - EXPORT_SYMBOL(default_wake_function); - --void __setscheduler_prio(struct task_struct *p, int prio) -+static void __setscheduler_prio(struct task_struct *p, int prio) - { -- /* -- * After switching all rt and fair class to ext, -- * stop class is switched to ext class too. Just -- * skip it. */ -+#ifdef CONFIG_HMBIRD_SCHED -+ bool on_hmbird = task_on_hmbird(p); -+ - if (p->sched_class == &stop_sched_class) -- ;/* do nothing */ -+ ; - else if (dl_prio(prio)) - p->sched_class = &dl_sched_class; --#ifdef CONFIG_SCHED_CLASS_EXT -- else if (task_on_scx(p)) -- p->sched_class = &ext_sched_class; --#endif -+ else if (rt_prio(prio) && on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#else -+ if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; - else if (rt_prio(prio)) - p->sched_class = &rt_sched_class; - else - p->sched_class = &fair_sched_class; -+#endif - - p->prio = prio; - } -@@ -7278,7 +7361,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - */ - void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - { -- int prio, oldprio, queue_flag = -+ int prio, oldprio, queued, running, queue_flag = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *prev_class; - struct rq_flags rf; -@@ -7341,42 +7424,52 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - queue_flag &= ~DEQUEUE_MOVE; - - prev_class = p->sched_class; -- SCHED_CHANGE_BLOCK(rq, p, queue_flag) { -- /* -- * Boosting condition are: -- * 1. -rt task is running and holds mutex A -- * --> -dl task blocks on mutex A -- * -- * 2. -dl task is running and holds mutex A -- * --> -dl task blocks on mutex A and could preempt the -- * running task -- */ -- if (dl_prio(prio)) { -- if (!dl_prio(p->normal_prio) || -- (pi_task && dl_prio(pi_task->prio) && -- dl_entity_preempt(&pi_task->dl, &p->dl))) { -- p->dl.pi_se = pi_task->dl.pi_se; -- queue_flag |= ENQUEUE_REPLENISH; -- } else { -- p->dl.pi_se = &p->dl; -- } -- } else if (rt_prio(prio)) { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- if (oldprio < prio) -- queue_flag |= ENQUEUE_HEAD; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flag); -+ if (running) -+ put_prev_task(rq, p); -+ -+ /* -+ * Boosting condition are: -+ * 1. -rt task is running and holds mutex A -+ * --> -dl task blocks on mutex A -+ * -+ * 2. -dl task is running and holds mutex A -+ * --> -dl task blocks on mutex A and could preempt the -+ * running task -+ */ -+ if (dl_prio(prio)) { -+ if (!dl_prio(p->normal_prio) || -+ (pi_task && dl_prio(pi_task->prio) && -+ dl_entity_preempt(&pi_task->dl, &p->dl))) { -+ p->dl.pi_se = pi_task->dl.pi_se; -+ queue_flag |= ENQUEUE_REPLENISH; - } else { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- else if (rt_prio(oldprio)) -- p->rt.timeout = 0; -- else if (!task_has_idle_policy(p)) -- reweight_task(p, prio - MAX_RT_PRIO); -+ p->dl.pi_se = &p->dl; - } -- -- __setscheduler_prio(p, prio); -+ } else if (rt_prio(prio)) { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ if (oldprio < prio) -+ queue_flag |= ENQUEUE_HEAD; -+ } else { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ else if (rt_prio(oldprio)) -+ p->rt.timeout = 0; -+ else if (!task_has_idle_policy(p)) -+ reweight_task(p, prio - MAX_RT_PRIO); - } - -+ __setscheduler_prio(p, prio); -+ -+ if (queued) -+ enqueue_task(rq, p, queue_flag); -+ if (running) -+ set_next_task(rq, p); -+ - check_class_changed(rq, p, prev_class, oldprio); - out_unlock: - /* Avoid rq from going away on us: */ -@@ -7397,7 +7490,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - - void set_user_nice(struct task_struct *p, long nice) - { -- bool allowed = false; -+ bool queued, running, allowed = false; - int old_prio; - struct rq_flags rf; - struct rq *rq; -@@ -7426,13 +7519,22 @@ void set_user_nice(struct task_struct *p, long nice) - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); -+ if (running) -+ put_prev_task(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->static_prio = NICE_TO_PRIO(nice); -- set_load_weight(p, true); -- old_prio = p->prio; -- p->prio = effective_prio(p); -- } -+ p->static_prio = NICE_TO_PRIO(nice); -+ set_load_weight(p, true); -+ old_prio = p->prio; -+ p->prio = effective_prio(p); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - - /* - * If the task increased its priority or is running and -@@ -7835,7 +7937,7 @@ static int __sched_setscheduler(struct task_struct *p, - bool user, bool pi) - { - int oldpolicy = -1, policy = attr->sched_policy; -- int retval, oldprio, newprio; -+ int retval, oldprio, newprio, queued, running; - const struct sched_class *prev_class; - struct balance_callback *head; - struct rq_flags rf; -@@ -7843,6 +7945,9 @@ static int __sched_setscheduler(struct task_struct *p, - int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct rq *rq; - bool cpuset_locked = false; -+#ifdef CONFIG_HMBIRD_SCHED -+ unsigned long flags; -+#endif - - /* The pi code expects interrupts enabled */ - BUG_ON(pi && in_interrupt()); -@@ -7908,6 +8013,9 @@ static int __sched_setscheduler(struct task_struct *p, - * To be able to change p->policy safely, the appropriate - * runqueue lock must be held. - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+#endif - rq = task_rq_lock(p, &rf); - update_rq_clock(rq); - -@@ -7919,10 +8027,11 @@ static int __sched_setscheduler(struct task_struct *p, - goto unlock; - } - -- retval = scx_check_setscheduler(p, policy); -+#ifdef CONFIG_HMBIRD_SCHED -+ retval = hmbird_check_setscheduler(p, policy); - if (retval) - goto unlock; -- -+#endif - /* - * If not changing anything there's no need to proceed further, - * but store a possible modification of reset_on_fork. -@@ -7979,6 +8088,9 @@ static int __sched_setscheduler(struct task_struct *p, - if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { - policy = oldpolicy = -1; - task_rq_unlock(rq, p, &rf); -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - goto recheck; -@@ -8011,16 +8123,23 @@ static int __sched_setscheduler(struct task_struct *p, - queue_flags &= ~DEQUEUE_MOVE; - } - -- SCHED_CHANGE_BLOCK(rq, p, queue_flags) { -- prev_class = p->sched_class; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flags); -+ if (running) -+ put_prev_task(rq, p); - -- if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -- __setscheduler_params(p, attr); -- __setscheduler_prio(p, newprio); -- trace_android_rvh_setscheduler(p); -- } -- __setscheduler_uclamp(p, attr); -+ prev_class = p->sched_class; - -+ if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -+ __setscheduler_params(p, attr); -+ __setscheduler_prio(p, newprio); -+ trace_android_rvh_setscheduler(p); -+ } -+ __setscheduler_uclamp(p, attr); -+ -+ if (queued) { - /* - * We enqueue to tail when the priority of a task is - * increased (user space view). -@@ -8028,7 +8147,10 @@ static int __sched_setscheduler(struct task_struct *p, - if (oldprio < p->prio) - queue_flags |= ENQUEUE_HEAD; - -+ enqueue_task(rq, p, queue_flags); - } -+ if (running) -+ set_next_task(rq, p); - - check_class_changed(rq, p, prev_class, oldprio); - -@@ -8036,7 +8158,9 @@ static int __sched_setscheduler(struct task_struct *p, - preempt_disable(); - head = splice_balance_callbacks(rq); - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (pi) { - if (cpuset_locked) - cpuset_unlock(); -@@ -8051,6 +8175,9 @@ static int __sched_setscheduler(struct task_struct *p, - - unlock: - task_rq_unlock(rq, p, &rf); -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - return retval; -@@ -9272,7 +9399,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - break; - } -@@ -9300,7 +9429,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - } - return ret; -@@ -9605,15 +9736,25 @@ int migrate_task_to(struct task_struct *p, int target_cpu) - */ - void sched_setnuma(struct task_struct *p, int nid) - { -+ bool queued, running; - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) { -- p->numa_preferred_nid = nid; -- } -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE); -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->numa_preferred_nid = nid; - -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - task_rq_unlock(rq, p, &rf); - } - #endif /* CONFIG_NUMA_BALANCING */ -@@ -10153,15 +10294,22 @@ void __init sched_init(void) - int i; - - /* Make sure the linker didn't screw up */ -+#ifdef CONFIG_HMBIRD_SCHED - #ifdef CONFIG_SMP -- BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+#endif -+ WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class)); -+#else -+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 || -+ &fair_sched_class != &rt_sched_class + 1 || -+ &rt_sched_class != &dl_sched_class + 1); -+#ifdef CONFIG_SMP -+ BUG_ON(&dl_sched_class != &stop_sched_class + 1); - #endif -- BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); -- BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); -- BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); --#ifdef CONFIG_SCHED_CLASS_EXT -- BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); -- BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); - #endif - - wait_bit_init(); -@@ -10332,8 +10480,9 @@ void __init sched_init(void) - balance_push_set(smp_processor_id(), false); - #endif - init_sched_fair_class(); -- init_sched_ext_class(); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ init_sched_hmbird_class(); -+#endif - psi_init(); - - init_uclamp(); -@@ -10743,6 +10892,8 @@ static void sched_change_group(struct task_struct *tsk, struct task_group *group - */ - void sched_move_task(struct task_struct *tsk) - { -+ int queued, running, queue_flags = -+ DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct task_group *group; - struct rq_flags rf; - struct rq *rq; -@@ -10759,18 +10910,27 @@ void sched_move_task(struct task_struct *tsk) - - update_rq_clock(rq); - -- SCHED_CHANGE_BLOCK(rq, tsk, -- DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { -- sched_change_group(tsk, group); -- } -+ running = task_current(rq, tsk); -+ queued = task_on_rq_queued(tsk); - -- /* -- * After changing group, the running task may have joined a throttled -- * one but it's still the running task. Trigger a resched to make sure -- * that task can still run. -- */ -- if (task_current(rq, tsk)) -+ if (queued) -+ dequeue_task(rq, tsk, queue_flags); -+ if (running) -+ put_prev_task(rq, tsk); -+ -+ sched_change_group(tsk, group); -+ -+ if (queued) -+ enqueue_task(rq, tsk, queue_flags); -+ if (running) { -+ set_next_task(rq, tsk); -+ /* -+ * After changing group, the running task may have joined a -+ * throttled one but it's still the running task. Trigger a -+ * resched to make sure that task can still run. -+ */ - resched_curr(rq); -+ } - - unlock: - task_rq_unlock(rq, tsk, &rf); -@@ -10808,6 +10968,10 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) - struct task_group *tg = css_tg(css); - struct task_group *parent = css_tg(css->parent); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_tg_online(tg); -+#endif -+ - if (parent) - sched_online_group(tg, parent); - -@@ -10857,13 +11021,79 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) - } - #endif - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 dl) -+{ -+ int i; -+ -+ for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) { -+ if (dl < HMBIRD_BPF_DSQS_DEADLINE[i]) -+ break; -+ } -+ -+ i = max_t(int, i-1, MIN_CGROUP_DL_IDX); -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return 0; -+ update_cgroup_ids_table(css->cgroup->kn->id, i); -+ -+ return 0; -+} -+ -+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ u8 i; -+ -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX]; -+ i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1); -+ if (i < 0) { -+ pr_err(" <%s> i is %d, less than 0, name is %s\n", -+ __func__, i, css->cgroup->kn->name); -+ i = DEFAULT_CGROUP_DL_IDX; -+ } -+ -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[i]; -+} -+ -+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css, -+ struct task_struct *p, int prio) -+{ -+ if (!css || !rt_prio(prio)) -+ return; -+ -+ if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) { -+ if (!strcmp(css->cgroup->kn->name, "display")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3); -+ else if (!strcmp(css->cgroup->kn->name, "touch")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2); -+ else if (!strcmp(css->cgroup->kn->name, "multimedia")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+ } -+} -+#endif -+ - static void cpu_cgroup_attach(struct cgroup_taskset *tset) - { - struct task_struct *task; - struct cgroup_subsys_state *css; - -- cgroup_taskset_for_each(task, css, tset) -+ cgroup_taskset_for_each(task, css, tset) { -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_change_rt_sched_prop(css, task, task->prio); -+#endif - sched_move_task(task); -+ } - - trace_android_rvh_cpu_cgroup_attach(tset); - } -@@ -11542,6 +11772,13 @@ static struct cftype cpu_legacy_files[] = { - .read_u64 = cpu_uclamp_ls_read_u64, - .write_u64 = cpu_uclamp_ls_write_u64, - }, -+#endif -+#ifdef CONFIG_HMBIRD_SCHED -+ { -+ .name = "hmbird.deadline", -+ .read_u64 = cgroup_read_hmbird_deadline, -+ .write_u64 = cgroup_write_hmbird_deadline, -+ }, - #endif - { } /* Terminate */ - }; -@@ -11591,7 +11828,6 @@ static int cpu_local_stat_show(struct seq_file *sf, - } - - #ifdef CONFIG_FAIR_GROUP_SCHED -- - static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, - struct cftype *cft) - { -diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c -index 423404100c9a..2fe1a734e640 100644 ---- a/kernel/sched/debug.c -+++ b/kernel/sched/debug.c -@@ -376,8 +376,8 @@ static __init int sched_init_debug(void) - - debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); - --#ifdef CONFIG_SCHED_CLASS_EXT -- debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops); -+#ifdef CONFIG_HMBIRD_SCHED -+ debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops); - #endif - return 0; - } -@@ -1006,9 +1006,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - SEQ_printf(m, - "---------------------------------------------------------" - "----------\n"); --#ifdef CONFIG_SLIM_SCHED -- SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop); --#endif - - #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) - #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) -@@ -1102,8 +1099,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - P(dl.runtime); - P(dl.deadline); - } --#ifdef CONFIG_SCHED_CLASS_EXT -- __PS("ext.enabled", p->sched_class == &ext_sched_class); -+#ifdef CONFIG_HMBIRD_SCHED -+ __PS("hmbird.enabled", p->sched_class == &hmbird_sched_class); -+ __PS("sched_prop", get_hmbird_ts(p)->sched_prop); -+ __PS("top_task_prop", get_hmbird_ts(p)->top_task_prop); - #endif - #undef PN_SCHEDSTAT - #undef P_SCHEDSTAT -diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c -deleted file mode 100755 -index 4845d441df2b..000000000000 ---- a/kernel/sched/ext.c -+++ /dev/null -@@ -1,3978 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -- --enum scx_internal_consts { -- SCX_NR_ONLINE_OPS = SCX_OP_IDX(init), -- SCX_DSP_DFL_MAX_BATCH = 32, -- SCX_DSP_MAX_LOOPS = 32, -- SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, --}; -- --enum scx_ops_enable_state { -- SCX_OPS_PREPPING, -- SCX_OPS_ENABLING, -- SCX_OPS_ENABLED, -- SCX_OPS_DISABLING, -- SCX_OPS_DISABLED, --}; -- --static inline struct task_group *css_tg(struct cgroup_subsys_state *css) --{ -- return css ? container_of(css, struct task_group, css) : NULL; --} -- --/* -- * sched_ext_entity->ops_state -- * -- * Used to track the task ownership between the SCX core and the BPF scheduler. -- * State transitions look as follows: -- * -- * NONE -> QUEUEING -> QUEUED -> DISPATCHING -- * ^ | | -- * | v v -- * \-------------------------------/ -- * -- * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -- * sites for explanations on the conditions being waited upon and why they are -- * safe. Transitions out of them into NONE or QUEUED must store_release and the -- * waiters should load_acquire. -- * -- * Tracking scx_ops_state enables sched_ext core to reliably determine whether -- * any given task can be dispatched by the BPF scheduler at all times and thus -- * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -- * to try to dispatch any task anytime regardless of its state as the SCX core -- * can safely reject invalid dispatches. -- */ --enum scx_ops_state { -- SCX_OPSS_NONE, /* owned by the SCX core */ -- SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -- SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ -- SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ -- -- /* -- * QSEQ brands each QUEUED instance so that, when dispatch races -- * dequeue/requeue, the dispatcher can tell whether it still has a claim -- * on the task being dispatched. -- */ -- SCX_OPSS_QSEQ_SHIFT = 2, -- SCX_OPSS_STATE_MASK = (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1, -- SCX_OPSS_QSEQ_MASK = ~SCX_OPSS_STATE_MASK, --}; -- --/* -- * During exit, a task may schedule after losing its PIDs. When disabling the -- * BPF scheduler, we need to be able to iterate tasks in every state to -- * guarantee system safety. Maintain a dedicated task list which contains every -- * task between its fork and eventual free. -- */ --static DEFINE_SPINLOCK(scx_tasks_lock); --static LIST_HEAD(scx_tasks); -- --/* ops enable/disable */ --static struct kthread_worker *scx_ops_helper; --static DEFINE_MUTEX(scx_ops_enable_mutex); --DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); --DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); --static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); --static bool scx_switch_all_req; --static bool scx_switching_all; --DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -- --static struct sched_ext_ops scx_ops; --static bool scx_warned_zero_slice; -- --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); --DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); --static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); -- --struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] = -- { [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT }; -- --static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE); --static struct scx_exit_info scx_exit_info; -- --static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0); -- --/* -- * The maximum amount of time in jiffies that a task may be runnable without -- * being scheduled on a CPU. If this timeout is exceeded, it will trigger -- * scx_ops_error(). -- */ --unsigned long scx_watchdog_timeout; -- --/* -- * The last time the delayed work was run. This delayed work relies on -- * ksoftirqd being able to run to service timer interrupts, so it's possible -- * that this work itself could get wedged. To account for this, we check that -- * it's not stalled in the timer tick, and trigger an error if it is. -- */ --unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; -- --static struct delayed_work scx_watchdog_work; -- --/* idle tracking */ --#ifdef CONFIG_SMP --#ifdef CONFIG_CPUMASK_OFFSTACK --#define CL_ALIGNED_IF_ONSTACK --#else --#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp --#endif -- --static struct { -- cpumask_var_t cpu; -- cpumask_var_t smt; --} idle_masks CL_ALIGNED_IF_ONSTACK; -- --static bool __cacheline_aligned_in_smp scx_has_idle_cpus; --#endif /* CONFIG_SMP */ -- --/* for %SCX_KICK_WAIT */ --static u64 __percpu *scx_kick_cpus_pnt_seqs; -- --/* -- * Direct dispatch marker. -- * -- * Non-NULL values are used for direct dispatch from enqueue path. A valid -- * pointer points to the task currently being enqueued. An ERR_PTR value is used -- * to indicate that direct dispatch has already happened. -- */ --static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); -- --/* dispatch queues */ --static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; -- --static LLIST_HEAD(dsqs_to_free); -- --/* dispatch buf */ --struct scx_dsp_buf_ent { -- struct task_struct *task; -- u64 qseq; -- u64 dsq_id; -- u64 enq_flags; --}; -- --static u32 scx_dsp_max_batch; --static struct scx_dsp_buf_ent __percpu *scx_dsp_buf; -- --struct scx_dsp_ctx { -- struct rq *rq; -- struct rq_flags *rf; -- u32 buf_cursor; -- u32 nr_tasks; --}; -- --static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx); -- --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags); --void scx_bpf_kick_cpu(s32 cpu, u64 flags); -- --struct scx_task_iter { -- struct sched_ext_entity cursor; -- struct task_struct *locked; -- struct rq *rq; -- struct rq_flags rf; --}; -- --#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) -- --/* if the highest set bit is N, return a mask with bits [N+1, 31] set */ --static u32 higher_bits(u32 flags) --{ -- return ~((1 << fls(flags)) - 1); --} -- --/* return the mask with only the highest bit set */ --static u32 highest_bit(u32 flags) --{ -- int bit = fls(flags); -- return bit ? 1 << (bit - 1) : 0; --} -- --/* -- * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX -- * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate -- * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check -- * whether it's running from an allowed context. -- * -- * @mask is constant, always inline to cull the mask calculations. -- */ --static __always_inline void scx_kf_allow(u32 mask) --{ -- /* nesting is allowed only in increasing scx_kf_mask order */ -- WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask, -- "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n", -- current->scx->kf_mask, mask); -- current->scx->kf_mask |= mask; --} -- --static void scx_kf_disallow(u32 mask) --{ -- current->scx->kf_mask &= ~mask; --} -- --#define SCX_CALL_OP(mask, op, args...) \ --do { \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- scx_ops.op(args); \ -- } \ --} while (0) -- --#define SCX_CALL_OP_RET(mask, op, args...) \ --({ \ -- __typeof__(scx_ops.op(args)) __ret; \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- __ret = scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- __ret = scx_ops.op(args); \ -- } \ -- __ret; \ --}) -- --/* -- * Some kfuncs are allowed only on the tasks that are subjects of the -- * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such -- * restrictions, the following SCX_CALL_OP_*() variants should be used when -- * invoking scx_ops operations that take task arguments. These can only be used -- * for non-nesting operations due to the way the tasks are tracked. -- * -- * kfuncs which can only operate on such tasks can in turn use -- * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on -- * the specific task. -- */ --#define SCX_CALL_OP_TASK(mask, op, task, args...) \ --do { \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- SCX_CALL_OP(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ --} while (0) -- --#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ --({ \ -- __typeof__(scx_ops.op(task, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- __ret; \ --}) -- --#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ --({ \ -- __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task0; \ -- current->scx->kf_tasks[1] = task1; \ -- __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- current->scx->kf_tasks[1] = NULL; \ -- __ret; \ --}) -- --//#include "./slim_walt.c" -- --/* @mask is constant, always inline to cull unnecessary branches */ --static __always_inline bool scx_kf_allowed(u32 mask) --{ -- if (unlikely(!(current->scx->kf_mask & mask))) { -- scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", -- mask, current->scx->kf_mask); -- return false; -- } -- -- if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) && -- in_interrupt())) { -- scx_ops_error("sleepable kfunc called from non-sleepable context"); -- return false; -- } -- -- /* -- * Enforce nesting boundaries. e.g. A kfunc which can be called from -- * DISPATCH must not be called if we're running DEQUEUE which is nested -- * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE -- * boundary thanks to the above in_interrupt() check. -- */ -- if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && -- (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { -- scx_ops_error("cpu_release kfunc called from a nested operation"); -- return false; -- } -- -- if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && -- (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) { -- scx_ops_error("dispatch kfunc called from a nested operation"); -- return false; -- } -- -- return true; --} -- --/* see SCX_CALL_OP_TASK() */ --static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, -- struct task_struct *p) --{ -- if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED)) -- return false; -- -- if (unlikely((p != current->scx->kf_tasks[0] && -- p != current->scx->kf_tasks[1]))) { -- scx_ops_error("called on a task not being operated on"); -- return false; -- } -- -- return true; --} -- --/** -- * scx_task_iter_init - Initialize a task iterator -- * @iter: iterator to init -- * -- * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized, -- * @iter must eventually be exited with scx_task_iter_exit(). -- * -- * scx_tasks_lock may be released between this and the first next() call or -- * between any two next() calls. If scx_tasks_lock is released between two -- * next() calls, the caller is responsible for ensuring that the task being -- * iterated remains accessible either through RCU read lock or obtaining a -- * reference count. -- * -- * All tasks which existed when the iteration started are guaranteed to be -- * visited as long as they still exist. -- */ --static void scx_task_iter_init(struct scx_task_iter *iter) --{ -- lockdep_assert_held(&scx_tasks_lock); -- -- iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; -- list_add(&iter->cursor.tasks_node, &scx_tasks); -- iter->locked = NULL; --} -- --/** -- * scx_task_iter_exit - Exit a task iterator -- * @iter: iterator to exit -- * -- * Exit a previously initialized @iter. Must be called with scx_tasks_lock held. -- * If the iterator holds a task's rq lock, that rq lock is released. See -- * scx_task_iter_init() for details. -- */ --static void scx_task_iter_exit(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- if (list_empty(cursor)) -- return; -- -- list_del_init(cursor); --} -- --/** -- * scx_task_iter_next - Next task -- * @iter: iterator to walk -- * -- * Visit the next task. See scx_task_iter_init() for details. -- */ --static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- struct sched_ext_entity *pos; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- list_for_each_entry(pos, cursor, tasks_node) { -- if (&pos->tasks_node == &scx_tasks) -- return NULL; -- if (!(pos->flags & SCX_TASK_CURSOR)) { -- list_move(cursor, &pos->tasks_node); -- return pos->task; -- } -- } -- -- /* can't happen, should always terminate at scx_tasks above */ -- BUG(); --} -- --/** -- * scx_task_iter_next_filtered - Next non-idle task -- * @iter: iterator to walk -- * -- * Visit the next non-idle task. See scx_task_iter_init() for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- while ((p = scx_task_iter_next(iter))) { -- if (!is_idle_task(p)) -- return p; -- } -- return NULL; --} -- --/** -- * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked -- * @iter: iterator to walk -- * -- * Visit the next non-idle task with its rq lock held. See scx_task_iter_init() -- * for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered_locked(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- p = scx_task_iter_next_filtered(iter); -- if (!p) -- return NULL; -- -- iter->rq = task_rq_lock(p, &iter->rf); -- iter->locked = p; -- return p; --} -- --static enum scx_ops_enable_state scx_ops_enable_state(void) --{ -- return atomic_read(&scx_ops_enable_state_var); --} -- --static enum scx_ops_enable_state --scx_ops_set_enable_state(enum scx_ops_enable_state to) --{ -- return atomic_xchg(&scx_ops_enable_state_var, to); --} -- --static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, -- enum scx_ops_enable_state from) --{ -- int from_v = from; -- -- return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); --} -- --static bool scx_ops_disabling(void) --{ -- return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING); --} -- --/** -- * wait_ops_state - Busy-wait the specified ops state to end -- * @p: target task -- * @opss: state to wait the end of -- * -- * Busy-wait for @p to transition out of @opss. This can only be used when the -- * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also -- * has load_acquire semantics to ensure that the caller can see the updates made -- * in the enqueueing and dispatching paths. -- */ --static void wait_ops_state(struct task_struct *p, u64 opss) --{ -- do { -- cpu_relax(); -- } while (atomic64_read_acquire(&p->scx->ops_state) == opss); --} -- --/** -- * ops_cpu_valid - Verify a cpu number -- * @cpu: cpu number which came from a BPF ops -- * -- * @cpu is a cpu number which came from the BPF scheduler and can be any value. -- * Verify that it is in range and one of the possible cpus. -- */ --static bool ops_cpu_valid(s32 cpu) --{ -- return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); --} -- --/** -- * ops_sanitize_err - Sanitize a -errno value -- * @ops_name: operation to blame on failure -- * @err: -errno value to sanitize -- * -- * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return -- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can -- * cause misbehaviors. For an example, a large negative return from -- * ops.prep_enable() triggers an oops when passed up the call chain because the -- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is -- * handled as a pointer. -- */ --static int ops_sanitize_err(const char *ops_name, s32 err) --{ -- if (err < 0 && err >= -MAX_ERRNO) -- return err; -- -- scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); -- return -EPROTO; --} -- --/** -- * touch_core_sched - Update timestamp used for core-sched task ordering -- * @rq: rq to read clock from, must be locked -- * @p: task to update the timestamp for -- * -- * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to -- * implement global or local-DSQ FIFO ordering for core-sched. Should be called -- * when a task becomes runnable and its turn on the CPU ends (e.g. slice -- * exhaustion). -- */ --static void touch_core_sched(struct rq *rq, struct task_struct *p) --{ --#ifdef CONFIG_SCHED_CORE -- /* -- * It's okay to update the timestamp spuriously. Use -- * sched_core_disabled() which is cheaper than enabled(). -- */ -- if (!sched_core_disabled()) -- p->scx->core_sched_at = rq_clock_task(rq); --#endif --} -- --/** -- * touch_core_sched_dispatch - Update core-sched timestamp on dispatch -- * @rq: rq to read clock from, must be locked -- * @p: task being dispatched -- * -- * If the BPF scheduler implements custom core-sched ordering via -- * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO -- * ordering within each local DSQ. This function is called from dispatch paths -- * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect. -- */ --static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- assert_clock_updated(rq); -- --#ifdef CONFIG_SCHED_CORE -- if (SCX_HAS_OP(core_sched_before)) -- touch_core_sched(rq, p); --#endif --} -- --static void update_curr_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- u64 now = rq_clock_task(rq); -- u64 delta_exec; -- -- if (time_before_eq64(now, curr->se.exec_start)) -- return; -- -- delta_exec = now - curr->se.exec_start; -- curr->se.exec_start = now; -- curr->se.sum_exec_runtime += delta_exec; -- account_group_exec_runtime(curr, delta_exec); -- cgroup_account_cputime(curr, delta_exec); -- -- if (curr->scx->slice != SCX_SLICE_INF) { -- curr->scx->slice -= min(curr->scx->slice, delta_exec); -- if (!curr->scx->slice) -- touch_core_sched(rq, curr); -- } --} -- --static bool scx_dsq_priq_less(struct rb_node *node_a, -- const struct rb_node *node_b) --{ -- const struct sched_ext_entity *a = -- container_of(node_a, struct sched_ext_entity, dsq_node.priq); -- const struct sched_ext_entity *b = -- container_of(node_b, struct sched_ext_entity, dsq_node.priq); -- -- return time_before64(a->dsq_vtime, b->dsq_vtime); --} -- --static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, -- u64 enq_flags) --{ -- bool is_local = dsq->id == SCX_DSQ_LOCAL; -- -- WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo)); -- WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq)); -- -- if (!is_local) { -- raw_spin_lock(&dsq->lock); -- if (unlikely(dsq->id == SCX_DSQ_INVALID)) { -- scx_ops_error("attempting to dispatch to a destroyed dsq"); -- /* fall back to the global dsq */ -- raw_spin_unlock(&dsq->lock); -- dsq = &scx_dsq_global; -- raw_spin_lock(&dsq->lock); -- } -- } -- -- if (enq_flags & SCX_ENQ_DSQ_PRIQ) { -- p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; -- rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq, -- scx_dsq_priq_less); -- } else { -- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) -- list_add(&p->scx->dsq_node.fifo, &dsq->fifo); -- else -- list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo); -- } -- dsq->nr++; -- p->scx->dsq = dsq; -- -- /* -- * We're transitioning out of QUEUEING or DISPATCHING. store_release to -- * match waiters' load_acquire. -- */ -- if (enq_flags & SCX_ENQ_CLEAR_OPSS) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- if (is_local) { -- struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq); -- struct rq *rq = scx->rq; -- bool preempt = false; -- -- if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && -- rq->curr->sched_class == &ext_sched_class) { -- rq->curr->scx->slice = 0; -- preempt = true; -- } -- -- if (preempt || sched_class_above(&ext_sched_class, -- rq->curr->sched_class)) -- resched_curr(rq); -- } else { -- raw_spin_unlock(&dsq->lock); -- } --} -- --static void task_unlink_from_dsq(struct task_struct *p, -- struct scx_dispatch_q *dsq) --{ -- if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { -- rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; -- } else { -- list_del_init(&p->scx->dsq_node.fifo); -- } -- --} -- --static bool task_linked_on_dsq(struct task_struct *p) --{ -- return !list_empty(&p->scx->dsq_node.fifo) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq); --} -- --static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p) --{ -- struct scx_dispatch_q *dsq = p->scx->dsq; -- bool is_local = dsq == &scx_rq->local_dsq; -- -- if (!dsq) { -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- /* -- * When dispatching directly from the BPF scheduler to a local -- * DSQ, the task isn't associated with any DSQ but -- * @p->scx->holding_cpu may be set under the protection of -- * %SCX_OPSS_DISPATCHING. -- */ -- if (p->scx->holding_cpu >= 0) -- p->scx->holding_cpu = -1; -- return; -- } -- -- if (!is_local) -- raw_spin_lock(&dsq->lock); -- -- /* -- * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node -- * can't change underneath us. -- */ -- if (p->scx->holding_cpu < 0) { -- /* @p must still be on @dsq, dequeue */ -- WARN_ON_ONCE(!task_linked_on_dsq(p)); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- } else { -- /* -- * We're racing against dispatch_to_local_dsq() which already -- * removed @p from @dsq and set @p->scx->holding_cpu. Clear the -- * holding_cpu which tells dispatch_to_local_dsq() that it lost -- * the race. -- */ -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- p->scx->holding_cpu = -1; -- } -- p->scx->dsq = NULL; -- -- if (!is_local) -- raw_spin_unlock(&dsq->lock); --} -- --static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) --{ -- lockdep_assert(rcu_read_lock_any_held()); -- -- return &scx_dsq_global; --} -- --static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, -- struct task_struct *p) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id == SCX_DSQ_LOCAL) -- return &rq->scx->local_dsq; -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", -- dsq_id, p->comm, p->pid); -- return &scx_dsq_global; -- } -- -- return dsq; --} -- --static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- -- /* @p must match the task which is being enqueued */ -- if (unlikely(p != ddsp_task)) { -- if (IS_ERR(ddsp_task)) -- scx_ops_error("%s[%d] already direct-dispatched", -- p->comm, p->pid); -- else -- scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]", -- ddsp_task->comm, ddsp_task->pid, -- p->comm, p->pid); -- return; -- } -- -- /* -- * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because -- * dispatching to the local DSQ of a different CPU requires unlocking -- * the current rq which isn't allowed in the enqueue path. Use -- * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL. -- */ -- if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) { -- scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch"); -- return; -- } -- -- touch_core_sched_dispatch(task_rq(p), p); -- -- dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- -- /* -- * Mark that dispatch already happened by spoiling direct_dispatch_task -- * with a non-NULL value which can never match a valid task pointer. -- */ -- __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); --} -- --static bool test_rq_online(struct rq *rq) --{ --#ifdef CONFIG_SMP -- return rq->online; --#else -- return true; --#endif --} -- --static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- int sticky_cpu) --{ -- struct task_struct **ddsp_taskp; -- u64 qseq; -- -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- if (p->scx->flags & SCX_TASK_ENQ_LOCAL) { -- enq_flags |= SCX_ENQ_LOCAL; -- p->scx->flags &= ~SCX_TASK_ENQ_LOCAL; -- } -- -- /* rq migration */ -- if (sticky_cpu == cpu_of(rq)) -- goto local_norefill; -- -- /* -- * If !rq->online, we already told the BPF scheduler that the CPU is -- * offline. We're just trying to on/offline the CPU. Don't bother the -- * BPF scheduler. -- */ -- if (unlikely(!test_rq_online(rq))) -- goto local; -- -- /* see %SCX_OPS_ENQ_EXITING */ -- if (!static_branch_unlikely(&scx_ops_enq_exiting) && -- unlikely(p->flags & PF_EXITING)) -- goto local; -- -- /* see %SCX_OPS_ENQ_LAST */ -- if (!static_branch_unlikely(&scx_ops_enq_last) && -- (enq_flags & SCX_ENQ_LAST)) -- goto local; -- -- if (!SCX_HAS_OP(enqueue)) { -- if (enq_flags & SCX_ENQ_LOCAL) -- goto local; -- else -- goto global; -- } -- -- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ -- qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; -- -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq); -- -- ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); -- WARN_ON_ONCE(*ddsp_taskp); -- *ddsp_taskp = p; -- -- SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); -- -- /* -- * If not directly dispatched, QUEUEING isn't clear yet and dispatch or -- * dequeue may be waiting. The store_release matches their load_acquire. -- */ -- if (*ddsp_taskp == p) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq); -- *ddsp_taskp = NULL; -- return; -- --local: -- /* -- * For task-ordering, slice refill must be treated as implying the end -- * of the current slice. Otherwise, the longer @p stays on the CPU, the -- * higher priority it becomes from scx_prio_less()'s POV. -- */ -- touch_core_sched(rq, p); -- p->scx->slice = SCX_SLICE_DFL; --local_norefill: -- dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags); -- return; -- --global: -- touch_core_sched(rq, p); /* see the comment in local: */ -- p->scx->slice = SCX_SLICE_DFL; -- dispatch_enqueue(&scx_dsq_global, p, enq_flags); --} -- --static bool watchdog_task_watched(const struct task_struct *p) --{ -- return !list_empty(&p->scx->watchdog_node); --} -- --static void watchdog_watch_task(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- if (p->scx->flags & SCX_TASK_WATCHDOG_RESET) -- p->scx->runnable_at = jiffies; -- p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET; -- list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list); --} -- --static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) --{ -- list_del_init(&p->scx->watchdog_node); -- if (reset_timeout) -- p->scx->flags |= SCX_TASK_WATCHDOG_RESET; --} -- --static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) --{ -- int sticky_cpu = p->scx->sticky_cpu; -- -- enq_flags |= rq->scx->extra_enq_flags; -- -- if (sticky_cpu >= 0) -- p->scx->sticky_cpu = -1; -- -- /* -- * Restoring a running task will be immediately followed by -- * set_next_task_scx() which expects the task to not be on the BPF -- * scheduler as tasks can only start running through local DSQs. Force -- * direct-dispatch into the local DSQ by setting the sticky_cpu. -- */ -- if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -- sticky_cpu = cpu_of(rq); -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- WARN_ON_ONCE(!watchdog_task_watched(p)); -- return; -- } -- -- watchdog_watch_task(rq, p); -- p->scx->flags |= SCX_TASK_QUEUED; -- rq->scx->nr_running++; -- add_nr_running(rq, 1); -- -- if (SCX_HAS_OP(runnable)) -- SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); -- -- if (enq_flags & SCX_ENQ_WAKEUP) -- touch_core_sched(rq, p); -- -- do_enqueue_task(rq, p, enq_flags, sticky_cpu); --} -- --static void ops_dequeue(struct task_struct *p, u64 deq_flags) --{ -- u64 opss; -- -- watchdog_unwatch_task(p, false); -- -- /* acquire ensures that we see the preceding updates on QUEUED */ -- opss = atomic64_read_acquire(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_NONE: -- break; -- case SCX_OPSS_QUEUEING: -- /* -- * QUEUEING is started and finished while holding @p's rq lock. -- * As we're holding the rq lock now, we shouldn't see QUEUEING. -- */ -- BUG(); -- case SCX_OPSS_QUEUED: -- if (SCX_HAS_OP(dequeue)) -- SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); -- -- if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_NONE)) -- break; -- fallthrough; -- case SCX_OPSS_DISPATCHING: -- /* -- * If @p is being dispatched from the BPF scheduler to a DSQ, -- * wait for the transfer to complete so that @p doesn't get -- * added to its DSQ after dequeueing is complete. -- * -- * As we're waiting on DISPATCHING with the rq locked, the -- * dispatching side shouldn't try to lock the rq while -- * DISPATCHING is set. See dispatch_to_local_dsq(). -- * -- * DISPATCHING shouldn't have qseq set and control can reach -- * here with NONE @opss from the above QUEUED case block. -- * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. -- */ -- wait_ops_state(p, SCX_OPSS_DISPATCHING); -- BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- break; -- } --} -- --static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) --{ -- struct scx_rq *scx_rq = rq->scx; -- -- if (!(p->scx->flags & SCX_TASK_QUEUED)) { -- WARN_ON_ONCE(watchdog_task_watched(p)); -- return; -- } -- -- ops_dequeue(p, deq_flags); -- -- /* -- * A currently running task which is going off @rq first gets dequeued -- * and then stops running. As we want running <-> stopping transitions -- * to be contained within runnable <-> quiescent transitions, trigger -- * ->stopping() early here instead of in put_prev_task_scx(). -- * -- * @p may go through multiple stopping <-> running transitions between -- * here and put_prev_task_scx() if task attribute changes occur while -- * balance_scx() leaves @rq unlocked. However, they don't contain any -- * information meaningful to the BPF scheduler and can be suppressed by -- * skipping the callbacks if the task is !QUEUED. -- */ -- if (SCX_HAS_OP(stopping) && task_current(rq, p)) { -- update_curr_scx(rq); -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); -- } -- -- //if(task_current(rq, p)) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- if (SCX_HAS_OP(quiescent)) -- SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); -- -- if (deq_flags & SCX_DEQ_SLEEP) -- p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP; -- else -- p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP; -- -- p->scx->flags &= ~SCX_TASK_QUEUED; -- BUG_ON(!scx_rq->nr_running); -- scx_rq->nr_running--; -- sub_nr_running(rq, 1); -- -- dispatch_dequeue(scx_rq, p); --} -- --static void yield_task_scx(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); -- else -- p->scx->slice = 0; --} -- --static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) --{ -- struct task_struct *from = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); -- else -- return false; --} -- --#ifdef CONFIG_SMP --/** -- * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -- * @rq: rq to move the task into, currently locked -- * @p: task to move -- * @enq_flags: %SCX_ENQ_* -- * -- * Move @p which is currently on a different rq to @rq's local DSQ. The caller -- * must: -- * -- * 1. Start with exclusive access to @p either through its DSQ lock or -- * %SCX_OPSS_DISPATCHING flag. -- * -- * 2. Set @p->scx->holding_cpu to raw_smp_processor_id(). -- * -- * 3. Remember task_rq(@p). Release the exclusive access so that we don't -- * deadlock with dequeue. -- * -- * 4. Lock @rq and the task_rq from #3. -- * -- * 5. Call this function. -- * -- * Returns %true if @p was successfully moved. %false after racing dequeue and -- * losing. -- */ --static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -- u64 enq_flags) --{ -- struct rq *task_rq; -- -- lockdep_assert_rq_held(rq); -- -- /* -- * If dequeue got to @p while we were trying to lock both rq's, it'd -- * have cleared @p->scx->holding_cpu to -1. While other cpus may have -- * updated it to different values afterwards, as this operation can't be -- * preempted or recurse, @p->scx->holding_cpu can never become -- * raw_smp_processor_id() again before we're done. Thus, we can tell -- * whether we lost to dequeue by testing whether @p->scx->holding_cpu is -- * still raw_smp_processor_id(). -- * -- * See dispatch_dequeue() for the counterpart. -- */ -- if (unlikely(p->scx->holding_cpu != raw_smp_processor_id())) -- return false; -- -- /* @p->rq couldn't have changed if we're still the holding cpu */ -- task_rq = task_rq(p); -- lockdep_assert_rq_held(task_rq); -- -- WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)); -- deactivate_task(task_rq, p, 0); -- set_task_cpu(p, cpu_of(rq)); -- p->scx->sticky_cpu = cpu_of(rq); -- -- /* -- * We want to pass scx-specific enq_flags but activate_task() will -- * truncate the upper 32 bit. As we own @rq, we can pass them through -- * @rq->scx->extra_enq_flags instead. -- */ -- WARN_ON_ONCE(rq->scx->extra_enq_flags); -- rq->scx->extra_enq_flags = enq_flags; -- activate_task(rq, p, 0); -- rq->scx->extra_enq_flags = 0; -- -- return true; --} -- --/** -- * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * We're holding @rq lock and trying to dispatch a task from @src_rq to -- * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether -- * @rq stays locked isn't important as long as the state is restored after -- * dispatch_to_local_dsq_unlock(). -- */ --static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- rq_unpin_lock(rq, rf); -- -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(rq); -- raw_spin_rq_lock(dst_rq); -- } else if (rq == src_rq) { -- double_lock_balance(rq, dst_rq); -- rq_repin_lock(rq, rf); -- } else if (rq == dst_rq) { -- double_lock_balance(rq, src_rq); -- rq_repin_lock(rq, rf); -- } else { -- raw_spin_rq_unlock(rq); -- double_rq_lock(src_rq, dst_rq); -- } --} -- --/** -- * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock() -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return. -- */ --static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } else if (rq == src_rq) { -- double_unlock_balance(rq, dst_rq); -- } else if (rq == dst_rq) { -- double_unlock_balance(rq, src_rq); -- } else { -- double_rq_unlock(src_rq, dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } --} --#endif /* CONFIG_SMP */ -- -- --static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq) --{ -- return likely(test_rq_online(rq)) && !is_migration_disabled(p) && -- cpumask_test_cpu(cpu_of(rq), p->cpus_ptr); --} -- --static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -- struct scx_dispatch_q *dsq) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rb_node *rb_node; -- struct rq *task_rq; -- bool moved = false; --retry: -- if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -- return false; -- -- raw_spin_lock(&dsq->lock); -- -- list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- for (rb_node = rb_first_cached(&dsq->priq); rb_node; -- rb_node = rb_next(rb_node)) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- raw_spin_unlock(&dsq->lock); -- return false; -- --this_rq: -- /* @dsq is locked and @p is on this rq */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo); -- dsq->nr--; -- scx_rq->local_dsq.nr++; -- p->scx->dsq = &scx_rq->local_dsq; -- raw_spin_unlock(&dsq->lock); -- return true; -- --remote_rq: --#ifdef CONFIG_SMP -- /* -- * @dsq is locked and @p is on a remote rq. @p is currently protected by -- * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -- * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -- * rq lock or fail, do a little dancing from our side. See -- * move_task_to_local_dsq(). -- */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- p->scx->holding_cpu = raw_smp_processor_id(); -- raw_spin_unlock(&dsq->lock); -- -- rq_unpin_lock(rq, rf); -- double_lock_balance(rq, task_rq); -- rq_repin_lock(rq, rf); -- -- moved = move_task_to_local_dsq(rq, p, 0); -- -- double_unlock_balance(rq, task_rq); --#endif /* CONFIG_SMP */ -- if (likely(moved)) -- return true; -- goto retry; --} -- --enum dispatch_to_local_dsq_ret { -- DTL_DISPATCHED, /* successfully dispatched */ -- DTL_LOST, /* lost race to dequeue */ -- DTL_NOT_LOCAL, /* destination is not a local DSQ */ -- DTL_INVALID, /* invalid local dsq_id */ --}; -- --/** -- * dispatch_to_local_dsq - Dispatch a task to a local dsq -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @dsq_id: destination dsq ID -- * @p: task to dispatch -- * @enq_flags: %SCX_ENQ_* -- * -- * We're holding @rq lock and want to dispatch @p to the local DSQ identified by -- * @dsq_id. This function performs all the synchronization dancing needed -- * because local DSQs are protected with rq locks. -- * -- * The caller must have exclusive ownership of @p (e.g. through -- * %SCX_OPSS_DISPATCHING). -- */ --static enum dispatch_to_local_dsq_ret --dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id, -- struct task_struct *p, u64 enq_flags) --{ -- struct rq *src_rq = task_rq(p); -- struct rq *dst_rq; -- -- /* -- * We're synchronized against dequeue through DISPATCHING. As @p can't -- * be dequeued, its task_rq and cpus_allowed are stable too. -- */ -- if (dsq_id == SCX_DSQ_LOCAL) { -- dst_rq = rq; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]", -- cpu, p->comm, p->pid); -- return DTL_INVALID; -- } -- dst_rq = cpu_rq(cpu); -- } else { -- return DTL_NOT_LOCAL; -- } -- -- /* if dispatching to @rq that @p is already on, no lock dancing needed */ -- if (rq == src_rq && rq == dst_rq) { -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags | SCX_ENQ_CLEAR_OPSS); -- return DTL_DISPATCHED; -- } -- --#ifdef CONFIG_SMP -- if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { -- struct rq *locked_dst_rq = dst_rq; -- bool dsp; -- -- /* -- * @p is on a possibly remote @src_rq which we need to lock to -- * move the task. If dequeue is in progress, it'd be locking -- * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq -- * lock while holding DISPATCHING. -- * -- * As DISPATCHING guarantees that @p is wholly ours, we can -- * pretend that we're moving from a DSQ and use the same -- * mechanism - mark the task under transfer with holding_cpu, -- * release DISPATCHING and then follow the same protocol. -- */ -- p->scx->holding_cpu = raw_smp_processor_id(); -- -- /* store_release ensures that dequeue sees the above */ -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq); -- -- /* -- * We don't require the BPF scheduler to avoid dispatching to -- * offline CPUs mostly for convenience but also because CPUs can -- * go offline between scx_bpf_dispatch() calls and here. If @p -- * is destined to an offline CPU, queue it on its current CPU -- * instead, which should always be safe. As this is an allowed -- * behavior, don't trigger an ops error. -- */ -- if (unlikely(!test_rq_online(dst_rq))) -- dst_rq = src_rq; -- -- if (src_rq == dst_rq) { -- /* -- * As @p is staying on the same rq, there's no need to -- * go through the full deactivate/activate cycle. -- * Optimize by abbreviating the operations in -- * move_task_to_local_dsq(). -- */ -- dsp = p->scx->holding_cpu == raw_smp_processor_id(); -- if (likely(dsp)) { -- p->scx->holding_cpu = -1; -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags); -- } -- } else { -- dsp = move_task_to_local_dsq(dst_rq, p, enq_flags); -- } -- -- /* if the destination CPU is idle, wake it up */ -- if (dsp && p->sched_class > dst_rq->curr->sched_class) -- resched_curr(dst_rq); -- -- dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq); -- -- return dsp ? DTL_DISPATCHED : DTL_LOST; -- } --#endif /* CONFIG_SMP */ -- -- scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", -- cpu_of(dst_rq), p->comm, p->pid); -- return DTL_INVALID; --} -- --/** -- * finish_dispatch - Asynchronously finish dispatching a task -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @p: task to finish dispatching -- * @qseq_at_dispatch: qseq when @p started getting dispatched -- * @dsq_id: destination DSQ ID -- * @enq_flags: %SCX_ENQ_* -- * -- * Dispatching to local DSQs may need to wait for queueing to complete or -- * require rq lock dancing. As we don't wanna do either while inside -- * ops.dispatch() to avoid locking order inversion, we split dispatching into -- * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the -- * task and its qseq. Once ops.dispatch() returns, this function is called to -- * finish up. -- * -- * There is no guarantee that @p is still valid for dispatching or even that it -- * was valid in the first place. Make sure that the task is still owned by the -- * BPF scheduler and claim the ownership before dispatching. -- */ --static void finish_dispatch(struct rq *rq, struct rq_flags *rf, -- struct task_struct *p, u64 qseq_at_dispatch, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- u64 opss; -- -- touch_core_sched_dispatch(rq, p); --retry: -- /* -- * No need for _acquire here. @p is accessed only after a successful -- * try_cmpxchg to DISPATCHING. -- */ -- opss = atomic64_read(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_DISPATCHING: -- case SCX_OPSS_NONE: -- /* someone else already got to it */ -- return; -- case SCX_OPSS_QUEUED: -- /* -- * If qseq doesn't match, @p has gone through at least one -- * dispatch/dequeue and re-enqueue cycle between -- * scx_bpf_dispatch() and here and we have no claim on it. -- */ -- if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) -- return; -- -- /* -- * While we know @p is accessible, we don't yet have a claim on -- * it - the BPF scheduler is allowed to dispatch tasks -- * spuriously and there can be a racing dequeue attempt. Let's -- * claim @p by atomically transitioning it from QUEUED to -- * DISPATCHING. -- */ -- if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_DISPATCHING))) -- break; -- goto retry; -- case SCX_OPSS_QUEUEING: -- /* -- * do_enqueue_task() is in the process of transferring the task -- * to the BPF scheduler while holding @p's rq lock. As we aren't -- * holding any kernel or BPF resource that the enqueue path may -- * depend upon, it's safe to wait. -- */ -- wait_ops_state(p, opss); -- goto retry; -- } -- -- BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) { -- case DTL_DISPATCHED: -- break; -- case DTL_LOST: -- break; -- case DTL_INVALID: -- dsq_id = SCX_DSQ_GLOBAL; -- fallthrough; -- case DTL_NOT_LOCAL: -- dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()), -- dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- break; -- } --} -- --static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- u32 u; -- -- for (u = 0; u < dspc->buf_cursor; u++) { -- struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u]; -- -- finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id, -- ent->enq_flags); -- } -- -- dspc->nr_tasks += dspc->buf_cursor; -- dspc->buf_cursor = 0; --} -- --static int balance_one(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf, bool local) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- bool prev_on_scx = prev->sched_class == &ext_sched_class; -- int nr_loops = SCX_DSP_MAX_LOOPS; -- -- lockdep_assert_rq_held(rq); -- -- if (static_branch_unlikely(&scx_ops_cpu_preempt) && -- unlikely(rq->scx->cpu_released)) { -- /* -- * If the previous sched_class for the current CPU was not SCX, -- * notify the BPF scheduler that it again has control of the -- * core. This callback complements ->cpu_release(), which is -- * emitted in scx_notify_pick_next_task(). -- */ -- if (SCX_HAS_OP(cpu_acquire)) -- SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq), -- NULL); -- rq->scx->cpu_released = false; -- } -- -- if (prev_on_scx) { -- WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP)); -- update_curr_scx(rq); -- -- /* -- * If @prev is runnable & has slice left, it has priority and -- * fetching more just increases latency for the fetched tasks. -- * Tell put_prev_task_scx() to put @prev on local_dsq. If the -- * BPF scheduler wants to handle this explicitly, it should -- * implement ->cpu_released(). -- * -- * See scx_ops_disable_workfn() for the explanation on the -- * disabling() test. -- * -- * When balancing a remote CPU for core-sched, there won't be a -- * following put_prev_task_scx() call and we don't own -- * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the -- * same conditions later and pick @rq->curr accordingly. -- */ -- if ((prev->scx->flags & SCX_TASK_QUEUED) && -- prev->scx->slice && !scx_ops_disabling()) { -- if (local) -- prev->scx->flags |= SCX_TASK_BAL_KEEP; -- return 1; -- } -- } -- -- /* if there already are tasks to run, nothing to do */ -- if (scx_rq->local_dsq.nr) -- return 1; -- -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- if (!SCX_HAS_OP(dispatch)) -- return 0; -- -- dspc->rq = rq; -- dspc->rf = rf; -- -- /* -- * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, -- * the local DSQ might still end up empty after a successful -- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() -- * produced some tasks, retry. The BPF scheduler may depend on this -- * looping behavior to simplify its implementation. -- */ -- do { -- dspc->nr_tasks = 0; -- -- SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), -- prev_on_scx ? prev : NULL); -- -- flush_dispatch_buf(rq, rf); -- -- if (scx_rq->local_dsq.nr) -- return 1; -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- /* -- * ops.dispatch() can trap us in this loop by repeatedly -- * dispatching ineligible tasks. Break out once in a while to -- * allow the watchdog to run. As IRQ can't be enabled in -- * balance(), we want to complete this scheduling cycle and then -- * start a new one. IOW, we want to call resched_curr() on the -- * next, most likely idle, task, not the current one. Use -- * scx_bpf_kick_cpu() for deferred kicking. -- */ -- if (unlikely(!--nr_loops)) { -- scx_bpf_kick_cpu(cpu_of(rq), 0); -- break; -- } -- } while (dspc->nr_tasks); -- -- return 0; --} -- --static int balance_scx(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf) --{ -- int ret; -- -- ret = balance_one(rq, prev, rf, true); --#ifdef CONFIG_SCHED_SMT -- /* -- * When core-sched is enabled, this ops.balance() call will be followed -- * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx() -- * on the SMT siblings. Balance the siblings too. -- */ -- if (sched_core_enabled(rq)) { -- const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); -- int scpu; -- -- for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { -- struct rq *srq = cpu_rq(scpu); -- struct rq_flags srf; -- struct task_struct *sprev = srq->curr; -- -- /* -- * While core-scheduling, rq lock is shared among -- * siblings but the debug annotations and rq clock -- * aren't. Do pinning dance to transfer the ownership. -- */ -- WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); -- rq_unpin_lock(rq, rf); -- rq_pin_lock(srq, &srf); -- -- update_rq_clock(srq); -- balance_one(srq, sprev, &srf, false); -- -- rq_unpin_lock(srq, &srf); -- rq_repin_lock(rq, rf); -- } -- } --#endif -- return ret; --} -- --static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) --{ -- if (p->scx->flags & SCX_TASK_QUEUED) { -- /* -- * Core-sched might decide to execute @p before it is -- * dispatched. Call ops_dequeue() to notify the BPF scheduler. -- */ -- ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); -- dispatch_dequeue(rq->scx, p); -- } -- -- p->se.exec_start = rq_clock_task(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, running, p); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock); -- -- watchdog_unwatch_task(p, true); -- -- /* -- * @p is getting newly scheduled or got kicked after someone updated its -- * slice. Refresh whether tick can be stopped. See can_stop_tick_scx(). -- */ -- if ((p->scx->slice == SCX_SLICE_INF) != -- (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) { -- if (p->scx->slice == SCX_SLICE_INF) -- rq->scx->flags |= SCX_RQ_CAN_STOP_TICK; -- else -- rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK; -- -- sched_update_tick_dependency(rq); -- } --} -- --static void put_prev_task_scx(struct rq *rq, struct task_struct *p) --{ --#ifndef CONFIG_SMP -- /* -- * UP workaround. -- * -- * Because SCX may transfer tasks across CPUs during dispatch, dispatch -- * is performed from its balance operation which isn't called in UP. -- * Let's work around by calling it from the operations which come right -- * after. -- * -- * 1. If the prev task is on SCX, pick_next_task() calls -- * .put_prev_task() right after. As .put_prev_task() is also called -- * from other places, we need to distinguish the calls which can be -- * done by looking at the previous task's state - if still queued or -- * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task(). -- * This case is handled here. -- * -- * 2. If the prev task is not on SCX, the first following call into SCX -- * will be .pick_next_task(), which is covered by calling -- * balance_scx() from pick_next_task_scx(). -- * -- * Note that we can't merge the first case into the second as -- * balance_scx() must be called before the previous SCX task goes -- * through put_prev_task_scx(). -- * -- * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf. -- * Pass in %NULL. -- */ -- if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP)) -- balance_scx(rq, p, NULL); --#endif -- -- update_curr_scx(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- -- /* -- * If we're being called from put_prev_task_balance(), balance_scx() may -- * have decided that @p should keep running. -- */ -- if (p->scx->flags & SCX_TASK_BAL_KEEP) { -- p->scx->flags &= ~SCX_TASK_BAL_KEEP; -- watchdog_watch_task(rq, p); -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- watchdog_watch_task(rq, p); -- -- /* -- * If @p has slice left and balance_scx() didn't tag it for -- * keeping, @p is getting preempted by a higher priority -- * scheduler class or core-sched forcing a different task. Leave -- * it at the head of the local DSQ. -- */ -- if (p->scx->slice && !scx_ops_disabling()) { -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- /* -- * If we're in the pick_next_task path, balance_scx() should -- * have already populated the local DSQ if there are any other -- * available tasks. If empty, tell ops.enqueue() that @p is the -- * only one available for this cpu. ops.enqueue() should put it -- * on the local DSQ so that the subsequent pick_next_task_scx() -- * can find the task unless it wants to trigger a separate -- * follow-up scheduling event. -- */ -- if (list_empty(&rq->scx->local_dsq.fifo)) -- do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1); -- else -- do_enqueue_task(rq, p, 0, -1); -- } --} -- --static struct task_struct *first_local_task(struct rq *rq) --{ -- struct rb_node *rb_node; -- struct sched_ext_entity *entity; -- -- if (!list_empty(&rq->scx->local_dsq.fifo)) { -- entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo); -- return entity->task; -- } -- -- rb_node = rb_first_cached(&rq->scx->local_dsq.priq); -- if (rb_node) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- return entity->task; -- } -- -- return NULL; --} -- --static struct task_struct *pick_next_task_scx(struct rq *rq) --{ -- struct task_struct *p; -- --#ifndef CONFIG_SMP -- /* UP workaround - see the comment at the head of put_prev_task_scx() */ -- if (unlikely(rq->curr->sched_class != &ext_sched_class)) -- balance_scx(rq, rq->curr, NULL); --#endif -- -- p = first_local_task(rq); -- if (!p) -- return NULL; -- -- if (unlikely(!p->scx->slice)) { -- if (!scx_ops_disabling() && !scx_warned_zero_slice) { -- printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n", -- p->comm, p->pid); -- scx_warned_zero_slice = true; -- } -- p->scx->slice = SCX_SLICE_DFL; -- } -- -- set_next_task_scx(rq, p, true); -- -- return p; --} -- --#ifdef CONFIG_SCHED_CORE --/** -- * scx_prio_less - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Core-sched is implemented as an additional scheduling layer on top of the -- * usual sched_class'es and needs to find out the expected task ordering. For -- * SCX, core-sched calls this function to interrogate the task ordering. -- * -- * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used -- * to implement the default task ordering. The older the timestamp, the higher -- * prority the task - the global FIFO ordering matching the default scheduling -- * behavior. -- * -- * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to -- * implement FIFO ordering within each local DSQ. See pick_task_scx(). -- */ --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi) --{ -- /* -- * The const qualifiers are dropped from task_struct pointers when -- * calling ops.core_sched_before(). Accesses are controlled by the -- * verifier. -- */ -- if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling()) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, -- (struct task_struct *)a, -- (struct task_struct *)b); -- else -- return time_after64(a->scx->core_sched_at, b->scx->core_sched_at); --} -- --/** -- * pick_task_scx - Pick a candidate task for core-sched -- * @rq: rq to pick the candidate task from -- * -- * Core-sched calls this function on each SMT sibling to determine the next -- * tasks to run on the SMT siblings. balance_one() has been called on all -- * siblings and put_prev_task_scx() has been called only for the current CPU. -- * -- * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look -- * at the first task in the local dsq. @rq->curr has to be considered explicitly -- * to mimic %SCX_TASK_BAL_KEEP. -- */ --static struct task_struct *pick_task_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- struct task_struct *first = first_local_task(rq); -- -- if (curr->scx->flags & SCX_TASK_QUEUED) { -- /* is curr the only runnable task? */ -- if (!first) -- return curr; -- -- /* -- * Does curr trump first? We can always go by core_sched_at for -- * this comparison as it represents global FIFO ordering when -- * the default core-sched ordering is used and local-DSQ FIFO -- * ordering otherwise. -- * -- * We can have a task with an earlier timestamp on the DSQ. For -- * example, when a current task is preempted by a sibling -- * picking a different cookie, the task would be requeued at the -- * head of the local DSQ with an earlier timestamp than the -- * core-sched picked next task. Besides, the BPF scheduler may -- * dispatch any tasks to the local DSQ anytime. -- */ -- if (curr->scx->slice && time_before64(curr->scx->core_sched_at, -- first->scx->core_sched_at)) -- return curr; -- } -- -- return first; /* this may be %NULL */ --} --#endif /* CONFIG_SCHED_CORE */ -- --static enum scx_cpu_preempt_reason --preempt_reason_from_class(const struct sched_class *class) --{ --#ifdef CONFIG_SMP -- if (class == &stop_sched_class) -- return SCX_CPU_PREEMPT_STOP; --#endif -- if (class == &dl_sched_class) -- return SCX_CPU_PREEMPT_DL; -- if (class == &rt_sched_class) -- return SCX_CPU_PREEMPT_RT; -- return SCX_CPU_PREEMPT_UNKNOWN; --} -- --void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task, -- const struct sched_class *active) --{ -- lockdep_assert_rq_held(rq); -- -- /* -- * The callback is conceptually meant to convey that the CPU is no -- * longer under the control of SCX. Therefore, don't invoke the -- * callback if the CPU is is staying on SCX, or going idle (in which -- * case the SCX scheduler has actively decided not to schedule any -- * tasks on the CPU). -- */ -- if (likely(active >= &ext_sched_class)) -- return; -- -- /* -- * At this point we know that SCX was preempted by a higher priority -- * sched_class, so invoke the ->cpu_release() callback if we have not -- * done so already. We only send the callback once between SCX being -- * preempted, and it regaining control of the CPU. -- * -- * ->cpu_release() complements ->cpu_acquire(), which is emitted the -- * next time that balance_scx() is invoked. -- */ -- if (!rq->scx->cpu_released) { -- if (SCX_HAS_OP(cpu_release)) { -- struct scx_cpu_release_args args = { -- .reason = preempt_reason_from_class(active), -- .task = task, -- }; -- -- SCX_CALL_OP(SCX_KF_CPU_RELEASE, -- cpu_release, cpu_of(rq), &args); -- } -- rq->scx->cpu_released = true; -- } --} -- --#ifdef CONFIG_SMP -- --static bool test_and_clear_cpu_idle(int cpu) --{ -- if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -- if (cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- return true; -- } else { -- return false; -- } --} -- --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- int cpu; -- -- do { -- cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); -- if (cpu < nr_cpu_ids) { -- const struct cpumask *sbm = topology_sibling_cpumask(cpu); -- -- /* -- * If offline, @cpu is not its own sibling and we can -- * get caught in an infinite loop as @cpu is never -- * cleared from idle_masks.smt. Clear @cpu directly in -- * such cases. -- */ -- if (likely(cpumask_test_cpu(cpu, sbm))) -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm); -- else -- cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu)); -- } else { -- cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -- if (cpu >= nr_cpu_ids) -- return -EBUSY; -- } -- } while (!test_and_clear_cpu_idle(cpu)); -- -- return cpu; --} -- --static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) --{ -- s32 cpu; -- -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return prev_cpu; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local DSQ of the waker. -- */ -- if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 && -- scx_has_idle_cpus && !(current->flags & PF_EXITING)) { -- cpu = smp_processor_id(); -- if (cpumask_test_cpu(cpu, p->cpus_ptr)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (test_and_clear_cpu_idle(prev_cpu)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return prev_cpu; -- } -- -- if (p->nr_cpus_allowed == 1) -- return prev_cpu; -- -- cpu = scx_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- -- return prev_cpu; --} -- --static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) --{ -- if (SCX_HAS_OP(select_cpu)) { -- s32 cpu; -- -- cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu, -- wake_flags); -- if (ops_cpu_valid(cpu)) { -- return cpu; -- } else { -- scx_ops_error("select_cpu returned invalid cpu %d", cpu); -- return prev_cpu; -- } -- } else { -- return scx_select_cpu_dfl(p, prev_cpu, wake_flags); -- } --} -- --static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx) --{ -- set_cpus_allowed_common(p, ctx); -- -- /* -- * The effective cpumask is stored in @p->cpus_ptr which may temporarily -- * differ from the configured one in @p->cpus_mask. Always tell the bpf -- * scheduler the effective one. -- * -- * Fine-grained memory write control is enforced by BPF making the const -- * designation pointless. Cast it away when calling the operation. -- */ -- if (SCX_HAS_OP(set_cpumask)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, -- (struct cpumask *)p->cpus_ptr); --} -- --static void reset_idle_masks(void) --{ -- /* consider all cpus idle, should converge to the actual state quickly */ -- cpumask_setall(idle_masks.cpu); -- cpumask_setall(idle_masks.smt); -- scx_has_idle_cpus = true; --} -- --void __scx_update_idle(struct rq *rq, bool idle) --{ -- int cpu = cpu_of(rq); -- struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -- -- if (SCX_HAS_OP(update_idle)) { -- SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); -- if (!static_branch_unlikely(&scx_builtin_idle_enabled)) -- return; -- } -- -- if (idle) { -- cpumask_set_cpu(cpu, idle_masks.cpu); -- if (!scx_has_idle_cpus) -- scx_has_idle_cpus = true; -- -- /* -- * idle_masks.smt handling is racy but that's fine as it's only -- * for optimization and self-correcting. -- */ -- for_each_cpu(cpu, sib_mask) { -- if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -- return; -- } -- cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -- } else { -- cpumask_clear_cpu(cpu, idle_masks.cpu); -- if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -- } --} -- --#else /* !CONFIG_SMP */ -- --static bool test_and_clear_cpu_idle(int cpu) { return false; } --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } --static void reset_idle_masks(void) {} -- --#endif /* CONFIG_SMP */ -- --static bool check_rq_for_timeouts(struct rq *rq) --{ -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rq_flags rf; -- bool timed_out = false; -- -- rq_lock_irqsave(rq, &rf); -- list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) { -- unsigned long last_runnable; -- -- p = entity->task; -- last_runnable = p->scx->runnable_at; -- -- if (unlikely(time_after(jiffies, -- last_runnable + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "%s[%d] failed to run for %u.%03us", -- p->comm, p->pid, -- dur_ms / 1000, dur_ms % 1000); -- timed_out = true; -- break; -- } -- } -- rq_unlock_irqrestore(rq, &rf); -- -- return timed_out; --} -- --static void scx_watchdog_workfn(struct work_struct *work) --{ -- int cpu; -- -- scx_watchdog_timestamp = jiffies; -- -- for_each_online_cpu(cpu) { -- if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) -- break; -- -- cond_resched(); -- } -- queue_delayed_work(system_unbound_wq, to_delayed_work(work), -- scx_watchdog_timeout / 2); --} -- --static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) --{ -- update_curr_scx(rq); -- //scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock); -- /* -- * While disabling, always resched and refresh core-sched timestamp as -- * we can't trust the slice management or ops.core_sched_before(). -- */ -- if (scx_ops_disabling()) { -- curr->scx->slice = 0; -- touch_core_sched(rq, curr); -- } -- -- if (!curr->scx->slice) -- resched_curr(rq); --} -- --#define SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- --static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg) --{ -- int ret; -- -- WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED); -- -- p->scx->disallow = false; -- -- if (SCX_HAS_OP(prep_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- }; -- -- ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args); -- if (unlikely(ret)) { -- ret = ops_sanitize_err("prep_enable", ret); -- return ret; -- } -- } -- //scx_sched_init_task(p); -- -- if (p->scx->disallow) { -- struct rq *rq; -- struct rq_flags rf; -- -- rq = task_rq_lock(p, &rf); -- -- /* -- * We're either in fork or load path and @p->policy will be -- * applied right after. Reverting @p->policy here and rejecting -- * %SCHED_EXT transitions from scx_check_setscheduler() -- * guarantees that if ops.prep_enable() sets @p->disallow, @p -- * can never be in SCX. -- */ -- if (p->policy == SCHED_EXT) { -- p->policy = SCHED_NORMAL; -- atomic64_inc(&scx_nr_rejected); -- } -- -- task_rq_unlock(rq, p, &rf); -- } -- -- p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET); -- return 0; --} -- --static void scx_ops_enable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED)); -- -- if (SCX_HAS_OP(enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- p->scx->flags |= SCX_TASK_OPS_ENABLED; --} -- --static void scx_ops_disable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- if (p->scx->flags & SCX_TASK_OPS_PREPPED) { -- if (SCX_HAS_OP(cancel_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- } else if (p->scx->flags & SCX_TASK_OPS_ENABLED) { -- if (SCX_HAS_OP(disable)) -- SCX_CALL_OP(SCX_KF_REST, disable, p); -- p->scx->flags &= ~SCX_TASK_OPS_ENABLED; -- } --} -- --static void set_task_scx_weight(struct task_struct *p) --{ -- u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -- -- p->scx->weight = sched_weight_to_cgroup(weight); --} -- --/** -- * refresh_scx_weight - Refresh a task's ext weight -- * @p: task to refresh ext weight for -- * -- * @p->scx->weight carries the task's static priority in cgroup weight scale to -- * enable easy access from the BPF scheduler. To keep it synchronized with the -- * current task priority, this function should be called when a new task is -- * created, priority is changed for a task on sched_ext, and a task is switched -- * to sched_ext from other classes. -- */ --static void refresh_scx_weight(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- set_task_scx_weight(p); -- if (SCX_HAS_OP(set_weight)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight); --} -- --void scx_pre_fork(struct task_struct *p) --{ -- p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL); -- if (!p->scx) { -- goto lock; -- } -- -- p->scx->dsq = NULL; -- INIT_LIST_HEAD(&p->scx->dsq_node.fifo); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- INIT_LIST_HEAD(&p->scx->watchdog_node); -- p->scx->flags = 0; -- p->scx->weight = 0; -- p->scx->sticky_cpu = -1; -- p->scx->holding_cpu = -1; -- p->scx->kf_mask = 0; -- atomic64_set(&p->scx->ops_state, 0); -- p->scx->runnable_at = INITIAL_JIFFIES; -- p->scx->slice = SCX_SLICE_DFL; -- p->scx->task = p; -- p->sched_prop = 0; -- -- /* -- * BPF scheduler enable/disable paths want to be able to iterate and -- * update all tasks which can become complex when racing forks. As -- * enable/disable are very cold paths, let's use a percpu_rwsem to -- * exclude forks. -- */ --lock: -- percpu_down_read(&scx_fork_rwsem); --} -- --int scx_fork(struct task_struct *p) --{ -- percpu_rwsem_assert_held(&scx_fork_rwsem); -- -- if (scx_enabled()) -- return scx_ops_prepare_task(p, task_group(p)); -- else -- return 0; --} -- --void scx_post_fork(struct task_struct *p) --{ -- if (scx_enabled()) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- /* -- * Set the weight manually before calling ops.enable() so that -- * the scheduler doesn't see a stale value if they inspect the -- * task struct. We'll invoke ops.set_weight() afterwards, as it -- * would be odd to receive a callback on the task before we -- * tell the scheduler that it's been fully enabled. -- */ -- set_task_scx_weight(p); -- scx_ops_enable_task(p); -- refresh_scx_weight(p); -- task_rq_unlock(rq, p, &rf); -- } -- -- spin_lock_irq(&scx_tasks_lock); -- list_add_tail(&p->scx->tasks_node, &scx_tasks); -- spin_unlock_irq(&scx_tasks_lock); -- -- percpu_up_read(&scx_fork_rwsem); --} -- --void scx_cancel_fork(struct task_struct *p) --{ -- if (scx_enabled()) -- scx_ops_disable_task(p); -- percpu_up_read(&scx_fork_rwsem); --} -- --void sched_ext_free(struct task_struct *p) --{ -- unsigned long flags; -- -- spin_lock_irqsave(&scx_tasks_lock, flags); -- list_del_init(&p->scx->tasks_node); -- spin_unlock_irqrestore(&scx_tasks_lock, flags); -- -- /* -- * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED -> -- * ENABLED transitions can't race us. Disable ops for @p. -- */ -- if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- scx_ops_disable_task(p); -- task_rq_unlock(rq, p, &rf); -- } --} -- --static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) --{ --} -- --static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} --static void switched_to_scx(struct rq *rq, struct task_struct *p) {} -- --int scx_check_setscheduler(struct task_struct *p, int policy) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- /* if disallow, reject transitioning into SCX */ -- if (scx_enabled() && READ_ONCE(p->scx->disallow) && -- p->policy != policy && policy == SCHED_EXT) -- return -EACCES; -- -- return 0; --} -- --#ifdef CONFIG_NO_HZ_FULL --bool scx_can_stop_tick(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (scx_ops_disabling()) -- return false; -- -- if (p->sched_class != &ext_sched_class) -- return true; -- -- /* -- * @rq can dispatch from different DSQs, so we can't tell whether it -- * needs the tick or not by looking at nr_running. Allow stopping ticks -- * iff the BPF scheduler indicated so. See set_next_task_scx(). -- */ -- return rq->scx->flags & SCX_RQ_CAN_STOP_TICK; --} --#endif -- --static inline void scx_cgroup_lock(void) {} --static inline void scx_cgroup_unlock(void) {} -- --/* -- * Omitted operations: -- * -- * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -- * task isn't tied to the CPU at that point. Preemption is implemented by -- * resetting the victim task's slice to 0 and triggering reschedule on the -- * target CPU. -- * -- * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -- * -- * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -- * their current sched_class. Call them directly from sched core instead. -- * -- * - task_woken, switched_from: Unnecessary. -- */ --DEFINE_SCHED_CLASS(ext) = { -- .enqueue_task = enqueue_task_scx, -- .dequeue_task = dequeue_task_scx, -- .yield_task = yield_task_scx, -- .yield_to_task = yield_to_task_scx, -- -- .check_preempt_curr = check_preempt_curr_scx, -- -- .pick_next_task = pick_next_task_scx, -- -- .put_prev_task = put_prev_task_scx, -- .set_next_task = set_next_task_scx, -- --#ifdef CONFIG_SMP -- .balance = balance_scx, -- .select_task_rq = select_task_rq_scx, -- .set_cpus_allowed = set_cpus_allowed_scx, --#endif -- --#ifdef CONFIG_SCHED_CORE -- .pick_task = pick_task_scx, --#endif -- -- .task_tick = task_tick_scx, -- -- .switched_to = switched_to_scx, -- .prio_changed = prio_changed_scx, -- -- .update_curr = update_curr_scx, -- --#ifdef CONFIG_UCLAMP_TASK -- .uclamp_enabled = 0, --#endif --}; -- --static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) --{ -- memset(dsq, 0, sizeof(*dsq)); -- -- raw_spin_lock_init(&dsq->lock); -- INIT_LIST_HEAD(&dsq->fifo); -- dsq->id = dsq_id; --} -- --static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id & SCX_DSQ_FLAG_BUILTIN) -- return ERR_PTR(-EINVAL); -- -- dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node); -- if (!dsq) -- return ERR_PTR(-ENOMEM); -- -- init_dsq(dsq, dsq_id); -- -- return dsq; --} -- --static void free_dsq_irq_workfn(struct irq_work *irq_work) --{ -- struct llist_node *to_free = llist_del_all(&dsqs_to_free); -- struct scx_dispatch_q *dsq, *tmp_dsq; -- -- llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) -- kfree_rcu(dsq, rcu); --} -- --static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); -- --static void destroy_dsq(u64 dsq_id) --{ --} -- --static void scx_cgroup_exit(void) {} --static int scx_cgroup_init(void) { return 0; } --static void scx_cgroup_config_knobs(void) {} -- --/* -- * Used by sched_fork() and __setscheduler_prio() to pick the matching -- * sched_class. dl/rt are already handled. -- */ --bool task_on_scx(struct task_struct *p) --{ -- if (!scx_enabled() || scx_ops_disabling()) -- return false; -- if (READ_ONCE(scx_switching_all)) -- return true; -- return p->policy == SCHED_EXT; --} -- --static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags) --{ -- if (enq_flags & SCX_ENQ_LAST) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); --} -- --static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {} -- --static void scx_ops_disable_workfn(struct kthread_work *work) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- struct scx_task_iter sti; -- struct task_struct *p; -- const char *reason; -- int i, cpu, type; -- -- type = atomic_read(&scx_exit_type); -- while (true) { -- /* -- * NONE indicates that a new scx_ops has been registered since -- * disable was scheduled - don't kill the new ops. DONE -- * indicates that the ops has already been disabled. -- */ -- if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE) -- return; -- if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE)) -- break; -- } -- -- cancel_delayed_work_sync(&scx_watchdog_work); -- -- switch (type) { -- case SCX_EXIT_UNREG: -- reason = "BPF scheduler unregistered"; -- break; -- case SCX_EXIT_SYSRQ: -- reason = "disabled by sysrq-S"; -- break; -- case SCX_EXIT_ERROR: -- reason = "runtime error"; -- break; -- case SCX_EXIT_ERROR_BPF: -- reason = "scx_bpf_error"; -- break; -- case SCX_EXIT_ERROR_STALL: -- reason = "runnable task stall"; -- break; -- default: -- reason = ""; -- } -- -- ei->type = type; -- strlcpy(ei->reason, reason, sizeof(ei->reason)); -- -- switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { -- case SCX_OPS_DISABLED: -- pr_warn("sched_ext: ops error detected without ops (%s)\n", -- scx_exit_info.msg); -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- return; -- case SCX_OPS_PREPPING: -- goto forward_progress_guaranteed; -- case SCX_OPS_DISABLING: -- /* shouldn't happen but handle it like ENABLING if it does */ -- WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); -- fallthrough; -- case SCX_OPS_ENABLING: -- case SCX_OPS_ENABLED: -- break; -- } -- -- /* -- * DISABLING is set and ops was either ENABLING or ENABLED indicating -- * that the ops and static branches are set. -- * -- * We must guarantee that all runnable tasks make forward progress -- * without trusting the BPF scheduler. We can't grab any mutexes or -- * rwsems as they might be held by tasks that the BPF scheduler is -- * forgetting to run, which unfortunately also excludes toggling the -- * static branches. -- * -- * Let's work around by overriding a couple ops and modifying behaviors -- * based on the DISABLING state and then cycling the tasks through -- * dequeue/enqueue to force global FIFO scheduling. -- * -- * a. ops.enqueue() and .dispatch() are overridden for simple global -- * FIFO scheduling. -- * -- * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value -- * can't be trusted. Whenever a tick triggers, the running task is -- * rotated to the tail of the queue with core_sched_at touched. -- * -- * c. pick_next_task() suppresses zero slice warning. -- * -- * d. scx_prio_less() reverts to the default core_sched_at order. -- */ -- scx_ops.enqueue = scx_ops_fallback_enqueue; -- scx_ops.dispatch = scx_ops_fallback_dispatch; -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- SCHED_CHANGE_BLOCK(task_rq(p), p, -- DEQUEUE_SAVE | DEQUEUE_MOVE) { -- /* cycling deq/enq is enough, see above */ -- } -- } -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* kick all CPUs to restore ticks */ -- for_each_possible_cpu(cpu) -- resched_cpu(cpu); -- --forward_progress_guaranteed: -- /* -- * Here, every runnable task is guaranteed to make forward progress and -- * we can safely use blocking synchronization constructs. Actually -- * disable ops. -- */ -- mutex_lock(&scx_ops_enable_mutex); -- -- static_branch_disable(&__scx_switched_all); -- WRITE_ONCE(scx_switching_all, false); -- -- /* avoid racing against fork and cgroup changes */ -- cpus_read_lock(); -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- bool alive = READ_ONCE(p->__state) != TASK_DEAD; -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL); -- -- __setscheduler_prio(p, p->prio); -- } -- -- if (alive) -- check_class_changed(task_rq(p), p, old_class, p->prio); -- -- scx_ops_disable_task(p); -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* no task is on scx, turn off all the switches and flush in-progress calls */ -- static_branch_disable_cpuslocked(&__scx_ops_enabled); -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- static_branch_disable_cpuslocked(&scx_has_op[i]); -- static_branch_disable_cpuslocked(&scx_ops_enq_last); -- static_branch_disable_cpuslocked(&scx_ops_enq_exiting); -- static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- synchronize_rcu(); -- -- scx_cgroup_exit(); -- -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- cpus_read_unlock(); -- -- if (ei->type >= SCX_EXIT_ERROR) { -- printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name); -- -- if (ei->msg[0] == '\0') -- printk(KERN_ERR "sched_ext: %s\n", ei->reason); -- else -- printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg); -- -- stack_trace_print(ei->bt, ei->bt_len, 2); -- } -- -- if (scx_ops.exit) -- SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); -- -- memset(&scx_ops, 0, sizeof(scx_ops)); -- -- free_percpu(scx_dsp_buf); -- scx_dsp_buf = NULL; -- scx_dsp_max_batch = 0; -- -- mutex_unlock(&scx_ops_enable_mutex); -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- -- scx_cgroup_config_knobs(); --} -- --static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); -- --static void schedule_scx_ops_disable_work(void) --{ -- struct kthread_worker *helper = READ_ONCE(scx_ops_helper); -- -- /* -- * We may be called spuriously before the first bpf_sched_ext_reg(). If -- * scx_ops_helper isn't set up yet, there's nothing to do. -- */ -- if (helper) -- kthread_queue_work(helper, &scx_ops_disable_work); --} -- --static void scx_ops_disable(enum scx_exit_type type) --{ -- int none = SCX_EXIT_NONE; -- -- if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)) -- type = SCX_EXIT_ERROR; -- -- atomic_try_cmpxchg(&scx_exit_type, &none, type); -- -- schedule_scx_ops_disable_work(); --} -- --static void scx_ops_error_irq_workfn(struct irq_work *irq_work) --{ -- schedule_scx_ops_disable_work(); --} -- --static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- int none = SCX_EXIT_NONE; -- va_list args; -- -- if (!atomic_try_cmpxchg(&scx_exit_type, &none, type)) -- return; -- -- ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1); -- -- va_start(args, fmt); -- vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args); -- va_end(args); -- -- irq_work_queue(&scx_ops_error_irq_work); --} -- --static struct kthread_worker *scx_create_rt_helper(const char *name) --{ -- struct kthread_worker *helper; -- -- helper = kthread_create_worker(0, name); -- if (helper) -- sched_set_fifo(helper->task); -- return helper; --} -- --static int scx_ops_enable(struct sched_ext_ops *ops) --{ -- struct scx_task_iter sti; -- struct task_struct *p; -- int i, ret; -- int tcnt = 0; -- unsigned long long start = 0; -- unsigned long long total_start = sched_clock(); -- -- mutex_lock(&scx_ops_enable_mutex); -- -- if (!scx_ops_helper) { -- WRITE_ONCE(scx_ops_helper, -- scx_create_rt_helper("sched_ext_ops_helper")); -- if (!scx_ops_helper) { -- ret = -ENOMEM; -- goto err_unlock; -- } -- } -- -- if (scx_ops_enable_state() != SCX_OPS_DISABLED) { -- ret = -EBUSY; -- goto err_unlock; -- } -- -- //slim_walt_enable(true); -- -- /* -- * Set scx_ops, transition to PREPPING and clear exit info to arm the -- * disable path. Failure triggers full disabling from here on. -- */ -- scx_ops = *ops; -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != -- SCX_OPS_DISABLED); -- -- memset(&scx_exit_info, 0, sizeof(scx_exit_info)); -- atomic_set(&scx_exit_type, SCX_EXIT_NONE); -- scx_warned_zero_slice = false; -- -- atomic64_set(&scx_nr_rejected, 0); -- -- /* -- * Keep CPUs stable during enable so that the BPF scheduler can track -- * online CPUs by watching ->on/offline_cpu() after ->init(). -- */ -- cpus_read_lock(); -- -- scx_switch_all_req = true; -- if (scx_ops.init) { -- ret = SCX_CALL_OP_RET(SCX_KF_INIT, init); -- if (ret) { -- ret = ops_sanitize_err("init", ret); -- goto err_disable; -- } -- -- /* -- * Exit early if ops.init() triggered scx_bpf_error(). Not -- * strictly necessary as we'll fail transitioning into ENABLING -- * later but that'd be after calling ops.prep_enable() on all -- * tasks and with -EBUSY which isn't very intuitive. Let's exit -- * early with success so that the condition is notified through -- * ops.exit() like other scx_bpf_error() invocations. -- */ -- if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE) -- goto err_disable; -- } -- -- WARN_ON_ONCE(scx_dsp_buf); -- scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; -- scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch, -- __alignof__(scx_dsp_buf[0])); -- if (!scx_dsp_buf) { -- ret = -ENOMEM; -- goto err_disable; -- } -- -- scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; -- if (ops->timeout_ms) -- scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); -- -- scx_watchdog_timestamp = jiffies; -- queue_delayed_work(system_unbound_wq, &scx_watchdog_work, -- scx_watchdog_timeout / 2); -- -- /* -- * Lock out forks, cgroup on/offlining and moves before opening the -- * floodgate so that they don't wander into the operations prematurely. -- */ -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- if (((void (**)(void))ops)[i]) -- static_branch_enable_cpuslocked(&scx_has_op[i]); -- -- if (ops->flags & SCX_OPS_ENQ_LAST) -- static_branch_enable_cpuslocked(&scx_ops_enq_last); -- -- if (ops->flags & SCX_OPS_ENQ_EXITING) -- static_branch_enable_cpuslocked(&scx_ops_enq_exiting); -- if (scx_ops.cpu_acquire || scx_ops.cpu_release) -- static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); -- -- if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { -- reset_idle_masks(); -- static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); -- } else { -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- } -- -- /* -- * All cgroups should be initialized before letting in tasks. cgroup -- * on/offlining and task migrations are already locked out. -- */ -- ret = scx_cgroup_init(); -- if (ret) -- goto err_disable_unlock; -- -- static_branch_enable_cpuslocked(&__scx_ops_enabled); -- -- /* -- * Enable ops for every task. Fork is excluded by scx_fork_rwsem -- * preventing new tasks from being added. No need to exclude tasks -- * leaving as sched_ext_free() can handle both prepped and enabled -- * tasks. Prep all tasks first and then enable them with preemption -- * disabled. -- */ -- spin_lock_irq(&scx_tasks_lock); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered(&sti))) { -- get_task_struct(p); -- spin_unlock_irq(&scx_tasks_lock); -- -- ret = scx_ops_prepare_task(p, task_group(p)); -- if (ret) { -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n", -- ret, p->comm, p->pid); -- goto err_disable_unlock; -- } -- -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- } -- scx_task_iter_exit(&sti); -- -- /* -- * All tasks are prepped but are still ops-disabled. Ensure that -- * %current can't be scheduled out and switch everyone. -- * preempt_disable() is necessary because we can't guarantee that -- * %current won't be starved if scheduled out while switching. -- */ -- preempt_disable(); -- -- /* -- * From here on, the disable path must assume that tasks have ops -- * enabled and need to be recovered. -- */ -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { -- preempt_enable(); -- spin_unlock_irq(&scx_tasks_lock); -- ret = -EBUSY; -- goto err_disable_unlock; -- } -- -- /* -- * We're fully committed and can't fail. The PREPPED -> ENABLED -- * transitions here are synchronized against sched_ext_free() through -- * scx_tasks_lock. -- */ -- WRITE_ONCE(scx_switching_all, scx_switch_all_req); -- start = sched_clock(); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- tcnt++; -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- scx_ops_enable_task(p); -- __setscheduler_prio(p, p->prio); -- } -- -- check_class_changed(task_rq(p), p, old_class, p->prio); -- } else { -- scx_ops_disable_task(p); -- } -- } -- printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id()); -- scx_task_iter_exit(&sti); -- -- spin_unlock_irq(&scx_tasks_lock); -- preempt_enable(); -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { -- ret = -EBUSY; -- goto err_disable; -- } -- -- if (scx_switch_all_req) -- static_branch_enable_cpuslocked(&__scx_switched_all); -- -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- -- scx_cgroup_config_knobs(); -- printk("\n total duration = %llu , current cpu = %d\n", sched_clock() - total_start, smp_processor_id()); -- -- return 0; -- --err_unlock: -- mutex_unlock(&scx_ops_enable_mutex); -- return ret; -- --err_disable_unlock: -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); --err_disable: -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- /* must be fully disabled before returning */ -- scx_ops_disable(SCX_EXIT_ERROR); -- kthread_flush_work(&scx_ops_disable_work); -- return ret; --} -- --#ifdef CONFIG_SCHED_DEBUG --static const char *scx_ops_enable_state_str[] = { -- [SCX_OPS_PREPPING] = "prepping", -- [SCX_OPS_ENABLING] = "enabling", -- [SCX_OPS_ENABLED] = "enabled", -- [SCX_OPS_DISABLING] = "disabling", -- [SCX_OPS_DISABLED] = "disabled", --}; -- --static int scx_debug_show(struct seq_file *m, void *v) --{ -- mutex_lock(&scx_ops_enable_mutex); -- seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name); -- seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled()); -- seq_printf(m, "%-30s: %d\n", "switching_all", -- READ_ONCE(scx_switching_all)); -- seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all()); -- seq_printf(m, "%-30s: %s\n", "enable_state", -- scx_ops_enable_state_str[scx_ops_enable_state()]); -- seq_printf(m, "%-30s: %llu\n", "nr_rejected", -- atomic64_read(&scx_nr_rejected)); -- mutex_unlock(&scx_ops_enable_mutex); -- return 0; --} -- --static int scx_debug_open(struct inode *inode, struct file *file) --{ -- return single_open(file, scx_debug_show, NULL); --} -- --const struct file_operations sched_ext_fops = { -- .open = scx_debug_open, -- .read = seq_read, -- .llseek = seq_lseek, -- .release = single_release, --}; --#endif -- --/******************************************************************************** -- * bpf_struct_ops plumbing. -- */ --#include --#include --#include -- --extern struct btf *btf_vmlinux; --static const struct btf_type *task_struct_type; -- --static bool bpf_scx_is_valid_access(int off, int size, -- enum bpf_access_type type, -- const struct bpf_prog *prog, -- struct bpf_insn_access_aux *info) --{ -- if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) -- return false; -- if (type != BPF_READ) -- return false; -- if (off % size != 0) -- return false; -- -- return btf_ctx_access(off, size, type, prog, info); --} -- --static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, -- const struct bpf_reg_state *reg, -- int off, int size) --{ -- return 0; --} -- --static const struct bpf_func_proto * --bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) --{ -- switch (func_id) { -- case BPF_FUNC_task_storage_get: -- return &bpf_task_storage_get_proto; -- case BPF_FUNC_task_storage_delete: -- return &bpf_task_storage_delete_proto; -- default: -- return bpf_base_func_proto(func_id); -- } --} -- --const struct bpf_verifier_ops bpf_scx_verifier_ops = { -- .get_func_proto = bpf_scx_get_func_proto, -- .is_valid_access = bpf_scx_is_valid_access, -- .btf_struct_access = bpf_scx_btf_struct_access, --}; -- --static int bpf_scx_init_member(const struct btf_type *t, -- const struct btf_member *member, -- void *kdata, const void *udata) --{ -- const struct sched_ext_ops *uops = udata; -- struct sched_ext_ops *ops = kdata; -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- int ret; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, dispatch_max_batch): -- if (*(u32 *)(udata + moff) > INT_MAX) -- return -E2BIG; -- ops->dispatch_max_batch = *(u32 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, flags): -- if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) -- return -EINVAL; -- ops->flags = *(u64 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, name): -- ret = bpf_obj_name_cpy(ops->name, uops->name, -- sizeof(ops->name)); -- if (ret < 0) -- return ret; -- if (ret == 0) -- return -EINVAL; -- return 1; -- case offsetof(struct sched_ext_ops, timeout_ms): -- if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT) -- return -E2BIG; -- ops->timeout_ms = *(u32 *)(udata + moff); -- return 1; -- } -- -- return 0; --} -- --static int bpf_scx_check_member(const struct btf_type *t, -- const struct btf_member *member, -- const struct bpf_prog *prog) --{ -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, prep_enable): -- case offsetof(struct sched_ext_ops, init): -- case offsetof(struct sched_ext_ops, exit): -- break; -- default: -- /* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */ -- /* if (prog->aux->sleepable) -- return -EINVAL; */ -- break; -- } -- -- return 0; --} -- --static int bpf_scx_reg(void *kdata) --{ -- return scx_ops_enable(kdata); --} -- --static void bpf_scx_unreg(void *kdata) --{ -- scx_ops_disable(SCX_EXIT_UNREG); -- kthread_flush_work(&scx_ops_disable_work); --} -- --static int bpf_scx_init(struct btf *btf) --{ -- u32 type_id; -- -- type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); -- if (type_id < 0) -- return -EINVAL; -- task_struct_type = btf_type_by_id(btf, type_id); -- -- return 0; --} -- --/* "extern" to avoid sparse warning, only used in this file */ --extern struct bpf_struct_ops bpf_sched_ext_ops; -- --struct bpf_struct_ops bpf_sched_ext_ops = { -- .verifier_ops = &bpf_scx_verifier_ops, -- .reg = bpf_scx_reg, -- .unreg = bpf_scx_unreg, -- .check_member = bpf_scx_check_member, -- .init_member = bpf_scx_init_member, -- .init = bpf_scx_init, -- .name = "sched_ext_ops", --}; -- --static void sysrq_handle_sched_ext_reset(u8 key) --{ -- if (scx_ops_helper) -- scx_ops_disable(SCX_EXIT_SYSRQ); -- else -- pr_info("sched_ext: BPF scheduler not yet used\n"); --} -- --static const struct sysrq_key_op sysrq_sched_ext_reset_op = { -- .handler = sysrq_handle_sched_ext_reset, -- .help_msg = "reset-sched-ext(S)", -- .action_msg = "Disable sched_ext and revert all tasks to CFS", -- .enable_mask = SYSRQ_ENABLE_RTNICE, --}; -- --static void kick_cpus_irq_workfn(struct irq_work *irq_work) --{ -- struct rq *this_rq = this_rq(); -- u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); -- int this_cpu = cpu_of(this_rq); -- int cpu; -- -- for_each_cpu(cpu, this_rq->scx->cpus_to_kick) { -- struct rq *rq = cpu_rq(cpu); -- unsigned long flags; -- -- raw_spin_rq_lock_irqsave(rq, flags); -- -- if (cpu_online(cpu) || cpu == this_cpu) { -- if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) && -- rq->curr->sched_class == &ext_sched_class) -- rq->curr->scx->slice = 0; -- pseqs[cpu] = rq->scx->pnt_seq; -- resched_curr(rq); -- } else { -- cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait); -- } -- -- raw_spin_rq_unlock_irqrestore(rq, flags); -- } -- -- for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait, -- cpumask_of(this_cpu)) { -- /* -- * Pairs with smp_store_release() issued by this CPU in -- * scx_notify_pick_next_task() on the resched path. -- * -- * We busy-wait here to guarantee that no other task can be -- * scheduled on our core before the target CPU has entered the -- * resched path. -- */ -- while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu]) -- cpu_relax(); -- } -- -- cpumask_clear(this_rq->scx->cpus_to_kick); -- cpumask_clear(this_rq->scx->cpus_to_preempt); -- cpumask_clear(this_rq->scx->cpus_to_wait); --} -- --void __init init_sched_ext_class(void) --{ -- int cpu; -- u32 v; -- -- /* -- * The following is to prevent the compiler from optimizing out the enum -- * definitions so that BPF scheduler implementations can use them -- * through the generated vmlinux.h. -- */ -- WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | -- SCX_TG_ONLINE | SCX_KICK_PREEMPT); -- -- init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); --#ifdef CONFIG_SMP -- BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -- BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); --#endif -- scx_kick_cpus_pnt_seqs = -- __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * -- num_possible_cpus(), -- __alignof__(scx_kick_cpus_pnt_seqs[0])); -- BUG_ON(!scx_kick_cpus_pnt_seqs); -- -- for_each_possible_cpu(cpu) { -- struct rq *rq = cpu_rq(cpu); -- -- rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL); -- if (rq->scx) -- rq->scx->rq = rq; -- else -- pr_err("fatal error : alloc rq->scx failed!!!\n"); -- -- init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL); -- INIT_LIST_HEAD(&rq->scx->watchdog_list); -- -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL)); -- init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn); -- } -- -- register_sysrq_key('S', &sysrq_sched_ext_reset_op); -- INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); -- scx_cgroup_config_knobs(); --} -- -- --/******************************************************************************** -- * Helpers that can be called from the BPF scheduler. -- */ --#include -- --/* Disables missing prototype warnings for kfuncs */ --__diag_push(); --__diag_ignore_all("-Wmissing-prototypes", -- "Global functions as their definitions will be in vmlinux BTF"); -- --/** -- * scx_bpf_switch_all - Switch all tasks into SCX -- * @into_scx: switch direction -- * -- * If @into_scx is %true, all existing and future non-dl/rt tasks are switched -- * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on -- * SCX. The actual switching is asynchronous. Can be called from ops.init(). -- */ --void scx_bpf_switch_all(void) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return; -- -- scx_switch_all_req = true; --} -- --BTF_SET8_START(scx_kfunc_ids_init) --BTF_ID_FLAGS(func, scx_bpf_switch_all) --BTF_SET8_END(scx_kfunc_ids_init) -- --static const struct btf_kfunc_id_set scx_kfunc_set_init = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_init, --}; -- --/** -- * scx_bpf_create_dsq - Create a custom DSQ -- * @dsq_id: DSQ to create -- * @node: NUMA node to allocate from -- * -- * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(), -- * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move(). -- */ --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return -EINVAL; -- -- if (unlikely(node >= (int)nr_node_ids || -- (node < 0 && node != NUMA_NO_NODE))) -- return -EINVAL; -- return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); --} -- --BTF_SET8_START(scx_kfunc_ids_sleepable) --BTF_ID_FLAGS(func, scx_bpf_create_dsq) --BTF_SET8_END(scx_kfunc_ids_sleepable) -- --static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_sleepable, --}; -- --static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) --{ -- if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) -- return false; -- -- lockdep_assert_irqs_disabled(); -- -- if (unlikely(!p)) { -- scx_ops_error("called with NULL task"); -- return false; -- } -- -- if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { -- scx_ops_error("invalid enq_flags 0x%llx", enq_flags); -- return false; -- } -- -- return true; --} -- --static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) --{ -- struct task_struct *ddsp_task; -- int idx; -- -- ddsp_task = __this_cpu_read(direct_dispatch_task); -- if (ddsp_task) { -- direct_dispatch(ddsp_task, p, dsq_id, enq_flags); -- return; -- } -- -- idx = __this_cpu_read(scx_dsp_ctx.buf_cursor); -- if (unlikely(idx >= scx_dsp_max_batch)) { -- scx_ops_error("dispatch buffer overflow"); -- return; -- } -- -- this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){ -- .task = p, -- .qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK, -- .dsq_id = dsq_id, -- .enq_flags = enq_flags, -- }; -- __this_cpu_inc(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe -- * to call this function spuriously. Can be called from ops.enqueue() and -- * ops.dispatch(). -- * -- * When called from ops.enqueue(), it's for direct dispatch and @p must match -- * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the -- * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be -- * on the target CPU in the first place. -- * -- * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id -- * and this function can be called upto ops.dispatch_max_batch times to dispatch -- * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the -- * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. -- * -- * This function doesn't have any locking restrictions and may be called under -- * BPF locks (in the future when BPF introduces more flexible locking). -- * -- * @p is allowed to run for @slice. The scheduling path is triggered on slice -- * exhaustion. If zero, the current residual slice is maintained. If -- * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with -- * scx_bpf_kick_cpu() to trigger scheduling. -- */ --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- scx_dispatch_commit(p, dsq_id, enq_flags); --} -- --/** -- * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. -- * Tasks queued into the priority queue are ordered by @vtime and always -- * consumed after the tasks in the FIFO queue. All other aspects are identical -- * to scx_bpf_dispatch(). -- * -- * @vtime ordering is according to time_before64() which considers wrapping. A -- * numerically larger vtime may indicate an earlier position in the ordering and -- * vice-versa. -- */ --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 vtime, u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- p->scx->dsq_vtime = vtime; -- -- scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); --} -- --BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime) --BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_enqueue_dispatch, --}; -- --/** -- * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots -- * -- * Can only be called from ops.dispatch(). -- */ --u32 scx_bpf_dispatch_nr_slots(void) --{ -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return 0; -- -- return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ -- * @dsq_id: DSQ to consume -- * -- * Consume a task from the non-local DSQ identified by @dsq_id and transfer it -- * to the current CPU's local DSQ for execution. Can only be called from -- * ops.dispatch(). -- * -- * This function flushes the in-flight dispatches from scx_bpf_dispatch() before -- * trying to consume the specified DSQ. It may also grab rq locks and thus can't -- * be called under any BPF locks. -- * -- * Returns %true if a task has been consumed, %false if there isn't any task to -- * consume. -- */ --bool scx_bpf_consume(u64 dsq_id) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- struct scx_dispatch_q *dsq; -- -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return false; -- -- flush_dispatch_buf(dspc->rq, dspc->rf); -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); -- return false; -- } -- -- if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) { -- /* -- * A successfully consumed task can be dequeued before it starts -- * running while the CPU is trying to migrate other dispatched -- * tasks. Bump nr_tasks to tell balance_scx() to retry on empty -- * local DSQ. -- */ -- dspc->nr_tasks++; -- return true; -- } else { -- return false; -- } --} -- --BTF_SET8_START(scx_kfunc_ids_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) --BTF_ID_FLAGS(func, scx_bpf_consume) --BTF_SET8_END(scx_kfunc_ids_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_dispatch, --}; -- --/** -- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ -- * -- * Iterate over all of the tasks currently enqueued on the local DSQ of the -- * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of -- * processed tasks. Can only be called from ops.cpu_release(). -- */ --u32 scx_bpf_reenqueue_local(void) --{ -- u32 nr_enqueued, i; -- struct rq *rq; -- struct scx_rq *scx_rq; -- -- if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) -- return 0; -- -- rq = cpu_rq(smp_processor_id()); -- lockdep_assert_rq_held(rq); -- scx_rq = rq->scx; -- -- /* -- * Get the number of tasks on the local DSQ before iterating over it to -- * pull off tasks. The enqueue callback below can signal that it wants -- * the task to stay on the local DSQ, and we want to prevent the BPF -- * scheduler from causing us to loop indefinitely. -- */ -- nr_enqueued = scx_rq->local_dsq.nr; -- for (i = 0; i < nr_enqueued; i++) { -- struct task_struct *p; -- -- p = first_local_task(rq); -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- WARN_ON_ONCE(p->scx->holding_cpu != -1); -- dispatch_dequeue(scx_rq, p); -- do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); -- } -- -- return nr_enqueued; --} -- --BTF_SET8_START(scx_kfunc_ids_cpu_release) --BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) --BTF_SET8_END(scx_kfunc_ids_cpu_release) -- --static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_cpu_release, --}; -- --/** -- * scx_bpf_kick_cpu - Trigger reschedule on a CPU -- * @cpu: cpu to kick -- * @flags: SCX_KICK_* flags -- * -- * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or -- * trigger rescheduling on a busy CPU. This can be called from any online -- * scx_ops operation and the actual kicking is performed asynchronously through -- * an irq work. -- */ --void scx_bpf_kick_cpu(s32 cpu, u64 flags) --{ -- struct rq *rq; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d", cpu); -- return; -- } -- -- preempt_disable(); -- rq = this_rq(); -- -- /* -- * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting -- * rq locks. We can probably be smarter and avoid bouncing if called -- * from ops which don't hold a rq lock. -- */ -- cpumask_set_cpu(cpu, rq->scx->cpus_to_kick); -- if (flags & SCX_KICK_PREEMPT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt); -- if (flags & SCX_KICK_WAIT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_wait); -- -- irq_work_queue(&rq->scx->kick_cpus_irq_work); -- preempt_enable(); --} -- --/** -- * scx_bpf_dsq_nr_queued - Return the number of queued tasks -- * @dsq_id: id of the DSQ -- * -- * Return the number of tasks in the DSQ matching @dsq_id. If not found, -- * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops -- * operations. -- */ --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) --{ -- struct scx_dispatch_q *dsq; -- -- lockdep_assert(rcu_read_lock_any_held()); -- -- if (dsq_id == SCX_DSQ_LOCAL) { -- return this_rq()->scx->local_dsq.nr; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (ops_cpu_valid(cpu)) -- return cpu_rq(cpu)->scx->local_dsq.nr; -- } else { -- dsq = find_non_local_dsq(dsq_id); -- if (dsq) -- return dsq->nr; -- } -- return -ENOENT; --} -- --/** -- * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state -- * @cpu: cpu to test and clear idle for -- * -- * Returns %true if @cpu was idle and its idle state was successfully cleared. -- * %false otherwise. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return false; -- } -- -- if (ops_cpu_valid(cpu)) -- return test_and_clear_cpu_idle(cpu); -- else -- return false; --} -- --/** -- * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu -- * @cpus_allowed: Allowed cpumask -- * -- * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked -- * idle cpu number on success. -%EBUSY if no matching cpu was found. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return -EBUSY; -- } -- -- return scx_pick_idle_cpu(cpus_allowed); --} -- --/** -- * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking -- * per-CPU cpumask. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_cpumask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.cpu; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, -- * per-physical-core cpumask. Can be used to determine if an entire physical -- * core is free. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_smtmask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.smt; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to -- * either the percpu, or SMT idle-tracking cpumask. -- */ --void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) --{ -- /* -- * Empty function body because we aren't actually acquiring or -- * releasing a reference to a global idle cpumask, which is read-only -- * in the caller and is never released. The acquire / release semantics -- * here are just used to make the cpumask is a trusted pointer in the -- * caller. -- */ --} -- --struct scx_bpf_error_bstr_bufs { -- u64 data[MAX_BPRINTF_VARARGS]; -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs); -- --/** -- * scx_bpf_error_bstr - Indicate fatal error -- * @fmt: error message format string -- * @data: format string parameters packaged using ___bpf_fill() macro -- * @data__sz: @data len, must end in '__sz' for the verifier -- * -- * Indicate that the BPF scheduler encountered a fatal error and initiate ops -- * disabling. -- */ --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) --{ -- struct scx_bpf_error_bstr_bufs *bufs; -- unsigned long flags; -- struct bpf_bprintf_data args = {}; -- int ret; -- -- local_irq_save(flags); -- bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs); -- -- if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || -- (data__sz && !data)) { -- scx_ops_error("invalid data=%p and data__sz=%u", -- (void *)data, data__sz); -- goto out_restore; -- } -- -- ret = copy_from_kernel_nofault(bufs->data, data, data__sz); -- if (ret) { -- scx_ops_error("failed to read data fields (%d)", ret); -- goto out_restore; -- } -- -- ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args); -- if (ret < 0) { -- scx_ops_error("failed to format prepration (%d)", ret); -- goto out_restore; -- } -- -- ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt, -- args.bin_args); -- bpf_bprintf_cleanup(&args); -- if (ret < 0) { -- scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format", -- fmt, data, data__sz); -- goto out_restore; -- } -- -- scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg); --out_restore: -- local_irq_restore(flags); --} -- --/** -- * scx_bpf_destroy_dsq - Destroy a custom DSQ -- * @dsq_id: DSQ to destroy -- * -- * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with -- * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is -- * empty and no further tasks are dispatched to it. Ignored if called on a DSQ -- * which doesn't exist. Can be called from any online scx_ops operations. -- */ --void scx_bpf_destroy_dsq(u64 dsq_id) --{ -- destroy_dsq(dsq_id); --} -- --/** -- * scx_bpf_task_running - Is task currently running? -- * @p: task of interest -- */ --bool scx_bpf_task_running(const struct task_struct *p) --{ -- return task_rq(p)->curr == p; --} -- --/** -- * scx_bpf_task_cpu - CPU a task is currently associated with -- * @p: task of interest -- */ --s32 scx_bpf_task_cpu(const struct task_struct *p) --{ -- return task_cpu(p); --} -- --/** -- * scx_bpf_task_cgroup - Return the sched cgroup of a task -- * @p: task of interest -- * -- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with -- * from the scheduler's POV. SCX operations should use this function to -- * determine @p's current cgroup as, unlike following @p->cgroups, -- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all -- * rq-locked operations. Can be called on the parameter tasks of rq-locked -- * operations. The restriction guarantees that @p's rq is locked by the caller. -- */ --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) --{ -- struct task_group *tg = p->sched_task_group; -- struct cgroup *cgrp = &cgrp_dfl_root.cgrp; -- -- if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) -- goto out; -- -- /* -- * A task_group may either be a cgroup or an autogroup. In the latter -- * case, @tg->css.cgroup is %NULL. A task_group can't become the other -- * kind once created. -- */ -- if (tg && tg->css.cgroup) -- cgrp = tg->css.cgroup; -- else -- cgrp = &cgrp_dfl_root.cgrp; --out: -- cgroup_get(cgrp); -- return cgrp; --} -- --BTF_SET8_START(scx_kfunc_ids_any) --BTF_ID_FLAGS(func, scx_bpf_kick_cpu) --BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) --BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) --BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu) --BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) --BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) --BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) --BTF_ID_FLAGS(func, scx_bpf_task_running) --BTF_ID_FLAGS(func, scx_bpf_task_cpu) --BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE) --BTF_SET8_END(scx_kfunc_ids_any) -- --static const struct btf_kfunc_id_set scx_kfunc_set_any = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_any, --}; -- --__diag_pop(); -- --/* -- * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set() -- * needs most of the system to be up. -- */ --static int __init register_ext_kfuncs(void) --{ -- int ret; -- -- /* -- * Some kfuncs are context-sensitive and can only be called from -- * specific SCX ops. They are grouped into BTF sets accordingly. -- * Unfortunately, BPF currently doesn't have a way of enforcing such -- * restrictions. Eventually, the verifier should be able to enforce -- * them. For now, register them the same and make each kfunc explicitly -- * check using scx_kf_allowed(). -- */ -- if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_init)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_sleepable)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_enqueue_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_cpu_release)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_any))) { -- pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret); -- return ret; -- } -- -- return 0; --} --__initcall(register_ext_kfuncs); -diff --git a/kernel/sched/ext.h b/kernel/sched/ext.h -deleted file mode 100755 -index fba8ed845f99..000000000000 ---- a/kernel/sched/ext.h -+++ /dev/null -@@ -1,257 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ -- -- --/* -- * Tag marking a kernel function as a kfunc. This is meant to minimize the -- * amount of copy-paste that kfunc authors have to include for correctness so -- * as to avoid issues such as the compiler inlining or eliding either a static -- * kfunc, or a global kfunc in an LTO build. -- */ --#define __bpf_kfunc __used noinline -- --enum scx_wake_flags { -- /* expose select WF_* flags as enums */ -- SCX_WAKE_EXEC = WF_EXEC, -- SCX_WAKE_FORK = WF_FORK, -- SCX_WAKE_TTWU = WF_TTWU, -- SCX_WAKE_SYNC = WF_SYNC, --}; -- --enum scx_enq_flags { -- /* expose select ENQUEUE_* flags as enums */ -- SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, -- SCX_ENQ_HEAD = ENQUEUE_HEAD, -- -- /* high 32bits are SCX specific */ -- -- /* -- * Set the following to trigger preemption when calling -- * scx_bpf_dispatch() with a local dsq as the target. The slice of the -- * current task is cleared to zero and the CPU is kicked into the -- * scheduling path. Implies %SCX_ENQ_HEAD. -- */ -- SCX_ENQ_PREEMPT = 1LLU << 32, -- -- /* -- * The task being enqueued was previously enqueued on the current CPU's -- * %SCX_DSQ_LOCAL, but was removed from it in a call to the -- * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was -- * invoked in a ->cpu_release() callback, and the task is again -- * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the -- * task will not be scheduled on the CPU until at least the next invocation -- * of the ->cpu_acquire() callback. -- */ -- SCX_ENQ_REENQ = 1LLU << 40, -- -- /* -- * The task being enqueued is the only task available for the cpu. By -- * default, ext core keeps executing such tasks but when -- * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with -- * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set. -- * -- * If the BPF scheduler wants to continue executing the task, -- * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately. -- * If the task gets queued on a different dsq or the BPF side, the BPF -- * scheduler is responsible for triggering a follow-up scheduling event. -- * Otherwise, Execution may stall. -- */ -- SCX_ENQ_LAST = 1LLU << 41, -- -- /* -- * A hint indicating that it's advisable to enqueue the task on the -- * local dsq of the currently selected CPU. Currently used by -- * select_cpu_dfl() and together with %SCX_ENQ_LAST. -- */ -- SCX_ENQ_LOCAL = 1LLU << 42, -- -- /* high 8 bits are internal */ -- __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, -- -- SCX_ENQ_CLEAR_OPSS = 1LLU << 56, -- SCX_ENQ_DSQ_PRIQ = 1LLU << 57, --}; -- --enum scx_deq_flags { -- /* expose select DEQUEUE_* flags as enums */ -- SCX_DEQ_SLEEP = DEQUEUE_SLEEP, -- -- /* high 32bits are SCX specific */ -- -- /* -- * The generic core-sched layer decided to execute the task even though -- * it hasn't been dispatched yet. Dequeue from the BPF side. -- */ -- SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, --}; -- --enum scx_tg_flags { -- SCX_TG_ONLINE = 1U << 0, -- SCX_TG_INITED = 1U << 1, --}; -- --enum scx_kick_flags { -- SCX_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -- SCX_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ --}; -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --extern const struct sched_class ext_sched_class; --extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops; --extern const struct file_operations sched_ext_fops; --extern unsigned long scx_watchdog_timeout; --extern unsigned long scx_watchdog_timestamp; -- --DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); --DECLARE_STATIC_KEY_FALSE(__scx_switched_all); --#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) --#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) -- --DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); -- --bool task_on_scx(struct task_struct *p); --void scx_pre_fork(struct task_struct *p); --int scx_fork(struct task_struct *p); --void scx_post_fork(struct task_struct *p); --void scx_cancel_fork(struct task_struct *p); --int scx_check_setscheduler(struct task_struct *p, int policy); --bool scx_can_stop_tick(struct rq *rq); --void init_sched_ext_class(void); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...); --#define scx_ops_error(fmt, args...) \ -- scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args) -- --void __scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active); -- --static inline void scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active) --{ -- if (!scx_enabled()) -- return; --#ifdef CONFIG_SMP -- /* -- * Pairs with the smp_load_acquire() issued by a CPU in -- * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -- * resched. -- */ -- smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1); --#endif -- if (!static_branch_unlikely(&scx_ops_cpu_preempt)) -- return; -- __scx_notify_pick_next_task(rq, p, active); --} -- --//extern void scx_scheduler_tick(void); --static inline void scx_notify_sched_tick(void) --{ -- unsigned long last_check; -- -- if (!scx_enabled()) -- return; -- //scx_scheduler_tick(); -- -- last_check = scx_watchdog_timestamp; -- if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "watchdog failed to check in for %u.%03us", -- dur_ms / 1000, dur_ms % 1000); -- } --} -- --static inline const struct sched_class *next_active_class(const struct sched_class *class) --{ -- class++; -- if (scx_switched_all() && class == &fair_sched_class) -- class++; -- if (!scx_enabled() && class == &ext_sched_class) -- class++; -- return class; --} -- --#define for_active_class_range(class, _from, _to) \ -- for (class = (_from); class != (_to); class = next_active_class(class)) -- --#define for_each_active_class(class) \ -- for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -- --/* -- * SCX requires a balance() call before every pick_next_task() call including -- * when waking up from idle. -- */ --#define for_balance_class_range(class, prev_class, end_class) \ -- for_active_class_range(class, (prev_class) > &ext_sched_class ? \ -- &ext_sched_class : (prev_class), (end_class)) -- --#ifdef CONFIG_SCHED_CORE --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi); --#endif -- --#else /* CONFIG_SCHED_CLASS_EXT */ -- --#define scx_enabled() false --#define scx_switched_all() false -- --static inline void scx_pre_fork(struct task_struct *p) {} --static inline int scx_fork(struct task_struct *p) { return 0; } --static inline void scx_post_fork(struct task_struct *p) {} --static inline void scx_cancel_fork(struct task_struct *p) {} --static inline int scx_check_setscheduler(struct task_struct *p, -- int policy) { return 0; } --static inline bool scx_can_stop_tick(struct rq *rq) { return true; } --static inline void init_sched_ext_class(void) {} --static inline void scx_notify_pick_next_task(struct rq *rq, -- const struct task_struct *p, -- const struct sched_class *active) {} --static inline void scx_notify_sched_tick(void) {} -- --#define for_each_active_class for_each_class --#define for_balance_class_range for_class_range -- --#endif /* CONFIG_SCHED_CLASS_EXT */ -- --#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) --void __scx_update_idle(struct rq *rq, bool idle); -- --static inline void scx_update_idle(struct rq *rq, bool idle) --{ -- if (scx_enabled()) -- __scx_update_idle(rq, idle); --} --#else --static inline void scx_update_idle(struct rq *rq, bool idle) {} --#endif -- --#ifdef CONFIG_CGROUP_SCHED --#ifdef CONFIG_EXT_GROUP_SCHED --int scx_tg_online(struct task_group *tg); --void scx_tg_offline(struct task_group *tg); --int scx_cgroup_can_attach(struct cgroup_taskset *tset); --void scx_move_task(struct task_struct *p); --void scx_cgroup_finish_attach(void); --void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); --void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); --#else /* CONFIG_EXT_GROUP_SCHED */ --static inline int scx_tg_online(struct task_group *tg) { return 0; } --static inline void scx_tg_offline(struct task_group *tg) {} --static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } --static inline void scx_move_task(struct task_struct *p) {} --static inline void scx_cgroup_finish_attach(void) {} --static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} --static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} --#endif /* CONFIG_EXT_GROUP_SCHED */ --#endif /* CONFIG_CGROUP_SCHED */ -diff --git a/kernel/sched/hmbird.h b/kernel/sched/hmbird.h -new file mode 100755 -index 000000000000..d0d84ddee13d ---- /dev/null -+++ b/kernel/sched/hmbird.h -@@ -0,0 +1,343 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#ifndef _HMBIRD_H_ -+#define _HMBIRD_H_ -+ -+/* -+ * Tag marking a kernel function as a kfunc. This is meant to minimize the -+ * amount of copy-paste that kfunc authors have to include for correctness so -+ * as to avoid issues such as the compiler inlining or eliding either a static -+ * kfunc, or a global kfunc in an LTO build. -+ */ -+ -+#define DEBUG_INTERNAL (1 << 0) -+#define DEBUG_INFO_TRACE (1 << 1) -+#define DEBUG_INFO_SYSTRACE (1 << 2) -+ -+#define NUMS_CGROUP_KINDS (512) -+#define SLIM_FOR_SGAME (1 << 0) -+#define SLIM_FOR_GENSHIN (1 << 1) -+ -+#ifdef MAX_TASK_NR -+#define MAX_KEY_THREAD_RECORD ((MAX_TASK_NR + 1) >> 1) -+#else -+#define MAX_KEY_THREAD_RECORD (1 << 3) -+#endif /* MAX_TASK_NR */ -+ -+#define TOP_TASK_SHIFT (8) -+#define TOP_TASK_MAX (1 << TOP_TASK_SHIFT) -+#ifndef TOP_TASK_BITS_MASK -+#define TOP_TASK_BITS_MASK (TOP_TASK_MAX - 1) -+#endif /* TOP_TASK_BITS_MASK */ -+ -+extern struct md_info_t *md_info; -+ -+#define debug_enabled() \ -+ (unlikely(hmbirdcore_debug)) -+ -+#define hmbird_debug(fmt, ...) \ -+ pr_info(":"fmt, ##__VA_ARGS__) -+ -+#define hmbird_err(id, fmt, ...) \ -+do { \ -+ pr_err(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_deferred_err(id, fmt, ...) \ -+do { \ -+ printk_deferred(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \ -+do { \ -+ if (unlikely(cond)) { \ -+ hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+ } \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_info_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_info_trace(fmt, ...) -+#endif -+ -+#define hmbird_info_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+ } \ -+} while (0) -+ -+#define hmbird_output_systrace(fmt, ...) \ -+do { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_internal_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_internal_trace(fmt, ...) -+#endif -+ -+#define hmbird_internal_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) { \ -+ hmbird_output_systrace(fmt, ##__VA_ARGS__); \ -+ } \ -+} while (0) -+ -+enum hmbird_wake_flags { -+ /* expose select WF_* flags as enums */ -+ HMBIRD_WAKE_EXEC = WF_EXEC, -+ HMBIRD_WAKE_FORK = WF_FORK, -+ HMBIRD_WAKE_TTWU = WF_TTWU, -+ HMBIRD_WAKE_SYNC = WF_SYNC, -+}; -+ -+enum hmbird_enq_flags { -+ /* expose select ENQUEUE_* flags as enums */ -+ HMBIRD_ENQ_WAKEUP = ENQUEUE_WAKEUP, -+ HMBIRD_ENQ_HEAD = ENQUEUE_HEAD, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * Set the following to trigger preemption when calling -+ * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the -+ * current task is cleared to zero and the CPU is kicked into the -+ * scheduling path. Implies %HMBIRD_ENQ_HEAD. -+ */ -+ HMBIRD_ENQ_PREEMPT = 1LLU << 32, -+ HMBIRD_ENQ_REENQ = 1LLU << 40, -+ HMBIRD_ENQ_LAST = 1LLU << 41, -+ -+ /* -+ * A hint indicating that it's advisable to enqueue the task on the -+ * local dsq of the currently selected CPU. Currently used by -+ * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST. -+ */ -+ HMBIRD_ENQ_LOCAL = 1LLU << 42, -+ -+ /* high 8 bits are internal */ -+ __HMBIRD_ENQ_INTERNAL_MASK = 0xffLLU << 56, -+ -+ HMBIRD_ENQ_CLEAR_OPSS = 1LLU << 56, -+ HMBIRD_ENQ_DSQ_PRIQ = 1LLU << 57, -+}; -+ -+enum hmbird_deq_flags { -+ /* expose select DEQUEUE_* flags as enums */ -+ HMBIRD_DEQ_SLEEP = DEQUEUE_SLEEP, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * The generic core-sched layer decided to execute the task even though -+ * it hasn't been dispatched yet. Dequeue from the BPF side. -+ */ -+ HMBIRD_DEQ_CORE_SCHED_EXEC = 1LLU << 32, -+}; -+ -+enum hmbird_kick_flags { -+ HMBIRD_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -+ HMBIRD_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ -+}; -+ -+enum hmbird_task_prop_type { -+ HMBIRD_TASK_PROP_COMMON, -+ HMBIRD_TASK_PROP_PIPELINE, -+ HMBIRD_TASK_PROP_DEBUG_OR_LOG, -+ HMBIRD_TASK_PROP_HIGH_LOAD, -+ HMBIRD_TASK_PROP_IO, -+ HMBIRD_TASK_PROP_NETWORK, -+ HMBIRD_TASK_PROP_PERIODIC, -+ HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL, -+ HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL, -+ HMBIRD_TASK_PROP_ISOLATE, -+ HMBIRD_TASK_PROP_MAX, -+}; -+ -+enum hmbird_preempt_policy_id { -+ HMBIRD_PREEMPT_POLICY_NONE, -+ HMBIRD_PREEMPT_POLICY_PRIO_BASED, -+}; -+ -+extern const struct sched_class hmbird_sched_class; -+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops; -+extern const struct file_operations sched_hmbird_fops; -+extern unsigned long hmbird_watchdog_timeout; -+extern unsigned long hmbird_watchdog_timestamp; -+ -+enum scx_rq_flags { -+ HMBIRD_RQ_CAN_STOP_TICK = 1 << 0, -+}; -+ -+struct hmbird_rq { -+ struct hmbird_dispatch_q local_dsq; -+ struct list_head watchdog_list; -+ u64 ops_qseq; -+ u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -+ u32 nr_running; -+ u32 flags; -+ bool cpu_released; -+ cpumask_var_t cpus_to_kick; -+ cpumask_var_t cpus_to_preempt; -+ cpumask_var_t cpus_to_wait; -+ u64 pnt_seq; -+ u64 *prev_runnable_sum_fixed; -+ u32 *prev_window_size; -+ struct irq_work kick_cpus_irq_work; -+ bool pipeline; -+ bool exclusive; -+ bool period_disallow; -+ bool nonperiod_disallow; -+ struct rq *rq; -+ struct hmbird_sched_rq_stats *srq; -+}; -+ -+struct hmbird_sched_change_guard { -+ struct task_struct *p; -+ struct rq *rq; -+ bool queued; -+ bool running; -+ bool done; -+}; -+ -+extern struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -+ -+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags); -+ -+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -+ for (struct hmbird_sched_change_guard __cg = \ -+ hmbird_sched_change_guard_init(__rq, __p, __flags); \ -+ !__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags)) -+ -+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+bool task_on_hmbird(struct task_struct *p); -+int hmbird_pre_fork(struct task_struct *p); -+int hmbird_fork(struct task_struct *p); -+void hmbird_post_fork(struct task_struct *p); -+void hmbird_cancel_fork(struct task_struct *p); -+int hmbird_check_setscheduler(struct task_struct *p, int policy); -+bool hmbird_can_stop_tick(struct rq *rq); -+void init_sched_hmbird_class(void); -+ -+void hmbird_ops_exit(void); -+#define hmbird_ops_error(fmt, ...) \ -+do { \ -+ hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__); \ -+ hmbird_ops_exit(); \ -+} while (0) -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active); -+ -+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight) -+{ -+ return clamp_t(unsigned long, -+ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -+ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); -+} -+ -+static inline void hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active) -+{ -+ if (!hmbird_enabled()) -+ return; -+#ifdef CONFIG_SMP -+ /* -+ * Pairs with the smp_load_acquire() issued by a CPU in -+ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -+ * resched. -+ */ -+ smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1); -+#endif -+ if (!static_branch_unlikely(&hmbird_ops_cpu_preempt)) -+ return; -+ __hmbird_notify_pick_next_task(rq, p, active); -+} -+ -+extern void hmbird_scheduler_tick(void); -+void scan_timeout(struct rq *rq); -+void hmbird_notify_sched_tick(void); -+ -+static inline const struct sched_class *next_active_class(const struct sched_class *class) -+{ -+ class++; -+ -+ if (hmbird_enabled() && class == &fair_sched_class) -+ class++; -+ if (!hmbird_enabled() && class == &hmbird_sched_class) -+ class++; -+ return class; -+} -+ -+#define for_active_class_range(class, _from, _to) \ -+ for (class = (_from); class != (_to); class = next_active_class(class)) -+ -+#define for_each_active_class(class) \ -+ for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -+ -+/* -+ * HMBIRD requires a balance() call before every pick_next_task() call including -+ * when waking up from idle. -+ */ -+#define for_balance_class_range(class, prev_class, end_class) \ -+ for_active_class_range(class, (prev_class) > &hmbird_sched_class ? \ -+ &hmbird_sched_class : (prev_class), (end_class)) -+ -+#define MIN_CGROUP_DL_IDX (5) /* 8ms */ -+#define DEFAULT_CGROUP_DL_IDX (8) /* 64ms */ -+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS]; -+ -+ -+void __hmbird_update_idle(struct rq *rq, bool idle); -+ -+static inline void hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ if (hmbird_enabled()) -+ __hmbird_update_idle(rq, idle); -+} -+ -+int hmbird_ctrl(bool enable); -+ -+int hmbird_tg_online(struct task_group *tg); -+ -+ -+static inline u16 hmbird_task_util(struct task_struct *p) -+{ -+ return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0; -+} -+u16 hmbird_cpu_util(int cpu); -+ -+bool get_hmbird_ops_enabled(void); -+bool get_non_hmbird_task(void); -+void set_cpu_cluster(u64 cpu_cluster); -+#endif /*_EXT_H_*/ -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -new file mode 100755 -index 000000000000..ce05928392bf ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird.c -@@ -0,0 +1,4313 @@ -+// SPDX-License-Identifier: GPL-2.0 -+ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#include -+#include -+ -+#include "slim.h" -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+#include -+ -+#define CLUSTER_SEPARATE -+ -+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0); -+atomic_t non_hmbird_task; -+atomic_t hmbird_module_loaded = ATOMIC_INIT(0); -+int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0}; -+ -+enum hmbird_internal_consts { -+ HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ, -+}; -+ -+enum hmbird_ops_enable_state { -+ HMBIRD_OPS_PREPPING, -+ HMBIRD_OPS_ENABLING, -+ HMBIRD_OPS_ENABLED, -+ HMBIRD_OPS_DISABLING, -+ HMBIRD_OPS_DISABLED, -+}; -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+/* -+ * hmbird_entity->ops_state -+ * -+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler. -+ * State transitions look as follows: -+ * -+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING -+ * ^ | | -+ * | v v -+ * \-------------------------------/ -+ * -+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -+ * sites for explanations on the conditions being waited upon and why they are -+ * safe. Transitions out of them into NONE or QUEUED must store_release and the -+ * waiters should load_acquire. -+ * -+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether -+ * any given task can be dispatched by the BPF scheduler at all times and thus -+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core -+ * can safely reject invalid dispatches. -+ */ -+enum hmbird_ops_state { -+ HMBIRD_OPSS_NONE, /* owned by the HMBIRD core */ -+ HMBIRD_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -+ HMBIRD_OPSS_QUEUED, /* owned by the BPF scheduler */ -+ HMBIRD_OPSS_DISPATCHING, /* in transit back to the HMBIRD core */ -+ -+ /* -+ * QSEQ brands each QUEUED instance so that, when dispatch races -+ * dequeue/requeue, the dispatcher can tell whether it still has a claim -+ * on the task being dispatched. -+ */ -+ HMBIRD_OPSS_QSEQ_SHIFT = 2, -+ HMBIRD_OPSS_STATE_MASK = (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1, -+ HMBIRD_OPSS_QSEQ_MASK = ~HMBIRD_OPSS_STATE_MASK, -+}; -+ -+enum switch_stat { -+ HMBIRD_DISABLED, -+ HMBIRD_SWITCH_PREP, -+ HMBIRD_RQ_SWITCH_BEGIN, -+ HMBIRD_RQ_SWITCH_DONE, -+ HMBIRD_ENABLED, -+}; -+enum switch_stat curr_ss; -+ -+/* -+ * During exit, a task may schedule after losing its PIDs. When disabling the -+ * BPF scheduler, we need to be able to iterate tasks in every state to -+ * guarantee system safety. Maintain a dedicated task list which contains every -+ * task between its fork and eventual free. -+ */ -+DEFINE_SPINLOCK(hmbird_tasks_lock); -+static LIST_HEAD(hmbird_tasks); -+ -+/* ops enable/disable */ -+static struct kthread_worker *hmbird_ops_helper; -+static DEFINE_MUTEX(hmbird_ops_enable_mutex); -+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem); -+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED); -+ -+static bool hmbird_warned_zero_slice; -+enum hmbird_switch_type sw_type; -+static int skip_num[MAX_GLOBAL_DSQS]; -+static int big_distribute_mask_prev; -+static int little_distribute_mask_prev; -+ -+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0); -+ -+/* -+ * The maximum amount of time in jiffies that a task may be runnable without -+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger -+ * hmbird_ops_error(). -+ */ -+unsigned long hmbird_watchdog_timeout; -+ -+/* -+ * The last time the delayed work was run. This delayed work relies on -+ * ksoftirqd being able to run to service timer interrupts, so it's possible -+ * that this work itself could get wedged. To account for this, we check that -+ * it's not stalled in the timer tick, and trigger an error if it is. -+ */ -+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES; -+ -+static struct delayed_work hmbird_watchdog_work; -+static struct work_struct hmbird_err_exit_work; -+ -+/* idle tracking */ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_CPUMASK_OFFSTACK -+#define CL_ALIGNED_IF_ONSTACK -+#else -+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp -+#endif -+ -+static struct { -+ cpumask_var_t cpu; -+ cpumask_var_t smt; -+} idle_masks CL_ALIGNED_IF_ONSTACK; -+ -+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus; -+#endif /* CONFIG_SMP */ -+ -+/* dispatch queues */ -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global; -+ -+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128}; -+u32 pcp_dsq_deadline = 20; -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS]; -+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq); -+ -+static u64 max_hmbird_dsq_internal_id; -+ -+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/ -+#define CLUSTER_SEPARATE_IDX (8) -+#define GDSQS_ID_BASE (3) -+#define UX_COMPATIBLE_IDX (4) -+#define NON_PERIOD_START (5) -+#define NON_PERIOD_END (MAX_GLOBAL_DSQS) -+#define CREATE_DSQ_LEVEL_WITHIN (1) -+ -+struct hmbird_sched_info { -+ spinlock_t lock; -+ int curr_idx[2]; -+ int rtime[MAX_GLOBAL_DSQS]; -+}; -+ -+struct pcp_sched_info { -+ s64 pcp_seq; -+ int rtime; -+ bool pcp_round; -+}; -+ -+/* -+ * pcp_info may rw by another cpu. -+ * protected by rq->lock. -+ */ -+atomic64_t pcp_dsq_round; -+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info); -+ -+struct md_info_t *md_info; -+ -+static int b_rescue_l, l_rescue_b; -+static struct hmbird_sched_info sinfo; -+ -+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC; -+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = { -+ 0, 0, 0, 0, 0, -+ 32 * NSEC_PER_MSEC, -+ 20 * NSEC_PER_MSEC, -+ 14 * NSEC_PER_MSEC, -+ 8 * NSEC_PER_MSEC, -+ 6 * NSEC_PER_MSEC -+}; -+ -+struct cluster_ctx { -+ /* cpu-dsq map must within [lower, upper) */ -+ int upper; -+ int lower; -+ int tidx; -+}; -+ -+enum stat_items { -+ GLOBAL_STAT, -+ CPU_ALLOW_FAIL, -+ RT_CNT, -+ KEY_TASK_CNT, -+ SWITCH_IDX, -+ TIMEOUT_CNT, -+ -+ TOTAL_DSP_CNT, -+ MOVE_RQ_CNT, -+ SELECT_CPU, -+ -+ DWORD_STAT_END = SELECT_CPU, -+ -+ GDSQ_CNT, -+ ERR_IDX, -+ PCP_TIMEOUT_CNT, -+ PCP_LDSQ_CNT, -+ PCP_ENQL_CNT, -+ -+ MAX_ITEMS, -+}; -+static DEFINE_SPINLOCK(stats_lock); -+static char *stats_str[MAX_ITEMS] = { -+ "global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt", -+ "switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt", -+ "select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt", -+ "pcp_ldsq_cnt", "pcp_enql_cnt" -+}; -+ -+ -+struct stats_struct { -+ u64 global_stat[2]; -+ u64 cpu_allow_fail[2]; -+ u64 rt_cnt[2]; -+ u64 key_task_cnt[2]; -+ u64 switch_idx[2]; -+ u64 timeout_cnt[2]; -+ -+ u64 total_dsp_cnt[2]; -+ u64 move_rq_cnt[2]; -+ u64 select_cpu[2]; -+ -+ u64 gdsq_count[MAX_GLOBAL_DSQS][2]; -+ u64 err_idx[5]; -+ u64 pcp_timeout_cnt[NR_CPUS]; -+ u64 pcp_ldsq_count[NR_CPUS][2]; -+ u64 pcp_enql_cnt[NR_CPUS]; -+} stats_data; -+ -+static struct { -+ cpumask_var_t ex_free; -+ cpumask_var_t exclusive; -+ cpumask_var_t partial; -+ cpumask_var_t big; -+ cpumask_var_t little; -+} iso_masks __read_mostly; -+ -+/* -+ * Need more synchronization for these two variables? -+ * I choose not to. -+ */ -+static int l_need_rescue, b_need_rescue; -+ -+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...) \ -+{ \ -+ char buf[MAX_FATAL_INFO]; \ -+ \ -+ scnprintf(buf, MAX_FATAL_INFO, fmt, ##args); \ -+ hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf); \ -+ trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable), \ -+ READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf); \ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); \ -+} \ -+ -+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2) -+{ -+ int c1, c2; -+ struct cpumask mask = {.bits = {0}}; -+ struct cpumask tmp = {.bits = {0}}; -+ -+ if (!slim_stats) -+ return false; -+ -+ if (!rq1 || !rq2) -+ return false; -+ -+ c1 = cpu_of(rq1); -+ c2 = cpu_of(rq2); -+ cpumask_set_cpu(c1, &mask); -+ cpumask_set_cpu(c2, &mask); -+ -+ if (cpumask_and(&tmp, iso_masks.little, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.big, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.partial, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ return false; -+} -+ -+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu) -+{ -+ unsigned long flags; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ if (!slim_stats) -+ return; -+ -+ switch (item) { -+ case GLOBAL_STAT: -+ fallthrough; -+ case CPU_ALLOW_FAIL: -+ fallthrough; -+ case RT_CNT: -+ fallthrough; -+ case KEY_TASK_CNT: -+ fallthrough; -+ case SWITCH_IDX: -+ fallthrough; -+ case TIMEOUT_CNT: -+ fallthrough; -+ case TOTAL_DSP_CNT: -+ fallthrough; -+ case MOVE_RQ_CNT: -+ fallthrough; -+ case SELECT_CPU: -+ pval = pbase + item * 2 + idx; -+ break; -+ case GDSQ_CNT: -+ pval = &stats_data.gdsq_count[dsq_id][idx]; -+ break; -+ case ERR_IDX: -+ pval = &stats_data.err_idx[idx]; -+ break; -+ case PCP_TIMEOUT_CNT: -+ pval = &stats_data.pcp_timeout_cnt[cpu]; -+ break; -+ case PCP_LDSQ_CNT: -+ pval = &stats_data.pcp_ldsq_count[cpu][idx]; -+ break; -+ case PCP_ENQL_CNT: -+ pval = &stats_data.pcp_enql_cnt[cpu]; -+ break; -+ default: -+ return; -+ } -+ -+ spin_lock_irqsave(&stats_lock, flags); -+ *pval += 1; -+ spin_unlock_irqrestore(&stats_lock, flags); -+} -+ -+static inline bool handle_ret(int ret, int *idx, int len) -+{ -+ if (ret < 0 || ret >= len - *idx) -+ return true; -+ *idx += ret; -+ return false; -+} -+ -+#define PRINT_INTV (5 * HZ) -+void stats_print(char *buf, int len) -+{ -+ int idx = 0, i, j, ret; -+ int item = 0; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ for (item = 0; item < MAX_ITEMS; item++) { -+ if (item <= DWORD_STAT_END) { -+ pval = pbase + item * 2; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n", -+ stats_str[item], pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == GDSQ_CNT) { -+ for (j = 0; j < MAX_GLOBAL_DSQS; j++) { -+ pval = (u64 *)&stats_data.gdsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == ERR_IDX) { -+ pval = (u64 *)&stats_data.err_idx; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n", -+ stats_str[item], pval[0], -+ pval[1], pval[2], pval[3], pval[4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == PCP_TIMEOUT_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_timeout_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_LDSQ_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_ldsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_ENQL_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_enql_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } -+ } -+ -+ if (!md_info) { -+ buf[idx] = '\0'; -+ return; -+ } -+ -+ ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_SWITCHS; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "sw_rec[%d] = %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.sw_rec[i].switch_at, -+ md_info->kern_dump.sw_rec[i].is_success, -+ md_info->kern_dump.sw_rec[i].end_state, -+ md_info->kern_dump.sw_rec[i].switch_reason); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_EXCEP_ID; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "excep[%d] = %llu %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.excep_rec[i][0], -+ md_info->kern_dump.excep_rec[i][1], -+ md_info->kern_dump.excep_rec[i][2], -+ md_info->kern_dump.excep_rec[i][3], -+ md_info->kern_dump.excep_rec[i][4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n", -+ i, md_info->kern_dump.excep_idx[i]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ -+ buf[idx] = '\0'; -+} -+ -+enum cpu_type { -+ LITTLE, -+ BIG, -+ PARTIAL, -+ EXCLUSIVE, -+ EX_FREE, -+ INVALID -+}; -+ -+enum dsq_type { -+ GLOBAL_DSQ, -+ PCP_DSQ, -+ OTHER, -+ MAX_DSQ_TYPE, -+}; -+ -+static enum cpu_type cpu_cluster(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (get_hmbird_rq(rq)->exclusive) { -+ return EXCLUSIVE; -+ } else { -+ if (cpumask_test_cpu(cpu, iso_masks.little)) -+ return LITTLE; -+ else if (cpumask_test_cpu(cpu, iso_masks.big)) -+ return BIG; -+ else if (cpumask_test_cpu(cpu, iso_masks.partial)) -+ return PARTIAL; -+ else if (cpumask_test_cpu(cpu, iso_masks.exclusive)) -+ return EXCLUSIVE; -+ else if (cpumask_test_cpu(cpu, iso_masks.ex_free)) -+ return EX_FREE; -+ } -+ return INVALID; -+} -+ -+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq) -+{ -+ if (!dsq) -+ return OTHER; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= GDSQS_ID_BASE) && -+ ((dsq->id & 0xff) < MAX_GLOBAL_DSQS)) -+ return GLOBAL_DSQ; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) && -+ ((dsq->id & 0xff) < max_hmbird_dsq_internal_id)) -+ return PCP_DSQ; -+ -+ return OTHER; -+} -+ -+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq) -+{ -+ enum dsq_type type; -+ -+ type = get_dsq_type(dsq); -+ switch (type) { -+ case GLOBAL_DSQ: -+ case PCP_DSQ: -+ return (dsq->id & 0xff) - GDSQS_ID_BASE; -+ default: -+ return -1; -+ } -+ return -1; -+} -+ -+static void update_cpus_idle(bool set, struct cpumask *mask) -+{ -+ int cpu; -+ struct rq *rq; -+ -+ if (set) { -+ for_each_cpu(cpu, mask) { -+ rq = cpu_rq(cpu); -+ if (is_idle_task(rq->curr)) -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ } -+ } else -+ cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask); -+} -+ -+static void set_partial_status(bool enable, bool little, bool big) -+{ -+ WRITE_ONCE(partial_enable, enable); -+ WRITE_ONCE(l_need_rescue, little); -+ WRITE_ONCE(b_need_rescue, big); -+} -+ -+static bool is_little_need_rescue(void) -+{ -+ return READ_ONCE(l_need_rescue); -+} -+ -+static bool is_big_need_rescue(void) -+{ -+ return READ_ONCE(b_need_rescue); -+} -+ -+static bool is_partial_enabled(void) -+{ -+ return READ_ONCE(partial_enable); -+} -+ -+static bool is_partial_cpu(int cpu) -+{ -+ return cpumask_test_cpu(cpu, iso_masks.partial); -+} -+ -+static void set_iso_par_free(bool enable) -+{ -+ WRITE_ONCE(isolate_ctrl, enable); -+} -+ -+static bool is_iso_par_free(void) -+{ -+ return READ_ONCE(isolate_ctrl); -+} -+ -+static bool skip_update_idle(void) -+{ -+ int cpu = smp_processor_id(); -+ enum cpu_type type = cpu_cluster(cpu); -+ -+ if ((type == EXCLUSIVE && !is_iso_par_free()) || -+ /* partial enable may changed during idle, it doesn't matter. */ -+ (!is_partial_enabled() && type == PARTIAL)) -+ return true; -+ -+ return false; -+} -+ -+static void init_isolate_cpus(void) -+{ -+ WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -+ cpumask_set_cpu(0, iso_masks.big); -+ cpumask_set_cpu(1, iso_masks.big); -+ cpumask_set_cpu(2, iso_masks.big); -+ cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(4, iso_masks.big); -+ cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(6, iso_masks.partial); -+ cpumask_set_cpu(7, iso_masks.ex_free); -+} -+ -+extern spinlock_t css_set_lock; -+ -+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp) -+{ -+ int i; -+ struct cgroup *anc; -+ -+ spin_lock_irq(&css_set_lock); -+ for (i = 0; i < cgrp->level; i++) { -+ anc = cgrp->ancestors[i]; -+ if (anc->level != CREATE_DSQ_LEVEL_WITHIN) -+ continue; -+ cgroup_get(anc); -+ spin_unlock_irq(&css_set_lock); -+ return anc; -+ } -+ spin_unlock_irq(&css_set_lock); -+ hmbird_err(NO_CGROUP_L1, ":error cgroup = %s\n", cgrp->kn->name); -+ return NULL; -+} -+ -+#define PCP_IDX_BIT (1 << 31) -+ -+static bool is_pcp_rt(struct task_struct *p) -+{ -+ return rt_prio(p->prio) && (p->nr_cpus_allowed == 1); -+} -+ -+static bool is_pcp_idx(int idx) -+{ -+ return idx >= MAX_GLOBAL_DSQS; -+} -+ -+static bool is_critical_system_task(struct task_struct *p) -+{ -+ int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK; -+ -+ return (p->prio < (MAX_RT_PRIO >> 1) && -+ (sp_dl < SCHED_PROP_DEADLINE_LEVEL3)); -+} -+ -+#define ISOLATE_TASK_TOP_BIT (1 << 17) -+#define PIPELINE_TASK_TOP_BIT (1 << 9) -+static bool is_pipeline_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT); -+} -+ -+static bool is_isolate_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT); -+} -+ -+static bool is_critical_app_task_without_isolate(struct task_struct *p) -+{ -+ return task_is_top_task(p) && !is_isolate_task(p); -+} -+ -+static int find_idx_from_task(struct task_struct *p) -+{ -+ int idx, cpu; -+ int sp_dl; -+ struct task_group *tg = p->sched_task_group; -+ -+ if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) { -+ cpu = cpumask_any(p->cpus_ptr); -+ idx = cpu + MAX_GLOBAL_DSQS; -+ return idx; -+ } -+ -+ if (is_critical_system_task(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL0; -+ goto done; -+ } -+ -+ if (is_critical_app_task_without_isolate(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL1; -+ goto done; -+ } -+ -+ sp_dl = hmbird_get_dsq_id(p); -+ if (sp_dl) { -+ idx = sp_dl; -+ goto done; -+ } -+ -+ if (rt_prio(p->prio)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL3; -+ goto done; -+ } -+ -+ if (tg && tg->css.cgroup && tg->css.cgroup->kn) { -+ if (likely(tg->css.cgroup->kn->id >= 0 && -+ tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS)) -+ idx = cgroup_ids_table[tg->css.cgroup->kn->id]; -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ -+done: -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) { -+ hmbird_err(DSQ_ID_ERR, " : idx error, idx = %d-----\n", idx); -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } -+ return idx; -+} -+ -+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p) -+{ -+ int idx; -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq; -+ -+ if (!p) -+ return NULL; -+ -+ idx = find_idx_from_task(p); -+ if (is_pcp_idx(idx)) { -+ idx -= MAX_GLOBAL_DSQS; -+ dsq = &per_cpu(pcp_ldsq, idx); -+ get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq); -+ slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx); -+ } else { -+ dsq = &gdsqs[idx]; -+ get_hmbird_ts(p)->gdsq_idx = idx; -+ slim_stats_record(GDSQ_CNT, 0, idx, 0); -+ } -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ dsq->last_consume_at = jiffies; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ return dsq; -+} -+ -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq); -+ -+static void set_partial_rescue(bool p_state, bool l_over, bool b_over) -+{ -+ set_partial_status(p_state, l_over, b_over); -+ update_cpus_idle(p_state, iso_masks.partial); -+ hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled()); -+ hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue()); -+ hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue()); -+} -+ -+static void free_isocpu(bool enable) -+{ -+ set_iso_par_free(enable); -+ update_cpus_idle(enable, iso_masks.ex_free); -+ hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free()); -+} -+ -+inline u64 get_hmbird_cpu_util(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed) -+ return 0; -+ u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed); -+ u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size); -+ -+ do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT); -+ -+ return prev_runnable_sum_fixed; -+} -+ -+static inline unsigned int get_scaling_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->max; -+} -+ -+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static u64 get_cpus_max_util(struct cpumask *mask) -+{ -+ int cpu; -+ u64 max = 0; -+ u64 util, ratio; -+ unsigned long effective_cap = 0; -+ for_each_cpu(cpu, mask) { -+ if (slim_walt_ctrl) -+ slim_get_cpu_util(cpu, &util); -+ else -+ util = get_hmbird_cpu_util(cpu); -+ -+ /* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/ -+ if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu))) -+ effective_cap = arch_scale_cpu_capacity(cpu); -+ else -+ effective_cap = arch_scale_cpu_capacity(cpu) * -+ get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu); -+ -+ ratio = util * 100 / effective_cap; -+ hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util); -+ hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n", -+ cpu, (u64)arch_scale_cpu_capacity(cpu)); -+ hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n", -+ cpu, (u64)effective_cap); -+ -+ if (ratio > 100) -+ ratio = 100; -+ -+ if (ratio > max) -+ max = ratio; -+ } -+ return max; -+} -+ -+static bool cluster_need_rescue(struct cpumask *mask, int hres) -+{ -+ return get_cpus_max_util(mask) > hres; -+} -+ -+void partial_dynamic_ctrl(void) -+{ -+ u64 lmax = 0, bmax = 0; -+ bool l_over, l_under, b_over, b_under; -+ static bool last_l_over, last_b_over; -+ static unsigned long last_check; -+ -+ /* Check partial every jiffies. need lock? */ -+ if (time_before_eq(jiffies, READ_ONCE(last_check))) -+ return; -+ -+ WRITE_ONCE(last_check, jiffies); -+ -+ lmax = get_cpus_max_util(iso_masks.little); -+ l_over = lmax > parctrl_high_ratio_l; -+ l_under = lmax < parctrl_low_ratio_l; -+ -+ bmax = get_cpus_max_util(iso_masks.big); -+ b_over = bmax > parctrl_high_ratio; -+ b_under = bmax < parctrl_low_ratio; -+ -+ if (is_partial_enabled() && (l_over || b_over)) { -+ if (last_l_over != l_over || last_b_over != b_over) -+ set_partial_rescue(true, l_over, b_over); -+ } else if (!is_partial_enabled() && (l_over || b_over)) { -+ set_partial_rescue(true, l_over, b_over); -+ } else if (is_partial_enabled() && l_under && b_under) { -+ set_partial_rescue(false, false, false); -+ } -+ last_l_over = l_over; -+ last_b_over = b_over; -+ -+ if (is_partial_enabled()) { -+ bmax = get_cpus_max_util(iso_masks.partial); -+ if (!is_iso_par_free() && bmax > isoctrl_high_ratio) { -+ hmbird_info_trace("partial max = %llu\n", bmax); -+ free_isocpu(true); -+ } else if (is_iso_par_free() && bmax < isoctrl_low_ratio) -+ free_isocpu(false); -+ } else if (is_iso_par_free()) { -+ free_isocpu(false); -+ } -+} -+ -+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx) -+{ -+ hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx); -+} -+ -+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx) -+{ -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[idx])) { -+ slim_stats_record(GDSQ_CNT, 1, idx, 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ int i; -+ -+ for (i = 0; i < UX_COMPATIBLE_IDX; i++) { -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ slim_stats_record(GDSQ_CNT, 1, i, 0); -+ return 1; -+ } -+ } -+ return 0; -+} -+ -+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) { -+ slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ goto clear_timeout; -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) -+ goto clear_timeout; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ hmbird_info_trace( -+ "dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n", -+ dsq_id_to_internal(dsq), entity->task->comm, -+ jiffies, msecs_to_jiffies(deadline), entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1); -+ return; -+ -+clear_timeout: -+ hmbird_info_trace("dsq[%d] clear timeout\n", -+ dsq_id_to_internal(dsq)); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0); -+ dsq->is_timeout = false; -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq)); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime) -+{ -+ hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime); -+} -+ -+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any) -+{ -+ bool is_timeout; -+ int cpu = cpu_of(rq); -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu); -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ is_timeout = dsq->is_timeout; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ /* -+ * dsq->is_timeout may change here, let it be. -+ * it won't cause serious problems. -+ * the same for consume_dispatch_q later. -+ */ -+ if (!is_timeout && !any) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, dsq)) { -+ if (is_timeout) { -+ hmbird_info_trace("dsq[%d] consume a pcp timeout task\n", -+ dsq_id_to_internal(dsq)); -+ update_timeout_stats(rq, dsq, pcp_dsq_deadline); -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu); -+ } -+ slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu); -+ return 1; -+ } -+ /* -+ * No pcp task, clear quota. -+ */ -+ if (any) { -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+ } -+ return 0; -+} -+ -+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf) -+{ -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ } -+ return 0; -+} -+ -+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf, -+ int tmp, int cidx, int tidx) -+{ -+ unsigned long flags; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) { -+ if (tmp != cidx) { -+ spin_lock(&sinfo.lock); -+ sinfo.curr_idx[tidx] = tmp; -+ spin_unlock(&sinfo.lock); -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]); -+ slim_stats_record(SWITCH_IDX, 0, 0, 0); -+ } -+ slim_stats_record(GDSQ_CNT, 1, tmp, 0); -+ -+ raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags); -+ gdsqs[tmp].last_consume_at = jiffies; -+ raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags); -+ return 1; -+ } -+ return 0; -+ -+} -+ -+static int get_cidx(struct cluster_ctx *ctx) -+{ -+ int cidx; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ if (cidx < ctx->lower || cidx >= ctx->upper) { -+ sinfo.curr_idx[ctx->tidx] = ctx->lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]); -+ slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ } -+ spin_unlock(&sinfo.lock); -+ -+ return cidx; -+} -+ -+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = CLUSTER_SEPARATE_IDX; -+ if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) { -+ pr_debug(" %s iso_masks.little first is %d\n", -+ __func__, cpumask_first(iso_masks.little)); -+ ctx->upper = NON_PERIOD_END; -+ } -+ ctx->tidx = 0; -+ break; -+ case LITTLE: -+ ctx->lower = CLUSTER_SEPARATE_IDX; -+ ctx->upper = NON_PERIOD_END; -+ if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) { -+ pr_debug(" %s iso_masks.big first is %d", -+ __func__, cpumask_first(iso_masks.big)); -+ ctx->lower = NON_PERIOD_START; -+ } -+ ctx->tidx = 1; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ case LITTLE: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = NON_PERIOD_END; -+ ctx->tidx = 0; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ if (cluster_separate) { -+ return gen_cluster_ctx_separate(ctx, type); -+ } else { -+ return gen_cluster_ctx_common(ctx, type); -+ } -+} -+ -+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ int i; -+ bool is_timeout; -+ unsigned long flags; -+ struct cluster_ctx ctx; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ /* Third param means only consume timeout pcp dsq. */ -+ if (consume_pcp_dsq(rq, rf, false)) -+ return 1; -+ -+ for (i = ctx.lower; i < ctx.upper; i++) { -+ raw_spin_lock_irqsave(&gdsqs[i].lock, flags); -+ is_timeout = gdsqs[i].is_timeout; -+ raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags); -+ /* gdsqs[i].is_timeout may change here, let it be... */ -+ if (is_timeout) { -+ /* -+ * consume_dispatch_q will acquire dsq-lock, -+ * So cannot keep lock here, annoy enough. -+ * may rewrite a consume_dispatch_q_locked, TODO. -+ */ -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ hmbird_info_trace("dsq[%d] consume a timeout task\n", i); -+ slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0); -+ update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]); -+ return 1; -+ } -+ } -+ } -+ return 0; -+} -+ -+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ struct cluster_ctx ctx; -+ int cidx; -+ int tmp; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ cidx = get_cidx(&ctx); -+ tmp = cidx; -+ do { -+ if (check_pcp_dsq_round(rq, rf)) -+ return 1; -+ if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx)) -+ return 1; -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[tmp] = 0; -+ systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]); -+ tmp++; -+ if (tmp >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ tmp = ctx.lower; -+ } -+ spin_unlock(&sinfo.lock); -+ } while (tmp != cidx); -+ -+ return consume_pcp_dsq(rq, rf, true); -+} -+ -+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ enum cpu_type type = cpu_cluster(cpu_of(rq)); -+ bool period_allowed = !get_hmbird_rq(rq)->period_disallow; -+ bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow; -+ -+ switch (type) { -+ case EX_FREE: -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ break; -+ case EXCLUSIVE: -+ if (!is_iso_par_free()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ } -+ /* Only non-period task can run on isolate cpus */ -+ if (!READ_ONCE(iso_free_rescue)) { -+ if (is_big_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ } else { -+ /* Free run, can run on any task. */ -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ case PARTIAL: -+ if (!is_partial_enabled()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ return 0; -+ } -+ if (is_big_need_rescue()) { -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ -+ case BIG: -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ if (is_iso_par_free() || (is_little_need_rescue() && -+ !cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) { -+ hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++); -+ return 1; -+ } -+ } -+ break; -+ -+ case LITTLE: -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ -+ if (is_iso_par_free() || (is_big_need_rescue() && -+ !cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) { -+ hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++); -+ return 1; -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return 0; -+} -+ -+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf) -+{ -+ return consume_hmbird_global_dsq(rq, rf); -+} -+ -+ -+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time) -+{ -+ int idx; -+ -+ /* which dsq belongs to while task enqueue, task will consume its running time. */ -+ idx = get_hmbird_ts(p)->gdsq_idx; -+ /* Only non-period dsq share running time between each other. */ -+ if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id) -+ return; -+ -+ if (idx >= MAX_GLOBAL_DSQS) { -+ per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } else { -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[idx] += exec_time; -+ spin_unlock(&sinfo.lock); -+ systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]); -+ } -+} -+ -+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type) -+{ -+ int cidx; -+ struct cluster_ctx ctx; -+ int cpu = cpu_of(rq); -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ if (cidx < ctx.lower || cidx >= ctx.upper) { -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ slim_stats_record(ERR_IDX, ctx.tidx, 0, 0); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ } -+ -+ while (1) { -+ if (per_cpu(pcp_info, cpu).pcp_round) { -+ if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) { -+ hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n", -+ cpu, per_cpu(pcp_info, cpu).rtime); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } -+ } -+ if (sinfo.rtime[cidx] < dsq_quota[cidx]) -+ break; -+ -+ /* clear current dsq rtime */ -+ sinfo.rtime[cidx] = 0; -+ systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]); -+ -+ sinfo.curr_idx[ctx.tidx]++; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", -+ ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ } -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ slim_stats_record(SWITCH_IDX, 1, 0, 0); -+ } -+ spin_unlock(&sinfo.lock); -+} -+ -+ -+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p) -+{ -+ enum cpu_type type; -+ -+ if (!rq || !p) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ switch (type) { -+ case PARTIAL: -+ return; -+ case EXCLUSIVE: -+ return; -+ case EX_FREE: -+ return; -+ default: -+ break; -+ } -+ update_dsq_idx(rq, p, type); -+} -+ -+ -+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ int dsq_id; -+ -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo) || dsq->is_timeout) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ hmbird_deferred_err(SCAN_ENTITY_NULL, -+ " : entity is NULL, dsq->id = %llu\n", dsq->id); -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ dsq->is_timeout = true; -+ dsq_id = dsq_id_to_internal(dsq); -+ hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n", -+ dsq_id, entity->task->comm, jiffies, entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1); -+ raw_spin_unlock(&dsq->lock); -+ -+ return true; -+} -+ -+void scan_timeout(struct rq *rq) -+{ -+ int i; -+ int cpu = cpu_of(rq); -+ struct hmbird_dispatch_q *dsq; -+ static u64 last_scan_at; -+ static DEFINE_PER_CPU(u64, pcp_last_scan_at); -+ -+ if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu))) -+ return; -+ per_cpu(pcp_last_scan_at, cpu) = jiffies; -+ -+ dsq = &per_cpu(pcp_ldsq, cpu); -+ scan_dsq_timeout(rq, dsq, pcp_dsq_deadline); -+ -+ if (time_before_eq(jiffies, (unsigned long)last_scan_at)) -+ return; -+ last_scan_at = jiffies; -+ -+ for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) { -+ dsq = &gdsqs[i]; -+ scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]); -+ } -+} -+ -+/*******************************Initialize***********************************/ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id); -+static void init_dsq_at_boot(void) -+{ -+ int i, cpu; -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i)); -+ } -+ for_each_possible_cpu(cpu) -+ init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i + cpu)); -+ -+ max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu; -+ spin_lock_init(&sinfo.lock); -+} -+ -+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgrp_name_to_idx(struct cgroup *cgrp) -+{ -+ int idx; -+ -+ if (!cgrp) -+ return -1; -+ -+ if (!strcmp(cgrp->kn->name, "display") -+ || !strcmp(cgrp->kn->name, "multimedia")) -+ idx = 5; /* 8ms */ -+ else if (!strcmp(cgrp->kn->name, "top-app") -+ || !strcmp(cgrp->kn->name, "ss-top")) -+ idx = 6; /* 16ms */ -+ else if (!strcmp(cgrp->kn->name, "ssfg") -+ || !strcmp(cgrp->kn->name, "foreground")) -+ idx = 7; /* 32ms */ -+ else if (!strcmp(cgrp->kn->name, "bg") -+ || !strcmp(cgrp->kn->name, "log") -+ || !strcmp(cgrp->kn->name, "dex2oat") -+ || !strcmp(cgrp->kn->name, "background")) -+ idx = 9; /* 128ms */ -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */ -+ -+ return idx; -+} -+ -+static void init_root_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX); -+} -+ -+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) { -+ pr_err("%s idx err!\n", __func__); -+ return; -+ } -+ -+ if (cgroup_ids_table[cgrp->kn->id] == -1) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp)); -+} -+ -+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ struct cgroup *l1cgrp; -+ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ l1cgrp = cgroup_ancestor_l1(cgrp); -+ if (l1cgrp) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp)); -+ cgroup_put(l1cgrp); -+} -+ -+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg) -+{ -+ switch (cgrp->level) { -+ case 0: -+ init_root_tg(cgrp, tg); -+ break; -+ case 1: -+ init_level1_tg(cgrp, tg); -+ break; -+ default: -+ init_child_tg(cgrp, tg); -+ break; -+ } -+} -+ -+/**************************************************************************/ -+ -+struct hmbird_task_iter { -+ struct hmbird_entity cursor; -+ struct task_struct *locked; -+ struct rq *rq; -+ struct rq_flags rf; -+}; -+ -+/** -+ * hmbird_task_iter_init - Initialize a task iterator -+ * @iter: iterator to init -+ * -+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized, -+ * @iter must eventually be exited with hmbird_task_iter_exit(). -+ * -+ * hmbird_tasks_lock may be released between this and the first next() call or -+ * between any two next() calls. If hmbird_tasks_lock is released between two -+ * next() calls, the caller is responsible for ensuring that the task being -+ * iterated remains accessible either through RCU read lock or obtaining a -+ * reference count. -+ * -+ * All tasks which existed when the iteration started are guaranteed to be -+ * visited as long as they still exist. -+ */ -+static void hmbird_task_iter_init(struct hmbird_task_iter *iter) -+{ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR }; -+ list_add(&iter->cursor.tasks_node, &hmbird_tasks); -+ iter->locked = NULL; -+} -+ -+/** -+ * hmbird_task_iter_exit - Exit a task iterator -+ * @iter: iterator to exit -+ * -+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held. -+ * If the iterator holds a task's rq lock, that rq lock is released. See -+ * hmbird_task_iter_init() for details. -+ */ -+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ if (list_empty(cursor)) -+ return; -+ -+ list_del_init(cursor); -+} -+ -+/** -+ * hmbird_task_iter_next - Next task -+ * @iter: iterator to walk -+ * -+ * Visit the next task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ struct hmbird_entity *pos; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ list_for_each_entry(pos, cursor, tasks_node) { -+ if (&pos->tasks_node == &hmbird_tasks) -+ return NULL; -+ if (!(pos->flags & HMBIRD_TASK_CURSOR)) { -+ list_move(cursor, &pos->tasks_node); -+ return pos->task; -+ } -+ } -+ -+ /* can't happen, should always terminate at hmbird_tasks above */ -+ hmbird_deferred_err(ITER_RET_NULL, " : unreachable path in scx_task_iter_next\n"); -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered - Next non-idle task -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ while ((p = hmbird_task_iter_next(iter))) { -+ if (!is_idle_task(p)) -+ return p; -+ } -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init() -+ * for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ p = hmbird_task_iter_next_filtered(iter); -+ if (!p) -+ return NULL; -+ -+ iter->rq = task_rq_lock(p, &iter->rf); -+ iter->locked = p; -+ return p; -+} -+ -+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void) -+{ -+ return atomic_read(&hmbird_ops_enable_state_var); -+} -+ -+static enum hmbird_ops_enable_state -+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to) -+{ -+ return atomic_xchg(&hmbird_ops_enable_state_var, to); -+} -+ -+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to, -+ enum hmbird_ops_enable_state from) -+{ -+ int from_v = from; -+ -+ return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to); -+} -+ -+static bool hmbird_ops_disabling(void) -+{ -+ return false; -+} -+ -+/** -+ * wait_ops_state - Busy-wait the specified ops state to end -+ * @p: target task -+ * @opss: state to wait the end of -+ * -+ * Busy-wait for @p to transition out of @opss. This can only be used when the -+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also -+ * has load_acquire semantics to ensure that the caller can see the updates made -+ * in the enqueueing and dispatching paths. -+ */ -+static void wait_ops_state(struct task_struct *p, u64 opss) -+{ -+ do { -+ cpu_relax(); -+ } while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss); -+} -+ -+ -+static void update_curr_hmbird(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ u64 now = rq_clock_task(rq); -+ u64 delta_exec; -+ -+ if (time_before_eq64(now, curr->se.exec_start)) -+ return; -+ -+ delta_exec = now - curr->se.exec_start; -+ curr->se.exec_start = now; -+ update_runningtime(rq, curr, delta_exec); -+ curr->se.sum_exec_runtime += delta_exec; -+ account_group_exec_runtime(curr, delta_exec); -+ cgroup_account_cputime(curr, delta_exec); -+ -+ if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF) -+ get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec); -+ -+ trace_sched_stat_runtime(curr, delta_exec, 0); -+} -+ -+static bool hmbird_dsq_priq_less(struct rb_node *node_a, -+ const struct rb_node *node_b) -+{ -+ const struct hmbird_entity *a = -+ container_of(node_a, struct hmbird_entity, dsq_node.priq); -+ const struct hmbird_entity *b = -+ container_of(node_b, struct hmbird_entity, dsq_node.priq); -+ -+ return time_before64(a->dsq_vtime, b->dsq_vtime); -+} -+ -+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ bool is_local = dsq->id == HMBIRD_DSQ_LOCAL; -+ unsigned long flags; -+ -+ hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq || -+ !list_empty(&get_hmbird_ts(p)->dsq_node.fifo), -+ "task = %s, dsq->id = %llu\n", p->comm, dsq->id); -+ hmbird_cond_deferred_err(ENQ_EXIST2, -+ (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq), -+ "task = %s\n", p->comm); -+ -+ if (!is_local) { -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ); -+ hmbird_ops_error(": %s\n", -+ "attempting to dispatch to a destroyed dsq"); -+ /* fall back to the global dsq */ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ dsq = &hmbird_dsq_global; -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ } -+ } -+ -+ if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) { -+ get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ; -+ rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq, -+ hmbird_dsq_priq_less); -+ } else { -+ if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT)) -+ list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ else -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ } -+ dsq->nr++; -+ get_hmbird_ts(p)->dsq = dsq; -+ -+ /* -+ * We're transitioning out of QUEUEING or DISPATCHING. store_release to -+ * match waiters' load_acquire. -+ */ -+ if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS) -+ atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE); -+ -+ if (is_local) { -+ struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq); -+ struct rq *rq = hmbird->rq; -+ bool preempt = false; -+ -+ if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr && -+ rq->curr->sched_class == &hmbird_sched_class) { -+ get_hmbird_ts(rq->curr)->slice = 0; -+ preempt = true; -+ } -+ -+ if (preempt || sched_class_above(&hmbird_sched_class, -+ rq->curr->sched_class)) -+ resched_curr(rq); -+ } else { -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ } -+} -+ -+static void task_unlink_from_dsq(struct task_struct *p, -+ struct hmbird_dispatch_q *dsq) -+{ -+ if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) { -+ rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ; -+ } else { -+ list_del_init(&get_hmbird_ts(p)->dsq_node.fifo); -+ } -+} -+ -+static bool task_linked_on_dsq(struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+} -+ -+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq; -+ bool is_local = dsq == &hmbird_rq->local_dsq; -+ -+ if (!dsq) { -+ hmbird_cond_deferred_err(TASK_LINKED1, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ /* -+ * When dispatching directly from the BPF scheduler to a local -+ * DSQ, the task isn't associated with any DSQ but -+ * @get_hmbird_ts(p)->holding_cpu may be set under the protection of -+ * %HMBIRD_OPSS_DISPATCHING. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu >= 0) -+ get_hmbird_ts(p)->holding_cpu = -1; -+ return; -+ } -+ -+ if (!is_local) -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ /* -+ * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node -+ * can't change underneath us. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu < 0) { -+ /* @p must still be on @dsq, dequeue */ -+ hmbird_cond_deferred_err(TASK_UNLINKED, -+ !task_linked_on_dsq(p), "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ } else { -+ /* -+ * We're racing against dispatch_to_local_dsq() which already -+ * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the -+ * holding_cpu which tells dispatch_to_local_dsq() that it lost -+ * the race. -+ */ -+ hmbird_cond_deferred_err(TASK_LINKED2, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ get_hmbird_ts(p)->holding_cpu = -1; -+ } -+ get_hmbird_ts(p)->dsq = NULL; -+ -+ if (!is_local) -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+ -+static bool test_rq_online(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ return rq->online; -+#else -+ return true; -+#endif -+} -+ -+static void refill_task_slice(struct task_struct *p) -+{ -+ if (is_isolate_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO; -+ else if (is_pipeline_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2; -+ else -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+} -+ -+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -+ int sticky_cpu) -+{ -+ struct hmbird_dispatch_q *d; -+ s32 cpu = -1; -+ -+ hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), -+ "task = %s\n", p->comm); -+ -+ if (is_pcp_rt(p)) { -+ /* Enqueue percpu rt task to local directly. */ -+ /* Or cause a bug when disable dispatch. */ -+ if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ } -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (cpu >= 0) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ -+ if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ /* rq migration */ -+ if (sticky_cpu == cpu_of(rq)) -+ goto local_norefill; -+ /* -+ * If !rq->online, we already told the BPF scheduler that the CPU is -+ * offline. We're just trying to on/offline the CPU. Don't bother the -+ * BPF scheduler. -+ */ -+ if (unlikely(!test_rq_online(rq))) -+ goto local; -+ -+ /* see %HMBIRD_OPS_ENQ_LAST */ -+ if (enq_flags & HMBIRD_ENQ_LAST) -+ goto local; -+ -+ if (enq_flags & HMBIRD_ENQ_LOCAL) -+ goto local; -+ else -+ goto global; -+local: -+ /* -+ * For task-ordering, slice refill must be treated as implying the end -+ * of the current slice. Otherwise, the longer @p stays on the CPU, the -+ * higher priority it becomes from hmbird_prio_less()'s POV. -+ */ -+ refill_task_slice(p); -+local_norefill: -+ dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags); -+ slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq)); -+ return; -+ -+global: -+ d = find_dsq_from_task(p); -+ if (d) { -+ refill_task_slice(p); -+ dispatch_enqueue(d, p, enq_flags); -+ return; -+ } -+ slim_stats_record(GLOBAL_STAT, 0, 0, 0); -+ refill_task_slice(p); -+ dispatch_enqueue(&hmbird_dsq_global, p, enq_flags); -+} -+ -+static bool watchdog_task_watched(const struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->watchdog_node); -+} -+ -+static void watchdog_watch_task(struct rq *rq, struct task_struct *p) -+{ -+ lockdep_assert_rq_held(rq); -+ if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ get_hmbird_ts(p)->runnable_at = jiffies; -+ clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list); -+} -+ -+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) -+{ -+ list_del_init(&get_hmbird_ts(p)->watchdog_node); -+ if (reset_timeout) -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags) -+{ -+ int sticky_cpu = get_hmbird_ts(p)->sticky_cpu; -+ -+ enq_flags |= get_hmbird_rq(rq)->extra_enq_flags; -+ -+ if (sticky_cpu >= 0) -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ -+ /* -+ * Restoring a running task will be immediately followed by -+ * set_next_task_hmbird() which expects the task to not be on the BPF -+ * scheduler as tasks can only start running through local DSQs. Force -+ * direct-dispatch into the local DSQ by setting the sticky_cpu. -+ */ -+ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -+ sticky_cpu = cpu_of(rq); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_UNWATCHED, -+ !watchdog_task_watched(p), "task = %s\n", p->comm); -+ return; -+ } -+ -+ watchdog_watch_task(rq, p); -+ set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ get_hmbird_rq(rq)->nr_running++; -+ add_nr_running(rq, 1); -+ -+ do_enqueue_task(rq, p, enq_flags, sticky_cpu); -+} -+ -+static void ops_dequeue(struct task_struct *p, u64 deq_flags) -+{ -+ u64 opss; -+ -+ watchdog_unwatch_task(p, false); -+ -+ /* acquire ensures that we see the preceding updates on QUEUED */ -+ opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state); -+ -+ switch (opss & HMBIRD_OPSS_STATE_MASK) { -+ case HMBIRD_OPSS_NONE: -+ break; -+ case HMBIRD_OPSS_QUEUEING: -+ /* -+ * QUEUEING is started and finished while holding @p's rq lock. -+ * As we're holding the rq lock now, we shouldn't see QUEUEING. -+ */ -+ hmbird_deferred_err(DEQ_DEQING, " : unreachable path in %s\n", __func__); -+ break; -+ case HMBIRD_OPSS_QUEUED: -+ if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss, -+ HMBIRD_OPSS_NONE)) -+ break; -+ fallthrough; -+ case HMBIRD_OPSS_DISPATCHING: -+ /* -+ * If @p is being dispatched from the BPF scheduler to a DSQ, -+ * wait for the transfer to complete so that @p doesn't get -+ * added to its DSQ after dequeueing is complete. -+ * -+ * As we're waiting on DISPATCHING with the rq locked, the -+ * dispatching side shouldn't try to lock the rq while -+ * DISPATCHING is set. See dispatch_to_local_dsq(). -+ * -+ * DISPATCHING shouldn't have qseq set and control can reach -+ * here with NONE @opss from the above QUEUED case block. -+ * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss. -+ */ -+ wait_ops_state(p, HMBIRD_OPSS_DISPATCHING); -+ hmbird_cond_deferred_err(HMBIRD_OPN, -+ atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE, -+ "task = %s\n", p->comm); -+ break; -+ } -+} -+ -+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ -+ if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p), -+ "task = %s\n", p->comm); -+ return; -+ } -+ -+ ops_dequeue(p, deq_flags); -+ -+ if (slim_walt_ctrl) { -+ if (task_current(rq, p)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ if (deq_flags & HMBIRD_DEQ_SLEEP) -+ set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else -+ clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), -+ (unsigned long *)&get_hmbird_ts(p)->flags); -+ -+ clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm); -+ hmbird_rq->nr_running--; -+ sub_nr_running(rq, 1); -+ dispatch_dequeue(hmbird_rq, p); -+} -+ -+static void yield_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ get_hmbird_ts(p)->slice = 0; -+} -+ -+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to) -+{ -+ return false; -+} -+ -+#ifdef CONFIG_SMP -+/** -+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -+ * @rq: rq to move the task into, currently locked -+ * @p: task to move -+ * @enq_flags: %HMBIRD_ENQ_* -+ * -+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller -+ * must: -+ * -+ * 1. Start with exclusive access to @p either through its DSQ lock or -+ * %HMBIRD_OPSS_DISPATCHING flag. -+ * -+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id(). -+ * -+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't -+ * deadlock with dequeue. -+ * -+ * 4. Lock @rq and the task_rq from #3. -+ * -+ * 5. Call this function. -+ * -+ * Returns %true if @p was successfully moved. %false after racing dequeue and -+ * losing. -+ */ -+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ struct rq *task_rq; -+ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * If dequeue got to @p while we were trying to lock both rq's, it'd -+ * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have -+ * updated it to different values afterwards, as this operation can't be -+ * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become -+ * raw_smp_processor_id() again before we're done. Thus, we can tell -+ * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is -+ * still raw_smp_processor_id(). -+ * -+ * See dispatch_dequeue() for the counterpart. -+ */ -+ if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id())) -+ return false; -+ -+ /* @p->rq couldn't have changed if we're still the holding cpu */ -+ task_rq = task_rq(p); -+ lockdep_assert_rq_held(task_rq); -+ deactivate_task(task_rq, p, 0); -+ set_task_cpu(p, cpu_of(rq)); -+ get_hmbird_ts(p)->sticky_cpu = cpu_of(rq); -+ -+ /* -+ * We want to pass hmbird-specific enq_flags but activate_task() will -+ * truncate the upper 32 bit. As we own @rq, we can pass them through -+ * @get_hmbird_rq(rq)->extra_enq_flags instead. -+ */ -+ hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags, -+ "task = %s\n", p->comm); -+ get_hmbird_rq(rq)->extra_enq_flags = enq_flags; -+ activate_task(rq, p, 0); -+ get_hmbird_rq(rq)->extra_enq_flags = 0; -+ -+ return true; -+} -+ -+#endif /* CONFIG_SMP */ -+ -+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu) -+{ -+ int fitable = 1; -+ -+ return fitable; -+} -+ -+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq, -+ struct hmbird_dispatch_q *dsq) -+{ -+ bool dsq_misfit; -+ int cpu = cpu_of(rq); -+ u64 task_util = 0; -+ struct cluster_ctx ctx; -+ int dsq_int = dsq_id_to_internal(dsq); -+ -+ if (!cpumask_test_cpu(cpu, iso_masks.little)) -+ return false; -+ -+ gen_cluster_ctx(&ctx, BIG); -+ dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 && -+ dsq_int <= SCHED_PROP_DEADLINE_LEVEL4); -+#ifdef CLUSTER_SEPARATE -+ /* In rescue mode, little will consume big cluster's dsq.*/ -+ dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper); -+#endif -+ if (!dsq_misfit) -+ return false; -+ -+ if (p) { -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &task_util); -+ else -+ task_util = get_hmbird_ts(p)->demand_scaled; -+ } -+ -+ if (task_util <= misfit_ds) -+ return false; -+ -+ hmbird_info_trace(":task %s can't run on cpu%d, util = %llu\n", -+ p->comm, cpu, task_util); -+ return true; -+} -+ -+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq) -+{ -+ if (!task_fits_cpu_hmbird(p, cpu_of(rq))) -+ return false; -+ -+ if (check_misfit_task_on_little(p, rq, dsq)) -+ return false; -+ -+ return likely(test_rq_online(rq)); -+} -+ -+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return; -+ -+ if (add) -+ skipn[idx]++; -+ else -+ skipn[idx] = 0; -+} -+ -+static bool skip_too_much(struct hmbird_dispatch_q *dsq) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return false; -+ -+ if (skip_num[idx] > 3) { -+ skip_num[idx] = 0; -+ return true; -+ } -+ -+ return false; -+} -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rb_node *rb_node; -+ struct rq *task_rq; -+ unsigned long flags; -+ bool moved = false; -+ struct task_struct *may_fit = NULL; -+ int skip = 0; -+ -+retry: -+ if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -+ return false; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) { -+ set_skip_num(dsq, skip_num, (bool)may_fit); -+ goto this_rq; -+ } -+ if (skip_too_much(dsq)) -+ goto remote_rq; -+ if (!may_fit) -+ may_fit = p; -+ if (++skip <= 3) -+ continue; -+ /* -+ * If the recent 3 tasks not fit, use the first one. -+ * and clear the skip, because the first one is consumed. -+ */ -+ set_skip_num(dsq, skip_num, false); -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ /* No more task, use the first may fit task.*/ -+ if (may_fit) { -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ -+ for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) -+ goto this_rq; -+ goto remote_rq; -+ } -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ return false; -+ -+this_rq: -+ /* @dsq is locked and @p is on this rq */ -+ hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo); -+ dsq->nr--; -+ hmbird_rq->local_dsq.nr++; -+ get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ -+remote_rq: -+#ifdef CONFIG_SMP -+ if (cpu_same_cluster_stat(p, rq, task_rq)) -+ slim_stats_record(MOVE_RQ_CNT, 0, 0, 0); -+ else -+ slim_stats_record(MOVE_RQ_CNT, 1, 0, 0); -+ /* -+ * @dsq is locked and @p is on a remote rq. @p is currently protected by -+ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -+ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -+ * rq lock or fail, do a little dancing from our side. See -+ * move_task_to_local_dsq(). -+ */ -+ hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id(); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ rq_unpin_lock(rq, rf); -+ double_lock_balance(rq, task_rq); -+ rq_repin_lock(rq, rf); -+ -+ moved = move_task_to_local_dsq(rq, p, 0); -+ -+ double_unlock_balance(rq, task_rq); -+#endif /* CONFIG_SMP */ -+ if (likely(moved)) { -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ } -+ may_fit = NULL; -+ goto retry; -+} -+ -+ -+static int balance_one(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf, bool local) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class; -+ -+ if (!hmbird_rq) -+ return 1; -+ -+ lockdep_assert_rq_held(rq); -+ -+ if (static_branch_unlikely(&hmbird_ops_cpu_preempt) && -+ unlikely(get_hmbird_rq(rq)->cpu_released)) { -+ /* -+ * If the previous sched_class for the current CPU was not HMBIRD, -+ * notify the BPF scheduler that it again has control of the -+ * core. This callback complements ->cpu_release(), which is -+ * emitted in hmbird_notify_pick_next_task(). -+ */ -+ get_hmbird_rq(rq)->cpu_released = false; -+ } -+ -+ if (prev_on_hmbird) -+ update_curr_hmbird(rq); -+ /* if there already are tasks to run, nothing to do */ -+ if (hmbird_rq->local_dsq.nr) -+ return 1; -+ -+ if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) { -+ slim_stats_record(GLOBAL_STAT, 1, 0, 0); -+ return 1; -+ } -+ -+ if (consume_dispatch_global(rq, rf)) -+ return 1; -+ -+ return 0; -+} -+ -+static int balance_hmbird(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf) -+{ -+ return balance_one(rq, prev, rf, true); -+} -+ -+/* -+ * output task util to systrace, only for debug mode. -+ * we can not output too many logs to systrace buffer even in debug mode -+ * only output debug-info while it exceed misfit_ds. -+ */ -+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p) -+{ -+ static DEFINE_PER_CPU(int, is_last_exceed); -+ int cpu = cpu_of(rq); -+ u64 util = 0; -+ -+ if (likely(!debug_enabled())) -+ return; -+ -+ if (!p) -+ return; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ util = uclamp_rq_util_with(rq, util, p); -+ -+ if (util >= misfit_ds) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util); -+ per_cpu(is_last_exceed, cpu) = true; -+ } else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0); -+ per_cpu(is_last_exceed, cpu) = false; -+ } else { -+ } -+} -+ -+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first) -+{ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ /* -+ * Core-sched might decide to execute @p before it is -+ * dispatched. Call ops_dequeue() to notify the BPF scheduler. -+ */ -+ ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC); -+ dispatch_dequeue(get_hmbird_rq(rq), p); -+ } -+ -+ p->se.exec_start = rq_clock_task(rq); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK); -+ } -+ -+ watchdog_unwatch_task(p, true); -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx); -+ systrace_output_cpu_ds(rq, p); -+ /* -+ * @p is getting newly scheduled or got kicked after someone updated its -+ * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird(). -+ */ -+ if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) != -+ (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) { -+ if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) -+ get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK; -+ else -+ get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK; -+ -+ sched_update_tick_dependency(rq); -+ } -+ -+ p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime; -+} -+ -+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p) -+{ -+ update_curr_hmbird(rq); -+ -+ update_dispatch_dsq_info(rq, p); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ watchdog_watch_task(rq, p); -+ -+ if (is_pipeline_task(p)) { -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1); -+ return; -+ } -+ -+ /* -+ * If we're in the pick_next_task path, balance_hmbird() should -+ * have already populated the local DSQ if there are any other -+ * available tasks. If empty, tell ops.enqueue() that @p is the -+ * only one available for this cpu. ops.enqueue() should put it -+ * on the local DSQ so that the subsequent pick_next_task_hmbird() -+ * can find the task unless it wants to trigger a separate -+ * follow-up scheduling event. -+ */ -+ if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1); -+ else -+ do_enqueue_task(rq, p, 0, -1); -+ } -+} -+ -+static struct task_struct *first_local_task(struct rq *rq) -+{ -+ struct rb_node *rb_node; -+ struct hmbird_entity *entity; -+ -+ if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) { -+ entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo, -+ struct hmbird_entity, dsq_node.fifo); -+ return entity->task; -+ } -+ -+ rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq); -+ if (rb_node) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ return entity->task; -+ } -+ return NULL; -+} -+ -+static struct task_struct *pick_next_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p; -+ -+ p = first_local_task(rq); -+ if (!p) -+ return NULL; -+ -+ if (unlikely(!get_hmbird_ts(p)->slice)) { -+ if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice) -+ hmbird_warned_zero_slice = true; -+ -+ refill_task_slice(p); -+ } -+ -+ set_next_task_hmbird(rq, p, true); -+ -+ return p; -+} -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task, -+ const struct sched_class *active) -+{ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * The callback is conceptually meant to convey that the CPU is no -+ * longer under the control of HMBIRD. Therefore, don't invoke the -+ * callback if the CPU is staying on HMBIRD, or going idle (in which -+ * case the HMBIRD scheduler has actively decided not to schedule any -+ * tasks on the CPU). -+ */ -+ if (likely(active >= &hmbird_sched_class)) -+ return; -+ -+ /* -+ * At this point we know that HMBIRD was preempted by a higher priority -+ * sched_class, so invoke the ->cpu_release() callback if we have not -+ * done so already. We only send the callback once between HMBIRD being -+ * preempted, and it regaining control of the CPU. -+ * -+ * ->cpu_release() complements ->cpu_acquire(), which is emitted the -+ * next time that balance_hmbird() is invoked. -+ */ -+ if (!get_hmbird_rq(rq)->cpu_released) -+ get_hmbird_rq(rq)->cpu_released = true; -+} -+ -+#ifdef CONFIG_SMP -+ -+static bool test_and_clear_cpu_idle(int cpu) -+{ -+ if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -+ if (cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ return true; -+ } else { -+ return false; -+ } -+} -+ -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) -+{ -+ int cpu; -+ -+ do { -+ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -+ if (cpu >= nr_cpu_ids) -+ return -EBUSY; -+ } while (!test_and_clear_cpu_idle(cpu)); -+ -+ return cpu; -+} -+ -+ -+static bool prev_cpu_misfit(int prev) -+{ -+ if (!is_partial_enabled() && is_partial_cpu(prev)) -+ return true; -+ -+ return false; -+} -+ -+static int heavy_rt_placement(struct task_struct *p, int prev) -+{ -+ struct cpumask tmp = {.bits = {0}}; -+ int cpu; -+ u64 util = 0; -+ -+ if (!rt_prio(p->prio)) -+ return -EFAULT; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ -+ if (util < misfit_ds) -+ return -EFAULT; -+ -+ if (is_partial_enabled()) -+ cpumask_or(&tmp, iso_masks.big, iso_masks.partial); -+ else -+ cpumask_copy(&tmp, iso_masks.big); -+ -+ cpu = hmbird_pick_idle_cpu(&tmp); -+ if (cpu >= 0) -+ return cpu; -+ -+ if (cpumask_test_cpu(prev, iso_masks.big) || -+ (is_partial_enabled() && is_partial_cpu(prev))) -+ return prev; -+ -+ return cpumask_first(&tmp); -+} -+ -+static int spec_task_before_pick_idle(struct task_struct *p, int prev) -+{ -+ int cpu; -+ -+ cpu = heavy_rt_placement(p, prev); -+ if (cpu >= 0) -+ return cpu; -+ return -EFAULT; -+} -+ -+static int cpumask_distribute_next(struct cpumask *mask, int *prev) -+{ -+ int p = *prev, n; -+ -+ n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1); -+ if (n < nr_cpu_ids) -+ WRITE_ONCE(*prev, n); -+ return n; -+} -+ -+static int repick_fallback_cpu(void) -+{ -+ /* -+ * partial cpu follow big cluster's Scheduling policy, -+ * simply return first bit cpu. -+ */ -+ return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev); -+} -+ -+/* -+ * Must return a valid cpu num, as this task's cpu. -+ */ -+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu, -+ struct cpumask *mask, int *prev_mask) -+{ -+ int cpu; -+ -+ cpu = hmbird_pick_idle_cpu(mask); -+ if (cpu >= 0) -+ return cpu; -+ return cpumask_distribute_next(mask, prev_mask); -+} -+ -+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type) -+{ -+ int idx; -+ struct cluster_ctx ctx; -+ -+ idx = find_idx_from_task(p); -+ if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS) -+ return false; -+ -+ gen_cluster_ctx(&ctx, type); -+ if (idx >= ctx.lower && idx < ctx.upper) -+ return true; -+ return false; -+} -+ -+static bool is_valid_cpu(int cpu) -+{ -+ return (cpu >= 0) && (cpu < nr_cpu_ids); -+} -+ -+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) -+{ -+ s32 cpu = -1; -+ struct cpumask mask = {.bits = {0}}; -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (is_valid_cpu(cpu)) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ partial_dynamic_ctrl(); -+ -+ if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ iso_masks.big, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ if (p->nr_cpus_allowed == 1) { -+ cpu = cpumask_any(p->cpus_ptr); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* For non-period global dsq, not contain pcp task. */ -+ if (task_only_blongs_to_cluster(p, LITTLE)) { -+ cpumask_copy(&mask, iso_masks.little); -+ if (unlikely(l_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &little_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ if (task_only_blongs_to_cluster(p, BIG)) { -+ cpumask_copy(&mask, iso_masks.big); -+ if (unlikely(b_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ cpu = spec_task_before_pick_idle(p, prev_cpu); -+ if (is_valid_cpu(cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* if the previous CPU is idle, dispatch directly to it */ -+ if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+ } -+ -+ cpu = hmbird_pick_idle_cpu(cpu_possible_mask); -+ if (is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ if (prev_cpu_misfit(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ cpu = repick_fallback_cpu(); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+} -+ -+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags) -+{ -+ return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags); -+} -+ -+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx) -+{ -+ set_cpus_allowed_common(p, ctx); -+} -+ -+static void reset_idle_masks(void) -+{ -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little); -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big); -+ if (is_partial_enabled()) -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial); -+ hmbird_has_idle_cpus = true; -+} -+ -+void __hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ int cpu = cpu_of(rq); -+ struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -+ -+ if (skip_update_idle()) -+ return; -+ -+ if (idle) { -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ if (!hmbird_has_idle_cpus) -+ hmbird_has_idle_cpus = true; -+ -+ /* -+ * idle_masks.smt handling is racy but that's fine as it's only -+ * for optimization and self-correcting. -+ */ -+ for_each_cpu(cpu, sib_mask) { -+ if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -+ return; -+ } -+ cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -+ } else { -+ cpumask_clear_cpu(cpu, idle_masks.cpu); -+ if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ -+ cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -+ } -+} -+ -+#else /* !CONFIG_SMP */ -+ -+static bool test_and_clear_cpu_idle(int cpu) { return false; } -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } -+static void reset_idle_masks(void) {} -+ -+#endif /* CONFIG_SMP */ -+ -+static bool check_rq_for_timeouts(struct rq *rq) -+{ -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rq_flags rf; -+ -+ rq_lock_irqsave(rq, &rf); -+ list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) { -+ unsigned long last_runnable; -+ -+ p = entity->task; -+ last_runnable = get_hmbird_ts(p)->runnable_at; -+ -+ if (unlikely(time_after(jiffies, -+ last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -+ -+ rq_unlock_irqrestore(rq, &rf); -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb", -+ p->comm, p->pid, -+ dur_ms / 1000, dur_ms % 1000, -+ get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0, -+ cpumask_pr_args(&p->cpus_mask)); -+ return 1; -+ } -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ return 0; -+} -+ -+static void hmbird_watchdog_workfn(struct work_struct *work) -+{ -+ int cpu; -+ bool timeout; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ -+ for_each_online_cpu(cpu) { -+ timeout = check_rq_for_timeouts(cpu_rq(cpu)); -+ if (unlikely(timeout)) -+ break; -+ -+ cond_resched(); -+ } -+ if (!timeout) -+ queue_delayed_work(system_unbound_wq, to_delayed_work(work), -+ hmbird_watchdog_timeout / 2); -+} -+ -+static void set_pcp_round(struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) { -+ per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round); -+ per_cpu(pcp_info, cpu).pcp_round = true; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+} -+ -+ -+/* -+ * Just for debug: output hmbird on/off state per 10s. -+ */ -+#define OUTPUT_INTVAL (msecs_to_jiffies(10 * 1000)) -+static void inform_hmbird_onoff_from_systrace(void) -+{ -+ static unsigned long __read_mostly next_print; -+ -+ if (time_before(jiffies, READ_ONCE(next_print))) -+ return; -+ -+ WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL); -+ hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio); -+} -+ -+void hmbird_notify_sched_tick(void) -+{ -+ unsigned long last_check; -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ hmbird_scheduler_tick(); -+ -+ if (!hmbird_enabled()) -+ return; -+ -+ last_check = hmbird_watchdog_timestamp; -+ if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -+ -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "watchdog failed to check in for %u.%03us", -+ dur_ms / 1000, dur_ms % 1000); -+ } -+ scan_timeout(rq); -+} -+ -+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued) -+{ -+ update_curr_hmbird(rq); -+ -+ set_pcp_round(rq); -+ -+ if (slim_walt_ctrl) -+ hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE); -+ /* -+ * While disabling, always resched and refresh core-sched timestamp as -+ * we can't trust the slice management or ops.core_sched_before(). -+ */ -+ if (hmbird_ops_disabling()) -+ get_hmbird_ts(curr)->slice = 0; -+ -+ if (!get_hmbird_ts(curr)->slice) -+ resched_curr(rq); -+ -+ inform_hmbird_onoff_from_systrace(); -+} -+ -+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg) -+{ -+ hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ get_hmbird_ts(p)->disallow = false; -+ -+ hmbird_sched_init_task(p); -+ -+ set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ return 0; -+} -+ -+static void hmbird_ops_enable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void hmbird_ops_disable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void set_task_hmbird_weight(struct task_struct *p) -+{ -+ u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -+ -+ get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight); -+} -+ -+/** -+ * refresh_hmbird_weight - Refresh a task's hmbird weight -+ * @p: task to refresh hmbird weight for -+ * -+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to -+ * enable easy access from the BPF scheduler. To keep it synchronized with the -+ * current task priority, this function should be called when a new task is -+ * created, priority is changed for a task on hmbird, and a task is switched -+ * to hmbird from other classes. -+ */ -+static void refresh_hmbird_weight(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ set_task_hmbird_weight(p); -+} -+ -+int hmbird_pre_fork(struct task_struct *p) -+{ -+ int ret = 0; -+ -+ p->android_oem_data1[HMBIRD_TS_IDX] = -+ (u64)(kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL)); -+ if (!get_hmbird_ts(p)) { -+ ret = -1; -+ goto lock; -+ } -+ -+ get_hmbird_ts(p)->dsq = NULL; -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node); -+ get_hmbird_ts(p)->flags = 0; -+ get_hmbird_ts(p)->weight = 0; -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ get_hmbird_ts(p)->holding_cpu = -1; -+ get_hmbird_ts(p)->kf_mask = 0; -+ atomic64_set(&get_hmbird_ts(p)->ops_state, 0); -+ get_hmbird_ts(p)->runnable_at = INITIAL_JIFFIES; -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+ get_hmbird_ts(p)->task = p; -+ hmbird_set_sched_prop(p, 0); -+ -+ get_hmbird_ts(p)->critical_affinity_cpu = -1; -+ get_hmbird_ts(p)->sched_class = &hmbird_sched_class; -+ -+ /* -+ * BPF scheduler enable/disable paths want to be able to iterate and -+ * update all tasks which can become complex when racing forks. As -+ * enable/disable are very cold paths, let's use a percpu_rwsem to -+ * exclude forks. -+ */ -+lock: -+ percpu_down_read(&hmbird_fork_rwsem); -+ -+ return ret; -+} -+ -+int hmbird_fork(struct task_struct *p) -+{ -+ percpu_rwsem_assert_held(&hmbird_fork_rwsem); -+ -+ if (hmbird_enabled()) -+ return hmbird_ops_prepare_task(p, task_group(p)); -+ else -+ return 0; -+} -+ -+void hmbird_post_fork(struct task_struct *p) -+{ -+ if (hmbird_enabled()) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ /* -+ * Set the weight manually before calling ops.enable() so that -+ * the scheduler doesn't see a stale value if they inspect the -+ * task struct. We'll invoke ops.set_weight() afterwards, as it -+ * would be odd to receive a callback on the task before we -+ * tell the scheduler that it's been fully enabled. -+ */ -+ set_task_hmbird_weight(p); -+ hmbird_ops_enable_task(p); -+ refresh_hmbird_weight(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ -+ spin_lock_irq(&hmbird_tasks_lock); -+ list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_cancel_fork(struct task_struct *p) -+{ -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ if (hmbird_enabled()) -+ hmbird_ops_disable_task(p); -+ -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_free(struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+ list_del_init(&get_hmbird_ts(p)->tasks_node); -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+ -+ /* -+ * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED -> -+ * ENABLED transitions can't race us. Disable ops for @p. -+ */ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) || -+ test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ hmbird_ops_disable_task(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+} -+ -+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio) -+{ -+} -+ -+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type) -+{ -+ return (prop >> TOP_TASK_SHIFT) & (1 << type); -+} -+ -+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p) -+{ -+ uint32_t prop = get_top_task_prop(p); -+ -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE)) -+ return HMBIRD_TASK_PROP_PIPELINE; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) || -+ !task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) { -+ return HMBIRD_TASK_PROP_COMMON; -+ } -+ return HMBIRD_TASK_PROP_DEBUG_OR_LOG; -+} -+ -+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b) -+{ -+ int type_a = hmbird_get_task_type(a); -+ int type_b = hmbird_get_task_type(b); -+ -+ return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b]; -+} -+ -+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ enum cpu_type type; -+ int sp_dl; -+ struct task_struct *curr = NULL; -+ -+ switch (hmbird_preempt_policy) { -+ case HMBIRD_PREEMPT_POLICY_PRIO_BASED: -+ curr = rq->curr; -+ if (curr && hmbird_prio_higher(p, curr)) -+ goto preempt; -+ break; -+ default: -+ break; -+ } -+ if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) || -+ (is_critical_system_task(p) && !is_critical_system_task(rq->curr))) -+ goto preempt; -+ -+ sp_dl = find_idx_from_task(p); -+ if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled())) -+ return; -+ -+ if (rq->curr->prio > p->prio) -+ goto preempt; -+ -+ return; -+ -+preempt: -+ resched_curr(rq); -+} -+ -+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {} -+ -+int hmbird_check_setscheduler(struct task_struct *p, int policy) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ /* if disallow, reject transitioning into HMBIRD */ -+ if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) && -+ p->policy != policy && policy == SCHED_HMBIRD) -+ return -EACCES; -+ -+ return 0; -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool hmbird_can_stop_tick(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ if (hmbird_ops_disabling()) -+ return false; -+ -+ if (p->sched_class != &hmbird_sched_class) -+ return true; -+ -+ return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK; -+} -+#endif -+ -+int hmbird_tg_online(struct task_group *tg) -+{ -+ struct cgroup *cgrp; -+ -+ if (!tg) -+ return 0; -+ cgrp = tg->css.cgroup; -+ if (!cgrp || !(cgrp->kn)) -+ return 0; -+ update_cgroup_ids_table(cgrp->kn->id, -1); -+ return 0; -+} -+ -+/* -+ * Omitted operations: -+ * -+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -+ * task isn't tied to the CPU at that point. Preemption is implemented by -+ * resetting the victim task's slice to 0 and triggering reschedule on the -+ * target CPU. -+ * -+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -+ * -+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -+ * their current sched_class. Call them directly from sched core instead. -+ * -+ * - task_woken, switched_from: Unnecessary. -+ */ -+DEFINE_SCHED_CLASS(hmbird) = { -+ .enqueue_task = enqueue_task_hmbird, -+ .dequeue_task = dequeue_task_hmbird, -+ .yield_task = yield_task_hmbird, -+ .yield_to_task = yield_to_task_hmbird, -+ -+ .check_preempt_curr = check_preempt_curr_hmbird, -+ -+ .pick_next_task = pick_next_task_hmbird, -+ -+ .put_prev_task = put_prev_task_hmbird, -+ .set_next_task = set_next_task_hmbird, -+ -+#ifdef CONFIG_SMP -+ .balance = balance_hmbird, -+ .select_task_rq = select_task_rq_hmbird, -+ .set_cpus_allowed = set_cpus_allowed_hmbird, -+#endif -+ .task_tick = task_tick_hmbird, -+ -+ .switched_to = switched_to_hmbird, -+ .prio_changed = prio_changed_hmbird, -+ -+ .update_curr = update_curr_hmbird, -+ -+#ifdef CONFIG_UCLAMP_TASK -+ .uclamp_enabled = 0, -+#endif -+}; -+ -+/* -+ * Must with rq lock held. -+ */ -+bool task_is_hmbird(struct task_struct *p) -+{ -+ return p->sched_class == &hmbird_sched_class; -+} -+ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id) -+{ -+ memset(dsq, 0, sizeof(*dsq)); -+ -+ raw_spin_lock_init(&dsq->lock); -+ INIT_LIST_HEAD(&dsq->fifo); -+ dsq->id = dsq_id; -+} -+ -+static int hmbird_cgroup_init(void) -+{ -+ struct cgroup_subsys_state *css; -+ -+ css_for_each_descendant_pre(css, &root_task_group.css) { -+ struct task_group *tg = css_tg(css); -+ -+ cgrp_dsq_idx_init(css->cgroup, tg); -+ } -+ return 0; -+} -+ -+/* -+ * Used by sched_fork() and __setscheduler_prio() to pick the matching -+ * sched_class. dl/rt are already handled. -+ */ -+bool task_on_hmbird(struct task_struct *p) -+{ -+ return hmbird_enabled(); -+} -+ -+static void __setscheduler_prio(struct task_struct *p, int prio) -+{ -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) { -+ p->prio = prio; -+ return; -+ } else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (on_hmbird && rt_prio(prio)) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ -+ p->prio = prio; -+} -+ -+/* -+ * Heartbeat, avoid humbird keep running while APP already exit. -+ * Check whether APP send alive-signal periodly. -+ */ -+#define HEARTBEAT_TIMEOUT (msecs_to_jiffies(2500)) -+#define HEARTBEAT_CHECK_INTERVAL (msecs_to_jiffies(1000)) -+static struct timer_list hb_timer; -+static unsigned long next_hb; -+ -+void hb_timer_handler(struct timer_list *timer) -+{ -+ if (!heartbeat_enable) -+ goto refill; -+ -+ pr_err(": enter timer.\n"); -+ if (heartbeat) { -+ heartbeat = 0; -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ } -+ -+ /* can't detect heartbeat, disable ext. */ -+ if (time_after(jiffies, READ_ONCE(next_hb))) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT, -+ "can't detect heartbeat, disable ext\n"); -+ } -+refill: -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_start(void) -+{ -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_init(void) -+{ -+ timer_setup(&hb_timer, hb_timer_handler, 0); -+} -+ -+static void hb_timer_exit(void) -+{ -+ del_timer(&hb_timer); -+} -+ -+static bool check_and_disable_cpuhp(void) -+{ -+ struct rq *rq; -+ struct rq_flags rf; -+ int cpu; -+ -+ cpu_hotplug_disable(); -+ cpus_read_lock(); -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ rq_lock_irqsave(rq, &rf); -+ if (!rq->online) { -+ rq_unlock_irqrestore(rq, &rf); -+ goto err_offline; -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ cpus_read_unlock(); -+ return true; -+ -+err_offline: -+ cpus_read_unlock(); -+ cpu_hotplug_enable(); -+ return false; -+} -+ -+static void reenable_cpuhp(void) -+{ -+ cpu_hotplug_enable(); -+} -+ -+static void scheduler_switch_done(bool final_state) -+{ -+ /* set scx_enable again, switch may fail. */ -+ scx_enable = final_state; -+ /* reset heartbeat*/ -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ -+ if (final_state) -+ hb_timer_start(); -+ else -+ hb_timer_exit(); -+} -+ -+/** -+ * ss : curr switch state -+ * finish : success or fail -+ * enable : curr operation is diabling or enabling? -+ * fail_reason : string output when failed, empty string when success. -+ */ -+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason) -+{ -+ char *s1 = finish ? "finished" : "failed"; -+ char *s2 = enable ? "enabled" : "disabled"; -+ -+ hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss); -+ if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) { -+ sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type)); -+ hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n", -+ s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason); -+ } -+ curr_ss = ss; -+} -+ -+bool get_hmbird_ops_enabled(void) -+{ -+ return atomic_read(&__hmbird_ops_enabled); -+} -+ -+bool get_non_hmbird_task(void) -+{ -+ return atomic_read(&non_hmbird_task); -+} -+ -+static void hmbird_ops_disable_workfn(struct kthread_work *work) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int cpu; -+ -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, ""); -+ cancel_delayed_work_sync(&hmbird_watchdog_work); -+ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) { -+ case HMBIRD_OPS_DISABLED: -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled"); -+ scheduler_switch_done(false); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return; -+ case HMBIRD_OPS_PREPPING: -+ fallthrough; -+ case HMBIRD_OPS_DISABLING: -+ /* shouldn't happen but handle it like ENABLING if it does */ -+ WARN_ONCE(true, "hmbird: duplicate disabling instance?"); -+ fallthrough; -+ case HMBIRD_OPS_ENABLING: -+ case HMBIRD_OPS_ENABLED: -+ break; -+ } -+ -+ /* kick all CPUs to restore ticks */ -+ for_each_possible_cpu(cpu) -+ resched_cpu(cpu); -+ -+ /* avoid racing against fork and cgroup changes */ -+ cpus_read_lock(); -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, ""); -+ spin_lock_irq(&hmbird_tasks_lock); -+ atomic_set(&__hmbird_ops_enabled, false); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ bool alive = READ_ONCE(p->__state) != TASK_DEAD; -+ -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ get_hmbird_ts(p)->slice = min_t(u64, -+ get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL); -+ -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ if (alive) -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ -+ hmbird_ops_disable_task(p); -+ } -+ hmbird_task_iter_exit(&sti); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, ""); -+ -+ atomic_set(&non_hmbird_task, true); -+ /* no task is on hmbird, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt); -+ synchronize_rcu(); -+ -+ percpu_up_write(&hmbird_fork_rwsem); -+ cpus_read_unlock(); -+ -+ if (slim_walt_ctrl) -+ slim_walt_enable(false); -+ -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 1, 0, ""); -+ scheduler_switch_done(false); -+ reenable_cpuhp(); -+} -+ -+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn); -+ -+static void schedule_hmbird_ops_disable_work(void) -+{ -+ struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper); -+ -+ /* -+ * We may be called spuriously before the first bpf_hmbird_reg(). If -+ * hmbird_ops_helper isn't set up yet, there's nothing to do. -+ */ -+ if (helper) -+ kthread_queue_work(helper, &hmbird_ops_disable_work); -+} -+ -+static void hmbird_ops_disable(void) -+{ -+ schedule_hmbird_ops_disable_work(); -+} -+ -+static void hmbird_err_exit_workfn(struct work_struct *work) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ hmbird_ctrl(false); -+ -+ for_each_present_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy) -+ continue; -+ if (cpu != policy->cpu) -+ goto put; -+ down_write(&policy->rwsem); -+ WARN_ON(store_scaling_governor(policy, -+ saved_gov[cpu], strlen(saved_gov[cpu])) <= 0); -+ up_write(&policy->rwsem); -+ hmbird_info_trace(":restore origin gov : %s\n", saved_gov[cpu]); -+put: -+ cpufreq_cpu_put(policy); -+ } -+ memset((char *)saved_gov, 0, sizeof(saved_gov)); -+} -+ -+void hmbird_ops_exit(void) -+{ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); -+} -+ -+static struct kthread_worker *hmbird_create_rt_helper(const char *name) -+{ -+ struct kthread_worker *helper; -+ -+ helper = kthread_create_worker(0, name); -+ if (helper) -+ sched_set_fifo(helper->task); -+ return helper; -+} -+ -+static inline void set_audio_thread_sched_prop(struct task_struct *p) -+{ -+ struct cgroup_subsys_state *css; -+ -+ if (likely(p->prio >= MAX_RT_PRIO)) -+ return; -+ -+ rcu_read_lock(); -+ css = task_css(p, cpuset_cgrp_id); -+ if (!css) { -+ rcu_read_unlock(); -+ return; -+ } -+ rcu_read_unlock(); -+ -+ if (!strcmp(css->cgroup->kn->name, "audio-app")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+} -+ -+static int hmbird_ops_enable(void *unused) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int ret; -+ int tcnt = 0; -+ unsigned long long start = 0; -+ -+ if (!check_and_disable_cpuhp()) { -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline"); -+ return -EBUSY; -+ } -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, ""); -+ mutex_lock(&hmbird_ops_enable_mutex); -+ -+ if (!hmbird_ops_helper) { -+ WRITE_ONCE(hmbird_ops_helper, -+ hmbird_create_rt_helper("hmbird_ops_helper")); -+ if (!hmbird_ops_helper) { -+ ret = -ENOMEM; -+ goto err_unlock; -+ } -+ } -+ -+ if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) { -+ ret = -EBUSY; -+ goto err_unlock; -+ } -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) != -+ HMBIRD_OPS_DISABLED); -+ -+ hmbird_warned_zero_slice = false; -+ -+ atomic64_set(&hmbird_nr_rejected, 0); -+ -+ /* -+ * Keep CPUs stable during enable so that the BPF scheduler can track -+ * online CPUs by watching ->on/offline_cpu() after ->init(). -+ */ -+ cpus_read_lock(); -+ -+ hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work, -+ hmbird_watchdog_timeout / 2); -+ -+ /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ */ -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ reset_idle_masks(); -+ -+ /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. -+ */ -+ ret = hmbird_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock; -+ -+ /* -+ * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem -+ * preventing new tasks from being added. No need to exclude tasks -+ * leaving as hmbird_free() can handle both prepped and enabled -+ * tasks. Prep all tasks first and then enable them with preemption -+ * disabled. -+ */ -+ spin_lock_irq(&hmbird_tasks_lock); -+ -+ atomic_set(&non_hmbird_task, false); -+ atomic_set(&__hmbird_ops_enabled, true); -+ -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered(&sti))) -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ hmbird_task_iter_exit(&sti); -+ -+ /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. -+ */ -+ preempt_disable(); -+ -+ /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ */ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) { -+ atomic_set(&non_hmbird_task, true); -+ atomic_set(&__hmbird_ops_enabled, false); -+ preempt_enable(); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ ret = -EBUSY; -+ goto err_disable_unlock; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED -+ * transitions here are synchronized against hmbird_free() through -+ * hmbird_tasks_lock. -+ */ -+ start = sched_clock(); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, ""); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ tcnt++; -+ if (READ_ONCE(p->__state) != TASK_DEAD) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ -+ set_audio_thread_sched_prop(p); -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ hmbird_ops_enable_task(p); -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ } else { -+ hmbird_ops_disable_task(p); -+ } -+ } -+ hmbird_task_iter_exit(&sti); -+ -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, ""); -+ preempt_enable(); -+ percpu_up_write(&hmbird_fork_rwsem); -+ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) { -+ ret = -EBUSY; -+ goto err_disable; -+ } -+ -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_ENABLED, 1, 1, ""); -+ scheduler_switch_done(true); -+ -+ return 0; -+ -+err_unlock: -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock"); -+ scheduler_switch_done(false); -+ return ret; -+ -+err_disable_unlock: -+ percpu_up_write(&hmbird_fork_rwsem); -+err_disable: -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ /* must be fully disabled before returning */ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable"); -+ scheduler_switch_done(false); -+ return ret; -+} -+ -+#ifdef CONFIG_SCHED_DEBUG -+static const char *hmbird_ops_enable_state_str[] = { -+ [HMBIRD_OPS_PREPPING] = "prepping", -+ [HMBIRD_OPS_ENABLING] = "enabling", -+ [HMBIRD_OPS_ENABLED] = "enabled", -+ [HMBIRD_OPS_DISABLING] = "disabling", -+ [HMBIRD_OPS_DISABLED] = "disabled", -+}; -+ -+static int hmbird_debug_show(struct seq_file *m, void *v) -+{ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled()); -+ seq_printf(m, "%-30s: %s\n", "enable_state", -+ hmbird_ops_enable_state_str[hmbird_ops_enable_state()]); -+ seq_printf(m, "%-30s: %llu\n", "nr_rejected", -+ atomic64_read(&hmbird_nr_rejected)); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return 0; -+} -+ -+static int hmbird_debug_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_debug_show, NULL); -+} -+ -+const struct file_operations sched_hmbird_fops = { -+ .open = hmbird_debug_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif -+ -+ -+static int bpf_hmbird_reg(void *kdata) -+{ -+ return hmbird_ops_enable(kdata); -+} -+ -+static int bpf_hmbird_unreg(void *kdata) -+{ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ return 0; -+} -+ -+void set_hmbird_module_loaded(int is_loaded) -+{ -+ atomic_set(&hmbird_module_loaded, is_loaded); -+} -+ -+/* -+ * MUST load hmbird module before enable hmbird scheduler -+ * load track & hmbird gover implemented in hmbird module -+ */ -+int hmbird_ctrl(bool enable) -+{ -+ if (!atomic_read(&hmbird_module_loaded)) { -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "ext module unloaded\n"); -+ return -EINVAL; -+ } -+ -+ if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED -+ || hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING -+ || hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already enabled(ing)\n"); -+ hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING || -+ hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already disabled(ing)\n"); -+ hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (enable) -+ return bpf_hmbird_reg(NULL); -+ else -+ return bpf_hmbird_unreg(NULL); -+} -+ -+void set_cpu_isomask(int cpu, cpumask_var_t *mask) -+{ -+ cpumask_clear_cpu(cpu, iso_masks.ex_free); -+ cpumask_clear_cpu(cpu, iso_masks.exclusive); -+ cpumask_clear_cpu(cpu, iso_masks.partial); -+ cpumask_clear_cpu(cpu, iso_masks.big); -+ cpumask_clear_cpu(cpu, iso_masks.little); -+ cpumask_set_cpu(cpu, *mask); -+} -+ -+void set_cpu_cluster(u64 cpu_cluster) -+{ -+ int cpu; -+ -+ for_each_present_cpu(cpu) { -+ u64 pos = 1 << cpu; -+ -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.ex_free)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.exclusive)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.partial)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.big)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) -+ set_cpu_isomask(cpu, &(iso_masks.little)); -+ } -+} -+ -+static void get_hmbird_snapshot(struct panic_snapshot_t *p) -+{ -+ struct hmbird_dispatch_q *dsq; -+ struct rq *rq; -+ int cpu, i; -+ -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ p->rq_nr[cpu] = rq->nr_running; -+ p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running; -+ } -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ struct hmbird_entity *entity; -+ -+ dsq = &gdsqs[i]; -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo)) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ p->runnable_at[i] = entity->runnable_at; -+ raw_spin_unlock(&dsq->lock); -+ -+ } -+ -+ p->snap_misc.hmbird_enabled = hmbird_enabled(); -+ p->snap_misc.curr_ss = curr_ss; -+ p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state(); -+ p->snap_misc.parctrl_high_ratio = parctrl_high_ratio; -+ p->snap_misc.parctrl_low_ratio = parctrl_low_ratio; -+ p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l; -+ p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l; -+ p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio; -+ p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio; -+ p->snap_misc.misfit_ds = misfit_ds; -+ p->snap_misc.partial_enable = partial_enable; -+ p->snap_misc.iso_free_rescue = iso_free_rescue; -+ p->snap_misc.isolate_ctrl = isolate_ctrl; -+ p->snap_misc.snap_jiffies = jiffies; -+ p->snap_misc.snap_time = local_clock(); -+} -+ -+// MTK minidump begin -+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3) -+{ -+ strscpy(m->desc_str, str, DESC_STR_LEN); -+ m->len = d1 * d2 * d3; -+ m->parse[0] = d1; -+ m->parse[1] = d2; -+ m->parse[2] = d3; -+} -+ -+static void init_desc_metas(struct md_info_t *m) -+{ -+ init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :", -+ 1, MAX_SWITCHS, SWITCH_ITEMS); -+ -+ init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1); -+ -+ init_desc_meta(&m->kern_dump.excep_rec_meta, -+ "excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS); -+ -+ init_desc_meta(&m->kern_dump.excep_idx_meta, -+ "excep idx :", 1, 1, MAX_EXCEP_ID); -+ -+ init_desc_meta(&m->kern_dump.snap.runnable_at_meta, -+ "each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS); -+ -+ init_desc_meta(&m->kern_dump.snap.rq_nr_meta, -+ "rq runnable task nr:", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta, -+ "scxrq runnable task nr :", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.snap_misc_meta, -+ "misc snap params :", 1, 1, SNAP_ITEMS); -+} -+ -+static void init_md_meta(struct md_info_t *m) -+{ -+ m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64); -+ m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64); -+ m->meta.unit_size = sizeof(u64); -+ m->meta.switches = MAX_SWITCHS; -+ m->meta.exceps = MAX_EXCEPS; -+ m->meta.global_dsqs = MAX_GLOBAL_DSQS; -+ m->meta.parse_dimens = PARSE_DIMENS; -+ m->meta.nr_cpus = num_possible_cpus(); -+ m->meta.real_cpus = nr_cpu_ids; -+ m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64); -+ m->meta.nr_meta_desc = 8; -+ m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64); -+ -+ init_desc_metas(m); -+} -+ -+#define MINIDUMP_DFL_SIZE (4 * 1024) -+ -+struct notifier_block hmbird_panic_blk; -+static int hmbird_panic_handler(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ if (!md_info) -+ return NOTIFY_DONE; -+ -+ get_hmbird_snapshot(&md_info->kern_dump.snap); -+ -+ return NOTIFY_DONE; -+} -+ -+static void panic_blk_init(void) -+{ -+ int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE); -+ -+ md_info = kzalloc(dump_size, GFP_KERNEL); -+ if (!md_info) -+ return; -+ init_md_meta(md_info); -+ -+ hmbird_panic_blk.notifier_call = hmbird_panic_handler; -+ /* make sure to execute before minidump. */ -+ hmbird_panic_blk.priority = INT_MAX; -+ atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk); -+ -+ hmbird_debug("register minidump.\n"); -+} -+ -+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size) -+{ -+ *vaddr = (unsigned long)md_info; -+ *size = sizeof(struct md_info_t); -+} -+// MTK minidump end -+ -+static inline void init_sched_prop_to_preempt_prio(void) -+{ -+ for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) { -+ switch (i) { -+ case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 5; -+ break; -+ -+ case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 4; -+ break; -+ -+ case HMBIRD_TASK_PROP_PIPELINE: -+ case HMBIRD_TASK_PROP_ISOLATE: -+ sched_prop_to_preempt_prio[i] = 3; -+ break; -+ -+ case HMBIRD_TASK_PROP_COMMON: -+ sched_prop_to_preempt_prio[i] = 1; -+ break; -+ -+ case HMBIRD_TASK_PROP_DEBUG_OR_LOG: -+ sched_prop_to_preempt_prio[i] = 0; -+ break; -+ -+ default: -+ sched_prop_to_preempt_prio[i] = 2; -+ break; -+ } -+ } -+} -+ -+void __init init_sched_hmbird_class(void) -+{ -+ int cpu; -+ u32 v; -+ struct hmbird_entity *init_hmbird; -+ -+ /* -+ * The following is to prevent the compiler from optimizing out the enum -+ * definitions so that BPF scheduler implementations can use them -+ * through the generated vmlinux.h. -+ */ -+ WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP | -+ HMBIRD_KICK_PREEMPT); -+ -+ init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL); -+ init_dsq_at_boot(); -+ init_isolate_cpus(); -+ hb_timer_init(); -+ init_sched_prop_to_preempt_prio(); -+#ifdef CONFIG_SMP -+ WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); -+#endif -+ -+ /* -+ * we can't static init init_task's hmbird struct, init here. -+ * init_task->hmbird would not use during boot. -+ */ -+ init_hmbird = kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL); -+ init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird; -+ if (init_hmbird) { -+ INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo); -+ INIT_LIST_HEAD(&init_hmbird->watchdog_node); -+ init_hmbird->sticky_cpu = -1; -+ init_hmbird->holding_cpu = -1; -+ atomic64_set(&init_hmbird->ops_state, 0); -+ init_hmbird->runnable_at = jiffies; -+ init_hmbird->slice = HMBIRD_SLICE_DFL; -+ init_hmbird->task = &init_task; -+ hmbird_set_sched_prop(&init_task, 0); -+ } else { -+ hmbird_err(INIT_TASK_FAIL, ":alloc init_task.scx failed!!!\n"); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ /* -+ * exec during boot phase, no need to care about alloc failed. -+ * lifecycle same to rq, no need to free. -+ */ -+ rq->android_oem_data1[HMBIRD_RQ_IDX] = -+ (u64)kmalloc(sizeof(struct hmbird_rq), GFP_KERNEL); -+ if (get_hmbird_rq(rq)) { -+ get_hmbird_rq(rq)->rq = rq; -+ get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window; -+ } else { -+ hmbird_err(ALLOC_RQSCX_FAIL, ":alloc rq->scx failed!!!\n"); -+ } -+ -+ rq->android_oem_data1[HMBIRD_OPS_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL); -+ if (!get_hmbird_ops(rq)) -+ pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n"); -+ init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL); -+ INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list); -+ -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL)); -+ -+ hmbird_ops_init(get_hmbird_ops(rq)); -+ } -+ -+ INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn); -+ INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn); -+ hmbird_misc_init(); -+ -+ panic_blk_init(); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_misc.c b/kernel/sched/hmbird/hmbird_misc.c -new file mode 100755 -index 000000000000..52582c22052c ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_misc.c -@@ -0,0 +1,124 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * @Description: -+ * @Version: -+ * @Author: wangrui8 -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include "hmbird_sched.h" -+#include -+#include -+#include "slim.h" -+ -+noinline int tracing_mark_write(const char *buf) -+{ -+ trace_printk(buf); -+ return 0; -+} -+ -+struct yield_opt_params yield_opt_params = { -+ .enable = 0, -+ .frame_per_sec = 120, -+ .frame_time_ns = NSEC_PER_SEC / 120, -+ .yield_headroom = 10, -+}; -+ -+DEFINE_PER_CPU(struct sched_yield_state, ystate); -+ -+static inline void hmbird_yield_state_update(struct sched_yield_state *ys) -+{ -+ if (!raw_spin_is_locked(&ys->lock)) -+ return; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ -+ if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1 -+ || ys->yield_cnt_after_sleep > yield_headroom) { -+ ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP); -+ } else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) { -+ ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP); -+ } -+ ys->yield_cnt = 0; -+ ys->sleep_times = 0; -+ ys->yield_cnt_after_sleep = 0; -+} -+ -+void hmbird_skip_yield(long *skip) -+{ -+ if (!get_hmbird_ops_enabled() || !yield_opt_params.enable) -+ return; -+ unsigned long flags, sleep_now = 0; -+ struct sched_yield_state *ys; -+ int cpu = raw_smp_processor_id(), cont_yield, new_frame; -+ int frame_time_ns = yield_opt_params.frame_time_ns; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ u64 wc; -+ -+ if (!(*skip)) { -+ wc = sched_clock(); -+ ys = &per_cpu(ystate, cpu); -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ -+ cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP; -+ new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1); -+ -+ if (!cont_yield && new_frame) { -+ hmbird_yield_state_update(ys); -+ ys->last_update_time = wc; -+ ys->sleep_end = ys->last_yield_time + frame_time_ns -+ - yield_headroom * YIELD_DURATION; -+ } -+ -+ if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) { -+ *skip = true; -+ -+ sleep_now = ys->sleep_times ? -+ max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep; -+ if (wc + sleep_now > ys->sleep_end) { -+ u64 delta = ys->sleep_end - wc; -+ -+ if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION) -+ sleep_now = delta; -+ else -+ sleep_now = 0; -+ } -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ if (sleep_now) { -+ sleep_now = div64_u64(sleep_now, 1000); -+ usleep_range_state(sleep_now, sleep_now, TASK_IDLE); -+ } -+ ys->sleep_times++; -+ ys->last_yield_time = sched_clock(); -+ return; -+ } -+ if (ys->sleep_times) -+ ys->yield_cnt_after_sleep++; -+ else -+ (ys->yield_cnt)++; -+ ys->last_yield_time = wc; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+} -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops) -+{ -+ hmbird_ops->scx_enable = get_hmbird_ops_enabled; -+ hmbird_ops->check_non_task = get_non_hmbird_task; -+ hmbird_ops->hmbird_get_md_info = hmbird_get_md_info; -+} -+ -+void hmbird_misc_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_init(&ys->lock); -+ } -+ -+ set_hmbird_module_loaded(1); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_sched.h b/kernel/sched/hmbird/hmbird_sched.h -new file mode 100755 -index 000000000000..6b5ef43cb827 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef __HMBIRD_SCHED__ -+#define __HMBIRD_SCHED__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include "../../time/tick-sched.h" -+#include "../../sched/sched.h" -+#include -+#include -+#include -+ -+#define REGISTER_TRACE_VH(vender_hook, handler) \ -+{ \ -+ ret = register_trace_##vender_hook(handler, NULL); \ -+ if (ret) { \ -+ pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \ -+ } \ -+} -+#define REGISTER_TRACE(vendor_hook, handler, data, err) \ -+do { \ -+ ret = register_trace_##vendor_hook(handler, data); \ -+ if (ret) { \ -+ pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret); \ -+ goto err; \ -+ } \ -+} while (0) -+ -+#define UNREGISTER_TRACE(vendor_hook, handler, data) \ -+ unregister_trace_##vendor_hook(handler, data) \ -+ -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+ -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int sched_ravg_window_frame_per_sec; -+extern int slim_gov_debug; -+extern int cpu7_tl; -+extern int scx_gov_ctrl; -+extern spinlock_t new_sched_ravg_window_lock; -+extern int cluster_separate; -+ -+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER BIT(31) -+#define MAX_YIELD_SLEEP (2000000ULL) -+#define MIN_YIELD_SLEEP (200000ULL) -+#define YIELD_DURATION (5000ULL) -+#define DEFAULT_YIELD_SLEEP_TH (10) -+ -+struct sched_yield_state { -+ raw_spinlock_t lock; -+ u64 last_yield_time; -+ u64 last_update_time; -+ u64 sleep_end; -+ unsigned long yield_cnt; -+ unsigned long yield_cnt_after_sleep; -+ unsigned long sleep; -+ int sleep_times; -+}; -+ -+DECLARE_PER_CPU(struct sched_yield_state, ystate); -+ -+void hmbird_window_rollover_run_once(struct rq *rq); -+void hmbird_yield_state_update_per_frame(void); -+void hmbird_misc_init(void); -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops); -+#endif /*__HMBIRD_SCHED__*/ -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.c b/kernel/sched/hmbird/hmbird_sched_proc.c -new file mode 100755 -index 000000000000..234768fc767a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.c -@@ -0,0 +1,527 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "hmbird_sched_proc.h" -+#include -+ -+#include "hmbird_util_track.h" -+#include "slim.h" -+ -+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched" -+#define SLIM_FREQ_GOV_DIR "slim_freq_gov" -+#define LOAD_TRACK_DIR "slim_walt" -+#define HMBIRD_PROC_PERMISSION 0666 -+ -+int scx_enable; -+int partial_enable; -+int cpuctrl_high_ratio = 55; -+int cpuctrl_low_ratio = 40; -+int slim_stats; -+int hmbirdcore_debug; -+int slim_for_app; -+int misfit_ds = 90; -+unsigned int highres_tick_ctrl; -+unsigned int highres_tick_ctrl_dbg; -+int cpu7_tl = 70; -+int slim_walt_ctrl; -+int slim_walt_dump; -+int slim_walt_policy; -+int slim_gov_debug; -+int scx_gov_ctrl = 1; -+int sched_ravg_window_frame_per_sec = 125; -+int parctrl_high_ratio = 55; -+int parctrl_low_ratio = 40; -+int parctrl_high_ratio_l = 65; -+int parctrl_low_ratio_l = 50; -+int isoctrl_high_ratio = 75; -+int isoctrl_low_ratio = 60; -+int isolate_ctrl; -+int iso_free_rescue; -+int heartbeat; -+int heartbeat_enable = 1; -+int watchdog_enable; -+int save_gov; -+u64 cpu_cluster_masks; -+int hmbird_preempt_policy; -+int cluster_separate; -+ -+char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= 32) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtoint(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoint\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+static int set_proc_buf_val_u64(struct file *file, const char __user *buf, -+ size_t count, u64 *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= sizeof(kbuf)) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtou64(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoul\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/* common ops begin */ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ return count; -+} -+ -+static int hmbird_common_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%d\n", *(int *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_show, pde_data(inode)); -+} -+ -+static int hmbird_common_ul_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%lu\n", *(unsigned long *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_ul_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_ul_show, pde_data(inode)); -+} -+ -+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write); -+/* common ops end */ -+ -+/* scx_enable ops begin */ -+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC); -+ if (hmbird_ctrl(*pval)) -+ return -EFAULT; -+ -+ return count; -+} -+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write); -+/* scx_enable ops end */ -+ -+/* hmbird_stats ops begin */ -+#define MAX_STATS_BUF (4096) -+static int hmbird_stats_proc_show(struct seq_file *m, void *v) -+{ -+ char *buf; -+ -+ buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ stats_print(buf, MAX_STATS_BUF); -+ -+ seq_printf(m, "%s\n", buf); -+ -+ kfree(buf); -+ return 0; -+} -+ -+static int hmbird_stats_proc_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_stats_proc_show, inode); -+} -+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL); -+/* hmbird_stats ops end */ -+ -+/* sched_ravg_window_frame_per_sec ops begin */ -+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file, -+ const char __user *buf, size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ sched_ravg_window_change(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open, -+ sched_ravg_window_frame_per_sec_proc_write); -+/* sched_ravg_window_frame_per_sec ops end */ -+ -+static ssize_t save_gov_str(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ for_each_possible_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy || (cpu != policy->cpu)) -+ continue; -+ WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0); -+ hmbird_info_systrace(":save origin gov : %s\n", saved_gov[cpu]); -+ } -+ return count; -+} -+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str); -+ -+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ u64 *pval = (u64 *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val_u64(file, buf, count, pval)) -+ return -EFAULT; -+ -+ if (scx_enable == 0) -+ set_cpu_cluster(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write); -+ -+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ int tmp_val; -+ -+ if (set_proc_buf_val(file, buf, count, &tmp_val)) -+ return -EFAULT; -+ -+ slim_walt_enable(tmp_val); -+ *pval = tmp_val; -+ -+ return count; -+} -+ -+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write); -+ -+/* yield_opt ops begin */ -+static int yield_opt_show(struct seq_file *m, void *v) -+{ -+ struct yield_opt_params *data = m->private; -+ -+ seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n", -+ data->enable, data->frame_per_sec, data->yield_headroom); -+ return 0; -+} -+ -+static int yield_opt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, yield_opt_show, pde_data(inode)); -+} -+ -+static ssize_t yield_opt_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu; -+ unsigned long flags; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if (copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp -+ != 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) || -+ (yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp; -+ yield_opt_params.frame_per_sec = frame_per_sec_tmp; -+ yield_opt_params.yield_headroom = yield_headroom_tmp; -+ yield_opt_params.enable = enable_tmp; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ ys->last_yield_time = 0; -+ ys->last_update_time = 0; -+ ys->sleep_end = 0; -+ ys->yield_cnt = 0; -+ ys->yield_cnt_after_sleep = 0; -+ ys->sleep = 0; -+ ys->sleep_times = 0; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write); -+ -+ -+static int __init hmbird_proc_init(void) -+{ -+ struct proc_dir_entry *hmbird_dir; -+ struct proc_dir_entry *load_track_dir; -+ struct proc_dir_entry *freq_gov_dir; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ -+ /* mkdir /proc/hmbird_sched */ -+ hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL); -+ if (!hmbird_dir) { -+ pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &scx_enable_proc_ops, -+ &scx_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &partial_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_stats); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbirdcore_debug); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_for_app); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &misfit_ds); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl_dbg); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpu7_tl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &cpu_cluster_masks_proc_ops, -+ &cpu_cluster_masks); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &save_gov_proc_ops, -+ &save_gov); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &watchdog_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isolate_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &iso_free_rescue); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_stats_proc_ops); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &yield_opt_proc_ops, -+ &yield_opt_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbird_preempt_policy); -+ /* /proc/hmbird_sched--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_walt */ -+ load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir); -+ if (!load_track_dir) { -+ pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_walt--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &slim_walt_ctrl_proc_ops, -+ &slim_walt_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_dump); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_policy); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &sched_ravg_window_frame_per_sec_proc_ops, -+ &sched_ravg_window_frame_per_sec); -+ /* /proc/hmbird_sched/slim_walt--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_freq_gov */ -+ freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir); -+ if (!freq_gov_dir) { -+ pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_freq_gov--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &slim_gov_debug); -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &scx_gov_ctrl); -+ /* /proc/hmbird_sched/slim_freq_gov--end */ -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cluster_separate); -+ -+ return 0; -+} -+ -+device_initcall(hmbird_proc_init); -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.h b/kernel/sched/hmbird/hmbird_sched_proc.h -new file mode 100755 -index 000000000000..e22bc75f1dd7 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SCHED_PROC_H__ -+#define __HMBIRD_SCHED_PROC_H__ -+ -+#include -+#include -+ -+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \ -+ do { \ -+ if (!proc_create(name, mode, parent, proc_ops)) { \ -+ pr_err("Error creating proc entry %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \ -+ do { \ -+ if (!proc_create_data(name, mode, parent, proc_ops, data)) { \ -+ pr_err("Error creating proc entry with data %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_PROC_OPS(name, open_func, write_func) \ -+ static const struct proc_ops name##_proc_ops = { \ -+ .proc_open = open_func, \ -+ .proc_write = write_func, \ -+ .proc_read = seq_read, \ -+ .proc_lseek = seq_lseek, \ -+ .proc_release = single_release, \ -+ } -+ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos); -+static int hmbird_common_show(struct seq_file *m, void *v); -+static int hmbird_common_open(struct inode *inode, struct file *file); -+ -+#endif -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.c b/kernel/sched/hmbird/hmbird_shadow_tick.c -new file mode 100755 -index 000000000000..21de551c5132 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.c -@@ -0,0 +1,159 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include -+#include "../../time/tick-sched.h" -+#include -+ -+#include "hmbird_shadow_tick.h" -+ -+#define HIGHRES_WATCH_CPU 0 -+ -+#include -+static bool shadow_tick_enable(void) {return highres_tick_ctrl; } -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; } -+#endif -+static bool shadow_tick_timer_init_flag; -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define shadow_tick_printk(fmt, args...) \ -+do { \ -+ int cpu = smp_processor_id(); \ -+ if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU) \ -+ trace_printk("hmbird shadow tick :"fmt, args); \ -+} while (0) -+#else -+#define shadow_tick_printk(fmt, args...) -+#endif -+ -+DEFINE_PER_CPU(struct hrtimer, stt); -+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu))) -+#define STOP_IDLE_TRIGGER (1) -+#define PERIODIC_TICK_TRIGGER (2) -+#define TICK_INTVAL (1000000ULL) -+/* -+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms, -+ * so we can't restart hrtimer in scheduler tick. -+ */ -+static DEFINE_PER_CPU(u8, trigger_event); -+ -+/* -+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick. -+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler. -+ */ -+static DEFINE_PER_CPU(u8, tick_phase); -+ -+static inline void highres_timer_ctrl(bool enable, int cpu) -+{ -+ if (enable && hmbird_enabled()) { -+ if (!hrtimer_active(shadow_tick_timer(cpu))) -+ hrtimer_start(shadow_tick_timer(cpu), -+ ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED); -+ } else { -+ if (!enable) -+ hrtimer_cancel(shadow_tick_timer(cpu)); -+ } -+} -+ -+static inline void high_res_clear_phase(int cpu) -+{ -+ per_cpu(tick_phase, cpu) = 0; -+} -+ -+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer) -+{ -+ per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3; -+ if (per_cpu(tick_phase, cpu)) { -+ hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) { -+ per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ struct rq_flags rf; -+ -+ rq_lock(rq, &rf); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ rq_unlock(rq, &rf); -+ -+ return highres_next_phase(cpu, timer); -+} -+ -+void shadow_tick_timer_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance; -+ } -+} -+ -+void start_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable()) { -+ if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER) -+ highres_timer_ctrl(false, cpu); -+ per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static void stop_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ per_cpu(trigger_event, cpu) = 0; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(false, cpu); -+} -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ stop_shadow_tick_timer(); -+} -+ -+void scheduler_tick_handler(void *unused, struct rq *rq) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ start_shadow_tick_timer(); -+} -+ -+static int __init hmbird_shadow_tick_init(void) -+{ -+ int ret = 0; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ shadow_tick_timer_init(); -+ shadow_tick_timer_init_flag = true; -+ return ret; -+} -+ -+device_initcall(hmbird_shadow_tick_init); -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.h b/kernel/sched/hmbird/hmbird_shadow_tick.h -new file mode 100755 -index 000000000000..0c26fd984f0a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.h -@@ -0,0 +1,11 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SHADOW_TICK_H__ -+#define __HMBIRD_SHADOW_TICK_H__ -+ -+#include -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+void scheduler_tick_handler(void *unused, struct rq *rq); -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state); -+#endif -diff --git a/kernel/sched/hmbird/hmbird_trace.h b/kernel/sched/hmbird/hmbird_trace.h -new file mode 100755 -index 000000000000..8468b406f347 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_trace.h -@@ -0,0 +1,92 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hmbird -+ -+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HMBIRD_H -+ -+#include -+#include -+#include -+ -+#define MAX_FATAL_INFO (64) -+ -+TRACE_EVENT(hmbird_fatal_info, -+ -+ TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info), -+ -+ TP_ARGS(type, pe, lnr, bnr, info), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, type) -+ __field(int, pe) -+ __field(int, lnr) -+ __field(int, bnr) -+ __array(char, info, MAX_FATAL_INFO)), -+ -+ TP_fast_assign( -+ __entry->type = type; -+ __entry->pe = pe; -+ __entry->lnr = lnr; -+ __entry->bnr = bnr; -+ memcpy(__entry->info, info, MAX_FATAL_INFO);), -+ -+ TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s", -+ __entry->type, __entry->pe, __entry->lnr, __entry->bnr, -+ __entry->info) -+); -+ -+TRACE_EVENT(hmbird_update_history, -+ -+ TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq, -+ struct task_struct *p, u32 runtime, int samples, int event), -+ -+ TP_ARGS(hmbird, rq, p, runtime, samples, event), -+ -+ TP_STRUCT__entry( -+ __array(char, comm, TASK_COMM_LEN) -+ __field(pid_t, pid) -+ __field(unsigned int, runtime) -+ __field(int, samples) -+ __field(int, event) -+ __field(unsigned int, demand) -+ __array(u32, hist, RAVG_HIST_SIZE) -+ __field(u16, task_util) -+ __field(int, cpu)), -+ -+ TP_fast_assign( -+ memcpy(__entry->comm, p->comm, TASK_COMM_LEN); -+ __entry->pid = p->pid; -+ __entry->runtime = runtime; -+ __entry->samples = samples; -+ __entry->event = event; -+ __entry->demand = hmbird->sts.demand; -+ memcpy(__entry->hist, hmbird->sts.sum_history, -+ RAVG_HIST_SIZE * sizeof(u32)); -+ __entry->task_util = hmbird->sts.demand_scaled; -+ __entry->cpu = rq->cpu;), -+ -+ TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d", -+ __entry->comm, __entry->pid, -+ __entry->runtime, __entry->samples, -+ __entry->event, -+ __entry->demand, -+ __entry->hist[0], __entry->hist[1], -+ __entry->hist[2], __entry->hist[3], -+ __entry->hist[4], -+ __entry->task_util, -+ __entry->cpu) -+); -+ -+#endif /*_TRACE_HMBIRD_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird -+ -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE hmbird_trace -+/* This part must be outside protection */ -+#include -diff --git a/kernel/sched/hmbird/hmbird_util_track.c b/kernel/sched/hmbird/hmbird_util_track.c -new file mode 100755 -index 000000000000..d520a8403401 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.c -@@ -0,0 +1,626 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+ -+#define CREATE_TRACE_POINTS -+#include "hmbird_trace.h" -+#undef CREATE_TRACE_POINTS -+ -+#define HMBIRD_DEBUG_PANIC (1 << 3) -+static bool init_irq_work_inited; -+ -+extern noinline int tracing_mark_write(const char *buf); -+ -+int hmbird_sched_ravg_window = 8000000; -+int new_hmbird_sched_ravg_window = 8000000; -+DEFINE_SPINLOCK(new_sched_ravg_window_lock); -+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+ -+static struct irq_work hmbird_slim_walt_irq_work; -+ -+/*Sysctl related interface*/ -+#define WINDOW_STATS_RECENT 0 -+#define WINDOW_STATS_MAX 1 -+#define WINDOW_STATS_MAX_RECENT_AVG 2 -+#define WINDOW_STATS_AVG 3 -+#define WINDOW_STATS_INVALID_POLICY 4 -+ -+ -+#define SCX_SCHED_CAPACITY_SHIFT 10 -+#define SCHED_ACCOUNT_WAIT_TIME 0 -+ -+atomic64_t hmbird_irq_work_lastq_ws; -+static u64 tick_sched_clock; -+ -+static inline u64 scale_exec_time(u64 delta, struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT; -+} -+ -+static inline u64 scale_time_to_util(u64 d) -+{ -+ do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT); -+ return d; -+} -+ -+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ delta = scale_exec_time(delta, rq); -+ sts->sum += delta; -+ if (unlikely(sts->sum > hmbird_sched_ravg_window)) -+ sts->sum = hmbird_sched_ravg_window; -+ -+ return delta; -+} -+ -+ -+static int -+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) -+{ -+ /* -+ * No need to bother updating task demand for the idle task. -+ */ -+ if (is_idle_task(p)) -+ return 0; -+ -+ /* -+ * When a task is waking up it is completing a segment of non-busy -+ * time. Likewise, if wait time is not treated as busy time, then -+ * when a task begins to run or is migrated, it is not running and -+ * is completing a segment of non-busy time. -+ */ -+ if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && -+ (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) -+ return 0; -+ -+ /* -+ * The idle exit time is not accounted for the first task _picked_ up to -+ * run on the idle CPU. -+ */ -+ if (event == PICK_NEXT_TASK && rq->curr == rq->idle) -+ return 0; -+ -+ /* -+ * TASK_UPDATE can be called on sleeping task, when its moved between -+ * related groups -+ */ -+ if (event == TASK_UPDATE) { -+ if (rq->curr == p) -+ return 1; -+ -+ return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; -+ } -+ -+ return 1; -+} -+ -+static void rollover_cpu_window(struct rq *rq, bool full_window) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 curr_sum = srq->curr_runnable_sum; -+ -+ if (unlikely(full_window)) -+ curr_sum = 0; -+ -+ srq->prev_runnable_sum = curr_sum; -+ srq->curr_runnable_sum = 0; -+} -+ -+static u64 -+update_window_start(struct rq *rq, u64 wallclock, int event) -+{ -+ s64 delta; -+ int nr_windows; -+ bool full_window; -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start = srq->window_start; -+ -+ if (wallclock < srq->latest_clock) -+ wallclock = srq->latest_clock; -+ delta = wallclock - srq->window_start; -+ if (delta < 0) { -+ delta = 0; -+ wallclock = srq->window_start; -+ } -+ srq->latest_clock = wallclock; -+ if (delta < hmbird_sched_ravg_window) -+ return old_window_start; -+ -+ nr_windows = div64_u64(delta, hmbird_sched_ravg_window); -+ srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window; -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ full_window = nr_windows > 1; -+ rollover_cpu_window(rq, full_window); -+ -+ return old_window_start; -+} -+ -+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -+ -+static inline unsigned int get_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static inline unsigned int cpu_cur_freq(int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cur; -+} -+ -+static void -+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, -+ u64 wallclock) -+{ -+ int cpu = cpu_of(rq); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * -+ arch_scale_cpu_capacity(cpu), get_max_freq(cpu)); -+} -+ -+/* -+ * Called when new window is starting for a task, to record cpu usage over -+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 -+ * when, say, a real-time task runs without preemption for several windows at a -+ * stretch. -+ */ -+static void update_history(struct rq *rq, struct task_struct *p, -+ u32 runtime, int samples, int event) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u32 *hist = &sts->sum_history[0]; -+ int i; -+ u32 max = 0, avg, demand; -+ u64 sum = 0; -+ u16 demand_scaled; -+ -+ /* Ignore windows where task had no activity */ -+ if (!runtime || is_idle_task(p) || !samples) -+ goto done; -+ -+ /* Push new 'runtime' value onto stack */ -+ for (; samples > 0; samples--) { -+ hist[sts->cidx] = runtime; -+ sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE; -+ } -+ -+ for (i = 0; i < RAVG_HIST_SIZE; i++) { -+ sum += hist[i]; -+ if (hist[i] > max) -+ max = hist[i]; -+ } -+ -+ sts->sum = 0; -+ avg = div64_u64(sum, RAVG_HIST_SIZE); -+ -+ switch (slim_walt_policy) { -+ case WINDOW_STATS_RECENT: -+ demand = runtime; -+ break; -+ case WINDOW_STATS_MAX: -+ demand = max; -+ break; -+ case WINDOW_STATS_AVG: -+ demand = avg; -+ break; -+ default: -+ demand = max(avg, runtime); -+ } -+ -+ demand_scaled = scale_time_to_util(demand); -+ -+ sts->demand = demand; -+ sts->demand_scaled = demand_scaled; -+ -+done: -+ return; -+} -+ -+ -+static u64 update_task_demand(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ -+ u64 delta, window_start = srq->window_start; -+ int new_window, nr_full_windows; -+ u32 window_size = hmbird_sched_ravg_window; -+ u64 runtime; -+ -+ new_window = mark_start < window_start; -+ if (!account_busy_for_task_demand(rq, p, event)) { -+ if (new_window) -+ /* -+ * If the time accounted isn't being accounted as -+ * busy time, and a new window started, only the -+ * previous window need be closed out with the -+ * pre-existing demand. Multiple windows may have -+ * elapsed, but since empty windows are dropped, -+ * it is not necessary to account those. -+ */ -+ update_history(rq, p, sts->sum, 1, event); -+ return 0; -+ } -+ -+ if (!new_window) { -+ /* -+ * The simple case - busy time contained within the existing -+ * window. -+ */ -+ return add_to_task_demand(rq, p, wallclock - mark_start); -+ } -+ -+ /* -+ * Busy time spans at least two windows. Temporarily rewind -+ * window_start to first window boundary after mark_start. -+ */ -+ delta = window_start - mark_start; -+ nr_full_windows = div64_u64(delta, window_size); -+ window_start -= (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (window_start - mark_start) first */ -+ runtime = add_to_task_demand(rq, p, window_start - mark_start); -+ -+ /* Push new sample(s) into task's demand history */ -+ update_history(rq, p, sts->sum, 1, event); -+ if (nr_full_windows) { -+ u64 scaled_window = scale_exec_time(window_size, rq); -+ -+ update_history(rq, p, scaled_window, nr_full_windows, event); -+ runtime += nr_full_windows * scaled_window; -+ } -+ -+ /* -+ * Roll window_start back to current to process any remainder -+ * in current window. -+ */ -+ window_start += (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (wallclock - window_start) next */ -+ mark_start = window_start; -+ runtime += add_to_task_demand(rq, p, wallclock - mark_start); -+ -+ return runtime; -+} -+ -+u16 slim_walt_cpu_util(int cpu) -+{ -+ u64 prev_runnable_sum; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ prev_runnable_sum = srq->prev_runnable_sum; -+ do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT); -+ -+ return (u16)prev_runnable_sum; -+} -+ -+static DEFINE_PER_CPU(u16, prev_cpu_util); -+static inline void cpu_util_update_systrace_c(int cpu) -+{ -+ char buf[256]; -+ u16 cpu_util = slim_walt_cpu_util(cpu); -+ -+ if (cpu_util != per_cpu(prev_cpu_util, cpu)) { -+ snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n", -+ cpu, cpu_util); -+ tracing_mark_write(buf); -+ per_cpu(prev_cpu_util, cpu) = cpu_util; -+ } -+} -+ -+ -+static inline int account_busy_for_cpu_time(struct rq *rq, -+ struct task_struct *p, -+ int event) -+{ -+ return !is_idle_task(p) && (event == PUT_PREV_TASK -+ || event == TASK_UPDATE); -+} -+ -+ -+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ int new_window, full_window = 0; -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 window_start = srq->window_start; -+ u32 window_size = srq->prev_window_size; -+ u64 delta; -+ u64 *curr_runnable_sum = &srq->curr_runnable_sum; -+ u64 *prev_runnable_sum = &srq->prev_runnable_sum; -+ -+ new_window = mark_start < window_start; -+ if (new_window) -+ full_window = (window_start - mark_start) >= window_size; -+ -+ -+ if (!account_busy_for_cpu_time(rq, p, event)) -+ goto done; -+ -+ -+ if (!new_window) { -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. No rollover -+ * since we didn't start a new window. An example of this is -+ * when a task starts execution and then sleeps within the -+ * same window. -+ */ -+ delta = wallclock - mark_start; -+ -+ delta = scale_exec_time(delta, rq); -+ *curr_runnable_sum += delta; -+ -+ goto done; -+ } -+ -+ /* -+ * situations below this need window rollover, -+ * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done -+ * in update_window_start() -+ * -+ * For task counters curr/prev_window[_cpu] are rolled over in the early part of -+ * this function. If full_window(s) have expired and time since last update needs -+ * to be accounted as busy time, set the prev to a complete window size time, else -+ * add the prev window portion. -+ * -+ * For task curr counters a new window has begun, always assign -+ */ -+ -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. A new window -+ * must have been started in udpate_window_start() -+ * If any of these three above conditions are true -+ * then this busy time can't be accounted as irqtime. -+ * -+ * Busy time for the idle task need not be accounted. -+ * -+ * An example of this would be a task that starts execution -+ * and then sleeps once a new window has begun. -+ */ -+ -+ /* -+ * A full window hasn't elapsed, account partial -+ * contribution to previous completed window. -+ */ -+ -+ -+ delta = full_window ? scale_exec_time(window_size, rq) : -+ scale_exec_time(window_start - mark_start, rq); -+ -+ *prev_runnable_sum += delta; -+ -+ /* Account piece of busy time in the current window. */ -+ delta = scale_exec_time(wallclock - window_start, rq); -+ *curr_runnable_sum += delta; -+ -+done: -+ if (slim_walt_dump && new_window) -+ cpu_util_update_systrace_c(rq->cpu); -+} -+ -+ -+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq) -+{ -+ u64 result; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 new_window_start = srq->window_start; -+ -+ if (old_window_start == new_window_start) -+ return; -+ -+ result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws, -+ old_window_start, new_window_start); -+ if (result != old_window_start) -+ return; -+ -+ if (likely(cpu_online(raw_smp_processor_id()))) -+ irq_work_queue(&hmbird_slim_walt_irq_work); -+ else -+ irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask)); -+} -+ -+/* -+ * In the core scheduler, most of the load update points update the rq_clock after -+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of -+ * obtaining the time, but to prevent the subsequent migration of the load update -+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock -+ * update here. -+ */ -+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p, -+ struct rq *rq, int event) -+{ -+ if (!(rq->clock_update_flags & RQCF_UPDATED)) -+ update_rq_clock(rq); -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock)); -+} -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start; -+ -+ if (!slim_walt_ctrl) -+ return; -+ -+ if (!srq->window_start || sts->mark_start == wallclock) -+ return; -+ -+ old_window_start = update_window_start(rq, wallclock, event); -+ -+ if (!sts->window_start) -+ sts->window_start = srq->window_start; -+ -+ if (!sts->mark_start) -+ goto done; -+ -+ update_task_rq_cpu_cycles(p, rq, event, wallclock); -+ update_task_demand(p, rq, event, wallclock); -+ update_cpu_busy_time(p, rq, event, wallclock); -+ -+ sts->window_start = srq->window_start; -+ -+done: -+ sts->mark_start = wallclock; -+ slim_walt_window_rollover_run_once(old_window_start, rq); -+} -+ -+static void slim_walt_irq_work(struct irq_work *irq_work) -+{ -+ cpumask_t lock_cpus; -+ struct hmbird_sched_rq_stats *srq; -+ struct rq *rq; -+ int cpu; -+ int level = 0; -+ u64 wc; -+ unsigned long flags; -+ -+ cpumask_copy(&lock_cpus, cpu_possible_mask); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ if (level == 0) -+ raw_spin_lock(&cpu_rq(cpu)->__lock); -+ else -+ raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); -+ level++; -+ } -+ -+ wc = sched_clock(); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ rq = cpu_rq(cpu); -+ hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc); -+ } -+ -+ cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER); -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) { -+ srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id()); -+ if (wc < srq->window_start + new_hmbird_sched_ravg_window) -+ hmbird_sched_ravg_window = new_hmbird_sched_ravg_window; -+ } -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ raw_spin_unlock(&cpu_rq(cpu)->__lock); -+ } -+} -+static void hmbird_sched_init_rq(struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ srq->task_exec_scale = 1024; -+ srq->window_start = 0; -+} -+ -+void hmbird_sched_init_task(struct task_struct *p) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ memset(sts, 0, sizeof(struct hmbird_sched_task_stats)); -+} -+ -+static void hmbird_sched_stats_init(void) -+{ -+ unsigned long flags; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ raw_spin_lock_irqsave(&rq->__lock, flags); -+ hmbird_sched_init_rq(rq); -+ raw_spin_unlock_irqrestore(&rq->__lock, flags); -+ } -+ slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG; -+ -+ if (false == init_irq_work_inited) { -+ init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work); -+ init_irq_work_inited = true; -+ } -+} -+ -+void hmbird_scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (unlikely(!tick_sched_clock)) { -+ /* -+ * Let the window begin 20us prior to the tick, -+ * that way we are guaranteed a rollover when the tick occurs. -+ * Use rq->clock directly instead of rq_clock() since -+ * we do not have the rq lock and -+ * rq->clock was updated in the tick callpath. -+ */ -+ if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000)) -+ return; -+ for_each_possible_cpu(cpu) { -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ srq->window_start = tick_sched_clock; -+ } -+ atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock); -+ } -+} -+ -+void slim_walt_enable(int enable) -+{ -+ if (1 == !!enable) { -+ hmbird_sched_stats_init(); -+ WRITE_ONCE(tick_sched_clock, 0); -+ } else -+ slim_walt_ctrl = 0; -+} -+ -+void slim_get_cpu_util(int cpu, u64 *util) -+{ -+ if (cpu < 0) -+ return; -+ -+ *util = slim_walt_cpu_util(cpu); -+} -+ -+void slim_get_task_util(struct task_struct *p, u64 *util) -+{ -+ if (p == NULL) -+ return; -+ -+ *util = get_hmbird_ts(p)->sts.demand_scaled; -+} -+ -+void sched_ravg_window_change(int frame_per_sec) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec; -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+} -diff --git a/kernel/sched/hmbird/hmbird_util_track.h b/kernel/sched/hmbird/hmbird_util_track.h -new file mode 100755 -index 000000000000..17b85df7f83b ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.h -@@ -0,0 +1,33 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef __HMBIRD_UTIL_TRACK_H__ -+#define __HMBIRD_UTIL_TRACK_H__ -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock); -+void hmbird_sched_init_task(struct task_struct *p); -+void slim_walt_enable(int enable); -+void slim_get_cpu_util(int cpu, u64 *util); -+void slim_get_task_util(struct task_struct *p, u64 *util); -+ -+extern atomic64_t hmbird_irq_work_lastq_ws; -+ -+enum task_event { -+ PUT_PREV_TASK = 0, -+ PICK_NEXT_TASK = 1, -+ TASK_WAKE = 2, -+ TASK_MIGRATE = 3, -+ TASK_UPDATE = 4, -+ IRQ_UPDATE = 5, -+}; -+ -+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+#endif /* __HMBIRD_UTIL_TRACK_H__ */ -diff --git a/kernel/sched/hmbird/slim.h b/kernel/sched/hmbird/slim.h -new file mode 100755 -index 000000000000..eb386260e950 ---- /dev/null -+++ b/kernel/sched/hmbird/slim.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+ -+#ifndef __SLIM_H -+#define __SLIM_H -+ -+extern atomic_t __hmbird_ops_enabled; -+extern atomic_t non_hmbird_task; -+extern int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+extern int heartbeat; -+extern int heartbeat_enable; -+extern int watchdog_enable; -+extern int isolate_ctrl; -+extern int parctrl_high_ratio; -+extern int parctrl_low_ratio; -+extern int isoctrl_high_ratio; -+extern int isoctrl_low_ratio; -+extern int iso_free_rescue; -+extern int yield_opt; -+ -+extern enum hmbird_switch_type sw_type; -+extern noinline int tracing_mark_write(const char *buf); -+int task_top_id(struct task_struct *p); -+void stats_print(char *buf, int len); -+void hmbird_skip_yield(long *skip); -+extern spinlock_t hmbird_tasks_lock; -+ -+struct yield_opt_params { -+ int enable; -+ int frame_per_sec; -+ u64 frame_time_ns; -+ int yield_headroom; -+}; -+ -+extern struct yield_opt_params yield_opt_params; -+ -+#define MAX_GOV_LEN (16) -+extern char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+#endif -diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c -index b33cefeb4188..487255ac6832 100644 ---- a/kernel/sched/idle.c -+++ b/kernel/sched/idle.c -@@ -408,13 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl - - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) - { -- scx_update_idle(rq, false); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, false); -+#endif - } - - static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) - { - update_idle_core(rq); -- scx_update_idle(rq, true); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, true); -+#endif - schedstat_inc(rq->sched_goidle); - } - -diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h -index cbc478992cc4..e35473a1f0ea 100644 ---- a/kernel/sched/sched.h -+++ b/kernel/sched/sched.h -@@ -187,18 +187,13 @@ static inline int idle_policy(int policy) - return policy == SCHED_IDLE; - } - --static inline int normal_policy(int policy) --{ --#ifdef CONFIG_SCHED_CLASS_EXT -- if (policy == SCHED_EXT) -- return true; --#endif -- return policy == SCHED_NORMAL; --} -- - static inline int fair_policy(int policy) - { -- return normal_policy(policy) || policy == SCHED_BATCH; -+#ifdef CONFIG_HMBIRD_SCHED -+ return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#else -+ return policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#endif - } - - static inline int rt_policy(int policy) -@@ -246,24 +241,6 @@ static inline void update_avg(u64 *avg, u64 sample) - #define shr_bound(val, shift) \ - (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) - --/* -- * cgroup weight knobs should use the common MIN, DFL and MAX values which are -- * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it -- * maps pretty well onto the shares value used by scheduler and the round-trip -- * conversions preserve the original value over the entire range. -- */ --static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) --{ -- return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); --} -- --static inline unsigned long sched_weight_to_cgroup(unsigned long weight) --{ -- return clamp_t(unsigned long, -- DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -- CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); --} -- - /* - * !! For sched_setattr_nocheck() (kernel) only !! - * -@@ -531,10 +508,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) { } - #endif /* CONFIG_SMP */ - #else /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_HMBIRD_SCHED - static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) - { - return 0; - } -+#endif - #endif /* CONFIG_FAIR_GROUP_SCHED */ - - #else /* CONFIG_CGROUP_SCHED */ -@@ -699,29 +678,6 @@ struct cfs_rq { - #endif /* CONFIG_FAIR_GROUP_SCHED */ - }; - --#ifdef CONFIG_SCHED_CLASS_EXT --/* scx_rq->flags, protected by the rq lock */ --enum scx_rq_flags { -- SCX_RQ_CAN_STOP_TICK = 1 << 0, --}; -- --struct scx_rq { -- struct scx_dispatch_q local_dsq; -- struct list_head watchdog_list; -- u64 ops_qseq; -- u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -- u32 nr_running; -- u32 flags; -- bool cpu_released; -- cpumask_var_t cpus_to_kick; -- cpumask_var_t cpus_to_preempt; -- cpumask_var_t cpus_to_wait; -- u64 pnt_seq; -- struct irq_work kick_cpus_irq_work; -- struct rq *rq; --}; --#endif /* CONFIG_SCHED_CLASS_EXT */ -- - static inline int rt_bandwidth_enabled(void) - { - return sysctl_sched_rt_runtime >= 0; -@@ -1257,11 +1213,7 @@ struct rq { - - ANDROID_OEM_DATA_ARRAY(1, 16); - --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, struct scx_rq *scx); --#else - ANDROID_KABI_RESERVE(1); --#endif - ANDROID_KABI_RESERVE(2); - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); -@@ -2339,11 +2291,6 @@ struct affinity_context { - unsigned int flags; - }; - --enum rq_onoff_reason { -- RQ_ONOFF_HOTPLUG, /* CPU is going on/offline */ -- RQ_ONOFF_TOPOLOGY, /* sched domain topology update */ --}; -- - struct sched_class { - - #ifdef CONFIG_UCLAMP_TASK -@@ -2550,7 +2497,6 @@ extern void init_sched_rt_class(void); - extern void init_sched_fair_class(void); - - extern void reweight_task(struct task_struct *p, int prio); --extern void __setscheduler_prio(struct task_struct *p, int prio); - - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); -@@ -2631,53 +2577,6 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) - extern void activate_task(struct rq *rq, struct task_struct *p, int flags); - extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); - --struct sched_change_guard { -- struct task_struct *p; -- struct rq *rq; -- bool queued; -- bool running; -- bool done; --}; -- --extern struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -- --extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags); -- --/** -- * SCHED_CHANGE_BLOCK - Nested block for task attribute updates -- * @__rq: Runqueue the target task belongs to -- * @__p: Target task -- * @__flags: DEQUEUE/ENQUEUE_* flags -- * -- * A task may need to be dequeued and put_prev_task'd for attribute updates and -- * set_next_task'd and re-enqueued afterwards. This helper defines a nested -- * block which automatically handles these preparation and cleanup operations. -- * -- * SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- * update_attribute(p); -- * ... -- * } -- * -- * If @__flags is a variable, the variable may be updated in the block body and -- * the updated value will be used when re-enqueueing @p. -- * -- * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling -- * update_rq_clock() beforehand. Otherwise, the rq clock is automatically -- * updated iff the task needs to be dequeued and re-enqueued. Only the former -- * case guarantees that the rq clock is up-to-date inside and after the block. -- */ --#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -- for (struct sched_change_guard __cg = \ -- sched_change_guard_init(__rq, __p, __flags); \ -- !__cg.done; sched_change_guard_fini(&__cg, __flags)) -- --extern void check_class_changing(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class); --extern void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio); -- - extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); - - #ifdef CONFIG_PREEMPT_RT -@@ -3709,6 +3608,8 @@ static inline bool cpu_busy_with_softirqs(int cpu) - } - #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ - --#include "ext.h" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird.h" -+#endif - - #endif /* _KERNEL_SCHED_SCHED_H */ -diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c -index 9c556c716382..7a26c09182e8 100644 ---- a/kernel/time/tick-sched.c -+++ b/kernel/time/tick-sched.c -@@ -34,6 +34,10 @@ - - #include - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "../sched/hmbird/hmbird_shadow_tick.h" -+#endif -+ - /* - * Per-CPU nohz control structure - */ -@@ -1103,6 +1107,9 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) - return true; - } - -+#ifdef CONFIG_HMBIRD_SCHED -+extern void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+#endif - /** - * tick_nohz_idle_stop_tick - stop the idle tick from the idle task - * -@@ -1115,7 +1122,9 @@ void tick_nohz_idle_stop_tick(void) - ktime_t expires; - - trace_android_vh_tick_nohz_idle_stop_tick(NULL); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL); -+#endif - /* - * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the - * tick timer expiration time is known already. -diff --git a/tools/sched_ext/.gitignore b/tools/sched_ext/.gitignore -deleted file mode 100644 -index a3240f9f7eba..000000000000 ---- a/tools/sched_ext/.gitignore -+++ /dev/null -@@ -1,9 +0,0 @@ --scx_example_simple --scx_example_qmap --scx_example_central --scx_example_pair --scx_example_flatcg --scx_example_userland --*.skel.h --*.subskel.h --/tools/ -diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile -deleted file mode 100644 -index 73c43782837d..000000000000 ---- a/tools/sched_ext/Makefile -+++ /dev/null -@@ -1,216 +0,0 @@ --# SPDX-License-Identifier: GPL-2.0 --# Copyright (c) 2022 Meta Platforms, Inc. and affiliates. --include ../build/Build.include --include ../scripts/Makefile.arch --include ../scripts/Makefile.include -- --ifneq ($(LLVM),) --ifneq ($(filter %/,$(LLVM)),) --LLVM_PREFIX := $(LLVM) --else ifneq ($(filter -%,$(LLVM)),) --LLVM_SUFFIX := $(LLVM) --endif -- --CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi --CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu --CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl --CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu --CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu --CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu --CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu --CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu --CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu --CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) -- --ifeq ($(CROSS_COMPILE),) --ifeq ($(CLANG_TARGET_FLAGS),) --$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk --else --CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) --endif # CLANG_TARGET_FLAGS --else --CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) --endif # CROSS_COMPILE -- --CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as --else --CC := $(CROSS_COMPILE)gcc --endif # LLVM -- --CURDIR := $(abspath .) --TOOLSDIR := $(abspath ..) --LIBDIR := $(TOOLSDIR)/lib --BPFDIR := $(LIBDIR)/bpf --TOOLSINCDIR := $(TOOLSDIR)/include --BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool --APIDIR := $(TOOLSINCDIR)/uapi --GENDIR := $(abspath ../../include/generated) --GENHDR := $(GENDIR)/autoconf.h -- --SCRATCH_DIR := $(CURDIR)/tools --BUILD_DIR := $(SCRATCH_DIR)/build --INCLUDE_DIR := $(SCRATCH_DIR)/include --BPFOBJ_DIR := $(BUILD_DIR)/libbpf --BPFOBJ := $(BPFOBJ_DIR)/libbpf.a --ifneq ($(CROSS_COMPILE),) --HOST_BUILD_DIR := $(BUILD_DIR)/host --HOST_SCRATCH_DIR := host-tools --HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include --else --HOST_BUILD_DIR := $(BUILD_DIR) --HOST_SCRATCH_DIR := $(SCRATCH_DIR) --HOST_INCLUDE_DIR := $(INCLUDE_DIR) --endif --HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a --RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids --DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -- --VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ -- $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ -- ../../vmlinux \ -- /sys/kernel/btf/vmlinux \ -- /boot/vmlinux-$(shell uname -r) --VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) --ifeq ($(VMLINUX_BTF),) --$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") --endif -- --BPFTOOL ?= $(DEFAULT_BPFTOOL) -- --ifneq ($(wildcard $(GENHDR)),) -- GENFLAGS := -DHAVE_GENHDR --endif -- --CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ -- -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -- -I$(TOOLSINCDIR) -I$(APIDIR) -- --CARGOFLAGS := --release -- --# Silence some warnings when compiled with clang --ifneq ($(LLVM),) --CFLAGS += -Wno-unused-command-line-argument --endif -- --LDFLAGS = -lelf -lz -lpthread -- --IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - &1 \ -- | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ --$(shell $(1) -dM -E - $@ --else -- $(call msg,CP,,$@) -- $(Q)cp "$(VMLINUX_H)" $@ --endif -- --%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \ -- | $(BPFOBJ) -- $(call msg,CLNG-BPF,,$@) -- $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ -- --%.skel.h: %.bpf.o $(BPFTOOL) -- $(call msg,GEN-SKEL,,$@) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) -- $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) -- $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@ -- $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h) -- --scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_userland: scx_example_userland.c scx_example_userland.skel.h \ -- scx_example_userland_common.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR) --atropos: export ATROPOS_CLANG = $(CLANG) --atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS) --atropos: $(INCLUDE_DIR)/vmlinux.h -- cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS) -- --clean: -- cargo clean --manifest-path=atropos/Cargo.toml -- rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) -- rm -f *.o *.bpf.o *.skel.h *.subskel.h -- rm -f scx_example_simple scx_example_qmap scx_example_central \ -- scx_example_pair scx_example_flatcg scx_example_userland -- --.PHONY: all atropos clean -- --# delete failed targets --.DELETE_ON_ERROR: -- --# keep intermediate (.skel.h, .bpf.o, etc) targets --.SECONDARY: -diff --git a/tools/sched_ext/atropos/.gitignore b/tools/sched_ext/atropos/.gitignore -deleted file mode 100644 -index 186dba259ec2..000000000000 ---- a/tools/sched_ext/atropos/.gitignore -+++ /dev/null -@@ -1,3 +0,0 @@ --src/bpf/.output --Cargo.lock --target -diff --git a/tools/sched_ext/atropos/Cargo.toml b/tools/sched_ext/atropos/Cargo.toml -deleted file mode 100644 -index 7462a836d53d..000000000000 ---- a/tools/sched_ext/atropos/Cargo.toml -+++ /dev/null -@@ -1,28 +0,0 @@ --[package] --name = "scx_atropos" --version = "0.5.0" --authors = ["Dan Schatzberg ", "Meta"] --edition = "2021" --description = "Userspace scheduling with BPF" --license = "GPL-2.0-only" -- --[dependencies] --anyhow = "1.0.65" --bitvec = { version = "1.0", features = ["serde"] } --clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } --ctrlc = { version = "3.1", features = ["termination"] } --fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"} --hex = "0.4.3" --libbpf-rs = "0.19.1" --libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] } --libc = "0.2.137" --log = "0.4.17" --ordered-float = "3.4.0" --simplelog = "0.12.0" -- --[build-dependencies] --bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false } --libbpf-cargo = "0.13.0" -- --[features] --enable_backtrace = [] -diff --git a/tools/sched_ext/atropos/build.rs b/tools/sched_ext/atropos/build.rs -deleted file mode 100644 -index 26e792c5e17e..000000000000 ---- a/tools/sched_ext/atropos/build.rs -+++ /dev/null -@@ -1,70 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --extern crate bindgen; -- --use std::env; --use std::fs::create_dir_all; --use std::path::Path; --use std::path::PathBuf; -- --use libbpf_cargo::SkeletonBuilder; -- --const HEADER_PATH: &str = "src/bpf/atropos.h"; -- --fn bindgen_atropos() { -- // Tell cargo to invalidate the built crate whenever the wrapper changes -- println!("cargo:rerun-if-changed={}", HEADER_PATH); -- -- // The bindgen::Builder is the main entry point -- // to bindgen, and lets you build up options for -- // the resulting bindings. -- let bindings = bindgen::Builder::default() -- // The input header we would like to generate -- // bindings for. -- .header(HEADER_PATH) -- // Tell cargo to invalidate the built crate whenever any of the -- // included header files changed. -- .parse_callbacks(Box::new(bindgen::CargoCallbacks)) -- // Finish the builder and generate the bindings. -- .generate() -- // Unwrap the Result and panic on failure. -- .expect("Unable to generate bindings"); -- -- // Write the bindings to the $OUT_DIR/bindings.rs file. -- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); -- bindings -- .write_to_file(out_path.join("atropos-sys.rs")) -- .expect("Couldn't write bindings!"); --} -- --fn gen_bpf_sched(name: &str) { -- let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap(); -- let clang = env::var("ATROPOS_CLANG").unwrap(); -- eprintln!("{}", clang); -- let outpath = format!("./src/bpf/.output/{}.skel.rs", name); -- let skel = Path::new(&outpath); -- let src = format!("./src/bpf/{}.bpf.c", name); -- SkeletonBuilder::new() -- .source(src.clone()) -- .clang(clang) -- .clang_args(bpf_cflags) -- .build_and_generate(&skel) -- .unwrap(); -- println!("cargo:rerun-if-changed={}", src); --} -- --fn main() { -- bindgen_atropos(); -- // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton. -- // Reasons are because the generated skeleton contains compiler attributes -- // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]` -- // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside -- // the path attribute either (see https://github.com/rust-lang/rust/pull/83366). -- // -- // However, there is hope! When the above feature stabilizes we can clean this -- // all up. -- create_dir_all("./src/bpf/.output").unwrap(); -- gen_bpf_sched("atropos"); --} -diff --git a/tools/sched_ext/atropos/rustfmt.toml b/tools/sched_ext/atropos/rustfmt.toml -deleted file mode 100644 -index b7258ed0a8d8..000000000000 ---- a/tools/sched_ext/atropos/rustfmt.toml -+++ /dev/null -@@ -1,8 +0,0 @@ --# Get help on options with `rustfmt --help=config` --# Please keep these in alphabetical order. --edition = "2021" --group_imports = "StdExternalCrate" --imports_granularity = "Item" --merge_derives = false --use_field_init_shorthand = true --version = "Two" -diff --git a/tools/sched_ext/atropos/src/atropos_sys.rs b/tools/sched_ext/atropos/src/atropos_sys.rs -deleted file mode 100644 -index bbeaf856d40e..000000000000 ---- a/tools/sched_ext/atropos/src/atropos_sys.rs -+++ /dev/null -@@ -1,10 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#![allow(non_upper_case_globals)] --#![allow(non_camel_case_types)] --#![allow(non_snake_case)] --#![allow(dead_code)] -- --include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs")); -diff --git a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -deleted file mode 100644 -index 3905a403e940..000000000000 ---- a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -+++ /dev/null -@@ -1,743 +0,0 @@ --/* Copyright (c) Meta Platforms, Inc. and affiliates. */ --/* -- * This software may be used and distributed according to the terms of the -- * GNU General Public License version 2. -- * -- * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF -- * part does simple round robin in each domain and the userspace part -- * calculates the load factor of each domain and tells the BPF part how to load -- * balance the domains. -- * -- * Every task has an entry in the task_data map which lists which domain the -- * task belongs to. When a task first enters the system (atropos_prep_enable), -- * they are round-robined to a domain. -- * -- * atropos_select_cpu is the primary scheduling logic, invoked when a task -- * becomes runnable. The lb_data map is populated by userspace to inform the BPF -- * scheduler that a task should be migrated to a new domain. Otherwise, the task -- * is scheduled in priority order as follows: -- * * The current core if the task was woken up synchronously and there are idle -- * cpus in the system -- * * The previous core, if idle -- * * The pinned-to core if the task is pinned to a specific core -- * * Any idle cpu in the domain -- * -- * If none of the above conditions are met, then the task is enqueued to a -- * dispatch queue corresponding to the domain (atropos_enqueue). -- * -- * atropos_dispatch will attempt to consume a task from its domain's -- * corresponding dispatch queue (this occurs after scheduling any tasks directly -- * assigned to it due to the logic in atropos_select_cpu). If no task is found, -- * then greedy load stealing will attempt to find a task on another dispatch -- * queue to run. -- * -- * Load balancing is almost entirely handled by userspace. BPF populates the -- * task weight, dom mask and current dom in the task_data map and executes the -- * load balance based on userspace populating the lb_data map. -- */ --#include "../../../scx_common.bpf.h" --#include "atropos.h" -- --#include --#include --#include --#include --#include --#include -- --char _license[] SEC("license") = "GPL"; -- --/* -- * const volatiles are set during initialization and treated as consts by the -- * jit compiler. -- */ -- --/* -- * Domains and cpus -- */ --const volatile __u32 nr_doms = 32; /* !0 for veristat, set during init */ --const volatile __u32 nr_cpus = 64; /* !0 for veristat, set during init */ --const volatile __u32 cpu_dom_id_map[MAX_CPUS]; --const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; -- --const volatile bool kthreads_local; --const volatile bool fifo_sched; --const volatile bool switch_partial; --const volatile __u32 greedy_threshold; -- --/* base slice duration */ --const volatile __u64 slice_us = 20000; -- --/* -- * Exit info -- */ --int exit_type = SCX_EXIT_NONE; --char exit_msg[SCX_EXIT_MSG_LEN]; -- --struct pcpu_ctx { -- __u32 dom_rr_cur; /* used when scanning other doms */ -- -- /* libbpf-rs does not respect the alignment, so pad out the struct explicitly */ -- __u8 _padding[CACHELINE_SIZE - sizeof(u64)]; --} __attribute__((aligned(CACHELINE_SIZE))); -- --struct pcpu_ctx pcpu_ctx[MAX_CPUS]; -- --/* -- * Domain context -- */ --struct dom_ctx { -- struct bpf_cpumask __kptr *cpumask; -- u64 vtime_now; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __type(key, u32); -- __type(value, struct dom_ctx); -- __uint(max_entries, MAX_DOMS); -- __uint(map_flags, 0); --} dom_ctx SEC(".maps"); -- --/* -- * Statistics -- */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, ATROPOS_NR_STATS); --} stats SEC(".maps"); -- --static inline void stat_add(enum stat_idx idx, u64 addend) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p) += addend; --} -- --/* Map pid -> task_ctx */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, struct task_ctx); -- __uint(max_entries, 1000000); -- __uint(map_flags, 0); --} task_data SEC(".maps"); -- --/* -- * This is populated from userspace to indicate which pids should be reassigned -- * to new doms. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, u32); -- __uint(max_entries, 1000); -- __uint(map_flags, 0); --} lb_data SEC(".maps"); -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p, -- u32 new_dom_id) --{ -- struct dom_ctx *old_domc, *new_domc; -- struct bpf_cpumask *d_cpumask, *t_cpumask; -- u32 old_dom_id = task_ctx->dom_id; -- s64 vtime_delta; -- -- old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id); -- if (!old_domc) { -- scx_bpf_error("No dom%u", old_dom_id); -- return false; -- } -- -- vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now; -- -- new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id); -- if (!new_domc) { -- scx_bpf_error("No dom%u", new_dom_id); -- return false; -- } -- -- d_cpumask = new_domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to get domain %u cpumask kptr", -- new_dom_id); -- return false; -- } -- -- t_cpumask = task_ctx->cpumask; -- if (!t_cpumask) { -- scx_bpf_error("Failed to look up task cpumask"); -- return false; -- } -- -- /* -- * set_cpumask might have happened between userspace requesting LB and -- * here and @p might not be able to run in @dom_id anymore. Verify. -- */ -- if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- p->cpus_ptr)) { -- p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta; -- task_ctx->dom_id = new_dom_id; -- bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask, -- p->cpus_ptr); -- } -- -- return task_ctx->dom_id == new_dom_id; --} -- --s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu, -- u32 wake_flags) --{ -- s32 cpu; -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- struct bpf_cpumask *p_cpumask; -- -- if (!task_ctx) -- return -ENOENT; -- -- if (kthreads_local && -- (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- cpu = prev_cpu; -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local dsq of the waker. -- */ -- if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) { -- struct task_struct *current = (void *)bpf_get_current_task(); -- -- if (!(BPF_CORE_READ(current, flags) & PF_EXITING) && -- task_ctx->dom_id < MAX_DOMS) { -- struct dom_ctx *domc; -- struct bpf_cpumask *d_cpumask; -- const struct cpumask *idle_cpumask; -- bool has_idle; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id); -- if (!domc) { -- scx_bpf_error("Failed to find dom%u", -- task_ctx->dom_id); -- return prev_cpu; -- } -- d_cpumask = domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to acquire domain %u cpumask kptr", -- task_ctx->dom_id); -- return prev_cpu; -- } -- -- idle_cpumask = scx_bpf_get_idle_cpumask(); -- -- has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- idle_cpumask); -- -- scx_bpf_put_idle_cpumask(idle_cpumask); -- -- if (has_idle) { -- cpu = bpf_get_smp_processor_id(); -- if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- stat_add(ATROPOS_STAT_WAKE_SYNC, 1); -- goto local; -- } -- } -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- stat_add(ATROPOS_STAT_PREV_IDLE, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- /* If only one core is allowed, dispatch */ -- if (p->nr_cpus_allowed == 1) { -- stat_add(ATROPOS_STAT_PINNED, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) -- return -ENOENT; -- -- /* If there is an eligible idle CPU, dispatch directly */ -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- if (cpu >= 0) { -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * @prev_cpu may be in a different domain. Returning an out-of-domain -- * CPU can lead to stalls as all in-domain CPUs may be idle by the time -- * @p gets enqueued. -- */ -- if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask)) -- cpu = prev_cpu; -- else -- cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask); -- -- return cpu; -- --local: -- task_ctx->dispatch_local = true; -- return cpu; --} -- --void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- u32 *new_dom; -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- new_dom = bpf_map_lookup_elem(&lb_data, &pid); -- if (new_dom && *new_dom != task_ctx->dom_id && -- task_set_dsq(task_ctx, p, *new_dom)) { -- struct bpf_cpumask *p_cpumask; -- s32 cpu; -- -- stat_add(ATROPOS_STAT_LOAD_BALANCE, 1); -- -- /* -- * If dispatch_local is set, We own @p's idle state but we are -- * not gonna put the task in the associated local dsq which can -- * cause the CPU to stall. Kick it. -- */ -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0); -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) { -- scx_bpf_error("Failed to get task_ctx->cpumask"); -- return; -- } -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- } -- -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags); -- return; -- } -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000, -- enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- u32 dom_id = task_ctx->dom_id; -- struct dom_ctx *domc; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, domc->vtime_now - slice_us * 1000)) -- vtime = domc->vtime_now - slice_us * 1000; -- -- scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --static u32 cpu_to_dom_id(s32 cpu) --{ -- const volatile u32 *dom_idp; -- -- if (nr_doms <= 1) -- return 0; -- -- dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]); -- if (!dom_idp) -- return MAX_DOMS; -- -- return *dom_idp; --} -- --static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) --{ -- s32 cpu; -- -- if (dom_id >= MAX_DOMS) -- return false; -- -- bpf_for(cpu, 0, nr_cpus) { -- if (bpf_cpumask_test_cpu(cpu, cpumask) && -- (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64)))) -- return true; -- } -- return false; --} -- --static u32 dom_rr_next(s32 cpu) --{ -- struct pcpu_ctx *pcpuc; -- u32 dom_id; -- -- pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]); -- if (!pcpuc) -- return 0; -- -- dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms; -- -- if (dom_id == cpu_to_dom_id(cpu)) -- dom_id = (dom_id + 1) % nr_doms; -- -- pcpuc->dom_rr_cur = dom_id; -- return dom_id; --} -- --void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 dom = cpu_to_dom_id(cpu); -- -- if (scx_bpf_consume(dom)) { -- stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1); -- return; -- } -- -- if (!greedy_threshold) -- return; -- -- bpf_repeat(nr_doms - 1) { -- u32 dom_id = dom_rr_next(cpu); -- -- if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold && -- scx_bpf_consume(dom_id)) { -- stat_add(ATROPOS_STAT_GREEDY, 1); -- break; -- } -- } --} -- --void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_at = bpf_ktime_get_ns(); --} -- --void BPF_STRUCT_OPS(atropos_running, struct task_struct *p) --{ -- struct task_ctx *taskc; -- struct dom_ctx *domc; -- pid_t pid = p->pid; -- u32 dom_id; -- -- if (fifo_sched) -- return; -- -- taskc = bpf_map_lookup_elem(&task_data, &pid); -- if (!taskc) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- dom_id = taskc->dom_id; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(domc->vtime_now, p->scx.dsq_vtime)) -- domc->vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at; -- task_ctx->runnable_at = 0; --} -- --void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->weight = weight; --} -- --struct pick_task_domain_loop_ctx { -- struct task_struct *p; -- const struct cpumask *cpumask; -- u64 dom_mask; -- u32 dom_rr_base; -- u32 dom_id; --}; -- --static int pick_task_domain_loopfn(u32 idx, void *data) --{ -- struct pick_task_domain_loop_ctx *lctx = data; -- u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms; -- -- if (dom_id >= MAX_DOMS) -- return 1; -- -- if (cpumask_intersects_domain(lctx->cpumask, dom_id)) { -- lctx->dom_mask |= 1LLU << dom_id; -- if (lctx->dom_id == MAX_DOMS) -- lctx->dom_id = dom_id; -- } -- return 0; --} -- --static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- struct pick_task_domain_loop_ctx lctx = { -- .p = p, -- .cpumask = cpumask, -- .dom_id = MAX_DOMS, -- }; -- s32 cpu = bpf_get_smp_processor_id(); -- -- if (cpu < 0 || cpu >= MAX_CPUS) -- return MAX_DOMS; -- -- lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur); -- -- bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0); -- task_ctx->dom_mask = lctx.dom_mask; -- -- return lctx.dom_id; --} -- --static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- u32 dom_id = 0; -- -- if (nr_doms > 1) -- dom_id = pick_task_domain(task_ctx, p, cpumask); -- -- if (!task_set_dsq(task_ctx, p, dom_id)) -- scx_bpf_error("Failed to set domain %d for %s[%d]", -- dom_id, p->comm, p->pid); --} -- --void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_set_domain(task_ctx, p, cpumask); --} -- --s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct bpf_cpumask *cpumask; -- struct task_ctx task_ctx, *map_value; -- long ret; -- pid_t pid; -- -- memset(&task_ctx, 0, sizeof(task_ctx)); -- -- pid = p->pid; -- ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return ret; -- } -- -- /* -- * Read the entry from the map immediately so we can add the cpumask -- * with bpf_kptr_xchg(). -- */ -- map_value = bpf_map_lookup_elem(&task_data, &pid); -- if (!map_value) -- /* Should never happen -- it was just inserted above. */ -- return -EINVAL; -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- bpf_map_delete_elem(&task_data, &pid); -- return -ENOMEM; -- } -- -- cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask); -- if (cpumask) { -- /* Should never happen as we just inserted it above. */ -- bpf_cpumask_release(cpumask); -- bpf_map_delete_elem(&task_data, &pid); -- return -EINVAL; -- } -- -- task_set_domain(map_value, p, p->cpus_ptr); -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p) --{ -- pid_t pid = p->pid; -- long ret = bpf_map_delete_elem(&task_data, &pid); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return; -- } --} -- --static int create_dom_dsq(u32 idx, void *data) --{ -- struct dom_ctx domc_init = {}, *domc; -- struct bpf_cpumask *cpumask; -- u32 cpu, dom_id = idx; -- s32 ret; -- -- ret = scx_bpf_create_dsq(dom_id, -1); -- if (ret < 0) { -- scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret); -- return 1; -- } -- -- ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0); -- if (ret) { -- scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret); -- return 1; -- } -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- /* Should never happen, we just inserted it above. */ -- scx_bpf_error("No dom%u", dom_id); -- return 1; -- } -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id); -- return 1; -- } -- -- for (cpu = 0; cpu < MAX_CPUS; cpu++) { -- const volatile __u64 *dmask; -- -- dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]); -- if (!dmask) { -- scx_bpf_error("array index error"); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- if (*dmask & (1LLU << (cpu % 64))) -- bpf_cpumask_set_cpu(cpu, cpumask); -- } -- -- cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask); -- if (cpumask) { -- scx_bpf_error("Domain %u was already present", dom_id); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(atropos_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- bpf_loop(nr_doms, create_dom_dsq, NULL, 0); -- -- for (u32 i = 0; i < nr_cpus; i++) -- pcpu_ctx[i].dom_rr_cur = i; -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg); -- exit_type = ei->type; --} -- --SEC(".struct_ops") --struct sched_ext_ops atropos = { -- .select_cpu = (void *)atropos_select_cpu, -- .enqueue = (void *)atropos_enqueue, -- .dispatch = (void *)atropos_dispatch, -- .runnable = (void *)atropos_runnable, -- .running = (void *)atropos_running, -- .stopping = (void *)atropos_stopping, -- .quiescent = (void *)atropos_quiescent, -- .set_weight = (void *)atropos_set_weight, -- .set_cpumask = (void *)atropos_set_cpumask, -- .prep_enable = (void *)atropos_prep_enable, -- .disable = (void *)atropos_disable, -- .init = (void *)atropos_init, -- .exit = (void *)atropos_exit, -- .flags = 0, -- .name = "atropos", --}; -diff --git a/tools/sched_ext/atropos/src/bpf/atropos.h b/tools/sched_ext/atropos/src/bpf/atropos.h -deleted file mode 100644 -index addf29ca104a..000000000000 ---- a/tools/sched_ext/atropos/src/bpf/atropos.h -+++ /dev/null -@@ -1,44 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#ifndef __ATROPOS_H --#define __ATROPOS_H -- --#include --#ifndef __kptr --#ifdef __KERNEL__ --#error "__kptr_ref not defined in the kernel" --#endif --#define __kptr --#endif -- --#define MAX_CPUS 512 --#define MAX_DOMS 64 /* limited to avoid complex bitmask ops */ --#define CACHELINE_SIZE 64 -- --/* Statistics */ --enum stat_idx { -- ATROPOS_STAT_TASK_GET_ERR, -- ATROPOS_STAT_WAKE_SYNC, -- ATROPOS_STAT_PREV_IDLE, -- ATROPOS_STAT_PINNED, -- ATROPOS_STAT_DIRECT_DISPATCH, -- ATROPOS_STAT_DSQ_DISPATCH, -- ATROPOS_STAT_GREEDY, -- ATROPOS_STAT_LOAD_BALANCE, -- ATROPOS_STAT_LAST_TASK, -- ATROPOS_NR_STATS, --}; -- --struct task_ctx { -- unsigned long long dom_mask; /* the domains this task can run on */ -- struct bpf_cpumask __kptr *cpumask; -- unsigned int dom_id; -- unsigned int weight; -- unsigned long long runnable_at; -- unsigned long long runnable_for; -- bool dispatch_local; --}; -- --#endif /* __ATROPOS_H */ -diff --git a/tools/sched_ext/atropos/src/main.rs b/tools/sched_ext/atropos/src/main.rs -deleted file mode 100644 -index 0d313662f713..000000000000 ---- a/tools/sched_ext/atropos/src/main.rs -+++ /dev/null -@@ -1,942 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#[path = "bpf/.output/atropos.skel.rs"] --mod atropos; --pub use atropos::*; --pub mod atropos_sys; -- --use std::cell::Cell; --use std::collections::{BTreeMap, BTreeSet}; --use std::ffi::CStr; --use std::ops::Bound::{Included, Unbounded}; --use std::sync::atomic::{AtomicBool, Ordering}; --use std::sync::Arc; --use std::time::{Duration, SystemTime}; -- --use ::fb_procfs as procfs; --use anyhow::{anyhow, bail, Context, Result}; --use bitvec::prelude::*; --use clap::Parser; --use log::{info, trace, warn}; --use ordered_float::OrderedFloat; -- --/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF --/// part does simple round robin in each domain and the userspace part --/// calculates the load factor of each domain and tells the BPF part how to load --/// balance the domains. -- --/// This scheduler demonstrates dividing scheduling logic between BPF and --/// userspace and using rust to build the userspace part. An earlier variant of --/// this scheduler was used to balance across six domains, each representing a --/// chiplet in a six-chiplet AMD processor, and could match the performance of --/// production setup using CFS. --#[derive(Debug, Parser)] --struct Opts { -- /// Scheduling slice duration in microseconds. -- #[clap(short, long, default_value = "20000")] -- slice_us: u64, -- -- /// Monitoring and load balance interval in seconds. -- #[clap(short, long, default_value = "2.0")] -- interval: f64, -- -- /// Build domains according to how CPUs are grouped at this cache level -- /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id. -- #[clap(short = 'c', long, default_value = "3")] -- cache_level: u32, -- -- /// Instead of using cache locality, set the cpumask for each domain -- /// manually, provide multiple --cpumasks, one for each domain. E.g. -- /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with -- /// the corresponding CPUs belonging to each domain. Each CPU must -- /// belong to precisely one domain. -- #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")] -- cpumasks: Vec, -- -- /// When non-zero, enable greedy task stealing. When a domain is idle, a -- /// cpu will attempt to steal tasks from a domain with at least -- /// greedy_threshold tasks enqueued. These tasks aren't permanently -- /// stolen from the domain. -- #[clap(short, long, default_value = "4")] -- greedy_threshold: u32, -- -- /// The load decay factor. Every interval, the existing load is decayed -- /// by this factor and new load is added. Must be in the range [0.0, -- /// 0.99]. The smaller the value, the more sensitive load calculation -- /// is to recent changes. When 0.0, history is ignored and the load -- /// value from the latest period is used directly. -- #[clap(short, long, default_value = "0.5")] -- load_decay_factor: f64, -- -- /// Disable load balancing. Unless disabled, periodically userspace will -- /// calculate the load factor of each domain and instruct BPF which -- /// processes to move. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- no_load_balance: bool, -- -- /// Put per-cpu kthreads directly into local dsq's. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- kthreads_local: bool, -- -- /// Use FIFO scheduling instead of weighted vtime scheduling. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- fifo_sched: bool, -- -- /// If specified, only tasks which have their scheduling policy set to -- /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all -- /// tasks are switched. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- partial: bool, -- -- /// Enable verbose output including libbpf details. Specify multiple -- /// times to increase verbosity. -- #[clap(short, long, action = clap::ArgAction::Count)] -- verbose: u8, --} -- --fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result { -- Ok(reader -- .read_stat() -- .context("Failed to read procfs")? -- .total_cpu -- .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?) --} -- --fn now_monotonic() -> u64 { -- let mut time = libc::timespec { -- tv_sec: 0, -- tv_nsec: 0, -- }; -- let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) }; -- assert!(ret == 0); -- time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 --} -- --fn clear_map(map: &mut libbpf_rs::Map) { -- // XXX: libbpf_rs has some design flaw that make it impossible to -- // delete while iterating despite it being safe so we alias it here -- let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) }; -- for key in map.keys() { -- let _ = deleter.delete(&key); -- } --} -- --#[derive(Debug)] --struct TaskLoad { -- runnable_for: u64, -- load: f64, --} -- --#[derive(Debug)] --struct TaskInfo { -- pid: i32, -- dom_mask: u64, -- migrated: Cell, --} -- --struct LoadBalancer<'a, 'b, 'c> { -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- -- tasks_by_load: Vec, TaskInfo>>, -- load_avg: f64, -- dom_loads: Vec, -- -- imbal: Vec, -- doms_to_push: BTreeMap, u32>, -- doms_to_pull: BTreeMap, u32>, -- -- nr_lb_data_errors: &'c mut u64, --} -- --impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> { -- const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10; -- const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1; -- const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50; -- -- fn new( -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- nr_lb_data_errors: &'c mut u64, -- ) -> Self { -- Self { -- maps, -- task_loads, -- nr_doms, -- load_decay_factor, -- -- tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(), -- load_avg: 0f64, -- dom_loads: vec![0.0; nr_doms], -- -- imbal: vec![0.0; nr_doms], -- doms_to_pull: BTreeMap::new(), -- doms_to_push: BTreeMap::new(), -- -- nr_lb_data_errors, -- } -- } -- -- fn read_task_loads(&mut self, period: Duration) -> Result<()> { -- let now_mono = now_monotonic(); -- let task_data = self.maps.task_data(); -- let mut this_task_loads = BTreeMap::::new(); -- let mut load_sum = 0.0f64; -- self.dom_loads = vec![0f64; self.nr_doms]; -- -- for key in task_data.keys() { -- if let Some(task_ctx_vec) = task_data -- .lookup(&key, libbpf_rs::MapFlags::ANY) -- .context("Failed to lookup task_data")? -- { -- let task_ctx = -- unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) }; -- let pid = i32::from_ne_bytes( -- key.as_slice() -- .try_into() -- .context("Invalid key length in task_data map")?, -- ); -- -- let (this_at, this_for, weight) = unsafe { -- ( -- std::ptr::read_volatile(&task_ctx.runnable_at as *const u64), -- std::ptr::read_volatile(&task_ctx.runnable_for as *const u64), -- std::ptr::read_volatile(&task_ctx.weight as *const u32), -- ) -- }; -- -- let (mut delta, prev_load) = match self.task_loads.get(&pid) { -- Some(prev) => (this_for - prev.runnable_for, Some(prev.load)), -- None => (this_for, None), -- }; -- -- // Non-zero this_at indicates that the task is currently -- // runnable. Note that we read runnable_at and runnable_for -- // without any synchronization and there is a small window -- // where we end up misaccounting. While this can cause -- // temporary error, it's unlikely to cause any noticeable -- // misbehavior especially given the load value clamping. -- if this_at > 0 && this_at < now_mono { -- delta += now_mono - this_at; -- } -- -- delta = delta.min(period.as_nanos() as u64); -- let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64) -- .clamp(0.0, weight as f64); -- -- let this_load = match prev_load { -- Some(prev_load) => { -- prev_load * self.load_decay_factor -- + this_load * (1.0 - self.load_decay_factor) -- } -- None => this_load, -- }; -- -- this_task_loads.insert( -- pid, -- TaskLoad { -- runnable_for: this_for, -- load: this_load, -- }, -- ); -- -- load_sum += this_load; -- self.dom_loads[task_ctx.dom_id as usize] += this_load; -- // Only record pids that are eligible for load balancing -- if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) { -- continue; -- } -- self.tasks_by_load[task_ctx.dom_id as usize].insert( -- OrderedFloat(this_load), -- TaskInfo { -- pid, -- dom_mask: task_ctx.dom_mask, -- migrated: Cell::new(false), -- }, -- ); -- } -- } -- -- self.load_avg = load_sum / self.nr_doms as f64; -- *self.task_loads = this_task_loads; -- Ok(()) -- } -- -- // To balance dom loads we identify doms with lower and higher load than average -- fn calculate_dom_load_balance(&mut self) -> Result<()> { -- for (dom, dom_load) in self.dom_loads.iter().enumerate() { -- let imbal = dom_load - self.load_avg; -- if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO { -- if imbal > 0f64 { -- self.doms_to_push.insert(OrderedFloat(imbal), dom as u32); -- } else { -- self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32); -- } -- self.imbal[dom] = imbal; -- } -- } -- Ok(()) -- } -- -- // Find the first candidate pid which hasn't already been migrated and -- // can run in @pull_dom. -- fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)> -- where -- I: IntoIterator, &'d TaskInfo)>, -- { -- match tasks_by_load -- .into_iter() -- .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0) -- .next() -- { -- Some((OrderedFloat(load), task)) => Some((*load, task)), -- None => None, -- } -- } -- -- fn pick_victim( -- &self, -- (push_dom, to_push): (u32, f64), -- (pull_dom, to_pull): (u32, f64), -- ) -> Option<(&TaskInfo, f64)> { -- let to_xfer = to_pull.min(to_push); -- -- trace!( -- "considering dom {}@{:.2} -> {}@{:.2}", -- push_dom, -- to_push, -- pull_dom, -- to_pull -- ); -- -- let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs(); -- -- trace!( -- "to_xfer={:.2} tasks_by_load={:?}", -- to_xfer, -- &self.tasks_by_load[push_dom as usize] -- ); -- -- // We want to pick a task to transfer from push_dom to pull_dom to -- // maximize the reduction of load imbalance between the two. IOW, -- // pick a task which has the closest load value to $to_xfer that can -- // be migrated. Find such task by locating the first migratable task -- // while scanning left from $to_xfer and the counterpart while -- // scanning right and picking the better of the two. -- let (load, task, new_imbal) = match ( -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Unbounded, Included(&OrderedFloat(to_xfer)))) -- .rev(), -- pull_dom, -- ), -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Included(&OrderedFloat(to_xfer)), Unbounded)), -- pull_dom, -- ), -- ) { -- (None, None) => return None, -- (Some((load, task)), None) | (None, Some((load, task))) => { -- (load, task, calc_new_imbal(load)) -- } -- (Some((load0, task0)), Some((load1, task1))) => { -- let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1)); -- if new_imbal0 <= new_imbal1 { -- (load0, task0, new_imbal0) -- } else { -- (load1, task1, new_imbal1) -- } -- } -- }; -- -- // If the best candidate can't reduce the imbalance, there's nothing -- // to do for this pair. -- let old_imbal = to_push + to_pull; -- if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal { -- trace!( -- "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal -- ); -- return None; -- } -- -- trace!( -- "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal, -- ); -- -- Some((task, load)) -- } -- -- // Actually execute the load balancing. Concretely this writes pid -> dom -- // entries into the lb_data map for bpf side to consume. -- fn load_balance(&mut self) -> Result<()> { -- clear_map(self.maps.lb_data()); -- -- trace!("imbal={:?}", &self.imbal); -- trace!("doms_to_push={:?}", &self.doms_to_push); -- trace!("doms_to_pull={:?}", &self.doms_to_pull); -- -- // Push from the most imbalanced to least. -- while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() { -- let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO; -- let mut pushed = 0f64; -- -- // Transfer tasks from push_dom to reduce imbalance. -- loop { -- let last_pushed = pushed; -- -- // Pull from the most imbalaned to least. -- let mut doms_to_pull = BTreeMap::<_, _>::new(); -- std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull); -- let mut pull_doms = doms_to_pull.into_iter().rev().collect::>(); -- -- for (to_pull, pull_dom) in pull_doms.iter_mut() { -- if let Some((task, load)) = -- self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull))) -- { -- // Execute migration. -- task.migrated.set(true); -- to_push -= load; -- *to_pull -= load; -- pushed += load; -- -- // Ask BPF code to execute the migration. -- let pid = task.pid; -- let cpid = (pid as libc::pid_t).to_ne_bytes(); -- if let Err(e) = self.maps.lb_data().update( -- &cpid, -- &pull_dom.to_ne_bytes(), -- libbpf_rs::MapFlags::NO_EXIST, -- ) { -- warn!( -- "Failed to update lb_data map for pid={} error={:?}", -- pid, &e -- ); -- *self.nr_lb_data_errors += 1; -- } -- -- // Always break after a successful migration so that -- // the pulling domains are always considered in the -- // descending imbalance order. -- break; -- } -- } -- -- pull_doms -- .into_iter() -- .map(|(k, v)| self.doms_to_pull.insert(k, v)) -- .count(); -- -- // Stop repeating if nothing got transferred or pushed enough. -- if pushed == last_pushed || pushed >= push_max { -- break; -- } -- } -- } -- Ok(()) -- } --} -- --struct Scheduler<'a> { -- skel: AtroposSkel<'a>, -- struct_ops: Option, -- -- nr_cpus: usize, -- nr_doms: usize, -- load_decay_factor: f64, -- balance_load: bool, -- -- proc_reader: procfs::ProcReader, -- -- prev_at: SystemTime, -- prev_total_cpu: procfs::CpuStat, -- task_loads: BTreeMap, -- -- nr_lb_data_errors: u64, --} -- --impl<'a> Scheduler<'a> { -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn parse_cpusets( -- cpumasks: &[String], -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- if cpumasks.len() > atropos_sys::MAX_DOMS as usize { -- bail!( -- "Number of requested DSQs ({}) is greater than MAX_DOMS ({})", -- cpumasks.len(), -- atropos_sys::MAX_DOMS -- ); -- } -- let mut cpus = vec![-1i32; nr_cpus]; -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()]; -- for (dq, cpumask) in cpumasks.iter().enumerate() { -- let hex_str = { -- let mut tmp_str = cpumask -- .strip_prefix("0x") -- .unwrap_or(cpumask) -- .replace('_', ""); -- if tmp_str.len() % 2 != 0 { -- tmp_str = "0".to_string() + &tmp_str; -- } -- tmp_str -- }; -- let byte_vec = hex::decode(&hex_str) -- .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?; -- -- for (index, &val) in byte_vec.iter().rev().enumerate() { -- let mut v = val; -- while v != 0 { -- let lsb = v.trailing_zeros() as usize; -- v &= !(1 << lsb); -- let cpu = index * 8 + lsb; -- if cpu > nr_cpus { -- bail!( -- concat!( -- "Found cpu ({}) in cpumask ({}) which is larger", -- " than the number of cpus on the machine ({})" -- ), -- cpu, -- cpumask, -- nr_cpus -- ); -- } -- if cpus[cpu] != -1 { -- bail!( -- "Found cpu ({}) with dq ({}) but also in cpumask ({})", -- cpu, -- cpus[cpu], -- cpumask -- ); -- } -- cpus[cpu] = dq as i32; -- cpusets[dq].set(cpu, true); -- } -- } -- cpusets[dq].set_uninitialized(false); -- } -- -- for (cpu, &dq) in cpus.iter().enumerate() { -- if dq < 0 { -- bail!( -- "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.", -- cpu -- ); -- } -- } -- -- Ok((cpusets, cpus)) -- } -- -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn cpusets_from_cache( -- level: u32, -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- let mut cpu_to_cache = vec![]; // (cpu_id, cache_id) -- let mut cache_ids = BTreeSet::::new(); -- let mut nr_not_found = 0; -- -- // Build cpu -> cache ID mapping. -- for cpu in 0..nr_cpus { -- let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level); -- let id = match std::fs::read_to_string(&path) { -- Ok(val) => val -- .trim() -- .parse::() -- .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?, -- Err(e) if e.kind() == std::io::ErrorKind::NotFound => { -- nr_not_found += 1; -- 0 -- } -- Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)), -- }; -- -- cpu_to_cache.push(id); -- cache_ids.insert(id); -- } -- -- if nr_not_found > 1 { -- warn!( -- "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0", -- level, nr_not_found, nr_cpus -- ); -- } -- -- // Cache IDs may have holes. Assign consecutive domain IDs to -- // existing cache IDs. -- let mut cache_to_dom = BTreeMap::::new(); -- let mut nr_doms = 0; -- for cache_id in cache_ids.iter() { -- cache_to_dom.insert(*cache_id, nr_doms); -- nr_doms += 1; -- } -- -- if nr_doms > atropos_sys::MAX_DOMS { -- bail!( -- "Total number of doms {} is greater than MAX_DOMS ({})", -- nr_doms, -- atropos_sys::MAX_DOMS -- ); -- } -- -- // Build and return dom -> cpumask and cpu -> dom mappings. -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize]; -- let mut cpu_to_dom = vec![]; -- -- for cpu in 0..nr_cpus { -- let dom_id = cache_to_dom[&cpu_to_cache[cpu]]; -- cpusets[dom_id as usize].set(cpu, true); -- cpu_to_dom.push(dom_id as i32); -- } -- -- Ok((cpusets, cpu_to_dom)) -- } -- -- fn init(opts: &Opts) -> Result { -- // Open the BPF prog first for verification. -- let mut skel_builder = AtroposSkelBuilder::default(); -- skel_builder.obj_builder.debug(opts.verbose > 0); -- let mut skel = skel_builder.open().context("Failed to open BPF program")?; -- -- let nr_cpus = libbpf_rs::num_possible_cpus().unwrap(); -- if nr_cpus > atropos_sys::MAX_CPUS as usize { -- bail!( -- "nr_cpus ({}) is greater than MAX_CPUS ({})", -- nr_cpus, -- atropos_sys::MAX_CPUS -- ); -- } -- -- // Initialize skel according to @opts. -- let (cpusets, cpus) = if opts.cpumasks.len() > 0 { -- Self::parse_cpusets(&opts.cpumasks, nr_cpus)? -- } else { -- Self::cpusets_from_cache(opts.cache_level, nr_cpus)? -- }; -- let nr_doms = cpusets.len(); -- skel.rodata().nr_doms = nr_doms as u32; -- skel.rodata().nr_cpus = nr_cpus as u32; -- -- for (cpu, dom) in cpus.iter().enumerate() { -- skel.rodata().cpu_dom_id_map[cpu] = *dom as u32; -- } -- -- for (dom, cpuset) in cpusets.iter().enumerate() { -- let raw_cpuset_slice = cpuset.as_raw_slice(); -- let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom]; -- let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len()); -- left.clone_from_slice(cpuset.as_raw_slice()); -- let cpumask_str = dom_cpumask_slice -- .iter() -- .take((nr_cpus + 63) / 64) -- .rev() -- .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x)); -- info!( -- "DOM[{:02}] cpumask{} ({} cpus)", -- dom, -- &cpumask_str, -- cpuset.count_ones() -- ); -- } -- -- skel.rodata().slice_us = opts.slice_us; -- skel.rodata().kthreads_local = opts.kthreads_local; -- skel.rodata().fifo_sched = opts.fifo_sched; -- skel.rodata().switch_partial = opts.partial; -- skel.rodata().greedy_threshold = opts.greedy_threshold; -- -- // Attach. -- let mut skel = skel.load().context("Failed to load BPF program")?; -- skel.attach().context("Failed to attach BPF program")?; -- let struct_ops = Some( -- skel.maps_mut() -- .atropos() -- .attach_struct_ops() -- .context("Failed to attach atropos struct ops")?, -- ); -- info!("Atropos Scheduler Attached"); -- -- // Other stuff. -- let mut proc_reader = procfs::ProcReader::new(); -- let prev_total_cpu = read_total_cpu(&mut proc_reader)?; -- -- Ok(Self { -- skel, -- struct_ops, // should be held to keep it attached -- -- nr_cpus, -- nr_doms, -- load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99), -- balance_load: !opts.no_load_balance, -- -- proc_reader, -- -- prev_at: SystemTime::now(), -- prev_total_cpu, -- task_loads: BTreeMap::new(), -- -- nr_lb_data_errors: 0, -- }) -- } -- -- fn get_cpu_busy(&mut self) -> Result { -- let total_cpu = read_total_cpu(&mut self.proc_reader)?; -- let busy = match (&self.prev_total_cpu, &total_cpu) { -- ( -- procfs::CpuStat { -- user_usec: Some(prev_user), -- nice_usec: Some(prev_nice), -- system_usec: Some(prev_system), -- idle_usec: Some(prev_idle), -- iowait_usec: Some(prev_iowait), -- irq_usec: Some(prev_irq), -- softirq_usec: Some(prev_softirq), -- stolen_usec: Some(prev_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- procfs::CpuStat { -- user_usec: Some(curr_user), -- nice_usec: Some(curr_nice), -- system_usec: Some(curr_system), -- idle_usec: Some(curr_idle), -- iowait_usec: Some(curr_iowait), -- irq_usec: Some(curr_irq), -- softirq_usec: Some(curr_softirq), -- stolen_usec: Some(curr_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- ) => { -- let idle_usec = curr_idle - prev_idle; -- let iowait_usec = curr_iowait - prev_iowait; -- let user_usec = curr_user - prev_user; -- let system_usec = curr_system - prev_system; -- let nice_usec = curr_nice - prev_nice; -- let irq_usec = curr_irq - prev_irq; -- let softirq_usec = curr_softirq - prev_softirq; -- let stolen_usec = curr_stolen - prev_stolen; -- -- let busy_usec = -- user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec; -- let total_usec = idle_usec + busy_usec + iowait_usec; -- busy_usec as f64 / total_usec as f64 -- } -- _ => { -- bail!("Some procfs stats are not populated!"); -- } -- }; -- -- self.prev_total_cpu = total_cpu; -- Ok(busy) -- } -- -- fn read_bpf_stats(&mut self) -> Result> { -- let mut maps = self.skel.maps_mut(); -- let stats_map = maps.stats(); -- let mut stats: Vec = Vec::new(); -- let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus]; -- -- for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS { -- let cpu_stat_vec = stats_map -- .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY) -- .with_context(|| format!("Failed to lookup stat {}", stat))? -- .expect("per-cpu stat should exist"); -- let sum = cpu_stat_vec -- .iter() -- .map(|val| { -- u64::from_ne_bytes( -- val.as_slice() -- .try_into() -- .expect("Invalid value length in stat map"), -- ) -- }) -- .sum(); -- stats_map -- .update_percpu( -- &(stat as u32).to_ne_bytes(), -- &zero_vec, -- libbpf_rs::MapFlags::ANY, -- ) -- .context("Failed to zero stat")?; -- stats.push(sum); -- } -- Ok(stats) -- } -- -- fn report( -- &self, -- stats: &Vec, -- cpu_busy: f64, -- processing_dur: Duration, -- load_avg: f64, -- dom_loads: &Vec, -- imbal: &Vec, -- ) { -- let stat = |idx| stats[idx as usize]; -- let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK); -- -- info!( -- "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms", -- cpu_busy * 100.0, -- load_avg, -- stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize], -- stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize], -- self.nr_lb_data_errors, -- processing_dur.as_millis(), -- ); -- -- let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0; -- -- info!( -- "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}", -- total, -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY), -- ); -- -- for i in 0..self.nr_doms { -- info!( -- "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}", -- i, -- dom_loads[i], -- if imbal[i] < 0.0 { -imbal[i] } else { 0.0 }, -- if imbal[i] > 0.0 { imbal[i] } else { 0.0 }, -- ); -- } -- } -- -- fn step(&mut self) -> Result<()> { -- let started_at = std::time::SystemTime::now(); -- let bpf_stats = self.read_bpf_stats()?; -- let cpu_busy = self.get_cpu_busy()?; -- -- let mut lb = LoadBalancer::new( -- self.skel.maps_mut(), -- &mut self.task_loads, -- self.nr_doms, -- self.load_decay_factor, -- &mut self.nr_lb_data_errors, -- ); -- -- lb.read_task_loads(started_at.duration_since(self.prev_at)?)?; -- lb.calculate_dom_load_balance()?; -- -- if self.balance_load { -- lb.load_balance()?; -- } -- -- // Extract fields needed for reporting and drop lb to release -- // mutable borrows. -- let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal); -- -- self.report( -- &bpf_stats, -- cpu_busy, -- std::time::SystemTime::now().duration_since(started_at)?, -- load_avg, -- &dom_loads, -- &imbal, -- ); -- -- self.prev_at = started_at; -- Ok(()) -- } -- -- fn read_bpf_exit_type(&mut self) -> i32 { -- unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) } -- } -- -- fn report_bpf_exit_type(&mut self) -> Result<()> { -- // Report msg if EXT_OPS_EXIT_ERROR. -- match self.read_bpf_exit_type() { -- 0 => Ok(()), -- etype if etype == 2 => { -- let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) }; -- let msg = cstr -- .to_str() -- .context("Failed to convert exit msg to string") -- .unwrap(); -- bail!("BPF exit_type={} msg={}", etype, msg); -- } -- etype => { -- info!("BPF exit_type={}", etype); -- Ok(()) -- } -- } -- } --} -- --impl<'a> Drop for Scheduler<'a> { -- fn drop(&mut self) { -- if let Some(struct_ops) = self.struct_ops.take() { -- drop(struct_ops); -- } -- } --} -- --fn main() -> Result<()> { -- let opts = Opts::parse(); -- -- let llv = match opts.verbose { -- 0 => simplelog::LevelFilter::Info, -- 1 => simplelog::LevelFilter::Debug, -- _ => simplelog::LevelFilter::Trace, -- }; -- let mut lcfg = simplelog::ConfigBuilder::new(); -- lcfg.set_time_level(simplelog::LevelFilter::Error) -- .set_location_level(simplelog::LevelFilter::Off) -- .set_target_level(simplelog::LevelFilter::Off) -- .set_thread_level(simplelog::LevelFilter::Off); -- simplelog::TermLogger::init( -- llv, -- lcfg.build(), -- simplelog::TerminalMode::Stderr, -- simplelog::ColorChoice::Auto, -- )?; -- -- let shutdown = Arc::new(AtomicBool::new(false)); -- let shutdown_clone = shutdown.clone(); -- ctrlc::set_handler(move || { -- shutdown_clone.store(true, Ordering::Relaxed); -- }) -- .context("Error setting Ctrl-C handler")?; -- -- let mut sched = Scheduler::init(&opts)?; -- -- while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 { -- std::thread::sleep(Duration::from_secs_f64(opts.interval)); -- sched.step()?; -- } -- -- sched.report_bpf_exit_type() --} -diff --git a/tools/sched_ext/gnu/stubs.h b/tools/sched_ext/gnu/stubs.h -deleted file mode 100644 -index 719225b16626..000000000000 ---- a/tools/sched_ext/gnu/stubs.h -+++ /dev/null -@@ -1 +0,0 @@ --/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ -diff --git a/tools/sched_ext/scx_common.bpf.h b/tools/sched_ext/scx_common.bpf.h -deleted file mode 100644 -index e56de9dc86f2..000000000000 ---- a/tools/sched_ext/scx_common.bpf.h -+++ /dev/null -@@ -1,288 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __SCHED_EXT_COMMON_BPF_H --#define __SCHED_EXT_COMMON_BPF_H -- --#include "vmlinux.h" --#include --#include --#include --#include "user_exit_info.h" -- --#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ --#define PF_EXITING 0x00000004 --#define CLOCK_MONOTONIC 1 -- --/* -- * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can -- * lead to really confusing misbehaviors. Let's trigger a build failure. -- */ --static inline void ___vmlinux_h_sanity_check___(void) --{ -- _Static_assert(SCX_DSQ_FLAG_BUILTIN, -- "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); --} -- --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; -- --static inline __attribute__((format(printf, 1, 2))) --void ___scx_bpf_error_format_checker(const char *fmt, ...) {} -- --/* -- * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments -- * instead of an array of u64. Note that __param[] must have at least one -- * element to keep the verifier happy. -- */ --#define scx_bpf_error(fmt, args...) \ --({ \ -- static char ___fmt[] = fmt; \ -- unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ -- \ -- _Pragma("GCC diagnostic push") \ -- _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ -- ___bpf_fill(___param, args); \ -- _Pragma("GCC diagnostic pop") \ -- \ -- scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ -- \ -- ___scx_bpf_error_format_checker(fmt, ##args); \ --}) -- --void scx_bpf_switch_all(void) __ksym; --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; --bool scx_bpf_consume(u64 dsq_id) __ksym; --u32 scx_bpf_dispatch_nr_slots(void) __ksym; --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym; --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym; --void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; --s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym; --const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; --const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; --void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; --void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; --bool scx_bpf_task_running(const struct task_struct *p) __ksym; --s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym; --u32 scx_bpf_reenqueue_local(void) __ksym; -- --#define BPF_STRUCT_OPS(name, args...) \ --SEC("struct_ops/"#name) \ --BPF_PROG(name, ##args) -- --#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ --SEC("struct_ops.s/"#name) \ --BPF_PROG(name, ##args) -- --/** -- * MEMBER_VPTR - Obtain the verified pointer to a struct or array member -- * @base: struct or array to index -- * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...) -- * -- * The verifier often gets confused by the instruction sequence the compiler -- * generates for indexing struct fields or arrays. This macro forces the -- * compiler to generate a code sequence which first calculates the byte offset, -- * checks it against the struct or array size and add that byte offset to -- * generate the pointer to the member to help the verifier. -- * -- * Ideally, we want to abort if the calculated offset is out-of-bounds. However, -- * BPF currently doesn't support abort, so evaluate to NULL instead. The caller -- * must check for NULL and take appropriate action to appease the verifier. To -- * avoid confusing the verifier, it's best to check for NULL and dereference -- * immediately. -- * -- * vptr = MEMBER_VPTR(my_array, [i][j]); -- * if (!vptr) -- * return error; -- * *vptr = new_value; -- */ --#define MEMBER_VPTR(base, member) (typeof(base member) *)({ \ -- u64 __base = (u64)base; \ -- u64 __addr = (u64)&(base member) - __base; \ -- asm volatile ( \ -- "if %0 <= %[max] goto +2\n" \ -- "%0 = 0\n" \ -- "goto +1\n" \ -- "%0 += %1\n" \ -- : "+r"(__addr) \ -- : "r"(__base), \ -- [max]"i"(sizeof(base) - sizeof(base member))); \ -- __addr; \ --}) -- --/* -- * BPF core and other generic helpers -- */ -- --/* list and rbtree */ --#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) --#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) -- --void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; --void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; -- --#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) --#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) -- --void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; --struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; --struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, -- struct bpf_rb_node *node) __ksym; --void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, -- bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym; --struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; -- --/* task */ --struct task_struct *bpf_task_from_pid(s32 pid) __ksym; --struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; --void bpf_task_release(struct task_struct *p) __ksym; -- --/* cgroup */ --struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; --void bpf_cgroup_release(struct cgroup *cgrp) __ksym; --struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; -- --/* cpumask */ --struct bpf_cpumask *bpf_cpumask_create(void) __ksym; --struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; --u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; --u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym; -- --/* rcu */ --void bpf_rcu_read_lock(void) __ksym; --void bpf_rcu_read_unlock(void) __ksym; -- --/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */ --struct bpf_iter_num; -- --extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym; --extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym; --extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym; -- --#ifndef bpf_for_each --/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for -- * using BPF open-coded iterators without having to write mundane explicit -- * low-level loop logic. Instead, it provides for()-like generic construct -- * that can be used pretty naturally. E.g., for some hypothetical cgroup -- * iterator, you'd write: -- * -- * struct cgroup *cg, *parent_cg = <...>; -- * -- * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { -- * bpf_printk("Child cgroup id = %d", cg->cgroup_id); -- * if (cg->cgroup_id == 123) -- * break; -- * } -- * -- * I.e., it looks almost like high-level for each loop in other languages, -- * supports continue/break, and is verifiable by BPF verifier. -- * -- * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) -- * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to -- * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int -- * *`, not just `int`. So for integers bpf_for() is more convenient. -- * -- * Note: this macro relies on C99 feature of allowing to declare variables -- * inside for() loop, bound to for() loop lifetime. It also utilizes GCC -- * extension: __attribute__((cleanup())), supported by both GCC and -- * Clang. -- */ --#define bpf_for_each(type, cur, args...) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ -- cleanup(bpf_iter_##type##_destroy))), \ -- /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_##type##_new(&___it, ##args), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_##type##_destroy, (void *)0); \ -- /* iteration and termination check */ \ -- (((cur) = bpf_iter_##type##_next(&___it))); \ --) --#endif /* bpf_for_each */ -- --#ifndef bpf_for --/* bpf_for(i, start, end) implements a for()-like looping construct that sets -- * provided integer variable *i* to values starting from *start* through, -- * but not including, *end*. It also proves to BPF verifier that *i* belongs -- * to range [start, end), so this can be used for accessing arrays without -- * extra checks. -- * -- * Note: *start* and *end* are assumed to be expressions with no side effects -- * and whose values do not change throughout bpf_for() loop execution. They do -- * not have to be statically known or constant, though. -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_for(i, start, end) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, (start), (end)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- ({ \ -- /* iteration step */ \ -- int *___t = bpf_iter_num_next(&___it); \ -- /* termination and bounds check */ \ -- (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ -- }); \ --) --#endif /* bpf_for */ -- --#ifndef bpf_repeat --/* bpf_repeat(N) performs N iterations without exposing iteration number -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_repeat(N) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, 0, (N)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- bpf_iter_num_next(&___it); \ -- /* nothing here */ \ --) --#endif /* bpf_repeat */ -- --#endif /* __SCHED_EXT_COMMON_BPF_H */ -diff --git a/tools/sched_ext/scx_example_central.bpf.c b/tools/sched_ext/scx_example_central.bpf.c -deleted file mode 100644 -index 4cec04b4c2ed..000000000000 ---- a/tools/sched_ext/scx_example_central.bpf.c -+++ /dev/null -@@ -1,334 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A central FIFO sched_ext scheduler which demonstrates the followings: -- * -- * a. Making all scheduling decisions from one CPU: -- * -- * The central CPU is the only one making scheduling decisions. All other -- * CPUs kick the central CPU when they run out of tasks to run. -- * -- * There is one global BPF queue and the central CPU schedules all CPUs by -- * dispatching from the global queue to each CPU's local dsq from dispatch(). -- * This isn't the most straightforward. e.g. It'd be easier to bounce -- * through per-CPU BPF queues. The current design is chosen to maximally -- * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching. -- * -- * b. Tickless operation -- * -- * All tasks are dispatched with the infinite slice which allows stopping the -- * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full -- * parameter. The tickless operation can be observed through -- * /proc/interrupts. -- * -- * Periodic switching is enforced by a periodic timer checking all CPUs and -- * preempting them as necessary. Unfortunately, BPF timer currently doesn't -- * have a way to pin to a specific CPU, so the periodic timer isn't pinned to -- * the central CPU. -- * -- * c. Preemption -- * -- * Kthreads are unconditionally queued to the head of a matching local dsq -- * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always -- * prioritized over user threads, which is required for ensuring forward -- * progress as e.g. the periodic timer may run on a ksoftirqd and if the -- * ksoftirqd gets starved by a user thread, there may not be anything else to -- * vacate that user thread. -- * -- * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the -- * next tasks. -- * -- * This scheduler is designed to maximize usage of various SCX mechanisms. A -- * more practical implementation would likely put the scheduling loop outside -- * the central CPU's dispatch() path and add some form of priority mechanism. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --enum { -- FALLBACK_DSQ_ID = 0, -- MAX_CPUS = 4096, -- MS_TO_NS = 1000LLU * 1000, -- TIMER_INTERVAL_NS = 1 * MS_TO_NS, --}; -- --const volatile bool switch_partial; --const volatile s32 central_cpu; --const volatile u32 nr_cpu_ids = 64; /* !0 for veristat, set during init */ -- --u64 nr_total, nr_locals, nr_queued, nr_lost_pids; --u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries; --u64 nr_overflows; -- --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, s32); --} central_q SEC(".maps"); -- --/* can't use percpu map due to bad lookups */ --static bool cpu_gimme_task[MAX_CPUS]; --static u64 cpu_started_at[MAX_CPUS]; -- --struct central_timer { -- struct bpf_timer timer; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, 1); -- __type(key, u32); -- __type(value, struct central_timer); --} central_timer SEC(".maps"); -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- /* -- * Steer wakeups to the central CPU as much as possible to avoid -- * disturbing other CPUs. It's safe to blindly return the central cpu as -- * select_cpu() is a hint and if @p can't be on it, the kernel will -- * automatically pick a fallback CPU. -- */ -- return central_cpu; --} -- --void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags) --{ -- s32 pid = p->pid; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- /* -- * Push per-cpu kthreads at the head of local dsq's and preempt the -- * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked -- * behind other threads which is necessary for forward progress -- * guarantee as we depend on the BPF timer which may run from ksoftirqd. -- */ -- if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- __sync_fetch_and_add(&nr_locals, 1); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF, -- enq_flags | SCX_ENQ_PREEMPT); -- return; -- } -- -- if (bpf_map_push_elem(¢ral_q, &pid, 0)) { -- __sync_fetch_and_add(&nr_overflows, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_queued, 1); -- -- if (!scx_bpf_task_running(p)) -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); --} -- --static bool dispatch_to_cpu(s32 cpu) --{ -- struct task_struct *p; -- s32 pid; -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (bpf_map_pop_elem(¢ral_q, &pid)) -- break; -- -- __sync_fetch_and_sub(&nr_queued, 1); -- -- p = bpf_task_from_pid(pid); -- if (!p) { -- __sync_fetch_and_add(&nr_lost_pids, 1); -- continue; -- } -- -- /* -- * If we can't run the task at the top, do the dumb thing and -- * bounce it to the fallback dsq. -- */ -- if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- __sync_fetch_and_add(&nr_mismatches, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0); -- bpf_task_release(p); -- continue; -- } -- -- /* dispatch to local and mark that @cpu doesn't need more */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0); -- -- if (cpu != central_cpu) -- scx_bpf_kick_cpu(cpu, 0); -- -- bpf_task_release(p); -- return true; -- } -- -- return false; --} -- --void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (cpu == central_cpu) { -- /* dispatch for all other CPUs first */ -- __sync_fetch_and_add(&nr_dispatches, 1); -- -- bpf_for(cpu, 0, nr_cpu_ids) { -- bool *gimme; -- -- if (!scx_bpf_dispatch_nr_slots()) -- break; -- -- /* central's gimme is never set */ -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme && !*gimme) -- continue; -- -- if (dispatch_to_cpu(cpu)) -- *gimme = false; -- } -- -- /* -- * Retry if we ran out of dispatch buffer slots as we might have -- * skipped some CPUs and also need to dispatch for self. The ext -- * core automatically retries if the local dsq is empty but we -- * can't rely on that as we're dispatching for other CPUs too. -- * Kick self explicitly to retry. -- */ -- if (!scx_bpf_dispatch_nr_slots()) { -- __sync_fetch_and_add(&nr_retries, 1); -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- return; -- } -- -- /* look for a task to run on the central CPU */ -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- dispatch_to_cpu(central_cpu); -- } else { -- bool *gimme; -- -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme) -- *gimme = true; -- -- /* -- * Force dispatch on the scheduling CPU so that it finds a task -- * to run for us. -- */ -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- } --} -- --void BPF_STRUCT_OPS(central_running, struct task_struct *p) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ --} -- --void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = 0; --} -- --static int central_timerfn(void *map, int *key, struct bpf_timer *timer) --{ -- u64 now = bpf_ktime_get_ns(); -- u64 nr_to_kick = nr_queued; -- s32 i; -- -- bpf_for(i, 0, nr_cpu_ids) { -- s32 cpu = (nr_timers + i) % nr_cpu_ids; -- u64 *started_at; -- -- if (cpu == central_cpu) -- continue; -- -- /* kick iff the current one exhausted its slice */ -- started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at && *started_at && -- vtime_before(now, *started_at + SCX_SLICE_DFL)) -- continue; -- -- /* and there's something pending */ -- if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) || -- scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu)) -- ; -- else if (nr_to_kick) -- nr_to_kick--; -- else -- continue; -- -- scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); -- } -- -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- -- bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- __sync_fetch_and_add(&nr_timers, 1); -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(central_init) --{ -- u32 key = 0; -- struct bpf_timer *timer; -- int ret; -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1); -- if (ret) -- return ret; -- -- timer = bpf_map_lookup_elem(¢ral_timer, &key); -- if (!timer) -- return -ESRCH; -- -- bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); -- bpf_timer_set_callback(timer, central_timerfn); -- ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- return ret; --} -- --void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops central_ops = { -- /* -- * We are offloading all scheduling decisions to the central CPU and -- * thus being the last task on a given CPU doesn't mean anything -- * special. Enqueue the last tasks like any other tasks. -- */ -- .flags = SCX_OPS_ENQ_LAST, -- -- .select_cpu = (void *)central_select_cpu, -- .enqueue = (void *)central_enqueue, -- .dispatch = (void *)central_dispatch, -- .running = (void *)central_running, -- .stopping = (void *)central_stopping, -- .init = (void *)central_init, -- .exit = (void *)central_exit, -- .name = "central", --}; -diff --git a/tools/sched_ext/scx_example_central.c b/tools/sched_ext/scx_example_central.c -deleted file mode 100644 -index 7ad591cbdc65..000000000000 ---- a/tools/sched_ext/scx_example_central.c -+++ /dev/null -@@ -1,94 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_central.skel.h" -- --const char help_fmt[] = --"A central FIFO sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-c CPU] [-p]\n" --"\n" --" -c CPU Override the central CPU (default: 0)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_central *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_central__open(); -- assert(skel); -- -- skel->rodata->central_cpu = 0; -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "c:ph")) != -1) { -- switch (opt) { -- case 'c': -- skel->rodata->central_cpu = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_central__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.central_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf("total :%10lu local:%10lu queued:%10lu lost:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_locals, -- skel->bss->nr_queued, -- skel->bss->nr_lost_pids); -- printf("timer :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n", -- skel->bss->nr_timers, -- skel->bss->nr_dispatches, -- skel->bss->nr_mismatches, -- skel->bss->nr_retries); -- printf("overflow:%10lu\n", -- skel->bss->nr_overflows); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_central__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_flatcg.bpf.c b/tools/sched_ext/scx_example_flatcg.bpf.c -deleted file mode 100644 -index f6078b9a681f..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.bpf.c -+++ /dev/null -@@ -1,872 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext flattened cgroup hierarchy scheduler. It implements -- * hierarchical weight-based cgroup CPU control by flattening the cgroup -- * hierarchy into a single layer by compounding the active weight share at each -- * level. Consider the following hierarchy with weights in parentheses: -- * -- * R + A (100) + B (100) -- * | \ C (100) -- * \ D (200) -- * -- * Ignoring the root and threaded cgroups, only B, C and D can contain tasks. -- * Let's say all three have runnable tasks. The total share that each of these -- * three cgroups is entitled to can be calculated by compounding its share at -- * each level. -- * -- * For example, B is competing against C and in that competition its share is -- * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's -- * share in that competition is 200/(200+100) == 1/3. B's eventual share in the -- * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's -- * eventual shaer is the same at 1/6. D is only competing at the top level and -- * its share is 200/(100+200) == 2/3. -- * -- * So, instead of hierarchically scheduling level-by-level, we can consider it -- * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3 -- * and keep updating the eventual shares as the cgroups' runnable states change. -- * -- * This flattening of hierarchy can bring a substantial performance gain when -- * the cgroup hierarchy is nested multiple levels. in a simple benchmark using -- * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it -- * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two -- * apache instances competing with 2:1 weight ratio nested four level deep. -- * -- * However, the gain comes at the cost of not being able to properly handle -- * thundering herd of cgroups. For example, if many cgroups which are nested -- * behind a low priority parent cgroup wake up around the same time, they may be -- * able to consume more CPU cycles than they are entitled to. In many use cases, -- * this isn't a real concern especially given the performance gain. Also, there -- * are ways to mitigate the problem further by e.g. introducing an extra -- * scheduling layer on cgroup delegation boundaries. -- * -- * The scheduler first picks the cgroup to run and then schedule the tasks -- * within by using nested weighted vtime scheduling by default. The -- * cgroup-internal scheduling can be switched to FIFO with the -f option. -- */ --#include "scx_common.bpf.h" --#include "user_exit_info.h" --#include "scx_example_flatcg.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */ --const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL; --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --u64 cvtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, u64); -- __uint(max_entries, FCG_NR_STATS); --} stats SEC(".maps"); -- --static void stat_inc(enum fcg_stat_idx idx) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p)++; --} -- --struct fcg_cpu_ctx { -- u64 cur_cgid; -- u64 cur_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, struct fcg_cpu_ctx); -- __uint(max_entries, 1); --} cpu_ctx SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_cgrp_ctx); --} cgrp_ctx SEC(".maps"); -- --struct cgv_node { -- struct bpf_rb_node rb_node; -- __u64 cvtime; -- __u64 cgid; --}; -- --private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock; --private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node); -- --struct cgv_node_stash { -- struct cgv_node __kptr *node; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, 16384); -- __type(key, __u64); -- __type(value, struct cgv_node_stash); --} cgv_node_stash SEC(".maps"); -- --struct fcg_task_ctx { -- u64 bypassed_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_task_ctx); --} task_ctx SEC(".maps"); -- --/* gets inc'd on weight tree changes to expire the cached hweights */ --unsigned long hweight_gen = 1; -- --static u64 div_round_up(u64 dividend, u64 divisor) --{ -- return (dividend + divisor - 1) / divisor; --} -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) --{ -- struct cgv_node *cgc_a, *cgc_b; -- -- cgc_a = container_of(a, struct cgv_node, rb_node); -- cgc_b = container_of(b, struct cgv_node, rb_node); -- -- return cgc_a->cvtime < cgc_b->cvtime; --} -- --static struct fcg_cpu_ctx *find_cpu_ctx(void) --{ -- struct fcg_cpu_ctx *cpuc; -- u32 idx = 0; -- -- cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx); -- if (!cpuc) { -- scx_bpf_error("cpu_ctx lookup failed"); -- return NULL; -- } -- return cpuc; --} -- --static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id); -- return NULL; -- } -- return cgc; --} -- --static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgrp = bpf_cgroup_ancestor(cgrp, level); -- if (!cgrp) { -- scx_bpf_error("ancestor cgroup lookup failed"); -- return NULL; -- } -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- scx_bpf_error("ancestor cgrp_ctx lookup failed"); -- bpf_cgroup_release(cgrp); -- return cgc; --} -- --static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- int level; -- -- if (!cgc->nr_active) { -- stat_inc(FCG_STAT_HWT_SKIP); -- return; -- } -- -- if (cgc->hweight_gen == hweight_gen) { -- stat_inc(FCG_STAT_HWT_CACHE); -- return; -- } -- -- stat_inc(FCG_STAT_HWT_UPDATES); -- bpf_for(level, 0, cgrp->level + 1) { -- struct fcg_cgrp_ctx *cgc; -- bool is_active; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- -- if (!level) { -- cgc->hweight = FCG_HWEIGHT_ONE; -- cgc->hweight_gen = hweight_gen; -- } else { -- struct fcg_cgrp_ctx *pcgc; -- -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- -- /* -- * We can be oppotunistic here and not grab the -- * cgv_tree_lock and deal with the occasional races. -- * However, hweight updates are already cached and -- * relatively low-frequency. Let's just do the -- * straightforward thing. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- is_active = cgc->nr_active; -- if (is_active) { -- cgc->hweight_gen = pcgc->hweight_gen; -- cgc->hweight = -- div_round_up(pcgc->hweight * cgc->weight, -- pcgc->child_weight_sum); -- } -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!is_active) { -- stat_inc(FCG_STAT_HWT_RACE); -- break; -- } -- } -- } --} -- --static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) --{ -- u64 delta, cvtime, max_budget; -- -- /* -- * A node which is on the rbtree can't be pointed to from elsewhere yet -- * and thus can't be updated and repositioned. Instead, we collect the -- * vtime deltas separately and apply it asynchronously here. -- */ -- delta = cgc->cvtime_delta; -- __sync_fetch_and_sub(&cgc->cvtime_delta, delta); -- cvtime = cgv_node->cvtime + delta; -- -- /* -- * Allow a cgroup to carry the maximum budget proportional to its -- * hweight such that a full-hweight cgroup can immediately take up half -- * of the CPUs at the most while staying at the front of the rbtree. -- */ -- max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / -- (2 * FCG_HWEIGHT_ONE); -- if (vtime_before(cvtime, cvtime_now - max_budget)) -- cvtime = cvtime_now - max_budget; -- -- cgv_node->cvtime = cvtime; --} -- --static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- u64 cgid = cgrp->kn->id; -- -- /* paired with cmpxchg in try_pick_next_cgroup() */ -- if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { -- stat_inc(FCG_STAT_ENQ_SKIP); -- return; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid); -- return; -- } -- -- /* NULL if the node is already on the rbtree */ -- cgv_node = bpf_kptr_xchg(&stash->node, NULL); -- if (!cgv_node) { -- stat_inc(FCG_STAT_ENQ_RACE); -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); --} -- --void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * If select_cpu_dfl() is recommending local enqueue, the target CPU is -- * idle. Follow it and charge the cgroup later in fcg_stopping() after -- * the fact. Use the same mechanism to deal with tasks with custom -- * affinities so that we don't have to worry about per-cgroup dq's -- * containing tasks that can't be executed from some CPUs. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) { -- /* -- * Tell fcg_stopping() that this bypassed the regular scheduling -- * path and should be force charged to the cgroup. 0 is used to -- * indicate that the task isn't bypassing, so if the current -- * runtime is 0, go back by one nanosecond. -- */ -- taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1; -- -- /* -- * The global dq is deprioritized as we don't want to let tasks -- * to boost themselves by constraining its cpumask. The -- * deprioritization is rather severe, so let's not apply that to -- * per-cpu kernel threads. This is ham-fisted. We probably wanna -- * implement per-cgroup fallback dq's instead so that we have -- * more control over when tasks with custom cpumask get issued. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || -- (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) { -- stat_inc(FCG_STAT_LOCAL); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- } else { -- stat_inc(FCG_STAT_GLOBAL); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } -- return; -- } -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- goto out_release; -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 tvtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) -- tvtime = cgc->tvtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, -- tvtime, enq_flags); -- } -- -- cgrp_enqueued(cgrp, cgc); --out_release: -- bpf_cgroup_release(cgrp); --} -- --/* -- * Walk the cgroup tree to update the active weight sums as tasks wake up and -- * sleep. The weight sums are used as the base when calculating the proportion a -- * given cgroup or task is entitled to at each level. -- */ --static void update_active_weight_sums(struct cgroup *cgrp, bool runnable) --{ -- struct fcg_cgrp_ctx *cgc; -- bool updated = false; -- int idx; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- /* -- * In most cases, a hot cgroup would have multiple threads going to -- * sleep and waking up while the whole cgroup stays active. In leaf -- * cgroups, ->nr_runnable which is updated with __sync operations gates -- * ->nr_active updates, so that we don't have to grab the cgv_tree_lock -- * repeatedly for a busy cgroup which is staying active. -- */ -- if (runnable) { -- if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_ACT); -- } else { -- if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_DEACT); -- } -- -- /* -- * If @cgrp is becoming runnable, its hweight should be refreshed after -- * it's added to the weight tree so that enqueue has the up-to-date -- * value. If @cgrp is becoming quiescent, the hweight should be -- * refreshed before it's removed from the weight tree so that the usage -- * charging which happens afterwards has access to the latest value. -- */ -- if (!runnable) -- cgrp_refresh_hweight(cgrp, cgc); -- -- /* propagate upwards */ -- bpf_for(idx, 0, cgrp->level) { -- int level = cgrp->level - idx; -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- bool propagate = false; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- if (level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- } -- -- /* -- * We need the propagation protected by a lock to synchronize -- * against weight changes. There's no reason to drop the lock at -- * each level but bpf_spin_lock() doesn't want any function -- * calls while locked. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- -- if (runnable) { -- if (!cgc->nr_active++) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum += cgc->weight; -- } -- } -- } else { -- if (!--cgc->nr_active) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum -= cgc->weight; -- } -- } -- } -- -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!propagate) -- break; -- } -- -- if (updated) -- __sync_fetch_and_add(&hweight_gen, 1); -- -- if (runnable) -- cgrp_refresh_hweight(cgrp, cgc); --} -- --void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, true); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) --{ -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- if (fifo_sched) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- /* -- * @cgc->tvtime_now always progresses forward as tasks start -- * executing. The test and update can be performed concurrently -- * from multiple CPUs and thus racy. Any error should be -- * contained and temporary. Let's just live with it. -- */ -- if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) -- cgc->tvtime_now = p->scx.dsq_vtime; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- /* scale the execution time by the inverse of the weight and charge */ -- if (!fifo_sched) -- p->scx.dsq_vtime += -- (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- if (!taskc->bypassed_at) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- __sync_fetch_and_add(&cgc->cvtime_delta, -- p->se.sum_exec_runtime - taskc->bypassed_at); -- taskc->bypassed_at = 0; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, false); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) --{ -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- if (cgrp->level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1); -- if (!pcgc) -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- if (pcgc && cgc->nr_active) -- pcgc->child_weight_sum += (s64)weight - cgc->weight; -- cgc->weight = weight; -- bpf_spin_unlock(&cgv_tree_lock); --} -- --static bool try_pick_next_cgroup(u64 *cgidp) --{ -- struct bpf_rb_node *rb_node; -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 cgid; -- -- /* pop the front cgroup and wind cvtime_now accordingly */ -- bpf_spin_lock(&cgv_tree_lock); -- -- rb_node = bpf_rbtree_first(&cgv_tree); -- if (!rb_node) { -- bpf_spin_unlock(&cgv_tree_lock); -- stat_inc(FCG_STAT_PNC_NO_CGRP); -- *cgidp = 0; -- return true; -- } -- -- rb_node = bpf_rbtree_remove(&cgv_tree, rb_node); -- bpf_spin_unlock(&cgv_tree_lock); -- -- cgv_node = container_of(rb_node, struct cgv_node, rb_node); -- cgid = cgv_node->cgid; -- -- if (vtime_before(cvtime_now, cgv_node->cvtime)) -- cvtime_now = cgv_node->cvtime; -- -- /* -- * If lookup fails, the cgroup's gone. Free and move on. See -- * fcg_cgroup_exit(). -- */ -- cgrp = bpf_cgroup_from_id(cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- if (!scx_bpf_consume(cgid)) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_EMPTY); -- goto out_stash; -- } -- -- /* -- * Successfully consumed from the cgroup. This will be our current -- * cgroup for the new slice. Refresh its hweight. -- */ -- cgrp_refresh_hweight(cgrp, cgc); -- -- bpf_cgroup_release(cgrp); -- -- /* -- * As the cgroup may have more tasks, add it back to the rbtree. Note -- * that here we charge the full slice upfront and then exact later -- * according to the actual consumption. This prevents lowpri thundering -- * herd from saturating the machine. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- -- *cgidp = cgid; -- stat_inc(FCG_STAT_PNC_NEXT); -- return true; -- --out_stash: -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- /* -- * Paired with cmpxchg in cgrp_enqueued(). If they see the following -- * transition, they'll enqueue the cgroup. If they are earlier, we'll -- * see their task in the dq below and requeue the cgroup. -- */ -- __sync_val_compare_and_swap(&cgc->queued, 1, 0); -- -- if (scx_bpf_dsq_nr_queued(cgid)) { -- bpf_spin_lock(&cgv_tree_lock); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- goto out_free; -- } -- } -- -- return false; -- --out_free: -- bpf_obj_drop(cgv_node); -- return false; --} -- --void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) --{ -- struct fcg_cpu_ctx *cpuc; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 now = bpf_ktime_get_ns(); -- -- cpuc = find_cpu_ctx(); -- if (!cpuc) -- return; -- -- if (!cpuc->cur_cgid) -- goto pick_next_cgroup; -- -- if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { -- if (scx_bpf_consume(cpuc->cur_cgid)) { -- stat_inc(FCG_STAT_CNS_KEEP); -- return; -- } -- stat_inc(FCG_STAT_CNS_EMPTY); -- } else { -- stat_inc(FCG_STAT_CNS_EXPIRE); -- } -- -- /* -- * The current cgroup is expiring. It was already charged a full slice. -- * Calculate the actual usage and accumulate the delta. -- */ -- cgrp = bpf_cgroup_from_id(cpuc->cur_cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_CNS_GONE); -- goto pick_next_cgroup; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (cgc) { -- /* -- * We want to update the vtime delta and then look for the next -- * cgroup to execute but the latter needs to be done in a loop -- * and we can't keep the lock held. Oh well... -- */ -- bpf_spin_lock(&cgv_tree_lock); -- __sync_fetch_and_add(&cgc->cvtime_delta, -- (cpuc->cur_at + cgrp_slice_ns - now) * -- FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- stat_inc(FCG_STAT_CNS_GONE); -- } -- -- bpf_cgroup_release(cgrp); -- --pick_next_cgroup: -- cpuc->cur_at = now; -- -- if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { -- cpuc->cur_cgid = 0; -- return; -- } -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_pick_next_cgroup(&cpuc->cur_cgid)) -- break; -- } --} -- --s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct fcg_task_ctx *taskc; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- taskc = bpf_task_storage_get(&task_ctx, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!taskc) -- return -ENOMEM; -- -- taskc->bypassed_at = 0; -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp, -- struct scx_cgroup_init_args *args) --{ -- struct fcg_cgrp_ctx *cgc; -- struct cgv_node *cgv_node; -- struct cgv_node_stash empty_stash = {}, *stash; -- u64 cgid = cgrp->kn->id; -- int ret; -- -- /* -- * Technically incorrect as cgroup ID is full 64bit while dq ID is -- * 63bit. Should not be a problem in practice and easy to spot in the -- * unlikely case that it breaks. -- */ -- ret = scx_bpf_create_dsq(cgid, -1); -- if (ret) -- return ret; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!cgc) { -- ret = -ENOMEM; -- goto err_destroy_dsq; -- } -- -- cgc->weight = args->weight; -- cgc->hweight = FCG_HWEIGHT_ONE; -- -- ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash, -- BPF_NOEXIST); -- if (ret) { -- if (ret != -ENOMEM) -- scx_bpf_error("unexpected stash creation error (%d)", -- ret); -- goto err_destroy_dsq; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("unexpected cgv_node stash lookup failure"); -- ret = -ENOENT; -- goto err_destroy_dsq; -- } -- -- cgv_node = bpf_obj_new(struct cgv_node); -- if (!cgv_node) { -- ret = -ENOMEM; -- goto err_del_cgv_node; -- } -- -- cgv_node->cgid = cgid; -- cgv_node->cvtime = cvtime_now; -- -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- ret = -EBUSY; -- goto err_drop; -- } -- -- return 0; -- --err_drop: -- bpf_obj_drop(cgv_node); --err_del_cgv_node: -- bpf_map_delete_elem(&cgv_node_stash, &cgid); --err_destroy_dsq: -- scx_bpf_destroy_dsq(cgid); -- return ret; --} -- --void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- -- /* -- * For now, there's no way find and remove the cgv_node if it's on the -- * cgv_tree. Let's drain them in the dispatch path as they get popped -- * off the front of the tree. -- */ -- bpf_map_delete_elem(&cgv_node_stash, &cgid); -- scx_bpf_destroy_dsq(cgid); --} -- --s32 BPF_STRUCT_OPS(fcg_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops flatcg_ops = { -- .enqueue = (void *)fcg_enqueue, -- .dispatch = (void *)fcg_dispatch, -- .runnable = (void *)fcg_runnable, -- .running = (void *)fcg_running, -- .stopping = (void *)fcg_stopping, -- .quiescent = (void *)fcg_quiescent, -- .prep_enable = (void *)fcg_prep_enable, -- .cgroup_set_weight = (void *)fcg_cgroup_set_weight, -- .cgroup_init = (void *)fcg_cgroup_init, -- .cgroup_exit = (void *)fcg_cgroup_exit, -- .init = (void *)fcg_init, -- .exit = (void *)fcg_exit, -- .flags = SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING, -- .name = "flatcg", --}; -diff --git a/tools/sched_ext/scx_example_flatcg.c b/tools/sched_ext/scx_example_flatcg.c -deleted file mode 100644 -index f9c8a5b84a70..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.c -+++ /dev/null -@@ -1,232 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2023 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2023 Tejun Heo -- * Copyright (c) 2023 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_flatcg.h" --#include "scx_example_flatcg.skel.h" -- --#ifndef FILEID_KERNFS --#define FILEID_KERNFS 0xfe --#endif -- --const char help_fmt[] = --"A flattened cgroup hierarchy sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -i INTERVAL Report interval\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --static float read_cpu_util(__u64 *last_sum, __u64 *last_idle) --{ -- FILE *fp; -- char buf[4096]; -- char *line, *cur = NULL, *tok; -- __u64 sum = 0, idle = 0; -- __u64 delta_sum, delta_idle; -- int idx; -- -- fp = fopen("/proc/stat", "r"); -- if (!fp) { -- perror("fopen(\"/proc/stat\")"); -- return 0.0; -- } -- -- if (!fgets(buf, sizeof(buf), fp)) { -- perror("fgets(\"/proc/stat\")"); -- fclose(fp); -- return 0.0; -- } -- fclose(fp); -- -- line = buf; -- for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) { -- char *endp = NULL; -- __u64 v; -- -- if (idx == 0) { -- line = NULL; -- continue; -- } -- v = strtoull(tok, &endp, 0); -- if (!endp || *endp != '\0') { -- fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n", -- idx, tok); -- continue; -- } -- sum += v; -- if (idx == 4) -- idle = v; -- } -- -- delta_sum = sum - *last_sum; -- delta_idle = idle - *last_idle; -- *last_sum = sum; -- *last_idle = idle; -- -- return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0; --} -- --static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats) --{ -- __u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus]; -- __u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); -- -- for (idx = 0; idx < FCG_NR_STATS; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_flatcg *skel; -- struct bpf_link *link; -- struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 }; -- bool dump_cgrps = false; -- __u64 last_cpu_sum = 0, last_cpu_idle = 0; -- __u64 last_stats[FCG_NR_STATS] = {}; -- unsigned long seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_flatcg__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open: %s\n", strerror(errno)); -- return 1; -- } -- -- skel->rodata->nr_cpus = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) { -- double v; -- -- switch (opt) { -- case 's': -- v = strtod(optarg, NULL); -- skel->rodata->cgrp_slice_ns = v * 1000; -- break; -- case 'i': -- v = strtod(optarg, NULL); -- intv_ts.tv_sec = v; -- intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000; -- break; -- case 'd': -- dump_cgrps = true; -- break; -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- case 'h': -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d", -- (double)skel->rodata->cgrp_slice_ns / 1000000.0, -- (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0, -- dump_cgrps); -- -- if (scx_example_flatcg__load(skel)) { -- fprintf(stderr, "Failed to load: %s\n", strerror(errno)); -- return 1; -- } -- -- link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops); -- if (!link) { -- fprintf(stderr, "Failed to attach_struct_ops: %s\n", -- strerror(errno)); -- return 1; -- } -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- __u64 acc_stats[FCG_NR_STATS]; -- __u64 stats[FCG_NR_STATS]; -- float cpu_util; -- int i; -- -- cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle); -- -- fcg_read_stats(skel, acc_stats); -- for (i = 0; i < FCG_NR_STATS; i++) -- stats[i] = acc_stats[i] - last_stats[i]; -- -- memcpy(last_stats, acc_stats, sizeof(acc_stats)); -- -- printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n", -- seq++, cpu_util * 100.0, skel->data->hweight_gen); -- printf(" act:%6llu deact:%6llu local:%6llu global:%6llu\n", -- stats[FCG_STAT_ACT], -- stats[FCG_STAT_DEACT], -- stats[FCG_STAT_LOCAL], -- stats[FCG_STAT_GLOBAL]); -- printf("HWT skip:%6llu race:%6llu cache:%6llu update:%6llu\n", -- stats[FCG_STAT_HWT_SKIP], -- stats[FCG_STAT_HWT_RACE], -- stats[FCG_STAT_HWT_CACHE], -- stats[FCG_STAT_HWT_UPDATES]); -- printf("ENQ skip:%6llu race:%6llu\n", -- stats[FCG_STAT_ENQ_SKIP], -- stats[FCG_STAT_ENQ_RACE]); -- printf("CNS keep:%6llu expire:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_CNS_KEEP], -- stats[FCG_STAT_CNS_EXPIRE], -- stats[FCG_STAT_CNS_EMPTY], -- stats[FCG_STAT_CNS_GONE]); -- printf("PNC nocgrp:%6llu next:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_PNC_NO_CGRP], -- stats[FCG_STAT_PNC_NEXT], -- stats[FCG_STAT_PNC_EMPTY], -- stats[FCG_STAT_PNC_GONE]); -- printf("BAD remove:%6llu\n", -- acc_stats[FCG_STAT_BAD_REMOVAL]); -- -- nanosleep(&intv_ts, NULL); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_flatcg__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_flatcg.h b/tools/sched_ext/scx_example_flatcg.h -deleted file mode 100644 -index 490758ed41f0..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.h -+++ /dev/null -@@ -1,49 +0,0 @@ --#ifndef __SCX_EXAMPLE_FLATCG_H --#define __SCX_EXAMPLE_FLATCG_H -- --enum { -- FCG_HWEIGHT_ONE = 1LLU << 16, --}; -- --enum fcg_stat_idx { -- FCG_STAT_ACT, -- FCG_STAT_DEACT, -- FCG_STAT_LOCAL, -- FCG_STAT_GLOBAL, -- -- FCG_STAT_HWT_UPDATES, -- FCG_STAT_HWT_CACHE, -- FCG_STAT_HWT_SKIP, -- FCG_STAT_HWT_RACE, -- -- FCG_STAT_ENQ_SKIP, -- FCG_STAT_ENQ_RACE, -- -- FCG_STAT_CNS_KEEP, -- FCG_STAT_CNS_EXPIRE, -- FCG_STAT_CNS_EMPTY, -- FCG_STAT_CNS_GONE, -- -- FCG_STAT_PNC_NO_CGRP, -- FCG_STAT_PNC_NEXT, -- FCG_STAT_PNC_EMPTY, -- FCG_STAT_PNC_GONE, -- -- FCG_STAT_BAD_REMOVAL, -- -- FCG_NR_STATS, --}; -- --struct fcg_cgrp_ctx { -- u32 nr_active; -- u32 nr_runnable; -- u32 queued; -- u32 weight; -- u32 hweight; -- u64 child_weight_sum; -- u64 hweight_gen; -- s64 cvtime_delta; -- u64 tvtime_now; --}; -- --#endif /* __SCX_EXAMPLE_FLATCG_H */ -diff --git a/tools/sched_ext/scx_example_pair.bpf.c b/tools/sched_ext/scx_example_pair.bpf.c -deleted file mode 100644 -index 279efe58b777..000000000000 ---- a/tools/sched_ext/scx_example_pair.bpf.c -+++ /dev/null -@@ -1,627 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext core-scheduler which always makes every sibling CPU pair -- * execute from the same CPU cgroup. -- * -- * This scheduler is a minimal implementation and would need some form of -- * priority handling both inside each cgroup and across the cgroups to be -- * practically useful. -- * -- * Each CPU in the system is paired with exactly one other CPU, according to a -- * "stride" value that can be specified when the BPF scheduler program is first -- * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee -- * that they will only ever schedule tasks that belong to the same CPU cgroup. -- * -- * Scheduler Initialization -- * ------------------------ -- * -- * The scheduler BPF program is first initialized from user space, before it is -- * enabled. During this initialization process, each CPU on the system is -- * assigned several values that are constant throughout its runtime: -- * -- * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling -- * decisions. Paired CPUs always schedule tasks from the same -- * CPU cgroup, and synchronize with each other to guarantee -- * that this constraint is not violated. -- * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access -- * a struct pair_ctx object that is shared between the pair. -- * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the -- * pair. Each struct pair_ctx has an active_mask field, -- * which is a bitmap used to indicate whether each core -- * in the pair currently has an actively running task. -- * This index specifies which entry in the bitmap corresponds -- * to each CPU in the pair. -- * -- * During this initialization, the CPUs are paired according to a "stride" that -- * may be specified when invoking the user space program that initializes and -- * loads the scheduler. By default, the stride is 1/2 the total number of CPUs. -- * -- * Tasks and cgroups -- * ----------------- -- * -- * Every cgroup in the system is registered with the scheduler using the -- * pair_cgroup_init() callback, and every task in the system is associated with -- * exactly one cgroup. At a high level, the idea with the pair scheduler is to -- * always schedule tasks from the same cgroup within a given CPU pair. When a -- * task is enqueued (i.e. passed to the pair_enqueue() callback function), its -- * cgroup ID is read from its task struct, and then a corresponding queue map -- * is used to FIFO-enqueue the task for that cgroup. -- * -- * If you look through the implementation of the scheduler, you'll notice that -- * there is quite a bit of complexity involved with looking up the per-cgroup -- * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash -- * BPF hash map that is used to map a cgroup ID to a globally unique ID that's -- * allocated in the BPF program. This is done because we use separate maps to -- * store the FIFO queue of tasks, and the length of that map, per cgroup. This -- * complexity is only present because of current deficiencies in BPF that will -- * soon be addressed. The main point to keep in mind is that newly enqueued -- * tasks are added to their cgroup's FIFO queue. -- * -- * Dispatching tasks -- * ----------------- -- * -- * This section will describe how enqueued tasks are dispatched and scheduled. -- * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is -- * as follows: -- * -- * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is -- * the structure that's used to synchronize amongst the two pair CPUs in their -- * scheduling decisions. After any of the following events have occurred: -- * -- * - The cgroup's slice run has expired, or -- * - The cgroup becomes empty, or -- * - Either CPU in the pair is preempted by a higher priority scheduling class -- * -- * The cgroup transitions to the draining state and stops executing new tasks -- * from the cgroup. -- * -- * 2. If the pair is still executing a task, mark the pair_ctx as draining, and -- * wait for the pair CPU to be preempted. -- * -- * 3. Otherwise, if the pair CPU is not running a task, we can move onto -- * scheduling new tasks. Pop the next cgroup id from the top_q queue. -- * -- * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it. -- * -- * Note again that this scheduling behavior is simple, but the implementation -- * is complex mostly because this it hits several BPF shortcomings and has to -- * work around in often awkward ways. Most of the shortcomings are expected to -- * be resolved in the near future which should allow greatly simplifying this -- * scheduler. -- * -- * Dealing with preemption -- * ----------------------- -- * -- * SCX is the lowest priority sched_class, and could be preempted by them at -- * any time. To address this, the scheduler implements pair_cpu_release() and -- * pair_cpu_acquire() callbacks which are invoked by the core scheduler when -- * the scheduler loses and gains control of the CPU respectively. -- * -- * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and -- * then invoke: -- * -- * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- * -- * This preempts the pair CPU, and waits until it has re-entered the scheduler -- * before returning. This is necessary to ensure that the higher priority -- * sched_class that preempted our scheduler does not schedule a task -- * concurrently with our pair CPU. -- * -- * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption -- * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable -- * pair scheduling. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include "scx_example_pair.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; -- --/* !0 for veristat, set during init */ --const volatile u32 nr_cpu_ids = 64; -- --/* a pair of CPUs stay on a cgroup for this duration */ --const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL; -- --/* cpu ID -> pair cpu ID */ --const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 }; -- --/* cpu ID -> pair_id */ --const volatile u32 pair_id[MAX_CPUS]; -- --/* CPU ID -> CPU # in the pair (0 or 1) */ --const volatile u32 in_pair_idx[MAX_CPUS]; -- --struct pair_ctx { -- struct bpf_spin_lock lock; -- -- /* the cgroup the pair is currently executing */ -- u64 cgid; -- -- /* the pair started executing the current cgroup at */ -- u64 started_at; -- -- /* whether the current cgroup is draining */ -- bool draining; -- -- /* the CPUs that are currently active on the cgroup */ -- u32 active_mask; -- -- /* -- * the CPUs that are currently preempted and running tasks in a -- * different scheduler. -- */ -- u32 preempted_mask; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, MAX_CPUS / 2); -- __type(key, u32); -- __type(value, struct pair_ctx); --} pair_ctx SEC(".maps"); -- --/* queue of cgrp_q's possibly with tasks on them */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- /* -- * Because it's difficult to build strong synchronization encompassing -- * multiple non-trivial operations in BPF, this queue is managed in an -- * opportunistic way so that we guarantee that a cgroup w/ active tasks -- * is always on it but possibly multiple times. Once we have more robust -- * synchronization constructs and e.g. linked list, we should be able to -- * do this in a prettier way but for now just size it big enough. -- */ -- __uint(max_entries, 4 * MAX_CGRPS); -- __type(value, u64); --} top_q SEC(".maps"); -- --/* per-cgroup q which FIFOs the tasks from the cgroup */ --struct cgrp_q { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, MAX_QUEUED); -- __type(value, u32); --}; -- --/* -- * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local -- * storage; however, a cgroup local storage can only be accessed from the BPF -- * progs attached to the cgroup. For now, work around by allocating array of -- * cgrp_q's and then allocating per-cgroup indices. -- * -- * Another caveat: It's difficult to populate a large array of maps statically -- * or from BPF. Initialize it from userland. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, MAX_CGRPS); -- __type(key, s32); -- __array(values, struct cgrp_q); --} cgrp_q_arr SEC(".maps"); -- --static u64 cgrp_q_len[MAX_CGRPS]; -- --/* -- * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be -- * useful to have as a map type. -- */ --static u32 cgrp_q_idx_cursor; --static u64 cgrp_q_idx_busy[MAX_CGRPS]; -- --/* -- * All added up, the following is what we do: -- * -- * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking -- * for a free ID. If not found, fail cgroup creation with -EBUSY. -- * -- * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following -- * cgrp_q_idx_hash. -- * -- * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from -- * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr. -- * -- * This is sadly complicated for something pretty simple. Hopefully, we should -- * be able to simplify in the future. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, MAX_CGRPS); -- __uint(key_size, sizeof(u64)); /* cgrp ID */ -- __uint(value_size, sizeof(s32)); /* cgrp_q idx */ --} cgrp_q_idx_hash SEC(".maps"); -- --/* statistics */ --u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions; --u64 nr_exps, nr_exp_waits, nr_exp_empty; --u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty; -- --struct user_exit_info uei; -- --static bool time_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- struct cgrp_q *cgq; -- s32 pid = p->pid; -- u64 cgid; -- u32 *q_idx; -- u64 *cgq_len; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- cgrp = scx_bpf_task_cgroup(p); -- cgid = cgrp->kn->id; -- bpf_cgroup_release(cgrp); -- -- /* find the cgroup's q and push @p into it */ -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!q_idx) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return; -- } -- -- cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx); -- if (!cgq) { -- scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]", -- cgid, *q_idx); -- return; -- } -- -- if (bpf_map_push_elem(cgq, &pid, 0)) { -- scx_bpf_error("cgroup[%llu] queue overflow", cgid); -- return; -- } -- -- /* bump q len, if going 0 -> 1, queue cgroup into the top_q */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len) { -- scx_bpf_error("MEMBER_VTPR malfunction"); -- return; -- } -- -- if (!__sync_fetch_and_add(cgq_len, 1) && -- bpf_map_push_elem(&top_q, &cgid, 0)) { -- scx_bpf_error("top_q overflow"); -- return; -- } --} -- --static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) --{ -- u32 *vptr; -- -- vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *pairc = bpf_map_lookup_elem(&pair_ctx, vptr); -- if (!(*pairc)) -- return -EINVAL; -- -- vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *mask = 1U << *vptr; -- -- return 0; --} -- --static int try_dispatch(s32 cpu) --{ -- struct pair_ctx *pairc; -- struct bpf_map *cgq_map; -- struct task_struct *p; -- u64 now = bpf_ktime_get_ns(); -- bool kick_pair = false; -- bool expired, pair_preempted; -- u32 *vptr, in_pair_mask; -- s32 pid, q_idx; -- u64 cgid; -- int ret; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) { -- scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]", -- cpu); -- return -ENOENT; -- } -- -- bpf_spin_lock(&pairc->lock); -- pairc->active_mask &= ~in_pair_mask; -- -- expired = time_before(pairc->started_at + pair_batch_dur_ns, now); -- if (expired || pairc->draining) { -- u64 new_cgid = 0; -- -- __sync_fetch_and_add(&nr_exps, 1); -- -- /* -- * We're done with the current cgid. An obvious optimization -- * would be not draining if the next cgroup is the current one. -- * For now, be dumb and always expire. -- */ -- pairc->draining = true; -- -- pair_preempted = pairc->preempted_mask; -- if (pairc->active_mask || pair_preempted) { -- /* -- * The other CPU is still active, or is no longer under -- * our control due to e.g. being preempted by a higher -- * priority sched_class. We want to wait until this -- * cgroup expires, or until control of our pair CPU has -- * been returned to us. -- * -- * If the pair controls its CPU, and the time already -- * expired, kick. When the other CPU arrives at -- * dispatch and clears its active mask, it'll push the -- * pair to the next cgroup and kick this CPU. -- */ -- __sync_fetch_and_add(&nr_exp_waits, 1); -- bpf_spin_unlock(&pairc->lock); -- if (expired && !pair_preempted) -- kick_pair = true; -- goto out_maybe_kick; -- } -- -- bpf_spin_unlock(&pairc->lock); -- -- /* -- * Pick the next cgroup. It'd be easier / cleaner to not drop -- * pairc->lock and use stronger synchronization here especially -- * given that we'll be switching cgroups significantly less -- * frequently than tasks. Unfortunately, bpf_spin_lock can't -- * really protect anything non-trivial. Let's do opportunistic -- * operations instead. -- */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u32 *q_idx; -- u64 *cgq_len; -- -- if (bpf_map_pop_elem(&top_q, &new_cgid)) { -- /* no active cgroup, go idle */ -- __sync_fetch_and_add(&nr_exp_empty, 1); -- return 0; -- } -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid); -- if (!q_idx) -- continue; -- -- /* -- * This is the only place where empty cgroups are taken -- * off the top_q. -- */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len || !*cgq_len) -- continue; -- -- /* -- * If it has any tasks, requeue as we may race and not -- * execute it. -- */ -- bpf_map_push_elem(&top_q, &new_cgid, 0); -- break; -- } -- -- bpf_spin_lock(&pairc->lock); -- -- /* -- * The other CPU may already have started on a new cgroup while -- * we dropped the lock. Make sure that we're still draining and -- * start on the new cgroup. -- */ -- if (pairc->draining && !pairc->active_mask) { -- __sync_fetch_and_add(&nr_cgrp_next, 1); -- pairc->cgid = new_cgid; -- pairc->started_at = now; -- pairc->draining = false; -- kick_pair = true; -- } else { -- __sync_fetch_and_add(&nr_cgrp_coll, 1); -- } -- } -- -- cgid = pairc->cgid; -- pairc->active_mask |= in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- -- /* again, it'd be better to do all these with the lock held, oh well */ -- vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!vptr) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return -ENOENT; -- } -- q_idx = *vptr; -- -- /* claim one task from cgrp_q w/ q_idx */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u64 *cgq_len, len; -- -- cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]); -- if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) { -- /* the cgroup must be empty, expire and repeat */ -- __sync_fetch_and_add(&nr_cgrp_empty, 1); -- bpf_spin_lock(&pairc->lock); -- pairc->draining = true; -- pairc->active_mask &= ~in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- return -EAGAIN; -- } -- -- if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len) -- continue; -- -- break; -- } -- -- cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx); -- if (!cgq_map) { -- scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- if (bpf_map_pop_elem(cgq_map, &pid)) { -- scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- p = bpf_task_from_pid(pid); -- if (p) { -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } else { -- /* we don't handle dequeues, retry on lost tasks */ -- __sync_fetch_and_add(&nr_missing, 1); -- return -EAGAIN; -- } -- --out_maybe_kick: -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } -- return 0; --} -- --void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) --{ -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_dispatch(cpu) != -EAGAIN) -- break; -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask &= ~in_pair_mask; -- /* Kick the pair CPU, unless it was also preempted. */ -- kick_pair = !pairc->preempted_mask; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask |= in_pair_mask; -- pairc->active_mask &= ~in_pair_mask; -- /* Kick the pair CPU if it's still running. */ -- kick_pair = pairc->active_mask; -- pairc->draining = true; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- } -- } -- __sync_fetch_and_add(&nr_preemptions, 1); --} -- --s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 i, q_idx; -- -- bpf_for(i, 0, MAX_CGRPS) { -- q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS; -- if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1)) -- break; -- } -- if (i == MAX_CGRPS) -- return -EBUSY; -- -- if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]); -- if (busy) -- *busy = 0; -- return -EBUSY; -- } -- -- return 0; --} -- --void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 *q_idx; -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (q_idx) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]); -- if (busy) -- *busy = 0; -- bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid); -- } --} -- --s32 BPF_STRUCT_OPS(pair_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops pair_ops = { -- .enqueue = (void *)pair_enqueue, -- .dispatch = (void *)pair_dispatch, -- .cpu_acquire = (void *)pair_cpu_acquire, -- .cpu_release = (void *)pair_cpu_release, -- .cgroup_init = (void *)pair_cgroup_init, -- .cgroup_exit = (void *)pair_cgroup_exit, -- .init = (void *)pair_init, -- .exit = (void *)pair_exit, -- .name = "pair", --}; -diff --git a/tools/sched_ext/scx_example_pair.c b/tools/sched_ext/scx_example_pair.c -deleted file mode 100644 -index 18e032bbc173..000000000000 ---- a/tools/sched_ext/scx_example_pair.c -+++ /dev/null -@@ -1,143 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_pair.h" --#include "scx_example_pair.skel.h" -- --const char help_fmt[] = --"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n" --"execute from the same CPU cgroup.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-S STRIDE] [-p]\n" --"\n" --" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_pair *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 stride, i, opt, outer_fd; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_pair__open(); -- assert(skel); -- -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- /* pair up the earlier half to the latter by default, override with -s */ -- stride = skel->rodata->nr_cpu_ids / 2; -- -- while ((opt = getopt(argc, argv, "S:ph")) != -1) { -- switch (opt) { -- case 'S': -- stride = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- for (i = 0; i < skel->rodata->nr_cpu_ids; i++) { -- if (skel->rodata->pair_cpu[i] < 0) { -- skel->rodata->pair_cpu[i] = i + stride; -- skel->rodata->pair_cpu[i + stride] = i; -- skel->rodata->pair_id[i] = i; -- skel->rodata->pair_id[i + stride] = i; -- skel->rodata->in_pair_idx[i] = 0; -- skel->rodata->in_pair_idx[i + stride] = 1; -- } -- } -- -- assert(!scx_example_pair__load(skel)); -- -- /* -- * Populate the cgrp_q_arr map which is an array containing per-cgroup -- * queues. It'd probably be better to do this from BPF but there are too -- * many to initialize statically and there's no way to dynamically -- * populate from BPF. -- */ -- outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr); -- assert(outer_fd >= 0); -- -- printf("Initializing"); -- for (i = 0; i < MAX_CGRPS; i++) { -- s32 inner_fd; -- -- if (exit_req) -- break; -- -- inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, -- sizeof(u32), MAX_QUEUED, NULL); -- assert(inner_fd >= 0); -- assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY)); -- close(inner_fd); -- -- if (!(i % 10)) -- printf("."); -- fflush(stdout); -- } -- printf("\n"); -- -- /* -- * Fully initialized, attach and run. -- */ -- link = bpf_map__attach_struct_ops(skel->maps.pair_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf(" total:%10lu dispatch:%10lu missing:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_dispatched, -- skel->bss->nr_missing); -- printf(" kicks:%10lu preemptions:%7lu\n", -- skel->bss->nr_kicks, -- skel->bss->nr_preemptions); -- printf(" exp:%10lu exp_wait:%10lu exp_empty:%10lu\n", -- skel->bss->nr_exps, -- skel->bss->nr_exp_waits, -- skel->bss->nr_exp_empty); -- printf("cgnext:%10lu cgcoll:%10lu cgempty:%10lu\n", -- skel->bss->nr_cgrp_next, -- skel->bss->nr_cgrp_coll, -- skel->bss->nr_cgrp_empty); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_pair__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_pair.h b/tools/sched_ext/scx_example_pair.h -deleted file mode 100644 -index f60b824272f7..000000000000 ---- a/tools/sched_ext/scx_example_pair.h -+++ /dev/null -@@ -1,10 +0,0 @@ --#ifndef __SCX_EXAMPLE_PAIR_H --#define __SCX_EXAMPLE_PAIR_H -- --enum { -- MAX_CPUS = 4096, -- MAX_QUEUED = 4096, -- MAX_CGRPS = 4096, --}; -- --#endif /* __SCX_EXAMPLE_PAIR_H */ -diff --git a/tools/sched_ext/scx_example_qmap.bpf.c b/tools/sched_ext/scx_example_qmap.bpf.c -deleted file mode 100644 -index 579ab21ae403..000000000000 ---- a/tools/sched_ext/scx_example_qmap.bpf.c -+++ /dev/null -@@ -1,401 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple five-level FIFO queue scheduler. -- * -- * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets -- * assigned to one depending on its compound weight. Each CPU round robins -- * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from -- * queue0, 2 from queue1, 4 from queue2 and so on. -- * -- * This scheduler demonstrates: -- * -- * - BPF-side queueing using PIDs. -- * - Sleepable per-task storage allocation using ops.prep_enable(). -- * - Using ops.cpu_release() to handle a higher priority scheduling class taking -- * the CPU away. -- * - Core-sched support. -- * -- * This scheduler is primarily for demonstration and testing of sched_ext -- * features and unlikely to be useful for actual workloads. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include -- --char _license[] SEC("license") = "GPL"; -- --const volatile u64 slice_ns = SCX_SLICE_DFL; --const volatile bool switch_partial; --const volatile u32 stall_user_nth; --const volatile u32 stall_kernel_nth; --const volatile u32 dsp_inf_loop_after; --const volatile s32 disallow_tgid; -- --u32 test_error_cnt; -- --struct user_exit_info uei; -- --struct qmap { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, u32); --} queue0 SEC(".maps"), -- queue1 SEC(".maps"), -- queue2 SEC(".maps"), -- queue3 SEC(".maps"), -- queue4 SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, 5); -- __type(key, int); -- __array(values, struct qmap); --} queue_arr SEC(".maps") = { -- .values = { -- [0] = &queue0, -- [1] = &queue1, -- [2] = &queue2, -- [3] = &queue3, -- [4] = &queue4, -- }, --}; -- --/* -- * Per-queue sequence numbers to implement core-sched ordering. -- * -- * Tail seq is assigned to each queued task and incremented. Head seq tracks the -- * sequence number of the latest dispatched task. The distance between the a -- * task's seq and the associated queue's head seq is called the queue distance -- * and used when comparing two tasks for ordering. See qmap_core_sched_before(). -- */ --static u64 core_sched_head_seqs[5]; --static u64 core_sched_tail_seqs[5]; -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local_dsq */ -- u64 core_sched_seq; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --/* Per-cpu dispatch index and remaining count */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(max_entries, 2); -- __type(key, u32); -- __type(value, u64); --} dispatch_idx_cnt SEC(".maps"); -- --/* Statistics */ --unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued; --unsigned long nr_core_sched_execed; -- --s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- struct task_ctx *tctx; -- s32 cpu; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- return cpu; -- -- return prev_cpu; --} -- --static int weight_to_idx(u32 weight) --{ -- /* Coarsely map the compound weight to a FIFO. */ -- if (weight <= 25) -- return 0; -- else if (weight <= 50) -- return 1; -- else if (weight < 200) -- return 2; -- else if (weight < 400) -- return 3; -- else -- return 4; --} -- --void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) --{ -- static u32 user_cnt, kernel_cnt; -- struct task_ctx *tctx; -- u32 pid = p->pid; -- int idx = weight_to_idx(p->scx.weight); -- void *ring; -- -- if (p->flags & PF_KTHREAD) { -- if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth)) -- return; -- } else { -- if (stall_user_nth && !(++user_cnt % stall_user_nth)) -- return; -- } -- -- if (test_error_cnt && !--test_error_cnt) -- scx_bpf_error("test triggering error"); -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * All enqueued tasks must have their core_sched_seq updated for correct -- * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in -- * qmap_ops.flags. -- */ -- tctx->core_sched_seq = core_sched_tail_seqs[idx]++; -- -- /* -- * If qmap_select_cpu() is telling us to or this is the last runnable -- * task on the CPU, enqueue locally. -- */ -- if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) { -- tctx->force_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); -- return; -- } -- -- /* -- * If the task was re-enqueued due to the CPU being preempted by a -- * higher priority scheduling class, just re-enqueue the task directly -- * on the global DSQ. As we want another CPU to pick it up, find and -- * kick an idle CPU. -- */ -- if (enq_flags & SCX_ENQ_REENQ) { -- s32 cpu; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags); -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- return; -- } -- -- ring = bpf_map_lookup_elem(&queue_arr, &idx); -- if (!ring) { -- scx_bpf_error("failed to find ring %d", idx); -- return; -- } -- -- /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */ -- if (bpf_map_push_elem(ring, &pid, 0)) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_enqueued, 1); --} -- --/* -- * The BPF queue map doesn't support removal and sched_ext can handle spurious -- * dispatches. qmap_dequeue() is only used to collect statistics. -- */ --void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags) --{ -- __sync_fetch_and_add(&nr_dequeued, 1); -- if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) -- __sync_fetch_and_add(&nr_core_sched_execed, 1); --} -- --static void update_core_sched_head_seq(struct task_struct *p) --{ -- struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- int idx = weight_to_idx(p->scx.weight); -- -- if (tctx) -- core_sched_head_seqs[idx] = tctx->core_sched_seq; -- else -- scx_bpf_error("task_ctx lookup failed"); --} -- --void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 zero = 0, one = 1; -- u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero); -- u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one); -- void *fifo; -- s32 pid; -- int i; -- -- if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) { -- struct task_struct *p; -- -- /* -- * PID 2 should be kthreadd which should mostly be idle and off -- * the scheduler. Let's keep dispatching it to force the kernel -- * to call this function over and over again. -- */ -- p = bpf_task_from_pid(2); -- if (p) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- if (!idx || !cnt) { -- scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt); -- return; -- } -- -- for (i = 0; i < 5; i++) { -- /* Advance the dispatch cursor and pick the fifo. */ -- if (!*cnt) { -- *idx = (*idx + 1) % 5; -- *cnt = 1 << *idx; -- } -- (*cnt)--; -- -- fifo = bpf_map_lookup_elem(&queue_arr, idx); -- if (!fifo) { -- scx_bpf_error("failed to find ring %llu", *idx); -- return; -- } -- -- /* Dispatch or advance. */ -- if (!bpf_map_pop_elem(fifo, &pid)) { -- struct task_struct *p; -- -- p = bpf_task_from_pid(pid); -- if (p) { -- update_core_sched_head_seq(p); -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- *cnt = 0; -- } --} -- --/* -- * The distance from the head of the queue scaled by the weight of the queue. -- * The lower the number, the older the task and the higher the priority. -- */ --static s64 task_qdist(struct task_struct *p) --{ -- int idx = weight_to_idx(p->scx.weight); -- struct task_ctx *tctx; -- s64 qdist; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return 0; -- } -- -- qdist = tctx->core_sched_seq - core_sched_head_seqs[idx]; -- -- /* -- * As queue index increments, the priority doubles. The queue w/ index 3 -- * is dispatched twice more frequently than 2. Reflect the difference by -- * scaling qdists accordingly. Note that the shift amount needs to be -- * flipped depending on the sign to avoid flipping priority direction. -- */ -- if (qdist >= 0) -- return qdist << (4 - idx); -- else -- return qdist << idx; --} -- --/* -- * This is called to determine the task ordering when core-sched is picking -- * tasks to execute on SMT siblings and should encode about the same ordering as -- * the regular scheduling path. Use the priority-scaled distances from the head -- * of the queues to compare the two tasks which should be consistent with the -- * dispatch path behavior. -- */ --bool BPF_STRUCT_OPS(qmap_core_sched_before, -- struct task_struct *a, struct task_struct *b) --{ -- return task_qdist(a) > task_qdist(b); --} -- --void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- u32 cnt; -- -- /* -- * Called when @cpu is taken by a higher priority scheduling class. This -- * makes @cpu no longer available for executing sched_ext tasks. As we -- * don't want the tasks in @cpu's local dsq to sit there until @cpu -- * becomes available again, re-enqueue them into the global dsq. See -- * %SCX_ENQ_REENQ handling in qmap_enqueue(). -- */ -- cnt = scx_bpf_reenqueue_local(); -- if (cnt) -- __sync_fetch_and_add(&nr_reenqueued, cnt); --} -- --s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (p->tgid == disallow_tgid) -- p->scx.disallow = true; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(qmap_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops qmap_ops = { -- .select_cpu = (void *)qmap_select_cpu, -- .enqueue = (void *)qmap_enqueue, -- .dequeue = (void *)qmap_dequeue, -- .dispatch = (void *)qmap_dispatch, -- .core_sched_before = (void *)qmap_core_sched_before, -- .cpu_release = (void *)qmap_cpu_release, -- .prep_enable = (void *)qmap_prep_enable, -- .init = (void *)qmap_init, -- .exit = (void *)qmap_exit, -- .flags = SCX_OPS_ENQ_LAST, -- .timeout_ms = 5000U, -- .name = "qmap", --}; -diff --git a/tools/sched_ext/scx_example_qmap.c b/tools/sched_ext/scx_example_qmap.c -deleted file mode 100644 -index ccb4814ee61b..000000000000 ---- a/tools/sched_ext/scx_example_qmap.c -+++ /dev/null -@@ -1,107 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_qmap.skel.h" -- --const char help_fmt[] = --"A simple five-level FIFO queue sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" --" -t COUNT Stall every COUNT'th user thread\n" --" -T COUNT Stall every COUNT'th kernel thread\n" --" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n" --" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_qmap *skel; -- struct bpf_link *link; -- int opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_qmap__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) { -- switch (opt) { -- case 's': -- skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; -- break; -- case 'e': -- skel->bss->test_error_cnt = strtoul(optarg, NULL, 0); -- break; -- case 't': -- skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0); -- break; -- case 'T': -- skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0); -- break; -- case 'l': -- skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0); -- break; -- case 'd': -- skel->rodata->disallow_tgid = strtol(optarg, NULL, 0); -- if (skel->rodata->disallow_tgid < 0) -- skel->rodata->disallow_tgid = getpid(); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_qmap__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.qmap_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- long nr_enqueued = skel->bss->nr_enqueued; -- long nr_dispatched = skel->bss->nr_dispatched; -- -- printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n", -- nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, -- skel->bss->nr_reenqueued, skel->bss->nr_dequeued, -- skel->bss->nr_core_sched_execed); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_qmap__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_simple.bpf.c b/tools/sched_ext/scx_example_simple.bpf.c -deleted file mode 100644 -index 4bccca3e2047..000000000000 ---- a/tools/sched_ext/scx_example_simple.bpf.c -+++ /dev/null -@@ -1,128 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple scheduler. -- * -- * By default, it operates as a simple global weighted vtime scheduler and can -- * be switched to FIFO scheduling. It also demonstrates the following niceties. -- * -- * - Statistics tracking how many tasks are queued to local and global dsq's. -- * - Termination notification for userspace. -- * -- * While very simple, this scheduler should work reasonably well on CPUs with a -- * uniform L3 cache topology. While preemption is not implemented, the fact that -- * the scheduling queue is shared across all CPUs means that whatever is at the -- * front of the queue is likely to be executed fairly quickly given enough -- * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads -- * but comes with the usual problems with FIFO scheduling where saturating -- * threads can easily drown out interactive ones. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --static u64 vtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, 2); /* [local, global] */ --} stats SEC(".maps"); -- --static void stat_inc(u32 idx) --{ -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); -- if (cnt_p) -- (*cnt_p)++; --} -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) --{ -- /* -- * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that -- * running @p on its CPU directly shouldn't affect fairness. Just queue -- * it on the local FIFO. -- */ -- if (enq_flags & SCX_ENQ_LOCAL) { -- stat_inc(0); /* count local queueing */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- return; -- } -- -- stat_inc(1); /* count global queueing */ -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) -- vtime = vtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --void BPF_STRUCT_OPS(simple_running, struct task_struct *p) --{ -- if (fifo_sched) -- return; -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(vtime_now, p->scx.dsq_vtime)) -- vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --s32 BPF_STRUCT_OPS(simple_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .running = (void *)simple_running, -- .stopping = (void *)simple_stopping, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", --}; -diff --git a/tools/sched_ext/scx_example_simple.c b/tools/sched_ext/scx_example_simple.c -deleted file mode 100644 -index 486b401f7c95..000000000000 ---- a/tools/sched_ext/scx_example_simple.c -+++ /dev/null -@@ -1,101 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_simple.skel.h" -- --const char help_fmt[] = --"A simple sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-f] [-p]\n" --"\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int simple) --{ -- exit_req = 1; --} -- --static void read_stats(struct scx_example_simple *skel, u64 *stats) --{ -- int nr_cpus = libbpf_num_possible_cpus(); -- u64 cnts[2][nr_cpus]; -- u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * 2); -- -- for (idx = 0; idx < 2; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_simple *skel; -- struct bpf_link *link; -- u32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_simple__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "fph")) != -1) { -- switch (opt) { -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_simple__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.simple_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- u64 stats[2]; -- -- read_stats(skel, stats); -- printf("local=%lu global=%lu\n", stats[0], stats[1]); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_simple__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_userland.bpf.c b/tools/sched_ext/scx_example_userland.bpf.c -deleted file mode 100644 -index a089bc6bbe86..000000000000 ---- a/tools/sched_ext/scx_example_userland.bpf.c -+++ /dev/null -@@ -1,269 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A minimal userland scheduler. -- * -- * In terms of scheduling, this provides two different types of behaviors: -- * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity. -- * All such tasks are direct-dispatched from the kernel, and are never -- * enqueued in user space. -- * 2. A primitive vruntime scheduler that is implemented in user space, for all -- * other tasks. -- * -- * Some parts of this example user space scheduler could be implemented more -- * efficiently using more complex and sophisticated data structures. For -- * example, rather than using BPF_MAP_TYPE_QUEUE's, -- * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between -- * user space and kernel space. Similarly, we use a simple vruntime-sorted list -- * in user space, but an rbtree could be used instead. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include --#include "scx_common.bpf.h" --#include "scx_example_userland_common.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; --const volatile s32 usersched_pid; -- --/* !0 for veristat, set during init */ --const volatile u32 num_possible_cpus = 64; -- --/* Stats that are printed by user space. */ --u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues; -- --struct user_exit_info uei; -- --/* -- * Whether the user space scheduler needs to be scheduled due to a task being -- * enqueued in user space. -- */ --static bool usersched_needed; -- --/* -- * The map containing tasks that are enqueued in user space from the kernel. -- * -- * This map is drained by the user space scheduler. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, struct scx_userland_enqueued_task); --} enqueued SEC(".maps"); -- --/* -- * The map containing tasks that are dispatched to the kernel from user space. -- * -- * Drained by the kernel in userland_dispatch(). -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, s32); --} dispatched SEC(".maps"); -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local DSQ */ --}; -- --/* Map that contains task-local storage. */ --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --static bool is_usersched_task(const struct task_struct *p) --{ -- return p->pid == usersched_pid; --} -- --static bool keep_in_kernel(const struct task_struct *p) --{ -- return p->nr_cpus_allowed < num_possible_cpus; --} -- --static struct task_struct *usersched_task(void) --{ -- struct task_struct *p; -- -- p = bpf_task_from_pid(usersched_pid); -- /* -- * Should never happen -- the usersched task should always be managed -- * by sched_ext. -- */ -- if (!p) { -- scx_bpf_error("Failed to find usersched task %d", usersched_pid); -- /* -- * We should never hit this path, and we error out of the -- * scheduler above just in case, so the scheduler will soon be -- * be evicted regardless. So as to simplify the logic in the -- * caller to not have to check for NULL, return an acquired -- * reference to the current task here rather than NULL. -- */ -- return bpf_task_acquire(bpf_get_current_task_btf()); -- } -- -- return p; --} -- --s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- if (keep_in_kernel(p)) { -- s32 cpu; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to look up task-local storage for %s", p->comm); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- tctx->force_local = true; -- return cpu; -- } -- } -- -- return prev_cpu; --} -- --static void dispatch_user_scheduler(void) --{ -- struct task_struct *p; -- -- usersched_needed = false; -- p = usersched_task(); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); --} -- --static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags) --{ -- struct scx_userland_enqueued_task task; -- -- memset(&task, 0, sizeof(task)); -- task.pid = p->pid; -- task.sum_exec_runtime = p->se.sum_exec_runtime; -- task.weight = p->scx.weight; -- -- if (bpf_map_push_elem(&enqueued, &task, 0)) { -- /* -- * If we fail to enqueue the task in user space, put it -- * directly on the global DSQ. -- */ -- __sync_fetch_and_add(&nr_failed_enqueues, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- __sync_fetch_and_add(&nr_user_enqueues, 1); -- usersched_needed = true; -- } --} -- --void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags) --{ -- if (keep_in_kernel(p)) { -- u64 dsq_id = SCX_DSQ_GLOBAL; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to lookup task ctx for %s", p->comm); -- return; -- } -- -- if (tctx->force_local) -- dsq_id = SCX_DSQ_LOCAL; -- tctx->force_local = false; -- scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags); -- __sync_fetch_and_add(&nr_kernel_enqueues, 1); -- return; -- } else if (!is_usersched_task(p)) { -- enqueue_task_in_user_space(p, enq_flags); -- } --} -- --void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (usersched_needed) -- dispatch_user_scheduler(); -- -- bpf_repeat(4096) { -- s32 pid; -- struct task_struct *p; -- -- if (bpf_map_pop_elem(&dispatched, &pid)) -- break; -- -- /* -- * The task could have exited by the time we get around to -- * dispatching it. Treat this as a normal occurrence, and simply -- * move onto the next iteration. -- */ -- p = bpf_task_from_pid(pid); -- if (!p) -- continue; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } --} -- --s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(userland_init) --{ -- if (num_possible_cpus == 0) { -- scx_bpf_error("User scheduler # CPUs uninitialized (%d)", -- num_possible_cpus); -- return -EINVAL; -- } -- -- if (usersched_pid <= 0) { -- scx_bpf_error("User scheduler pid uninitialized (%d)", -- usersched_pid); -- return -EINVAL; -- } -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops userland_ops = { -- .select_cpu = (void *)userland_select_cpu, -- .enqueue = (void *)userland_enqueue, -- .dispatch = (void *)userland_dispatch, -- .prep_enable = (void *)userland_prep_enable, -- .init = (void *)userland_init, -- .exit = (void *)userland_exit, -- .timeout_ms = 3000, -- .name = "userland", --}; -diff --git a/tools/sched_ext/scx_example_userland.c b/tools/sched_ext/scx_example_userland.c -deleted file mode 100644 -index 4152b1e65fe1..000000000000 ---- a/tools/sched_ext/scx_example_userland.c -+++ /dev/null -@@ -1,402 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext user space scheduler which provides vruntime semantics -- * using a simple ordered-list implementation. -- * -- * Each CPU in the system resides in a single, global domain. This precludes -- * the need to do any load balancing between domains. The scheduler could -- * easily be extended to support multiple domains, with load balancing -- * happening in user space. -- * -- * Any task which has any CPU affinity is scheduled entirely in BPF. This -- * program only schedules tasks which may run on any CPU. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include "user_exit_info.h" --#include "scx_example_userland_common.h" --#include "scx_example_userland.skel.h" -- --const char help_fmt[] = --"A minimal userland sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-b BATCH] [-p]\n" --"\n" --" -b BATCH The number of tasks to batch when dispatching (default: 8)\n" --" -p Don't switch all, switch only tasks on SCHED_EXT policy\n" --" -h Display this help and exit\n"; -- --/* Defined in UAPI */ --#define SCHED_EXT 7 -- --/* Number of tasks to batch when dispatching to user space. */ --static __u32 batch_size = 8; -- --static volatile int exit_req; --static int enqueued_fd, dispatched_fd; -- --static struct scx_example_userland *skel; --static struct bpf_link *ops_link; -- --/* Stats collected in user space. */ --static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches; -- --/* The data structure containing tasks that are enqueued in user space. */ --struct enqueued_task { -- LIST_ENTRY(enqueued_task) entries; -- __u64 sum_exec_runtime; -- double vruntime; --}; -- --/* -- * Use a vruntime-sorted list to store tasks. This could easily be extended to -- * a more optimal data structure, such as an rbtree as is done in CFS. We -- * currently elect to use a sorted list to simplify the example for -- * illustrative purposes. -- */ --LIST_HEAD(listhead, enqueued_task); -- --/* -- * A vruntime-sorted list of tasks. The head of the list contains the task with -- * the lowest vruntime. That is, the task that has the "highest" claim to be -- * scheduled. -- */ --static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head); -- --/* -- * The statically allocated array of tasks. We use a statically allocated list -- * here to avoid having to allocate on the enqueue path, which could cause a -- * deadlock. A more substantive user space scheduler could e.g. provide a hook -- * for newly enabled tasks that are passed to the scheduler from the -- * .prep_enable() callback to allows the scheduler to allocate on safe paths. -- */ --struct enqueued_task tasks[USERLAND_MAX_TASKS]; -- --static double min_vruntime; -- --static void sigint_handler(int userland) --{ -- exit_req = 1; --} -- --static __u32 task_pid(const struct enqueued_task *task) --{ -- return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task); --} -- --static int dispatch_task(s32 pid) --{ -- int err; -- -- err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d\n", pid); -- exit_req = 1; -- } else { -- nr_vruntime_dispatches++; -- } -- -- return err; --} -- --static struct enqueued_task *get_enqueued_task(__s32 pid) --{ -- if (pid >= USERLAND_MAX_TASKS) -- return NULL; -- -- return &tasks[pid]; --} -- --static double calc_vruntime_delta(__u64 weight, __u64 delta) --{ -- double weight_f = (double)weight / 100.0; -- double delta_f = (double)delta; -- -- return delta_f / weight_f; --} -- --static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task) --{ -- __u64 delta; -- -- delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime; -- -- enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta); -- if (min_vruntime > enqueued->vruntime) -- enqueued->vruntime = min_vruntime; -- enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime; --} -- --static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task) --{ -- struct enqueued_task *curr, *enqueued, *prev; -- -- curr = get_enqueued_task(bpf_task->pid); -- if (!curr) -- return ENOENT; -- -- update_enqueued(curr, bpf_task); -- nr_vruntime_enqueues++; -- -- /* -- * Enqueue the task in a vruntime-sorted list. A more optimal data -- * structure such as an rbtree could easily be used as well. We elect -- * to use a list here simply because it's less code, and thus the -- * example is less convoluted and better serves to illustrate what a -- * user space scheduler could look like. -- */ -- -- if (LIST_EMPTY(&vruntime_head)) { -- LIST_INSERT_HEAD(&vruntime_head, curr, entries); -- return 0; -- } -- -- LIST_FOREACH(enqueued, &vruntime_head, entries) { -- if (curr->vruntime <= enqueued->vruntime) { -- LIST_INSERT_BEFORE(enqueued, curr, entries); -- return 0; -- } -- prev = enqueued; -- } -- -- LIST_INSERT_AFTER(prev, curr, entries); -- -- return 0; --} -- --static void drain_enqueued_map(void) --{ -- while (1) { -- struct scx_userland_enqueued_task task; -- int err; -- -- if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) -- return; -- -- err = vruntime_enqueue(&task); -- if (err) { -- fprintf(stderr, "Failed to enqueue task %d: %s\n", -- task.pid, strerror(err)); -- exit_req = 1; -- return; -- } -- } --} -- --static void dispatch_batch(void) --{ -- __u32 i; -- -- for (i = 0; i < batch_size; i++) { -- struct enqueued_task *task; -- int err; -- __s32 pid; -- -- task = LIST_FIRST(&vruntime_head); -- if (!task) -- return; -- -- min_vruntime = task->vruntime; -- pid = task_pid(task); -- LIST_REMOVE(task, entries); -- err = dispatch_task(pid); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d in %u\n", -- pid, i); -- return; -- } -- } --} -- --static void *run_stats_printer(void *arg) --{ -- while (!exit_req) { -- __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; -- -- nr_failed_enqueues = skel->bss->nr_failed_enqueues; -- nr_kernel_enqueues = skel->bss->nr_kernel_enqueues; -- nr_user_enqueues = skel->bss->nr_user_enqueues; -- total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues; -- -- printf("o-----------------------o\n"); -- printf("| BPF ENQUEUES |\n"); -- printf("|-----------------------|\n"); -- printf("| kern: %10llu |\n", nr_kernel_enqueues); -- printf("| user: %10llu |\n", nr_user_enqueues); -- printf("| failed: %10llu |\n", nr_failed_enqueues); -- printf("| -------------------- |\n"); -- printf("| total: %10llu |\n", total); -- printf("| |\n"); -- printf("|-----------------------|\n"); -- printf("| VRUNTIME / USER |\n"); -- printf("|-----------------------|\n"); -- printf("| enq: %10llu |\n", nr_vruntime_enqueues); -- printf("| disp: %10llu |\n", nr_vruntime_dispatches); -- printf("o-----------------------o\n"); -- printf("\n\n"); -- sleep(1); -- } -- -- return NULL; --} -- --static int spawn_stats_thread(void) --{ -- pthread_t stats_printer; -- -- return pthread_create(&stats_printer, NULL, run_stats_printer, NULL); --} -- --static int bootstrap(int argc, char **argv) --{ -- int err; -- __u32 opt; -- struct sched_param sched_param = { -- .sched_priority = sched_get_priority_max(SCHED_EXT), -- }; -- bool switch_partial = false; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- /* -- * Enforce that the user scheduler task is managed by sched_ext. The -- * task eagerly drains the list of enqueued tasks in its main work -- * loop, and then yields the CPU. The BPF scheduler only schedules the -- * user space scheduler task when at least one other task in the system -- * needs to be scheduled. -- */ -- err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param); -- if (err) { -- fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err)); -- return err; -- } -- -- while ((opt = getopt(argc, argv, "b:ph")) != -1) { -- switch (opt) { -- case 'b': -- batch_size = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- exit(opt != 'h'); -- } -- } -- -- /* -- * It's not always safe to allocate in a user space scheduler, as an -- * enqueued task could hold a lock that we require in order to be able -- * to allocate. -- */ -- err = mlockall(MCL_CURRENT | MCL_FUTURE); -- if (err) { -- fprintf(stderr, "Failed to prefault and lock address space: %s\n", -- strerror(err)); -- return err; -- } -- -- skel = scx_example_userland__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno)); -- return errno; -- } -- skel->rodata->num_possible_cpus = libbpf_num_possible_cpus(); -- assert(skel->rodata->num_possible_cpus > 0); -- skel->rodata->usersched_pid = getpid(); -- assert(skel->rodata->usersched_pid > 0); -- skel->rodata->switch_partial = switch_partial; -- -- err = scx_example_userland__load(skel); -- if (err) { -- fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- enqueued_fd = bpf_map__fd(skel->maps.enqueued); -- dispatched_fd = bpf_map__fd(skel->maps.dispatched); -- assert(enqueued_fd > 0); -- assert(dispatched_fd > 0); -- -- err = spawn_stats_thread(); -- if (err) { -- fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops); -- if (!ops_link) { -- fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno)); -- err = errno; -- goto destroy_skel; -- } -- -- return 0; -- --destroy_skel: -- scx_example_userland__destroy(skel); -- exit_req = 1; -- return err; --} -- --static void sched_main_loop(void) --{ -- while (!exit_req) { -- /* -- * Perform the following work in the main user space scheduler -- * loop: -- * -- * 1. Drain all tasks from the enqueued map, and enqueue them -- * to the vruntime sorted list. -- * -- * 2. Dispatch a batch of tasks from the vruntime sorted list -- * down to the kernel. -- * -- * 3. Yield the CPU back to the system. The BPF scheduler will -- * reschedule the user space scheduler once another task has -- * been enqueued to user space. -- */ -- drain_enqueued_map(); -- dispatch_batch(); -- sched_yield(); -- } --} -- --int main(int argc, char **argv) --{ -- int err; -- -- err = bootstrap(argc, argv); -- if (err) { -- fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err)); -- return err; -- } -- -- sched_main_loop(); -- -- exit_req = 1; -- bpf_link__destroy(ops_link); -- uei_print(&skel->bss->uei); -- scx_example_userland__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_userland_common.h b/tools/sched_ext/scx_example_userland_common.h -deleted file mode 100644 -index 639c6809c5ff..000000000000 ---- a/tools/sched_ext/scx_example_userland_common.h -+++ /dev/null -@@ -1,19 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* Copyright (c) 2022 Meta, Inc */ -- --#ifndef __SCX_USERLAND_COMMON_H --#define __SCX_USERLAND_COMMON_H -- --#define USERLAND_MAX_TASKS 8192 -- --/* -- * An instance of a task that has been enqueued by the kernel for consumption -- * by a user space global scheduler thread. -- */ --struct scx_userland_enqueued_task { -- __s32 pid; -- u64 sum_exec_runtime; -- u64 weight; --}; -- --#endif // __SCX_USERLAND_COMMON_H -diff --git a/tools/sched_ext/user_exit_info.h b/tools/sched_ext/user_exit_info.h -deleted file mode 100644 -index e701ef0e0b86..000000000000 ---- a/tools/sched_ext/user_exit_info.h -+++ /dev/null -@@ -1,50 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Define struct user_exit_info which is shared between BPF and userspace parts -- * to communicate exit status and other information. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __USER_EXIT_INFO_H --#define __USER_EXIT_INFO_H -- --struct user_exit_info { -- int type; -- char reason[128]; -- char msg[1024]; --}; -- --#ifdef __bpf__ -- --#include "vmlinux.h" --#include -- --static inline void uei_record(struct user_exit_info *uei, -- const struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason); -- bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg); -- /* use __sync to force memory barrier */ -- __sync_val_compare_and_swap(&uei->type, uei->type, ei->type); --} -- --#else /* !__bpf__ */ -- --static inline bool uei_exited(struct user_exit_info *uei) --{ -- /* use __sync to force memory barrier */ -- return __sync_val_compare_and_swap(&uei->type, -1, -1); --} -- --static inline void uei_print(const struct user_exit_info *uei) --{ -- fprintf(stderr, "EXIT: %s", uei->reason); -- if (uei->msg[0] != '\0') -- fprintf(stderr, " (%s)", uei->msg); -- fputs("\n", stderr); --} -- --#endif /* __bpf__ */ --#endif /* __USER_EXIT_INFO_H */ -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -index ce05928392bf..f47f79079297 100755 ---- a/kernel/sched/hmbird/hmbird.c -+++ b/kernel/sched/hmbird/hmbird.c -@@ -633,14 +633,14 @@ static void init_isolate_cpus(void) - WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -- cpumask_set_cpu(0, iso_masks.big); -- cpumask_set_cpu(1, iso_masks.big); -- cpumask_set_cpu(2, iso_masks.big); -- cpumask_set_cpu(3, iso_masks.big); -- cpumask_set_cpu(4, iso_masks.big); -- cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(0, iso_masks.little); -+ cpumask_set_cpu(1, iso_masks.little); -+ cpumask_set_cpu(2, iso_masks.little); -+ cpumask_set_cpu(3, iso_masks.little); -+ cpumask_set_cpu(4, iso_masks.little); -+ cpumask_set_cpu(5, iso_masks.little); - cpumask_set_cpu(6, iso_masks.partial); -- cpumask_set_cpu(7, iso_masks.ex_free); -+ cpumask_set_cpu(7, iso_masks.big); - } - - extern spinlock_t css_set_lock; diff --git a/oneplus/hmbird/fengchi_OP-ACE-5-PRO_A16.patch b/oneplus/hmbird/fengchi_OP-ACE-5-PRO_A16.patch deleted file mode 100644 index 81dfa588..00000000 --- a/oneplus/hmbird/fengchi_OP-ACE-5-PRO_A16.patch +++ /dev/null @@ -1,19895 +0,0 @@ -diff --git b/Documentation/scheduler/index.rst a/Documentation/scheduler/index.rst -index 0b650bb550e6..3170747226f6 100644 ---- b/Documentation/scheduler/index.rst -+++ a/Documentation/scheduler/index.rst -@@ -19,7 +19,6 @@ Scheduler - sched-nice-design - sched-rt-group - sched-stats -- sched-ext - sched-debug - - text_files -diff --git b/Documentation/scheduler/sched-ext.rst a/Documentation/scheduler/sched-ext.rst -deleted file mode 100644 -index 84c30b44f104..000000000000 ---- b/Documentation/scheduler/sched-ext.rst -+++ /dev/null -@@ -1,230 +0,0 @@ --========================== --Extensible Scheduler Class --========================== -- --sched_ext is a scheduler class whose behavior can be defined by a set of BPF --programs - the BPF scheduler. -- --* sched_ext exports a full scheduling interface so that any scheduling -- algorithm can be implemented on top. -- --* The BPF scheduler can group CPUs however it sees fit and schedule them -- together, as tasks aren't tied to specific CPUs at the time of wakeup. -- --* The BPF scheduler can be turned on and off dynamically anytime. -- --* The system integrity is maintained no matter what the BPF scheduler does. -- The default scheduling behavior is restored anytime an error is detected, -- a runnable task stalls, or on invoking the SysRq key sequence -- :kbd:`SysRq-S`. -- --Switching to and from sched_ext --=============================== -- --``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and --``tools/sched_ext`` contains the example schedulers. -- --sched_ext is used only when the BPF scheduler is loaded and running. -- --If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be --treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is --loaded. On load, such tasks will be switched to and scheduled by sched_ext. -- --The BPF scheduler can choose to schedule all normal and lower class tasks by --calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this --case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and --``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers, --this mode can be selected with the ``-a`` option. -- --Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or --detection of any internal error including stalled runnable tasks aborts the --BPF scheduler and reverts all tasks back to CFS. -- --.. code-block:: none -- -- # make -j16 -C tools/sched_ext -- # tools/sched_ext/scx_example_simple -- local=0 global=3 -- local=5 global=24 -- local=9 global=44 -- local=13 global=56 -- local=17 global=72 -- ^CEXIT: BPF scheduler unregistered -- --If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler --and whether a given task is on sched_ext can be determined as follows: -- --.. code-block:: none -- -- # cat /sys/kernel/debug/sched/ext -- ops : simple -- enabled : 1 -- switching_all : 1 -- switched_all : 1 -- enable_state : enabled -- -- # grep ext /proc/self/sched -- ext.enabled : 1 -- --The Basics --========== -- --Userspace can implement an arbitrary BPF scheduler by loading a set of BPF --programs that implement ``struct sched_ext_ops``. The only mandatory field --is ``ops.name`` which must be a valid BPF object name. All operations are --optional. The following modified excerpt is from --``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO --scheduler. -- --.. code-block:: c -- -- s32 BPF_STRUCT_OPS(simple_init) -- { -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; -- } -- -- void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) -- { -- if (enq_flags & SCX_ENQ_LOCAL) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags); -- } -- -- void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) -- { -- exit_type = ei->type; -- } -- -- SEC(".struct_ops") -- struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", -- }; -- --Dispatch Queues ----------------- -- --To match the impedance between the scheduler core and the BPF scheduler, --sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a --priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``), --and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage --an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and --``scx_bpf_destroy_dsq()``. -- --A CPU always executes a task from its local DSQ. A task is "dispatched" to a --DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's --local DSQ. -- --When a CPU is looking for the next task to run, if the local DSQ is not --empty, the first task is picked. Otherwise, the CPU tries to consume the --global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()`` --is invoked. -- --Scheduling Cycle ------------------ -- --The following briefly shows how a waking task is scheduled and executed. -- --1. When a task is waking up, ``ops.select_cpu()`` is the first operation -- invoked. This serves two purposes. First, CPU selection optimization -- hint. Second, waking up the selected CPU if idle. -- -- The CPU selected by ``ops.select_cpu()`` is an optimization hint and not -- binding. The actual decision is made at the last step of scheduling. -- However, there is a small performance gain if the CPU -- ``ops.select_cpu()`` returns matches the CPU the task eventually runs on. -- -- A side-effect of selecting a CPU is waking it up from idle. While a BPF -- scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper, -- using ``ops.select_cpu()`` judiciously can be simpler and more efficient. -- -- Note that the scheduler core will ignore an invalid CPU selection, for -- example, if it's outside the allowed cpumask of the task. -- --2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can -- make one of the following decisions: -- -- * Immediately dispatch the task to either the global or local DSQ by -- calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or -- ``SCX_DSQ_LOCAL``, respectively. -- -- * Immediately dispatch the task to a custom DSQ by calling -- ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63. -- -- * Queue the task on the BPF side. -- --3. When a CPU is ready to schedule, it first looks at its local DSQ. If -- empty, it then looks at the global DSQ. If there still isn't a task to -- run, ``ops.dispatch()`` is invoked which can use the following two -- functions to populate the local DSQ. -- -- * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can -- be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``, -- ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()`` -- currently can't be called with BPF locks held, this is being worked on -- and will be supported. ``scx_bpf_dispatch()`` schedules dispatching -- rather than performing them immediately. There can be up to -- ``ops.dispatch_max_batch`` pending tasks. -- -- * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ -- to the dispatching DSQ. This function cannot be called with any BPF -- locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks -- before trying to consume the specified DSQ. -- --4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ, -- the CPU runs the first one. If empty, the following steps are taken: -- -- * Try to consume the global DSQ. If successful, run the task. -- -- * If ``ops.dispatch()`` has dispatched any tasks, retry #3. -- -- * If the previous task is an SCX task and still runnable, keep executing -- it (see ``SCX_OPS_ENQ_LAST``). -- -- * Go idle. -- --Note that the BPF scheduler can always choose to dispatch tasks immediately --in ``ops.enqueue()`` as illustrated in the above simple example. If only the --built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as --a task is never queued on the BPF scheduler and both the local and global --DSQs are consumed automatically. -- --``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use --``scx_bpf_dispatch_vtime()`` for the priority queue. See the function --documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for --more information. -- --Where to Look --============= -- --* ``include/linux/sched/ext.h`` defines the core data structures, ops table -- and constants. -- --* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. -- The functions prefixed with ``scx_bpf_`` can be called from the BPF -- scheduler. -- --* ``tools/sched_ext/`` hosts example BPF scheduler implementations. -- -- * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example -- using a custom DSQ. -- -- * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting -- five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. -- --ABI Instability --=============== -- --The APIs provided by sched_ext to BPF schedulers programs have no stability --guarantees. This includes the ops table callbacks and constants defined in --``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in --``kernel/sched/ext.c``. -- --While we will attempt to provide a relatively stable API surface when --possible, they are subject to change without warning between kernel --versions. -diff --git b/MAINTAINERS a/MAINTAINERS -index 9fc6108503c3..2384b55682ee 100644 ---- b/MAINTAINERS -+++ a/MAINTAINERS -@@ -19132,8 +19132,6 @@ R: Ben Segall (CONFIG_CFS_BANDWIDTH) - R: Mel Gorman (CONFIG_NUMA_BALANCING) - R: Daniel Bristot de Oliveira (SCHED_DEADLINE) - R: Valentin Schneider (TOPOLOGY) --R: Tejun Heo (SCHED_EXT) --R: David Vernet (SCHED_EXT) - L: linux-kernel@vger.kernel.org - S: Maintained - T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core -@@ -19142,7 +19140,6 @@ F: include/linux/sched.h - F: include/linux/wait.h - F: include/uapi/linux/sched.h - F: kernel/sched/ --F: tools/sched_ext/ - - SCSI LIBSAS SUBSYSTEM - R: John Garry -diff --git b/arch/arm64/configs/defconfig a/arch/arm64/configs/defconfig -index f93980c09d7f..c2d530535980 100644 ---- b/arch/arm64/configs/defconfig -+++ a/arch/arm64/configs/defconfig -@@ -6,8 +6,6 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_BPF_SYSCALL=y - CONFIG_BPF_JIT=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_BSD_PROCESS_ACCT=y - CONFIG_BSD_PROCESS_ACCT_V3=y -@@ -1587,3 +1585,4 @@ CONFIG_CORESIGHT_STM=m - CONFIG_CORESIGHT_CPU_DEBUG=m - CONFIG_CORESIGHT_CTI=m - CONFIG_MEMTEST=y -+CONFIG_HMBIRD_SCHED=y -diff --git b/arch/arm64/configs/gki_defconfig a/arch/arm64/configs/gki_defconfig -index 4f756dca5e05..6c1bb94f875d 100644 ---- b/arch/arm64/configs/gki_defconfig -+++ a/arch/arm64/configs/gki_defconfig -@@ -10,8 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y - # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set - CONFIG_BPF_LSM=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_TASKSTATS=y - CONFIG_TASK_DELAY_ACCT=y -diff --git b/drivers/gpu/drm/msm/msm_drv.c a/drivers/gpu/drm/msm/msm_drv.c -index 4bd028fa7500..5825ce9d6d7b 100755 ---- b/drivers/gpu/drm/msm/msm_drv.c -+++ a/drivers/gpu/drm/msm/msm_drv.c -@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) - } - - sched_set_fifo(ev_thread->worker->task); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3); -+#endif - } - - ret = drm_vblank_init(ddev, priv->num_crtcs); -diff --git b/drivers/tty/sysrq.c a/drivers/tty/sysrq.c -index bd001445815e..dcf2e1ebf9c5 100644 ---- b/drivers/tty/sysrq.c -+++ a/drivers/tty/sysrq.c -@@ -524,7 +524,6 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { - NULL, /* P */ - NULL, /* Q */ - NULL, /* R */ -- /* S: May be registered by sched_ext for resetting */ - NULL, /* S */ - NULL, /* T */ - NULL, /* U */ -diff --git b/include/asm-generic/vmlinux.lds.h a/include/asm-generic/vmlinux.lds.h -index c45078a445c8..6c15659f874e 100755 ---- b/include/asm-generic/vmlinux.lds.h -+++ a/include/asm-generic/vmlinux.lds.h -@@ -131,7 +131,7 @@ - *(__dl_sched_class) \ - *(__rt_sched_class) \ - *(__fair_sched_class) \ -- *(__ext_sched_class) \ -+ *(__hmbird_sched_class) \ - *(__idle_sched_class) \ - __sched_class_lowest = .; - -diff --git b/include/linux/sched.h a/include/linux/sched.h -index ba49d7bd2b67..dd6b1d58af01 100755 ---- b/include/linux/sched.h -+++ a/include/linux/sched.h -@@ -73,7 +73,9 @@ struct task_delay_info; - struct task_group; - struct user_event_mm; - --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - - /* - * Task state bitmask. NOTE! These bits are also -@@ -298,11 +300,6 @@ enum { - - extern void scheduler_tick(void); - --#ifdef CONFIG_SLIM_SCHED --extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer); --extern void stop_shadow_tick_timer(void); --#endif -- - #define MAX_SCHEDULE_TIMEOUT LONG_MAX - - extern long schedule_timeout(long timeout); -@@ -1933,6 +1925,91 @@ static inline int task_nice(const struct task_struct *p) - return PRIO_TO_NICE((p)->static_prio); - } - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline bool task_is_top_task(struct task_struct *p) -+{ -+ return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ ->top_task_prop & TOP_TASK_BITS_MASK); -+} -+ -+static inline int get_top_task_prop(struct task_struct *p) -+{ -+ return get_hmbird_ts(p)->top_task_prop; -+} -+ -+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear) -+{ -+ if (set) -+ get_hmbird_ts(p)->top_task_prop |= set; -+ if (clear) -+ get_hmbird_ts(p)->top_task_prop &= ~clear; -+ return 0; -+} -+ -+static inline void reset_top_task_prop(struct task_struct *p) -+{ -+ get_hmbird_ts(p)->top_task_prop = 0; -+} -+ -+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ entity->sched_prop = sp; -+ } -+ return 0; -+} -+ -+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->sched_prop; -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq) -+{ -+ unsigned long new_dsq; -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq; -+ entity->sched_prop = new_dsq; -+ } -+} -+ -+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK); -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ entity->dsq_sync_ux = val; -+} -+ -+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->dsq_sync_ux; -+ else -+ return 0; -+} -+#endif - - extern int can_nice(const struct task_struct *p, const int nice); - extern int task_curr(const struct task_struct *p); -diff --git b/include/linux/sched/ext.h a/include/linux/sched/ext.h -deleted file mode 100755 -index b737a00c1373..000000000000 ---- b/include/linux/sched/ext.h -+++ /dev/null -@@ -1,647 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef _LINUX_SCHED_EXT_H --#define _LINUX_SCHED_EXT_H -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --#include -- --enum scx_consts { -- SCX_OPS_NAME_LEN = 128, -- SCX_EXIT_REASON_LEN = 128, -- SCX_EXIT_BT_LEN = 64, -- SCX_EXIT_MSG_LEN = 1024, -- -- SCX_SLICE_DFL = 20 * NSEC_PER_MSEC, -- SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */ --}; -- --/* -- * DSQ (dispatch queue) IDs are 64bit of the format: -- * -- * Bits: [63] [62 .. 0] -- * [ B] [ ID ] -- * -- * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -- * ID: 63 bit ID -- * -- * Built-in IDs: -- * -- * Bits: [63] [62] [61..32] [31 .. 0] -- * [ 1] [ L] [ R ] [ V ] -- * -- * 1: 1 for built-in DSQs. -- * L: 1 for LOCAL_ON DSQ IDs, 0 for others -- * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -- */ --enum scx_dsq_id_flags { -- SCX_DSQ_FLAG_BUILTIN = 1LLU << 63, -- SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -- -- SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0, -- SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, -- SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, -- SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, -- SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, --}; -- --enum scx_exit_type { -- SCX_EXIT_NONE, -- SCX_EXIT_DONE, -- -- SCX_EXIT_UNREG = 64, /* BPF unregistration */ -- SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -- SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ -- SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ --}; -- --/* -- * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is -- * being disabled. -- */ --struct scx_exit_info { -- /* %SCX_EXIT_* - broad category of the exit reason */ -- enum scx_exit_type type; -- /* textual representation of the above */ -- char reason[SCX_EXIT_REASON_LEN]; -- /* number of entries in the backtrace */ -- u32 bt_len; -- /* backtrace if exiting due to an error */ -- unsigned long bt[SCX_EXIT_BT_LEN]; -- /* extra message */ -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --/* sched_ext_ops.flags */ --enum scx_ops_flags { -- /* -- * Keep built-in idle tracking even if ops.update_idle() is implemented. -- */ -- SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, -- -- /* -- * By default, if there are no other task to run on the CPU, ext core -- * keeps running the current task even after its slice expires. If this -- * flag is specified, such tasks are passed to ops.enqueue() with -- * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. -- */ -- SCX_OPS_ENQ_LAST = 1LLU << 1, -- -- /* -- * An exiting task may schedule after PF_EXITING is set. In such cases, -- * bpf_task_from_pid() may not be able to find the task and if the BPF -- * scheduler depends on pid lookup for dispatching, the task will be -- * lost leading to various issues including RCU grace period stalls. -- * -- * To mask this problem, by default, unhashed tasks are automatically -- * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't -- * depend on pid lookups and wants to handle these tasks directly, the -- * following flag can be used. -- */ -- SCX_OPS_ENQ_EXITING = 1LLU << 2, -- -- /* -- * CPU cgroup knob enable flags -- */ -- SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16, /* cpu.weight */ -- -- SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | -- SCX_OPS_ENQ_LAST | -- SCX_OPS_ENQ_EXITING | -- SCX_OPS_CGROUP_KNOB_WEIGHT, --}; -- --/* argument container for ops.enable() and friends */ --struct scx_enable_args { --}; -- --/* argument container for ops->cgroup_init() */ --struct scx_cgroup_init_args { -- /* the weight of the cgroup [1..10000] */ -- u32 weight; --}; -- --enum scx_cpu_preempt_reason { -- /* next task is being scheduled by &sched_class_rt */ -- SCX_CPU_PREEMPT_RT, -- /* next task is being scheduled by &sched_class_dl */ -- SCX_CPU_PREEMPT_DL, -- /* next task is being scheduled by &sched_class_stop */ -- SCX_CPU_PREEMPT_STOP, -- /* unknown reason for SCX being preempted */ -- SCX_CPU_PREEMPT_UNKNOWN, --}; -- --/* -- * Argument container for ops->cpu_acquire(). Currently empty, but may be -- * expanded in the future. -- */ --struct scx_cpu_acquire_args {}; -- --/* argument container for ops->cpu_release() */ --struct scx_cpu_release_args { -- /* the reason the CPU was preempted */ -- enum scx_cpu_preempt_reason reason; -- -- /* the task that's going to be scheduled on the CPU */ -- struct task_struct *task; --}; -- --/** -- * struct sched_ext_ops - Operation table for BPF scheduler implementation -- * -- * Userland can implement an arbitrary scheduling policy by implementing and -- * loading operations in this table. -- */ --struct sched_ext_ops { -- /** -- * select_cpu - Pick the target CPU for a task which is being woken up -- * @p: task being woken up -- * @prev_cpu: the cpu @p was on before sleeping -- * @wake_flags: SCX_WAKE_* -- * -- * Decision made here isn't final. @p may be moved to any CPU while it -- * is getting dispatched for execution later. However, as @p is not on -- * the rq at this point, getting the eventual execution CPU right here -- * saves a small bit of overhead down the line. -- * -- * If an idle CPU is returned, the CPU is kicked and will try to -- * dispatch. While an explicit custom mechanism can be added, -- * select_cpu() serves as the default way to wake up idle CPUs. -- */ -- s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); -- -- /** -- * enqueue - Enqueue a task on the BPF scheduler -- * @p: task being enqueued -- * @enq_flags: %SCX_ENQ_* -- * -- * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() -- * or enqueue on the BPF scheduler. If not directly dispatched, the bpf -- * scheduler owns @p and if it fails to dispatch @p, the task will -- * stall. -- */ -- void (*enqueue)(struct task_struct *p, u64 enq_flags); -- -- /** -- * dequeue - Remove a task from the BPF scheduler -- * @p: task being dequeued -- * @deq_flags: %SCX_DEQ_* -- * -- * Remove @p from the BPF scheduler. This is usually called to isolate -- * the task while updating its scheduling properties (e.g. priority). -- * -- * The ext core keeps track of whether the BPF side owns a given task or -- * not and can gracefully ignore spurious dispatches from BPF side, -- * which makes it safe to not implement this method. However, depending -- * on the scheduling logic, this can lead to confusing behaviors - e.g. -- * scheduling position not being updated across a priority change. -- */ -- void (*dequeue)(struct task_struct *p, u64 deq_flags); -- -- /** -- * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs -- * @cpu: CPU to dispatch tasks for -- * @prev: previous task being switched out -- * -- * Called when a CPU's local dsq is empty. The operation should dispatch -- * one or more tasks from the BPF scheduler into the DSQs using -- * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using -- * scx_bpf_consume(). -- * -- * The maximum number of times scx_bpf_dispatch() can be called without -- * an intervening scx_bpf_consume() is specified by -- * ops.dispatch_max_batch. See the comments on top of the two functions -- * for more details. -- * -- * When not %NULL, @prev is an SCX task with its slice depleted. If -- * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in -- * @prev->scx.flags, it is not enqueued yet and will be enqueued after -- * ops.dispatch() returns. To keep executing @prev, return without -- * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. -- */ -- void (*dispatch)(s32 cpu, struct task_struct *prev); -- -- /** -- * runnable - A task is becoming runnable on its associated CPU -- * @p: task becoming runnable -- * @enq_flags: %SCX_ENQ_* -- * -- * This and the following three functions can be used to track a task's -- * execution state transitions. A task becomes ->runnable() on a CPU, -- * and then goes through one or more ->running() and ->stopping() pairs -- * as it runs on the CPU, and eventually becomes ->quiescent() when it's -- * done running on the CPU. -- * -- * @p is becoming runnable on the CPU because it's -- * -- * - waking up (%SCX_ENQ_WAKEUP) -- * - being moved from another CPU -- * - being restored after temporarily taken off the queue for an -- * attribute change. -- * -- * This and ->enqueue() are related but not coupled. This operation -- * notifies @p's state transition and may not be followed by ->enqueue() -- * e.g. when @p is being dispatched to a remote CPU. Likewise, a task -- * may be ->enqueue()'d without being preceded by this operation e.g. -- * after exhausting its slice. -- */ -- void (*runnable)(struct task_struct *p, u64 enq_flags); -- -- /** -- * running - A task is starting to run on its associated CPU -- * @p: task starting to run -- * -- * See ->runnable() for explanation on the task state notifiers. -- */ -- void (*running)(struct task_struct *p); -- -- /** -- * stopping - A task is stopping execution -- * @p: task stopping to run -- * @runnable: is task @p still runnable? -- * -- * See ->runnable() for explanation on the task state notifiers. If -- * !@runnable, ->quiescent() will be invoked after this operation -- * returns. -- */ -- void (*stopping)(struct task_struct *p, bool runnable); -- -- /** -- * quiescent - A task is becoming not runnable on its associated CPU -- * @p: task becoming not runnable -- * @deq_flags: %SCX_DEQ_* -- * -- * See ->runnable() for explanation on the task state notifiers. -- * -- * @p is becoming quiescent on the CPU because it's -- * -- * - sleeping (%SCX_DEQ_SLEEP) -- * - being moved to another CPU -- * - being temporarily taken off the queue for an attribute change -- * (%SCX_DEQ_SAVE) -- * -- * This and ->dequeue() are related but not coupled. This operation -- * notifies @p's state transition and may not be preceded by ->dequeue() -- * e.g. when @p is being dispatched to a remote CPU. -- */ -- void (*quiescent)(struct task_struct *p, u64 deq_flags); -- -- /** -- * yield - Yield CPU -- * @from: yielding task -- * @to: optional yield target task -- * -- * If @to is NULL, @from is yielding the CPU to other runnable tasks. -- * The BPF scheduler should ensure that other available tasks are -- * dispatched before the yielding task. Return value is ignored in this -- * case. -- * -- * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf -- * scheduler can implement the request, return %true; otherwise, %false. -- */ -- bool (*yield)(struct task_struct *from, struct task_struct *to); -- -- /** -- * core_sched_before - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Used by core-sched to determine the ordering between two tasks. See -- * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on -- * core-sched. -- * -- * Both @a and @b are runnable and may or may not currently be queued on -- * the BPF scheduler. Should return %true if @a should run before @b. -- * %false if there's no required ordering or @b should run before @a. -- * -- * If not specified, the default is ordering them according to when they -- * became runnable. -- */ -- bool (*core_sched_before)(struct task_struct *a,struct task_struct *b); -- -- /** -- * set_weight - Set task weight -- * @p: task to set weight for -- * @weight: new eight [1..10000] -- * -- * Update @p's weight to @weight. -- */ -- void (*set_weight)(struct task_struct *p, u32 weight); -- -- /** -- * set_cpumask - Set CPU affinity -- * @p: task to set CPU affinity for -- * @cpumask: cpumask of cpus that @p can run on -- * -- * Update @p's CPU affinity to @cpumask. -- */ -- void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask); -- -- /** -- * update_idle - Update the idle state of a CPU -- * @cpu: CPU to udpate the idle state for -- * @idle: whether entering or exiting the idle state -- * -- * This operation is called when @rq's CPU goes or leaves the idle -- * state. By default, implementing this operation disables the built-in -- * idle CPU tracking and the following helpers become unavailable: -- * -- * - scx_bpf_select_cpu_dfl() -- * - scx_bpf_test_and_clear_cpu_idle() -- * - scx_bpf_pick_idle_cpu() -- * - scx_bpf_any_idle_cpu() -- * -- * The user also must implement ops.select_cpu() as the default -- * implementation relies on scx_bpf_select_cpu_dfl(). -- * -- * If you keep the built-in idle tracking, specify the -- * %SCX_OPS_KEEP_BUILTIN_IDLE flag. -- */ -- void (*update_idle)(s32 cpu, bool idle); -- -- /** -- * cpu_acquire - A CPU is becoming available to the BPF scheduler -- * @cpu: The CPU being acquired by the BPF scheduler. -- * @args: Acquire arguments, see the struct definition. -- * -- * A CPU that was previously released from the BPF scheduler is now once -- * again under its control. -- */ -- void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); -- -- /** -- * cpu_release - A CPU is taken away from the BPF scheduler -- * @cpu: The CPU being released by the BPF scheduler. -- * @args: Release arguments, see the struct definition. -- * -- * The specified CPU is no longer under the control of the BPF -- * scheduler. This could be because it was preempted by a higher -- * priority sched_class, though there may be other reasons as well. The -- * caller should consult @args->reason to determine the cause. -- */ -- void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -- -- /** -- * cpu_online - A CPU became online -- * @cpu: CPU which just came up -- * -- * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs beforehand. -- */ -- void (*cpu_online)(s32 cpu); -- -- /** -- * cpu_offline - A CPU is going offline -- * @cpu: CPU which is going offline -- * -- * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs afterwards. -- */ -- void (*cpu_offline)(s32 cpu); -- -- /** -- * prep_enable - Prepare to enable BPF scheduling for a task -- * @p: task to prepare BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Either we're loading a BPF scheduler or a new task is being forked. -- * Prepare BPF scheduling for @p. This operation may block and can be -- * used for allocations. -- * -- * Return 0 for success, -errno for failure. An error return while -- * loading will abort loading of the BPF scheduler. During a fork, will -- * abort the specific fork. -- */ -- s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * enable - Enable BPF scheduling for a task -- * @p: task to enable BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Enable @p for BPF scheduling. @p is now in the cgroup specified for -- * the preceding prep_enable() and will start running soon. -- */ -- void (*enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * cancel_enable - Cancel prep_enable() -- * @p: task being canceled -- * @args: enable arguments, see the struct definition -- * -- * @p was prep_enable()'d but failed before reaching enable(). Undo the -- * preparation. -- */ -- void (*cancel_enable)(struct task_struct *p, -- struct scx_enable_args *args); -- -- /** -- * disable - Disable BPF scheduling for a task -- * @p: task to disable BPF scheduling for -- * -- * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. -- * Disable BPF scheduling for @p. -- */ -- void (*disable)(struct task_struct *p); -- -- /* -- * All online ops must come before ops.init(). -- */ -- -- /** -- * init - Initialize the BPF scheduler -- */ -- s32 (*init)(void); -- -- /** -- * exit - Clean up after the BPF scheduler -- * @info: Exit info -- */ -- void (*exit)(struct scx_exit_info *info); -- -- /** -- * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch -- */ -- u32 dispatch_max_batch; -- -- /** -- * flags - %SCX_OPS_* flags -- */ -- u64 flags; -- -- /** -- * timeout_ms - The maximum amount of time, in milliseconds, that a -- * runnable task should be able to wait before being scheduled. The -- * maximum timeout may not exceed the default timeout of 30 seconds. -- * -- * Defaults to the maximum allowed timeout value of 30 seconds. -- */ -- u32 timeout_ms; -- -- /** -- * name - BPF scheduler's name -- * -- * Must be a non-zero valid BPF object name including only isalnum(), -- * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the -- * BPF scheduler is enabled. -- */ -- char name[SCX_OPS_NAME_LEN]; --}; -- --/* -- * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -- * scheduler core and the BPF scheduler. See the documentation for more details. -- */ --struct scx_dispatch_q { -- raw_spinlock_t lock; -- struct list_head fifo; /* processed in dispatching order */ -- struct rb_root_cached priq; /* processed in p->scx.dsq_vtime order */ -- u32 nr; -- u64 id; -- struct llist_node free_node; -- struct rcu_head rcu; --}; -- --/* scx_entity.flags */ --enum scx_ent_flags { -- SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ -- SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -- SCX_TASK_ENQ_LOCAL = 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */ -- -- SCX_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -- SCX_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -- -- SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -- SCX_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -- -- SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ --}; -- --/* scx_entity.dsq_flags */ --enum scx_ent_dsq_flags { -- SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ --}; -- --/* -- * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from -- * everywhere and the following bits track which kfunc sets are currently -- * allowed for %current. This simple per-task tracking works because SCX ops -- * nest in a limited way. BPF will likely implement a way to allow and disallow -- * kfuncs depending on the calling context which will replace this manual -- * mechanism. See scx_kf_allow(). -- */ --enum scx_kf_mask { -- SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ -- /* all non-sleepables may be nested inside INIT and SLEEPABLE */ -- SCX_KF_INIT = 1 << 0, /* running ops.init() */ -- SCX_KF_SLEEPABLE = 1 << 1, /* other sleepable init operations */ -- /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ -- SCX_KF_CPU_RELEASE = 1 << 2, /* ops.cpu_release() */ -- /* ops.dequeue (in REST) may be nested inside DISPATCH */ -- SCX_KF_DISPATCH = 1 << 3, /* ops.dispatch() */ -- SCX_KF_ENQUEUE = 1 << 4, /* ops.enqueue() */ -- SCX_KF_REST = 1 << 5, /* other rq-locked operations */ -- -- __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | -- SCX_KF_ENQUEUE | SCX_KF_REST, -- __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_REST, --}; -- --#define RAVG_HIST_SIZE 5 --struct scx_sched_task_stats { -- u64 mark_start; -- u64 window_start; -- u32 sum; -- u32 sum_history[RAVG_HIST_SIZE]; -- int cidx; -- -- u32 demand; -- u16 demand_scaled; -- void *sdsq; --}; -- -- -- --/* -- * The following is embedded in task_struct and contains all fields necessary -- * for a task to be scheduled by SCX. -- */ --struct sched_ext_entity { -- struct scx_dispatch_q *dsq; -- struct { -- struct list_head fifo; /* dispatch order */ -- struct rb_node priq; /* p->scx.dsq_vtime order */ -- } dsq_node; -- struct list_head watchdog_node; -- u32 flags; /* protected by rq lock */ -- u32 dsq_flags; /* protected by dsq lock */ -- u32 weight; -- s32 sticky_cpu; -- s32 holding_cpu; -- u32 kf_mask; /* see scx_kf_mask above */ -- struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ -- atomic64_t ops_state; -- unsigned long runnable_at; --#ifdef CONFIG_SCHED_CORE -- u64 core_sched_at; /* see scx_prio_less() */ --#endif -- -- /* BPF scheduler modifiable fields */ -- -- /* -- * Runtime budget in nsecs. This is usually set through -- * scx_bpf_dispatch() but can also be modified directly by the BPF -- * scheduler. Automatically decreased by SCX as the task executes. On -- * depletion, a scheduling event is triggered. -- * -- * This value is cleared to zero if the task is preempted by -- * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the -- * task ran. Use p->se.sum_exec_runtime instead. -- */ -- u64 slice; -- -- /* -- * Used to order tasks when dispatching to the vtime-ordered priority -- * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime() -- * but can also be modified directly by the BPF scheduler. Modifying it -- * while a task is queued on a dsq may mangle the ordering and is not -- * recommended. -- */ -- u64 dsq_vtime; -- -- /* -- * If set, reject future sched_setscheduler(2) calls updating the policy -- * to %SCHED_EXT with -%EACCES. -- * -- * If set from ops.prep_enable() and the task's policy is already -- * %SCHED_EXT, which can happen while the BPF scheduler is being loaded -- * or by inhering the parent's policy during fork, the task's policy is -- * rejected and forcefully reverted to %SCHED_NORMAL. The number of such -- * events are reported through /sys/kernel/debug/sched_ext::nr_rejected. -- */ -- bool disallow; /* reject switching into SCX */ -- -- /* cold fields */ -- struct list_head tasks_node; -- struct task_struct *task; -- struct scx_sched_task_stats sts; --}; -- --void sched_ext_free(struct task_struct *p); -- --#else /* !CONFIG_SCHED_CLASS_EXT */ -- --static inline void sched_ext_free(struct task_struct *p) {} -- --#endif /* CONFIG_SCHED_CLASS_EXT */ --#endif /* _LINUX_SCHED_EXT_H */ -diff --git b/include/linux/sched/hmbird_proc_val.h a/include/linux/sched/hmbird_proc_val.h -new file mode 100755 -index 000000000000..e59708b16ea3 ---- /dev/null -+++ a/include/linux/sched/hmbird_proc_val.h -@@ -0,0 +1,22 @@ -+#ifndef __HMBORD_PROC_VAL_H__ -+#define __HMBORD_PROC_VAL_H__ -+ -+extern int scx_enable; -+extern int partial_enable; -+extern int cpuctrl_high_ratio; -+extern int cpuctrl_low_ratio; -+extern int slim_stats; -+extern int hmbirdcore_debug; -+extern int slim_for_app; -+extern int misfit_ds; -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+extern int cpu7_tl; -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int slim_gov_debug; -+extern int scx_gov_ctrl; -+extern int sched_ravg_window_frame_per_sec; -+ -+#endif -\ No newline at end of file -diff --git b/include/linux/sched/hmbird_version.h a/include/linux/sched/hmbird_version.h -new file mode 100755 -index 000000000000..b2843881c26f ---- /dev/null -+++ a/include/linux/sched/hmbird_version.h -@@ -0,0 +1,52 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#ifndef _OPLUS_HMBIRD_VERSION_H_ -+#define _OPLUS_HMBIRD_VERSION_H_ -+#include -+#include -+#include -+ -+enum hmbird_version { -+ HMBIRD_UNINIT, -+ HMBIRD_GKI_VERSION, -+ HMBIRD_OGKI_VERSION, -+ HMBIRD_UNKNOW_VERSION, -+}; -+ -+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT; -+ -+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type" -+ -+static inline enum hmbird_version get_hmbird_version_type(void) -+{ -+ struct device_node *np = NULL; -+ const char *hmbird_version_str = NULL; -+ if (HMBIRD_UNINIT != hmbird_version_type) -+ return hmbird_version_type; -+ np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH); -+ if (np) { -+ of_property_read_string(np, "type", &hmbird_version_str); -+ if (NULL != hmbird_version_str) { -+ if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) { -+ hmbird_version_type = HMBIRD_OGKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi"); -+ } else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) { -+ hmbird_version_type = HMBIRD_GKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi"); -+ } else { -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi"); -+ } -+ return hmbird_version_type; -+ } -+ } -+ -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION"); -+ return hmbird_version_type; -+} -+ -+#endif /*_OPLUS_HMBIRD_VERSION_H_ */ -diff --git b/include/linux/sched/task.h a/include/linux/sched/task.h -index 03d35e3eda3c..6a5f0c0d74bd 100644 ---- b/include/linux/sched/task.h -+++ a/include/linux/sched/task.h -@@ -61,8 +61,10 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); - extern void init_idle(struct task_struct *idle, int cpu); - - extern int sched_fork(unsigned long clone_flags, struct task_struct *p); --extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); --extern void sched_cancel_fork(struct task_struct *p); -+extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); -+#ifdef CONFIG_HMBIRD_SCHED -+extern void hmbird_cancel_fork(struct task_struct *p); -+#endif - extern void sched_post_fork(struct task_struct *p); - extern void sched_dead(struct task_struct *p); - -diff --git b/include/uapi/linux/sched.h a/include/uapi/linux/sched.h -index 359a14cc76a4..969716ab4320 100755 ---- b/include/uapi/linux/sched.h -+++ a/include/uapi/linux/sched.h -@@ -118,7 +118,7 @@ struct clone_args { - /* SCHED_ISO: reserved but not implemented yet */ - #define SCHED_IDLE 5 - #define SCHED_DEADLINE 6 --#define SCHED_EXT 7 -+#define SCHED_HMBIRD 7 - - /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ - #define SCHED_RESET_ON_FORK 0x40000000 -diff --git b/init/Kconfig a/init/Kconfig -index 337276cbc7f8..bf052ca532b4 100644 ---- b/init/Kconfig -+++ a/init/Kconfig -@@ -1030,10 +1030,6 @@ config RT_GROUP_SCHED - realtime bandwidth for them. - See Documentation/scheduler/sched-rt-group.rst for more information. - --config EXT_GROUP_SCHED -- bool -- depends on SCHED_CLASS_EXT && CGROUP_SCHED -- default y - endif #CGROUP_SCHED - - config SCHED_MM_CID -diff --git b/init/init_task.c a/init/init_task.c -index 69a046388995..31ceb0e469f7 100755 ---- b/init/init_task.c -+++ a/init/init_task.c -@@ -6,7 +6,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -102,9 +101,6 @@ struct task_struct init_task - #endif - #ifdef CONFIG_CGROUP_SCHED - .sched_task_group = &root_task_group, --#endif --#ifdef CONFIG_SLIM_SCHED -- .scx = NULL, - #endif - .ptraced = LIST_HEAD_INIT(init_task.ptraced), - .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), -diff --git b/kernel/Kconfig.preempt a/kernel/Kconfig.preempt -index cf22ef6107b8..ca8de6eb1e16 100755 ---- b/kernel/Kconfig.preempt -+++ a/kernel/Kconfig.preempt -@@ -133,12 +133,8 @@ config SCHED_CORE - which is the likely usage by Linux distributions, there should - be no measurable impact on performance. - --config SLIM_SCHED -- bool "slim sched" -- default n --config SCHED_CLASS_EXT -- bool "Extensible Scheduling Class" -- depends on BPF_SYSCALL && BPF_JIT -+config HMBIRD_SCHED -+ bool "hmbird Scheduling Class(base on ext scheduler)" - help - This option enables a new scheduler class sched_ext (SCX), which - allows scheduling policies to be implemented as BPF programs to -diff --git b/kernel/bpf/bpf_struct_ops_types.h a/kernel/bpf/bpf_struct_ops_types.h -index 3618769d853d..5678a9ddf817 100644 ---- b/kernel/bpf/bpf_struct_ops_types.h -+++ a/kernel/bpf/bpf_struct_ops_types.h -@@ -9,8 +9,4 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) - #include - BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) - #endif --#ifdef CONFIG_SCHED_CLASS_EXT --#include --BPF_STRUCT_OPS_TYPE(sched_ext_ops) --#endif - #endif -diff --git b/kernel/fork.c a/kernel/fork.c -index a43f97302332..7a91505b9378 100755 ---- b/kernel/fork.c -+++ a/kernel/fork.c -@@ -23,7 +23,9 @@ - #include - #include - #include --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - #include - #include - #include -@@ -999,8 +1001,9 @@ void __put_task_struct(struct task_struct *tsk) - WARN_ON(!tsk->exit_state); - WARN_ON(refcount_read(&tsk->usage)); - WARN_ON(tsk == current); -- -- sched_ext_free(tsk); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_free(tsk); -+#endif - io_uring_free(tsk); - cgroup_free(tsk); - task_numa_free(tsk, true); -@@ -2517,7 +2520,11 @@ __latent_entropy struct task_struct *copy_process( - - retval = perf_event_init_task(p, clone_flags); - if (retval) -- goto bad_fork_sched_cancel_fork; -+#ifdef CONFIG_HMBIRD_SCHED -+ goto bad_fork_hmbird_cancel_fork; -+#else -+ goto bad_fork_cleanup_policy; -+#endif - retval = audit_alloc(p); - if (retval) - goto bad_fork_cleanup_perf; -@@ -2649,9 +2656,7 @@ __latent_entropy struct task_struct *copy_process( - * cgroup specific, it unconditionally needs to place the task on a - * runqueue. - */ -- retval = sched_cgroup_fork(p, args); -- if (retval) -- goto bad_fork_cancel_cgroup; -+ sched_cgroup_fork(p, args); - - /* - * From this point on we must avoid any synchronous user-space -@@ -2697,13 +2702,13 @@ __latent_entropy struct task_struct *copy_process( - /* Don't start children in a dying pid namespace */ - if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { - retval = -ENOMEM; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* Let kill terminate clone/fork in the middle */ - if (fatal_signal_pending(current)) { - retval = -EINTR; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* No more failure paths after this point. */ -@@ -2780,11 +2785,10 @@ __latent_entropy struct task_struct *copy_process( - - return p; - --bad_fork_core_free: -+bad_fork_cancel_cgroup: - sched_core_free(p); - spin_unlock(¤t->sighand->siglock); - write_unlock_irq(&tasklist_lock); --bad_fork_cancel_cgroup: - cgroup_cancel_fork(p, args); - bad_fork_put_pidfd: - if (clone_flags & CLONE_PIDFD) { -@@ -2823,8 +2827,10 @@ __latent_entropy struct task_struct *copy_process( - audit_free(p); - bad_fork_cleanup_perf: - perf_event_free_task(p); --bad_fork_sched_cancel_fork: -- sched_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+bad_fork_hmbird_cancel_fork: -+ hmbird_cancel_fork(p); -+#endif - bad_fork_cleanup_policy: - lockdep_free_task(p); - #ifdef CONFIG_NUMA -diff --git b/kernel/sched/build_policy.c a/kernel/sched/build_policy.c -index 005025f55bea..c091539a0f60 100755 ---- b/kernel/sched/build_policy.c -+++ a/kernel/sched/build_policy.c -@@ -28,8 +28,10 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED - #include - #include -+#endif - - #include - -@@ -54,6 +56,10 @@ - #include "cputime.c" - #include "deadline.c" - --#ifdef CONFIG_SCHED_CLASS_EXT --# include "ext.c" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/hmbird_util_track.c" -+#include "hmbird/hmbird_sched_proc.c" -+#include "hmbird/hmbird_shadow_tick.c" -+#include "hmbird/hmbird.c" -+#include "hmbird/hmbird_misc.c" - #endif -diff --git b/kernel/sched/core.c a/kernel/sched/core.c -index 25d5703df992..b894417ce7df 100755 ---- b/kernel/sched/core.c -+++ a/kernel/sched/core.c -@@ -96,6 +96,12 @@ - #include "../../io_uring/io-wq.h" - #include "../smpboot.h" - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/slim.h" -+#include "hmbird/hmbird_shadow_tick.h" -+#include "hmbird.h" -+#endif -+ - #include - #include - #include -@@ -170,7 +176,6 @@ __read_mostly int scheduler_running; - - DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); - -- - /* kernel prio, less is more */ - static inline int __task_prio(const struct task_struct *p) - { -@@ -183,11 +188,10 @@ static inline int __task_prio(const struct task_struct *p) - if (p->sched_class == &idle_sched_class) - return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - --#ifdef CONFIG_SCHED_CLASS_EXT -- if (p->sched_class == &ext_sched_class) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (p->sched_class == &hmbird_sched_class) - return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ - #endif -- - return MAX_RT_PRIO + MAX_NICE; /* 119, squash fair */ - } - -@@ -217,9 +221,9 @@ static inline bool prio_less(const struct task_struct *a, - if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ - return cfs_prio_less(a, b, in_fi); - --#ifdef CONFIG_SCHED_CLASS_EXT -+#ifdef CONFIG_HMBIRD_SCHED - if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ -- return scx_prio_less(a, b, in_fi); -+ return hmbird_prio_less(a, b, in_fi); - #endif - - return false; -@@ -1279,17 +1283,26 @@ bool sched_can_stop_tick(struct rq *rq) - if (fifo_nr_running) - return true; - -+#ifdef CONFIG_HMBIRD_SCHED - /* -- * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks -+ * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks - * left. For CFS, if there's more than one we need the tick for -- * involuntary preemption. For SCX, ask. -+ * involuntary preemption. For HMBIRD, ask. - */ -- if (!scx_switched_all() && rq->nr_running > 1) -+ if (!hmbird_enabled() && rq->nr_running > 1) - return false; - -- if (scx_enabled() && !scx_can_stop_tick(rq)) -+ if (hmbird_enabled() && !hmbird_can_stop_tick(rq)) - return false; -- -+#else -+ /* -+ * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; -+ * if there's more than one we need the tick for involuntary -+ * preemption. -+ */ -+ if (rq->nr_running > 1) -+ return false; -+#endif - /* - * If there is one task and it has CFS runtime bandwidth constraints - * and it's on the cpu now we don't want to stop the tick. -@@ -2222,10 +2235,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) - } - EXPORT_SYMBOL_GPL(deactivate_task); - --struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) -+#ifdef CONFIG_HMBIRD_SCHED -+struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - { -- struct sched_change_guard cg = { -+ struct hmbird_sched_change_guard cg = { - .rq = rq, - .p = p, - .queued = task_on_rq_queued(p), -@@ -2247,7 +2261,7 @@ sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - return cg; - } - --void sched_change_guard_fini(struct sched_change_guard *cg, int flags) -+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags) - { - if (cg->queued) - enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); -@@ -2255,6 +2269,7 @@ void sched_change_guard_fini(struct sched_change_guard *cg, int flags) - set_next_task(cg->rq, cg->p); - cg->done = true; - } -+#endif - - static inline int __normal_prio(int policy, int rt_prio, int nice) - { -@@ -2320,9 +2335,9 @@ inline int task_curr(const struct task_struct *p) - * this means any call to check_class_changed() must be followed by a call to - * balance_callback(). - */ --void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio) -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) - { - if (prev_class != p->sched_class) { - if (prev_class->switched_from) -@@ -2885,6 +2900,7 @@ static void - __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - { - struct rq *rq = task_rq(p); -+ bool queued, running; - - /* - * This here violates the locking rules for affinity, since we're only -@@ -2903,9 +2919,26 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - else - lockdep_assert_held(&p->pi_lock); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->sched_class->set_cpus_allowed(p, ctx); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ -+ if (queued) { -+ /* -+ * Because __kthread_bind() calls this on blocked tasks without -+ * holding rq->lock. -+ */ -+ lockdep_assert_rq_held(rq); -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); - } -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->sched_class->set_cpus_allowed(p, ctx); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - } - - /* -@@ -3772,7 +3805,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class)) -+#else - if (unlikely(!is_cpu_allowed(p, cpu))) -+#endif - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -@@ -4891,8 +4928,9 @@ late_initcall(sched_core_sysctl_init); - */ - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { -- -+#ifdef CONFIG_HMBIRD_SCHED - int ret; -+#endif - - trace_android_rvh_sched_fork(p); - -@@ -4933,21 +4971,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_reset_on_fork = 0; - } - -- scx_pre_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ ret = hmbird_pre_fork(p); -+ if (ret) -+ goto out_cancel; - - if (dl_prio(p->prio)) { - ret = -EAGAIN; - goto out_cancel; --#ifdef CONFIG_SCHED_CLASS_EXT -- } else if (task_on_scx(p)) { -- p->sched_class = &ext_sched_class; --#endif -+ } else if (task_on_hmbird(p)) { -+ p->sched_class = &hmbird_sched_class; - } else if (rt_prio(p->prio)) { - p->sched_class = &rt_sched_class; - } else { - p->sched_class = &fair_sched_class; - } -- -+#else -+ if (dl_prio(p->prio)) -+ return -EAGAIN; -+ else if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - init_entity_runnable_average(&p->se); - trace_android_rvh_finish_prio_fork(p); - -@@ -4966,12 +5012,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - #endif - return 0; - -+#ifdef CONFIG_HMBIRD_SCHED - out_cancel: -- scx_cancel_fork(p); -+ hmbird_cancel_fork(p); - return ret; -+#endif - } - --int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) -+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - { - unsigned long flags; - -@@ -4998,20 +5046,14 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -- -- return scx_fork(p); --} -- --void sched_cancel_fork(struct task_struct *p) --{ -- scx_cancel_fork(p); - } - - void sched_post_fork(struct task_struct *p) - { - uclamp_post_fork(p); -- -- scx_post_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_post_fork(p); -+#endif - } - - unsigned long to_ratio(u64 period, u64 runtime) -@@ -5875,21 +5917,28 @@ void scheduler_tick(void) - - if (sched_feat(LATENCY_WARN) && resched_latency) - resched_latency_warn(cpu, resched_latency); -- -- scx_notify_sched_tick(); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_notify_sched_tick(); -+#endif - perf_event_task_tick(); - - if (curr->flags & PF_WQ_WORKER) - wq_worker_tick(curr); - - #ifdef CONFIG_SMP -- if (!scx_switched_all()) { -+#ifdef CONFIG_HMBIRD_SCHED -+ if (!hmbird_enabled()) { -+#endif - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); -+#ifdef CONFIG_HMBIRD_SCHED - } -+#endif - #endif - trace_android_vh_scheduler_tick(rq); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ scheduler_tick_handler(NULL, NULL); -+#endif - } - - #ifdef CONFIG_NO_HZ_FULL -@@ -6191,7 +6240,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ -+#ifdef CONFIG_HMBIRD_SCHED - for_balance_class_range(class, prev->sched_class, &idle_sched_class) { -+#else -+ for_class_range(class, prev->sched_class, &idle_sched_class) { -+#endif - if (class->balance(rq, prev, rf)) - break; - } -@@ -6209,9 +6262,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - const struct sched_class *class; - struct task_struct *p; - -- if (scx_enabled()) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (hmbird_enabled()) - goto restart; -- -+#endif - /* - * Optimization: we know that if all tasks are in the fair class we can - * call that function directly, but only if the @prev task wasn't of a -@@ -6237,13 +6291,21 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - restart: - put_prev_task_balance(rq, prev, rf); - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { - p = class->pick_next_task(rq); - if (p) { -- scx_notify_pick_next_task(rq, p, class); -+ hmbird_notify_pick_next_task(rq, p, class); - return p; - } - } -+#else -+ for_each_class(class) { -+ p = class->pick_next_task(rq); -+ if (p) -+ return p; -+ } -+#endif - - BUG(); /* The idle class should always have a runnable task. */ - } -@@ -6272,7 +6334,11 @@ static inline struct task_struct *pick_task(struct rq *rq) - const struct sched_class *class; - struct task_struct *p; - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { -+#else -+ for_each_class(class) { -+#endif - p = class->pick_task(rq); - if (p) - return p; -@@ -6916,7 +6982,9 @@ static void __sched notrace __schedule(unsigned int sched_mode) - psi_sched_switch(prev, next, !task_on_rq_queued(prev)); - - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#endif - /* Also unlocks the rq: */ - rq = context_switch(rq, prev, next, &rf); - } else { -@@ -7244,24 +7312,31 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag - } - EXPORT_SYMBOL(default_wake_function); - --void __setscheduler_prio(struct task_struct *p, int prio) -+static void __setscheduler_prio(struct task_struct *p, int prio) - { -- /* -- * After switching all rt and fair class to ext, -- * stop class is switched to ext class too. Just -- * skip it. */ -+#ifdef CONFIG_HMBIRD_SCHED -+ bool on_hmbird = task_on_hmbird(p); -+ - if (p->sched_class == &stop_sched_class) -- ;/* do nothing */ -+ ; - else if (dl_prio(prio)) - p->sched_class = &dl_sched_class; --#ifdef CONFIG_SCHED_CLASS_EXT -- else if (task_on_scx(p)) -- p->sched_class = &ext_sched_class; --#endif -+ else if (rt_prio(prio) && on_hmbird) -+ p->sched_class = &hmbird_sched_class; - else if (rt_prio(prio)) - p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; - else - p->sched_class = &fair_sched_class; -+#else -+ if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - - p->prio = prio; - trace_android_rvh_setscheduler_prio(p); -@@ -7297,7 +7372,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - */ - void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - { -- int prio, oldprio, queue_flag = -+ int prio, oldprio, queued, running, queue_flag = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *prev_class; - struct rq_flags rf; -@@ -7360,42 +7435,52 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - queue_flag &= ~DEQUEUE_MOVE; - - prev_class = p->sched_class; -- SCHED_CHANGE_BLOCK(rq, p, queue_flag) { -- /* -- * Boosting condition are: -- * 1. -rt task is running and holds mutex A -- * --> -dl task blocks on mutex A -- * -- * 2. -dl task is running and holds mutex A -- * --> -dl task blocks on mutex A and could preempt the -- * running task -- */ -- if (dl_prio(prio)) { -- if (!dl_prio(p->normal_prio) || -- (pi_task && dl_prio(pi_task->prio) && -- dl_entity_preempt(&pi_task->dl, &p->dl))) { -- p->dl.pi_se = pi_task->dl.pi_se; -- queue_flag |= ENQUEUE_REPLENISH; -- } else { -- p->dl.pi_se = &p->dl; -- } -- } else if (rt_prio(prio)) { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- if (oldprio < prio) -- queue_flag |= ENQUEUE_HEAD; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flag); -+ if (running) -+ put_prev_task(rq, p); -+ -+ /* -+ * Boosting condition are: -+ * 1. -rt task is running and holds mutex A -+ * --> -dl task blocks on mutex A -+ * -+ * 2. -dl task is running and holds mutex A -+ * --> -dl task blocks on mutex A and could preempt the -+ * running task -+ */ -+ if (dl_prio(prio)) { -+ if (!dl_prio(p->normal_prio) || -+ (pi_task && dl_prio(pi_task->prio) && -+ dl_entity_preempt(&pi_task->dl, &p->dl))) { -+ p->dl.pi_se = pi_task->dl.pi_se; -+ queue_flag |= ENQUEUE_REPLENISH; - } else { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- else if (rt_prio(oldprio)) -- p->rt.timeout = 0; -- else if (!task_has_idle_policy(p)) -- reweight_task(p, prio - MAX_RT_PRIO); -+ p->dl.pi_se = &p->dl; - } -- -- __setscheduler_prio(p, prio); -+ } else if (rt_prio(prio)) { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ if (oldprio < prio) -+ queue_flag |= ENQUEUE_HEAD; -+ } else { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ else if (rt_prio(oldprio)) -+ p->rt.timeout = 0; -+ else if (!task_has_idle_policy(p)) -+ reweight_task(p, prio - MAX_RT_PRIO); - } - -+ __setscheduler_prio(p, prio); -+ -+ if (queued) -+ enqueue_task(rq, p, queue_flag); -+ if (running) -+ set_next_task(rq, p); -+ - check_class_changed(rq, p, prev_class, oldprio); - out_unlock: - /* Avoid rq from going away on us: */ -@@ -7416,7 +7501,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - - void set_user_nice(struct task_struct *p, long nice) - { -- bool allowed = false; -+ bool queued, running, allowed = false; - int old_prio; - struct rq_flags rf; - struct rq *rq; -@@ -7445,13 +7530,22 @@ void set_user_nice(struct task_struct *p, long nice) - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); -+ if (running) -+ put_prev_task(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->static_prio = NICE_TO_PRIO(nice); -- set_load_weight(p, true); -- old_prio = p->prio; -- p->prio = effective_prio(p); -- } -+ p->static_prio = NICE_TO_PRIO(nice); -+ set_load_weight(p, true); -+ old_prio = p->prio; -+ p->prio = effective_prio(p); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - - /* - * If the task increased its priority or is running and -@@ -7868,7 +7962,7 @@ static int __sched_setscheduler(struct task_struct *p, - bool user, bool pi) - { - int oldpolicy = -1, policy = attr->sched_policy; -- int retval, oldprio, newprio; -+ int retval, oldprio, newprio, queued, running; - const struct sched_class *prev_class; - struct balance_callback *head; - struct rq_flags rf; -@@ -7876,7 +7970,9 @@ static int __sched_setscheduler(struct task_struct *p, - int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct rq *rq; - bool cpuset_locked = false; -- -+#ifdef CONFIG_HMBIRD_SCHED -+ unsigned long flags; -+#endif - - /* The pi code expects interrupts enabled */ - BUG_ON(pi && in_interrupt()); -@@ -7942,7 +8038,9 @@ static int __sched_setscheduler(struct task_struct *p, - * To be able to change p->policy safely, the appropriate - * runqueue lock must be held. - */ -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+#endif - rq = task_rq_lock(p, &rf); - update_rq_clock(rq); - -@@ -7954,10 +8052,11 @@ static int __sched_setscheduler(struct task_struct *p, - goto unlock; - } - -- retval = scx_check_setscheduler(p, policy); -+#ifdef CONFIG_HMBIRD_SCHED -+ retval = hmbird_check_setscheduler(p, policy); - if (retval) - goto unlock; -- -+#endif - /* - * If not changing anything there's no need to proceed further, - * but store a possible modification of reset_on_fork. -@@ -8014,7 +8113,9 @@ static int __sched_setscheduler(struct task_struct *p, - if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { - policy = oldpolicy = -1; - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - goto recheck; -@@ -8047,16 +8148,23 @@ static int __sched_setscheduler(struct task_struct *p, - queue_flags &= ~DEQUEUE_MOVE; - } - -- SCHED_CHANGE_BLOCK(rq, p, queue_flags) { -- prev_class = p->sched_class; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flags); -+ if (running) -+ put_prev_task(rq, p); -+ -+ prev_class = p->sched_class; - -- if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -- __setscheduler_params(p, attr); -- __setscheduler_prio(p, newprio); -- trace_android_rvh_setscheduler(p); -- } -- __setscheduler_uclamp(p, attr); -+ if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -+ __setscheduler_params(p, attr); -+ __setscheduler_prio(p, newprio); -+ trace_android_rvh_setscheduler(p); -+ } -+ __setscheduler_uclamp(p, attr); - -+ if (queued) { - /* - * We enqueue to tail when the priority of a task is - * increased (user space view). -@@ -8064,7 +8172,10 @@ static int __sched_setscheduler(struct task_struct *p, - if (oldprio < p->prio) - queue_flags |= ENQUEUE_HEAD; - -+ enqueue_task(rq, p, queue_flags); - } -+ if (running) -+ set_next_task(rq, p); - - check_class_changed(rq, p, prev_class, oldprio); - -@@ -8072,7 +8183,9 @@ static int __sched_setscheduler(struct task_struct *p, - preempt_disable(); - head = splice_balance_callbacks(rq); - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (pi) { - if (cpuset_locked) - cpuset_unlock(); -@@ -8087,7 +8200,9 @@ static int __sched_setscheduler(struct task_struct *p, - - unlock: - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - return retval; -@@ -8785,6 +8900,9 @@ static void do_sched_yield(void) - long skip = 0; - - trace_android_rvh_before_do_sched_yield(&skip); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_skip_yield(&skip); -+#endif - if (skip) - return; - -@@ -9309,7 +9427,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - break; - } -@@ -9337,7 +9457,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - } - return ret; -@@ -9638,15 +9760,25 @@ int migrate_task_to(struct task_struct *p, int target_cpu) - */ - void sched_setnuma(struct task_struct *p, int nid) - { -+ bool queued, running; - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) { -- p->numa_preferred_nid = nid; -- } -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE); -+ if (running) -+ put_prev_task(rq, p); - -+ p->numa_preferred_nid = nid; -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - task_rq_unlock(rq, p, &rf); - } - #endif /* CONFIG_NUMA_BALANCING */ -@@ -10212,17 +10344,23 @@ void __init sched_init(void) - int i; - - /* Make sure the linker didn't screw up */ -+#ifdef CONFIG_HMBIRD_SCHED - #ifdef CONFIG_SMP -- BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+#endif -+ WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class)); -+#else -+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 || -+ &fair_sched_class != &rt_sched_class + 1 || -+ &rt_sched_class != &dl_sched_class + 1); -+#ifdef CONFIG_SMP -+ BUG_ON(&dl_sched_class != &stop_sched_class + 1); - #endif -- BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); -- BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); -- BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); --#ifdef CONFIG_SCHED_CLASS_EXT -- BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); -- BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); - #endif -- - wait_bit_init(); - - #ifdef CONFIG_FAIR_GROUP_SCHED -@@ -10392,8 +10530,9 @@ void __init sched_init(void) - balance_push_set(smp_processor_id(), false); - #endif - init_sched_fair_class(); -- init_sched_ext_class(); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ init_sched_hmbird_class(); -+#endif - psi_init(); - - init_uclamp(); -@@ -10863,6 +11002,11 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) - struct task_group *tg = css_tg(css); - struct task_group *parent = css_tg(css->parent); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_tg_online(tg); -+#endif -+ -+ - if (parent) - sched_online_group(tg, parent); - -@@ -10912,13 +11056,77 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) - } - #endif - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 dl) -+{ -+ int i; -+ -+ for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) { -+ if (dl < HMBIRD_BPF_DSQS_DEADLINE[i]) -+ break; -+ } -+ -+ i = max_t(int, i-1, MIN_CGROUP_DL_IDX); -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return 0; -+ update_cgroup_ids_table(css->cgroup->kn->id, i); -+ -+ return 0; -+} -+ -+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ u8 i; -+ -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX]; -+ i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1); -+ if (i < 0) { -+ pr_err(" <%s> i is %d, less than 0, name is %s\n", -+ __func__, i, css->cgroup->kn->name); -+ i = DEFAULT_CGROUP_DL_IDX; -+ } -+ -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[i]; -+} -+ -+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css, -+ struct task_struct *p, int prio) -+{ -+ if (!css || !rt_prio(prio)) -+ return; -+ -+ if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) { -+ if (!strcmp(css->cgroup->kn->name, "display")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3); -+ else if (!strcmp(css->cgroup->kn->name, "touch")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2); -+ else if (!strcmp(css->cgroup->kn->name, "multimedia")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+ } -+} -+#endif -+ - static void cpu_cgroup_attach(struct cgroup_taskset *tset) - { - struct task_struct *task; - struct cgroup_subsys_state *css; - - cgroup_taskset_for_each(task, css, tset) { -- -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_change_rt_sched_prop(css, task, task->prio); -+#endif - sched_move_task(task); - } - -@@ -11600,7 +11808,13 @@ static struct cftype cpu_legacy_files[] = { - .write_u64 = cpu_uclamp_ls_write_u64, - }, - #endif -- -+#ifdef CONFIG_HMBIRD_SCHED -+ { -+ .name = "hmbird.deadline", -+ .read_u64 = cgroup_read_hmbird_deadline, -+ .write_u64 = cgroup_write_hmbird_deadline, -+ }, -+#endif - { } /* Terminate */ - }; - -@@ -11649,7 +11863,6 @@ static int cpu_local_stat_show(struct seq_file *sf, - } - - #ifdef CONFIG_FAIR_GROUP_SCHED -- - static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, - struct cftype *cft) - { -diff --git b/kernel/sched/debug.c a/kernel/sched/debug.c -index a6c99df9edf9..e09904f4d6e5 100755 ---- b/kernel/sched/debug.c -+++ a/kernel/sched/debug.c -@@ -375,9 +375,8 @@ static __init int sched_init_debug(void) - #endif - - debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); -- --#ifdef CONFIG_SCHED_CLASS_EXT -- debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops); -+#ifdef CONFIG_HMBIRD_SCHED -+ debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops); - #endif - return 0; - } -@@ -1006,9 +1005,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - SEQ_printf(m, - "---------------------------------------------------------" - "----------\n"); --#ifdef CONFIG_SLIM_SCHED -- SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop); --#endif - - #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) - #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) -@@ -1104,8 +1100,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - } else if (fair_policy(p->policy)) { - P(se.slice); - } --#ifdef CONFIG_SCHED_CLASS_EXT -- __PS("ext.enabled", p->sched_class == &ext_sched_class); -+#ifdef CONFIG_HMBIRD_SCHED -+ __PS("hmbird.enabled", p->sched_class == &hmbird_sched_class); -+ __PS("sched_prop", get_hmbird_ts(p)->sched_prop); -+ __PS("top_task_prop", get_hmbird_ts(p)->top_task_prop); - #endif - #undef PN_SCHEDSTAT - #undef P_SCHEDSTAT -diff --git b/kernel/sched/ext.c a/kernel/sched/ext.c -deleted file mode 100755 -index 4845d441df2b..000000000000 ---- b/kernel/sched/ext.c -+++ /dev/null -@@ -1,3978 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -- --enum scx_internal_consts { -- SCX_NR_ONLINE_OPS = SCX_OP_IDX(init), -- SCX_DSP_DFL_MAX_BATCH = 32, -- SCX_DSP_MAX_LOOPS = 32, -- SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, --}; -- --enum scx_ops_enable_state { -- SCX_OPS_PREPPING, -- SCX_OPS_ENABLING, -- SCX_OPS_ENABLED, -- SCX_OPS_DISABLING, -- SCX_OPS_DISABLED, --}; -- --static inline struct task_group *css_tg(struct cgroup_subsys_state *css) --{ -- return css ? container_of(css, struct task_group, css) : NULL; --} -- --/* -- * sched_ext_entity->ops_state -- * -- * Used to track the task ownership between the SCX core and the BPF scheduler. -- * State transitions look as follows: -- * -- * NONE -> QUEUEING -> QUEUED -> DISPATCHING -- * ^ | | -- * | v v -- * \-------------------------------/ -- * -- * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -- * sites for explanations on the conditions being waited upon and why they are -- * safe. Transitions out of them into NONE or QUEUED must store_release and the -- * waiters should load_acquire. -- * -- * Tracking scx_ops_state enables sched_ext core to reliably determine whether -- * any given task can be dispatched by the BPF scheduler at all times and thus -- * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -- * to try to dispatch any task anytime regardless of its state as the SCX core -- * can safely reject invalid dispatches. -- */ --enum scx_ops_state { -- SCX_OPSS_NONE, /* owned by the SCX core */ -- SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -- SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ -- SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ -- -- /* -- * QSEQ brands each QUEUED instance so that, when dispatch races -- * dequeue/requeue, the dispatcher can tell whether it still has a claim -- * on the task being dispatched. -- */ -- SCX_OPSS_QSEQ_SHIFT = 2, -- SCX_OPSS_STATE_MASK = (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1, -- SCX_OPSS_QSEQ_MASK = ~SCX_OPSS_STATE_MASK, --}; -- --/* -- * During exit, a task may schedule after losing its PIDs. When disabling the -- * BPF scheduler, we need to be able to iterate tasks in every state to -- * guarantee system safety. Maintain a dedicated task list which contains every -- * task between its fork and eventual free. -- */ --static DEFINE_SPINLOCK(scx_tasks_lock); --static LIST_HEAD(scx_tasks); -- --/* ops enable/disable */ --static struct kthread_worker *scx_ops_helper; --static DEFINE_MUTEX(scx_ops_enable_mutex); --DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); --DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); --static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); --static bool scx_switch_all_req; --static bool scx_switching_all; --DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -- --static struct sched_ext_ops scx_ops; --static bool scx_warned_zero_slice; -- --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); --DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); --static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); -- --struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] = -- { [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT }; -- --static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE); --static struct scx_exit_info scx_exit_info; -- --static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0); -- --/* -- * The maximum amount of time in jiffies that a task may be runnable without -- * being scheduled on a CPU. If this timeout is exceeded, it will trigger -- * scx_ops_error(). -- */ --unsigned long scx_watchdog_timeout; -- --/* -- * The last time the delayed work was run. This delayed work relies on -- * ksoftirqd being able to run to service timer interrupts, so it's possible -- * that this work itself could get wedged. To account for this, we check that -- * it's not stalled in the timer tick, and trigger an error if it is. -- */ --unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; -- --static struct delayed_work scx_watchdog_work; -- --/* idle tracking */ --#ifdef CONFIG_SMP --#ifdef CONFIG_CPUMASK_OFFSTACK --#define CL_ALIGNED_IF_ONSTACK --#else --#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp --#endif -- --static struct { -- cpumask_var_t cpu; -- cpumask_var_t smt; --} idle_masks CL_ALIGNED_IF_ONSTACK; -- --static bool __cacheline_aligned_in_smp scx_has_idle_cpus; --#endif /* CONFIG_SMP */ -- --/* for %SCX_KICK_WAIT */ --static u64 __percpu *scx_kick_cpus_pnt_seqs; -- --/* -- * Direct dispatch marker. -- * -- * Non-NULL values are used for direct dispatch from enqueue path. A valid -- * pointer points to the task currently being enqueued. An ERR_PTR value is used -- * to indicate that direct dispatch has already happened. -- */ --static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); -- --/* dispatch queues */ --static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; -- --static LLIST_HEAD(dsqs_to_free); -- --/* dispatch buf */ --struct scx_dsp_buf_ent { -- struct task_struct *task; -- u64 qseq; -- u64 dsq_id; -- u64 enq_flags; --}; -- --static u32 scx_dsp_max_batch; --static struct scx_dsp_buf_ent __percpu *scx_dsp_buf; -- --struct scx_dsp_ctx { -- struct rq *rq; -- struct rq_flags *rf; -- u32 buf_cursor; -- u32 nr_tasks; --}; -- --static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx); -- --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags); --void scx_bpf_kick_cpu(s32 cpu, u64 flags); -- --struct scx_task_iter { -- struct sched_ext_entity cursor; -- struct task_struct *locked; -- struct rq *rq; -- struct rq_flags rf; --}; -- --#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) -- --/* if the highest set bit is N, return a mask with bits [N+1, 31] set */ --static u32 higher_bits(u32 flags) --{ -- return ~((1 << fls(flags)) - 1); --} -- --/* return the mask with only the highest bit set */ --static u32 highest_bit(u32 flags) --{ -- int bit = fls(flags); -- return bit ? 1 << (bit - 1) : 0; --} -- --/* -- * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX -- * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate -- * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check -- * whether it's running from an allowed context. -- * -- * @mask is constant, always inline to cull the mask calculations. -- */ --static __always_inline void scx_kf_allow(u32 mask) --{ -- /* nesting is allowed only in increasing scx_kf_mask order */ -- WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask, -- "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n", -- current->scx->kf_mask, mask); -- current->scx->kf_mask |= mask; --} -- --static void scx_kf_disallow(u32 mask) --{ -- current->scx->kf_mask &= ~mask; --} -- --#define SCX_CALL_OP(mask, op, args...) \ --do { \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- scx_ops.op(args); \ -- } \ --} while (0) -- --#define SCX_CALL_OP_RET(mask, op, args...) \ --({ \ -- __typeof__(scx_ops.op(args)) __ret; \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- __ret = scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- __ret = scx_ops.op(args); \ -- } \ -- __ret; \ --}) -- --/* -- * Some kfuncs are allowed only on the tasks that are subjects of the -- * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such -- * restrictions, the following SCX_CALL_OP_*() variants should be used when -- * invoking scx_ops operations that take task arguments. These can only be used -- * for non-nesting operations due to the way the tasks are tracked. -- * -- * kfuncs which can only operate on such tasks can in turn use -- * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on -- * the specific task. -- */ --#define SCX_CALL_OP_TASK(mask, op, task, args...) \ --do { \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- SCX_CALL_OP(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ --} while (0) -- --#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ --({ \ -- __typeof__(scx_ops.op(task, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- __ret; \ --}) -- --#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ --({ \ -- __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task0; \ -- current->scx->kf_tasks[1] = task1; \ -- __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- current->scx->kf_tasks[1] = NULL; \ -- __ret; \ --}) -- --//#include "./slim_walt.c" -- --/* @mask is constant, always inline to cull unnecessary branches */ --static __always_inline bool scx_kf_allowed(u32 mask) --{ -- if (unlikely(!(current->scx->kf_mask & mask))) { -- scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", -- mask, current->scx->kf_mask); -- return false; -- } -- -- if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) && -- in_interrupt())) { -- scx_ops_error("sleepable kfunc called from non-sleepable context"); -- return false; -- } -- -- /* -- * Enforce nesting boundaries. e.g. A kfunc which can be called from -- * DISPATCH must not be called if we're running DEQUEUE which is nested -- * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE -- * boundary thanks to the above in_interrupt() check. -- */ -- if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && -- (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { -- scx_ops_error("cpu_release kfunc called from a nested operation"); -- return false; -- } -- -- if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && -- (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) { -- scx_ops_error("dispatch kfunc called from a nested operation"); -- return false; -- } -- -- return true; --} -- --/* see SCX_CALL_OP_TASK() */ --static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, -- struct task_struct *p) --{ -- if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED)) -- return false; -- -- if (unlikely((p != current->scx->kf_tasks[0] && -- p != current->scx->kf_tasks[1]))) { -- scx_ops_error("called on a task not being operated on"); -- return false; -- } -- -- return true; --} -- --/** -- * scx_task_iter_init - Initialize a task iterator -- * @iter: iterator to init -- * -- * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized, -- * @iter must eventually be exited with scx_task_iter_exit(). -- * -- * scx_tasks_lock may be released between this and the first next() call or -- * between any two next() calls. If scx_tasks_lock is released between two -- * next() calls, the caller is responsible for ensuring that the task being -- * iterated remains accessible either through RCU read lock or obtaining a -- * reference count. -- * -- * All tasks which existed when the iteration started are guaranteed to be -- * visited as long as they still exist. -- */ --static void scx_task_iter_init(struct scx_task_iter *iter) --{ -- lockdep_assert_held(&scx_tasks_lock); -- -- iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; -- list_add(&iter->cursor.tasks_node, &scx_tasks); -- iter->locked = NULL; --} -- --/** -- * scx_task_iter_exit - Exit a task iterator -- * @iter: iterator to exit -- * -- * Exit a previously initialized @iter. Must be called with scx_tasks_lock held. -- * If the iterator holds a task's rq lock, that rq lock is released. See -- * scx_task_iter_init() for details. -- */ --static void scx_task_iter_exit(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- if (list_empty(cursor)) -- return; -- -- list_del_init(cursor); --} -- --/** -- * scx_task_iter_next - Next task -- * @iter: iterator to walk -- * -- * Visit the next task. See scx_task_iter_init() for details. -- */ --static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- struct sched_ext_entity *pos; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- list_for_each_entry(pos, cursor, tasks_node) { -- if (&pos->tasks_node == &scx_tasks) -- return NULL; -- if (!(pos->flags & SCX_TASK_CURSOR)) { -- list_move(cursor, &pos->tasks_node); -- return pos->task; -- } -- } -- -- /* can't happen, should always terminate at scx_tasks above */ -- BUG(); --} -- --/** -- * scx_task_iter_next_filtered - Next non-idle task -- * @iter: iterator to walk -- * -- * Visit the next non-idle task. See scx_task_iter_init() for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- while ((p = scx_task_iter_next(iter))) { -- if (!is_idle_task(p)) -- return p; -- } -- return NULL; --} -- --/** -- * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked -- * @iter: iterator to walk -- * -- * Visit the next non-idle task with its rq lock held. See scx_task_iter_init() -- * for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered_locked(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- p = scx_task_iter_next_filtered(iter); -- if (!p) -- return NULL; -- -- iter->rq = task_rq_lock(p, &iter->rf); -- iter->locked = p; -- return p; --} -- --static enum scx_ops_enable_state scx_ops_enable_state(void) --{ -- return atomic_read(&scx_ops_enable_state_var); --} -- --static enum scx_ops_enable_state --scx_ops_set_enable_state(enum scx_ops_enable_state to) --{ -- return atomic_xchg(&scx_ops_enable_state_var, to); --} -- --static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, -- enum scx_ops_enable_state from) --{ -- int from_v = from; -- -- return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); --} -- --static bool scx_ops_disabling(void) --{ -- return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING); --} -- --/** -- * wait_ops_state - Busy-wait the specified ops state to end -- * @p: target task -- * @opss: state to wait the end of -- * -- * Busy-wait for @p to transition out of @opss. This can only be used when the -- * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also -- * has load_acquire semantics to ensure that the caller can see the updates made -- * in the enqueueing and dispatching paths. -- */ --static void wait_ops_state(struct task_struct *p, u64 opss) --{ -- do { -- cpu_relax(); -- } while (atomic64_read_acquire(&p->scx->ops_state) == opss); --} -- --/** -- * ops_cpu_valid - Verify a cpu number -- * @cpu: cpu number which came from a BPF ops -- * -- * @cpu is a cpu number which came from the BPF scheduler and can be any value. -- * Verify that it is in range and one of the possible cpus. -- */ --static bool ops_cpu_valid(s32 cpu) --{ -- return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); --} -- --/** -- * ops_sanitize_err - Sanitize a -errno value -- * @ops_name: operation to blame on failure -- * @err: -errno value to sanitize -- * -- * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return -- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can -- * cause misbehaviors. For an example, a large negative return from -- * ops.prep_enable() triggers an oops when passed up the call chain because the -- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is -- * handled as a pointer. -- */ --static int ops_sanitize_err(const char *ops_name, s32 err) --{ -- if (err < 0 && err >= -MAX_ERRNO) -- return err; -- -- scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); -- return -EPROTO; --} -- --/** -- * touch_core_sched - Update timestamp used for core-sched task ordering -- * @rq: rq to read clock from, must be locked -- * @p: task to update the timestamp for -- * -- * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to -- * implement global or local-DSQ FIFO ordering for core-sched. Should be called -- * when a task becomes runnable and its turn on the CPU ends (e.g. slice -- * exhaustion). -- */ --static void touch_core_sched(struct rq *rq, struct task_struct *p) --{ --#ifdef CONFIG_SCHED_CORE -- /* -- * It's okay to update the timestamp spuriously. Use -- * sched_core_disabled() which is cheaper than enabled(). -- */ -- if (!sched_core_disabled()) -- p->scx->core_sched_at = rq_clock_task(rq); --#endif --} -- --/** -- * touch_core_sched_dispatch - Update core-sched timestamp on dispatch -- * @rq: rq to read clock from, must be locked -- * @p: task being dispatched -- * -- * If the BPF scheduler implements custom core-sched ordering via -- * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO -- * ordering within each local DSQ. This function is called from dispatch paths -- * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect. -- */ --static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- assert_clock_updated(rq); -- --#ifdef CONFIG_SCHED_CORE -- if (SCX_HAS_OP(core_sched_before)) -- touch_core_sched(rq, p); --#endif --} -- --static void update_curr_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- u64 now = rq_clock_task(rq); -- u64 delta_exec; -- -- if (time_before_eq64(now, curr->se.exec_start)) -- return; -- -- delta_exec = now - curr->se.exec_start; -- curr->se.exec_start = now; -- curr->se.sum_exec_runtime += delta_exec; -- account_group_exec_runtime(curr, delta_exec); -- cgroup_account_cputime(curr, delta_exec); -- -- if (curr->scx->slice != SCX_SLICE_INF) { -- curr->scx->slice -= min(curr->scx->slice, delta_exec); -- if (!curr->scx->slice) -- touch_core_sched(rq, curr); -- } --} -- --static bool scx_dsq_priq_less(struct rb_node *node_a, -- const struct rb_node *node_b) --{ -- const struct sched_ext_entity *a = -- container_of(node_a, struct sched_ext_entity, dsq_node.priq); -- const struct sched_ext_entity *b = -- container_of(node_b, struct sched_ext_entity, dsq_node.priq); -- -- return time_before64(a->dsq_vtime, b->dsq_vtime); --} -- --static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, -- u64 enq_flags) --{ -- bool is_local = dsq->id == SCX_DSQ_LOCAL; -- -- WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo)); -- WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq)); -- -- if (!is_local) { -- raw_spin_lock(&dsq->lock); -- if (unlikely(dsq->id == SCX_DSQ_INVALID)) { -- scx_ops_error("attempting to dispatch to a destroyed dsq"); -- /* fall back to the global dsq */ -- raw_spin_unlock(&dsq->lock); -- dsq = &scx_dsq_global; -- raw_spin_lock(&dsq->lock); -- } -- } -- -- if (enq_flags & SCX_ENQ_DSQ_PRIQ) { -- p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; -- rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq, -- scx_dsq_priq_less); -- } else { -- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) -- list_add(&p->scx->dsq_node.fifo, &dsq->fifo); -- else -- list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo); -- } -- dsq->nr++; -- p->scx->dsq = dsq; -- -- /* -- * We're transitioning out of QUEUEING or DISPATCHING. store_release to -- * match waiters' load_acquire. -- */ -- if (enq_flags & SCX_ENQ_CLEAR_OPSS) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- if (is_local) { -- struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq); -- struct rq *rq = scx->rq; -- bool preempt = false; -- -- if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && -- rq->curr->sched_class == &ext_sched_class) { -- rq->curr->scx->slice = 0; -- preempt = true; -- } -- -- if (preempt || sched_class_above(&ext_sched_class, -- rq->curr->sched_class)) -- resched_curr(rq); -- } else { -- raw_spin_unlock(&dsq->lock); -- } --} -- --static void task_unlink_from_dsq(struct task_struct *p, -- struct scx_dispatch_q *dsq) --{ -- if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { -- rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; -- } else { -- list_del_init(&p->scx->dsq_node.fifo); -- } -- --} -- --static bool task_linked_on_dsq(struct task_struct *p) --{ -- return !list_empty(&p->scx->dsq_node.fifo) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq); --} -- --static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p) --{ -- struct scx_dispatch_q *dsq = p->scx->dsq; -- bool is_local = dsq == &scx_rq->local_dsq; -- -- if (!dsq) { -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- /* -- * When dispatching directly from the BPF scheduler to a local -- * DSQ, the task isn't associated with any DSQ but -- * @p->scx->holding_cpu may be set under the protection of -- * %SCX_OPSS_DISPATCHING. -- */ -- if (p->scx->holding_cpu >= 0) -- p->scx->holding_cpu = -1; -- return; -- } -- -- if (!is_local) -- raw_spin_lock(&dsq->lock); -- -- /* -- * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node -- * can't change underneath us. -- */ -- if (p->scx->holding_cpu < 0) { -- /* @p must still be on @dsq, dequeue */ -- WARN_ON_ONCE(!task_linked_on_dsq(p)); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- } else { -- /* -- * We're racing against dispatch_to_local_dsq() which already -- * removed @p from @dsq and set @p->scx->holding_cpu. Clear the -- * holding_cpu which tells dispatch_to_local_dsq() that it lost -- * the race. -- */ -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- p->scx->holding_cpu = -1; -- } -- p->scx->dsq = NULL; -- -- if (!is_local) -- raw_spin_unlock(&dsq->lock); --} -- --static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) --{ -- lockdep_assert(rcu_read_lock_any_held()); -- -- return &scx_dsq_global; --} -- --static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, -- struct task_struct *p) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id == SCX_DSQ_LOCAL) -- return &rq->scx->local_dsq; -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", -- dsq_id, p->comm, p->pid); -- return &scx_dsq_global; -- } -- -- return dsq; --} -- --static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- -- /* @p must match the task which is being enqueued */ -- if (unlikely(p != ddsp_task)) { -- if (IS_ERR(ddsp_task)) -- scx_ops_error("%s[%d] already direct-dispatched", -- p->comm, p->pid); -- else -- scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]", -- ddsp_task->comm, ddsp_task->pid, -- p->comm, p->pid); -- return; -- } -- -- /* -- * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because -- * dispatching to the local DSQ of a different CPU requires unlocking -- * the current rq which isn't allowed in the enqueue path. Use -- * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL. -- */ -- if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) { -- scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch"); -- return; -- } -- -- touch_core_sched_dispatch(task_rq(p), p); -- -- dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- -- /* -- * Mark that dispatch already happened by spoiling direct_dispatch_task -- * with a non-NULL value which can never match a valid task pointer. -- */ -- __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); --} -- --static bool test_rq_online(struct rq *rq) --{ --#ifdef CONFIG_SMP -- return rq->online; --#else -- return true; --#endif --} -- --static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- int sticky_cpu) --{ -- struct task_struct **ddsp_taskp; -- u64 qseq; -- -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- if (p->scx->flags & SCX_TASK_ENQ_LOCAL) { -- enq_flags |= SCX_ENQ_LOCAL; -- p->scx->flags &= ~SCX_TASK_ENQ_LOCAL; -- } -- -- /* rq migration */ -- if (sticky_cpu == cpu_of(rq)) -- goto local_norefill; -- -- /* -- * If !rq->online, we already told the BPF scheduler that the CPU is -- * offline. We're just trying to on/offline the CPU. Don't bother the -- * BPF scheduler. -- */ -- if (unlikely(!test_rq_online(rq))) -- goto local; -- -- /* see %SCX_OPS_ENQ_EXITING */ -- if (!static_branch_unlikely(&scx_ops_enq_exiting) && -- unlikely(p->flags & PF_EXITING)) -- goto local; -- -- /* see %SCX_OPS_ENQ_LAST */ -- if (!static_branch_unlikely(&scx_ops_enq_last) && -- (enq_flags & SCX_ENQ_LAST)) -- goto local; -- -- if (!SCX_HAS_OP(enqueue)) { -- if (enq_flags & SCX_ENQ_LOCAL) -- goto local; -- else -- goto global; -- } -- -- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ -- qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; -- -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq); -- -- ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); -- WARN_ON_ONCE(*ddsp_taskp); -- *ddsp_taskp = p; -- -- SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); -- -- /* -- * If not directly dispatched, QUEUEING isn't clear yet and dispatch or -- * dequeue may be waiting. The store_release matches their load_acquire. -- */ -- if (*ddsp_taskp == p) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq); -- *ddsp_taskp = NULL; -- return; -- --local: -- /* -- * For task-ordering, slice refill must be treated as implying the end -- * of the current slice. Otherwise, the longer @p stays on the CPU, the -- * higher priority it becomes from scx_prio_less()'s POV. -- */ -- touch_core_sched(rq, p); -- p->scx->slice = SCX_SLICE_DFL; --local_norefill: -- dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags); -- return; -- --global: -- touch_core_sched(rq, p); /* see the comment in local: */ -- p->scx->slice = SCX_SLICE_DFL; -- dispatch_enqueue(&scx_dsq_global, p, enq_flags); --} -- --static bool watchdog_task_watched(const struct task_struct *p) --{ -- return !list_empty(&p->scx->watchdog_node); --} -- --static void watchdog_watch_task(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- if (p->scx->flags & SCX_TASK_WATCHDOG_RESET) -- p->scx->runnable_at = jiffies; -- p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET; -- list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list); --} -- --static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) --{ -- list_del_init(&p->scx->watchdog_node); -- if (reset_timeout) -- p->scx->flags |= SCX_TASK_WATCHDOG_RESET; --} -- --static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) --{ -- int sticky_cpu = p->scx->sticky_cpu; -- -- enq_flags |= rq->scx->extra_enq_flags; -- -- if (sticky_cpu >= 0) -- p->scx->sticky_cpu = -1; -- -- /* -- * Restoring a running task will be immediately followed by -- * set_next_task_scx() which expects the task to not be on the BPF -- * scheduler as tasks can only start running through local DSQs. Force -- * direct-dispatch into the local DSQ by setting the sticky_cpu. -- */ -- if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -- sticky_cpu = cpu_of(rq); -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- WARN_ON_ONCE(!watchdog_task_watched(p)); -- return; -- } -- -- watchdog_watch_task(rq, p); -- p->scx->flags |= SCX_TASK_QUEUED; -- rq->scx->nr_running++; -- add_nr_running(rq, 1); -- -- if (SCX_HAS_OP(runnable)) -- SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); -- -- if (enq_flags & SCX_ENQ_WAKEUP) -- touch_core_sched(rq, p); -- -- do_enqueue_task(rq, p, enq_flags, sticky_cpu); --} -- --static void ops_dequeue(struct task_struct *p, u64 deq_flags) --{ -- u64 opss; -- -- watchdog_unwatch_task(p, false); -- -- /* acquire ensures that we see the preceding updates on QUEUED */ -- opss = atomic64_read_acquire(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_NONE: -- break; -- case SCX_OPSS_QUEUEING: -- /* -- * QUEUEING is started and finished while holding @p's rq lock. -- * As we're holding the rq lock now, we shouldn't see QUEUEING. -- */ -- BUG(); -- case SCX_OPSS_QUEUED: -- if (SCX_HAS_OP(dequeue)) -- SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); -- -- if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_NONE)) -- break; -- fallthrough; -- case SCX_OPSS_DISPATCHING: -- /* -- * If @p is being dispatched from the BPF scheduler to a DSQ, -- * wait for the transfer to complete so that @p doesn't get -- * added to its DSQ after dequeueing is complete. -- * -- * As we're waiting on DISPATCHING with the rq locked, the -- * dispatching side shouldn't try to lock the rq while -- * DISPATCHING is set. See dispatch_to_local_dsq(). -- * -- * DISPATCHING shouldn't have qseq set and control can reach -- * here with NONE @opss from the above QUEUED case block. -- * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. -- */ -- wait_ops_state(p, SCX_OPSS_DISPATCHING); -- BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- break; -- } --} -- --static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) --{ -- struct scx_rq *scx_rq = rq->scx; -- -- if (!(p->scx->flags & SCX_TASK_QUEUED)) { -- WARN_ON_ONCE(watchdog_task_watched(p)); -- return; -- } -- -- ops_dequeue(p, deq_flags); -- -- /* -- * A currently running task which is going off @rq first gets dequeued -- * and then stops running. As we want running <-> stopping transitions -- * to be contained within runnable <-> quiescent transitions, trigger -- * ->stopping() early here instead of in put_prev_task_scx(). -- * -- * @p may go through multiple stopping <-> running transitions between -- * here and put_prev_task_scx() if task attribute changes occur while -- * balance_scx() leaves @rq unlocked. However, they don't contain any -- * information meaningful to the BPF scheduler and can be suppressed by -- * skipping the callbacks if the task is !QUEUED. -- */ -- if (SCX_HAS_OP(stopping) && task_current(rq, p)) { -- update_curr_scx(rq); -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); -- } -- -- //if(task_current(rq, p)) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- if (SCX_HAS_OP(quiescent)) -- SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); -- -- if (deq_flags & SCX_DEQ_SLEEP) -- p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP; -- else -- p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP; -- -- p->scx->flags &= ~SCX_TASK_QUEUED; -- BUG_ON(!scx_rq->nr_running); -- scx_rq->nr_running--; -- sub_nr_running(rq, 1); -- -- dispatch_dequeue(scx_rq, p); --} -- --static void yield_task_scx(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); -- else -- p->scx->slice = 0; --} -- --static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) --{ -- struct task_struct *from = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); -- else -- return false; --} -- --#ifdef CONFIG_SMP --/** -- * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -- * @rq: rq to move the task into, currently locked -- * @p: task to move -- * @enq_flags: %SCX_ENQ_* -- * -- * Move @p which is currently on a different rq to @rq's local DSQ. The caller -- * must: -- * -- * 1. Start with exclusive access to @p either through its DSQ lock or -- * %SCX_OPSS_DISPATCHING flag. -- * -- * 2. Set @p->scx->holding_cpu to raw_smp_processor_id(). -- * -- * 3. Remember task_rq(@p). Release the exclusive access so that we don't -- * deadlock with dequeue. -- * -- * 4. Lock @rq and the task_rq from #3. -- * -- * 5. Call this function. -- * -- * Returns %true if @p was successfully moved. %false after racing dequeue and -- * losing. -- */ --static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -- u64 enq_flags) --{ -- struct rq *task_rq; -- -- lockdep_assert_rq_held(rq); -- -- /* -- * If dequeue got to @p while we were trying to lock both rq's, it'd -- * have cleared @p->scx->holding_cpu to -1. While other cpus may have -- * updated it to different values afterwards, as this operation can't be -- * preempted or recurse, @p->scx->holding_cpu can never become -- * raw_smp_processor_id() again before we're done. Thus, we can tell -- * whether we lost to dequeue by testing whether @p->scx->holding_cpu is -- * still raw_smp_processor_id(). -- * -- * See dispatch_dequeue() for the counterpart. -- */ -- if (unlikely(p->scx->holding_cpu != raw_smp_processor_id())) -- return false; -- -- /* @p->rq couldn't have changed if we're still the holding cpu */ -- task_rq = task_rq(p); -- lockdep_assert_rq_held(task_rq); -- -- WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)); -- deactivate_task(task_rq, p, 0); -- set_task_cpu(p, cpu_of(rq)); -- p->scx->sticky_cpu = cpu_of(rq); -- -- /* -- * We want to pass scx-specific enq_flags but activate_task() will -- * truncate the upper 32 bit. As we own @rq, we can pass them through -- * @rq->scx->extra_enq_flags instead. -- */ -- WARN_ON_ONCE(rq->scx->extra_enq_flags); -- rq->scx->extra_enq_flags = enq_flags; -- activate_task(rq, p, 0); -- rq->scx->extra_enq_flags = 0; -- -- return true; --} -- --/** -- * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * We're holding @rq lock and trying to dispatch a task from @src_rq to -- * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether -- * @rq stays locked isn't important as long as the state is restored after -- * dispatch_to_local_dsq_unlock(). -- */ --static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- rq_unpin_lock(rq, rf); -- -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(rq); -- raw_spin_rq_lock(dst_rq); -- } else if (rq == src_rq) { -- double_lock_balance(rq, dst_rq); -- rq_repin_lock(rq, rf); -- } else if (rq == dst_rq) { -- double_lock_balance(rq, src_rq); -- rq_repin_lock(rq, rf); -- } else { -- raw_spin_rq_unlock(rq); -- double_rq_lock(src_rq, dst_rq); -- } --} -- --/** -- * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock() -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return. -- */ --static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } else if (rq == src_rq) { -- double_unlock_balance(rq, dst_rq); -- } else if (rq == dst_rq) { -- double_unlock_balance(rq, src_rq); -- } else { -- double_rq_unlock(src_rq, dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } --} --#endif /* CONFIG_SMP */ -- -- --static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq) --{ -- return likely(test_rq_online(rq)) && !is_migration_disabled(p) && -- cpumask_test_cpu(cpu_of(rq), p->cpus_ptr); --} -- --static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -- struct scx_dispatch_q *dsq) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rb_node *rb_node; -- struct rq *task_rq; -- bool moved = false; --retry: -- if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -- return false; -- -- raw_spin_lock(&dsq->lock); -- -- list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- for (rb_node = rb_first_cached(&dsq->priq); rb_node; -- rb_node = rb_next(rb_node)) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- raw_spin_unlock(&dsq->lock); -- return false; -- --this_rq: -- /* @dsq is locked and @p is on this rq */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo); -- dsq->nr--; -- scx_rq->local_dsq.nr++; -- p->scx->dsq = &scx_rq->local_dsq; -- raw_spin_unlock(&dsq->lock); -- return true; -- --remote_rq: --#ifdef CONFIG_SMP -- /* -- * @dsq is locked and @p is on a remote rq. @p is currently protected by -- * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -- * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -- * rq lock or fail, do a little dancing from our side. See -- * move_task_to_local_dsq(). -- */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- p->scx->holding_cpu = raw_smp_processor_id(); -- raw_spin_unlock(&dsq->lock); -- -- rq_unpin_lock(rq, rf); -- double_lock_balance(rq, task_rq); -- rq_repin_lock(rq, rf); -- -- moved = move_task_to_local_dsq(rq, p, 0); -- -- double_unlock_balance(rq, task_rq); --#endif /* CONFIG_SMP */ -- if (likely(moved)) -- return true; -- goto retry; --} -- --enum dispatch_to_local_dsq_ret { -- DTL_DISPATCHED, /* successfully dispatched */ -- DTL_LOST, /* lost race to dequeue */ -- DTL_NOT_LOCAL, /* destination is not a local DSQ */ -- DTL_INVALID, /* invalid local dsq_id */ --}; -- --/** -- * dispatch_to_local_dsq - Dispatch a task to a local dsq -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @dsq_id: destination dsq ID -- * @p: task to dispatch -- * @enq_flags: %SCX_ENQ_* -- * -- * We're holding @rq lock and want to dispatch @p to the local DSQ identified by -- * @dsq_id. This function performs all the synchronization dancing needed -- * because local DSQs are protected with rq locks. -- * -- * The caller must have exclusive ownership of @p (e.g. through -- * %SCX_OPSS_DISPATCHING). -- */ --static enum dispatch_to_local_dsq_ret --dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id, -- struct task_struct *p, u64 enq_flags) --{ -- struct rq *src_rq = task_rq(p); -- struct rq *dst_rq; -- -- /* -- * We're synchronized against dequeue through DISPATCHING. As @p can't -- * be dequeued, its task_rq and cpus_allowed are stable too. -- */ -- if (dsq_id == SCX_DSQ_LOCAL) { -- dst_rq = rq; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]", -- cpu, p->comm, p->pid); -- return DTL_INVALID; -- } -- dst_rq = cpu_rq(cpu); -- } else { -- return DTL_NOT_LOCAL; -- } -- -- /* if dispatching to @rq that @p is already on, no lock dancing needed */ -- if (rq == src_rq && rq == dst_rq) { -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags | SCX_ENQ_CLEAR_OPSS); -- return DTL_DISPATCHED; -- } -- --#ifdef CONFIG_SMP -- if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { -- struct rq *locked_dst_rq = dst_rq; -- bool dsp; -- -- /* -- * @p is on a possibly remote @src_rq which we need to lock to -- * move the task. If dequeue is in progress, it'd be locking -- * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq -- * lock while holding DISPATCHING. -- * -- * As DISPATCHING guarantees that @p is wholly ours, we can -- * pretend that we're moving from a DSQ and use the same -- * mechanism - mark the task under transfer with holding_cpu, -- * release DISPATCHING and then follow the same protocol. -- */ -- p->scx->holding_cpu = raw_smp_processor_id(); -- -- /* store_release ensures that dequeue sees the above */ -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq); -- -- /* -- * We don't require the BPF scheduler to avoid dispatching to -- * offline CPUs mostly for convenience but also because CPUs can -- * go offline between scx_bpf_dispatch() calls and here. If @p -- * is destined to an offline CPU, queue it on its current CPU -- * instead, which should always be safe. As this is an allowed -- * behavior, don't trigger an ops error. -- */ -- if (unlikely(!test_rq_online(dst_rq))) -- dst_rq = src_rq; -- -- if (src_rq == dst_rq) { -- /* -- * As @p is staying on the same rq, there's no need to -- * go through the full deactivate/activate cycle. -- * Optimize by abbreviating the operations in -- * move_task_to_local_dsq(). -- */ -- dsp = p->scx->holding_cpu == raw_smp_processor_id(); -- if (likely(dsp)) { -- p->scx->holding_cpu = -1; -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags); -- } -- } else { -- dsp = move_task_to_local_dsq(dst_rq, p, enq_flags); -- } -- -- /* if the destination CPU is idle, wake it up */ -- if (dsp && p->sched_class > dst_rq->curr->sched_class) -- resched_curr(dst_rq); -- -- dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq); -- -- return dsp ? DTL_DISPATCHED : DTL_LOST; -- } --#endif /* CONFIG_SMP */ -- -- scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", -- cpu_of(dst_rq), p->comm, p->pid); -- return DTL_INVALID; --} -- --/** -- * finish_dispatch - Asynchronously finish dispatching a task -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @p: task to finish dispatching -- * @qseq_at_dispatch: qseq when @p started getting dispatched -- * @dsq_id: destination DSQ ID -- * @enq_flags: %SCX_ENQ_* -- * -- * Dispatching to local DSQs may need to wait for queueing to complete or -- * require rq lock dancing. As we don't wanna do either while inside -- * ops.dispatch() to avoid locking order inversion, we split dispatching into -- * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the -- * task and its qseq. Once ops.dispatch() returns, this function is called to -- * finish up. -- * -- * There is no guarantee that @p is still valid for dispatching or even that it -- * was valid in the first place. Make sure that the task is still owned by the -- * BPF scheduler and claim the ownership before dispatching. -- */ --static void finish_dispatch(struct rq *rq, struct rq_flags *rf, -- struct task_struct *p, u64 qseq_at_dispatch, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- u64 opss; -- -- touch_core_sched_dispatch(rq, p); --retry: -- /* -- * No need for _acquire here. @p is accessed only after a successful -- * try_cmpxchg to DISPATCHING. -- */ -- opss = atomic64_read(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_DISPATCHING: -- case SCX_OPSS_NONE: -- /* someone else already got to it */ -- return; -- case SCX_OPSS_QUEUED: -- /* -- * If qseq doesn't match, @p has gone through at least one -- * dispatch/dequeue and re-enqueue cycle between -- * scx_bpf_dispatch() and here and we have no claim on it. -- */ -- if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) -- return; -- -- /* -- * While we know @p is accessible, we don't yet have a claim on -- * it - the BPF scheduler is allowed to dispatch tasks -- * spuriously and there can be a racing dequeue attempt. Let's -- * claim @p by atomically transitioning it from QUEUED to -- * DISPATCHING. -- */ -- if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_DISPATCHING))) -- break; -- goto retry; -- case SCX_OPSS_QUEUEING: -- /* -- * do_enqueue_task() is in the process of transferring the task -- * to the BPF scheduler while holding @p's rq lock. As we aren't -- * holding any kernel or BPF resource that the enqueue path may -- * depend upon, it's safe to wait. -- */ -- wait_ops_state(p, opss); -- goto retry; -- } -- -- BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) { -- case DTL_DISPATCHED: -- break; -- case DTL_LOST: -- break; -- case DTL_INVALID: -- dsq_id = SCX_DSQ_GLOBAL; -- fallthrough; -- case DTL_NOT_LOCAL: -- dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()), -- dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- break; -- } --} -- --static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- u32 u; -- -- for (u = 0; u < dspc->buf_cursor; u++) { -- struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u]; -- -- finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id, -- ent->enq_flags); -- } -- -- dspc->nr_tasks += dspc->buf_cursor; -- dspc->buf_cursor = 0; --} -- --static int balance_one(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf, bool local) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- bool prev_on_scx = prev->sched_class == &ext_sched_class; -- int nr_loops = SCX_DSP_MAX_LOOPS; -- -- lockdep_assert_rq_held(rq); -- -- if (static_branch_unlikely(&scx_ops_cpu_preempt) && -- unlikely(rq->scx->cpu_released)) { -- /* -- * If the previous sched_class for the current CPU was not SCX, -- * notify the BPF scheduler that it again has control of the -- * core. This callback complements ->cpu_release(), which is -- * emitted in scx_notify_pick_next_task(). -- */ -- if (SCX_HAS_OP(cpu_acquire)) -- SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq), -- NULL); -- rq->scx->cpu_released = false; -- } -- -- if (prev_on_scx) { -- WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP)); -- update_curr_scx(rq); -- -- /* -- * If @prev is runnable & has slice left, it has priority and -- * fetching more just increases latency for the fetched tasks. -- * Tell put_prev_task_scx() to put @prev on local_dsq. If the -- * BPF scheduler wants to handle this explicitly, it should -- * implement ->cpu_released(). -- * -- * See scx_ops_disable_workfn() for the explanation on the -- * disabling() test. -- * -- * When balancing a remote CPU for core-sched, there won't be a -- * following put_prev_task_scx() call and we don't own -- * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the -- * same conditions later and pick @rq->curr accordingly. -- */ -- if ((prev->scx->flags & SCX_TASK_QUEUED) && -- prev->scx->slice && !scx_ops_disabling()) { -- if (local) -- prev->scx->flags |= SCX_TASK_BAL_KEEP; -- return 1; -- } -- } -- -- /* if there already are tasks to run, nothing to do */ -- if (scx_rq->local_dsq.nr) -- return 1; -- -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- if (!SCX_HAS_OP(dispatch)) -- return 0; -- -- dspc->rq = rq; -- dspc->rf = rf; -- -- /* -- * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, -- * the local DSQ might still end up empty after a successful -- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() -- * produced some tasks, retry. The BPF scheduler may depend on this -- * looping behavior to simplify its implementation. -- */ -- do { -- dspc->nr_tasks = 0; -- -- SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), -- prev_on_scx ? prev : NULL); -- -- flush_dispatch_buf(rq, rf); -- -- if (scx_rq->local_dsq.nr) -- return 1; -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- /* -- * ops.dispatch() can trap us in this loop by repeatedly -- * dispatching ineligible tasks. Break out once in a while to -- * allow the watchdog to run. As IRQ can't be enabled in -- * balance(), we want to complete this scheduling cycle and then -- * start a new one. IOW, we want to call resched_curr() on the -- * next, most likely idle, task, not the current one. Use -- * scx_bpf_kick_cpu() for deferred kicking. -- */ -- if (unlikely(!--nr_loops)) { -- scx_bpf_kick_cpu(cpu_of(rq), 0); -- break; -- } -- } while (dspc->nr_tasks); -- -- return 0; --} -- --static int balance_scx(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf) --{ -- int ret; -- -- ret = balance_one(rq, prev, rf, true); --#ifdef CONFIG_SCHED_SMT -- /* -- * When core-sched is enabled, this ops.balance() call will be followed -- * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx() -- * on the SMT siblings. Balance the siblings too. -- */ -- if (sched_core_enabled(rq)) { -- const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); -- int scpu; -- -- for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { -- struct rq *srq = cpu_rq(scpu); -- struct rq_flags srf; -- struct task_struct *sprev = srq->curr; -- -- /* -- * While core-scheduling, rq lock is shared among -- * siblings but the debug annotations and rq clock -- * aren't. Do pinning dance to transfer the ownership. -- */ -- WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); -- rq_unpin_lock(rq, rf); -- rq_pin_lock(srq, &srf); -- -- update_rq_clock(srq); -- balance_one(srq, sprev, &srf, false); -- -- rq_unpin_lock(srq, &srf); -- rq_repin_lock(rq, rf); -- } -- } --#endif -- return ret; --} -- --static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) --{ -- if (p->scx->flags & SCX_TASK_QUEUED) { -- /* -- * Core-sched might decide to execute @p before it is -- * dispatched. Call ops_dequeue() to notify the BPF scheduler. -- */ -- ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); -- dispatch_dequeue(rq->scx, p); -- } -- -- p->se.exec_start = rq_clock_task(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, running, p); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock); -- -- watchdog_unwatch_task(p, true); -- -- /* -- * @p is getting newly scheduled or got kicked after someone updated its -- * slice. Refresh whether tick can be stopped. See can_stop_tick_scx(). -- */ -- if ((p->scx->slice == SCX_SLICE_INF) != -- (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) { -- if (p->scx->slice == SCX_SLICE_INF) -- rq->scx->flags |= SCX_RQ_CAN_STOP_TICK; -- else -- rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK; -- -- sched_update_tick_dependency(rq); -- } --} -- --static void put_prev_task_scx(struct rq *rq, struct task_struct *p) --{ --#ifndef CONFIG_SMP -- /* -- * UP workaround. -- * -- * Because SCX may transfer tasks across CPUs during dispatch, dispatch -- * is performed from its balance operation which isn't called in UP. -- * Let's work around by calling it from the operations which come right -- * after. -- * -- * 1. If the prev task is on SCX, pick_next_task() calls -- * .put_prev_task() right after. As .put_prev_task() is also called -- * from other places, we need to distinguish the calls which can be -- * done by looking at the previous task's state - if still queued or -- * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task(). -- * This case is handled here. -- * -- * 2. If the prev task is not on SCX, the first following call into SCX -- * will be .pick_next_task(), which is covered by calling -- * balance_scx() from pick_next_task_scx(). -- * -- * Note that we can't merge the first case into the second as -- * balance_scx() must be called before the previous SCX task goes -- * through put_prev_task_scx(). -- * -- * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf. -- * Pass in %NULL. -- */ -- if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP)) -- balance_scx(rq, p, NULL); --#endif -- -- update_curr_scx(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- -- /* -- * If we're being called from put_prev_task_balance(), balance_scx() may -- * have decided that @p should keep running. -- */ -- if (p->scx->flags & SCX_TASK_BAL_KEEP) { -- p->scx->flags &= ~SCX_TASK_BAL_KEEP; -- watchdog_watch_task(rq, p); -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- watchdog_watch_task(rq, p); -- -- /* -- * If @p has slice left and balance_scx() didn't tag it for -- * keeping, @p is getting preempted by a higher priority -- * scheduler class or core-sched forcing a different task. Leave -- * it at the head of the local DSQ. -- */ -- if (p->scx->slice && !scx_ops_disabling()) { -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- /* -- * If we're in the pick_next_task path, balance_scx() should -- * have already populated the local DSQ if there are any other -- * available tasks. If empty, tell ops.enqueue() that @p is the -- * only one available for this cpu. ops.enqueue() should put it -- * on the local DSQ so that the subsequent pick_next_task_scx() -- * can find the task unless it wants to trigger a separate -- * follow-up scheduling event. -- */ -- if (list_empty(&rq->scx->local_dsq.fifo)) -- do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1); -- else -- do_enqueue_task(rq, p, 0, -1); -- } --} -- --static struct task_struct *first_local_task(struct rq *rq) --{ -- struct rb_node *rb_node; -- struct sched_ext_entity *entity; -- -- if (!list_empty(&rq->scx->local_dsq.fifo)) { -- entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo); -- return entity->task; -- } -- -- rb_node = rb_first_cached(&rq->scx->local_dsq.priq); -- if (rb_node) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- return entity->task; -- } -- -- return NULL; --} -- --static struct task_struct *pick_next_task_scx(struct rq *rq) --{ -- struct task_struct *p; -- --#ifndef CONFIG_SMP -- /* UP workaround - see the comment at the head of put_prev_task_scx() */ -- if (unlikely(rq->curr->sched_class != &ext_sched_class)) -- balance_scx(rq, rq->curr, NULL); --#endif -- -- p = first_local_task(rq); -- if (!p) -- return NULL; -- -- if (unlikely(!p->scx->slice)) { -- if (!scx_ops_disabling() && !scx_warned_zero_slice) { -- printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n", -- p->comm, p->pid); -- scx_warned_zero_slice = true; -- } -- p->scx->slice = SCX_SLICE_DFL; -- } -- -- set_next_task_scx(rq, p, true); -- -- return p; --} -- --#ifdef CONFIG_SCHED_CORE --/** -- * scx_prio_less - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Core-sched is implemented as an additional scheduling layer on top of the -- * usual sched_class'es and needs to find out the expected task ordering. For -- * SCX, core-sched calls this function to interrogate the task ordering. -- * -- * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used -- * to implement the default task ordering. The older the timestamp, the higher -- * prority the task - the global FIFO ordering matching the default scheduling -- * behavior. -- * -- * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to -- * implement FIFO ordering within each local DSQ. See pick_task_scx(). -- */ --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi) --{ -- /* -- * The const qualifiers are dropped from task_struct pointers when -- * calling ops.core_sched_before(). Accesses are controlled by the -- * verifier. -- */ -- if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling()) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, -- (struct task_struct *)a, -- (struct task_struct *)b); -- else -- return time_after64(a->scx->core_sched_at, b->scx->core_sched_at); --} -- --/** -- * pick_task_scx - Pick a candidate task for core-sched -- * @rq: rq to pick the candidate task from -- * -- * Core-sched calls this function on each SMT sibling to determine the next -- * tasks to run on the SMT siblings. balance_one() has been called on all -- * siblings and put_prev_task_scx() has been called only for the current CPU. -- * -- * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look -- * at the first task in the local dsq. @rq->curr has to be considered explicitly -- * to mimic %SCX_TASK_BAL_KEEP. -- */ --static struct task_struct *pick_task_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- struct task_struct *first = first_local_task(rq); -- -- if (curr->scx->flags & SCX_TASK_QUEUED) { -- /* is curr the only runnable task? */ -- if (!first) -- return curr; -- -- /* -- * Does curr trump first? We can always go by core_sched_at for -- * this comparison as it represents global FIFO ordering when -- * the default core-sched ordering is used and local-DSQ FIFO -- * ordering otherwise. -- * -- * We can have a task with an earlier timestamp on the DSQ. For -- * example, when a current task is preempted by a sibling -- * picking a different cookie, the task would be requeued at the -- * head of the local DSQ with an earlier timestamp than the -- * core-sched picked next task. Besides, the BPF scheduler may -- * dispatch any tasks to the local DSQ anytime. -- */ -- if (curr->scx->slice && time_before64(curr->scx->core_sched_at, -- first->scx->core_sched_at)) -- return curr; -- } -- -- return first; /* this may be %NULL */ --} --#endif /* CONFIG_SCHED_CORE */ -- --static enum scx_cpu_preempt_reason --preempt_reason_from_class(const struct sched_class *class) --{ --#ifdef CONFIG_SMP -- if (class == &stop_sched_class) -- return SCX_CPU_PREEMPT_STOP; --#endif -- if (class == &dl_sched_class) -- return SCX_CPU_PREEMPT_DL; -- if (class == &rt_sched_class) -- return SCX_CPU_PREEMPT_RT; -- return SCX_CPU_PREEMPT_UNKNOWN; --} -- --void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task, -- const struct sched_class *active) --{ -- lockdep_assert_rq_held(rq); -- -- /* -- * The callback is conceptually meant to convey that the CPU is no -- * longer under the control of SCX. Therefore, don't invoke the -- * callback if the CPU is is staying on SCX, or going idle (in which -- * case the SCX scheduler has actively decided not to schedule any -- * tasks on the CPU). -- */ -- if (likely(active >= &ext_sched_class)) -- return; -- -- /* -- * At this point we know that SCX was preempted by a higher priority -- * sched_class, so invoke the ->cpu_release() callback if we have not -- * done so already. We only send the callback once between SCX being -- * preempted, and it regaining control of the CPU. -- * -- * ->cpu_release() complements ->cpu_acquire(), which is emitted the -- * next time that balance_scx() is invoked. -- */ -- if (!rq->scx->cpu_released) { -- if (SCX_HAS_OP(cpu_release)) { -- struct scx_cpu_release_args args = { -- .reason = preempt_reason_from_class(active), -- .task = task, -- }; -- -- SCX_CALL_OP(SCX_KF_CPU_RELEASE, -- cpu_release, cpu_of(rq), &args); -- } -- rq->scx->cpu_released = true; -- } --} -- --#ifdef CONFIG_SMP -- --static bool test_and_clear_cpu_idle(int cpu) --{ -- if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -- if (cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- return true; -- } else { -- return false; -- } --} -- --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- int cpu; -- -- do { -- cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); -- if (cpu < nr_cpu_ids) { -- const struct cpumask *sbm = topology_sibling_cpumask(cpu); -- -- /* -- * If offline, @cpu is not its own sibling and we can -- * get caught in an infinite loop as @cpu is never -- * cleared from idle_masks.smt. Clear @cpu directly in -- * such cases. -- */ -- if (likely(cpumask_test_cpu(cpu, sbm))) -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm); -- else -- cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu)); -- } else { -- cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -- if (cpu >= nr_cpu_ids) -- return -EBUSY; -- } -- } while (!test_and_clear_cpu_idle(cpu)); -- -- return cpu; --} -- --static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) --{ -- s32 cpu; -- -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return prev_cpu; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local DSQ of the waker. -- */ -- if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 && -- scx_has_idle_cpus && !(current->flags & PF_EXITING)) { -- cpu = smp_processor_id(); -- if (cpumask_test_cpu(cpu, p->cpus_ptr)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (test_and_clear_cpu_idle(prev_cpu)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return prev_cpu; -- } -- -- if (p->nr_cpus_allowed == 1) -- return prev_cpu; -- -- cpu = scx_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- -- return prev_cpu; --} -- --static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) --{ -- if (SCX_HAS_OP(select_cpu)) { -- s32 cpu; -- -- cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu, -- wake_flags); -- if (ops_cpu_valid(cpu)) { -- return cpu; -- } else { -- scx_ops_error("select_cpu returned invalid cpu %d", cpu); -- return prev_cpu; -- } -- } else { -- return scx_select_cpu_dfl(p, prev_cpu, wake_flags); -- } --} -- --static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx) --{ -- set_cpus_allowed_common(p, ctx); -- -- /* -- * The effective cpumask is stored in @p->cpus_ptr which may temporarily -- * differ from the configured one in @p->cpus_mask. Always tell the bpf -- * scheduler the effective one. -- * -- * Fine-grained memory write control is enforced by BPF making the const -- * designation pointless. Cast it away when calling the operation. -- */ -- if (SCX_HAS_OP(set_cpumask)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, -- (struct cpumask *)p->cpus_ptr); --} -- --static void reset_idle_masks(void) --{ -- /* consider all cpus idle, should converge to the actual state quickly */ -- cpumask_setall(idle_masks.cpu); -- cpumask_setall(idle_masks.smt); -- scx_has_idle_cpus = true; --} -- --void __scx_update_idle(struct rq *rq, bool idle) --{ -- int cpu = cpu_of(rq); -- struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -- -- if (SCX_HAS_OP(update_idle)) { -- SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); -- if (!static_branch_unlikely(&scx_builtin_idle_enabled)) -- return; -- } -- -- if (idle) { -- cpumask_set_cpu(cpu, idle_masks.cpu); -- if (!scx_has_idle_cpus) -- scx_has_idle_cpus = true; -- -- /* -- * idle_masks.smt handling is racy but that's fine as it's only -- * for optimization and self-correcting. -- */ -- for_each_cpu(cpu, sib_mask) { -- if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -- return; -- } -- cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -- } else { -- cpumask_clear_cpu(cpu, idle_masks.cpu); -- if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -- } --} -- --#else /* !CONFIG_SMP */ -- --static bool test_and_clear_cpu_idle(int cpu) { return false; } --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } --static void reset_idle_masks(void) {} -- --#endif /* CONFIG_SMP */ -- --static bool check_rq_for_timeouts(struct rq *rq) --{ -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rq_flags rf; -- bool timed_out = false; -- -- rq_lock_irqsave(rq, &rf); -- list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) { -- unsigned long last_runnable; -- -- p = entity->task; -- last_runnable = p->scx->runnable_at; -- -- if (unlikely(time_after(jiffies, -- last_runnable + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "%s[%d] failed to run for %u.%03us", -- p->comm, p->pid, -- dur_ms / 1000, dur_ms % 1000); -- timed_out = true; -- break; -- } -- } -- rq_unlock_irqrestore(rq, &rf); -- -- return timed_out; --} -- --static void scx_watchdog_workfn(struct work_struct *work) --{ -- int cpu; -- -- scx_watchdog_timestamp = jiffies; -- -- for_each_online_cpu(cpu) { -- if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) -- break; -- -- cond_resched(); -- } -- queue_delayed_work(system_unbound_wq, to_delayed_work(work), -- scx_watchdog_timeout / 2); --} -- --static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) --{ -- update_curr_scx(rq); -- //scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock); -- /* -- * While disabling, always resched and refresh core-sched timestamp as -- * we can't trust the slice management or ops.core_sched_before(). -- */ -- if (scx_ops_disabling()) { -- curr->scx->slice = 0; -- touch_core_sched(rq, curr); -- } -- -- if (!curr->scx->slice) -- resched_curr(rq); --} -- --#define SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- --static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg) --{ -- int ret; -- -- WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED); -- -- p->scx->disallow = false; -- -- if (SCX_HAS_OP(prep_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- }; -- -- ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args); -- if (unlikely(ret)) { -- ret = ops_sanitize_err("prep_enable", ret); -- return ret; -- } -- } -- //scx_sched_init_task(p); -- -- if (p->scx->disallow) { -- struct rq *rq; -- struct rq_flags rf; -- -- rq = task_rq_lock(p, &rf); -- -- /* -- * We're either in fork or load path and @p->policy will be -- * applied right after. Reverting @p->policy here and rejecting -- * %SCHED_EXT transitions from scx_check_setscheduler() -- * guarantees that if ops.prep_enable() sets @p->disallow, @p -- * can never be in SCX. -- */ -- if (p->policy == SCHED_EXT) { -- p->policy = SCHED_NORMAL; -- atomic64_inc(&scx_nr_rejected); -- } -- -- task_rq_unlock(rq, p, &rf); -- } -- -- p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET); -- return 0; --} -- --static void scx_ops_enable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED)); -- -- if (SCX_HAS_OP(enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- p->scx->flags |= SCX_TASK_OPS_ENABLED; --} -- --static void scx_ops_disable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- if (p->scx->flags & SCX_TASK_OPS_PREPPED) { -- if (SCX_HAS_OP(cancel_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- } else if (p->scx->flags & SCX_TASK_OPS_ENABLED) { -- if (SCX_HAS_OP(disable)) -- SCX_CALL_OP(SCX_KF_REST, disable, p); -- p->scx->flags &= ~SCX_TASK_OPS_ENABLED; -- } --} -- --static void set_task_scx_weight(struct task_struct *p) --{ -- u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -- -- p->scx->weight = sched_weight_to_cgroup(weight); --} -- --/** -- * refresh_scx_weight - Refresh a task's ext weight -- * @p: task to refresh ext weight for -- * -- * @p->scx->weight carries the task's static priority in cgroup weight scale to -- * enable easy access from the BPF scheduler. To keep it synchronized with the -- * current task priority, this function should be called when a new task is -- * created, priority is changed for a task on sched_ext, and a task is switched -- * to sched_ext from other classes. -- */ --static void refresh_scx_weight(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- set_task_scx_weight(p); -- if (SCX_HAS_OP(set_weight)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight); --} -- --void scx_pre_fork(struct task_struct *p) --{ -- p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL); -- if (!p->scx) { -- goto lock; -- } -- -- p->scx->dsq = NULL; -- INIT_LIST_HEAD(&p->scx->dsq_node.fifo); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- INIT_LIST_HEAD(&p->scx->watchdog_node); -- p->scx->flags = 0; -- p->scx->weight = 0; -- p->scx->sticky_cpu = -1; -- p->scx->holding_cpu = -1; -- p->scx->kf_mask = 0; -- atomic64_set(&p->scx->ops_state, 0); -- p->scx->runnable_at = INITIAL_JIFFIES; -- p->scx->slice = SCX_SLICE_DFL; -- p->scx->task = p; -- p->sched_prop = 0; -- -- /* -- * BPF scheduler enable/disable paths want to be able to iterate and -- * update all tasks which can become complex when racing forks. As -- * enable/disable are very cold paths, let's use a percpu_rwsem to -- * exclude forks. -- */ --lock: -- percpu_down_read(&scx_fork_rwsem); --} -- --int scx_fork(struct task_struct *p) --{ -- percpu_rwsem_assert_held(&scx_fork_rwsem); -- -- if (scx_enabled()) -- return scx_ops_prepare_task(p, task_group(p)); -- else -- return 0; --} -- --void scx_post_fork(struct task_struct *p) --{ -- if (scx_enabled()) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- /* -- * Set the weight manually before calling ops.enable() so that -- * the scheduler doesn't see a stale value if they inspect the -- * task struct. We'll invoke ops.set_weight() afterwards, as it -- * would be odd to receive a callback on the task before we -- * tell the scheduler that it's been fully enabled. -- */ -- set_task_scx_weight(p); -- scx_ops_enable_task(p); -- refresh_scx_weight(p); -- task_rq_unlock(rq, p, &rf); -- } -- -- spin_lock_irq(&scx_tasks_lock); -- list_add_tail(&p->scx->tasks_node, &scx_tasks); -- spin_unlock_irq(&scx_tasks_lock); -- -- percpu_up_read(&scx_fork_rwsem); --} -- --void scx_cancel_fork(struct task_struct *p) --{ -- if (scx_enabled()) -- scx_ops_disable_task(p); -- percpu_up_read(&scx_fork_rwsem); --} -- --void sched_ext_free(struct task_struct *p) --{ -- unsigned long flags; -- -- spin_lock_irqsave(&scx_tasks_lock, flags); -- list_del_init(&p->scx->tasks_node); -- spin_unlock_irqrestore(&scx_tasks_lock, flags); -- -- /* -- * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED -> -- * ENABLED transitions can't race us. Disable ops for @p. -- */ -- if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- scx_ops_disable_task(p); -- task_rq_unlock(rq, p, &rf); -- } --} -- --static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) --{ --} -- --static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} --static void switched_to_scx(struct rq *rq, struct task_struct *p) {} -- --int scx_check_setscheduler(struct task_struct *p, int policy) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- /* if disallow, reject transitioning into SCX */ -- if (scx_enabled() && READ_ONCE(p->scx->disallow) && -- p->policy != policy && policy == SCHED_EXT) -- return -EACCES; -- -- return 0; --} -- --#ifdef CONFIG_NO_HZ_FULL --bool scx_can_stop_tick(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (scx_ops_disabling()) -- return false; -- -- if (p->sched_class != &ext_sched_class) -- return true; -- -- /* -- * @rq can dispatch from different DSQs, so we can't tell whether it -- * needs the tick or not by looking at nr_running. Allow stopping ticks -- * iff the BPF scheduler indicated so. See set_next_task_scx(). -- */ -- return rq->scx->flags & SCX_RQ_CAN_STOP_TICK; --} --#endif -- --static inline void scx_cgroup_lock(void) {} --static inline void scx_cgroup_unlock(void) {} -- --/* -- * Omitted operations: -- * -- * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -- * task isn't tied to the CPU at that point. Preemption is implemented by -- * resetting the victim task's slice to 0 and triggering reschedule on the -- * target CPU. -- * -- * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -- * -- * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -- * their current sched_class. Call them directly from sched core instead. -- * -- * - task_woken, switched_from: Unnecessary. -- */ --DEFINE_SCHED_CLASS(ext) = { -- .enqueue_task = enqueue_task_scx, -- .dequeue_task = dequeue_task_scx, -- .yield_task = yield_task_scx, -- .yield_to_task = yield_to_task_scx, -- -- .check_preempt_curr = check_preempt_curr_scx, -- -- .pick_next_task = pick_next_task_scx, -- -- .put_prev_task = put_prev_task_scx, -- .set_next_task = set_next_task_scx, -- --#ifdef CONFIG_SMP -- .balance = balance_scx, -- .select_task_rq = select_task_rq_scx, -- .set_cpus_allowed = set_cpus_allowed_scx, --#endif -- --#ifdef CONFIG_SCHED_CORE -- .pick_task = pick_task_scx, --#endif -- -- .task_tick = task_tick_scx, -- -- .switched_to = switched_to_scx, -- .prio_changed = prio_changed_scx, -- -- .update_curr = update_curr_scx, -- --#ifdef CONFIG_UCLAMP_TASK -- .uclamp_enabled = 0, --#endif --}; -- --static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) --{ -- memset(dsq, 0, sizeof(*dsq)); -- -- raw_spin_lock_init(&dsq->lock); -- INIT_LIST_HEAD(&dsq->fifo); -- dsq->id = dsq_id; --} -- --static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id & SCX_DSQ_FLAG_BUILTIN) -- return ERR_PTR(-EINVAL); -- -- dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node); -- if (!dsq) -- return ERR_PTR(-ENOMEM); -- -- init_dsq(dsq, dsq_id); -- -- return dsq; --} -- --static void free_dsq_irq_workfn(struct irq_work *irq_work) --{ -- struct llist_node *to_free = llist_del_all(&dsqs_to_free); -- struct scx_dispatch_q *dsq, *tmp_dsq; -- -- llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) -- kfree_rcu(dsq, rcu); --} -- --static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); -- --static void destroy_dsq(u64 dsq_id) --{ --} -- --static void scx_cgroup_exit(void) {} --static int scx_cgroup_init(void) { return 0; } --static void scx_cgroup_config_knobs(void) {} -- --/* -- * Used by sched_fork() and __setscheduler_prio() to pick the matching -- * sched_class. dl/rt are already handled. -- */ --bool task_on_scx(struct task_struct *p) --{ -- if (!scx_enabled() || scx_ops_disabling()) -- return false; -- if (READ_ONCE(scx_switching_all)) -- return true; -- return p->policy == SCHED_EXT; --} -- --static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags) --{ -- if (enq_flags & SCX_ENQ_LAST) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); --} -- --static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {} -- --static void scx_ops_disable_workfn(struct kthread_work *work) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- struct scx_task_iter sti; -- struct task_struct *p; -- const char *reason; -- int i, cpu, type; -- -- type = atomic_read(&scx_exit_type); -- while (true) { -- /* -- * NONE indicates that a new scx_ops has been registered since -- * disable was scheduled - don't kill the new ops. DONE -- * indicates that the ops has already been disabled. -- */ -- if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE) -- return; -- if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE)) -- break; -- } -- -- cancel_delayed_work_sync(&scx_watchdog_work); -- -- switch (type) { -- case SCX_EXIT_UNREG: -- reason = "BPF scheduler unregistered"; -- break; -- case SCX_EXIT_SYSRQ: -- reason = "disabled by sysrq-S"; -- break; -- case SCX_EXIT_ERROR: -- reason = "runtime error"; -- break; -- case SCX_EXIT_ERROR_BPF: -- reason = "scx_bpf_error"; -- break; -- case SCX_EXIT_ERROR_STALL: -- reason = "runnable task stall"; -- break; -- default: -- reason = ""; -- } -- -- ei->type = type; -- strlcpy(ei->reason, reason, sizeof(ei->reason)); -- -- switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { -- case SCX_OPS_DISABLED: -- pr_warn("sched_ext: ops error detected without ops (%s)\n", -- scx_exit_info.msg); -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- return; -- case SCX_OPS_PREPPING: -- goto forward_progress_guaranteed; -- case SCX_OPS_DISABLING: -- /* shouldn't happen but handle it like ENABLING if it does */ -- WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); -- fallthrough; -- case SCX_OPS_ENABLING: -- case SCX_OPS_ENABLED: -- break; -- } -- -- /* -- * DISABLING is set and ops was either ENABLING or ENABLED indicating -- * that the ops and static branches are set. -- * -- * We must guarantee that all runnable tasks make forward progress -- * without trusting the BPF scheduler. We can't grab any mutexes or -- * rwsems as they might be held by tasks that the BPF scheduler is -- * forgetting to run, which unfortunately also excludes toggling the -- * static branches. -- * -- * Let's work around by overriding a couple ops and modifying behaviors -- * based on the DISABLING state and then cycling the tasks through -- * dequeue/enqueue to force global FIFO scheduling. -- * -- * a. ops.enqueue() and .dispatch() are overridden for simple global -- * FIFO scheduling. -- * -- * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value -- * can't be trusted. Whenever a tick triggers, the running task is -- * rotated to the tail of the queue with core_sched_at touched. -- * -- * c. pick_next_task() suppresses zero slice warning. -- * -- * d. scx_prio_less() reverts to the default core_sched_at order. -- */ -- scx_ops.enqueue = scx_ops_fallback_enqueue; -- scx_ops.dispatch = scx_ops_fallback_dispatch; -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- SCHED_CHANGE_BLOCK(task_rq(p), p, -- DEQUEUE_SAVE | DEQUEUE_MOVE) { -- /* cycling deq/enq is enough, see above */ -- } -- } -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* kick all CPUs to restore ticks */ -- for_each_possible_cpu(cpu) -- resched_cpu(cpu); -- --forward_progress_guaranteed: -- /* -- * Here, every runnable task is guaranteed to make forward progress and -- * we can safely use blocking synchronization constructs. Actually -- * disable ops. -- */ -- mutex_lock(&scx_ops_enable_mutex); -- -- static_branch_disable(&__scx_switched_all); -- WRITE_ONCE(scx_switching_all, false); -- -- /* avoid racing against fork and cgroup changes */ -- cpus_read_lock(); -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- bool alive = READ_ONCE(p->__state) != TASK_DEAD; -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL); -- -- __setscheduler_prio(p, p->prio); -- } -- -- if (alive) -- check_class_changed(task_rq(p), p, old_class, p->prio); -- -- scx_ops_disable_task(p); -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* no task is on scx, turn off all the switches and flush in-progress calls */ -- static_branch_disable_cpuslocked(&__scx_ops_enabled); -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- static_branch_disable_cpuslocked(&scx_has_op[i]); -- static_branch_disable_cpuslocked(&scx_ops_enq_last); -- static_branch_disable_cpuslocked(&scx_ops_enq_exiting); -- static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- synchronize_rcu(); -- -- scx_cgroup_exit(); -- -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- cpus_read_unlock(); -- -- if (ei->type >= SCX_EXIT_ERROR) { -- printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name); -- -- if (ei->msg[0] == '\0') -- printk(KERN_ERR "sched_ext: %s\n", ei->reason); -- else -- printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg); -- -- stack_trace_print(ei->bt, ei->bt_len, 2); -- } -- -- if (scx_ops.exit) -- SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); -- -- memset(&scx_ops, 0, sizeof(scx_ops)); -- -- free_percpu(scx_dsp_buf); -- scx_dsp_buf = NULL; -- scx_dsp_max_batch = 0; -- -- mutex_unlock(&scx_ops_enable_mutex); -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- -- scx_cgroup_config_knobs(); --} -- --static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); -- --static void schedule_scx_ops_disable_work(void) --{ -- struct kthread_worker *helper = READ_ONCE(scx_ops_helper); -- -- /* -- * We may be called spuriously before the first bpf_sched_ext_reg(). If -- * scx_ops_helper isn't set up yet, there's nothing to do. -- */ -- if (helper) -- kthread_queue_work(helper, &scx_ops_disable_work); --} -- --static void scx_ops_disable(enum scx_exit_type type) --{ -- int none = SCX_EXIT_NONE; -- -- if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)) -- type = SCX_EXIT_ERROR; -- -- atomic_try_cmpxchg(&scx_exit_type, &none, type); -- -- schedule_scx_ops_disable_work(); --} -- --static void scx_ops_error_irq_workfn(struct irq_work *irq_work) --{ -- schedule_scx_ops_disable_work(); --} -- --static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- int none = SCX_EXIT_NONE; -- va_list args; -- -- if (!atomic_try_cmpxchg(&scx_exit_type, &none, type)) -- return; -- -- ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1); -- -- va_start(args, fmt); -- vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args); -- va_end(args); -- -- irq_work_queue(&scx_ops_error_irq_work); --} -- --static struct kthread_worker *scx_create_rt_helper(const char *name) --{ -- struct kthread_worker *helper; -- -- helper = kthread_create_worker(0, name); -- if (helper) -- sched_set_fifo(helper->task); -- return helper; --} -- --static int scx_ops_enable(struct sched_ext_ops *ops) --{ -- struct scx_task_iter sti; -- struct task_struct *p; -- int i, ret; -- int tcnt = 0; -- unsigned long long start = 0; -- unsigned long long total_start = sched_clock(); -- -- mutex_lock(&scx_ops_enable_mutex); -- -- if (!scx_ops_helper) { -- WRITE_ONCE(scx_ops_helper, -- scx_create_rt_helper("sched_ext_ops_helper")); -- if (!scx_ops_helper) { -- ret = -ENOMEM; -- goto err_unlock; -- } -- } -- -- if (scx_ops_enable_state() != SCX_OPS_DISABLED) { -- ret = -EBUSY; -- goto err_unlock; -- } -- -- //slim_walt_enable(true); -- -- /* -- * Set scx_ops, transition to PREPPING and clear exit info to arm the -- * disable path. Failure triggers full disabling from here on. -- */ -- scx_ops = *ops; -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != -- SCX_OPS_DISABLED); -- -- memset(&scx_exit_info, 0, sizeof(scx_exit_info)); -- atomic_set(&scx_exit_type, SCX_EXIT_NONE); -- scx_warned_zero_slice = false; -- -- atomic64_set(&scx_nr_rejected, 0); -- -- /* -- * Keep CPUs stable during enable so that the BPF scheduler can track -- * online CPUs by watching ->on/offline_cpu() after ->init(). -- */ -- cpus_read_lock(); -- -- scx_switch_all_req = true; -- if (scx_ops.init) { -- ret = SCX_CALL_OP_RET(SCX_KF_INIT, init); -- if (ret) { -- ret = ops_sanitize_err("init", ret); -- goto err_disable; -- } -- -- /* -- * Exit early if ops.init() triggered scx_bpf_error(). Not -- * strictly necessary as we'll fail transitioning into ENABLING -- * later but that'd be after calling ops.prep_enable() on all -- * tasks and with -EBUSY which isn't very intuitive. Let's exit -- * early with success so that the condition is notified through -- * ops.exit() like other scx_bpf_error() invocations. -- */ -- if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE) -- goto err_disable; -- } -- -- WARN_ON_ONCE(scx_dsp_buf); -- scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; -- scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch, -- __alignof__(scx_dsp_buf[0])); -- if (!scx_dsp_buf) { -- ret = -ENOMEM; -- goto err_disable; -- } -- -- scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; -- if (ops->timeout_ms) -- scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); -- -- scx_watchdog_timestamp = jiffies; -- queue_delayed_work(system_unbound_wq, &scx_watchdog_work, -- scx_watchdog_timeout / 2); -- -- /* -- * Lock out forks, cgroup on/offlining and moves before opening the -- * floodgate so that they don't wander into the operations prematurely. -- */ -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- if (((void (**)(void))ops)[i]) -- static_branch_enable_cpuslocked(&scx_has_op[i]); -- -- if (ops->flags & SCX_OPS_ENQ_LAST) -- static_branch_enable_cpuslocked(&scx_ops_enq_last); -- -- if (ops->flags & SCX_OPS_ENQ_EXITING) -- static_branch_enable_cpuslocked(&scx_ops_enq_exiting); -- if (scx_ops.cpu_acquire || scx_ops.cpu_release) -- static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); -- -- if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { -- reset_idle_masks(); -- static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); -- } else { -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- } -- -- /* -- * All cgroups should be initialized before letting in tasks. cgroup -- * on/offlining and task migrations are already locked out. -- */ -- ret = scx_cgroup_init(); -- if (ret) -- goto err_disable_unlock; -- -- static_branch_enable_cpuslocked(&__scx_ops_enabled); -- -- /* -- * Enable ops for every task. Fork is excluded by scx_fork_rwsem -- * preventing new tasks from being added. No need to exclude tasks -- * leaving as sched_ext_free() can handle both prepped and enabled -- * tasks. Prep all tasks first and then enable them with preemption -- * disabled. -- */ -- spin_lock_irq(&scx_tasks_lock); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered(&sti))) { -- get_task_struct(p); -- spin_unlock_irq(&scx_tasks_lock); -- -- ret = scx_ops_prepare_task(p, task_group(p)); -- if (ret) { -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n", -- ret, p->comm, p->pid); -- goto err_disable_unlock; -- } -- -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- } -- scx_task_iter_exit(&sti); -- -- /* -- * All tasks are prepped but are still ops-disabled. Ensure that -- * %current can't be scheduled out and switch everyone. -- * preempt_disable() is necessary because we can't guarantee that -- * %current won't be starved if scheduled out while switching. -- */ -- preempt_disable(); -- -- /* -- * From here on, the disable path must assume that tasks have ops -- * enabled and need to be recovered. -- */ -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { -- preempt_enable(); -- spin_unlock_irq(&scx_tasks_lock); -- ret = -EBUSY; -- goto err_disable_unlock; -- } -- -- /* -- * We're fully committed and can't fail. The PREPPED -> ENABLED -- * transitions here are synchronized against sched_ext_free() through -- * scx_tasks_lock. -- */ -- WRITE_ONCE(scx_switching_all, scx_switch_all_req); -- start = sched_clock(); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- tcnt++; -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- scx_ops_enable_task(p); -- __setscheduler_prio(p, p->prio); -- } -- -- check_class_changed(task_rq(p), p, old_class, p->prio); -- } else { -- scx_ops_disable_task(p); -- } -- } -- printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id()); -- scx_task_iter_exit(&sti); -- -- spin_unlock_irq(&scx_tasks_lock); -- preempt_enable(); -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { -- ret = -EBUSY; -- goto err_disable; -- } -- -- if (scx_switch_all_req) -- static_branch_enable_cpuslocked(&__scx_switched_all); -- -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- -- scx_cgroup_config_knobs(); -- printk("\n total duration = %llu , current cpu = %d\n", sched_clock() - total_start, smp_processor_id()); -- -- return 0; -- --err_unlock: -- mutex_unlock(&scx_ops_enable_mutex); -- return ret; -- --err_disable_unlock: -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); --err_disable: -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- /* must be fully disabled before returning */ -- scx_ops_disable(SCX_EXIT_ERROR); -- kthread_flush_work(&scx_ops_disable_work); -- return ret; --} -- --#ifdef CONFIG_SCHED_DEBUG --static const char *scx_ops_enable_state_str[] = { -- [SCX_OPS_PREPPING] = "prepping", -- [SCX_OPS_ENABLING] = "enabling", -- [SCX_OPS_ENABLED] = "enabled", -- [SCX_OPS_DISABLING] = "disabling", -- [SCX_OPS_DISABLED] = "disabled", --}; -- --static int scx_debug_show(struct seq_file *m, void *v) --{ -- mutex_lock(&scx_ops_enable_mutex); -- seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name); -- seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled()); -- seq_printf(m, "%-30s: %d\n", "switching_all", -- READ_ONCE(scx_switching_all)); -- seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all()); -- seq_printf(m, "%-30s: %s\n", "enable_state", -- scx_ops_enable_state_str[scx_ops_enable_state()]); -- seq_printf(m, "%-30s: %llu\n", "nr_rejected", -- atomic64_read(&scx_nr_rejected)); -- mutex_unlock(&scx_ops_enable_mutex); -- return 0; --} -- --static int scx_debug_open(struct inode *inode, struct file *file) --{ -- return single_open(file, scx_debug_show, NULL); --} -- --const struct file_operations sched_ext_fops = { -- .open = scx_debug_open, -- .read = seq_read, -- .llseek = seq_lseek, -- .release = single_release, --}; --#endif -- --/******************************************************************************** -- * bpf_struct_ops plumbing. -- */ --#include --#include --#include -- --extern struct btf *btf_vmlinux; --static const struct btf_type *task_struct_type; -- --static bool bpf_scx_is_valid_access(int off, int size, -- enum bpf_access_type type, -- const struct bpf_prog *prog, -- struct bpf_insn_access_aux *info) --{ -- if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) -- return false; -- if (type != BPF_READ) -- return false; -- if (off % size != 0) -- return false; -- -- return btf_ctx_access(off, size, type, prog, info); --} -- --static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, -- const struct bpf_reg_state *reg, -- int off, int size) --{ -- return 0; --} -- --static const struct bpf_func_proto * --bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) --{ -- switch (func_id) { -- case BPF_FUNC_task_storage_get: -- return &bpf_task_storage_get_proto; -- case BPF_FUNC_task_storage_delete: -- return &bpf_task_storage_delete_proto; -- default: -- return bpf_base_func_proto(func_id); -- } --} -- --const struct bpf_verifier_ops bpf_scx_verifier_ops = { -- .get_func_proto = bpf_scx_get_func_proto, -- .is_valid_access = bpf_scx_is_valid_access, -- .btf_struct_access = bpf_scx_btf_struct_access, --}; -- --static int bpf_scx_init_member(const struct btf_type *t, -- const struct btf_member *member, -- void *kdata, const void *udata) --{ -- const struct sched_ext_ops *uops = udata; -- struct sched_ext_ops *ops = kdata; -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- int ret; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, dispatch_max_batch): -- if (*(u32 *)(udata + moff) > INT_MAX) -- return -E2BIG; -- ops->dispatch_max_batch = *(u32 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, flags): -- if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) -- return -EINVAL; -- ops->flags = *(u64 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, name): -- ret = bpf_obj_name_cpy(ops->name, uops->name, -- sizeof(ops->name)); -- if (ret < 0) -- return ret; -- if (ret == 0) -- return -EINVAL; -- return 1; -- case offsetof(struct sched_ext_ops, timeout_ms): -- if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT) -- return -E2BIG; -- ops->timeout_ms = *(u32 *)(udata + moff); -- return 1; -- } -- -- return 0; --} -- --static int bpf_scx_check_member(const struct btf_type *t, -- const struct btf_member *member, -- const struct bpf_prog *prog) --{ -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, prep_enable): -- case offsetof(struct sched_ext_ops, init): -- case offsetof(struct sched_ext_ops, exit): -- break; -- default: -- /* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */ -- /* if (prog->aux->sleepable) -- return -EINVAL; */ -- break; -- } -- -- return 0; --} -- --static int bpf_scx_reg(void *kdata) --{ -- return scx_ops_enable(kdata); --} -- --static void bpf_scx_unreg(void *kdata) --{ -- scx_ops_disable(SCX_EXIT_UNREG); -- kthread_flush_work(&scx_ops_disable_work); --} -- --static int bpf_scx_init(struct btf *btf) --{ -- u32 type_id; -- -- type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); -- if (type_id < 0) -- return -EINVAL; -- task_struct_type = btf_type_by_id(btf, type_id); -- -- return 0; --} -- --/* "extern" to avoid sparse warning, only used in this file */ --extern struct bpf_struct_ops bpf_sched_ext_ops; -- --struct bpf_struct_ops bpf_sched_ext_ops = { -- .verifier_ops = &bpf_scx_verifier_ops, -- .reg = bpf_scx_reg, -- .unreg = bpf_scx_unreg, -- .check_member = bpf_scx_check_member, -- .init_member = bpf_scx_init_member, -- .init = bpf_scx_init, -- .name = "sched_ext_ops", --}; -- --static void sysrq_handle_sched_ext_reset(u8 key) --{ -- if (scx_ops_helper) -- scx_ops_disable(SCX_EXIT_SYSRQ); -- else -- pr_info("sched_ext: BPF scheduler not yet used\n"); --} -- --static const struct sysrq_key_op sysrq_sched_ext_reset_op = { -- .handler = sysrq_handle_sched_ext_reset, -- .help_msg = "reset-sched-ext(S)", -- .action_msg = "Disable sched_ext and revert all tasks to CFS", -- .enable_mask = SYSRQ_ENABLE_RTNICE, --}; -- --static void kick_cpus_irq_workfn(struct irq_work *irq_work) --{ -- struct rq *this_rq = this_rq(); -- u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); -- int this_cpu = cpu_of(this_rq); -- int cpu; -- -- for_each_cpu(cpu, this_rq->scx->cpus_to_kick) { -- struct rq *rq = cpu_rq(cpu); -- unsigned long flags; -- -- raw_spin_rq_lock_irqsave(rq, flags); -- -- if (cpu_online(cpu) || cpu == this_cpu) { -- if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) && -- rq->curr->sched_class == &ext_sched_class) -- rq->curr->scx->slice = 0; -- pseqs[cpu] = rq->scx->pnt_seq; -- resched_curr(rq); -- } else { -- cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait); -- } -- -- raw_spin_rq_unlock_irqrestore(rq, flags); -- } -- -- for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait, -- cpumask_of(this_cpu)) { -- /* -- * Pairs with smp_store_release() issued by this CPU in -- * scx_notify_pick_next_task() on the resched path. -- * -- * We busy-wait here to guarantee that no other task can be -- * scheduled on our core before the target CPU has entered the -- * resched path. -- */ -- while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu]) -- cpu_relax(); -- } -- -- cpumask_clear(this_rq->scx->cpus_to_kick); -- cpumask_clear(this_rq->scx->cpus_to_preempt); -- cpumask_clear(this_rq->scx->cpus_to_wait); --} -- --void __init init_sched_ext_class(void) --{ -- int cpu; -- u32 v; -- -- /* -- * The following is to prevent the compiler from optimizing out the enum -- * definitions so that BPF scheduler implementations can use them -- * through the generated vmlinux.h. -- */ -- WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | -- SCX_TG_ONLINE | SCX_KICK_PREEMPT); -- -- init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); --#ifdef CONFIG_SMP -- BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -- BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); --#endif -- scx_kick_cpus_pnt_seqs = -- __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * -- num_possible_cpus(), -- __alignof__(scx_kick_cpus_pnt_seqs[0])); -- BUG_ON(!scx_kick_cpus_pnt_seqs); -- -- for_each_possible_cpu(cpu) { -- struct rq *rq = cpu_rq(cpu); -- -- rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL); -- if (rq->scx) -- rq->scx->rq = rq; -- else -- pr_err("fatal error : alloc rq->scx failed!!!\n"); -- -- init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL); -- INIT_LIST_HEAD(&rq->scx->watchdog_list); -- -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL)); -- init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn); -- } -- -- register_sysrq_key('S', &sysrq_sched_ext_reset_op); -- INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); -- scx_cgroup_config_knobs(); --} -- -- --/******************************************************************************** -- * Helpers that can be called from the BPF scheduler. -- */ --#include -- --/* Disables missing prototype warnings for kfuncs */ --__diag_push(); --__diag_ignore_all("-Wmissing-prototypes", -- "Global functions as their definitions will be in vmlinux BTF"); -- --/** -- * scx_bpf_switch_all - Switch all tasks into SCX -- * @into_scx: switch direction -- * -- * If @into_scx is %true, all existing and future non-dl/rt tasks are switched -- * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on -- * SCX. The actual switching is asynchronous. Can be called from ops.init(). -- */ --void scx_bpf_switch_all(void) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return; -- -- scx_switch_all_req = true; --} -- --BTF_SET8_START(scx_kfunc_ids_init) --BTF_ID_FLAGS(func, scx_bpf_switch_all) --BTF_SET8_END(scx_kfunc_ids_init) -- --static const struct btf_kfunc_id_set scx_kfunc_set_init = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_init, --}; -- --/** -- * scx_bpf_create_dsq - Create a custom DSQ -- * @dsq_id: DSQ to create -- * @node: NUMA node to allocate from -- * -- * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(), -- * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move(). -- */ --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return -EINVAL; -- -- if (unlikely(node >= (int)nr_node_ids || -- (node < 0 && node != NUMA_NO_NODE))) -- return -EINVAL; -- return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); --} -- --BTF_SET8_START(scx_kfunc_ids_sleepable) --BTF_ID_FLAGS(func, scx_bpf_create_dsq) --BTF_SET8_END(scx_kfunc_ids_sleepable) -- --static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_sleepable, --}; -- --static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) --{ -- if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) -- return false; -- -- lockdep_assert_irqs_disabled(); -- -- if (unlikely(!p)) { -- scx_ops_error("called with NULL task"); -- return false; -- } -- -- if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { -- scx_ops_error("invalid enq_flags 0x%llx", enq_flags); -- return false; -- } -- -- return true; --} -- --static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) --{ -- struct task_struct *ddsp_task; -- int idx; -- -- ddsp_task = __this_cpu_read(direct_dispatch_task); -- if (ddsp_task) { -- direct_dispatch(ddsp_task, p, dsq_id, enq_flags); -- return; -- } -- -- idx = __this_cpu_read(scx_dsp_ctx.buf_cursor); -- if (unlikely(idx >= scx_dsp_max_batch)) { -- scx_ops_error("dispatch buffer overflow"); -- return; -- } -- -- this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){ -- .task = p, -- .qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK, -- .dsq_id = dsq_id, -- .enq_flags = enq_flags, -- }; -- __this_cpu_inc(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe -- * to call this function spuriously. Can be called from ops.enqueue() and -- * ops.dispatch(). -- * -- * When called from ops.enqueue(), it's for direct dispatch and @p must match -- * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the -- * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be -- * on the target CPU in the first place. -- * -- * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id -- * and this function can be called upto ops.dispatch_max_batch times to dispatch -- * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the -- * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. -- * -- * This function doesn't have any locking restrictions and may be called under -- * BPF locks (in the future when BPF introduces more flexible locking). -- * -- * @p is allowed to run for @slice. The scheduling path is triggered on slice -- * exhaustion. If zero, the current residual slice is maintained. If -- * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with -- * scx_bpf_kick_cpu() to trigger scheduling. -- */ --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- scx_dispatch_commit(p, dsq_id, enq_flags); --} -- --/** -- * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. -- * Tasks queued into the priority queue are ordered by @vtime and always -- * consumed after the tasks in the FIFO queue. All other aspects are identical -- * to scx_bpf_dispatch(). -- * -- * @vtime ordering is according to time_before64() which considers wrapping. A -- * numerically larger vtime may indicate an earlier position in the ordering and -- * vice-versa. -- */ --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 vtime, u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- p->scx->dsq_vtime = vtime; -- -- scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); --} -- --BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime) --BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_enqueue_dispatch, --}; -- --/** -- * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots -- * -- * Can only be called from ops.dispatch(). -- */ --u32 scx_bpf_dispatch_nr_slots(void) --{ -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return 0; -- -- return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ -- * @dsq_id: DSQ to consume -- * -- * Consume a task from the non-local DSQ identified by @dsq_id and transfer it -- * to the current CPU's local DSQ for execution. Can only be called from -- * ops.dispatch(). -- * -- * This function flushes the in-flight dispatches from scx_bpf_dispatch() before -- * trying to consume the specified DSQ. It may also grab rq locks and thus can't -- * be called under any BPF locks. -- * -- * Returns %true if a task has been consumed, %false if there isn't any task to -- * consume. -- */ --bool scx_bpf_consume(u64 dsq_id) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- struct scx_dispatch_q *dsq; -- -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return false; -- -- flush_dispatch_buf(dspc->rq, dspc->rf); -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); -- return false; -- } -- -- if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) { -- /* -- * A successfully consumed task can be dequeued before it starts -- * running while the CPU is trying to migrate other dispatched -- * tasks. Bump nr_tasks to tell balance_scx() to retry on empty -- * local DSQ. -- */ -- dspc->nr_tasks++; -- return true; -- } else { -- return false; -- } --} -- --BTF_SET8_START(scx_kfunc_ids_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) --BTF_ID_FLAGS(func, scx_bpf_consume) --BTF_SET8_END(scx_kfunc_ids_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_dispatch, --}; -- --/** -- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ -- * -- * Iterate over all of the tasks currently enqueued on the local DSQ of the -- * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of -- * processed tasks. Can only be called from ops.cpu_release(). -- */ --u32 scx_bpf_reenqueue_local(void) --{ -- u32 nr_enqueued, i; -- struct rq *rq; -- struct scx_rq *scx_rq; -- -- if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) -- return 0; -- -- rq = cpu_rq(smp_processor_id()); -- lockdep_assert_rq_held(rq); -- scx_rq = rq->scx; -- -- /* -- * Get the number of tasks on the local DSQ before iterating over it to -- * pull off tasks. The enqueue callback below can signal that it wants -- * the task to stay on the local DSQ, and we want to prevent the BPF -- * scheduler from causing us to loop indefinitely. -- */ -- nr_enqueued = scx_rq->local_dsq.nr; -- for (i = 0; i < nr_enqueued; i++) { -- struct task_struct *p; -- -- p = first_local_task(rq); -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- WARN_ON_ONCE(p->scx->holding_cpu != -1); -- dispatch_dequeue(scx_rq, p); -- do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); -- } -- -- return nr_enqueued; --} -- --BTF_SET8_START(scx_kfunc_ids_cpu_release) --BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) --BTF_SET8_END(scx_kfunc_ids_cpu_release) -- --static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_cpu_release, --}; -- --/** -- * scx_bpf_kick_cpu - Trigger reschedule on a CPU -- * @cpu: cpu to kick -- * @flags: SCX_KICK_* flags -- * -- * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or -- * trigger rescheduling on a busy CPU. This can be called from any online -- * scx_ops operation and the actual kicking is performed asynchronously through -- * an irq work. -- */ --void scx_bpf_kick_cpu(s32 cpu, u64 flags) --{ -- struct rq *rq; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d", cpu); -- return; -- } -- -- preempt_disable(); -- rq = this_rq(); -- -- /* -- * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting -- * rq locks. We can probably be smarter and avoid bouncing if called -- * from ops which don't hold a rq lock. -- */ -- cpumask_set_cpu(cpu, rq->scx->cpus_to_kick); -- if (flags & SCX_KICK_PREEMPT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt); -- if (flags & SCX_KICK_WAIT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_wait); -- -- irq_work_queue(&rq->scx->kick_cpus_irq_work); -- preempt_enable(); --} -- --/** -- * scx_bpf_dsq_nr_queued - Return the number of queued tasks -- * @dsq_id: id of the DSQ -- * -- * Return the number of tasks in the DSQ matching @dsq_id. If not found, -- * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops -- * operations. -- */ --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) --{ -- struct scx_dispatch_q *dsq; -- -- lockdep_assert(rcu_read_lock_any_held()); -- -- if (dsq_id == SCX_DSQ_LOCAL) { -- return this_rq()->scx->local_dsq.nr; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (ops_cpu_valid(cpu)) -- return cpu_rq(cpu)->scx->local_dsq.nr; -- } else { -- dsq = find_non_local_dsq(dsq_id); -- if (dsq) -- return dsq->nr; -- } -- return -ENOENT; --} -- --/** -- * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state -- * @cpu: cpu to test and clear idle for -- * -- * Returns %true if @cpu was idle and its idle state was successfully cleared. -- * %false otherwise. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return false; -- } -- -- if (ops_cpu_valid(cpu)) -- return test_and_clear_cpu_idle(cpu); -- else -- return false; --} -- --/** -- * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu -- * @cpus_allowed: Allowed cpumask -- * -- * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked -- * idle cpu number on success. -%EBUSY if no matching cpu was found. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return -EBUSY; -- } -- -- return scx_pick_idle_cpu(cpus_allowed); --} -- --/** -- * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking -- * per-CPU cpumask. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_cpumask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.cpu; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, -- * per-physical-core cpumask. Can be used to determine if an entire physical -- * core is free. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_smtmask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.smt; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to -- * either the percpu, or SMT idle-tracking cpumask. -- */ --void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) --{ -- /* -- * Empty function body because we aren't actually acquiring or -- * releasing a reference to a global idle cpumask, which is read-only -- * in the caller and is never released. The acquire / release semantics -- * here are just used to make the cpumask is a trusted pointer in the -- * caller. -- */ --} -- --struct scx_bpf_error_bstr_bufs { -- u64 data[MAX_BPRINTF_VARARGS]; -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs); -- --/** -- * scx_bpf_error_bstr - Indicate fatal error -- * @fmt: error message format string -- * @data: format string parameters packaged using ___bpf_fill() macro -- * @data__sz: @data len, must end in '__sz' for the verifier -- * -- * Indicate that the BPF scheduler encountered a fatal error and initiate ops -- * disabling. -- */ --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) --{ -- struct scx_bpf_error_bstr_bufs *bufs; -- unsigned long flags; -- struct bpf_bprintf_data args = {}; -- int ret; -- -- local_irq_save(flags); -- bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs); -- -- if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || -- (data__sz && !data)) { -- scx_ops_error("invalid data=%p and data__sz=%u", -- (void *)data, data__sz); -- goto out_restore; -- } -- -- ret = copy_from_kernel_nofault(bufs->data, data, data__sz); -- if (ret) { -- scx_ops_error("failed to read data fields (%d)", ret); -- goto out_restore; -- } -- -- ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args); -- if (ret < 0) { -- scx_ops_error("failed to format prepration (%d)", ret); -- goto out_restore; -- } -- -- ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt, -- args.bin_args); -- bpf_bprintf_cleanup(&args); -- if (ret < 0) { -- scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format", -- fmt, data, data__sz); -- goto out_restore; -- } -- -- scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg); --out_restore: -- local_irq_restore(flags); --} -- --/** -- * scx_bpf_destroy_dsq - Destroy a custom DSQ -- * @dsq_id: DSQ to destroy -- * -- * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with -- * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is -- * empty and no further tasks are dispatched to it. Ignored if called on a DSQ -- * which doesn't exist. Can be called from any online scx_ops operations. -- */ --void scx_bpf_destroy_dsq(u64 dsq_id) --{ -- destroy_dsq(dsq_id); --} -- --/** -- * scx_bpf_task_running - Is task currently running? -- * @p: task of interest -- */ --bool scx_bpf_task_running(const struct task_struct *p) --{ -- return task_rq(p)->curr == p; --} -- --/** -- * scx_bpf_task_cpu - CPU a task is currently associated with -- * @p: task of interest -- */ --s32 scx_bpf_task_cpu(const struct task_struct *p) --{ -- return task_cpu(p); --} -- --/** -- * scx_bpf_task_cgroup - Return the sched cgroup of a task -- * @p: task of interest -- * -- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with -- * from the scheduler's POV. SCX operations should use this function to -- * determine @p's current cgroup as, unlike following @p->cgroups, -- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all -- * rq-locked operations. Can be called on the parameter tasks of rq-locked -- * operations. The restriction guarantees that @p's rq is locked by the caller. -- */ --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) --{ -- struct task_group *tg = p->sched_task_group; -- struct cgroup *cgrp = &cgrp_dfl_root.cgrp; -- -- if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) -- goto out; -- -- /* -- * A task_group may either be a cgroup or an autogroup. In the latter -- * case, @tg->css.cgroup is %NULL. A task_group can't become the other -- * kind once created. -- */ -- if (tg && tg->css.cgroup) -- cgrp = tg->css.cgroup; -- else -- cgrp = &cgrp_dfl_root.cgrp; --out: -- cgroup_get(cgrp); -- return cgrp; --} -- --BTF_SET8_START(scx_kfunc_ids_any) --BTF_ID_FLAGS(func, scx_bpf_kick_cpu) --BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) --BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) --BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu) --BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) --BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) --BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) --BTF_ID_FLAGS(func, scx_bpf_task_running) --BTF_ID_FLAGS(func, scx_bpf_task_cpu) --BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE) --BTF_SET8_END(scx_kfunc_ids_any) -- --static const struct btf_kfunc_id_set scx_kfunc_set_any = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_any, --}; -- --__diag_pop(); -- --/* -- * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set() -- * needs most of the system to be up. -- */ --static int __init register_ext_kfuncs(void) --{ -- int ret; -- -- /* -- * Some kfuncs are context-sensitive and can only be called from -- * specific SCX ops. They are grouped into BTF sets accordingly. -- * Unfortunately, BPF currently doesn't have a way of enforcing such -- * restrictions. Eventually, the verifier should be able to enforce -- * them. For now, register them the same and make each kfunc explicitly -- * check using scx_kf_allowed(). -- */ -- if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_init)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_sleepable)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_enqueue_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_cpu_release)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_any))) { -- pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret); -- return ret; -- } -- -- return 0; --} --__initcall(register_ext_kfuncs); -diff --git b/kernel/sched/ext.h a/kernel/sched/ext.h -deleted file mode 100755 -index fba8ed845f99..000000000000 ---- b/kernel/sched/ext.h -+++ /dev/null -@@ -1,257 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ -- -- --/* -- * Tag marking a kernel function as a kfunc. This is meant to minimize the -- * amount of copy-paste that kfunc authors have to include for correctness so -- * as to avoid issues such as the compiler inlining or eliding either a static -- * kfunc, or a global kfunc in an LTO build. -- */ --#define __bpf_kfunc __used noinline -- --enum scx_wake_flags { -- /* expose select WF_* flags as enums */ -- SCX_WAKE_EXEC = WF_EXEC, -- SCX_WAKE_FORK = WF_FORK, -- SCX_WAKE_TTWU = WF_TTWU, -- SCX_WAKE_SYNC = WF_SYNC, --}; -- --enum scx_enq_flags { -- /* expose select ENQUEUE_* flags as enums */ -- SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, -- SCX_ENQ_HEAD = ENQUEUE_HEAD, -- -- /* high 32bits are SCX specific */ -- -- /* -- * Set the following to trigger preemption when calling -- * scx_bpf_dispatch() with a local dsq as the target. The slice of the -- * current task is cleared to zero and the CPU is kicked into the -- * scheduling path. Implies %SCX_ENQ_HEAD. -- */ -- SCX_ENQ_PREEMPT = 1LLU << 32, -- -- /* -- * The task being enqueued was previously enqueued on the current CPU's -- * %SCX_DSQ_LOCAL, but was removed from it in a call to the -- * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was -- * invoked in a ->cpu_release() callback, and the task is again -- * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the -- * task will not be scheduled on the CPU until at least the next invocation -- * of the ->cpu_acquire() callback. -- */ -- SCX_ENQ_REENQ = 1LLU << 40, -- -- /* -- * The task being enqueued is the only task available for the cpu. By -- * default, ext core keeps executing such tasks but when -- * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with -- * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set. -- * -- * If the BPF scheduler wants to continue executing the task, -- * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately. -- * If the task gets queued on a different dsq or the BPF side, the BPF -- * scheduler is responsible for triggering a follow-up scheduling event. -- * Otherwise, Execution may stall. -- */ -- SCX_ENQ_LAST = 1LLU << 41, -- -- /* -- * A hint indicating that it's advisable to enqueue the task on the -- * local dsq of the currently selected CPU. Currently used by -- * select_cpu_dfl() and together with %SCX_ENQ_LAST. -- */ -- SCX_ENQ_LOCAL = 1LLU << 42, -- -- /* high 8 bits are internal */ -- __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, -- -- SCX_ENQ_CLEAR_OPSS = 1LLU << 56, -- SCX_ENQ_DSQ_PRIQ = 1LLU << 57, --}; -- --enum scx_deq_flags { -- /* expose select DEQUEUE_* flags as enums */ -- SCX_DEQ_SLEEP = DEQUEUE_SLEEP, -- -- /* high 32bits are SCX specific */ -- -- /* -- * The generic core-sched layer decided to execute the task even though -- * it hasn't been dispatched yet. Dequeue from the BPF side. -- */ -- SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, --}; -- --enum scx_tg_flags { -- SCX_TG_ONLINE = 1U << 0, -- SCX_TG_INITED = 1U << 1, --}; -- --enum scx_kick_flags { -- SCX_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -- SCX_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ --}; -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --extern const struct sched_class ext_sched_class; --extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops; --extern const struct file_operations sched_ext_fops; --extern unsigned long scx_watchdog_timeout; --extern unsigned long scx_watchdog_timestamp; -- --DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); --DECLARE_STATIC_KEY_FALSE(__scx_switched_all); --#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) --#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) -- --DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); -- --bool task_on_scx(struct task_struct *p); --void scx_pre_fork(struct task_struct *p); --int scx_fork(struct task_struct *p); --void scx_post_fork(struct task_struct *p); --void scx_cancel_fork(struct task_struct *p); --int scx_check_setscheduler(struct task_struct *p, int policy); --bool scx_can_stop_tick(struct rq *rq); --void init_sched_ext_class(void); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...); --#define scx_ops_error(fmt, args...) \ -- scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args) -- --void __scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active); -- --static inline void scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active) --{ -- if (!scx_enabled()) -- return; --#ifdef CONFIG_SMP -- /* -- * Pairs with the smp_load_acquire() issued by a CPU in -- * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -- * resched. -- */ -- smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1); --#endif -- if (!static_branch_unlikely(&scx_ops_cpu_preempt)) -- return; -- __scx_notify_pick_next_task(rq, p, active); --} -- --//extern void scx_scheduler_tick(void); --static inline void scx_notify_sched_tick(void) --{ -- unsigned long last_check; -- -- if (!scx_enabled()) -- return; -- //scx_scheduler_tick(); -- -- last_check = scx_watchdog_timestamp; -- if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "watchdog failed to check in for %u.%03us", -- dur_ms / 1000, dur_ms % 1000); -- } --} -- --static inline const struct sched_class *next_active_class(const struct sched_class *class) --{ -- class++; -- if (scx_switched_all() && class == &fair_sched_class) -- class++; -- if (!scx_enabled() && class == &ext_sched_class) -- class++; -- return class; --} -- --#define for_active_class_range(class, _from, _to) \ -- for (class = (_from); class != (_to); class = next_active_class(class)) -- --#define for_each_active_class(class) \ -- for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -- --/* -- * SCX requires a balance() call before every pick_next_task() call including -- * when waking up from idle. -- */ --#define for_balance_class_range(class, prev_class, end_class) \ -- for_active_class_range(class, (prev_class) > &ext_sched_class ? \ -- &ext_sched_class : (prev_class), (end_class)) -- --#ifdef CONFIG_SCHED_CORE --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi); --#endif -- --#else /* CONFIG_SCHED_CLASS_EXT */ -- --#define scx_enabled() false --#define scx_switched_all() false -- --static inline void scx_pre_fork(struct task_struct *p) {} --static inline int scx_fork(struct task_struct *p) { return 0; } --static inline void scx_post_fork(struct task_struct *p) {} --static inline void scx_cancel_fork(struct task_struct *p) {} --static inline int scx_check_setscheduler(struct task_struct *p, -- int policy) { return 0; } --static inline bool scx_can_stop_tick(struct rq *rq) { return true; } --static inline void init_sched_ext_class(void) {} --static inline void scx_notify_pick_next_task(struct rq *rq, -- const struct task_struct *p, -- const struct sched_class *active) {} --static inline void scx_notify_sched_tick(void) {} -- --#define for_each_active_class for_each_class --#define for_balance_class_range for_class_range -- --#endif /* CONFIG_SCHED_CLASS_EXT */ -- --#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) --void __scx_update_idle(struct rq *rq, bool idle); -- --static inline void scx_update_idle(struct rq *rq, bool idle) --{ -- if (scx_enabled()) -- __scx_update_idle(rq, idle); --} --#else --static inline void scx_update_idle(struct rq *rq, bool idle) {} --#endif -- --#ifdef CONFIG_CGROUP_SCHED --#ifdef CONFIG_EXT_GROUP_SCHED --int scx_tg_online(struct task_group *tg); --void scx_tg_offline(struct task_group *tg); --int scx_cgroup_can_attach(struct cgroup_taskset *tset); --void scx_move_task(struct task_struct *p); --void scx_cgroup_finish_attach(void); --void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); --void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); --#else /* CONFIG_EXT_GROUP_SCHED */ --static inline int scx_tg_online(struct task_group *tg) { return 0; } --static inline void scx_tg_offline(struct task_group *tg) {} --static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } --static inline void scx_move_task(struct task_struct *p) {} --static inline void scx_cgroup_finish_attach(void) {} --static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} --static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} --#endif /* CONFIG_EXT_GROUP_SCHED */ --#endif /* CONFIG_CGROUP_SCHED */ -diff --git b/kernel/sched/hmbird.h a/kernel/sched/hmbird.h -new file mode 100755 -index 000000000000..fa76c7f09977 ---- /dev/null -+++ a/kernel/sched/hmbird.h -@@ -0,0 +1,342 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#ifndef _HMBIRD_H_ -+#define _HMBIRD_H_ -+ -+/* -+ * Tag marking a kernel function as a kfunc. This is meant to minimize the -+ * amount of copy-paste that kfunc authors have to include for correctness so -+ * as to avoid issues such as the compiler inlining or eliding either a static -+ * kfunc, or a global kfunc in an LTO build. -+ */ -+ -+#define DEBUG_INTERNAL (1 << 0) -+#define DEBUG_INFO_TRACE (1 << 1) -+#define DEBUG_INFO_SYSTRACE (1 << 2) -+ -+#define NUMS_CGROUP_KINDS (512) -+#define SLIM_FOR_SGAME (1 << 0) -+#define SLIM_FOR_GENSHIN (1 << 1) -+ -+#ifdef MAX_TASK_NR -+#define MAX_KEY_THREAD_RECORD ((MAX_TASK_NR + 1) >> 1) -+#else -+#define MAX_KEY_THREAD_RECORD (1 << 3) -+#endif /* MAX_TASK_NR */ -+ -+#define TOP_TASK_SHIFT (8) -+#define TOP_TASK_MAX (1 << TOP_TASK_SHIFT) -+#ifndef TOP_TASK_BITS_MASK -+#define TOP_TASK_BITS_MASK (TOP_TASK_MAX - 1) -+#endif /* TOP_TASK_BITS_MASK */ -+ -+extern struct md_info_t *md_info; -+ -+#define debug_enabled() \ -+ (unlikely(hmbirdcore_debug)) -+ -+#define hmbird_debug(fmt, ...) \ -+ pr_info(":"fmt, ##__VA_ARGS__) -+ -+#define hmbird_err(id, fmt, ...) \ -+do { \ -+ pr_err(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_deferred_err(id, fmt, ...) \ -+do { \ -+ printk_deferred(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \ -+do { \ -+ if (unlikely(cond)) { \ -+ hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+ } \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_info_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_info_trace(fmt, ...) -+#endif -+ -+#define hmbird_info_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+ } \ -+} while (0) -+ -+#define hmbird_output_systrace(fmt, ...) \ -+do { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_internal_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_internal_trace(fmt, ...) -+#endif -+ -+#define hmbird_internal_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) { \ -+ hmbird_output_systrace(fmt, ##__VA_ARGS__); \ -+ } \ -+} while (0) -+ -+enum hmbird_wake_flags { -+ /* expose select WF_* flags as enums */ -+ HMBIRD_WAKE_EXEC = WF_EXEC, -+ HMBIRD_WAKE_FORK = WF_FORK, -+ HMBIRD_WAKE_TTWU = WF_TTWU, -+ HMBIRD_WAKE_SYNC = WF_SYNC, -+}; -+ -+enum hmbird_enq_flags { -+ /* expose select ENQUEUE_* flags as enums */ -+ HMBIRD_ENQ_WAKEUP = ENQUEUE_WAKEUP, -+ HMBIRD_ENQ_HEAD = ENQUEUE_HEAD, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * Set the following to trigger preemption when calling -+ * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the -+ * current task is cleared to zero and the CPU is kicked into the -+ * scheduling path. Implies %HMBIRD_ENQ_HEAD. -+ */ -+ HMBIRD_ENQ_PREEMPT = 1LLU << 32, -+ HMBIRD_ENQ_REENQ = 1LLU << 40, -+ HMBIRD_ENQ_LAST = 1LLU << 41, -+ -+ /* -+ * A hint indicating that it's advisable to enqueue the task on the -+ * local dsq of the currently selected CPU. Currently used by -+ * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST. -+ */ -+ HMBIRD_ENQ_LOCAL = 1LLU << 42, -+ -+ /* high 8 bits are internal */ -+ __HMBIRD_ENQ_INTERNAL_MASK = 0xffLLU << 56, -+ -+ HMBIRD_ENQ_CLEAR_OPSS = 1LLU << 56, -+ HMBIRD_ENQ_DSQ_PRIQ = 1LLU << 57, -+}; -+ -+enum hmbird_deq_flags { -+ /* expose select DEQUEUE_* flags as enums */ -+ HMBIRD_DEQ_SLEEP = DEQUEUE_SLEEP, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * The generic core-sched layer decided to execute the task even though -+ * it hasn't been dispatched yet. Dequeue from the BPF side. -+ */ -+ HMBIRD_DEQ_CORE_SCHED_EXEC = 1LLU << 32, -+}; -+ -+enum hmbird_kick_flags { -+ HMBIRD_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -+ HMBIRD_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ -+}; -+ -+enum hmbird_task_prop_type { -+ HMBIRD_TASK_PROP_COMMON, -+ HMBIRD_TASK_PROP_PIPELINE, -+ HMBIRD_TASK_PROP_DEBUG_OR_LOG, -+ HMBIRD_TASK_PROP_HIGH_LOAD, -+ HMBIRD_TASK_PROP_IO, -+ HMBIRD_TASK_PROP_NETWORK, -+ HMBIRD_TASK_PROP_PERIODIC, -+ HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL, -+ HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL, -+ HMBIRD_TASK_PROP_ISOLATE, -+ HMBIRD_TASK_PROP_MAX, -+}; -+ -+enum hmbird_preempt_policy_id { -+ HMBIRD_PREEMPT_POLICY_NONE, -+ HMBIRD_PREEMPT_POLICY_PRIO_BASED, -+}; -+ -+extern const struct sched_class hmbird_sched_class; -+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops; -+extern const struct file_operations sched_hmbird_fops; -+extern unsigned long hmbird_watchdog_timeout; -+extern unsigned long hmbird_watchdog_timestamp; -+ -+enum scx_rq_flags { -+ HMBIRD_RQ_CAN_STOP_TICK = 1 << 0, -+}; -+ -+struct hmbird_rq { -+ struct hmbird_dispatch_q local_dsq; -+ struct list_head watchdog_list; -+ u64 ops_qseq; -+ u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -+ u32 nr_running; -+ u32 flags; -+ bool cpu_released; -+ cpumask_var_t cpus_to_kick; -+ cpumask_var_t cpus_to_preempt; -+ cpumask_var_t cpus_to_wait; -+ u64 pnt_seq; -+ u64 *prev_runnable_sum_fixed; -+ u32 *prev_window_size; -+ struct irq_work kick_cpus_irq_work; -+ bool pipeline; -+ bool exclusive; -+ bool period_disallow; -+ bool nonperiod_disallow; -+ struct rq *rq; -+ struct hmbird_sched_rq_stats *srq; -+}; -+ -+struct hmbird_sched_change_guard { -+ struct task_struct *p; -+ struct rq *rq; -+ bool queued; -+ bool running; -+ bool done; -+}; -+ -+extern struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -+ -+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags); -+ -+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -+ for (struct hmbird_sched_change_guard __cg = \ -+ hmbird_sched_change_guard_init(__rq, __p, __flags); \ -+ !__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags)) -+ -+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+bool task_on_hmbird(struct task_struct *p); -+int hmbird_pre_fork(struct task_struct *p); -+void hmbird_post_fork(struct task_struct *p); -+void hmbird_cancel_fork(struct task_struct *p); -+int hmbird_check_setscheduler(struct task_struct *p, int policy); -+bool hmbird_can_stop_tick(struct rq *rq); -+void init_sched_hmbird_class(void); -+ -+void hmbird_ops_exit(void); -+#define hmbird_ops_error(fmt, ...) \ -+do { \ -+ hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__); \ -+ hmbird_ops_exit(); \ -+} while (0) -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active); -+ -+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight) -+{ -+ return clamp_t(unsigned long, -+ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -+ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); -+} -+ -+static inline void hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active) -+{ -+ if (!hmbird_enabled()) -+ return; -+#ifdef CONFIG_SMP -+ /* -+ * Pairs with the smp_load_acquire() issued by a CPU in -+ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -+ * resched. -+ */ -+ smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1); -+#endif -+ if (!static_branch_unlikely(&hmbird_ops_cpu_preempt)) -+ return; -+ __hmbird_notify_pick_next_task(rq, p, active); -+} -+ -+extern void hmbird_scheduler_tick(void); -+void scan_timeout(struct rq *rq); -+void hmbird_notify_sched_tick(void); -+ -+static inline const struct sched_class *next_active_class(const struct sched_class *class) -+{ -+ class++; -+ -+ if (hmbird_enabled() && class == &fair_sched_class) -+ class++; -+ if (!hmbird_enabled() && class == &hmbird_sched_class) -+ class++; -+ return class; -+} -+ -+#define for_active_class_range(class, _from, _to) \ -+ for (class = (_from); class != (_to); class = next_active_class(class)) -+ -+#define for_each_active_class(class) \ -+ for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -+ -+/* -+ * HMBIRD requires a balance() call before every pick_next_task() call including -+ * when waking up from idle. -+ */ -+#define for_balance_class_range(class, prev_class, end_class) \ -+ for_active_class_range(class, (prev_class) > &hmbird_sched_class ? \ -+ &hmbird_sched_class : (prev_class), (end_class)) -+ -+#define MIN_CGROUP_DL_IDX (5) /* 8ms */ -+#define DEFAULT_CGROUP_DL_IDX (8) /* 64ms */ -+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS]; -+ -+ -+void __hmbird_update_idle(struct rq *rq, bool idle); -+ -+static inline void hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ if (hmbird_enabled()) -+ __hmbird_update_idle(rq, idle); -+} -+ -+int hmbird_ctrl(bool enable); -+ -+int hmbird_tg_online(struct task_group *tg); -+ -+ -+static inline u16 hmbird_task_util(struct task_struct *p) -+{ -+ return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0; -+} -+u16 hmbird_cpu_util(int cpu); -+ -+bool get_hmbird_ops_enabled(void); -+bool get_non_hmbird_task(void); -+void set_cpu_cluster(u64 cpu_cluster); -+#endif /*_EXT_H_*/ -diff --git b/kernel/sched/hmbird/hmbird.c a/kernel/sched/hmbird/hmbird.c -new file mode 100755 -index 000000000000..86f9fd092424 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird.c -@@ -0,0 +1,4354 @@ -+// SPDX-License-Identifier: GPL-2.0 -+ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#include -+#include -+ -+#include "slim.h" -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+#include -+ -+#define CLUSTER_SEPARATE -+ -+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0); -+atomic_t non_hmbird_task; -+atomic_t hmbird_module_loaded = ATOMIC_INIT(0); -+int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0}; -+ -+enum hmbird_internal_consts { -+ HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ, -+}; -+ -+enum hmbird_ops_enable_state { -+ HMBIRD_OPS_PREPPING, -+ HMBIRD_OPS_ENABLING, -+ HMBIRD_OPS_ENABLED, -+ HMBIRD_OPS_DISABLING, -+ HMBIRD_OPS_DISABLED, -+}; -+ -+static inline void put_hmbird_ts(struct task_struct *p) -+{ -+ kfree((void *)p->android_oem_data1[HMBIRD_TS_IDX]); -+ p->android_oem_data1[HMBIRD_TS_IDX] = 0; -+} -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+/* -+ * hmbird_entity->ops_state -+ * -+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler. -+ * State transitions look as follows: -+ * -+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING -+ * ^ | | -+ * | v v -+ * \-------------------------------/ -+ * -+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -+ * sites for explanations on the conditions being waited upon and why they are -+ * safe. Transitions out of them into NONE or QUEUED must store_release and the -+ * waiters should load_acquire. -+ * -+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether -+ * any given task can be dispatched by the BPF scheduler at all times and thus -+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core -+ * can safely reject invalid dispatches. -+ */ -+enum hmbird_ops_state { -+ HMBIRD_OPSS_NONE, /* owned by the HMBIRD core */ -+ HMBIRD_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -+ HMBIRD_OPSS_QUEUED, /* owned by the BPF scheduler */ -+ HMBIRD_OPSS_DISPATCHING, /* in transit back to the HMBIRD core */ -+ -+ /* -+ * QSEQ brands each QUEUED instance so that, when dispatch races -+ * dequeue/requeue, the dispatcher can tell whether it still has a claim -+ * on the task being dispatched. -+ */ -+ HMBIRD_OPSS_QSEQ_SHIFT = 2, -+ HMBIRD_OPSS_STATE_MASK = (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1, -+ HMBIRD_OPSS_QSEQ_MASK = ~HMBIRD_OPSS_STATE_MASK, -+}; -+ -+enum switch_stat { -+ HMBIRD_DISABLED, -+ HMBIRD_SWITCH_PREP, -+ HMBIRD_RQ_SWITCH_BEGIN, -+ HMBIRD_RQ_SWITCH_DONE, -+ HMBIRD_ENABLED, -+}; -+enum switch_stat curr_ss; -+ -+/* -+ * During exit, a task may schedule after losing its PIDs. When disabling the -+ * BPF scheduler, we need to be able to iterate tasks in every state to -+ * guarantee system safety. Maintain a dedicated task list which contains every -+ * task between its fork and eventual free. -+ */ -+DEFINE_SPINLOCK(hmbird_tasks_lock); -+static LIST_HEAD(hmbird_tasks); -+ -+/* ops enable/disable */ -+static struct kthread_worker *hmbird_ops_helper; -+static DEFINE_MUTEX(hmbird_ops_enable_mutex); -+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem); -+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED); -+ -+static bool hmbird_warned_zero_slice; -+enum hmbird_switch_type sw_type; -+static int skip_num[MAX_GLOBAL_DSQS]; -+static int big_distribute_mask_prev; -+static int little_distribute_mask_prev; -+ -+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0); -+ -+/* -+ * The maximum amount of time in jiffies that a task may be runnable without -+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger -+ * hmbird_ops_error(). -+ */ -+unsigned long hmbird_watchdog_timeout; -+ -+/* -+ * The last time the delayed work was run. This delayed work relies on -+ * ksoftirqd being able to run to service timer interrupts, so it's possible -+ * that this work itself could get wedged. To account for this, we check that -+ * it's not stalled in the timer tick, and trigger an error if it is. -+ */ -+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES; -+ -+static struct delayed_work hmbird_watchdog_work; -+static struct work_struct hmbird_err_exit_work; -+ -+/* idle tracking */ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_CPUMASK_OFFSTACK -+#define CL_ALIGNED_IF_ONSTACK -+#else -+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp -+#endif -+ -+static struct { -+ cpumask_var_t cpu; -+ cpumask_var_t smt; -+} idle_masks CL_ALIGNED_IF_ONSTACK; -+ -+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus; -+#endif /* CONFIG_SMP */ -+ -+/* dispatch queues */ -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global; -+ -+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128}; -+u32 pcp_dsq_deadline = 20; -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS]; -+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq); -+ -+static u64 max_hmbird_dsq_internal_id; -+ -+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/ -+#define CLUSTER_SEPARATE_IDX (8) -+#define GDSQS_ID_BASE (3) -+#define UX_COMPATIBLE_IDX (4) -+#define NON_PERIOD_START (5) -+#define NON_PERIOD_END (MAX_GLOBAL_DSQS) -+#define CREATE_DSQ_LEVEL_WITHIN (1) -+ -+struct hmbird_sched_info { -+ spinlock_t lock; -+ int curr_idx[2]; -+ int rtime[MAX_GLOBAL_DSQS]; -+}; -+ -+struct pcp_sched_info { -+ s64 pcp_seq; -+ int rtime; -+ bool pcp_round; -+}; -+ -+/* -+ * pcp_info may rw by another cpu. -+ * protected by rq->lock. -+ */ -+atomic64_t pcp_dsq_round; -+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info); -+ -+struct md_info_t *md_info; -+ -+static int b_rescue_l, l_rescue_b; -+static struct hmbird_sched_info sinfo; -+ -+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC; -+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = { -+ 0, 0, 0, 0, 0, -+ 32 * NSEC_PER_MSEC, -+ 20 * NSEC_PER_MSEC, -+ 14 * NSEC_PER_MSEC, -+ 8 * NSEC_PER_MSEC, -+ 6 * NSEC_PER_MSEC -+}; -+ -+struct cluster_ctx { -+ /* cpu-dsq map must within [lower, upper) */ -+ int upper; -+ int lower; -+ int tidx; -+}; -+ -+enum stat_items { -+ GLOBAL_STAT, -+ CPU_ALLOW_FAIL, -+ RT_CNT, -+ KEY_TASK_CNT, -+ SWITCH_IDX, -+ TIMEOUT_CNT, -+ -+ TOTAL_DSP_CNT, -+ MOVE_RQ_CNT, -+ SELECT_CPU, -+ -+ DWORD_STAT_END = SELECT_CPU, -+ -+ GDSQ_CNT, -+ ERR_IDX, -+ PCP_TIMEOUT_CNT, -+ PCP_LDSQ_CNT, -+ PCP_ENQL_CNT, -+ -+ MAX_ITEMS, -+}; -+static DEFINE_SPINLOCK(stats_lock); -+static char *stats_str[MAX_ITEMS] = { -+ "global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt", -+ "switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt", -+ "select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt", -+ "pcp_ldsq_cnt", "pcp_enql_cnt" -+}; -+ -+ -+struct stats_struct { -+ u64 global_stat[2]; -+ u64 cpu_allow_fail[2]; -+ u64 rt_cnt[2]; -+ u64 key_task_cnt[2]; -+ u64 switch_idx[2]; -+ u64 timeout_cnt[2]; -+ -+ u64 total_dsp_cnt[2]; -+ u64 move_rq_cnt[2]; -+ u64 select_cpu[2]; -+ -+ u64 gdsq_count[MAX_GLOBAL_DSQS][2]; -+ u64 err_idx[5]; -+ u64 pcp_timeout_cnt[NR_CPUS]; -+ u64 pcp_ldsq_count[NR_CPUS][2]; -+ u64 pcp_enql_cnt[NR_CPUS]; -+} stats_data; -+ -+static struct { -+ cpumask_var_t ex_free; -+ cpumask_var_t exclusive; -+ cpumask_var_t partial; -+ cpumask_var_t big; -+ cpumask_var_t little; -+} iso_masks __read_mostly; -+ -+/* -+ * Need more synchronization for these two variables? -+ * I choose not to. -+ */ -+static int l_need_rescue, b_need_rescue; -+ -+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...) \ -+{ \ -+ char buf[MAX_FATAL_INFO]; \ -+ \ -+ scnprintf(buf, MAX_FATAL_INFO, fmt, ##args); \ -+ hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf); \ -+ trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable), \ -+ READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf); \ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); \ -+} \ -+ -+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2) -+{ -+ int c1, c2; -+ struct cpumask mask = {.bits = {0}}; -+ struct cpumask tmp = {.bits = {0}}; -+ -+ if (!slim_stats) -+ return false; -+ -+ if (!rq1 || !rq2) -+ return false; -+ -+ c1 = cpu_of(rq1); -+ c2 = cpu_of(rq2); -+ cpumask_set_cpu(c1, &mask); -+ cpumask_set_cpu(c2, &mask); -+ -+ if (cpumask_and(&tmp, iso_masks.little, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.big, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.partial, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ return false; -+} -+ -+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu) -+{ -+ unsigned long flags; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ if (!slim_stats) -+ return; -+ -+ switch (item) { -+ case GLOBAL_STAT: -+ fallthrough; -+ case CPU_ALLOW_FAIL: -+ fallthrough; -+ case RT_CNT: -+ fallthrough; -+ case KEY_TASK_CNT: -+ fallthrough; -+ case SWITCH_IDX: -+ fallthrough; -+ case TIMEOUT_CNT: -+ fallthrough; -+ case TOTAL_DSP_CNT: -+ fallthrough; -+ case MOVE_RQ_CNT: -+ fallthrough; -+ case SELECT_CPU: -+ pval = pbase + item * 2 + idx; -+ break; -+ case GDSQ_CNT: -+ pval = &stats_data.gdsq_count[dsq_id][idx]; -+ break; -+ case ERR_IDX: -+ pval = &stats_data.err_idx[idx]; -+ break; -+ case PCP_TIMEOUT_CNT: -+ pval = &stats_data.pcp_timeout_cnt[cpu]; -+ break; -+ case PCP_LDSQ_CNT: -+ pval = &stats_data.pcp_ldsq_count[cpu][idx]; -+ break; -+ case PCP_ENQL_CNT: -+ pval = &stats_data.pcp_enql_cnt[cpu]; -+ break; -+ default: -+ return; -+ } -+ -+ spin_lock_irqsave(&stats_lock, flags); -+ *pval += 1; -+ spin_unlock_irqrestore(&stats_lock, flags); -+} -+ -+static inline bool handle_ret(int ret, int *idx, int len) -+{ -+ if (ret < 0 || ret >= len - *idx) -+ return true; -+ *idx += ret; -+ return false; -+} -+ -+#define PRINT_INTV (5 * HZ) -+void stats_print(char *buf, int len) -+{ -+ int idx = 0, i, j, ret; -+ int item = 0; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ for (item = 0; item < MAX_ITEMS; item++) { -+ if (item <= DWORD_STAT_END) { -+ pval = pbase + item * 2; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n", -+ stats_str[item], pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == GDSQ_CNT) { -+ for (j = 0; j < MAX_GLOBAL_DSQS; j++) { -+ pval = (u64 *)&stats_data.gdsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == ERR_IDX) { -+ pval = (u64 *)&stats_data.err_idx; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n", -+ stats_str[item], pval[0], -+ pval[1], pval[2], pval[3], pval[4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == PCP_TIMEOUT_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_timeout_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_LDSQ_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_ldsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_ENQL_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_enql_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } -+ } -+ -+ if (!md_info) { -+ buf[idx] = '\0'; -+ return; -+ } -+ -+ ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_SWITCHS; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "sw_rec[%d] = %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.sw_rec[i].switch_at, -+ md_info->kern_dump.sw_rec[i].is_success, -+ md_info->kern_dump.sw_rec[i].end_state, -+ md_info->kern_dump.sw_rec[i].switch_reason); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_EXCEP_ID; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "excep[%d] = %llu %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.excep_rec[i][0], -+ md_info->kern_dump.excep_rec[i][1], -+ md_info->kern_dump.excep_rec[i][2], -+ md_info->kern_dump.excep_rec[i][3], -+ md_info->kern_dump.excep_rec[i][4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n", -+ i, md_info->kern_dump.excep_idx[i]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ -+ buf[idx] = '\0'; -+} -+ -+enum cpu_type { -+ LITTLE, -+ BIG, -+ PARTIAL, -+ EXCLUSIVE, -+ EX_FREE, -+ INVALID -+}; -+ -+enum dsq_type { -+ GLOBAL_DSQ, -+ PCP_DSQ, -+ OTHER, -+ MAX_DSQ_TYPE, -+}; -+ -+static enum cpu_type cpu_cluster(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (get_hmbird_rq(rq)->exclusive) { -+ return EXCLUSIVE; -+ } else { -+ if (cpumask_test_cpu(cpu, iso_masks.little)) -+ return LITTLE; -+ else if (cpumask_test_cpu(cpu, iso_masks.big)) -+ return BIG; -+ else if (cpumask_test_cpu(cpu, iso_masks.partial)) -+ return PARTIAL; -+ else if (cpumask_test_cpu(cpu, iso_masks.exclusive)) -+ return EXCLUSIVE; -+ else if (cpumask_test_cpu(cpu, iso_masks.ex_free)) -+ return EX_FREE; -+ } -+ return INVALID; -+} -+ -+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq) -+{ -+ if (!dsq) -+ return OTHER; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= GDSQS_ID_BASE) && -+ ((dsq->id & 0xff) < MAX_GLOBAL_DSQS)) -+ return GLOBAL_DSQ; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) && -+ ((dsq->id & 0xff) < max_hmbird_dsq_internal_id)) -+ return PCP_DSQ; -+ -+ return OTHER; -+} -+ -+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq) -+{ -+ enum dsq_type type; -+ -+ type = get_dsq_type(dsq); -+ switch (type) { -+ case GLOBAL_DSQ: -+ case PCP_DSQ: -+ return (dsq->id & 0xff) - GDSQS_ID_BASE; -+ default: -+ return -1; -+ } -+ return -1; -+} -+ -+static void update_cpus_idle(bool set, struct cpumask *mask) -+{ -+ int cpu; -+ struct rq *rq; -+ -+ if (set) { -+ for_each_cpu(cpu, mask) { -+ rq = cpu_rq(cpu); -+ if (is_idle_task(rq->curr)) -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ } -+ } else -+ cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask); -+} -+ -+static void set_partial_status(bool enable, bool little, bool big) -+{ -+ WRITE_ONCE(partial_enable, enable); -+ WRITE_ONCE(l_need_rescue, little); -+ WRITE_ONCE(b_need_rescue, big); -+} -+ -+static bool is_little_need_rescue(void) -+{ -+ return READ_ONCE(l_need_rescue); -+} -+ -+static bool is_big_need_rescue(void) -+{ -+ return READ_ONCE(b_need_rescue); -+} -+ -+static bool is_partial_enabled(void) -+{ -+ return READ_ONCE(partial_enable); -+} -+ -+static bool is_partial_cpu(int cpu) -+{ -+ return cpumask_test_cpu(cpu, iso_masks.partial); -+} -+ -+static void set_iso_par_free(bool enable) -+{ -+ WRITE_ONCE(isolate_ctrl, enable); -+} -+ -+static bool is_iso_par_free(void) -+{ -+ return READ_ONCE(isolate_ctrl); -+} -+ -+static bool skip_update_idle(void) -+{ -+ int cpu = smp_processor_id(); -+ enum cpu_type type = cpu_cluster(cpu); -+ -+ if ((type == EXCLUSIVE && !is_iso_par_free()) || -+ /* partial enable may changed during idle, it doesn't matter. */ -+ (!is_partial_enabled() && type == PARTIAL)) -+ return true; -+ -+ return false; -+} -+ -+static void init_isolate_cpus(void) -+{ -+ WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -+ cpumask_set_cpu(0, iso_masks.big); -+ cpumask_set_cpu(1, iso_masks.big); -+ cpumask_set_cpu(2, iso_masks.big); -+ cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(4, iso_masks.big); -+ cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(6, iso_masks.partial); -+ cpumask_set_cpu(7, iso_masks.ex_free); -+} -+ -+extern spinlock_t css_set_lock; -+ -+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp) -+{ -+ int i; -+ struct cgroup *anc; -+ -+ spin_lock_irq(&css_set_lock); -+ for (i = 0; i < cgrp->level; i++) { -+ anc = cgrp->ancestors[i]; -+ if (anc->level != CREATE_DSQ_LEVEL_WITHIN) -+ continue; -+ cgroup_get(anc); -+ spin_unlock_irq(&css_set_lock); -+ return anc; -+ } -+ spin_unlock_irq(&css_set_lock); -+ hmbird_err(NO_CGROUP_L1, ":error cgroup = %s\n", cgrp->kn->name); -+ return NULL; -+} -+ -+#define PCP_IDX_BIT (1 << 31) -+ -+static bool is_pcp_rt(struct task_struct *p) -+{ -+ return rt_prio(p->prio) && (p->nr_cpus_allowed == 1); -+} -+ -+static bool is_pcp_idx(int idx) -+{ -+ return idx >= MAX_GLOBAL_DSQS; -+} -+ -+static bool is_critical_system_task(struct task_struct *p) -+{ -+ int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK; -+ -+ return (p->prio < (MAX_RT_PRIO >> 1) && -+ (sp_dl < SCHED_PROP_DEADLINE_LEVEL3)); -+} -+ -+#define ISOLATE_TASK_TOP_BIT (1 << 17) -+#define PIPELINE_TASK_TOP_BIT (1 << 9) -+static bool is_pipeline_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT); -+} -+ -+static bool is_isolate_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT); -+} -+ -+static bool is_critical_app_task_without_isolate(struct task_struct *p) -+{ -+ return task_is_top_task(p) && !is_isolate_task(p); -+} -+ -+static int find_idx_from_task(struct task_struct *p) -+{ -+ int idx, cpu; -+ int sp_dl; -+ struct task_group *tg = p->sched_task_group; -+ -+ if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) { -+ cpu = cpumask_any(p->cpus_ptr); -+ idx = cpu + MAX_GLOBAL_DSQS; -+ return idx; -+ } -+ -+ if (is_critical_system_task(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL0; -+ goto done; -+ } -+ -+ if (is_critical_app_task_without_isolate(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL1; -+ goto done; -+ } -+ -+ if (p->pid == scx_systemui_pid) { -+ idx = SCHED_PROP_DEADLINE_LEVEL4; -+ goto done; -+ } -+ -+ sp_dl = hmbird_get_dsq_id(p); -+ if (sp_dl) { -+ idx = sp_dl; -+ goto done; -+ } -+ -+ if (rt_prio(p->prio)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL3; -+ goto done; -+ } -+ -+ if (tg && tg->css.cgroup && tg->css.cgroup->kn) { -+ if (likely(tg->css.cgroup->kn->id >= 0 && -+ tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS)) -+ idx = cgroup_ids_table[tg->css.cgroup->kn->id]; -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ -+done: -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) { -+ hmbird_err(DSQ_ID_ERR, " : idx error, idx = %d-----\n", idx); -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } -+ return idx; -+} -+ -+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p) -+{ -+ int idx; -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq; -+ -+ if (!p) -+ return NULL; -+ -+ idx = find_idx_from_task(p); -+ if (is_pcp_idx(idx)) { -+ idx -= MAX_GLOBAL_DSQS; -+ dsq = &per_cpu(pcp_ldsq, idx); -+ get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq); -+ slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx); -+ } else { -+ dsq = &gdsqs[idx]; -+ get_hmbird_ts(p)->gdsq_idx = idx; -+ slim_stats_record(GDSQ_CNT, 0, idx, 0); -+ } -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ dsq->last_consume_at = jiffies; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ return dsq; -+} -+ -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq); -+ -+static void set_partial_rescue(bool p_state, bool l_over, bool b_over) -+{ -+ set_partial_status(p_state, l_over, b_over); -+ update_cpus_idle(p_state, iso_masks.partial); -+ hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled()); -+ hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue()); -+ hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue()); -+} -+ -+static void free_isocpu(bool enable) -+{ -+ set_iso_par_free(enable); -+ update_cpus_idle(enable, iso_masks.ex_free); -+ hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free()); -+} -+ -+inline u64 get_hmbird_cpu_util(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed) -+ return 0; -+ u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed); -+ u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size); -+ -+ do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT); -+ -+ return prev_runnable_sum_fixed; -+} -+ -+static inline unsigned int get_scaling_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->max; -+} -+ -+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static u64 get_cpus_max_util(struct cpumask *mask) -+{ -+ int cpu; -+ u64 max = 0; -+ u64 util, ratio; -+ unsigned long effective_cap = 0; -+ -+ for_each_cpu(cpu, mask) { -+ if (slim_walt_ctrl) -+ slim_get_cpu_util(cpu, &util); -+ else -+ util = get_hmbird_cpu_util(cpu); -+ -+ /* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/ -+ if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu))) -+ effective_cap = arch_scale_cpu_capacity(cpu); -+ else -+ effective_cap = arch_scale_cpu_capacity(cpu) * -+ get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu); -+ -+ ratio = util * 100 / effective_cap; -+ hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util); -+ hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n", -+ cpu, (u64)arch_scale_cpu_capacity(cpu)); -+ hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n", -+ cpu, (u64)effective_cap); -+ -+ if (ratio > 100) -+ ratio = 100; -+ -+ if (ratio > max) -+ max = ratio; -+ } -+ return max; -+} -+ -+static bool cluster_need_rescue(struct cpumask *mask, int hres) -+{ -+ return get_cpus_max_util(mask) > hres; -+} -+ -+void partial_dynamic_ctrl(void) -+{ -+ u64 lmax = 0, bmax = 0; -+ bool l_over, l_under, b_over, b_under; -+ static bool last_l_over, last_b_over; -+ static unsigned long last_check; -+ -+ /* Check partial every jiffies. need lock? */ -+ if (time_before_eq(jiffies, READ_ONCE(last_check))) -+ return; -+ -+ WRITE_ONCE(last_check, jiffies); -+ -+ lmax = get_cpus_max_util(iso_masks.little); -+ l_over = lmax > parctrl_high_ratio_l; -+ l_under = lmax < parctrl_low_ratio_l; -+ -+ bmax = get_cpus_max_util(iso_masks.big); -+ b_over = bmax > parctrl_high_ratio; -+ b_under = bmax < parctrl_low_ratio; -+ -+ if (is_partial_enabled() && (l_over || b_over)) { -+ if (last_l_over != l_over || last_b_over != b_over) -+ set_partial_rescue(true, l_over, b_over); -+ } else if (!is_partial_enabled() && (l_over || b_over)) { -+ set_partial_rescue(true, l_over, b_over); -+ } else if (is_partial_enabled() && l_under && b_under) { -+ set_partial_rescue(false, false, false); -+ } -+ last_l_over = l_over; -+ last_b_over = b_over; -+ -+ if (is_partial_enabled()) { -+ if (cpumask_empty(iso_masks.partial)) { -+ bmax = bmax > lmax ? bmax : lmax; -+ } else { -+ bmax = get_cpus_max_util(iso_masks.partial); -+ } -+ if (!is_iso_par_free() && bmax > isoctrl_high_ratio) { -+ hmbird_info_trace("partial max = %llu\n", bmax); -+ free_isocpu(true); -+ } else if (is_iso_par_free() && bmax < isoctrl_low_ratio) -+ free_isocpu(false); -+ } else if (is_iso_par_free()) { -+ free_isocpu(false); -+ } -+} -+ -+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx) -+{ -+ hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx); -+} -+ -+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx) -+{ -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[idx])) { -+ slim_stats_record(GDSQ_CNT, 1, idx, 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ int i; -+ -+ for (i = 0; i < UX_COMPATIBLE_IDX; i++) { -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ slim_stats_record(GDSQ_CNT, 1, i, 0); -+ return 1; -+ } -+ } -+ return 0; -+} -+ -+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) { -+ slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ goto clear_timeout; -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) -+ goto clear_timeout; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ hmbird_info_trace( -+ "dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n", -+ dsq_id_to_internal(dsq), entity->task->comm, -+ jiffies, msecs_to_jiffies(deadline), entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1); -+ return; -+ -+clear_timeout: -+ hmbird_info_trace("dsq[%d] clear timeout\n", -+ dsq_id_to_internal(dsq)); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0); -+ dsq->is_timeout = false; -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq)); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime) -+{ -+ hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime); -+} -+ -+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any) -+{ -+ bool is_timeout; -+ int cpu = cpu_of(rq); -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu); -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ is_timeout = dsq->is_timeout; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ /* -+ * dsq->is_timeout may change here, let it be. -+ * it won't cause serious problems. -+ * the same for consume_dispatch_q later. -+ */ -+ if (!is_timeout && !any) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, dsq)) { -+ if (is_timeout) { -+ hmbird_info_trace("dsq[%d] consume a pcp timeout task\n", -+ dsq_id_to_internal(dsq)); -+ update_timeout_stats(rq, dsq, pcp_dsq_deadline); -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu); -+ } -+ slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu); -+ return 1; -+ } -+ /* -+ * No pcp task, clear quota. -+ */ -+ if (any) { -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+ } -+ return 0; -+} -+ -+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf) -+{ -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ } -+ return 0; -+} -+ -+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf, -+ int tmp, int cidx, int tidx) -+{ -+ unsigned long flags; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) { -+ if (tmp != cidx) { -+ spin_lock(&sinfo.lock); -+ sinfo.curr_idx[tidx] = tmp; -+ spin_unlock(&sinfo.lock); -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]); -+ slim_stats_record(SWITCH_IDX, 0, 0, 0); -+ } -+ slim_stats_record(GDSQ_CNT, 1, tmp, 0); -+ -+ raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags); -+ gdsqs[tmp].last_consume_at = jiffies; -+ raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags); -+ return 1; -+ } -+ return 0; -+ -+} -+ -+static int get_cidx(struct cluster_ctx *ctx) -+{ -+ int cidx; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ if (cidx < ctx->lower || cidx >= ctx->upper) { -+ sinfo.curr_idx[ctx->tidx] = ctx->lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]); -+ slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ } -+ spin_unlock(&sinfo.lock); -+ -+ return cidx; -+} -+ -+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = CLUSTER_SEPARATE_IDX; -+ if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) { -+ pr_debug(" %s iso_masks.little first is %d\n", -+ __func__, cpumask_first(iso_masks.little)); -+ ctx->upper = NON_PERIOD_END; -+ } -+ ctx->tidx = 0; -+ break; -+ case LITTLE: -+ ctx->lower = CLUSTER_SEPARATE_IDX; -+ ctx->upper = NON_PERIOD_END; -+ if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) { -+ pr_debug(" %s iso_masks.big first is %d", -+ __func__, cpumask_first(iso_masks.big)); -+ ctx->lower = NON_PERIOD_START; -+ } -+ ctx->tidx = 1; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ case LITTLE: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = NON_PERIOD_END; -+ ctx->tidx = 0; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ if (cluster_separate) { -+ return gen_cluster_ctx_separate(ctx, type); -+ } else { -+ return gen_cluster_ctx_common(ctx, type); -+ } -+} -+ -+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ int i; -+ bool is_timeout; -+ unsigned long flags; -+ struct cluster_ctx ctx; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ /* Third param means only consume timeout pcp dsq. */ -+ if (consume_pcp_dsq(rq, rf, false)) -+ return 1; -+ -+ for (i = ctx.lower; i < ctx.upper; i++) { -+ raw_spin_lock_irqsave(&gdsqs[i].lock, flags); -+ is_timeout = gdsqs[i].is_timeout; -+ raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags); -+ /* gdsqs[i].is_timeout may change here, let it be... */ -+ if (is_timeout) { -+ /* -+ * consume_dispatch_q will acquire dsq-lock, -+ * So cannot keep lock here, annoy enough. -+ * may rewrite a consume_dispatch_q_locked, TODO. -+ */ -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ hmbird_info_trace("dsq[%d] consume a timeout task\n", i); -+ slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0); -+ update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]); -+ return 1; -+ } -+ } -+ } -+ return 0; -+} -+ -+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ struct cluster_ctx ctx; -+ int cidx; -+ int tmp; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ cidx = get_cidx(&ctx); -+ tmp = cidx; -+ do { -+ if (check_pcp_dsq_round(rq, rf)) -+ return 1; -+ if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx)) -+ return 1; -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[tmp] = 0; -+ systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]); -+ tmp++; -+ if (tmp >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ tmp = ctx.lower; -+ } -+ spin_unlock(&sinfo.lock); -+ } while (tmp != cidx); -+ -+ return consume_pcp_dsq(rq, rf, true); -+} -+ -+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ enum cpu_type type = cpu_cluster(cpu_of(rq)); -+ bool period_allowed = !get_hmbird_rq(rq)->period_disallow; -+ bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow; -+ -+ switch (type) { -+ case EX_FREE: -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ break; -+ case EXCLUSIVE: -+ if (!is_iso_par_free()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ } -+ /* Only non-period task can run on isolate cpus */ -+ if (!READ_ONCE(iso_free_rescue)) { -+ if (is_big_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ } else { -+ /* Free run, can run on any task. */ -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ case PARTIAL: -+ if (!is_partial_enabled()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ return 0; -+ } -+ if (is_big_need_rescue()) { -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ -+ case BIG: -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ if (is_iso_par_free() || (is_little_need_rescue() && -+ !cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) { -+ hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++); -+ return 1; -+ } -+ } -+ break; -+ -+ case LITTLE: -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ -+ if (is_iso_par_free() || (is_big_need_rescue() && -+ !cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) { -+ hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++); -+ return 1; -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return 0; -+} -+ -+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf) -+{ -+ return consume_hmbird_global_dsq(rq, rf); -+} -+ -+ -+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time) -+{ -+ int idx; -+ -+ /* which dsq belongs to while task enqueue, task will consume its running time. */ -+ idx = get_hmbird_ts(p)->gdsq_idx; -+ /* Only non-period dsq share running time between each other. */ -+ if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id) -+ return; -+ -+ if (idx >= MAX_GLOBAL_DSQS) { -+ per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } else { -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[idx] += exec_time; -+ spin_unlock(&sinfo.lock); -+ systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]); -+ } -+} -+ -+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type) -+{ -+ int cidx; -+ struct cluster_ctx ctx; -+ int cpu = cpu_of(rq); -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ if (cidx < ctx.lower || cidx >= ctx.upper) { -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ slim_stats_record(ERR_IDX, ctx.tidx, 0, 0); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ } -+ -+ while (1) { -+ if (per_cpu(pcp_info, cpu).pcp_round) { -+ if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) { -+ hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n", -+ cpu, per_cpu(pcp_info, cpu).rtime); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } -+ } -+ if (sinfo.rtime[cidx] < dsq_quota[cidx]) -+ break; -+ -+ /* clear current dsq rtime */ -+ sinfo.rtime[cidx] = 0; -+ systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]); -+ -+ sinfo.curr_idx[ctx.tidx]++; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", -+ ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ } -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ slim_stats_record(SWITCH_IDX, 1, 0, 0); -+ } -+ spin_unlock(&sinfo.lock); -+} -+ -+ -+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p) -+{ -+ enum cpu_type type; -+ -+ if (!rq || !p) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ switch (type) { -+ case PARTIAL: -+ return; -+ case EXCLUSIVE: -+ return; -+ case EX_FREE: -+ return; -+ default: -+ break; -+ } -+ update_dsq_idx(rq, p, type); -+} -+ -+ -+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ int dsq_id; -+ -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo) || dsq->is_timeout) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ hmbird_deferred_err(SCAN_ENTITY_NULL, -+ " : entity is NULL, dsq->id = %llu\n", dsq->id); -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ dsq->is_timeout = true; -+ dsq_id = dsq_id_to_internal(dsq); -+ hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n", -+ dsq_id, entity->task->comm, jiffies, entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1); -+ raw_spin_unlock(&dsq->lock); -+ -+ return true; -+} -+ -+void scan_timeout(struct rq *rq) -+{ -+ int i; -+ int cpu = cpu_of(rq); -+ struct hmbird_dispatch_q *dsq; -+ static u64 last_scan_at; -+ static DEFINE_PER_CPU(u64, pcp_last_scan_at); -+ -+ if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu))) -+ return; -+ per_cpu(pcp_last_scan_at, cpu) = jiffies; -+ -+ dsq = &per_cpu(pcp_ldsq, cpu); -+ scan_dsq_timeout(rq, dsq, pcp_dsq_deadline); -+ -+ if (time_before_eq(jiffies, (unsigned long)last_scan_at)) -+ return; -+ last_scan_at = jiffies; -+ -+ for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) { -+ dsq = &gdsqs[i]; -+ scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]); -+ } -+} -+ -+/*******************************Initialize***********************************/ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id); -+static void init_dsq_at_boot(void) -+{ -+ int i, cpu; -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i)); -+ } -+ for_each_possible_cpu(cpu) -+ init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i + cpu)); -+ -+ max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu; -+ spin_lock_init(&sinfo.lock); -+} -+ -+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgrp_name_to_idx(struct cgroup *cgrp) -+{ -+ int idx; -+ -+ if (!cgrp) -+ return -1; -+ -+ if (!strcmp(cgrp->kn->name, "display") -+ || !strcmp(cgrp->kn->name, "multimedia")) -+ idx = 5; /* 8ms */ -+ else if (!strcmp(cgrp->kn->name, "top-app") -+ || !strcmp(cgrp->kn->name, "ss-top")) -+ idx = 6; /* 16ms */ -+ else if (!strcmp(cgrp->kn->name, "ssfg") -+ || !strcmp(cgrp->kn->name, "foreground")) -+ idx = 7; /* 32ms */ -+ else if (!strcmp(cgrp->kn->name, "bg") -+ || !strcmp(cgrp->kn->name, "log") -+ || !strcmp(cgrp->kn->name, "dex2oat") -+ || !strcmp(cgrp->kn->name, "background")) -+ idx = 9; /* 128ms */ -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */ -+ -+ return idx; -+} -+ -+static void init_root_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX); -+} -+ -+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) { -+ pr_err("%s idx err!\n", __func__); -+ return; -+ } -+ -+ if (cgroup_ids_table[cgrp->kn->id] == -1) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp)); -+} -+ -+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ struct cgroup *l1cgrp; -+ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ l1cgrp = cgroup_ancestor_l1(cgrp); -+ if (l1cgrp) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp)); -+ cgroup_put(l1cgrp); -+} -+ -+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg) -+{ -+ switch (cgrp->level) { -+ case 0: -+ init_root_tg(cgrp, tg); -+ break; -+ case 1: -+ init_level1_tg(cgrp, tg); -+ break; -+ default: -+ init_child_tg(cgrp, tg); -+ break; -+ } -+} -+ -+/**************************************************************************/ -+ -+struct hmbird_task_iter { -+ struct hmbird_entity cursor; -+ struct task_struct *locked; -+ struct rq *rq; -+ struct rq_flags rf; -+}; -+ -+/** -+ * hmbird_task_iter_init - Initialize a task iterator -+ * @iter: iterator to init -+ * -+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized, -+ * @iter must eventually be exited with hmbird_task_iter_exit(). -+ * -+ * hmbird_tasks_lock may be released between this and the first next() call or -+ * between any two next() calls. If hmbird_tasks_lock is released between two -+ * next() calls, the caller is responsible for ensuring that the task being -+ * iterated remains accessible either through RCU read lock or obtaining a -+ * reference count. -+ * -+ * All tasks which existed when the iteration started are guaranteed to be -+ * visited as long as they still exist. -+ */ -+static void hmbird_task_iter_init(struct hmbird_task_iter *iter) -+{ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR }; -+ list_add(&iter->cursor.tasks_node, &hmbird_tasks); -+ iter->locked = NULL; -+} -+ -+/** -+ * hmbird_task_iter_exit - Exit a task iterator -+ * @iter: iterator to exit -+ * -+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held. -+ * If the iterator holds a task's rq lock, that rq lock is released. See -+ * hmbird_task_iter_init() for details. -+ */ -+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ if (list_empty(cursor)) -+ return; -+ -+ list_del_init(cursor); -+} -+ -+/** -+ * hmbird_task_iter_next - Next task -+ * @iter: iterator to walk -+ * -+ * Visit the next task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ struct hmbird_entity *pos; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ list_for_each_entry(pos, cursor, tasks_node) { -+ if (&pos->tasks_node == &hmbird_tasks) -+ return NULL; -+ if (!(pos->flags & HMBIRD_TASK_CURSOR)) { -+ list_move(cursor, &pos->tasks_node); -+ return pos->task; -+ } -+ } -+ -+ /* can't happen, should always terminate at hmbird_tasks above */ -+ hmbird_deferred_err(ITER_RET_NULL, " : unreachable path in scx_task_iter_next\n"); -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered - Next non-idle task -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ while ((p = hmbird_task_iter_next(iter))) { -+ if (!is_idle_task(p)) -+ return p; -+ } -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init() -+ * for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ p = hmbird_task_iter_next_filtered(iter); -+ if (!p) -+ return NULL; -+ -+ iter->rq = task_rq_lock(p, &iter->rf); -+ iter->locked = p; -+ return p; -+} -+ -+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void) -+{ -+ return atomic_read(&hmbird_ops_enable_state_var); -+} -+ -+static enum hmbird_ops_enable_state -+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to) -+{ -+ return atomic_xchg(&hmbird_ops_enable_state_var, to); -+} -+ -+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to, -+ enum hmbird_ops_enable_state from) -+{ -+ int from_v = from; -+ -+ return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to); -+} -+ -+static bool hmbird_ops_disabling(void) -+{ -+ return false; -+} -+ -+/** -+ * wait_ops_state - Busy-wait the specified ops state to end -+ * @p: target task -+ * @opss: state to wait the end of -+ * -+ * Busy-wait for @p to transition out of @opss. This can only be used when the -+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also -+ * has load_acquire semantics to ensure that the caller can see the updates made -+ * in the enqueueing and dispatching paths. -+ */ -+static void wait_ops_state(struct task_struct *p, u64 opss) -+{ -+ do { -+ cpu_relax(); -+ } while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss); -+} -+ -+ -+static void update_curr_hmbird(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ u64 now = rq_clock_task(rq); -+ u64 delta_exec; -+ -+ if (time_before_eq64(now, curr->se.exec_start)) -+ return; -+ -+ delta_exec = now - curr->se.exec_start; -+ curr->se.exec_start = now; -+ update_runningtime(rq, curr, delta_exec); -+ curr->se.sum_exec_runtime += delta_exec; -+ account_group_exec_runtime(curr, delta_exec); -+ cgroup_account_cputime(curr, delta_exec); -+ -+ if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF) -+ get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec); -+ -+ trace_sched_stat_runtime(curr, delta_exec, 0); -+} -+ -+static bool hmbird_dsq_priq_less(struct rb_node *node_a, -+ const struct rb_node *node_b) -+{ -+ const struct hmbird_entity *a = -+ container_of(node_a, struct hmbird_entity, dsq_node.priq); -+ const struct hmbird_entity *b = -+ container_of(node_b, struct hmbird_entity, dsq_node.priq); -+ -+ return time_before64(a->dsq_vtime, b->dsq_vtime); -+} -+ -+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ bool is_local = dsq->id == HMBIRD_DSQ_LOCAL; -+ unsigned long flags; -+ -+ hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq || -+ !list_empty(&get_hmbird_ts(p)->dsq_node.fifo), -+ "task = %s, dsq->id = %llu\n", p->comm, dsq->id); -+ hmbird_cond_deferred_err(ENQ_EXIST2, -+ (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq), -+ "task = %s\n", p->comm); -+ -+ if (!is_local) { -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ); -+ hmbird_ops_error(": %s\n", -+ "attempting to dispatch to a destroyed dsq"); -+ /* fall back to the global dsq */ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ dsq = &hmbird_dsq_global; -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ } -+ } -+ -+ if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) { -+ get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ; -+ rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq, -+ hmbird_dsq_priq_less); -+ } else { -+ if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT)) -+ list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ else -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ } -+ dsq->nr++; -+ get_hmbird_ts(p)->dsq = dsq; -+ -+ /* -+ * We're transitioning out of QUEUEING or DISPATCHING. store_release to -+ * match waiters' load_acquire. -+ */ -+ if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS) -+ atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE); -+ -+ if (is_local) { -+ struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq); -+ struct rq *rq = hmbird->rq; -+ bool preempt = false; -+ -+ if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr && -+ rq->curr->sched_class == &hmbird_sched_class) { -+ get_hmbird_ts(rq->curr)->slice = 0; -+ preempt = true; -+ } -+ -+ if (preempt || sched_class_above(&hmbird_sched_class, -+ rq->curr->sched_class)) -+ resched_curr(rq); -+ } else { -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ } -+} -+ -+static void task_unlink_from_dsq(struct task_struct *p, -+ struct hmbird_dispatch_q *dsq) -+{ -+ if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) { -+ rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ; -+ } else { -+ list_del_init(&get_hmbird_ts(p)->dsq_node.fifo); -+ } -+} -+ -+static bool task_linked_on_dsq(struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+} -+ -+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq; -+ bool is_local = dsq == &hmbird_rq->local_dsq; -+ -+ if (!dsq) { -+ hmbird_cond_deferred_err(TASK_LINKED1, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ /* -+ * When dispatching directly from the BPF scheduler to a local -+ * DSQ, the task isn't associated with any DSQ but -+ * @get_hmbird_ts(p)->holding_cpu may be set under the protection of -+ * %HMBIRD_OPSS_DISPATCHING. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu >= 0) -+ get_hmbird_ts(p)->holding_cpu = -1; -+ return; -+ } -+ -+ if (!is_local) -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ /* -+ * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node -+ * can't change underneath us. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu < 0) { -+ /* @p must still be on @dsq, dequeue */ -+ hmbird_cond_deferred_err(TASK_UNLINKED, -+ !task_linked_on_dsq(p), "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ } else { -+ /* -+ * We're racing against dispatch_to_local_dsq() which already -+ * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the -+ * holding_cpu which tells dispatch_to_local_dsq() that it lost -+ * the race. -+ */ -+ hmbird_cond_deferred_err(TASK_LINKED2, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ get_hmbird_ts(p)->holding_cpu = -1; -+ } -+ get_hmbird_ts(p)->dsq = NULL; -+ -+ if (!is_local) -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+ -+static bool test_rq_online(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ return rq->online; -+#else -+ return true; -+#endif -+} -+ -+static void refill_task_slice(struct task_struct *p) -+{ -+ if (is_isolate_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO; -+ else if (is_pipeline_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2; -+ else -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+} -+ -+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -+ int sticky_cpu) -+{ -+ struct hmbird_dispatch_q *d; -+ s32 cpu = -1; -+ -+ hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), -+ "task = %s\n", p->comm); -+ -+ if (is_pcp_rt(p)) { -+ /* Enqueue percpu rt task to local directly. */ -+ /* Or cause a bug when disable dispatch. */ -+ if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ } -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (cpu >= 0) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ -+ if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ /* rq migration */ -+ if (sticky_cpu == cpu_of(rq)) -+ goto local_norefill; -+ /* -+ * If !rq->online, we already told the BPF scheduler that the CPU is -+ * offline. We're just trying to on/offline the CPU. Don't bother the -+ * BPF scheduler. -+ */ -+ if (unlikely(!test_rq_online(rq))) -+ goto local; -+ -+ /* see %HMBIRD_OPS_ENQ_LAST */ -+ if (enq_flags & HMBIRD_ENQ_LAST) -+ goto local; -+ -+ if (enq_flags & HMBIRD_ENQ_LOCAL) -+ goto local; -+ else -+ goto global; -+local: -+ /* -+ * For task-ordering, slice refill must be treated as implying the end -+ * of the current slice. Otherwise, the longer @p stays on the CPU, the -+ * higher priority it becomes from hmbird_prio_less()'s POV. -+ */ -+ refill_task_slice(p); -+local_norefill: -+ dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags); -+ slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq)); -+ return; -+ -+global: -+ d = find_dsq_from_task(p); -+ if (d) { -+ refill_task_slice(p); -+ dispatch_enqueue(d, p, enq_flags); -+ return; -+ } -+ slim_stats_record(GLOBAL_STAT, 0, 0, 0); -+ refill_task_slice(p); -+ dispatch_enqueue(&hmbird_dsq_global, p, enq_flags); -+} -+ -+static bool watchdog_task_watched(const struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->watchdog_node); -+} -+ -+static void watchdog_watch_task(struct rq *rq, struct task_struct *p) -+{ -+ lockdep_assert_rq_held(rq); -+ if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ get_hmbird_ts(p)->runnable_at = jiffies; -+ clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list); -+} -+ -+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) -+{ -+ list_del_init(&get_hmbird_ts(p)->watchdog_node); -+ if (reset_timeout) -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags) -+{ -+ int sticky_cpu = get_hmbird_ts(p)->sticky_cpu; -+ -+ enq_flags |= get_hmbird_rq(rq)->extra_enq_flags; -+ -+ if (sticky_cpu >= 0) -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ -+ /* -+ * Restoring a running task will be immediately followed by -+ * set_next_task_hmbird() which expects the task to not be on the BPF -+ * scheduler as tasks can only start running through local DSQs. Force -+ * direct-dispatch into the local DSQ by setting the sticky_cpu. -+ */ -+ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -+ sticky_cpu = cpu_of(rq); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_UNWATCHED, -+ !watchdog_task_watched(p), "task = %s\n", p->comm); -+ return; -+ } -+ -+ watchdog_watch_task(rq, p); -+ set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ get_hmbird_rq(rq)->nr_running++; -+ add_nr_running(rq, 1); -+ -+ do_enqueue_task(rq, p, enq_flags, sticky_cpu); -+} -+ -+static void ops_dequeue(struct task_struct *p, u64 deq_flags) -+{ -+ u64 opss; -+ -+ watchdog_unwatch_task(p, false); -+ -+ /* acquire ensures that we see the preceding updates on QUEUED */ -+ opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state); -+ -+ switch (opss & HMBIRD_OPSS_STATE_MASK) { -+ case HMBIRD_OPSS_NONE: -+ break; -+ case HMBIRD_OPSS_QUEUEING: -+ /* -+ * QUEUEING is started and finished while holding @p's rq lock. -+ * As we're holding the rq lock now, we shouldn't see QUEUEING. -+ */ -+ hmbird_deferred_err(DEQ_DEQING, " : unreachable path in %s\n", __func__); -+ break; -+ case HMBIRD_OPSS_QUEUED: -+ if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss, -+ HMBIRD_OPSS_NONE)) -+ break; -+ fallthrough; -+ case HMBIRD_OPSS_DISPATCHING: -+ /* -+ * If @p is being dispatched from the BPF scheduler to a DSQ, -+ * wait for the transfer to complete so that @p doesn't get -+ * added to its DSQ after dequeueing is complete. -+ * -+ * As we're waiting on DISPATCHING with the rq locked, the -+ * dispatching side shouldn't try to lock the rq while -+ * DISPATCHING is set. See dispatch_to_local_dsq(). -+ * -+ * DISPATCHING shouldn't have qseq set and control can reach -+ * here with NONE @opss from the above QUEUED case block. -+ * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss. -+ */ -+ wait_ops_state(p, HMBIRD_OPSS_DISPATCHING); -+ hmbird_cond_deferred_err(HMBIRD_OPN, -+ atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE, -+ "task = %s\n", p->comm); -+ break; -+ } -+} -+ -+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ -+ if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p), -+ "task = %s\n", p->comm); -+ return; -+ } -+ -+ ops_dequeue(p, deq_flags); -+ -+ if (slim_walt_ctrl) { -+ if (task_current(rq, p)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ if (deq_flags & HMBIRD_DEQ_SLEEP) -+ set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else -+ clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), -+ (unsigned long *)&get_hmbird_ts(p)->flags); -+ -+ clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm); -+ hmbird_rq->nr_running--; -+ sub_nr_running(rq, 1); -+ dispatch_dequeue(hmbird_rq, p); -+} -+ -+static void yield_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ get_hmbird_ts(p)->slice = 0; -+} -+ -+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to) -+{ -+ return false; -+} -+ -+#ifdef CONFIG_SMP -+/** -+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -+ * @rq: rq to move the task into, currently locked -+ * @p: task to move -+ * @enq_flags: %HMBIRD_ENQ_* -+ * -+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller -+ * must: -+ * -+ * 1. Start with exclusive access to @p either through its DSQ lock or -+ * %HMBIRD_OPSS_DISPATCHING flag. -+ * -+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id(). -+ * -+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't -+ * deadlock with dequeue. -+ * -+ * 4. Lock @rq and the task_rq from #3. -+ * -+ * 5. Call this function. -+ * -+ * Returns %true if @p was successfully moved. %false after racing dequeue and -+ * losing. -+ */ -+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ struct rq *task_rq; -+ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * If dequeue got to @p while we were trying to lock both rq's, it'd -+ * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have -+ * updated it to different values afterwards, as this operation can't be -+ * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become -+ * raw_smp_processor_id() again before we're done. Thus, we can tell -+ * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is -+ * still raw_smp_processor_id(). -+ * -+ * See dispatch_dequeue() for the counterpart. -+ */ -+ if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id())) -+ return false; -+ -+ /* @p->rq couldn't have changed if we're still the holding cpu */ -+ task_rq = task_rq(p); -+ lockdep_assert_rq_held(task_rq); -+ deactivate_task(task_rq, p, 0); -+ set_task_cpu(p, cpu_of(rq)); -+ get_hmbird_ts(p)->sticky_cpu = cpu_of(rq); -+ -+ /* -+ * We want to pass hmbird-specific enq_flags but activate_task() will -+ * truncate the upper 32 bit. As we own @rq, we can pass them through -+ * @get_hmbird_rq(rq)->extra_enq_flags instead. -+ */ -+ hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags, -+ "task = %s\n", p->comm); -+ get_hmbird_rq(rq)->extra_enq_flags = enq_flags; -+ activate_task(rq, p, 0); -+ get_hmbird_rq(rq)->extra_enq_flags = 0; -+ -+ return true; -+} -+ -+#endif /* CONFIG_SMP */ -+ -+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu) -+{ -+ int fitable = 1; -+ -+ return fitable; -+} -+ -+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq, -+ struct hmbird_dispatch_q *dsq) -+{ -+ bool dsq_misfit; -+ int cpu = cpu_of(rq); -+ u64 task_util = 0; -+ struct cluster_ctx ctx; -+ int dsq_int = dsq_id_to_internal(dsq); -+ -+ if (!cpumask_test_cpu(cpu, iso_masks.little)) -+ return false; -+ if (p->pid == scx_systemui_pid) -+ return true; -+ -+ gen_cluster_ctx(&ctx, BIG); -+ dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 && -+ dsq_int <= SCHED_PROP_DEADLINE_LEVEL4); -+#ifdef CLUSTER_SEPARATE -+ /* In rescue mode, little will consume big cluster's dsq.*/ -+ dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper); -+#endif -+ if (!dsq_misfit) -+ return false; -+ -+ if (p) { -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &task_util); -+ else -+ task_util = get_hmbird_ts(p)->demand_scaled; -+ } -+ -+ if (task_util <= misfit_ds) -+ return false; -+ -+ hmbird_info_trace(":task %s can't run on cpu%d, util = %llu\n", -+ p->comm, cpu, task_util); -+ return true; -+} -+ -+static int check_misfit_task_on_fake_big(struct task_struct *p, struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (likely(p->pid != scx_systemui_pid)) -+ return false; -+ -+ if (topology_cluster_id(num_possible_cpus() - 1) > 1 && -+ arch_scale_cpu_capacity(cpu) == arch_scale_cpu_capacity(0) && -+ (parctrl_high_ratio <= 0 || parctrl_high_ratio_l <= 0)) -+ return true; -+ -+ return false; -+} -+ -+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq) -+{ -+ if (!cpumask_test_cpu(cpu_of(rq), task_cpu_possible_mask(p))) -+ return false; -+ -+ if (!task_fits_cpu_hmbird(p, cpu_of(rq))) -+ return false; -+ -+ if (check_misfit_task_on_little(p, rq, dsq)) -+ return false; -+ -+ if (check_misfit_task_on_fake_big(p, rq)) -+ return false; -+ -+ return likely(test_rq_online(rq)); -+} -+ -+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return; -+ -+ if (add) -+ skipn[idx]++; -+ else -+ skipn[idx] = 0; -+} -+ -+static bool skip_too_much(struct hmbird_dispatch_q *dsq) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return false; -+ -+ if (skip_num[idx] > 3) { -+ skip_num[idx] = 0; -+ return true; -+ } -+ -+ return false; -+} -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rb_node *rb_node; -+ struct rq *task_rq; -+ unsigned long flags; -+ bool moved = false; -+ struct task_struct *may_fit = NULL; -+ int skip = 0; -+ -+retry: -+ if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -+ return false; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) { -+ set_skip_num(dsq, skip_num, (bool)may_fit); -+ goto this_rq; -+ } -+ if (skip_too_much(dsq)) -+ goto remote_rq; -+ if (!may_fit) -+ may_fit = p; -+ if (++skip <= 3) -+ continue; -+ /* -+ * If the recent 3 tasks not fit, use the first one. -+ * and clear the skip, because the first one is consumed. -+ */ -+ set_skip_num(dsq, skip_num, false); -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ /* No more task, use the first may fit task.*/ -+ if (may_fit) { -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ -+ for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) -+ goto this_rq; -+ goto remote_rq; -+ } -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ return false; -+ -+this_rq: -+ /* @dsq is locked and @p is on this rq */ -+ hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo); -+ dsq->nr--; -+ hmbird_rq->local_dsq.nr++; -+ get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ -+remote_rq: -+#ifdef CONFIG_SMP -+ if (cpu_same_cluster_stat(p, rq, task_rq)) -+ slim_stats_record(MOVE_RQ_CNT, 0, 0, 0); -+ else -+ slim_stats_record(MOVE_RQ_CNT, 1, 0, 0); -+ /* -+ * @dsq is locked and @p is on a remote rq. @p is currently protected by -+ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -+ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -+ * rq lock or fail, do a little dancing from our side. See -+ * move_task_to_local_dsq(). -+ */ -+ hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id(); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ rq_unpin_lock(rq, rf); -+ double_lock_balance(rq, task_rq); -+ rq_repin_lock(rq, rf); -+ -+ moved = move_task_to_local_dsq(rq, p, 0); -+ -+ double_unlock_balance(rq, task_rq); -+#endif /* CONFIG_SMP */ -+ if (likely(moved)) { -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ } -+ may_fit = NULL; -+ goto retry; -+} -+ -+ -+static int balance_one(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf, bool local) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class; -+ -+ if (!hmbird_rq) -+ return 1; -+ -+ lockdep_assert_rq_held(rq); -+ -+ if (static_branch_unlikely(&hmbird_ops_cpu_preempt) && -+ unlikely(get_hmbird_rq(rq)->cpu_released)) { -+ /* -+ * If the previous sched_class for the current CPU was not HMBIRD, -+ * notify the BPF scheduler that it again has control of the -+ * core. This callback complements ->cpu_release(), which is -+ * emitted in hmbird_notify_pick_next_task(). -+ */ -+ get_hmbird_rq(rq)->cpu_released = false; -+ } -+ -+ if (prev_on_hmbird) -+ update_curr_hmbird(rq); -+ /* if there already are tasks to run, nothing to do */ -+ if (hmbird_rq->local_dsq.nr) -+ return 1; -+ -+ if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) { -+ slim_stats_record(GLOBAL_STAT, 1, 0, 0); -+ return 1; -+ } -+ -+ if (consume_dispatch_global(rq, rf)) -+ return 1; -+ -+ return 0; -+} -+ -+static int balance_hmbird(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf) -+{ -+ return balance_one(rq, prev, rf, true); -+} -+ -+/* -+ * output task util to systrace, only for debug mode. -+ * we can not output too many logs to systrace buffer even in debug mode -+ * only output debug-info while it exceed misfit_ds. -+ */ -+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p) -+{ -+ static DEFINE_PER_CPU(int, is_last_exceed); -+ int cpu = cpu_of(rq); -+ u64 util = 0; -+ -+ if (likely(!debug_enabled())) -+ return; -+ -+ if (!p) -+ return; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ util = uclamp_rq_util_with(rq, util, p); -+ -+ if (util >= misfit_ds) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util); -+ per_cpu(is_last_exceed, cpu) = true; -+ } else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0); -+ per_cpu(is_last_exceed, cpu) = false; -+ } else { -+ } -+} -+ -+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first) -+{ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ /* -+ * Core-sched might decide to execute @p before it is -+ * dispatched. Call ops_dequeue() to notify the BPF scheduler. -+ */ -+ ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC); -+ dispatch_dequeue(get_hmbird_rq(rq), p); -+ } -+ -+ p->se.exec_start = rq_clock_task(rq); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK); -+ } -+ -+ watchdog_unwatch_task(p, true); -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx); -+ systrace_output_cpu_ds(rq, p); -+ /* -+ * @p is getting newly scheduled or got kicked after someone updated its -+ * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird(). -+ */ -+ if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) != -+ (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) { -+ if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) -+ get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK; -+ else -+ get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK; -+ -+ sched_update_tick_dependency(rq); -+ } -+ -+ p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime; -+} -+ -+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p) -+{ -+ update_curr_hmbird(rq); -+ -+ update_dispatch_dsq_info(rq, p); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ watchdog_watch_task(rq, p); -+ -+ if (is_pipeline_task(p)) { -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1); -+ return; -+ } -+ -+ /* -+ * If we're in the pick_next_task path, balance_hmbird() should -+ * have already populated the local DSQ if there are any other -+ * available tasks. If empty, tell ops.enqueue() that @p is the -+ * only one available for this cpu. ops.enqueue() should put it -+ * on the local DSQ so that the subsequent pick_next_task_hmbird() -+ * can find the task unless it wants to trigger a separate -+ * follow-up scheduling event. -+ */ -+ if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1); -+ else -+ do_enqueue_task(rq, p, 0, -1); -+ } -+} -+ -+static struct task_struct *first_local_task(struct rq *rq) -+{ -+ struct rb_node *rb_node; -+ struct hmbird_entity *entity; -+ -+ if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) { -+ entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo, -+ struct hmbird_entity, dsq_node.fifo); -+ return entity->task; -+ } -+ -+ rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq); -+ if (rb_node) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ return entity->task; -+ } -+ return NULL; -+} -+ -+static struct task_struct *pick_next_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p; -+ -+ p = first_local_task(rq); -+ if (!p) -+ return NULL; -+ -+ if (unlikely(!get_hmbird_ts(p)->slice)) { -+ if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice) -+ hmbird_warned_zero_slice = true; -+ -+ refill_task_slice(p); -+ } -+ -+ set_next_task_hmbird(rq, p, true); -+ -+ return p; -+} -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task, -+ const struct sched_class *active) -+{ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * The callback is conceptually meant to convey that the CPU is no -+ * longer under the control of HMBIRD. Therefore, don't invoke the -+ * callback if the CPU is staying on HMBIRD, or going idle (in which -+ * case the HMBIRD scheduler has actively decided not to schedule any -+ * tasks on the CPU). -+ */ -+ if (likely(active >= &hmbird_sched_class)) -+ return; -+ -+ /* -+ * At this point we know that HMBIRD was preempted by a higher priority -+ * sched_class, so invoke the ->cpu_release() callback if we have not -+ * done so already. We only send the callback once between HMBIRD being -+ * preempted, and it regaining control of the CPU. -+ * -+ * ->cpu_release() complements ->cpu_acquire(), which is emitted the -+ * next time that balance_hmbird() is invoked. -+ */ -+ if (!get_hmbird_rq(rq)->cpu_released) -+ get_hmbird_rq(rq)->cpu_released = true; -+} -+ -+#ifdef CONFIG_SMP -+ -+static bool test_and_clear_cpu_idle(int cpu) -+{ -+ if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -+ if (cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ return true; -+ } else { -+ return false; -+ } -+} -+ -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) -+{ -+ int cpu; -+ -+ do { -+ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -+ if (cpu >= nr_cpu_ids) -+ return -EBUSY; -+ } while (!test_and_clear_cpu_idle(cpu)); -+ -+ return cpu; -+} -+ -+ -+static bool prev_cpu_misfit(int prev) -+{ -+ if (!is_partial_enabled() && is_partial_cpu(prev)) -+ return true; -+ -+ return false; -+} -+ -+static int heavy_rt_placement(struct task_struct *p, int prev) -+{ -+ struct cpumask tmp = {.bits = {0}}; -+ int cpu; -+ u64 util = 0; -+ -+ if (!rt_prio(p->prio)) -+ return -EFAULT; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ -+ if (util < misfit_ds) -+ return -EFAULT; -+ -+ if (is_partial_enabled()) -+ cpumask_or(&tmp, iso_masks.big, iso_masks.partial); -+ else -+ cpumask_copy(&tmp, iso_masks.big); -+ -+ cpu = hmbird_pick_idle_cpu(&tmp); -+ if (cpu >= 0) -+ return cpu; -+ -+ if (cpumask_test_cpu(prev, iso_masks.big) || -+ (is_partial_enabled() && is_partial_cpu(prev))) -+ return prev; -+ -+ return cpumask_first(&tmp); -+} -+ -+static int spec_task_before_pick_idle(struct task_struct *p, int prev) -+{ -+ int cpu; -+ -+ cpu = heavy_rt_placement(p, prev); -+ if (cpu >= 0) -+ return cpu; -+ return -EFAULT; -+} -+ -+static int cpumask_distribute_next(struct cpumask *mask, int *prev) -+{ -+ int p = *prev, n; -+ -+ n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1); -+ if (n < nr_cpu_ids) -+ WRITE_ONCE(*prev, n); -+ return n; -+} -+ -+static int repick_fallback_cpu(void) -+{ -+ /* -+ * partial cpu follow big cluster's Scheduling policy, -+ * simply return first bit cpu. -+ */ -+ return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev); -+} -+ -+/* -+ * Must return a valid cpu num, as this task's cpu. -+ */ -+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu, -+ struct cpumask *mask, int *prev_mask) -+{ -+ int cpu; -+ -+ cpu = hmbird_pick_idle_cpu(mask); -+ if (cpu >= 0) -+ return cpu; -+ return cpumask_distribute_next(mask, prev_mask); -+} -+ -+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type) -+{ -+ int idx; -+ struct cluster_ctx ctx; -+ -+ idx = find_idx_from_task(p); -+ if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS) -+ return false; -+ -+ gen_cluster_ctx(&ctx, type); -+ if (idx >= ctx.lower && idx < ctx.upper) -+ return true; -+ return false; -+} -+ -+static bool is_valid_cpu(int cpu) -+{ -+ return (cpu >= 0) && (cpu < nr_cpu_ids); -+} -+ -+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) -+{ -+ s32 cpu = -1; -+ struct cpumask mask = {.bits = {0}}; -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (is_valid_cpu(cpu)) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ partial_dynamic_ctrl(); -+ -+ if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ iso_masks.big, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ if (p->nr_cpus_allowed == 1) { -+ cpu = cpumask_any(p->cpus_ptr); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* For non-period global dsq, not contain pcp task. */ -+ if (task_only_blongs_to_cluster(p, LITTLE)) { -+ cpumask_copy(&mask, iso_masks.little); -+ if (unlikely(l_need_rescue)) { -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ cpumask_or(&mask, iso_masks.ex_free, &mask); -+ } -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &little_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ if (task_only_blongs_to_cluster(p, BIG)) { -+ cpumask_copy(&mask, iso_masks.big); -+ if (unlikely(b_need_rescue)) { -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ cpumask_or(&mask, iso_masks.ex_free, &mask); -+ } -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ cpu = spec_task_before_pick_idle(p, prev_cpu); -+ if (is_valid_cpu(cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* if the previous CPU is idle, dispatch directly to it */ -+ if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+ } -+ -+ cpu = hmbird_pick_idle_cpu(cpu_possible_mask); -+ if (is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ if (prev_cpu_misfit(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ cpu = repick_fallback_cpu(); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+} -+ -+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags) -+{ -+ return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags); -+} -+ -+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx) -+{ -+ set_cpus_allowed_common(p, ctx); -+} -+ -+static void reset_idle_masks(void) -+{ -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little); -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big); -+ if (is_partial_enabled()) -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial); -+ hmbird_has_idle_cpus = true; -+} -+ -+void __hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ int cpu = cpu_of(rq); -+ struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -+ -+ if (skip_update_idle()) -+ return; -+ -+ if (idle) { -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ if (!hmbird_has_idle_cpus) -+ hmbird_has_idle_cpus = true; -+ -+ /* -+ * idle_masks.smt handling is racy but that's fine as it's only -+ * for optimization and self-correcting. -+ */ -+ for_each_cpu(cpu, sib_mask) { -+ if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -+ return; -+ } -+ cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -+ } else { -+ cpumask_clear_cpu(cpu, idle_masks.cpu); -+ if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ -+ cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -+ } -+} -+ -+#else /* !CONFIG_SMP */ -+ -+static bool test_and_clear_cpu_idle(int cpu) { return false; } -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } -+static void reset_idle_masks(void) {} -+ -+#endif /* CONFIG_SMP */ -+ -+static bool check_rq_for_timeouts(struct rq *rq) -+{ -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rq_flags rf; -+ -+ rq_lock_irqsave(rq, &rf); -+ list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) { -+ unsigned long last_runnable; -+ -+ p = entity->task; -+ last_runnable = get_hmbird_ts(p)->runnable_at; -+ -+ if (unlikely(time_after(jiffies, -+ last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -+ -+ rq_unlock_irqrestore(rq, &rf); -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb", -+ p->comm, p->pid, -+ dur_ms / 1000, dur_ms % 1000, -+ get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0, -+ cpumask_pr_args(&p->cpus_mask)); -+ return 1; -+ } -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ return 0; -+} -+ -+static void hmbird_watchdog_workfn(struct work_struct *work) -+{ -+ int cpu; -+ bool timeout; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ -+ for_each_online_cpu(cpu) { -+ timeout = check_rq_for_timeouts(cpu_rq(cpu)); -+ if (unlikely(timeout)) -+ break; -+ -+ cond_resched(); -+ } -+ if (!timeout) -+ queue_delayed_work(system_unbound_wq, to_delayed_work(work), -+ hmbird_watchdog_timeout / 2); -+} -+ -+static void set_pcp_round(struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) { -+ per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round); -+ per_cpu(pcp_info, cpu).pcp_round = true; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+} -+ -+ -+/* -+ * Just for debug: output hmbird on/off state per 10s. -+ */ -+#define OUTPUT_INTVAL (msecs_to_jiffies(10 * 1000)) -+static void inform_hmbird_onoff_from_systrace(void) -+{ -+ static unsigned long __read_mostly next_print; -+ -+ if (time_before(jiffies, READ_ONCE(next_print))) -+ return; -+ -+ WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL); -+ hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio); -+} -+ -+void hmbird_notify_sched_tick(void) -+{ -+ unsigned long last_check; -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ hmbird_scheduler_tick(); -+ -+ if (!hmbird_enabled()) -+ return; -+ -+ last_check = hmbird_watchdog_timestamp; -+ if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -+ -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "watchdog failed to check in for %u.%03us", -+ dur_ms / 1000, dur_ms % 1000); -+ } -+ scan_timeout(rq); -+} -+ -+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued) -+{ -+ update_curr_hmbird(rq); -+ -+ set_pcp_round(rq); -+ -+ if (slim_walt_ctrl) -+ hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE); -+ /* -+ * While disabling, always resched and refresh core-sched timestamp as -+ * we can't trust the slice management or ops.core_sched_before(). -+ */ -+ if (hmbird_ops_disabling()) -+ get_hmbird_ts(curr)->slice = 0; -+ -+ if (!get_hmbird_ts(curr)->slice) -+ resched_curr(rq); -+ -+ inform_hmbird_onoff_from_systrace(); -+} -+ -+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg) -+{ -+ hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ get_hmbird_ts(p)->disallow = false; -+ -+ hmbird_sched_init_task(p); -+ -+ set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ return 0; -+} -+ -+static void hmbird_ops_enable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void hmbird_ops_disable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void set_task_hmbird_weight(struct task_struct *p) -+{ -+ u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -+ -+ get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight); -+} -+ -+/** -+ * refresh_hmbird_weight - Refresh a task's hmbird weight -+ * @p: task to refresh hmbird weight for -+ * -+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to -+ * enable easy access from the BPF scheduler. To keep it synchronized with the -+ * current task priority, this function should be called when a new task is -+ * created, priority is changed for a task on hmbird, and a task is switched -+ * to hmbird from other classes. -+ */ -+static void refresh_hmbird_weight(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ set_task_hmbird_weight(p); -+} -+ -+int hmbird_pre_fork(struct task_struct *p) -+{ -+ int ret = 0; -+ -+ p->android_oem_data1[HMBIRD_TS_IDX] = -+ (u64)(kzalloc(sizeof(struct hmbird_entity), GFP_KERNEL)); -+ if (!get_hmbird_ts(p)) -+ return -1; -+ -+ get_hmbird_ts(p)->dsq = NULL; -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node); -+ get_hmbird_ts(p)->flags = 0; -+ get_hmbird_ts(p)->weight = 0; -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ get_hmbird_ts(p)->holding_cpu = -1; -+ get_hmbird_ts(p)->kf_mask = 0; -+ atomic64_set(&get_hmbird_ts(p)->ops_state, 0); -+ get_hmbird_ts(p)->runnable_at = INITIAL_JIFFIES; -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+ get_hmbird_ts(p)->task = p; -+ hmbird_set_sched_prop(p, 0); -+ -+ get_hmbird_ts(p)->critical_affinity_cpu = -1; -+ get_hmbird_ts(p)->sched_class = &hmbird_sched_class; -+ get_hmbird_ts(p)->tick_hit_count = 0; -+ get_hmbird_ts(p)->start_jiffies = 0; -+ -+ /* -+ * BPF scheduler enable/disable paths want to be able to iterate and -+ * update all tasks which can become complex when racing forks. As -+ * enable/disable are very cold paths, let's use a percpu_rwsem to -+ * exclude forks. -+ */ -+ -+ return ret; -+} -+ -+void hmbird_post_fork(struct task_struct *p) -+{ -+ percpu_down_read(&hmbird_fork_rwsem); -+ -+ if (hmbird_enabled()) { -+ struct rq_flags rf; -+ struct rq *rq; -+ p->sched_class = &hmbird_sched_class; -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ rq = task_rq_lock(p, &rf); -+ /* -+ * Set the weight manually before calling ops.enable() so that -+ * the scheduler doesn't see a stale value if they inspect the -+ * task struct. We'll invoke ops.set_weight() afterwards, as it -+ * would be odd to receive a callback on the task before we -+ * tell the scheduler that it's been fully enabled. -+ */ -+ set_task_hmbird_weight(p); -+ hmbird_ops_enable_task(p); -+ refresh_hmbird_weight(p); -+ task_rq_unlock(rq, p, &rf); -+ } else { -+ if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ } -+ -+ spin_lock_irq(&hmbird_tasks_lock); -+ list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_cancel_fork(struct task_struct *p) -+{ -+ if (hmbird_enabled()) -+ hmbird_ops_disable_task(p); -+ put_hmbird_ts(p); -+} -+ -+void hmbird_free(struct task_struct *p) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+ list_del_init(&get_hmbird_ts(p)->tasks_node); -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+ -+ /* -+ * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED -> -+ * ENABLED transitions can't race us. Disable ops for @p. -+ */ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) || -+ test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ hmbird_ops_disable_task(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ put_hmbird_ts(p); -+} -+ -+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio) -+{ -+} -+ -+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type) -+{ -+ return (prop >> TOP_TASK_SHIFT) & (1 << type); -+} -+ -+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p) -+{ -+ uint32_t prop = get_top_task_prop(p); -+ -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE)) -+ return HMBIRD_TASK_PROP_PIPELINE; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) || -+ !task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) { -+ return HMBIRD_TASK_PROP_COMMON; -+ } -+ return HMBIRD_TASK_PROP_DEBUG_OR_LOG; -+} -+ -+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b) -+{ -+ int type_a = hmbird_get_task_type(a); -+ int type_b = hmbird_get_task_type(b); -+ -+ return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b]; -+} -+ -+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ enum cpu_type type; -+ int sp_dl; -+ struct task_struct *curr = NULL; -+ -+ switch (hmbird_preempt_policy) { -+ case HMBIRD_PREEMPT_POLICY_PRIO_BASED: -+ curr = rq->curr; -+ if (curr && hmbird_prio_higher(p, curr)) -+ goto preempt; -+ break; -+ default: -+ break; -+ } -+ if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) || -+ (is_critical_system_task(p) && !is_critical_system_task(rq->curr))) -+ goto preempt; -+ -+ sp_dl = find_idx_from_task(p); -+ if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled())) -+ return; -+ -+ if (rq->curr->prio > p->prio) -+ goto preempt; -+ -+ return; -+ -+preempt: -+ resched_curr(rq); -+} -+ -+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {} -+ -+int hmbird_check_setscheduler(struct task_struct *p, int policy) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ /* if disallow, reject transitioning into HMBIRD */ -+ if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) && -+ p->policy != policy && policy == SCHED_HMBIRD) -+ return -EACCES; -+ -+ return 0; -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool hmbird_can_stop_tick(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ if (hmbird_ops_disabling()) -+ return false; -+ -+ if (p->sched_class != &hmbird_sched_class) -+ return true; -+ -+ return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK; -+} -+#endif -+ -+int hmbird_tg_online(struct task_group *tg) -+{ -+ struct cgroup *cgrp; -+ -+ if (!tg) -+ return 0; -+ cgrp = tg->css.cgroup; -+ if (!cgrp || !(cgrp->kn)) -+ return 0; -+ update_cgroup_ids_table(cgrp->kn->id, -1); -+ return 0; -+} -+ -+/* -+ * Omitted operations: -+ * -+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -+ * task isn't tied to the CPU at that point. Preemption is implemented by -+ * resetting the victim task's slice to 0 and triggering reschedule on the -+ * target CPU. -+ * -+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -+ * -+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -+ * their current sched_class. Call them directly from sched core instead. -+ * -+ * - task_woken, switched_from: Unnecessary. -+ */ -+DEFINE_SCHED_CLASS(hmbird) = { -+ .enqueue_task = enqueue_task_hmbird, -+ .dequeue_task = dequeue_task_hmbird, -+ .yield_task = yield_task_hmbird, -+ .yield_to_task = yield_to_task_hmbird, -+ -+ .check_preempt_curr = check_preempt_curr_hmbird, -+ -+ .pick_next_task = pick_next_task_hmbird, -+ -+ .put_prev_task = put_prev_task_hmbird, -+ .set_next_task = set_next_task_hmbird, -+ -+#ifdef CONFIG_SMP -+ .balance = balance_hmbird, -+ .select_task_rq = select_task_rq_hmbird, -+ .set_cpus_allowed = set_cpus_allowed_hmbird, -+#endif -+ .task_tick = task_tick_hmbird, -+ -+ .switched_to = switched_to_hmbird, -+ .prio_changed = prio_changed_hmbird, -+ -+ .update_curr = update_curr_hmbird, -+ -+#ifdef CONFIG_UCLAMP_TASK -+ .uclamp_enabled = 0, -+#endif -+}; -+ -+/* -+ * Must with rq lock held. -+ */ -+bool task_is_hmbird(struct task_struct *p) -+{ -+ return p->sched_class == &hmbird_sched_class; -+} -+ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id) -+{ -+ memset(dsq, 0, sizeof(*dsq)); -+ -+ raw_spin_lock_init(&dsq->lock); -+ INIT_LIST_HEAD(&dsq->fifo); -+ dsq->id = dsq_id; -+} -+ -+static int hmbird_cgroup_init(void) -+{ -+ struct cgroup_subsys_state *css; -+ -+ css_for_each_descendant_pre(css, &root_task_group.css) { -+ struct task_group *tg = css_tg(css); -+ -+ cgrp_dsq_idx_init(css->cgroup, tg); -+ } -+ return 0; -+} -+ -+/* -+ * Used by sched_fork() and __setscheduler_prio() to pick the matching -+ * sched_class. dl/rt are already handled. -+ */ -+bool task_on_hmbird(struct task_struct *p) -+{ -+ return hmbird_enabled(); -+} -+ -+static void __setscheduler_prio(struct task_struct *p, int prio) -+{ -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) { -+ p->prio = prio; -+ return; -+ } else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (on_hmbird && rt_prio(prio)) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ -+ p->prio = prio; -+} -+ -+/* -+ * Heartbeat, avoid humbird keep running while APP already exit. -+ * Check whether APP send alive-signal periodly. -+ */ -+#define HEARTBEAT_TIMEOUT (msecs_to_jiffies(2500)) -+#define HEARTBEAT_CHECK_INTERVAL (msecs_to_jiffies(1000)) -+static struct timer_list hb_timer; -+static unsigned long next_hb; -+ -+void hb_timer_handler(struct timer_list *timer) -+{ -+ if (!heartbeat_enable) -+ goto refill; -+ -+ pr_info(": enter timer.\n"); -+ if (heartbeat) { -+ heartbeat = 0; -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ } -+ -+ /* can't detect heartbeat, disable ext. */ -+ if (time_after(jiffies, READ_ONCE(next_hb))) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT, -+ "can't detect heartbeat, disable ext\n"); -+ } -+refill: -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_start(void) -+{ -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_init(void) -+{ -+ timer_setup(&hb_timer, hb_timer_handler, 0); -+} -+ -+static void hb_timer_exit(void) -+{ -+ del_timer(&hb_timer); -+} -+ -+static bool check_and_disable_cpuhp(void) -+{ -+ struct rq *rq; -+ struct rq_flags rf; -+ int cpu; -+ -+ cpu_hotplug_disable(); -+ cpus_read_lock(); -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ rq_lock_irqsave(rq, &rf); -+ if (!rq->online) { -+ rq_unlock_irqrestore(rq, &rf); -+ goto err_offline; -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ cpus_read_unlock(); -+ return true; -+ -+err_offline: -+ cpus_read_unlock(); -+ cpu_hotplug_enable(); -+ return false; -+} -+ -+static void reenable_cpuhp(void) -+{ -+ cpu_hotplug_enable(); -+} -+ -+static void scheduler_switch_done(bool final_state) -+{ -+ /* set scx_enable again, switch may fail. */ -+ scx_enable = final_state; -+ /* reset heartbeat*/ -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ -+ if (final_state) -+ hb_timer_start(); -+ else -+ hb_timer_exit(); -+} -+ -+/** -+ * ss : curr switch state -+ * finish : success or fail -+ * enable : curr operation is diabling or enabling? -+ * fail_reason : string output when failed, empty string when success. -+ */ -+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason) -+{ -+ char *s1 = finish ? "finished" : "failed"; -+ char *s2 = enable ? "enabled" : "disabled"; -+ -+ hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss); -+ if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) { -+ sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type)); -+ hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n", -+ s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason); -+ } -+ curr_ss = ss; -+} -+ -+bool get_hmbird_ops_enabled(void) -+{ -+ return atomic_read(&__hmbird_ops_enabled); -+} -+ -+bool get_non_hmbird_task(void) -+{ -+ return atomic_read(&non_hmbird_task); -+} -+ -+static void hmbird_ops_disable_workfn(struct kthread_work *work) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int cpu; -+ -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, ""); -+ cancel_delayed_work_sync(&hmbird_watchdog_work); -+ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) { -+ case HMBIRD_OPS_DISABLED: -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled"); -+ scheduler_switch_done(false); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return; -+ case HMBIRD_OPS_PREPPING: -+ fallthrough; -+ case HMBIRD_OPS_DISABLING: -+ /* shouldn't happen but handle it like ENABLING if it does */ -+ WARN_ONCE(true, "hmbird: duplicate disabling instance?"); -+ fallthrough; -+ case HMBIRD_OPS_ENABLING: -+ case HMBIRD_OPS_ENABLED: -+ break; -+ } -+ -+ /* kick all CPUs to restore ticks */ -+ for_each_possible_cpu(cpu) -+ resched_cpu(cpu); -+ -+ /* avoid racing against fork and cgroup changes */ -+ cpus_read_lock(); -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, ""); -+ spin_lock_irq(&hmbird_tasks_lock); -+ atomic_set(&__hmbird_ops_enabled, false); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ bool alive = READ_ONCE(p->__state) != TASK_DEAD; -+ -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ get_hmbird_ts(p)->slice = min_t(u64, -+ get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL); -+ -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ if (alive) -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ -+ hmbird_ops_disable_task(p); -+ } -+ hmbird_task_iter_exit(&sti); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, ""); -+ -+ atomic_set(&non_hmbird_task, true); -+ /* no task is on hmbird, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt); -+ synchronize_rcu(); -+ -+ percpu_up_write(&hmbird_fork_rwsem); -+ cpus_read_unlock(); -+ -+ if (slim_walt_ctrl) -+ slim_walt_enable(false); -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ -+ hmbird_switch_log(HMBIRD_DISABLED, 1, 0, ""); -+ scheduler_switch_done(false); -+ reenable_cpuhp(); -+} -+ -+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn); -+ -+static void schedule_hmbird_ops_disable_work(void) -+{ -+ struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper); -+ -+ /* -+ * We may be called spuriously before the first bpf_hmbird_reg(). If -+ * hmbird_ops_helper isn't set up yet, there's nothing to do. -+ */ -+ if (helper) -+ kthread_queue_work(helper, &hmbird_ops_disable_work); -+} -+ -+static void hmbird_ops_disable(void) -+{ -+ schedule_hmbird_ops_disable_work(); -+} -+ -+static void hmbird_err_exit_workfn(struct work_struct *work) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ hmbird_ctrl(false); -+ -+ for_each_present_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy) -+ continue; -+ if (cpu != policy->cpu) -+ goto put; -+ down_write(&policy->rwsem); -+ WARN_ON(store_scaling_governor(policy, -+ saved_gov[cpu], strlen(saved_gov[cpu])) <= 0); -+ up_write(&policy->rwsem); -+ hmbird_info_trace(":restore origin gov : %s\n", saved_gov[cpu]); -+put: -+ cpufreq_cpu_put(policy); -+ } -+ memset((char *)saved_gov, 0, sizeof(saved_gov)); -+} -+ -+void hmbird_ops_exit(void) -+{ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); -+} -+ -+static struct kthread_worker *hmbird_create_rt_helper(const char *name) -+{ -+ struct kthread_worker *helper; -+ -+ helper = kthread_create_worker(KTW_FREEZABLE, name); -+ if (helper) -+ sched_set_fifo(helper->task); -+ return helper; -+} -+ -+static inline void set_audio_thread_sched_prop(struct task_struct *p) -+{ -+ struct cgroup_subsys_state *css; -+ -+ if (likely(p->prio >= MAX_RT_PRIO)) -+ return; -+ -+ rcu_read_lock(); -+ css = task_css(p, cpuset_cgrp_id); -+ if (!css) { -+ rcu_read_unlock(); -+ return; -+ } -+ rcu_read_unlock(); -+ -+ if (!strcmp(css->cgroup->kn->name, "audio-app")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+} -+ -+int scx_systemui_pid = -1; -+void set_systemui_thread_pid(struct task_struct *p) -+{ -+ if ((strcmp(p->comm, "ndroid.systemui") == 0) && (p->pid == p->tgid)) -+ scx_systemui_pid = p->pid; -+} -+ -+static int hmbird_ops_enable(void *unused) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int ret; -+ int tcnt = 0; -+ unsigned long long start = 0; -+ -+ if (!check_and_disable_cpuhp()) { -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline"); -+ return -EBUSY; -+ } -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, ""); -+ mutex_lock(&hmbird_ops_enable_mutex); -+ -+ if (!hmbird_ops_helper) { -+ WRITE_ONCE(hmbird_ops_helper, -+ hmbird_create_rt_helper("hmbird_ops_helper")); -+ if (!hmbird_ops_helper) { -+ ret = -ENOMEM; -+ goto err_unlock; -+ } -+ } -+ -+ if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) { -+ ret = -EBUSY; -+ goto err_unlock; -+ } -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) != -+ HMBIRD_OPS_DISABLED); -+ -+ hmbird_warned_zero_slice = false; -+ -+ atomic64_set(&hmbird_nr_rejected, 0); -+ -+ /* -+ * Keep CPUs stable during enable so that the BPF scheduler can track -+ * online CPUs by watching ->on/offline_cpu() after ->init(). -+ */ -+ cpus_read_lock(); -+ -+ hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work, -+ hmbird_watchdog_timeout / 2); -+ -+ /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ */ -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ reset_idle_masks(); -+ -+ /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. -+ */ -+ ret = hmbird_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock; -+ -+ /* -+ * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem -+ * preventing new tasks from being added. No need to exclude tasks -+ * leaving as hmbird_free() can handle both prepped and enabled -+ * tasks. Prep all tasks first and then enable them with preemption -+ * disabled. -+ */ -+ spin_lock_irq(&hmbird_tasks_lock); -+ -+ atomic_set(&non_hmbird_task, false); -+ atomic_set(&__hmbird_ops_enabled, true); -+ -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered(&sti))) -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ hmbird_task_iter_exit(&sti); -+ -+ /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. -+ */ -+ preempt_disable(); -+ -+ /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ */ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) { -+ atomic_set(&non_hmbird_task, true); -+ atomic_set(&__hmbird_ops_enabled, false); -+ preempt_enable(); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ ret = -EBUSY; -+ goto err_disable_unlock; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED -+ * transitions here are synchronized against hmbird_free() through -+ * hmbird_tasks_lock. -+ */ -+ start = sched_clock(); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, ""); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ tcnt++; -+ if (READ_ONCE(p->__state) != TASK_DEAD) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ -+ set_audio_thread_sched_prop(p); -+ set_systemui_thread_pid(p); -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ hmbird_ops_enable_task(p); -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ } else { -+ hmbird_ops_disable_task(p); -+ } -+ } -+ hmbird_task_iter_exit(&sti); -+ -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, ""); -+ preempt_enable(); -+ percpu_up_write(&hmbird_fork_rwsem); -+ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) { -+ ret = -EBUSY; -+ goto err_disable; -+ } -+ -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_ENABLED, 1, 1, ""); -+ scheduler_switch_done(true); -+ -+ return 0; -+ -+err_unlock: -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock"); -+ scheduler_switch_done(false); -+ return ret; -+ -+err_disable_unlock: -+ percpu_up_write(&hmbird_fork_rwsem); -+err_disable: -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ /* must be fully disabled before returning */ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable"); -+ scheduler_switch_done(false); -+ return ret; -+} -+ -+#ifdef CONFIG_SCHED_DEBUG -+static const char *hmbird_ops_enable_state_str[] = { -+ [HMBIRD_OPS_PREPPING] = "prepping", -+ [HMBIRD_OPS_ENABLING] = "enabling", -+ [HMBIRD_OPS_ENABLED] = "enabled", -+ [HMBIRD_OPS_DISABLING] = "disabling", -+ [HMBIRD_OPS_DISABLED] = "disabled", -+}; -+ -+static int hmbird_debug_show(struct seq_file *m, void *v) -+{ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled()); -+ seq_printf(m, "%-30s: %s\n", "enable_state", -+ hmbird_ops_enable_state_str[hmbird_ops_enable_state()]); -+ seq_printf(m, "%-30s: %llu\n", "nr_rejected", -+ atomic64_read(&hmbird_nr_rejected)); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return 0; -+} -+ -+static int hmbird_debug_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_debug_show, NULL); -+} -+ -+const struct file_operations sched_hmbird_fops = { -+ .open = hmbird_debug_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif -+ -+ -+static int bpf_hmbird_reg(void *kdata) -+{ -+ return hmbird_ops_enable(kdata); -+} -+ -+static int bpf_hmbird_unreg(void *kdata) -+{ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ return 0; -+} -+ -+void set_hmbird_module_loaded(int is_loaded) -+{ -+ atomic_set(&hmbird_module_loaded, is_loaded); -+} -+ -+/* -+ * MUST load hmbird module before enable hmbird scheduler -+ * load track & hmbird gover implemented in hmbird module -+ */ -+int hmbird_ctrl(bool enable) -+{ -+ if (!atomic_read(&hmbird_module_loaded)) { -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "ext module unloaded\n"); -+ return -EINVAL; -+ } -+ -+ if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED -+ || hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING -+ || hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already enabled(ing)\n"); -+ hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING || -+ hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already disabled(ing)\n"); -+ hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (enable) -+ return bpf_hmbird_reg(NULL); -+ else -+ return bpf_hmbird_unreg(NULL); -+} -+ -+void set_cpu_isomask(int cpu, cpumask_var_t *mask) -+{ -+ cpumask_clear_cpu(cpu, iso_masks.ex_free); -+ cpumask_clear_cpu(cpu, iso_masks.exclusive); -+ cpumask_clear_cpu(cpu, iso_masks.partial); -+ cpumask_clear_cpu(cpu, iso_masks.big); -+ cpumask_clear_cpu(cpu, iso_masks.little); -+ cpumask_set_cpu(cpu, *mask); -+} -+ -+void set_cpu_cluster(u64 cpu_cluster) -+{ -+ int cpu; -+ -+ for_each_present_cpu(cpu) { -+ u64 pos = 1 << cpu; -+ -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.ex_free)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.exclusive)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.partial)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.big)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) -+ set_cpu_isomask(cpu, &(iso_masks.little)); -+ } -+} -+ -+static void get_hmbird_snapshot(struct panic_snapshot_t *p) -+{ -+ struct hmbird_dispatch_q *dsq; -+ struct rq *rq; -+ int cpu, i; -+ -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ p->rq_nr[cpu] = rq->nr_running; -+ p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running; -+ } -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ struct hmbird_entity *entity; -+ -+ dsq = &gdsqs[i]; -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo)) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ p->runnable_at[i] = entity->runnable_at; -+ raw_spin_unlock(&dsq->lock); -+ -+ } -+ -+ p->snap_misc.hmbird_enabled = hmbird_enabled(); -+ p->snap_misc.curr_ss = curr_ss; -+ p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state(); -+ p->snap_misc.parctrl_high_ratio = parctrl_high_ratio; -+ p->snap_misc.parctrl_low_ratio = parctrl_low_ratio; -+ p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l; -+ p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l; -+ p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio; -+ p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio; -+ p->snap_misc.misfit_ds = misfit_ds; -+ p->snap_misc.partial_enable = partial_enable; -+ p->snap_misc.iso_free_rescue = iso_free_rescue; -+ p->snap_misc.isolate_ctrl = isolate_ctrl; -+ p->snap_misc.snap_jiffies = jiffies; -+ p->snap_misc.snap_time = local_clock(); -+} -+ -+// MTK minidump begin -+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3) -+{ -+ strscpy(m->desc_str, str, DESC_STR_LEN); -+ m->len = d1 * d2 * d3; -+ m->parse[0] = d1; -+ m->parse[1] = d2; -+ m->parse[2] = d3; -+} -+ -+static void init_desc_metas(struct md_info_t *m) -+{ -+ init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :", -+ 1, MAX_SWITCHS, SWITCH_ITEMS); -+ -+ init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1); -+ -+ init_desc_meta(&m->kern_dump.excep_rec_meta, -+ "excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS); -+ -+ init_desc_meta(&m->kern_dump.excep_idx_meta, -+ "excep idx :", 1, 1, MAX_EXCEP_ID); -+ -+ init_desc_meta(&m->kern_dump.snap.runnable_at_meta, -+ "each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS); -+ -+ init_desc_meta(&m->kern_dump.snap.rq_nr_meta, -+ "rq runnable task nr:", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta, -+ "scxrq runnable task nr :", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.snap_misc_meta, -+ "misc snap params :", 1, 1, SNAP_ITEMS); -+} -+ -+static void init_md_meta(struct md_info_t *m) -+{ -+ m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64); -+ m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64); -+ m->meta.unit_size = sizeof(u64); -+ m->meta.switches = MAX_SWITCHS; -+ m->meta.exceps = MAX_EXCEPS; -+ m->meta.global_dsqs = MAX_GLOBAL_DSQS; -+ m->meta.parse_dimens = PARSE_DIMENS; -+ m->meta.nr_cpus = num_possible_cpus(); -+ m->meta.real_cpus = nr_cpu_ids; -+ m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64); -+ m->meta.nr_meta_desc = 8; -+ m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64); -+ -+ init_desc_metas(m); -+} -+ -+#define MINIDUMP_DFL_SIZE (4 * 1024) -+ -+struct notifier_block hmbird_panic_blk; -+static int hmbird_panic_handler(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ if (!md_info) -+ return NOTIFY_DONE; -+ -+ get_hmbird_snapshot(&md_info->kern_dump.snap); -+ -+ return NOTIFY_DONE; -+} -+ -+static void panic_blk_init(void) -+{ -+ int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE); -+ -+ md_info = kzalloc(dump_size, GFP_KERNEL); -+ if (!md_info) -+ return; -+ init_md_meta(md_info); -+ -+ hmbird_panic_blk.notifier_call = hmbird_panic_handler; -+ /* make sure to execute before minidump. */ -+ hmbird_panic_blk.priority = INT_MAX; -+ atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk); -+ -+ hmbird_debug("register minidump.\n"); -+} -+ -+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size) -+{ -+ *vaddr = (unsigned long)md_info; -+ *size = sizeof(struct md_info_t); -+} -+// MTK minidump end -+ -+static inline void init_sched_prop_to_preempt_prio(void) -+{ -+ for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) { -+ switch (i) { -+ case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 5; -+ break; -+ -+ case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 4; -+ break; -+ -+ case HMBIRD_TASK_PROP_PIPELINE: -+ case HMBIRD_TASK_PROP_ISOLATE: -+ sched_prop_to_preempt_prio[i] = 3; -+ break; -+ -+ case HMBIRD_TASK_PROP_COMMON: -+ sched_prop_to_preempt_prio[i] = 1; -+ break; -+ -+ case HMBIRD_TASK_PROP_DEBUG_OR_LOG: -+ sched_prop_to_preempt_prio[i] = 0; -+ break; -+ -+ default: -+ sched_prop_to_preempt_prio[i] = 2; -+ break; -+ } -+ } -+} -+ -+void __init init_sched_hmbird_class(void) -+{ -+ int cpu; -+ u32 v; -+ struct hmbird_entity *init_hmbird; -+ -+ /* -+ * The following is to prevent the compiler from optimizing out the enum -+ * definitions so that BPF scheduler implementations can use them -+ * through the generated vmlinux.h. -+ */ -+ WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP | -+ HMBIRD_KICK_PREEMPT); -+ -+ init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL); -+ init_dsq_at_boot(); -+ init_isolate_cpus(); -+ hb_timer_init(); -+ init_sched_prop_to_preempt_prio(); -+#ifdef CONFIG_SMP -+ WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); -+#endif -+ -+ /* -+ * we can't static init init_task's hmbird struct, init here. -+ * init_task->hmbird would not use during boot. -+ */ -+ init_hmbird = kzalloc(sizeof(struct hmbird_entity), GFP_KERNEL); -+ init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird; -+ if (init_hmbird) { -+ INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo); -+ INIT_LIST_HEAD(&init_hmbird->watchdog_node); -+ init_hmbird->sticky_cpu = -1; -+ init_hmbird->holding_cpu = -1; -+ atomic64_set(&init_hmbird->ops_state, 0); -+ init_hmbird->runnable_at = jiffies; -+ init_hmbird->slice = HMBIRD_SLICE_DFL; -+ init_hmbird->task = &init_task; -+ hmbird_set_sched_prop(&init_task, 0); -+ } else { -+ hmbird_err(INIT_TASK_FAIL, ":alloc init_task.scx failed!!!\n"); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ /* -+ * exec during boot phase, no need to care about alloc failed. -+ * lifecycle same to rq, no need to free. -+ */ -+ rq->android_oem_data1[HMBIRD_RQ_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_rq), GFP_KERNEL); -+ if (get_hmbird_rq(rq)) { -+ get_hmbird_rq(rq)->rq = rq; -+ get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window; -+ } else { -+ hmbird_err(ALLOC_RQSCX_FAIL, ":alloc rq->scx failed!!!\n"); -+ } -+ -+ rq->android_oem_data1[HMBIRD_OPS_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL); -+ if (!get_hmbird_ops(rq)) -+ pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n"); -+ init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL); -+ INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list); -+ -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL)); -+ -+ hmbird_ops_init(get_hmbird_ops(rq)); -+ } -+ -+ INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn); -+ INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn); -+ hmbird_misc_init(); -+ -+ panic_blk_init(); -+} -+ -diff --git b/kernel/sched/hmbird/hmbird_misc.c a/kernel/sched/hmbird/hmbird_misc.c -new file mode 100755 -index 000000000000..895e8a29df33 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_misc.c -@@ -0,0 +1,148 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * @Description: -+ * @Version: -+ * @Author: wangrui8 -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include "hmbird_sched.h" -+#include -+#include -+#include "slim.h" -+ -+noinline int tracing_mark_write(const char *buf) -+{ -+ trace_printk(buf); -+ return 0; -+} -+ -+struct yield_opt_params yield_opt_params = { -+ .enable = 0, -+ .frame_per_sec = 120, -+ .frame_time_ns = NSEC_PER_SEC / 120, -+ .yield_headroom = 10, -+}; -+ -+DEFINE_PER_CPU(struct sched_yield_state, ystate); -+ -+static inline void hmbird_yield_state_update(struct sched_yield_state *ys) -+{ -+ if (!raw_spin_is_locked(&ys->lock)) -+ return; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ -+ if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1 -+ || ys->yield_cnt_after_sleep > yield_headroom) { -+ ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP); -+ } else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) { -+ ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP); -+ } -+ ys->yield_cnt = 0; -+ ys->sleep_times = 0; -+ ys->yield_cnt_after_sleep = 0; -+} -+ -+void hmbird_skip_yield(long *skip) -+{ -+ if (!get_hmbird_ops_enabled() || !yield_opt_params.enable) -+ return; -+ unsigned long flags, sleep_now = 0; -+ struct sched_yield_state *ys; -+ int cpu = raw_smp_processor_id(), cont_yield, new_frame; -+ int frame_time_ns = yield_opt_params.frame_time_ns; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ u64 wc; -+ -+ if (!(*skip)) { -+ wc = sched_clock(); -+ ys = &per_cpu(ystate, cpu); -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ -+ cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP; -+ new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1); -+ -+ if (!cont_yield && new_frame) { -+ hmbird_yield_state_update(ys); -+ ys->last_update_time = wc; -+ ys->sleep_end = ys->last_yield_time + frame_time_ns -+ - yield_headroom * YIELD_DURATION; -+ } -+ -+ if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) { -+ *skip = true; -+ -+ sleep_now = ys->sleep_times ? -+ max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep; -+ if (wc + sleep_now > ys->sleep_end) { -+ u64 delta = ys->sleep_end - wc; -+ -+ if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION) -+ sleep_now = delta; -+ else -+ sleep_now = 0; -+ } -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ if (sleep_now) { -+ sleep_now = div64_u64(sleep_now, 1000); -+ usleep_range_state(sleep_now, sleep_now, TASK_IDLE); -+ } -+ ys->sleep_times++; -+ ys->last_yield_time = sched_clock(); -+ return; -+ } -+ if (ys->sleep_times) -+ ys->yield_cnt_after_sleep++; -+ else -+ (ys->yield_cnt)++; -+ ys->last_yield_time = wc; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+} -+ -+struct boost_policy_params boost_policy_params = { -+ .enable = 0, -+ .bottom_freq = 1200000, -+ .boost_weight = 120, -+}; -+ -+int hmbird_get_boost_enable(void) -+{ -+ return boost_policy_params.enable; -+} -+ -+unsigned int hmbird_get_boost_bottom_freq(void) -+{ -+ return boost_policy_params.bottom_freq; -+} -+ -+int hmbird_get_boost_weight(void) -+{ -+ return boost_policy_params.boost_weight; -+} -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops) -+{ -+ hmbird_ops->scx_enable = get_hmbird_ops_enabled; -+ hmbird_ops->check_non_task = get_non_hmbird_task; -+ hmbird_ops->hmbird_get_md_info = hmbird_get_md_info; -+ hmbird_ops->hmbird_get_boost_enable = hmbird_get_boost_enable; -+ hmbird_ops->hmbird_get_boost_bottom_freq = hmbird_get_boost_bottom_freq; -+ hmbird_ops->hmbird_get_boost_weight = hmbird_get_boost_weight; -+} -+ -+void hmbird_misc_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_init(&ys->lock); -+ } -+ -+ set_hmbird_module_loaded(1); -+} -+ -diff --git b/kernel/sched/hmbird/hmbird_sched.h a/kernel/sched/hmbird/hmbird_sched.h -new file mode 100755 -index 000000000000..76acbe9685e4 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef __HMBIRD_SCHED__ -+#define __HMBIRD_SCHED__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include <../../kernel/time/tick-sched.h> -+#include <../../kernel/sched/sched.h> -+#include -+#include -+#include -+ -+#define REGISTER_TRACE_VH(vender_hook, handler) \ -+{ \ -+ ret = register_trace_##vender_hook(handler, NULL); \ -+ if (ret) { \ -+ pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \ -+ } \ -+} -+#define REGISTER_TRACE(vendor_hook, handler, data, err) \ -+do { \ -+ ret = register_trace_##vendor_hook(handler, data); \ -+ if (ret) { \ -+ pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret); \ -+ goto err; \ -+ } \ -+} while (0) -+ -+#define UNREGISTER_TRACE(vendor_hook, handler, data) \ -+ unregister_trace_##vendor_hook(handler, data) \ -+ -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+ -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int sched_ravg_window_frame_per_sec; -+extern int slim_gov_debug; -+extern int cpu7_tl; -+extern int scx_gov_ctrl; -+extern spinlock_t new_sched_ravg_window_lock; -+extern int cluster_separate; -+ -+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER BIT(31) -+#define MAX_YIELD_SLEEP (2000000ULL) -+#define MIN_YIELD_SLEEP (200000ULL) -+#define YIELD_DURATION (5000ULL) -+#define DEFAULT_YIELD_SLEEP_TH (10) -+ -+struct sched_yield_state { -+ raw_spinlock_t lock; -+ u64 last_yield_time; -+ u64 last_update_time; -+ u64 sleep_end; -+ unsigned long yield_cnt; -+ unsigned long yield_cnt_after_sleep; -+ unsigned long sleep; -+ int sleep_times; -+}; -+ -+DECLARE_PER_CPU(struct sched_yield_state, ystate); -+ -+void hmbird_window_rollover_run_once(struct rq *rq); -+void hmbird_yield_state_update_per_frame(void); -+void hmbird_misc_init(void); -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops); -+#endif /*__HMBIRD_SCHED__*/ -diff --git b/kernel/sched/hmbird/hmbird_sched_proc.c a/kernel/sched/hmbird/hmbird_sched_proc.c -new file mode 100755 -index 000000000000..295d45404608 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched_proc.c -@@ -0,0 +1,640 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "hmbird_sched_proc.h" -+#include -+ -+#include "hmbird_util_track.h" -+#include "slim.h" -+ -+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched" -+#define SLIM_FREQ_GOV_DIR "slim_freq_gov" -+#define LOAD_TRACK_DIR "slim_walt" -+#define HMBIRD_PROC_PERMISSION 0666 -+ -+int scx_enable; -+int partial_enable; -+int cpuctrl_high_ratio = 55; -+int cpuctrl_low_ratio = 40; -+int slim_stats; -+int hmbirdcore_debug; -+int slim_for_app; -+int misfit_ds = 90; -+unsigned int highres_tick_ctrl; -+unsigned int highres_tick_ctrl_dbg; -+int cpu7_tl = 70; -+int slim_walt_ctrl; -+int slim_walt_dump; -+int slim_walt_policy; -+int slim_gov_debug; -+int scx_gov_ctrl = 1; -+int sched_ravg_window_frame_per_sec = 125; -+int parctrl_high_ratio = 55; -+int parctrl_low_ratio = 40; -+int parctrl_high_ratio_l = 65; -+int parctrl_low_ratio_l = 50; -+int isoctrl_high_ratio = 75; -+int isoctrl_low_ratio = 60; -+int isolate_ctrl; -+int iso_free_rescue; -+int heartbeat; -+int heartbeat_enable = 1; -+int watchdog_enable; -+int save_gov; -+u64 cpu_cluster_masks; -+int hmbird_preempt_policy; -+int cluster_separate; -+ -+char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= 32) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtoint(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoint\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+static int set_proc_buf_val_u64(struct file *file, const char __user *buf, -+ size_t count, u64 *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= sizeof(kbuf)) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtou64(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoul\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/* common ops begin */ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ return count; -+} -+ -+static int hmbird_common_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%d\n", *(int *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_show, pde_data(inode)); -+} -+ -+static int hmbird_common_ul_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%lu\n", *(unsigned long *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_ul_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_ul_show, pde_data(inode)); -+} -+ -+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write); -+/* common ops end */ -+ -+/* scx_enable ops begin */ -+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC); -+ if (hmbird_ctrl(*pval)) -+ return -EFAULT; -+ -+ return count; -+} -+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write); -+/* scx_enable ops end */ -+ -+/* hmbird_stats ops begin */ -+#define MAX_STATS_BUF (4096) -+static int hmbird_stats_proc_show(struct seq_file *m, void *v) -+{ -+ char *buf; -+ -+ buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ stats_print(buf, MAX_STATS_BUF); -+ -+ seq_printf(m, "%s\n", buf); -+ -+ kfree(buf); -+ return 0; -+} -+ -+static int hmbird_stats_proc_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_stats_proc_show, inode); -+} -+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL); -+/* hmbird_stats ops end */ -+ -+/* sched_ravg_window_frame_per_sec ops begin */ -+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file, -+ const char __user *buf, size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ sched_ravg_window_change(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open, -+ sched_ravg_window_frame_per_sec_proc_write); -+/* sched_ravg_window_frame_per_sec ops end */ -+ -+static ssize_t save_gov_str(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ for_each_possible_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy || (cpu != policy->cpu)) -+ continue; -+ WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0); -+ hmbird_info_systrace(":save origin gov : %s\n", saved_gov[cpu]); -+ } -+ return count; -+} -+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str); -+ -+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ u64 *pval = (u64 *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val_u64(file, buf, count, pval)) -+ return -EFAULT; -+ -+ if (scx_enable == 0) -+ set_cpu_cluster(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write); -+ -+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ int tmp_val; -+ -+ if (set_proc_buf_val(file, buf, count, &tmp_val)) -+ return -EFAULT; -+ -+ slim_walt_enable(tmp_val); -+ *pval = tmp_val; -+ -+ return count; -+} -+ -+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write); -+ -+/* yield_opt ops begin */ -+static int yield_opt_show(struct seq_file *m, void *v) -+{ -+ struct yield_opt_params *data = m->private; -+ -+ seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n", -+ data->enable, data->frame_per_sec, data->yield_headroom); -+ return 0; -+} -+ -+static int yield_opt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, yield_opt_show, pde_data(inode)); -+} -+ -+static ssize_t yield_opt_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu; -+ unsigned long flags; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if (copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp -+ != 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) || -+ (yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp; -+ yield_opt_params.frame_per_sec = frame_per_sec_tmp; -+ yield_opt_params.yield_headroom = yield_headroom_tmp; -+ yield_opt_params.enable = enable_tmp; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ ys->last_yield_time = 0; -+ ys->last_update_time = 0; -+ ys->sleep_end = 0; -+ ys->yield_cnt = 0; -+ ys->yield_cnt_after_sleep = 0; -+ ys->sleep = 0; -+ ys->sleep_times = 0; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write); -+ -+/* boost_policy ops begin */ -+static int boost_policy_show(struct seq_file *m, void *v) -+{ -+ struct boost_policy_params *data = m->private; -+ -+ seq_printf(m, "boost_policy:{\"enable\":%d; \"bottom_freq\":%u; \"boost_weight\":%d}\n", -+ data->enable, data->bottom_freq, data->boost_weight); -+ return 0; -+} -+ -+static int boost_policy_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, boost_policy_show, pde_data(inode)); -+} -+ -+static ssize_t boost_policy_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, bottom_freq_tmp, boost_weight_tmp; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if(copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &bottom_freq_tmp, &boost_weight_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || -+ (boost_weight_tmp < 50 || boost_weight_tmp > 300) || -+ (bottom_freq_tmp < 400000 || bottom_freq_tmp > 2200000)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ boost_policy_params.bottom_freq = bottom_freq_tmp; -+ boost_policy_params.boost_weight = boost_weight_tmp; -+ boost_policy_params.enable = enable_tmp; -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(boost_policy, boost_policy_open, boost_policy_write); -+ -+/* tick_hit ops begin */ -+static int tick_hit_show(struct seq_file *m, void *v) -+{ -+ struct tick_hit_params *data = m->private; -+ -+ seq_printf(m, "tick_hit:{\"enable\":%d; \"hit_count_thres\":%d; \"jiffies_num\":%lu}\n", -+ data->enable, data->hit_count_thres, data->jiffies_num); -+ return 0; -+} -+ -+static int tick_hit_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, tick_hit_show, pde_data(inode)); -+} -+ -+static ssize_t tick_hit_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, hit_count_thres_tmp, jiffies_num_tmp; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if(copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &hit_count_thres_tmp, &jiffies_num_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ tick_hit_params.hit_count_thres = hit_count_thres_tmp; -+ tick_hit_params.jiffies_num = jiffies_num_tmp; -+ tick_hit_params.enable = enable_tmp; -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(tick_hit, tick_hit_open, tick_hit_write); -+ -+static int __init hmbird_proc_init(void) -+{ -+ struct proc_dir_entry *hmbird_dir; -+ struct proc_dir_entry *load_track_dir; -+ struct proc_dir_entry *freq_gov_dir; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ -+ /* mkdir /proc/hmbird_sched */ -+ hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL); -+ if (!hmbird_dir) { -+ pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &scx_enable_proc_ops, -+ &scx_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &partial_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_stats); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbirdcore_debug); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_for_app); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &misfit_ds); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl_dbg); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpu7_tl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &cpu_cluster_masks_proc_ops, -+ &cpu_cluster_masks); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &save_gov_proc_ops, -+ &save_gov); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &watchdog_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isolate_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &iso_free_rescue); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_stats_proc_ops); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &yield_opt_proc_ops, -+ &yield_opt_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("boost_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &boost_policy_proc_ops, -+ &boost_policy_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("tick_hit", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &tick_hit_proc_ops, -+ &tick_hit_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbird_preempt_policy); -+ /* /proc/hmbird_sched--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_walt */ -+ load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir); -+ if (!load_track_dir) { -+ pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_walt--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &slim_walt_ctrl_proc_ops, -+ &slim_walt_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_dump); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_policy); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &sched_ravg_window_frame_per_sec_proc_ops, -+ &sched_ravg_window_frame_per_sec); -+ /* /proc/hmbird_sched/slim_walt--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_freq_gov */ -+ freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir); -+ if (!freq_gov_dir) { -+ pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_freq_gov--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &slim_gov_debug); -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &scx_gov_ctrl); -+ /* /proc/hmbird_sched/slim_freq_gov--end */ -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cluster_separate); -+ -+ return 0; -+} -+ -+device_initcall(hmbird_proc_init); -diff --git b/kernel/sched/hmbird/hmbird_sched_proc.h a/kernel/sched/hmbird/hmbird_sched_proc.h -new file mode 100755 -index 000000000000..e22bc75f1dd7 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched_proc.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SCHED_PROC_H__ -+#define __HMBIRD_SCHED_PROC_H__ -+ -+#include -+#include -+ -+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \ -+ do { \ -+ if (!proc_create(name, mode, parent, proc_ops)) { \ -+ pr_err("Error creating proc entry %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \ -+ do { \ -+ if (!proc_create_data(name, mode, parent, proc_ops, data)) { \ -+ pr_err("Error creating proc entry with data %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_PROC_OPS(name, open_func, write_func) \ -+ static const struct proc_ops name##_proc_ops = { \ -+ .proc_open = open_func, \ -+ .proc_write = write_func, \ -+ .proc_read = seq_read, \ -+ .proc_lseek = seq_lseek, \ -+ .proc_release = single_release, \ -+ } -+ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos); -+static int hmbird_common_show(struct seq_file *m, void *v); -+static int hmbird_common_open(struct inode *inode, struct file *file); -+ -+#endif -diff --git b/kernel/sched/hmbird/hmbird_shadow_tick.c a/kernel/sched/hmbird/hmbird_shadow_tick.c -new file mode 100755 -index 000000000000..2744cd42bbfd ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_shadow_tick.c -@@ -0,0 +1,198 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include -+#include <../../kernel/time/tick-sched.h> -+#include -+ -+#include "hmbird_shadow_tick.h" -+#include "slim.h" -+ -+#define HIGHRES_WATCH_CPU 0 -+ -+#include -+static bool shadow_tick_enable(void) {return highres_tick_ctrl; } -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; } -+#endif -+static bool shadow_tick_timer_init_flag; -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define shadow_tick_printk(fmt, args...) \ -+do { \ -+ int cpu = smp_processor_id(); \ -+ if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU) \ -+ trace_printk("hmbird shadow tick :"fmt, args); \ -+} while (0) -+#else -+#define shadow_tick_printk(fmt, args...) -+#endif -+ -+DEFINE_PER_CPU(struct hrtimer, stt); -+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu))) -+#define STOP_IDLE_TRIGGER (1) -+#define PERIODIC_TICK_TRIGGER (2) -+#define TICK_INTVAL (1000000ULL) -+#define HMBIRD_TICK_HIT_BOOST BIT(29) -+/* -+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms, -+ * so we can't restart hrtimer in scheduler tick. -+ */ -+static DEFINE_PER_CPU(u8, trigger_event); -+ -+/* -+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick. -+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler. -+ */ -+static DEFINE_PER_CPU(u8, tick_phase); -+ -+static inline void highres_timer_ctrl(bool enable, int cpu) -+{ -+ if (enable && hmbird_enabled()) { -+ if (!hrtimer_active(shadow_tick_timer(cpu))) -+ hrtimer_start(shadow_tick_timer(cpu), -+ ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED); -+ } else { -+ if (!enable) -+ hrtimer_cancel(shadow_tick_timer(cpu)); -+ } -+} -+ -+static inline void high_res_clear_phase(int cpu) -+{ -+ per_cpu(tick_phase, cpu) = 0; -+} -+ -+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer) -+{ -+ per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3; -+ if (per_cpu(tick_phase, cpu)) { -+ hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) { -+ per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+struct tick_hit_params tick_hit_params = { -+ .enable = 0, -+ .jiffies_num = 2, -+ .hit_count_thres = 6, -+}; -+ -+static void tick_hit_critical_task(struct task_struct *curr, struct rq *rq) -+{ -+ struct hmbird_entity *see = get_hmbird_ts(curr); -+ if (!tick_hit_params.enable || !see) -+ return; -+ -+ if ((!task_is_top_task(curr) || curr->pid != curr->tgid) && (curr->pid != scx_systemui_pid)) -+ return; -+ -+ if (see->tick_hit_count == 0) { -+ if (see->start_jiffies == 0) { -+ see->start_jiffies = jiffies; -+ see->tick_hit_count = 1; -+ } else { -+ see->start_jiffies = 0; /* status reset */ -+ see->tick_hit_count = 0; -+ } -+ } else { -+ if (time_before_eq(jiffies, see->start_jiffies + tick_hit_params.jiffies_num)) { -+ see->tick_hit_count++; -+ if (see->tick_hit_count >= tick_hit_params.hit_count_thres) { -+ cpufreq_update_util(rq, HMBIRD_TICK_HIT_BOOST); -+ } -+ } else { -+ see->tick_hit_count = 1; -+ see->start_jiffies = jiffies; -+ } -+ } -+} -+ -+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ struct rq_flags rf; -+ -+ rq_lock(rq, &rf); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ tick_hit_critical_task(curr, rq); -+ rq_unlock(rq, &rf); -+ -+ return highres_next_phase(cpu, timer); -+} -+ -+void shadow_tick_timer_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance; -+ } -+} -+ -+void start_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable()) { -+ if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER) -+ highres_timer_ctrl(false, cpu); -+ per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static void stop_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ per_cpu(trigger_event, cpu) = 0; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(false, cpu); -+} -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ stop_shadow_tick_timer(); -+} -+ -+void scheduler_tick_handler(void *unused, struct rq *rq) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ start_shadow_tick_timer(); -+} -+ -+static int __init hmbird_shadow_tick_init(void) -+{ -+ int ret = 0; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ shadow_tick_timer_init(); -+ shadow_tick_timer_init_flag = true; -+ return ret; -+} -+ -+device_initcall(hmbird_shadow_tick_init); -diff --git b/kernel/sched/hmbird/hmbird_shadow_tick.h a/kernel/sched/hmbird/hmbird_shadow_tick.h -new file mode 100755 -index 000000000000..0c26fd984f0a ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_shadow_tick.h -@@ -0,0 +1,11 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SHADOW_TICK_H__ -+#define __HMBIRD_SHADOW_TICK_H__ -+ -+#include -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+void scheduler_tick_handler(void *unused, struct rq *rq); -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state); -+#endif -diff --git b/kernel/sched/hmbird/hmbird_trace.h a/kernel/sched/hmbird/hmbird_trace.h -new file mode 100755 -index 000000000000..8468b406f347 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_trace.h -@@ -0,0 +1,92 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hmbird -+ -+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HMBIRD_H -+ -+#include -+#include -+#include -+ -+#define MAX_FATAL_INFO (64) -+ -+TRACE_EVENT(hmbird_fatal_info, -+ -+ TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info), -+ -+ TP_ARGS(type, pe, lnr, bnr, info), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, type) -+ __field(int, pe) -+ __field(int, lnr) -+ __field(int, bnr) -+ __array(char, info, MAX_FATAL_INFO)), -+ -+ TP_fast_assign( -+ __entry->type = type; -+ __entry->pe = pe; -+ __entry->lnr = lnr; -+ __entry->bnr = bnr; -+ memcpy(__entry->info, info, MAX_FATAL_INFO);), -+ -+ TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s", -+ __entry->type, __entry->pe, __entry->lnr, __entry->bnr, -+ __entry->info) -+); -+ -+TRACE_EVENT(hmbird_update_history, -+ -+ TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq, -+ struct task_struct *p, u32 runtime, int samples, int event), -+ -+ TP_ARGS(hmbird, rq, p, runtime, samples, event), -+ -+ TP_STRUCT__entry( -+ __array(char, comm, TASK_COMM_LEN) -+ __field(pid_t, pid) -+ __field(unsigned int, runtime) -+ __field(int, samples) -+ __field(int, event) -+ __field(unsigned int, demand) -+ __array(u32, hist, RAVG_HIST_SIZE) -+ __field(u16, task_util) -+ __field(int, cpu)), -+ -+ TP_fast_assign( -+ memcpy(__entry->comm, p->comm, TASK_COMM_LEN); -+ __entry->pid = p->pid; -+ __entry->runtime = runtime; -+ __entry->samples = samples; -+ __entry->event = event; -+ __entry->demand = hmbird->sts.demand; -+ memcpy(__entry->hist, hmbird->sts.sum_history, -+ RAVG_HIST_SIZE * sizeof(u32)); -+ __entry->task_util = hmbird->sts.demand_scaled; -+ __entry->cpu = rq->cpu;), -+ -+ TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d", -+ __entry->comm, __entry->pid, -+ __entry->runtime, __entry->samples, -+ __entry->event, -+ __entry->demand, -+ __entry->hist[0], __entry->hist[1], -+ __entry->hist[2], __entry->hist[3], -+ __entry->hist[4], -+ __entry->task_util, -+ __entry->cpu) -+); -+ -+#endif /*_TRACE_HMBIRD_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird -+ -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE hmbird_trace -+/* This part must be outside protection */ -+#include -diff --git b/kernel/sched/hmbird/hmbird_util_track.c a/kernel/sched/hmbird/hmbird_util_track.c -new file mode 100755 -index 000000000000..d520a8403401 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_util_track.c -@@ -0,0 +1,626 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+ -+#define CREATE_TRACE_POINTS -+#include "hmbird_trace.h" -+#undef CREATE_TRACE_POINTS -+ -+#define HMBIRD_DEBUG_PANIC (1 << 3) -+static bool init_irq_work_inited; -+ -+extern noinline int tracing_mark_write(const char *buf); -+ -+int hmbird_sched_ravg_window = 8000000; -+int new_hmbird_sched_ravg_window = 8000000; -+DEFINE_SPINLOCK(new_sched_ravg_window_lock); -+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+ -+static struct irq_work hmbird_slim_walt_irq_work; -+ -+/*Sysctl related interface*/ -+#define WINDOW_STATS_RECENT 0 -+#define WINDOW_STATS_MAX 1 -+#define WINDOW_STATS_MAX_RECENT_AVG 2 -+#define WINDOW_STATS_AVG 3 -+#define WINDOW_STATS_INVALID_POLICY 4 -+ -+ -+#define SCX_SCHED_CAPACITY_SHIFT 10 -+#define SCHED_ACCOUNT_WAIT_TIME 0 -+ -+atomic64_t hmbird_irq_work_lastq_ws; -+static u64 tick_sched_clock; -+ -+static inline u64 scale_exec_time(u64 delta, struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT; -+} -+ -+static inline u64 scale_time_to_util(u64 d) -+{ -+ do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT); -+ return d; -+} -+ -+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ delta = scale_exec_time(delta, rq); -+ sts->sum += delta; -+ if (unlikely(sts->sum > hmbird_sched_ravg_window)) -+ sts->sum = hmbird_sched_ravg_window; -+ -+ return delta; -+} -+ -+ -+static int -+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) -+{ -+ /* -+ * No need to bother updating task demand for the idle task. -+ */ -+ if (is_idle_task(p)) -+ return 0; -+ -+ /* -+ * When a task is waking up it is completing a segment of non-busy -+ * time. Likewise, if wait time is not treated as busy time, then -+ * when a task begins to run or is migrated, it is not running and -+ * is completing a segment of non-busy time. -+ */ -+ if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && -+ (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) -+ return 0; -+ -+ /* -+ * The idle exit time is not accounted for the first task _picked_ up to -+ * run on the idle CPU. -+ */ -+ if (event == PICK_NEXT_TASK && rq->curr == rq->idle) -+ return 0; -+ -+ /* -+ * TASK_UPDATE can be called on sleeping task, when its moved between -+ * related groups -+ */ -+ if (event == TASK_UPDATE) { -+ if (rq->curr == p) -+ return 1; -+ -+ return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; -+ } -+ -+ return 1; -+} -+ -+static void rollover_cpu_window(struct rq *rq, bool full_window) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 curr_sum = srq->curr_runnable_sum; -+ -+ if (unlikely(full_window)) -+ curr_sum = 0; -+ -+ srq->prev_runnable_sum = curr_sum; -+ srq->curr_runnable_sum = 0; -+} -+ -+static u64 -+update_window_start(struct rq *rq, u64 wallclock, int event) -+{ -+ s64 delta; -+ int nr_windows; -+ bool full_window; -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start = srq->window_start; -+ -+ if (wallclock < srq->latest_clock) -+ wallclock = srq->latest_clock; -+ delta = wallclock - srq->window_start; -+ if (delta < 0) { -+ delta = 0; -+ wallclock = srq->window_start; -+ } -+ srq->latest_clock = wallclock; -+ if (delta < hmbird_sched_ravg_window) -+ return old_window_start; -+ -+ nr_windows = div64_u64(delta, hmbird_sched_ravg_window); -+ srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window; -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ full_window = nr_windows > 1; -+ rollover_cpu_window(rq, full_window); -+ -+ return old_window_start; -+} -+ -+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -+ -+static inline unsigned int get_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static inline unsigned int cpu_cur_freq(int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cur; -+} -+ -+static void -+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, -+ u64 wallclock) -+{ -+ int cpu = cpu_of(rq); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * -+ arch_scale_cpu_capacity(cpu), get_max_freq(cpu)); -+} -+ -+/* -+ * Called when new window is starting for a task, to record cpu usage over -+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 -+ * when, say, a real-time task runs without preemption for several windows at a -+ * stretch. -+ */ -+static void update_history(struct rq *rq, struct task_struct *p, -+ u32 runtime, int samples, int event) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u32 *hist = &sts->sum_history[0]; -+ int i; -+ u32 max = 0, avg, demand; -+ u64 sum = 0; -+ u16 demand_scaled; -+ -+ /* Ignore windows where task had no activity */ -+ if (!runtime || is_idle_task(p) || !samples) -+ goto done; -+ -+ /* Push new 'runtime' value onto stack */ -+ for (; samples > 0; samples--) { -+ hist[sts->cidx] = runtime; -+ sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE; -+ } -+ -+ for (i = 0; i < RAVG_HIST_SIZE; i++) { -+ sum += hist[i]; -+ if (hist[i] > max) -+ max = hist[i]; -+ } -+ -+ sts->sum = 0; -+ avg = div64_u64(sum, RAVG_HIST_SIZE); -+ -+ switch (slim_walt_policy) { -+ case WINDOW_STATS_RECENT: -+ demand = runtime; -+ break; -+ case WINDOW_STATS_MAX: -+ demand = max; -+ break; -+ case WINDOW_STATS_AVG: -+ demand = avg; -+ break; -+ default: -+ demand = max(avg, runtime); -+ } -+ -+ demand_scaled = scale_time_to_util(demand); -+ -+ sts->demand = demand; -+ sts->demand_scaled = demand_scaled; -+ -+done: -+ return; -+} -+ -+ -+static u64 update_task_demand(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ -+ u64 delta, window_start = srq->window_start; -+ int new_window, nr_full_windows; -+ u32 window_size = hmbird_sched_ravg_window; -+ u64 runtime; -+ -+ new_window = mark_start < window_start; -+ if (!account_busy_for_task_demand(rq, p, event)) { -+ if (new_window) -+ /* -+ * If the time accounted isn't being accounted as -+ * busy time, and a new window started, only the -+ * previous window need be closed out with the -+ * pre-existing demand. Multiple windows may have -+ * elapsed, but since empty windows are dropped, -+ * it is not necessary to account those. -+ */ -+ update_history(rq, p, sts->sum, 1, event); -+ return 0; -+ } -+ -+ if (!new_window) { -+ /* -+ * The simple case - busy time contained within the existing -+ * window. -+ */ -+ return add_to_task_demand(rq, p, wallclock - mark_start); -+ } -+ -+ /* -+ * Busy time spans at least two windows. Temporarily rewind -+ * window_start to first window boundary after mark_start. -+ */ -+ delta = window_start - mark_start; -+ nr_full_windows = div64_u64(delta, window_size); -+ window_start -= (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (window_start - mark_start) first */ -+ runtime = add_to_task_demand(rq, p, window_start - mark_start); -+ -+ /* Push new sample(s) into task's demand history */ -+ update_history(rq, p, sts->sum, 1, event); -+ if (nr_full_windows) { -+ u64 scaled_window = scale_exec_time(window_size, rq); -+ -+ update_history(rq, p, scaled_window, nr_full_windows, event); -+ runtime += nr_full_windows * scaled_window; -+ } -+ -+ /* -+ * Roll window_start back to current to process any remainder -+ * in current window. -+ */ -+ window_start += (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (wallclock - window_start) next */ -+ mark_start = window_start; -+ runtime += add_to_task_demand(rq, p, wallclock - mark_start); -+ -+ return runtime; -+} -+ -+u16 slim_walt_cpu_util(int cpu) -+{ -+ u64 prev_runnable_sum; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ prev_runnable_sum = srq->prev_runnable_sum; -+ do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT); -+ -+ return (u16)prev_runnable_sum; -+} -+ -+static DEFINE_PER_CPU(u16, prev_cpu_util); -+static inline void cpu_util_update_systrace_c(int cpu) -+{ -+ char buf[256]; -+ u16 cpu_util = slim_walt_cpu_util(cpu); -+ -+ if (cpu_util != per_cpu(prev_cpu_util, cpu)) { -+ snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n", -+ cpu, cpu_util); -+ tracing_mark_write(buf); -+ per_cpu(prev_cpu_util, cpu) = cpu_util; -+ } -+} -+ -+ -+static inline int account_busy_for_cpu_time(struct rq *rq, -+ struct task_struct *p, -+ int event) -+{ -+ return !is_idle_task(p) && (event == PUT_PREV_TASK -+ || event == TASK_UPDATE); -+} -+ -+ -+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ int new_window, full_window = 0; -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 window_start = srq->window_start; -+ u32 window_size = srq->prev_window_size; -+ u64 delta; -+ u64 *curr_runnable_sum = &srq->curr_runnable_sum; -+ u64 *prev_runnable_sum = &srq->prev_runnable_sum; -+ -+ new_window = mark_start < window_start; -+ if (new_window) -+ full_window = (window_start - mark_start) >= window_size; -+ -+ -+ if (!account_busy_for_cpu_time(rq, p, event)) -+ goto done; -+ -+ -+ if (!new_window) { -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. No rollover -+ * since we didn't start a new window. An example of this is -+ * when a task starts execution and then sleeps within the -+ * same window. -+ */ -+ delta = wallclock - mark_start; -+ -+ delta = scale_exec_time(delta, rq); -+ *curr_runnable_sum += delta; -+ -+ goto done; -+ } -+ -+ /* -+ * situations below this need window rollover, -+ * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done -+ * in update_window_start() -+ * -+ * For task counters curr/prev_window[_cpu] are rolled over in the early part of -+ * this function. If full_window(s) have expired and time since last update needs -+ * to be accounted as busy time, set the prev to a complete window size time, else -+ * add the prev window portion. -+ * -+ * For task curr counters a new window has begun, always assign -+ */ -+ -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. A new window -+ * must have been started in udpate_window_start() -+ * If any of these three above conditions are true -+ * then this busy time can't be accounted as irqtime. -+ * -+ * Busy time for the idle task need not be accounted. -+ * -+ * An example of this would be a task that starts execution -+ * and then sleeps once a new window has begun. -+ */ -+ -+ /* -+ * A full window hasn't elapsed, account partial -+ * contribution to previous completed window. -+ */ -+ -+ -+ delta = full_window ? scale_exec_time(window_size, rq) : -+ scale_exec_time(window_start - mark_start, rq); -+ -+ *prev_runnable_sum += delta; -+ -+ /* Account piece of busy time in the current window. */ -+ delta = scale_exec_time(wallclock - window_start, rq); -+ *curr_runnable_sum += delta; -+ -+done: -+ if (slim_walt_dump && new_window) -+ cpu_util_update_systrace_c(rq->cpu); -+} -+ -+ -+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq) -+{ -+ u64 result; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 new_window_start = srq->window_start; -+ -+ if (old_window_start == new_window_start) -+ return; -+ -+ result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws, -+ old_window_start, new_window_start); -+ if (result != old_window_start) -+ return; -+ -+ if (likely(cpu_online(raw_smp_processor_id()))) -+ irq_work_queue(&hmbird_slim_walt_irq_work); -+ else -+ irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask)); -+} -+ -+/* -+ * In the core scheduler, most of the load update points update the rq_clock after -+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of -+ * obtaining the time, but to prevent the subsequent migration of the load update -+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock -+ * update here. -+ */ -+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p, -+ struct rq *rq, int event) -+{ -+ if (!(rq->clock_update_flags & RQCF_UPDATED)) -+ update_rq_clock(rq); -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock)); -+} -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start; -+ -+ if (!slim_walt_ctrl) -+ return; -+ -+ if (!srq->window_start || sts->mark_start == wallclock) -+ return; -+ -+ old_window_start = update_window_start(rq, wallclock, event); -+ -+ if (!sts->window_start) -+ sts->window_start = srq->window_start; -+ -+ if (!sts->mark_start) -+ goto done; -+ -+ update_task_rq_cpu_cycles(p, rq, event, wallclock); -+ update_task_demand(p, rq, event, wallclock); -+ update_cpu_busy_time(p, rq, event, wallclock); -+ -+ sts->window_start = srq->window_start; -+ -+done: -+ sts->mark_start = wallclock; -+ slim_walt_window_rollover_run_once(old_window_start, rq); -+} -+ -+static void slim_walt_irq_work(struct irq_work *irq_work) -+{ -+ cpumask_t lock_cpus; -+ struct hmbird_sched_rq_stats *srq; -+ struct rq *rq; -+ int cpu; -+ int level = 0; -+ u64 wc; -+ unsigned long flags; -+ -+ cpumask_copy(&lock_cpus, cpu_possible_mask); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ if (level == 0) -+ raw_spin_lock(&cpu_rq(cpu)->__lock); -+ else -+ raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); -+ level++; -+ } -+ -+ wc = sched_clock(); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ rq = cpu_rq(cpu); -+ hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc); -+ } -+ -+ cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER); -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) { -+ srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id()); -+ if (wc < srq->window_start + new_hmbird_sched_ravg_window) -+ hmbird_sched_ravg_window = new_hmbird_sched_ravg_window; -+ } -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ raw_spin_unlock(&cpu_rq(cpu)->__lock); -+ } -+} -+static void hmbird_sched_init_rq(struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ srq->task_exec_scale = 1024; -+ srq->window_start = 0; -+} -+ -+void hmbird_sched_init_task(struct task_struct *p) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ memset(sts, 0, sizeof(struct hmbird_sched_task_stats)); -+} -+ -+static void hmbird_sched_stats_init(void) -+{ -+ unsigned long flags; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ raw_spin_lock_irqsave(&rq->__lock, flags); -+ hmbird_sched_init_rq(rq); -+ raw_spin_unlock_irqrestore(&rq->__lock, flags); -+ } -+ slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG; -+ -+ if (false == init_irq_work_inited) { -+ init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work); -+ init_irq_work_inited = true; -+ } -+} -+ -+void hmbird_scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (unlikely(!tick_sched_clock)) { -+ /* -+ * Let the window begin 20us prior to the tick, -+ * that way we are guaranteed a rollover when the tick occurs. -+ * Use rq->clock directly instead of rq_clock() since -+ * we do not have the rq lock and -+ * rq->clock was updated in the tick callpath. -+ */ -+ if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000)) -+ return; -+ for_each_possible_cpu(cpu) { -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ srq->window_start = tick_sched_clock; -+ } -+ atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock); -+ } -+} -+ -+void slim_walt_enable(int enable) -+{ -+ if (1 == !!enable) { -+ hmbird_sched_stats_init(); -+ WRITE_ONCE(tick_sched_clock, 0); -+ } else -+ slim_walt_ctrl = 0; -+} -+ -+void slim_get_cpu_util(int cpu, u64 *util) -+{ -+ if (cpu < 0) -+ return; -+ -+ *util = slim_walt_cpu_util(cpu); -+} -+ -+void slim_get_task_util(struct task_struct *p, u64 *util) -+{ -+ if (p == NULL) -+ return; -+ -+ *util = get_hmbird_ts(p)->sts.demand_scaled; -+} -+ -+void sched_ravg_window_change(int frame_per_sec) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec; -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+} -diff --git b/kernel/sched/hmbird/hmbird_util_track.h a/kernel/sched/hmbird/hmbird_util_track.h -new file mode 100644 -index 000000000000..17b85df7f83b ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_util_track.h -@@ -0,0 +1,33 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef __HMBIRD_UTIL_TRACK_H__ -+#define __HMBIRD_UTIL_TRACK_H__ -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock); -+void hmbird_sched_init_task(struct task_struct *p); -+void slim_walt_enable(int enable); -+void slim_get_cpu_util(int cpu, u64 *util); -+void slim_get_task_util(struct task_struct *p, u64 *util); -+ -+extern atomic64_t hmbird_irq_work_lastq_ws; -+ -+enum task_event { -+ PUT_PREV_TASK = 0, -+ PICK_NEXT_TASK = 1, -+ TASK_WAKE = 2, -+ TASK_MIGRATE = 3, -+ TASK_UPDATE = 4, -+ IRQ_UPDATE = 5, -+}; -+ -+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+#endif /* __HMBIRD_UTIL_TRACK_H__ */ -diff --git b/kernel/sched/hmbird/slim.h a/kernel/sched/hmbird/slim.h -new file mode 100755 -index 000000000000..dffe6506658e ---- /dev/null -+++ a/kernel/sched/hmbird/slim.h -@@ -0,0 +1,56 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+ -+#ifndef __SLIM_H -+#define __SLIM_H -+ -+extern atomic_t __hmbird_ops_enabled; -+extern atomic_t non_hmbird_task; -+extern int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+extern int heartbeat; -+extern int heartbeat_enable; -+extern int watchdog_enable; -+extern int isolate_ctrl; -+extern int parctrl_high_ratio; -+extern int parctrl_low_ratio; -+extern int isoctrl_high_ratio; -+extern int isoctrl_low_ratio; -+extern int iso_free_rescue; -+extern int yield_opt; -+ -+extern enum hmbird_switch_type sw_type; -+extern noinline int tracing_mark_write(const char *buf); -+int task_top_id(struct task_struct *p); -+void stats_print(char *buf, int len); -+void hmbird_skip_yield(long *skip); -+extern spinlock_t hmbird_tasks_lock; -+extern int scx_systemui_pid; -+ -+struct yield_opt_params { -+ int enable; -+ int frame_per_sec; -+ u64 frame_time_ns; -+ int yield_headroom; -+}; -+ -+extern struct yield_opt_params yield_opt_params; -+ -+struct tick_hit_params { -+ int enable; -+ unsigned long jiffies_num; -+ int hit_count_thres; -+}; -+ -+extern struct tick_hit_params tick_hit_params; -+ -+struct boost_policy_params { -+ int enable; -+ unsigned int bottom_freq; -+ int boost_weight; -+}; -+ -+extern struct boost_policy_params boost_policy_params; -+ -+#define MAX_GOV_LEN (16) -+extern char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+#endif -diff --git b/kernel/sched/idle.c a/kernel/sched/idle.c -index b33cefeb4188..487255ac6832 100755 ---- b/kernel/sched/idle.c -+++ a/kernel/sched/idle.c -@@ -408,13 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl - - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) - { -- scx_update_idle(rq, false); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, false); -+#endif - } - - static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) - { - update_idle_core(rq); -- scx_update_idle(rq, true); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, true); -+#endif - schedstat_inc(rq->sched_goidle); - } - -diff --git b/kernel/sched/sched.h a/kernel/sched/sched.h -index 509e8dc616ab..a7e9caea1efe 100755 ---- b/kernel/sched/sched.h -+++ a/kernel/sched/sched.h -@@ -188,18 +188,13 @@ static inline int idle_policy(int policy) - return policy == SCHED_IDLE; - } - --static inline int normal_policy(int policy) --{ --#ifdef CONFIG_SCHED_CLASS_EXT -- if (policy == SCHED_EXT) -- return true; --#endif -- return policy == SCHED_NORMAL; --} -- - static inline int fair_policy(int policy) - { -- return normal_policy(policy) || policy == SCHED_BATCH; -+#ifdef CONFIG_HMBIRD_SCHED -+ return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#else -+ return policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#endif - } - - static inline int rt_policy(int policy) -@@ -247,25 +242,7 @@ static inline void update_avg(u64 *avg, u64 sample) - #define shr_bound(val, shift) \ - (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) - --/* -- * cgroup weight knobs should use the common MIN, DFL and MAX values which are -- * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it -- * maps pretty well onto the shares value used by scheduler and the round-trip -- * conversions preserve the original value over the entire range. -- */ --static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) --{ -- return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); --} -- --static inline unsigned long sched_weight_to_cgroup(unsigned long weight) --{ -- return clamp_t(unsigned long, -- DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -- CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); --} -- --/* -+/* - * !! For sched_setattr_nocheck() (kernel) only !! - * - * This is actually gross. :( -@@ -532,10 +509,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) { } - #endif /* CONFIG_SMP */ - #else /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_HMBIRD_SCHED - static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) - { - return 0; - } -+#endif - #endif /* CONFIG_FAIR_GROUP_SCHED */ - - #else /* CONFIG_CGROUP_SCHED */ -@@ -700,29 +679,6 @@ struct cfs_rq { - #endif /* CONFIG_FAIR_GROUP_SCHED */ - }; - --#ifdef CONFIG_SCHED_CLASS_EXT --/* scx_rq->flags, protected by the rq lock */ --enum scx_rq_flags { -- SCX_RQ_CAN_STOP_TICK = 1 << 0, --}; -- --struct scx_rq { -- struct scx_dispatch_q local_dsq; -- struct list_head watchdog_list; -- u64 ops_qseq; -- u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -- u32 nr_running; -- u32 flags; -- bool cpu_released; -- cpumask_var_t cpus_to_kick; -- cpumask_var_t cpus_to_preempt; -- cpumask_var_t cpus_to_wait; -- u64 pnt_seq; -- struct irq_work kick_cpus_irq_work; -- struct rq *rq; --}; --#endif /* CONFIG_SCHED_CLASS_EXT */ -- - static inline int rt_bandwidth_enabled(void) - { - return sysctl_sched_rt_runtime >= 0; -@@ -1258,11 +1214,7 @@ struct rq { - - ANDROID_OEM_DATA_ARRAY(1, 16); - --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, struct scx_rq *scx); --#else - ANDROID_KABI_RESERVE(1); --#endif - ANDROID_KABI_RESERVE(2); - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); -@@ -2340,11 +2292,6 @@ struct affinity_context { - unsigned int flags; - }; - --enum rq_onoff_reason { -- RQ_ONOFF_HOTPLUG, /* CPU is going on/offline */ -- RQ_ONOFF_TOPOLOGY, /* sched domain topology update */ --}; -- - struct sched_class { - - #ifdef CONFIG_UCLAMP_TASK -@@ -2551,7 +2498,6 @@ extern void init_sched_rt_class(void); - extern void init_sched_fair_class(void); - - extern void reweight_task(struct task_struct *p, int prio); --extern void __setscheduler_prio(struct task_struct *p, int prio); - - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); -@@ -2632,53 +2578,6 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) - extern void activate_task(struct rq *rq, struct task_struct *p, int flags); - extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); - --struct sched_change_guard { -- struct task_struct *p; -- struct rq *rq; -- bool queued; -- bool running; -- bool done; --}; -- --extern struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -- --extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags); -- --/** -- * SCHED_CHANGE_BLOCK - Nested block for task attribute updates -- * @__rq: Runqueue the target task belongs to -- * @__p: Target task -- * @__flags: DEQUEUE/ENQUEUE_* flags -- * -- * A task may need to be dequeued and put_prev_task'd for attribute updates and -- * set_next_task'd and re-enqueued afterwards. This helper defines a nested -- * block which automatically handles these preparation and cleanup operations. -- * -- * SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- * update_attribute(p); -- * ... -- * } -- * -- * If @__flags is a variable, the variable may be updated in the block body and -- * the updated value will be used when re-enqueueing @p. -- * -- * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling -- * update_rq_clock() beforehand. Otherwise, the rq clock is automatically -- * updated iff the task needs to be dequeued and re-enqueued. Only the former -- * case guarantees that the rq clock is up-to-date inside and after the block. -- */ --#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -- for (struct sched_change_guard __cg = \ -- sched_change_guard_init(__rq, __p, __flags); \ -- !__cg.done; sched_change_guard_fini(&__cg, __flags)) -- --extern void check_class_changing(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class); --extern void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio); -- - extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); - - #ifdef CONFIG_PREEMPT_RT -@@ -3710,6 +3609,8 @@ static inline bool cpu_busy_with_softirqs(int cpu) - } - #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ - --#include "ext.h" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird.h" -+#endif - - #endif /* _KERNEL_SCHED_SCHED_H */ -diff --git b/kernel/time/tick-sched.c a/kernel/time/tick-sched.c -index 998c2eefe173..c13772ee823c 100755 ---- b/kernel/time/tick-sched.c -+++ a/kernel/time/tick-sched.c -@@ -34,6 +34,10 @@ - - #include - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "../sched/hmbird/hmbird_shadow_tick.h" -+#endif -+ - /* - * Per-CPU nohz control structure - */ -@@ -1124,6 +1128,9 @@ void tick_nohz_idle_stop_tick(void) - ktime_t expires; - - trace_android_vh_tick_nohz_idle_stop_tick(NULL); -+#ifdef CONFIG_HMBIRD_SCHED -+ android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL); -+#endif - /* - * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the - * tick timer expiration time is known already. -diff --git b/tools/sched_ext/.gitignore a/tools/sched_ext/.gitignore -deleted file mode 100644 -index a3240f9f7eba..000000000000 ---- b/tools/sched_ext/.gitignore -+++ /dev/null -@@ -1,9 +0,0 @@ --scx_example_simple --scx_example_qmap --scx_example_central --scx_example_pair --scx_example_flatcg --scx_example_userland --*.skel.h --*.subskel.h --/tools/ -diff --git b/tools/sched_ext/Makefile a/tools/sched_ext/Makefile -deleted file mode 100644 -index 73c43782837d..000000000000 ---- b/tools/sched_ext/Makefile -+++ /dev/null -@@ -1,216 +0,0 @@ --# SPDX-License-Identifier: GPL-2.0 --# Copyright (c) 2022 Meta Platforms, Inc. and affiliates. --include ../build/Build.include --include ../scripts/Makefile.arch --include ../scripts/Makefile.include -- --ifneq ($(LLVM),) --ifneq ($(filter %/,$(LLVM)),) --LLVM_PREFIX := $(LLVM) --else ifneq ($(filter -%,$(LLVM)),) --LLVM_SUFFIX := $(LLVM) --endif -- --CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi --CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu --CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl --CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu --CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu --CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu --CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu --CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu --CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu --CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) -- --ifeq ($(CROSS_COMPILE),) --ifeq ($(CLANG_TARGET_FLAGS),) --$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk --else --CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) --endif # CLANG_TARGET_FLAGS --else --CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) --endif # CROSS_COMPILE -- --CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as --else --CC := $(CROSS_COMPILE)gcc --endif # LLVM -- --CURDIR := $(abspath .) --TOOLSDIR := $(abspath ..) --LIBDIR := $(TOOLSDIR)/lib --BPFDIR := $(LIBDIR)/bpf --TOOLSINCDIR := $(TOOLSDIR)/include --BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool --APIDIR := $(TOOLSINCDIR)/uapi --GENDIR := $(abspath ../../include/generated) --GENHDR := $(GENDIR)/autoconf.h -- --SCRATCH_DIR := $(CURDIR)/tools --BUILD_DIR := $(SCRATCH_DIR)/build --INCLUDE_DIR := $(SCRATCH_DIR)/include --BPFOBJ_DIR := $(BUILD_DIR)/libbpf --BPFOBJ := $(BPFOBJ_DIR)/libbpf.a --ifneq ($(CROSS_COMPILE),) --HOST_BUILD_DIR := $(BUILD_DIR)/host --HOST_SCRATCH_DIR := host-tools --HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include --else --HOST_BUILD_DIR := $(BUILD_DIR) --HOST_SCRATCH_DIR := $(SCRATCH_DIR) --HOST_INCLUDE_DIR := $(INCLUDE_DIR) --endif --HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a --RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids --DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -- --VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ -- $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ -- ../../vmlinux \ -- /sys/kernel/btf/vmlinux \ -- /boot/vmlinux-$(shell uname -r) --VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) --ifeq ($(VMLINUX_BTF),) --$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") --endif -- --BPFTOOL ?= $(DEFAULT_BPFTOOL) -- --ifneq ($(wildcard $(GENHDR)),) -- GENFLAGS := -DHAVE_GENHDR --endif -- --CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ -- -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -- -I$(TOOLSINCDIR) -I$(APIDIR) -- --CARGOFLAGS := --release -- --# Silence some warnings when compiled with clang --ifneq ($(LLVM),) --CFLAGS += -Wno-unused-command-line-argument --endif -- --LDFLAGS = -lelf -lz -lpthread -- --IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - &1 \ -- | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ --$(shell $(1) -dM -E - $@ --else -- $(call msg,CP,,$@) -- $(Q)cp "$(VMLINUX_H)" $@ --endif -- --%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \ -- | $(BPFOBJ) -- $(call msg,CLNG-BPF,,$@) -- $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ -- --%.skel.h: %.bpf.o $(BPFTOOL) -- $(call msg,GEN-SKEL,,$@) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) -- $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) -- $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@ -- $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h) -- --scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_userland: scx_example_userland.c scx_example_userland.skel.h \ -- scx_example_userland_common.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR) --atropos: export ATROPOS_CLANG = $(CLANG) --atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS) --atropos: $(INCLUDE_DIR)/vmlinux.h -- cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS) -- --clean: -- cargo clean --manifest-path=atropos/Cargo.toml -- rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) -- rm -f *.o *.bpf.o *.skel.h *.subskel.h -- rm -f scx_example_simple scx_example_qmap scx_example_central \ -- scx_example_pair scx_example_flatcg scx_example_userland -- --.PHONY: all atropos clean -- --# delete failed targets --.DELETE_ON_ERROR: -- --# keep intermediate (.skel.h, .bpf.o, etc) targets --.SECONDARY: -diff --git b/tools/sched_ext/atropos/.gitignore a/tools/sched_ext/atropos/.gitignore -deleted file mode 100644 -index 186dba259ec2..000000000000 ---- b/tools/sched_ext/atropos/.gitignore -+++ /dev/null -@@ -1,3 +0,0 @@ --src/bpf/.output --Cargo.lock --target -diff --git b/tools/sched_ext/atropos/Cargo.toml a/tools/sched_ext/atropos/Cargo.toml -deleted file mode 100644 -index 7462a836d53d..000000000000 ---- b/tools/sched_ext/atropos/Cargo.toml -+++ /dev/null -@@ -1,28 +0,0 @@ --[package] --name = "scx_atropos" --version = "0.5.0" --authors = ["Dan Schatzberg ", "Meta"] --edition = "2021" --description = "Userspace scheduling with BPF" --license = "GPL-2.0-only" -- --[dependencies] --anyhow = "1.0.65" --bitvec = { version = "1.0", features = ["serde"] } --clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } --ctrlc = { version = "3.1", features = ["termination"] } --fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"} --hex = "0.4.3" --libbpf-rs = "0.19.1" --libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] } --libc = "0.2.137" --log = "0.4.17" --ordered-float = "3.4.0" --simplelog = "0.12.0" -- --[build-dependencies] --bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false } --libbpf-cargo = "0.13.0" -- --[features] --enable_backtrace = [] -diff --git b/tools/sched_ext/atropos/build.rs a/tools/sched_ext/atropos/build.rs -deleted file mode 100644 -index 26e792c5e17e..000000000000 ---- b/tools/sched_ext/atropos/build.rs -+++ /dev/null -@@ -1,70 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --extern crate bindgen; -- --use std::env; --use std::fs::create_dir_all; --use std::path::Path; --use std::path::PathBuf; -- --use libbpf_cargo::SkeletonBuilder; -- --const HEADER_PATH: &str = "src/bpf/atropos.h"; -- --fn bindgen_atropos() { -- // Tell cargo to invalidate the built crate whenever the wrapper changes -- println!("cargo:rerun-if-changed={}", HEADER_PATH); -- -- // The bindgen::Builder is the main entry point -- // to bindgen, and lets you build up options for -- // the resulting bindings. -- let bindings = bindgen::Builder::default() -- // The input header we would like to generate -- // bindings for. -- .header(HEADER_PATH) -- // Tell cargo to invalidate the built crate whenever any of the -- // included header files changed. -- .parse_callbacks(Box::new(bindgen::CargoCallbacks)) -- // Finish the builder and generate the bindings. -- .generate() -- // Unwrap the Result and panic on failure. -- .expect("Unable to generate bindings"); -- -- // Write the bindings to the $OUT_DIR/bindings.rs file. -- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); -- bindings -- .write_to_file(out_path.join("atropos-sys.rs")) -- .expect("Couldn't write bindings!"); --} -- --fn gen_bpf_sched(name: &str) { -- let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap(); -- let clang = env::var("ATROPOS_CLANG").unwrap(); -- eprintln!("{}", clang); -- let outpath = format!("./src/bpf/.output/{}.skel.rs", name); -- let skel = Path::new(&outpath); -- let src = format!("./src/bpf/{}.bpf.c", name); -- SkeletonBuilder::new() -- .source(src.clone()) -- .clang(clang) -- .clang_args(bpf_cflags) -- .build_and_generate(&skel) -- .unwrap(); -- println!("cargo:rerun-if-changed={}", src); --} -- --fn main() { -- bindgen_atropos(); -- // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton. -- // Reasons are because the generated skeleton contains compiler attributes -- // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]` -- // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside -- // the path attribute either (see https://github.com/rust-lang/rust/pull/83366). -- // -- // However, there is hope! When the above feature stabilizes we can clean this -- // all up. -- create_dir_all("./src/bpf/.output").unwrap(); -- gen_bpf_sched("atropos"); --} -diff --git b/tools/sched_ext/atropos/rustfmt.toml a/tools/sched_ext/atropos/rustfmt.toml -deleted file mode 100644 -index b7258ed0a8d8..000000000000 ---- b/tools/sched_ext/atropos/rustfmt.toml -+++ /dev/null -@@ -1,8 +0,0 @@ --# Get help on options with `rustfmt --help=config` --# Please keep these in alphabetical order. --edition = "2021" --group_imports = "StdExternalCrate" --imports_granularity = "Item" --merge_derives = false --use_field_init_shorthand = true --version = "Two" -diff --git b/tools/sched_ext/atropos/src/atropos_sys.rs a/tools/sched_ext/atropos/src/atropos_sys.rs -deleted file mode 100644 -index bbeaf856d40e..000000000000 ---- b/tools/sched_ext/atropos/src/atropos_sys.rs -+++ /dev/null -@@ -1,10 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#![allow(non_upper_case_globals)] --#![allow(non_camel_case_types)] --#![allow(non_snake_case)] --#![allow(dead_code)] -- --include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs")); -diff --git b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -deleted file mode 100644 -index 3905a403e940..000000000000 ---- b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -+++ /dev/null -@@ -1,743 +0,0 @@ --/* Copyright (c) Meta Platforms, Inc. and affiliates. */ --/* -- * This software may be used and distributed according to the terms of the -- * GNU General Public License version 2. -- * -- * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF -- * part does simple round robin in each domain and the userspace part -- * calculates the load factor of each domain and tells the BPF part how to load -- * balance the domains. -- * -- * Every task has an entry in the task_data map which lists which domain the -- * task belongs to. When a task first enters the system (atropos_prep_enable), -- * they are round-robined to a domain. -- * -- * atropos_select_cpu is the primary scheduling logic, invoked when a task -- * becomes runnable. The lb_data map is populated by userspace to inform the BPF -- * scheduler that a task should be migrated to a new domain. Otherwise, the task -- * is scheduled in priority order as follows: -- * * The current core if the task was woken up synchronously and there are idle -- * cpus in the system -- * * The previous core, if idle -- * * The pinned-to core if the task is pinned to a specific core -- * * Any idle cpu in the domain -- * -- * If none of the above conditions are met, then the task is enqueued to a -- * dispatch queue corresponding to the domain (atropos_enqueue). -- * -- * atropos_dispatch will attempt to consume a task from its domain's -- * corresponding dispatch queue (this occurs after scheduling any tasks directly -- * assigned to it due to the logic in atropos_select_cpu). If no task is found, -- * then greedy load stealing will attempt to find a task on another dispatch -- * queue to run. -- * -- * Load balancing is almost entirely handled by userspace. BPF populates the -- * task weight, dom mask and current dom in the task_data map and executes the -- * load balance based on userspace populating the lb_data map. -- */ --#include "../../../scx_common.bpf.h" --#include "atropos.h" -- --#include --#include --#include --#include --#include --#include -- --char _license[] SEC("license") = "GPL"; -- --/* -- * const volatiles are set during initialization and treated as consts by the -- * jit compiler. -- */ -- --/* -- * Domains and cpus -- */ --const volatile __u32 nr_doms = 32; /* !0 for veristat, set during init */ --const volatile __u32 nr_cpus = 64; /* !0 for veristat, set during init */ --const volatile __u32 cpu_dom_id_map[MAX_CPUS]; --const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; -- --const volatile bool kthreads_local; --const volatile bool fifo_sched; --const volatile bool switch_partial; --const volatile __u32 greedy_threshold; -- --/* base slice duration */ --const volatile __u64 slice_us = 20000; -- --/* -- * Exit info -- */ --int exit_type = SCX_EXIT_NONE; --char exit_msg[SCX_EXIT_MSG_LEN]; -- --struct pcpu_ctx { -- __u32 dom_rr_cur; /* used when scanning other doms */ -- -- /* libbpf-rs does not respect the alignment, so pad out the struct explicitly */ -- __u8 _padding[CACHELINE_SIZE - sizeof(u64)]; --} __attribute__((aligned(CACHELINE_SIZE))); -- --struct pcpu_ctx pcpu_ctx[MAX_CPUS]; -- --/* -- * Domain context -- */ --struct dom_ctx { -- struct bpf_cpumask __kptr *cpumask; -- u64 vtime_now; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __type(key, u32); -- __type(value, struct dom_ctx); -- __uint(max_entries, MAX_DOMS); -- __uint(map_flags, 0); --} dom_ctx SEC(".maps"); -- --/* -- * Statistics -- */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, ATROPOS_NR_STATS); --} stats SEC(".maps"); -- --static inline void stat_add(enum stat_idx idx, u64 addend) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p) += addend; --} -- --/* Map pid -> task_ctx */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, struct task_ctx); -- __uint(max_entries, 1000000); -- __uint(map_flags, 0); --} task_data SEC(".maps"); -- --/* -- * This is populated from userspace to indicate which pids should be reassigned -- * to new doms. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, u32); -- __uint(max_entries, 1000); -- __uint(map_flags, 0); --} lb_data SEC(".maps"); -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p, -- u32 new_dom_id) --{ -- struct dom_ctx *old_domc, *new_domc; -- struct bpf_cpumask *d_cpumask, *t_cpumask; -- u32 old_dom_id = task_ctx->dom_id; -- s64 vtime_delta; -- -- old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id); -- if (!old_domc) { -- scx_bpf_error("No dom%u", old_dom_id); -- return false; -- } -- -- vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now; -- -- new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id); -- if (!new_domc) { -- scx_bpf_error("No dom%u", new_dom_id); -- return false; -- } -- -- d_cpumask = new_domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to get domain %u cpumask kptr", -- new_dom_id); -- return false; -- } -- -- t_cpumask = task_ctx->cpumask; -- if (!t_cpumask) { -- scx_bpf_error("Failed to look up task cpumask"); -- return false; -- } -- -- /* -- * set_cpumask might have happened between userspace requesting LB and -- * here and @p might not be able to run in @dom_id anymore. Verify. -- */ -- if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- p->cpus_ptr)) { -- p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta; -- task_ctx->dom_id = new_dom_id; -- bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask, -- p->cpus_ptr); -- } -- -- return task_ctx->dom_id == new_dom_id; --} -- --s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu, -- u32 wake_flags) --{ -- s32 cpu; -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- struct bpf_cpumask *p_cpumask; -- -- if (!task_ctx) -- return -ENOENT; -- -- if (kthreads_local && -- (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- cpu = prev_cpu; -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local dsq of the waker. -- */ -- if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) { -- struct task_struct *current = (void *)bpf_get_current_task(); -- -- if (!(BPF_CORE_READ(current, flags) & PF_EXITING) && -- task_ctx->dom_id < MAX_DOMS) { -- struct dom_ctx *domc; -- struct bpf_cpumask *d_cpumask; -- const struct cpumask *idle_cpumask; -- bool has_idle; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id); -- if (!domc) { -- scx_bpf_error("Failed to find dom%u", -- task_ctx->dom_id); -- return prev_cpu; -- } -- d_cpumask = domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to acquire domain %u cpumask kptr", -- task_ctx->dom_id); -- return prev_cpu; -- } -- -- idle_cpumask = scx_bpf_get_idle_cpumask(); -- -- has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- idle_cpumask); -- -- scx_bpf_put_idle_cpumask(idle_cpumask); -- -- if (has_idle) { -- cpu = bpf_get_smp_processor_id(); -- if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- stat_add(ATROPOS_STAT_WAKE_SYNC, 1); -- goto local; -- } -- } -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- stat_add(ATROPOS_STAT_PREV_IDLE, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- /* If only one core is allowed, dispatch */ -- if (p->nr_cpus_allowed == 1) { -- stat_add(ATROPOS_STAT_PINNED, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) -- return -ENOENT; -- -- /* If there is an eligible idle CPU, dispatch directly */ -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- if (cpu >= 0) { -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * @prev_cpu may be in a different domain. Returning an out-of-domain -- * CPU can lead to stalls as all in-domain CPUs may be idle by the time -- * @p gets enqueued. -- */ -- if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask)) -- cpu = prev_cpu; -- else -- cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask); -- -- return cpu; -- --local: -- task_ctx->dispatch_local = true; -- return cpu; --} -- --void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- u32 *new_dom; -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- new_dom = bpf_map_lookup_elem(&lb_data, &pid); -- if (new_dom && *new_dom != task_ctx->dom_id && -- task_set_dsq(task_ctx, p, *new_dom)) { -- struct bpf_cpumask *p_cpumask; -- s32 cpu; -- -- stat_add(ATROPOS_STAT_LOAD_BALANCE, 1); -- -- /* -- * If dispatch_local is set, We own @p's idle state but we are -- * not gonna put the task in the associated local dsq which can -- * cause the CPU to stall. Kick it. -- */ -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0); -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) { -- scx_bpf_error("Failed to get task_ctx->cpumask"); -- return; -- } -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- } -- -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags); -- return; -- } -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000, -- enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- u32 dom_id = task_ctx->dom_id; -- struct dom_ctx *domc; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, domc->vtime_now - slice_us * 1000)) -- vtime = domc->vtime_now - slice_us * 1000; -- -- scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --static u32 cpu_to_dom_id(s32 cpu) --{ -- const volatile u32 *dom_idp; -- -- if (nr_doms <= 1) -- return 0; -- -- dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]); -- if (!dom_idp) -- return MAX_DOMS; -- -- return *dom_idp; --} -- --static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) --{ -- s32 cpu; -- -- if (dom_id >= MAX_DOMS) -- return false; -- -- bpf_for(cpu, 0, nr_cpus) { -- if (bpf_cpumask_test_cpu(cpu, cpumask) && -- (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64)))) -- return true; -- } -- return false; --} -- --static u32 dom_rr_next(s32 cpu) --{ -- struct pcpu_ctx *pcpuc; -- u32 dom_id; -- -- pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]); -- if (!pcpuc) -- return 0; -- -- dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms; -- -- if (dom_id == cpu_to_dom_id(cpu)) -- dom_id = (dom_id + 1) % nr_doms; -- -- pcpuc->dom_rr_cur = dom_id; -- return dom_id; --} -- --void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 dom = cpu_to_dom_id(cpu); -- -- if (scx_bpf_consume(dom)) { -- stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1); -- return; -- } -- -- if (!greedy_threshold) -- return; -- -- bpf_repeat(nr_doms - 1) { -- u32 dom_id = dom_rr_next(cpu); -- -- if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold && -- scx_bpf_consume(dom_id)) { -- stat_add(ATROPOS_STAT_GREEDY, 1); -- break; -- } -- } --} -- --void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_at = bpf_ktime_get_ns(); --} -- --void BPF_STRUCT_OPS(atropos_running, struct task_struct *p) --{ -- struct task_ctx *taskc; -- struct dom_ctx *domc; -- pid_t pid = p->pid; -- u32 dom_id; -- -- if (fifo_sched) -- return; -- -- taskc = bpf_map_lookup_elem(&task_data, &pid); -- if (!taskc) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- dom_id = taskc->dom_id; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(domc->vtime_now, p->scx.dsq_vtime)) -- domc->vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at; -- task_ctx->runnable_at = 0; --} -- --void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->weight = weight; --} -- --struct pick_task_domain_loop_ctx { -- struct task_struct *p; -- const struct cpumask *cpumask; -- u64 dom_mask; -- u32 dom_rr_base; -- u32 dom_id; --}; -- --static int pick_task_domain_loopfn(u32 idx, void *data) --{ -- struct pick_task_domain_loop_ctx *lctx = data; -- u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms; -- -- if (dom_id >= MAX_DOMS) -- return 1; -- -- if (cpumask_intersects_domain(lctx->cpumask, dom_id)) { -- lctx->dom_mask |= 1LLU << dom_id; -- if (lctx->dom_id == MAX_DOMS) -- lctx->dom_id = dom_id; -- } -- return 0; --} -- --static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- struct pick_task_domain_loop_ctx lctx = { -- .p = p, -- .cpumask = cpumask, -- .dom_id = MAX_DOMS, -- }; -- s32 cpu = bpf_get_smp_processor_id(); -- -- if (cpu < 0 || cpu >= MAX_CPUS) -- return MAX_DOMS; -- -- lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur); -- -- bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0); -- task_ctx->dom_mask = lctx.dom_mask; -- -- return lctx.dom_id; --} -- --static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- u32 dom_id = 0; -- -- if (nr_doms > 1) -- dom_id = pick_task_domain(task_ctx, p, cpumask); -- -- if (!task_set_dsq(task_ctx, p, dom_id)) -- scx_bpf_error("Failed to set domain %d for %s[%d]", -- dom_id, p->comm, p->pid); --} -- --void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_set_domain(task_ctx, p, cpumask); --} -- --s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct bpf_cpumask *cpumask; -- struct task_ctx task_ctx, *map_value; -- long ret; -- pid_t pid; -- -- memset(&task_ctx, 0, sizeof(task_ctx)); -- -- pid = p->pid; -- ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return ret; -- } -- -- /* -- * Read the entry from the map immediately so we can add the cpumask -- * with bpf_kptr_xchg(). -- */ -- map_value = bpf_map_lookup_elem(&task_data, &pid); -- if (!map_value) -- /* Should never happen -- it was just inserted above. */ -- return -EINVAL; -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- bpf_map_delete_elem(&task_data, &pid); -- return -ENOMEM; -- } -- -- cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask); -- if (cpumask) { -- /* Should never happen as we just inserted it above. */ -- bpf_cpumask_release(cpumask); -- bpf_map_delete_elem(&task_data, &pid); -- return -EINVAL; -- } -- -- task_set_domain(map_value, p, p->cpus_ptr); -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p) --{ -- pid_t pid = p->pid; -- long ret = bpf_map_delete_elem(&task_data, &pid); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return; -- } --} -- --static int create_dom_dsq(u32 idx, void *data) --{ -- struct dom_ctx domc_init = {}, *domc; -- struct bpf_cpumask *cpumask; -- u32 cpu, dom_id = idx; -- s32 ret; -- -- ret = scx_bpf_create_dsq(dom_id, -1); -- if (ret < 0) { -- scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret); -- return 1; -- } -- -- ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0); -- if (ret) { -- scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret); -- return 1; -- } -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- /* Should never happen, we just inserted it above. */ -- scx_bpf_error("No dom%u", dom_id); -- return 1; -- } -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id); -- return 1; -- } -- -- for (cpu = 0; cpu < MAX_CPUS; cpu++) { -- const volatile __u64 *dmask; -- -- dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]); -- if (!dmask) { -- scx_bpf_error("array index error"); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- if (*dmask & (1LLU << (cpu % 64))) -- bpf_cpumask_set_cpu(cpu, cpumask); -- } -- -- cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask); -- if (cpumask) { -- scx_bpf_error("Domain %u was already present", dom_id); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(atropos_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- bpf_loop(nr_doms, create_dom_dsq, NULL, 0); -- -- for (u32 i = 0; i < nr_cpus; i++) -- pcpu_ctx[i].dom_rr_cur = i; -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg); -- exit_type = ei->type; --} -- --SEC(".struct_ops") --struct sched_ext_ops atropos = { -- .select_cpu = (void *)atropos_select_cpu, -- .enqueue = (void *)atropos_enqueue, -- .dispatch = (void *)atropos_dispatch, -- .runnable = (void *)atropos_runnable, -- .running = (void *)atropos_running, -- .stopping = (void *)atropos_stopping, -- .quiescent = (void *)atropos_quiescent, -- .set_weight = (void *)atropos_set_weight, -- .set_cpumask = (void *)atropos_set_cpumask, -- .prep_enable = (void *)atropos_prep_enable, -- .disable = (void *)atropos_disable, -- .init = (void *)atropos_init, -- .exit = (void *)atropos_exit, -- .flags = 0, -- .name = "atropos", --}; -diff --git b/tools/sched_ext/atropos/src/bpf/atropos.h a/tools/sched_ext/atropos/src/bpf/atropos.h -deleted file mode 100644 -index addf29ca104a..000000000000 ---- b/tools/sched_ext/atropos/src/bpf/atropos.h -+++ /dev/null -@@ -1,44 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#ifndef __ATROPOS_H --#define __ATROPOS_H -- --#include --#ifndef __kptr --#ifdef __KERNEL__ --#error "__kptr_ref not defined in the kernel" --#endif --#define __kptr --#endif -- --#define MAX_CPUS 512 --#define MAX_DOMS 64 /* limited to avoid complex bitmask ops */ --#define CACHELINE_SIZE 64 -- --/* Statistics */ --enum stat_idx { -- ATROPOS_STAT_TASK_GET_ERR, -- ATROPOS_STAT_WAKE_SYNC, -- ATROPOS_STAT_PREV_IDLE, -- ATROPOS_STAT_PINNED, -- ATROPOS_STAT_DIRECT_DISPATCH, -- ATROPOS_STAT_DSQ_DISPATCH, -- ATROPOS_STAT_GREEDY, -- ATROPOS_STAT_LOAD_BALANCE, -- ATROPOS_STAT_LAST_TASK, -- ATROPOS_NR_STATS, --}; -- --struct task_ctx { -- unsigned long long dom_mask; /* the domains this task can run on */ -- struct bpf_cpumask __kptr *cpumask; -- unsigned int dom_id; -- unsigned int weight; -- unsigned long long runnable_at; -- unsigned long long runnable_for; -- bool dispatch_local; --}; -- --#endif /* __ATROPOS_H */ -diff --git b/tools/sched_ext/atropos/src/main.rs a/tools/sched_ext/atropos/src/main.rs -deleted file mode 100644 -index 0d313662f713..000000000000 ---- b/tools/sched_ext/atropos/src/main.rs -+++ /dev/null -@@ -1,942 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#[path = "bpf/.output/atropos.skel.rs"] --mod atropos; --pub use atropos::*; --pub mod atropos_sys; -- --use std::cell::Cell; --use std::collections::{BTreeMap, BTreeSet}; --use std::ffi::CStr; --use std::ops::Bound::{Included, Unbounded}; --use std::sync::atomic::{AtomicBool, Ordering}; --use std::sync::Arc; --use std::time::{Duration, SystemTime}; -- --use ::fb_procfs as procfs; --use anyhow::{anyhow, bail, Context, Result}; --use bitvec::prelude::*; --use clap::Parser; --use log::{info, trace, warn}; --use ordered_float::OrderedFloat; -- --/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF --/// part does simple round robin in each domain and the userspace part --/// calculates the load factor of each domain and tells the BPF part how to load --/// balance the domains. -- --/// This scheduler demonstrates dividing scheduling logic between BPF and --/// userspace and using rust to build the userspace part. An earlier variant of --/// this scheduler was used to balance across six domains, each representing a --/// chiplet in a six-chiplet AMD processor, and could match the performance of --/// production setup using CFS. --#[derive(Debug, Parser)] --struct Opts { -- /// Scheduling slice duration in microseconds. -- #[clap(short, long, default_value = "20000")] -- slice_us: u64, -- -- /// Monitoring and load balance interval in seconds. -- #[clap(short, long, default_value = "2.0")] -- interval: f64, -- -- /// Build domains according to how CPUs are grouped at this cache level -- /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id. -- #[clap(short = 'c', long, default_value = "3")] -- cache_level: u32, -- -- /// Instead of using cache locality, set the cpumask for each domain -- /// manually, provide multiple --cpumasks, one for each domain. E.g. -- /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with -- /// the corresponding CPUs belonging to each domain. Each CPU must -- /// belong to precisely one domain. -- #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")] -- cpumasks: Vec, -- -- /// When non-zero, enable greedy task stealing. When a domain is idle, a -- /// cpu will attempt to steal tasks from a domain with at least -- /// greedy_threshold tasks enqueued. These tasks aren't permanently -- /// stolen from the domain. -- #[clap(short, long, default_value = "4")] -- greedy_threshold: u32, -- -- /// The load decay factor. Every interval, the existing load is decayed -- /// by this factor and new load is added. Must be in the range [0.0, -- /// 0.99]. The smaller the value, the more sensitive load calculation -- /// is to recent changes. When 0.0, history is ignored and the load -- /// value from the latest period is used directly. -- #[clap(short, long, default_value = "0.5")] -- load_decay_factor: f64, -- -- /// Disable load balancing. Unless disabled, periodically userspace will -- /// calculate the load factor of each domain and instruct BPF which -- /// processes to move. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- no_load_balance: bool, -- -- /// Put per-cpu kthreads directly into local dsq's. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- kthreads_local: bool, -- -- /// Use FIFO scheduling instead of weighted vtime scheduling. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- fifo_sched: bool, -- -- /// If specified, only tasks which have their scheduling policy set to -- /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all -- /// tasks are switched. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- partial: bool, -- -- /// Enable verbose output including libbpf details. Specify multiple -- /// times to increase verbosity. -- #[clap(short, long, action = clap::ArgAction::Count)] -- verbose: u8, --} -- --fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result { -- Ok(reader -- .read_stat() -- .context("Failed to read procfs")? -- .total_cpu -- .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?) --} -- --fn now_monotonic() -> u64 { -- let mut time = libc::timespec { -- tv_sec: 0, -- tv_nsec: 0, -- }; -- let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) }; -- assert!(ret == 0); -- time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 --} -- --fn clear_map(map: &mut libbpf_rs::Map) { -- // XXX: libbpf_rs has some design flaw that make it impossible to -- // delete while iterating despite it being safe so we alias it here -- let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) }; -- for key in map.keys() { -- let _ = deleter.delete(&key); -- } --} -- --#[derive(Debug)] --struct TaskLoad { -- runnable_for: u64, -- load: f64, --} -- --#[derive(Debug)] --struct TaskInfo { -- pid: i32, -- dom_mask: u64, -- migrated: Cell, --} -- --struct LoadBalancer<'a, 'b, 'c> { -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- -- tasks_by_load: Vec, TaskInfo>>, -- load_avg: f64, -- dom_loads: Vec, -- -- imbal: Vec, -- doms_to_push: BTreeMap, u32>, -- doms_to_pull: BTreeMap, u32>, -- -- nr_lb_data_errors: &'c mut u64, --} -- --impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> { -- const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10; -- const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1; -- const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50; -- -- fn new( -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- nr_lb_data_errors: &'c mut u64, -- ) -> Self { -- Self { -- maps, -- task_loads, -- nr_doms, -- load_decay_factor, -- -- tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(), -- load_avg: 0f64, -- dom_loads: vec![0.0; nr_doms], -- -- imbal: vec![0.0; nr_doms], -- doms_to_pull: BTreeMap::new(), -- doms_to_push: BTreeMap::new(), -- -- nr_lb_data_errors, -- } -- } -- -- fn read_task_loads(&mut self, period: Duration) -> Result<()> { -- let now_mono = now_monotonic(); -- let task_data = self.maps.task_data(); -- let mut this_task_loads = BTreeMap::::new(); -- let mut load_sum = 0.0f64; -- self.dom_loads = vec![0f64; self.nr_doms]; -- -- for key in task_data.keys() { -- if let Some(task_ctx_vec) = task_data -- .lookup(&key, libbpf_rs::MapFlags::ANY) -- .context("Failed to lookup task_data")? -- { -- let task_ctx = -- unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) }; -- let pid = i32::from_ne_bytes( -- key.as_slice() -- .try_into() -- .context("Invalid key length in task_data map")?, -- ); -- -- let (this_at, this_for, weight) = unsafe { -- ( -- std::ptr::read_volatile(&task_ctx.runnable_at as *const u64), -- std::ptr::read_volatile(&task_ctx.runnable_for as *const u64), -- std::ptr::read_volatile(&task_ctx.weight as *const u32), -- ) -- }; -- -- let (mut delta, prev_load) = match self.task_loads.get(&pid) { -- Some(prev) => (this_for - prev.runnable_for, Some(prev.load)), -- None => (this_for, None), -- }; -- -- // Non-zero this_at indicates that the task is currently -- // runnable. Note that we read runnable_at and runnable_for -- // without any synchronization and there is a small window -- // where we end up misaccounting. While this can cause -- // temporary error, it's unlikely to cause any noticeable -- // misbehavior especially given the load value clamping. -- if this_at > 0 && this_at < now_mono { -- delta += now_mono - this_at; -- } -- -- delta = delta.min(period.as_nanos() as u64); -- let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64) -- .clamp(0.0, weight as f64); -- -- let this_load = match prev_load { -- Some(prev_load) => { -- prev_load * self.load_decay_factor -- + this_load * (1.0 - self.load_decay_factor) -- } -- None => this_load, -- }; -- -- this_task_loads.insert( -- pid, -- TaskLoad { -- runnable_for: this_for, -- load: this_load, -- }, -- ); -- -- load_sum += this_load; -- self.dom_loads[task_ctx.dom_id as usize] += this_load; -- // Only record pids that are eligible for load balancing -- if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) { -- continue; -- } -- self.tasks_by_load[task_ctx.dom_id as usize].insert( -- OrderedFloat(this_load), -- TaskInfo { -- pid, -- dom_mask: task_ctx.dom_mask, -- migrated: Cell::new(false), -- }, -- ); -- } -- } -- -- self.load_avg = load_sum / self.nr_doms as f64; -- *self.task_loads = this_task_loads; -- Ok(()) -- } -- -- // To balance dom loads we identify doms with lower and higher load than average -- fn calculate_dom_load_balance(&mut self) -> Result<()> { -- for (dom, dom_load) in self.dom_loads.iter().enumerate() { -- let imbal = dom_load - self.load_avg; -- if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO { -- if imbal > 0f64 { -- self.doms_to_push.insert(OrderedFloat(imbal), dom as u32); -- } else { -- self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32); -- } -- self.imbal[dom] = imbal; -- } -- } -- Ok(()) -- } -- -- // Find the first candidate pid which hasn't already been migrated and -- // can run in @pull_dom. -- fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)> -- where -- I: IntoIterator, &'d TaskInfo)>, -- { -- match tasks_by_load -- .into_iter() -- .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0) -- .next() -- { -- Some((OrderedFloat(load), task)) => Some((*load, task)), -- None => None, -- } -- } -- -- fn pick_victim( -- &self, -- (push_dom, to_push): (u32, f64), -- (pull_dom, to_pull): (u32, f64), -- ) -> Option<(&TaskInfo, f64)> { -- let to_xfer = to_pull.min(to_push); -- -- trace!( -- "considering dom {}@{:.2} -> {}@{:.2}", -- push_dom, -- to_push, -- pull_dom, -- to_pull -- ); -- -- let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs(); -- -- trace!( -- "to_xfer={:.2} tasks_by_load={:?}", -- to_xfer, -- &self.tasks_by_load[push_dom as usize] -- ); -- -- // We want to pick a task to transfer from push_dom to pull_dom to -- // maximize the reduction of load imbalance between the two. IOW, -- // pick a task which has the closest load value to $to_xfer that can -- // be migrated. Find such task by locating the first migratable task -- // while scanning left from $to_xfer and the counterpart while -- // scanning right and picking the better of the two. -- let (load, task, new_imbal) = match ( -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Unbounded, Included(&OrderedFloat(to_xfer)))) -- .rev(), -- pull_dom, -- ), -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Included(&OrderedFloat(to_xfer)), Unbounded)), -- pull_dom, -- ), -- ) { -- (None, None) => return None, -- (Some((load, task)), None) | (None, Some((load, task))) => { -- (load, task, calc_new_imbal(load)) -- } -- (Some((load0, task0)), Some((load1, task1))) => { -- let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1)); -- if new_imbal0 <= new_imbal1 { -- (load0, task0, new_imbal0) -- } else { -- (load1, task1, new_imbal1) -- } -- } -- }; -- -- // If the best candidate can't reduce the imbalance, there's nothing -- // to do for this pair. -- let old_imbal = to_push + to_pull; -- if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal { -- trace!( -- "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal -- ); -- return None; -- } -- -- trace!( -- "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal, -- ); -- -- Some((task, load)) -- } -- -- // Actually execute the load balancing. Concretely this writes pid -> dom -- // entries into the lb_data map for bpf side to consume. -- fn load_balance(&mut self) -> Result<()> { -- clear_map(self.maps.lb_data()); -- -- trace!("imbal={:?}", &self.imbal); -- trace!("doms_to_push={:?}", &self.doms_to_push); -- trace!("doms_to_pull={:?}", &self.doms_to_pull); -- -- // Push from the most imbalanced to least. -- while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() { -- let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO; -- let mut pushed = 0f64; -- -- // Transfer tasks from push_dom to reduce imbalance. -- loop { -- let last_pushed = pushed; -- -- // Pull from the most imbalaned to least. -- let mut doms_to_pull = BTreeMap::<_, _>::new(); -- std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull); -- let mut pull_doms = doms_to_pull.into_iter().rev().collect::>(); -- -- for (to_pull, pull_dom) in pull_doms.iter_mut() { -- if let Some((task, load)) = -- self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull))) -- { -- // Execute migration. -- task.migrated.set(true); -- to_push -= load; -- *to_pull -= load; -- pushed += load; -- -- // Ask BPF code to execute the migration. -- let pid = task.pid; -- let cpid = (pid as libc::pid_t).to_ne_bytes(); -- if let Err(e) = self.maps.lb_data().update( -- &cpid, -- &pull_dom.to_ne_bytes(), -- libbpf_rs::MapFlags::NO_EXIST, -- ) { -- warn!( -- "Failed to update lb_data map for pid={} error={:?}", -- pid, &e -- ); -- *self.nr_lb_data_errors += 1; -- } -- -- // Always break after a successful migration so that -- // the pulling domains are always considered in the -- // descending imbalance order. -- break; -- } -- } -- -- pull_doms -- .into_iter() -- .map(|(k, v)| self.doms_to_pull.insert(k, v)) -- .count(); -- -- // Stop repeating if nothing got transferred or pushed enough. -- if pushed == last_pushed || pushed >= push_max { -- break; -- } -- } -- } -- Ok(()) -- } --} -- --struct Scheduler<'a> { -- skel: AtroposSkel<'a>, -- struct_ops: Option, -- -- nr_cpus: usize, -- nr_doms: usize, -- load_decay_factor: f64, -- balance_load: bool, -- -- proc_reader: procfs::ProcReader, -- -- prev_at: SystemTime, -- prev_total_cpu: procfs::CpuStat, -- task_loads: BTreeMap, -- -- nr_lb_data_errors: u64, --} -- --impl<'a> Scheduler<'a> { -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn parse_cpusets( -- cpumasks: &[String], -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- if cpumasks.len() > atropos_sys::MAX_DOMS as usize { -- bail!( -- "Number of requested DSQs ({}) is greater than MAX_DOMS ({})", -- cpumasks.len(), -- atropos_sys::MAX_DOMS -- ); -- } -- let mut cpus = vec![-1i32; nr_cpus]; -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()]; -- for (dq, cpumask) in cpumasks.iter().enumerate() { -- let hex_str = { -- let mut tmp_str = cpumask -- .strip_prefix("0x") -- .unwrap_or(cpumask) -- .replace('_', ""); -- if tmp_str.len() % 2 != 0 { -- tmp_str = "0".to_string() + &tmp_str; -- } -- tmp_str -- }; -- let byte_vec = hex::decode(&hex_str) -- .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?; -- -- for (index, &val) in byte_vec.iter().rev().enumerate() { -- let mut v = val; -- while v != 0 { -- let lsb = v.trailing_zeros() as usize; -- v &= !(1 << lsb); -- let cpu = index * 8 + lsb; -- if cpu > nr_cpus { -- bail!( -- concat!( -- "Found cpu ({}) in cpumask ({}) which is larger", -- " than the number of cpus on the machine ({})" -- ), -- cpu, -- cpumask, -- nr_cpus -- ); -- } -- if cpus[cpu] != -1 { -- bail!( -- "Found cpu ({}) with dq ({}) but also in cpumask ({})", -- cpu, -- cpus[cpu], -- cpumask -- ); -- } -- cpus[cpu] = dq as i32; -- cpusets[dq].set(cpu, true); -- } -- } -- cpusets[dq].set_uninitialized(false); -- } -- -- for (cpu, &dq) in cpus.iter().enumerate() { -- if dq < 0 { -- bail!( -- "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.", -- cpu -- ); -- } -- } -- -- Ok((cpusets, cpus)) -- } -- -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn cpusets_from_cache( -- level: u32, -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- let mut cpu_to_cache = vec![]; // (cpu_id, cache_id) -- let mut cache_ids = BTreeSet::::new(); -- let mut nr_not_found = 0; -- -- // Build cpu -> cache ID mapping. -- for cpu in 0..nr_cpus { -- let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level); -- let id = match std::fs::read_to_string(&path) { -- Ok(val) => val -- .trim() -- .parse::() -- .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?, -- Err(e) if e.kind() == std::io::ErrorKind::NotFound => { -- nr_not_found += 1; -- 0 -- } -- Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)), -- }; -- -- cpu_to_cache.push(id); -- cache_ids.insert(id); -- } -- -- if nr_not_found > 1 { -- warn!( -- "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0", -- level, nr_not_found, nr_cpus -- ); -- } -- -- // Cache IDs may have holes. Assign consecutive domain IDs to -- // existing cache IDs. -- let mut cache_to_dom = BTreeMap::::new(); -- let mut nr_doms = 0; -- for cache_id in cache_ids.iter() { -- cache_to_dom.insert(*cache_id, nr_doms); -- nr_doms += 1; -- } -- -- if nr_doms > atropos_sys::MAX_DOMS { -- bail!( -- "Total number of doms {} is greater than MAX_DOMS ({})", -- nr_doms, -- atropos_sys::MAX_DOMS -- ); -- } -- -- // Build and return dom -> cpumask and cpu -> dom mappings. -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize]; -- let mut cpu_to_dom = vec![]; -- -- for cpu in 0..nr_cpus { -- let dom_id = cache_to_dom[&cpu_to_cache[cpu]]; -- cpusets[dom_id as usize].set(cpu, true); -- cpu_to_dom.push(dom_id as i32); -- } -- -- Ok((cpusets, cpu_to_dom)) -- } -- -- fn init(opts: &Opts) -> Result { -- // Open the BPF prog first for verification. -- let mut skel_builder = AtroposSkelBuilder::default(); -- skel_builder.obj_builder.debug(opts.verbose > 0); -- let mut skel = skel_builder.open().context("Failed to open BPF program")?; -- -- let nr_cpus = libbpf_rs::num_possible_cpus().unwrap(); -- if nr_cpus > atropos_sys::MAX_CPUS as usize { -- bail!( -- "nr_cpus ({}) is greater than MAX_CPUS ({})", -- nr_cpus, -- atropos_sys::MAX_CPUS -- ); -- } -- -- // Initialize skel according to @opts. -- let (cpusets, cpus) = if opts.cpumasks.len() > 0 { -- Self::parse_cpusets(&opts.cpumasks, nr_cpus)? -- } else { -- Self::cpusets_from_cache(opts.cache_level, nr_cpus)? -- }; -- let nr_doms = cpusets.len(); -- skel.rodata().nr_doms = nr_doms as u32; -- skel.rodata().nr_cpus = nr_cpus as u32; -- -- for (cpu, dom) in cpus.iter().enumerate() { -- skel.rodata().cpu_dom_id_map[cpu] = *dom as u32; -- } -- -- for (dom, cpuset) in cpusets.iter().enumerate() { -- let raw_cpuset_slice = cpuset.as_raw_slice(); -- let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom]; -- let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len()); -- left.clone_from_slice(cpuset.as_raw_slice()); -- let cpumask_str = dom_cpumask_slice -- .iter() -- .take((nr_cpus + 63) / 64) -- .rev() -- .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x)); -- info!( -- "DOM[{:02}] cpumask{} ({} cpus)", -- dom, -- &cpumask_str, -- cpuset.count_ones() -- ); -- } -- -- skel.rodata().slice_us = opts.slice_us; -- skel.rodata().kthreads_local = opts.kthreads_local; -- skel.rodata().fifo_sched = opts.fifo_sched; -- skel.rodata().switch_partial = opts.partial; -- skel.rodata().greedy_threshold = opts.greedy_threshold; -- -- // Attach. -- let mut skel = skel.load().context("Failed to load BPF program")?; -- skel.attach().context("Failed to attach BPF program")?; -- let struct_ops = Some( -- skel.maps_mut() -- .atropos() -- .attach_struct_ops() -- .context("Failed to attach atropos struct ops")?, -- ); -- info!("Atropos Scheduler Attached"); -- -- // Other stuff. -- let mut proc_reader = procfs::ProcReader::new(); -- let prev_total_cpu = read_total_cpu(&mut proc_reader)?; -- -- Ok(Self { -- skel, -- struct_ops, // should be held to keep it attached -- -- nr_cpus, -- nr_doms, -- load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99), -- balance_load: !opts.no_load_balance, -- -- proc_reader, -- -- prev_at: SystemTime::now(), -- prev_total_cpu, -- task_loads: BTreeMap::new(), -- -- nr_lb_data_errors: 0, -- }) -- } -- -- fn get_cpu_busy(&mut self) -> Result { -- let total_cpu = read_total_cpu(&mut self.proc_reader)?; -- let busy = match (&self.prev_total_cpu, &total_cpu) { -- ( -- procfs::CpuStat { -- user_usec: Some(prev_user), -- nice_usec: Some(prev_nice), -- system_usec: Some(prev_system), -- idle_usec: Some(prev_idle), -- iowait_usec: Some(prev_iowait), -- irq_usec: Some(prev_irq), -- softirq_usec: Some(prev_softirq), -- stolen_usec: Some(prev_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- procfs::CpuStat { -- user_usec: Some(curr_user), -- nice_usec: Some(curr_nice), -- system_usec: Some(curr_system), -- idle_usec: Some(curr_idle), -- iowait_usec: Some(curr_iowait), -- irq_usec: Some(curr_irq), -- softirq_usec: Some(curr_softirq), -- stolen_usec: Some(curr_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- ) => { -- let idle_usec = curr_idle - prev_idle; -- let iowait_usec = curr_iowait - prev_iowait; -- let user_usec = curr_user - prev_user; -- let system_usec = curr_system - prev_system; -- let nice_usec = curr_nice - prev_nice; -- let irq_usec = curr_irq - prev_irq; -- let softirq_usec = curr_softirq - prev_softirq; -- let stolen_usec = curr_stolen - prev_stolen; -- -- let busy_usec = -- user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec; -- let total_usec = idle_usec + busy_usec + iowait_usec; -- busy_usec as f64 / total_usec as f64 -- } -- _ => { -- bail!("Some procfs stats are not populated!"); -- } -- }; -- -- self.prev_total_cpu = total_cpu; -- Ok(busy) -- } -- -- fn read_bpf_stats(&mut self) -> Result> { -- let mut maps = self.skel.maps_mut(); -- let stats_map = maps.stats(); -- let mut stats: Vec = Vec::new(); -- let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus]; -- -- for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS { -- let cpu_stat_vec = stats_map -- .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY) -- .with_context(|| format!("Failed to lookup stat {}", stat))? -- .expect("per-cpu stat should exist"); -- let sum = cpu_stat_vec -- .iter() -- .map(|val| { -- u64::from_ne_bytes( -- val.as_slice() -- .try_into() -- .expect("Invalid value length in stat map"), -- ) -- }) -- .sum(); -- stats_map -- .update_percpu( -- &(stat as u32).to_ne_bytes(), -- &zero_vec, -- libbpf_rs::MapFlags::ANY, -- ) -- .context("Failed to zero stat")?; -- stats.push(sum); -- } -- Ok(stats) -- } -- -- fn report( -- &self, -- stats: &Vec, -- cpu_busy: f64, -- processing_dur: Duration, -- load_avg: f64, -- dom_loads: &Vec, -- imbal: &Vec, -- ) { -- let stat = |idx| stats[idx as usize]; -- let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK); -- -- info!( -- "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms", -- cpu_busy * 100.0, -- load_avg, -- stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize], -- stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize], -- self.nr_lb_data_errors, -- processing_dur.as_millis(), -- ); -- -- let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0; -- -- info!( -- "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}", -- total, -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY), -- ); -- -- for i in 0..self.nr_doms { -- info!( -- "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}", -- i, -- dom_loads[i], -- if imbal[i] < 0.0 { -imbal[i] } else { 0.0 }, -- if imbal[i] > 0.0 { imbal[i] } else { 0.0 }, -- ); -- } -- } -- -- fn step(&mut self) -> Result<()> { -- let started_at = std::time::SystemTime::now(); -- let bpf_stats = self.read_bpf_stats()?; -- let cpu_busy = self.get_cpu_busy()?; -- -- let mut lb = LoadBalancer::new( -- self.skel.maps_mut(), -- &mut self.task_loads, -- self.nr_doms, -- self.load_decay_factor, -- &mut self.nr_lb_data_errors, -- ); -- -- lb.read_task_loads(started_at.duration_since(self.prev_at)?)?; -- lb.calculate_dom_load_balance()?; -- -- if self.balance_load { -- lb.load_balance()?; -- } -- -- // Extract fields needed for reporting and drop lb to release -- // mutable borrows. -- let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal); -- -- self.report( -- &bpf_stats, -- cpu_busy, -- std::time::SystemTime::now().duration_since(started_at)?, -- load_avg, -- &dom_loads, -- &imbal, -- ); -- -- self.prev_at = started_at; -- Ok(()) -- } -- -- fn read_bpf_exit_type(&mut self) -> i32 { -- unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) } -- } -- -- fn report_bpf_exit_type(&mut self) -> Result<()> { -- // Report msg if EXT_OPS_EXIT_ERROR. -- match self.read_bpf_exit_type() { -- 0 => Ok(()), -- etype if etype == 2 => { -- let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) }; -- let msg = cstr -- .to_str() -- .context("Failed to convert exit msg to string") -- .unwrap(); -- bail!("BPF exit_type={} msg={}", etype, msg); -- } -- etype => { -- info!("BPF exit_type={}", etype); -- Ok(()) -- } -- } -- } --} -- --impl<'a> Drop for Scheduler<'a> { -- fn drop(&mut self) { -- if let Some(struct_ops) = self.struct_ops.take() { -- drop(struct_ops); -- } -- } --} -- --fn main() -> Result<()> { -- let opts = Opts::parse(); -- -- let llv = match opts.verbose { -- 0 => simplelog::LevelFilter::Info, -- 1 => simplelog::LevelFilter::Debug, -- _ => simplelog::LevelFilter::Trace, -- }; -- let mut lcfg = simplelog::ConfigBuilder::new(); -- lcfg.set_time_level(simplelog::LevelFilter::Error) -- .set_location_level(simplelog::LevelFilter::Off) -- .set_target_level(simplelog::LevelFilter::Off) -- .set_thread_level(simplelog::LevelFilter::Off); -- simplelog::TermLogger::init( -- llv, -- lcfg.build(), -- simplelog::TerminalMode::Stderr, -- simplelog::ColorChoice::Auto, -- )?; -- -- let shutdown = Arc::new(AtomicBool::new(false)); -- let shutdown_clone = shutdown.clone(); -- ctrlc::set_handler(move || { -- shutdown_clone.store(true, Ordering::Relaxed); -- }) -- .context("Error setting Ctrl-C handler")?; -- -- let mut sched = Scheduler::init(&opts)?; -- -- while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 { -- std::thread::sleep(Duration::from_secs_f64(opts.interval)); -- sched.step()?; -- } -- -- sched.report_bpf_exit_type() --} -diff --git b/tools/sched_ext/gnu/stubs.h a/tools/sched_ext/gnu/stubs.h -deleted file mode 100644 -index 719225b16626..000000000000 ---- b/tools/sched_ext/gnu/stubs.h -+++ /dev/null -@@ -1 +0,0 @@ --/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ -diff --git b/tools/sched_ext/scx_common.bpf.h a/tools/sched_ext/scx_common.bpf.h -deleted file mode 100644 -index e56de9dc86f2..000000000000 ---- b/tools/sched_ext/scx_common.bpf.h -+++ /dev/null -@@ -1,288 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __SCHED_EXT_COMMON_BPF_H --#define __SCHED_EXT_COMMON_BPF_H -- --#include "vmlinux.h" --#include --#include --#include --#include "user_exit_info.h" -- --#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ --#define PF_EXITING 0x00000004 --#define CLOCK_MONOTONIC 1 -- --/* -- * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can -- * lead to really confusing misbehaviors. Let's trigger a build failure. -- */ --static inline void ___vmlinux_h_sanity_check___(void) --{ -- _Static_assert(SCX_DSQ_FLAG_BUILTIN, -- "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); --} -- --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; -- --static inline __attribute__((format(printf, 1, 2))) --void ___scx_bpf_error_format_checker(const char *fmt, ...) {} -- --/* -- * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments -- * instead of an array of u64. Note that __param[] must have at least one -- * element to keep the verifier happy. -- */ --#define scx_bpf_error(fmt, args...) \ --({ \ -- static char ___fmt[] = fmt; \ -- unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ -- \ -- _Pragma("GCC diagnostic push") \ -- _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ -- ___bpf_fill(___param, args); \ -- _Pragma("GCC diagnostic pop") \ -- \ -- scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ -- \ -- ___scx_bpf_error_format_checker(fmt, ##args); \ --}) -- --void scx_bpf_switch_all(void) __ksym; --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; --bool scx_bpf_consume(u64 dsq_id) __ksym; --u32 scx_bpf_dispatch_nr_slots(void) __ksym; --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym; --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym; --void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; --s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym; --const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; --const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; --void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; --void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; --bool scx_bpf_task_running(const struct task_struct *p) __ksym; --s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym; --u32 scx_bpf_reenqueue_local(void) __ksym; -- --#define BPF_STRUCT_OPS(name, args...) \ --SEC("struct_ops/"#name) \ --BPF_PROG(name, ##args) -- --#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ --SEC("struct_ops.s/"#name) \ --BPF_PROG(name, ##args) -- --/** -- * MEMBER_VPTR - Obtain the verified pointer to a struct or array member -- * @base: struct or array to index -- * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...) -- * -- * The verifier often gets confused by the instruction sequence the compiler -- * generates for indexing struct fields or arrays. This macro forces the -- * compiler to generate a code sequence which first calculates the byte offset, -- * checks it against the struct or array size and add that byte offset to -- * generate the pointer to the member to help the verifier. -- * -- * Ideally, we want to abort if the calculated offset is out-of-bounds. However, -- * BPF currently doesn't support abort, so evaluate to NULL instead. The caller -- * must check for NULL and take appropriate action to appease the verifier. To -- * avoid confusing the verifier, it's best to check for NULL and dereference -- * immediately. -- * -- * vptr = MEMBER_VPTR(my_array, [i][j]); -- * if (!vptr) -- * return error; -- * *vptr = new_value; -- */ --#define MEMBER_VPTR(base, member) (typeof(base member) *)({ \ -- u64 __base = (u64)base; \ -- u64 __addr = (u64)&(base member) - __base; \ -- asm volatile ( \ -- "if %0 <= %[max] goto +2\n" \ -- "%0 = 0\n" \ -- "goto +1\n" \ -- "%0 += %1\n" \ -- : "+r"(__addr) \ -- : "r"(__base), \ -- [max]"i"(sizeof(base) - sizeof(base member))); \ -- __addr; \ --}) -- --/* -- * BPF core and other generic helpers -- */ -- --/* list and rbtree */ --#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) --#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) -- --void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; --void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; -- --#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) --#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) -- --void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; --struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; --struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, -- struct bpf_rb_node *node) __ksym; --void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, -- bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym; --struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; -- --/* task */ --struct task_struct *bpf_task_from_pid(s32 pid) __ksym; --struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; --void bpf_task_release(struct task_struct *p) __ksym; -- --/* cgroup */ --struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; --void bpf_cgroup_release(struct cgroup *cgrp) __ksym; --struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; -- --/* cpumask */ --struct bpf_cpumask *bpf_cpumask_create(void) __ksym; --struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; --u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; --u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym; -- --/* rcu */ --void bpf_rcu_read_lock(void) __ksym; --void bpf_rcu_read_unlock(void) __ksym; -- --/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */ --struct bpf_iter_num; -- --extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym; --extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym; --extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym; -- --#ifndef bpf_for_each --/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for -- * using BPF open-coded iterators without having to write mundane explicit -- * low-level loop logic. Instead, it provides for()-like generic construct -- * that can be used pretty naturally. E.g., for some hypothetical cgroup -- * iterator, you'd write: -- * -- * struct cgroup *cg, *parent_cg = <...>; -- * -- * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { -- * bpf_printk("Child cgroup id = %d", cg->cgroup_id); -- * if (cg->cgroup_id == 123) -- * break; -- * } -- * -- * I.e., it looks almost like high-level for each loop in other languages, -- * supports continue/break, and is verifiable by BPF verifier. -- * -- * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) -- * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to -- * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int -- * *`, not just `int`. So for integers bpf_for() is more convenient. -- * -- * Note: this macro relies on C99 feature of allowing to declare variables -- * inside for() loop, bound to for() loop lifetime. It also utilizes GCC -- * extension: __attribute__((cleanup())), supported by both GCC and -- * Clang. -- */ --#define bpf_for_each(type, cur, args...) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ -- cleanup(bpf_iter_##type##_destroy))), \ -- /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_##type##_new(&___it, ##args), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_##type##_destroy, (void *)0); \ -- /* iteration and termination check */ \ -- (((cur) = bpf_iter_##type##_next(&___it))); \ --) --#endif /* bpf_for_each */ -- --#ifndef bpf_for --/* bpf_for(i, start, end) implements a for()-like looping construct that sets -- * provided integer variable *i* to values starting from *start* through, -- * but not including, *end*. It also proves to BPF verifier that *i* belongs -- * to range [start, end), so this can be used for accessing arrays without -- * extra checks. -- * -- * Note: *start* and *end* are assumed to be expressions with no side effects -- * and whose values do not change throughout bpf_for() loop execution. They do -- * not have to be statically known or constant, though. -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_for(i, start, end) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, (start), (end)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- ({ \ -- /* iteration step */ \ -- int *___t = bpf_iter_num_next(&___it); \ -- /* termination and bounds check */ \ -- (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ -- }); \ --) --#endif /* bpf_for */ -- --#ifndef bpf_repeat --/* bpf_repeat(N) performs N iterations without exposing iteration number -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_repeat(N) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, 0, (N)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- bpf_iter_num_next(&___it); \ -- /* nothing here */ \ --) --#endif /* bpf_repeat */ -- --#endif /* __SCHED_EXT_COMMON_BPF_H */ -diff --git b/tools/sched_ext/scx_example_central.bpf.c a/tools/sched_ext/scx_example_central.bpf.c -deleted file mode 100644 -index 4cec04b4c2ed..000000000000 ---- b/tools/sched_ext/scx_example_central.bpf.c -+++ /dev/null -@@ -1,334 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A central FIFO sched_ext scheduler which demonstrates the followings: -- * -- * a. Making all scheduling decisions from one CPU: -- * -- * The central CPU is the only one making scheduling decisions. All other -- * CPUs kick the central CPU when they run out of tasks to run. -- * -- * There is one global BPF queue and the central CPU schedules all CPUs by -- * dispatching from the global queue to each CPU's local dsq from dispatch(). -- * This isn't the most straightforward. e.g. It'd be easier to bounce -- * through per-CPU BPF queues. The current design is chosen to maximally -- * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching. -- * -- * b. Tickless operation -- * -- * All tasks are dispatched with the infinite slice which allows stopping the -- * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full -- * parameter. The tickless operation can be observed through -- * /proc/interrupts. -- * -- * Periodic switching is enforced by a periodic timer checking all CPUs and -- * preempting them as necessary. Unfortunately, BPF timer currently doesn't -- * have a way to pin to a specific CPU, so the periodic timer isn't pinned to -- * the central CPU. -- * -- * c. Preemption -- * -- * Kthreads are unconditionally queued to the head of a matching local dsq -- * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always -- * prioritized over user threads, which is required for ensuring forward -- * progress as e.g. the periodic timer may run on a ksoftirqd and if the -- * ksoftirqd gets starved by a user thread, there may not be anything else to -- * vacate that user thread. -- * -- * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the -- * next tasks. -- * -- * This scheduler is designed to maximize usage of various SCX mechanisms. A -- * more practical implementation would likely put the scheduling loop outside -- * the central CPU's dispatch() path and add some form of priority mechanism. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --enum { -- FALLBACK_DSQ_ID = 0, -- MAX_CPUS = 4096, -- MS_TO_NS = 1000LLU * 1000, -- TIMER_INTERVAL_NS = 1 * MS_TO_NS, --}; -- --const volatile bool switch_partial; --const volatile s32 central_cpu; --const volatile u32 nr_cpu_ids = 64; /* !0 for veristat, set during init */ -- --u64 nr_total, nr_locals, nr_queued, nr_lost_pids; --u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries; --u64 nr_overflows; -- --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, s32); --} central_q SEC(".maps"); -- --/* can't use percpu map due to bad lookups */ --static bool cpu_gimme_task[MAX_CPUS]; --static u64 cpu_started_at[MAX_CPUS]; -- --struct central_timer { -- struct bpf_timer timer; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, 1); -- __type(key, u32); -- __type(value, struct central_timer); --} central_timer SEC(".maps"); -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- /* -- * Steer wakeups to the central CPU as much as possible to avoid -- * disturbing other CPUs. It's safe to blindly return the central cpu as -- * select_cpu() is a hint and if @p can't be on it, the kernel will -- * automatically pick a fallback CPU. -- */ -- return central_cpu; --} -- --void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags) --{ -- s32 pid = p->pid; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- /* -- * Push per-cpu kthreads at the head of local dsq's and preempt the -- * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked -- * behind other threads which is necessary for forward progress -- * guarantee as we depend on the BPF timer which may run from ksoftirqd. -- */ -- if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- __sync_fetch_and_add(&nr_locals, 1); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF, -- enq_flags | SCX_ENQ_PREEMPT); -- return; -- } -- -- if (bpf_map_push_elem(¢ral_q, &pid, 0)) { -- __sync_fetch_and_add(&nr_overflows, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_queued, 1); -- -- if (!scx_bpf_task_running(p)) -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); --} -- --static bool dispatch_to_cpu(s32 cpu) --{ -- struct task_struct *p; -- s32 pid; -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (bpf_map_pop_elem(¢ral_q, &pid)) -- break; -- -- __sync_fetch_and_sub(&nr_queued, 1); -- -- p = bpf_task_from_pid(pid); -- if (!p) { -- __sync_fetch_and_add(&nr_lost_pids, 1); -- continue; -- } -- -- /* -- * If we can't run the task at the top, do the dumb thing and -- * bounce it to the fallback dsq. -- */ -- if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- __sync_fetch_and_add(&nr_mismatches, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0); -- bpf_task_release(p); -- continue; -- } -- -- /* dispatch to local and mark that @cpu doesn't need more */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0); -- -- if (cpu != central_cpu) -- scx_bpf_kick_cpu(cpu, 0); -- -- bpf_task_release(p); -- return true; -- } -- -- return false; --} -- --void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (cpu == central_cpu) { -- /* dispatch for all other CPUs first */ -- __sync_fetch_and_add(&nr_dispatches, 1); -- -- bpf_for(cpu, 0, nr_cpu_ids) { -- bool *gimme; -- -- if (!scx_bpf_dispatch_nr_slots()) -- break; -- -- /* central's gimme is never set */ -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme && !*gimme) -- continue; -- -- if (dispatch_to_cpu(cpu)) -- *gimme = false; -- } -- -- /* -- * Retry if we ran out of dispatch buffer slots as we might have -- * skipped some CPUs and also need to dispatch for self. The ext -- * core automatically retries if the local dsq is empty but we -- * can't rely on that as we're dispatching for other CPUs too. -- * Kick self explicitly to retry. -- */ -- if (!scx_bpf_dispatch_nr_slots()) { -- __sync_fetch_and_add(&nr_retries, 1); -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- return; -- } -- -- /* look for a task to run on the central CPU */ -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- dispatch_to_cpu(central_cpu); -- } else { -- bool *gimme; -- -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme) -- *gimme = true; -- -- /* -- * Force dispatch on the scheduling CPU so that it finds a task -- * to run for us. -- */ -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- } --} -- --void BPF_STRUCT_OPS(central_running, struct task_struct *p) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ --} -- --void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = 0; --} -- --static int central_timerfn(void *map, int *key, struct bpf_timer *timer) --{ -- u64 now = bpf_ktime_get_ns(); -- u64 nr_to_kick = nr_queued; -- s32 i; -- -- bpf_for(i, 0, nr_cpu_ids) { -- s32 cpu = (nr_timers + i) % nr_cpu_ids; -- u64 *started_at; -- -- if (cpu == central_cpu) -- continue; -- -- /* kick iff the current one exhausted its slice */ -- started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at && *started_at && -- vtime_before(now, *started_at + SCX_SLICE_DFL)) -- continue; -- -- /* and there's something pending */ -- if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) || -- scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu)) -- ; -- else if (nr_to_kick) -- nr_to_kick--; -- else -- continue; -- -- scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); -- } -- -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- -- bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- __sync_fetch_and_add(&nr_timers, 1); -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(central_init) --{ -- u32 key = 0; -- struct bpf_timer *timer; -- int ret; -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1); -- if (ret) -- return ret; -- -- timer = bpf_map_lookup_elem(¢ral_timer, &key); -- if (!timer) -- return -ESRCH; -- -- bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); -- bpf_timer_set_callback(timer, central_timerfn); -- ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- return ret; --} -- --void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops central_ops = { -- /* -- * We are offloading all scheduling decisions to the central CPU and -- * thus being the last task on a given CPU doesn't mean anything -- * special. Enqueue the last tasks like any other tasks. -- */ -- .flags = SCX_OPS_ENQ_LAST, -- -- .select_cpu = (void *)central_select_cpu, -- .enqueue = (void *)central_enqueue, -- .dispatch = (void *)central_dispatch, -- .running = (void *)central_running, -- .stopping = (void *)central_stopping, -- .init = (void *)central_init, -- .exit = (void *)central_exit, -- .name = "central", --}; -diff --git b/tools/sched_ext/scx_example_central.c a/tools/sched_ext/scx_example_central.c -deleted file mode 100644 -index 7ad591cbdc65..000000000000 ---- b/tools/sched_ext/scx_example_central.c -+++ /dev/null -@@ -1,94 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_central.skel.h" -- --const char help_fmt[] = --"A central FIFO sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-c CPU] [-p]\n" --"\n" --" -c CPU Override the central CPU (default: 0)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_central *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_central__open(); -- assert(skel); -- -- skel->rodata->central_cpu = 0; -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "c:ph")) != -1) { -- switch (opt) { -- case 'c': -- skel->rodata->central_cpu = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_central__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.central_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf("total :%10lu local:%10lu queued:%10lu lost:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_locals, -- skel->bss->nr_queued, -- skel->bss->nr_lost_pids); -- printf("timer :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n", -- skel->bss->nr_timers, -- skel->bss->nr_dispatches, -- skel->bss->nr_mismatches, -- skel->bss->nr_retries); -- printf("overflow:%10lu\n", -- skel->bss->nr_overflows); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_central__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_flatcg.bpf.c a/tools/sched_ext/scx_example_flatcg.bpf.c -deleted file mode 100644 -index f6078b9a681f..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.bpf.c -+++ /dev/null -@@ -1,872 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext flattened cgroup hierarchy scheduler. It implements -- * hierarchical weight-based cgroup CPU control by flattening the cgroup -- * hierarchy into a single layer by compounding the active weight share at each -- * level. Consider the following hierarchy with weights in parentheses: -- * -- * R + A (100) + B (100) -- * | \ C (100) -- * \ D (200) -- * -- * Ignoring the root and threaded cgroups, only B, C and D can contain tasks. -- * Let's say all three have runnable tasks. The total share that each of these -- * three cgroups is entitled to can be calculated by compounding its share at -- * each level. -- * -- * For example, B is competing against C and in that competition its share is -- * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's -- * share in that competition is 200/(200+100) == 1/3. B's eventual share in the -- * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's -- * eventual shaer is the same at 1/6. D is only competing at the top level and -- * its share is 200/(100+200) == 2/3. -- * -- * So, instead of hierarchically scheduling level-by-level, we can consider it -- * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3 -- * and keep updating the eventual shares as the cgroups' runnable states change. -- * -- * This flattening of hierarchy can bring a substantial performance gain when -- * the cgroup hierarchy is nested multiple levels. in a simple benchmark using -- * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it -- * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two -- * apache instances competing with 2:1 weight ratio nested four level deep. -- * -- * However, the gain comes at the cost of not being able to properly handle -- * thundering herd of cgroups. For example, if many cgroups which are nested -- * behind a low priority parent cgroup wake up around the same time, they may be -- * able to consume more CPU cycles than they are entitled to. In many use cases, -- * this isn't a real concern especially given the performance gain. Also, there -- * are ways to mitigate the problem further by e.g. introducing an extra -- * scheduling layer on cgroup delegation boundaries. -- * -- * The scheduler first picks the cgroup to run and then schedule the tasks -- * within by using nested weighted vtime scheduling by default. The -- * cgroup-internal scheduling can be switched to FIFO with the -f option. -- */ --#include "scx_common.bpf.h" --#include "user_exit_info.h" --#include "scx_example_flatcg.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */ --const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL; --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --u64 cvtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, u64); -- __uint(max_entries, FCG_NR_STATS); --} stats SEC(".maps"); -- --static void stat_inc(enum fcg_stat_idx idx) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p)++; --} -- --struct fcg_cpu_ctx { -- u64 cur_cgid; -- u64 cur_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, struct fcg_cpu_ctx); -- __uint(max_entries, 1); --} cpu_ctx SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_cgrp_ctx); --} cgrp_ctx SEC(".maps"); -- --struct cgv_node { -- struct bpf_rb_node rb_node; -- __u64 cvtime; -- __u64 cgid; --}; -- --private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock; --private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node); -- --struct cgv_node_stash { -- struct cgv_node __kptr *node; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, 16384); -- __type(key, __u64); -- __type(value, struct cgv_node_stash); --} cgv_node_stash SEC(".maps"); -- --struct fcg_task_ctx { -- u64 bypassed_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_task_ctx); --} task_ctx SEC(".maps"); -- --/* gets inc'd on weight tree changes to expire the cached hweights */ --unsigned long hweight_gen = 1; -- --static u64 div_round_up(u64 dividend, u64 divisor) --{ -- return (dividend + divisor - 1) / divisor; --} -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) --{ -- struct cgv_node *cgc_a, *cgc_b; -- -- cgc_a = container_of(a, struct cgv_node, rb_node); -- cgc_b = container_of(b, struct cgv_node, rb_node); -- -- return cgc_a->cvtime < cgc_b->cvtime; --} -- --static struct fcg_cpu_ctx *find_cpu_ctx(void) --{ -- struct fcg_cpu_ctx *cpuc; -- u32 idx = 0; -- -- cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx); -- if (!cpuc) { -- scx_bpf_error("cpu_ctx lookup failed"); -- return NULL; -- } -- return cpuc; --} -- --static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id); -- return NULL; -- } -- return cgc; --} -- --static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgrp = bpf_cgroup_ancestor(cgrp, level); -- if (!cgrp) { -- scx_bpf_error("ancestor cgroup lookup failed"); -- return NULL; -- } -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- scx_bpf_error("ancestor cgrp_ctx lookup failed"); -- bpf_cgroup_release(cgrp); -- return cgc; --} -- --static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- int level; -- -- if (!cgc->nr_active) { -- stat_inc(FCG_STAT_HWT_SKIP); -- return; -- } -- -- if (cgc->hweight_gen == hweight_gen) { -- stat_inc(FCG_STAT_HWT_CACHE); -- return; -- } -- -- stat_inc(FCG_STAT_HWT_UPDATES); -- bpf_for(level, 0, cgrp->level + 1) { -- struct fcg_cgrp_ctx *cgc; -- bool is_active; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- -- if (!level) { -- cgc->hweight = FCG_HWEIGHT_ONE; -- cgc->hweight_gen = hweight_gen; -- } else { -- struct fcg_cgrp_ctx *pcgc; -- -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- -- /* -- * We can be oppotunistic here and not grab the -- * cgv_tree_lock and deal with the occasional races. -- * However, hweight updates are already cached and -- * relatively low-frequency. Let's just do the -- * straightforward thing. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- is_active = cgc->nr_active; -- if (is_active) { -- cgc->hweight_gen = pcgc->hweight_gen; -- cgc->hweight = -- div_round_up(pcgc->hweight * cgc->weight, -- pcgc->child_weight_sum); -- } -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!is_active) { -- stat_inc(FCG_STAT_HWT_RACE); -- break; -- } -- } -- } --} -- --static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) --{ -- u64 delta, cvtime, max_budget; -- -- /* -- * A node which is on the rbtree can't be pointed to from elsewhere yet -- * and thus can't be updated and repositioned. Instead, we collect the -- * vtime deltas separately and apply it asynchronously here. -- */ -- delta = cgc->cvtime_delta; -- __sync_fetch_and_sub(&cgc->cvtime_delta, delta); -- cvtime = cgv_node->cvtime + delta; -- -- /* -- * Allow a cgroup to carry the maximum budget proportional to its -- * hweight such that a full-hweight cgroup can immediately take up half -- * of the CPUs at the most while staying at the front of the rbtree. -- */ -- max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / -- (2 * FCG_HWEIGHT_ONE); -- if (vtime_before(cvtime, cvtime_now - max_budget)) -- cvtime = cvtime_now - max_budget; -- -- cgv_node->cvtime = cvtime; --} -- --static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- u64 cgid = cgrp->kn->id; -- -- /* paired with cmpxchg in try_pick_next_cgroup() */ -- if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { -- stat_inc(FCG_STAT_ENQ_SKIP); -- return; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid); -- return; -- } -- -- /* NULL if the node is already on the rbtree */ -- cgv_node = bpf_kptr_xchg(&stash->node, NULL); -- if (!cgv_node) { -- stat_inc(FCG_STAT_ENQ_RACE); -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); --} -- --void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * If select_cpu_dfl() is recommending local enqueue, the target CPU is -- * idle. Follow it and charge the cgroup later in fcg_stopping() after -- * the fact. Use the same mechanism to deal with tasks with custom -- * affinities so that we don't have to worry about per-cgroup dq's -- * containing tasks that can't be executed from some CPUs. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) { -- /* -- * Tell fcg_stopping() that this bypassed the regular scheduling -- * path and should be force charged to the cgroup. 0 is used to -- * indicate that the task isn't bypassing, so if the current -- * runtime is 0, go back by one nanosecond. -- */ -- taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1; -- -- /* -- * The global dq is deprioritized as we don't want to let tasks -- * to boost themselves by constraining its cpumask. The -- * deprioritization is rather severe, so let's not apply that to -- * per-cpu kernel threads. This is ham-fisted. We probably wanna -- * implement per-cgroup fallback dq's instead so that we have -- * more control over when tasks with custom cpumask get issued. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || -- (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) { -- stat_inc(FCG_STAT_LOCAL); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- } else { -- stat_inc(FCG_STAT_GLOBAL); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } -- return; -- } -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- goto out_release; -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 tvtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) -- tvtime = cgc->tvtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, -- tvtime, enq_flags); -- } -- -- cgrp_enqueued(cgrp, cgc); --out_release: -- bpf_cgroup_release(cgrp); --} -- --/* -- * Walk the cgroup tree to update the active weight sums as tasks wake up and -- * sleep. The weight sums are used as the base when calculating the proportion a -- * given cgroup or task is entitled to at each level. -- */ --static void update_active_weight_sums(struct cgroup *cgrp, bool runnable) --{ -- struct fcg_cgrp_ctx *cgc; -- bool updated = false; -- int idx; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- /* -- * In most cases, a hot cgroup would have multiple threads going to -- * sleep and waking up while the whole cgroup stays active. In leaf -- * cgroups, ->nr_runnable which is updated with __sync operations gates -- * ->nr_active updates, so that we don't have to grab the cgv_tree_lock -- * repeatedly for a busy cgroup which is staying active. -- */ -- if (runnable) { -- if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_ACT); -- } else { -- if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_DEACT); -- } -- -- /* -- * If @cgrp is becoming runnable, its hweight should be refreshed after -- * it's added to the weight tree so that enqueue has the up-to-date -- * value. If @cgrp is becoming quiescent, the hweight should be -- * refreshed before it's removed from the weight tree so that the usage -- * charging which happens afterwards has access to the latest value. -- */ -- if (!runnable) -- cgrp_refresh_hweight(cgrp, cgc); -- -- /* propagate upwards */ -- bpf_for(idx, 0, cgrp->level) { -- int level = cgrp->level - idx; -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- bool propagate = false; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- if (level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- } -- -- /* -- * We need the propagation protected by a lock to synchronize -- * against weight changes. There's no reason to drop the lock at -- * each level but bpf_spin_lock() doesn't want any function -- * calls while locked. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- -- if (runnable) { -- if (!cgc->nr_active++) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum += cgc->weight; -- } -- } -- } else { -- if (!--cgc->nr_active) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum -= cgc->weight; -- } -- } -- } -- -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!propagate) -- break; -- } -- -- if (updated) -- __sync_fetch_and_add(&hweight_gen, 1); -- -- if (runnable) -- cgrp_refresh_hweight(cgrp, cgc); --} -- --void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, true); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) --{ -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- if (fifo_sched) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- /* -- * @cgc->tvtime_now always progresses forward as tasks start -- * executing. The test and update can be performed concurrently -- * from multiple CPUs and thus racy. Any error should be -- * contained and temporary. Let's just live with it. -- */ -- if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) -- cgc->tvtime_now = p->scx.dsq_vtime; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- /* scale the execution time by the inverse of the weight and charge */ -- if (!fifo_sched) -- p->scx.dsq_vtime += -- (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- if (!taskc->bypassed_at) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- __sync_fetch_and_add(&cgc->cvtime_delta, -- p->se.sum_exec_runtime - taskc->bypassed_at); -- taskc->bypassed_at = 0; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, false); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) --{ -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- if (cgrp->level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1); -- if (!pcgc) -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- if (pcgc && cgc->nr_active) -- pcgc->child_weight_sum += (s64)weight - cgc->weight; -- cgc->weight = weight; -- bpf_spin_unlock(&cgv_tree_lock); --} -- --static bool try_pick_next_cgroup(u64 *cgidp) --{ -- struct bpf_rb_node *rb_node; -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 cgid; -- -- /* pop the front cgroup and wind cvtime_now accordingly */ -- bpf_spin_lock(&cgv_tree_lock); -- -- rb_node = bpf_rbtree_first(&cgv_tree); -- if (!rb_node) { -- bpf_spin_unlock(&cgv_tree_lock); -- stat_inc(FCG_STAT_PNC_NO_CGRP); -- *cgidp = 0; -- return true; -- } -- -- rb_node = bpf_rbtree_remove(&cgv_tree, rb_node); -- bpf_spin_unlock(&cgv_tree_lock); -- -- cgv_node = container_of(rb_node, struct cgv_node, rb_node); -- cgid = cgv_node->cgid; -- -- if (vtime_before(cvtime_now, cgv_node->cvtime)) -- cvtime_now = cgv_node->cvtime; -- -- /* -- * If lookup fails, the cgroup's gone. Free and move on. See -- * fcg_cgroup_exit(). -- */ -- cgrp = bpf_cgroup_from_id(cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- if (!scx_bpf_consume(cgid)) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_EMPTY); -- goto out_stash; -- } -- -- /* -- * Successfully consumed from the cgroup. This will be our current -- * cgroup for the new slice. Refresh its hweight. -- */ -- cgrp_refresh_hweight(cgrp, cgc); -- -- bpf_cgroup_release(cgrp); -- -- /* -- * As the cgroup may have more tasks, add it back to the rbtree. Note -- * that here we charge the full slice upfront and then exact later -- * according to the actual consumption. This prevents lowpri thundering -- * herd from saturating the machine. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- -- *cgidp = cgid; -- stat_inc(FCG_STAT_PNC_NEXT); -- return true; -- --out_stash: -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- /* -- * Paired with cmpxchg in cgrp_enqueued(). If they see the following -- * transition, they'll enqueue the cgroup. If they are earlier, we'll -- * see their task in the dq below and requeue the cgroup. -- */ -- __sync_val_compare_and_swap(&cgc->queued, 1, 0); -- -- if (scx_bpf_dsq_nr_queued(cgid)) { -- bpf_spin_lock(&cgv_tree_lock); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- goto out_free; -- } -- } -- -- return false; -- --out_free: -- bpf_obj_drop(cgv_node); -- return false; --} -- --void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) --{ -- struct fcg_cpu_ctx *cpuc; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 now = bpf_ktime_get_ns(); -- -- cpuc = find_cpu_ctx(); -- if (!cpuc) -- return; -- -- if (!cpuc->cur_cgid) -- goto pick_next_cgroup; -- -- if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { -- if (scx_bpf_consume(cpuc->cur_cgid)) { -- stat_inc(FCG_STAT_CNS_KEEP); -- return; -- } -- stat_inc(FCG_STAT_CNS_EMPTY); -- } else { -- stat_inc(FCG_STAT_CNS_EXPIRE); -- } -- -- /* -- * The current cgroup is expiring. It was already charged a full slice. -- * Calculate the actual usage and accumulate the delta. -- */ -- cgrp = bpf_cgroup_from_id(cpuc->cur_cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_CNS_GONE); -- goto pick_next_cgroup; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (cgc) { -- /* -- * We want to update the vtime delta and then look for the next -- * cgroup to execute but the latter needs to be done in a loop -- * and we can't keep the lock held. Oh well... -- */ -- bpf_spin_lock(&cgv_tree_lock); -- __sync_fetch_and_add(&cgc->cvtime_delta, -- (cpuc->cur_at + cgrp_slice_ns - now) * -- FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- stat_inc(FCG_STAT_CNS_GONE); -- } -- -- bpf_cgroup_release(cgrp); -- --pick_next_cgroup: -- cpuc->cur_at = now; -- -- if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { -- cpuc->cur_cgid = 0; -- return; -- } -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_pick_next_cgroup(&cpuc->cur_cgid)) -- break; -- } --} -- --s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct fcg_task_ctx *taskc; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- taskc = bpf_task_storage_get(&task_ctx, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!taskc) -- return -ENOMEM; -- -- taskc->bypassed_at = 0; -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp, -- struct scx_cgroup_init_args *args) --{ -- struct fcg_cgrp_ctx *cgc; -- struct cgv_node *cgv_node; -- struct cgv_node_stash empty_stash = {}, *stash; -- u64 cgid = cgrp->kn->id; -- int ret; -- -- /* -- * Technically incorrect as cgroup ID is full 64bit while dq ID is -- * 63bit. Should not be a problem in practice and easy to spot in the -- * unlikely case that it breaks. -- */ -- ret = scx_bpf_create_dsq(cgid, -1); -- if (ret) -- return ret; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!cgc) { -- ret = -ENOMEM; -- goto err_destroy_dsq; -- } -- -- cgc->weight = args->weight; -- cgc->hweight = FCG_HWEIGHT_ONE; -- -- ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash, -- BPF_NOEXIST); -- if (ret) { -- if (ret != -ENOMEM) -- scx_bpf_error("unexpected stash creation error (%d)", -- ret); -- goto err_destroy_dsq; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("unexpected cgv_node stash lookup failure"); -- ret = -ENOENT; -- goto err_destroy_dsq; -- } -- -- cgv_node = bpf_obj_new(struct cgv_node); -- if (!cgv_node) { -- ret = -ENOMEM; -- goto err_del_cgv_node; -- } -- -- cgv_node->cgid = cgid; -- cgv_node->cvtime = cvtime_now; -- -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- ret = -EBUSY; -- goto err_drop; -- } -- -- return 0; -- --err_drop: -- bpf_obj_drop(cgv_node); --err_del_cgv_node: -- bpf_map_delete_elem(&cgv_node_stash, &cgid); --err_destroy_dsq: -- scx_bpf_destroy_dsq(cgid); -- return ret; --} -- --void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- -- /* -- * For now, there's no way find and remove the cgv_node if it's on the -- * cgv_tree. Let's drain them in the dispatch path as they get popped -- * off the front of the tree. -- */ -- bpf_map_delete_elem(&cgv_node_stash, &cgid); -- scx_bpf_destroy_dsq(cgid); --} -- --s32 BPF_STRUCT_OPS(fcg_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops flatcg_ops = { -- .enqueue = (void *)fcg_enqueue, -- .dispatch = (void *)fcg_dispatch, -- .runnable = (void *)fcg_runnable, -- .running = (void *)fcg_running, -- .stopping = (void *)fcg_stopping, -- .quiescent = (void *)fcg_quiescent, -- .prep_enable = (void *)fcg_prep_enable, -- .cgroup_set_weight = (void *)fcg_cgroup_set_weight, -- .cgroup_init = (void *)fcg_cgroup_init, -- .cgroup_exit = (void *)fcg_cgroup_exit, -- .init = (void *)fcg_init, -- .exit = (void *)fcg_exit, -- .flags = SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING, -- .name = "flatcg", --}; -diff --git b/tools/sched_ext/scx_example_flatcg.c a/tools/sched_ext/scx_example_flatcg.c -deleted file mode 100644 -index f9c8a5b84a70..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.c -+++ /dev/null -@@ -1,232 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2023 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2023 Tejun Heo -- * Copyright (c) 2023 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_flatcg.h" --#include "scx_example_flatcg.skel.h" -- --#ifndef FILEID_KERNFS --#define FILEID_KERNFS 0xfe --#endif -- --const char help_fmt[] = --"A flattened cgroup hierarchy sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -i INTERVAL Report interval\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --static float read_cpu_util(__u64 *last_sum, __u64 *last_idle) --{ -- FILE *fp; -- char buf[4096]; -- char *line, *cur = NULL, *tok; -- __u64 sum = 0, idle = 0; -- __u64 delta_sum, delta_idle; -- int idx; -- -- fp = fopen("/proc/stat", "r"); -- if (!fp) { -- perror("fopen(\"/proc/stat\")"); -- return 0.0; -- } -- -- if (!fgets(buf, sizeof(buf), fp)) { -- perror("fgets(\"/proc/stat\")"); -- fclose(fp); -- return 0.0; -- } -- fclose(fp); -- -- line = buf; -- for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) { -- char *endp = NULL; -- __u64 v; -- -- if (idx == 0) { -- line = NULL; -- continue; -- } -- v = strtoull(tok, &endp, 0); -- if (!endp || *endp != '\0') { -- fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n", -- idx, tok); -- continue; -- } -- sum += v; -- if (idx == 4) -- idle = v; -- } -- -- delta_sum = sum - *last_sum; -- delta_idle = idle - *last_idle; -- *last_sum = sum; -- *last_idle = idle; -- -- return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0; --} -- --static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats) --{ -- __u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus]; -- __u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); -- -- for (idx = 0; idx < FCG_NR_STATS; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_flatcg *skel; -- struct bpf_link *link; -- struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 }; -- bool dump_cgrps = false; -- __u64 last_cpu_sum = 0, last_cpu_idle = 0; -- __u64 last_stats[FCG_NR_STATS] = {}; -- unsigned long seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_flatcg__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open: %s\n", strerror(errno)); -- return 1; -- } -- -- skel->rodata->nr_cpus = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) { -- double v; -- -- switch (opt) { -- case 's': -- v = strtod(optarg, NULL); -- skel->rodata->cgrp_slice_ns = v * 1000; -- break; -- case 'i': -- v = strtod(optarg, NULL); -- intv_ts.tv_sec = v; -- intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000; -- break; -- case 'd': -- dump_cgrps = true; -- break; -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- case 'h': -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d", -- (double)skel->rodata->cgrp_slice_ns / 1000000.0, -- (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0, -- dump_cgrps); -- -- if (scx_example_flatcg__load(skel)) { -- fprintf(stderr, "Failed to load: %s\n", strerror(errno)); -- return 1; -- } -- -- link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops); -- if (!link) { -- fprintf(stderr, "Failed to attach_struct_ops: %s\n", -- strerror(errno)); -- return 1; -- } -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- __u64 acc_stats[FCG_NR_STATS]; -- __u64 stats[FCG_NR_STATS]; -- float cpu_util; -- int i; -- -- cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle); -- -- fcg_read_stats(skel, acc_stats); -- for (i = 0; i < FCG_NR_STATS; i++) -- stats[i] = acc_stats[i] - last_stats[i]; -- -- memcpy(last_stats, acc_stats, sizeof(acc_stats)); -- -- printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n", -- seq++, cpu_util * 100.0, skel->data->hweight_gen); -- printf(" act:%6llu deact:%6llu local:%6llu global:%6llu\n", -- stats[FCG_STAT_ACT], -- stats[FCG_STAT_DEACT], -- stats[FCG_STAT_LOCAL], -- stats[FCG_STAT_GLOBAL]); -- printf("HWT skip:%6llu race:%6llu cache:%6llu update:%6llu\n", -- stats[FCG_STAT_HWT_SKIP], -- stats[FCG_STAT_HWT_RACE], -- stats[FCG_STAT_HWT_CACHE], -- stats[FCG_STAT_HWT_UPDATES]); -- printf("ENQ skip:%6llu race:%6llu\n", -- stats[FCG_STAT_ENQ_SKIP], -- stats[FCG_STAT_ENQ_RACE]); -- printf("CNS keep:%6llu expire:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_CNS_KEEP], -- stats[FCG_STAT_CNS_EXPIRE], -- stats[FCG_STAT_CNS_EMPTY], -- stats[FCG_STAT_CNS_GONE]); -- printf("PNC nocgrp:%6llu next:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_PNC_NO_CGRP], -- stats[FCG_STAT_PNC_NEXT], -- stats[FCG_STAT_PNC_EMPTY], -- stats[FCG_STAT_PNC_GONE]); -- printf("BAD remove:%6llu\n", -- acc_stats[FCG_STAT_BAD_REMOVAL]); -- -- nanosleep(&intv_ts, NULL); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_flatcg__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_flatcg.h a/tools/sched_ext/scx_example_flatcg.h -deleted file mode 100644 -index 490758ed41f0..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.h -+++ /dev/null -@@ -1,49 +0,0 @@ --#ifndef __SCX_EXAMPLE_FLATCG_H --#define __SCX_EXAMPLE_FLATCG_H -- --enum { -- FCG_HWEIGHT_ONE = 1LLU << 16, --}; -- --enum fcg_stat_idx { -- FCG_STAT_ACT, -- FCG_STAT_DEACT, -- FCG_STAT_LOCAL, -- FCG_STAT_GLOBAL, -- -- FCG_STAT_HWT_UPDATES, -- FCG_STAT_HWT_CACHE, -- FCG_STAT_HWT_SKIP, -- FCG_STAT_HWT_RACE, -- -- FCG_STAT_ENQ_SKIP, -- FCG_STAT_ENQ_RACE, -- -- FCG_STAT_CNS_KEEP, -- FCG_STAT_CNS_EXPIRE, -- FCG_STAT_CNS_EMPTY, -- FCG_STAT_CNS_GONE, -- -- FCG_STAT_PNC_NO_CGRP, -- FCG_STAT_PNC_NEXT, -- FCG_STAT_PNC_EMPTY, -- FCG_STAT_PNC_GONE, -- -- FCG_STAT_BAD_REMOVAL, -- -- FCG_NR_STATS, --}; -- --struct fcg_cgrp_ctx { -- u32 nr_active; -- u32 nr_runnable; -- u32 queued; -- u32 weight; -- u32 hweight; -- u64 child_weight_sum; -- u64 hweight_gen; -- s64 cvtime_delta; -- u64 tvtime_now; --}; -- --#endif /* __SCX_EXAMPLE_FLATCG_H */ -diff --git b/tools/sched_ext/scx_example_pair.bpf.c a/tools/sched_ext/scx_example_pair.bpf.c -deleted file mode 100644 -index 279efe58b777..000000000000 ---- b/tools/sched_ext/scx_example_pair.bpf.c -+++ /dev/null -@@ -1,627 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext core-scheduler which always makes every sibling CPU pair -- * execute from the same CPU cgroup. -- * -- * This scheduler is a minimal implementation and would need some form of -- * priority handling both inside each cgroup and across the cgroups to be -- * practically useful. -- * -- * Each CPU in the system is paired with exactly one other CPU, according to a -- * "stride" value that can be specified when the BPF scheduler program is first -- * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee -- * that they will only ever schedule tasks that belong to the same CPU cgroup. -- * -- * Scheduler Initialization -- * ------------------------ -- * -- * The scheduler BPF program is first initialized from user space, before it is -- * enabled. During this initialization process, each CPU on the system is -- * assigned several values that are constant throughout its runtime: -- * -- * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling -- * decisions. Paired CPUs always schedule tasks from the same -- * CPU cgroup, and synchronize with each other to guarantee -- * that this constraint is not violated. -- * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access -- * a struct pair_ctx object that is shared between the pair. -- * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the -- * pair. Each struct pair_ctx has an active_mask field, -- * which is a bitmap used to indicate whether each core -- * in the pair currently has an actively running task. -- * This index specifies which entry in the bitmap corresponds -- * to each CPU in the pair. -- * -- * During this initialization, the CPUs are paired according to a "stride" that -- * may be specified when invoking the user space program that initializes and -- * loads the scheduler. By default, the stride is 1/2 the total number of CPUs. -- * -- * Tasks and cgroups -- * ----------------- -- * -- * Every cgroup in the system is registered with the scheduler using the -- * pair_cgroup_init() callback, and every task in the system is associated with -- * exactly one cgroup. At a high level, the idea with the pair scheduler is to -- * always schedule tasks from the same cgroup within a given CPU pair. When a -- * task is enqueued (i.e. passed to the pair_enqueue() callback function), its -- * cgroup ID is read from its task struct, and then a corresponding queue map -- * is used to FIFO-enqueue the task for that cgroup. -- * -- * If you look through the implementation of the scheduler, you'll notice that -- * there is quite a bit of complexity involved with looking up the per-cgroup -- * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash -- * BPF hash map that is used to map a cgroup ID to a globally unique ID that's -- * allocated in the BPF program. This is done because we use separate maps to -- * store the FIFO queue of tasks, and the length of that map, per cgroup. This -- * complexity is only present because of current deficiencies in BPF that will -- * soon be addressed. The main point to keep in mind is that newly enqueued -- * tasks are added to their cgroup's FIFO queue. -- * -- * Dispatching tasks -- * ----------------- -- * -- * This section will describe how enqueued tasks are dispatched and scheduled. -- * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is -- * as follows: -- * -- * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is -- * the structure that's used to synchronize amongst the two pair CPUs in their -- * scheduling decisions. After any of the following events have occurred: -- * -- * - The cgroup's slice run has expired, or -- * - The cgroup becomes empty, or -- * - Either CPU in the pair is preempted by a higher priority scheduling class -- * -- * The cgroup transitions to the draining state and stops executing new tasks -- * from the cgroup. -- * -- * 2. If the pair is still executing a task, mark the pair_ctx as draining, and -- * wait for the pair CPU to be preempted. -- * -- * 3. Otherwise, if the pair CPU is not running a task, we can move onto -- * scheduling new tasks. Pop the next cgroup id from the top_q queue. -- * -- * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it. -- * -- * Note again that this scheduling behavior is simple, but the implementation -- * is complex mostly because this it hits several BPF shortcomings and has to -- * work around in often awkward ways. Most of the shortcomings are expected to -- * be resolved in the near future which should allow greatly simplifying this -- * scheduler. -- * -- * Dealing with preemption -- * ----------------------- -- * -- * SCX is the lowest priority sched_class, and could be preempted by them at -- * any time. To address this, the scheduler implements pair_cpu_release() and -- * pair_cpu_acquire() callbacks which are invoked by the core scheduler when -- * the scheduler loses and gains control of the CPU respectively. -- * -- * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and -- * then invoke: -- * -- * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- * -- * This preempts the pair CPU, and waits until it has re-entered the scheduler -- * before returning. This is necessary to ensure that the higher priority -- * sched_class that preempted our scheduler does not schedule a task -- * concurrently with our pair CPU. -- * -- * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption -- * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable -- * pair scheduling. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include "scx_example_pair.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; -- --/* !0 for veristat, set during init */ --const volatile u32 nr_cpu_ids = 64; -- --/* a pair of CPUs stay on a cgroup for this duration */ --const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL; -- --/* cpu ID -> pair cpu ID */ --const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 }; -- --/* cpu ID -> pair_id */ --const volatile u32 pair_id[MAX_CPUS]; -- --/* CPU ID -> CPU # in the pair (0 or 1) */ --const volatile u32 in_pair_idx[MAX_CPUS]; -- --struct pair_ctx { -- struct bpf_spin_lock lock; -- -- /* the cgroup the pair is currently executing */ -- u64 cgid; -- -- /* the pair started executing the current cgroup at */ -- u64 started_at; -- -- /* whether the current cgroup is draining */ -- bool draining; -- -- /* the CPUs that are currently active on the cgroup */ -- u32 active_mask; -- -- /* -- * the CPUs that are currently preempted and running tasks in a -- * different scheduler. -- */ -- u32 preempted_mask; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, MAX_CPUS / 2); -- __type(key, u32); -- __type(value, struct pair_ctx); --} pair_ctx SEC(".maps"); -- --/* queue of cgrp_q's possibly with tasks on them */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- /* -- * Because it's difficult to build strong synchronization encompassing -- * multiple non-trivial operations in BPF, this queue is managed in an -- * opportunistic way so that we guarantee that a cgroup w/ active tasks -- * is always on it but possibly multiple times. Once we have more robust -- * synchronization constructs and e.g. linked list, we should be able to -- * do this in a prettier way but for now just size it big enough. -- */ -- __uint(max_entries, 4 * MAX_CGRPS); -- __type(value, u64); --} top_q SEC(".maps"); -- --/* per-cgroup q which FIFOs the tasks from the cgroup */ --struct cgrp_q { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, MAX_QUEUED); -- __type(value, u32); --}; -- --/* -- * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local -- * storage; however, a cgroup local storage can only be accessed from the BPF -- * progs attached to the cgroup. For now, work around by allocating array of -- * cgrp_q's and then allocating per-cgroup indices. -- * -- * Another caveat: It's difficult to populate a large array of maps statically -- * or from BPF. Initialize it from userland. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, MAX_CGRPS); -- __type(key, s32); -- __array(values, struct cgrp_q); --} cgrp_q_arr SEC(".maps"); -- --static u64 cgrp_q_len[MAX_CGRPS]; -- --/* -- * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be -- * useful to have as a map type. -- */ --static u32 cgrp_q_idx_cursor; --static u64 cgrp_q_idx_busy[MAX_CGRPS]; -- --/* -- * All added up, the following is what we do: -- * -- * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking -- * for a free ID. If not found, fail cgroup creation with -EBUSY. -- * -- * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following -- * cgrp_q_idx_hash. -- * -- * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from -- * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr. -- * -- * This is sadly complicated for something pretty simple. Hopefully, we should -- * be able to simplify in the future. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, MAX_CGRPS); -- __uint(key_size, sizeof(u64)); /* cgrp ID */ -- __uint(value_size, sizeof(s32)); /* cgrp_q idx */ --} cgrp_q_idx_hash SEC(".maps"); -- --/* statistics */ --u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions; --u64 nr_exps, nr_exp_waits, nr_exp_empty; --u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty; -- --struct user_exit_info uei; -- --static bool time_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- struct cgrp_q *cgq; -- s32 pid = p->pid; -- u64 cgid; -- u32 *q_idx; -- u64 *cgq_len; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- cgrp = scx_bpf_task_cgroup(p); -- cgid = cgrp->kn->id; -- bpf_cgroup_release(cgrp); -- -- /* find the cgroup's q and push @p into it */ -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!q_idx) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return; -- } -- -- cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx); -- if (!cgq) { -- scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]", -- cgid, *q_idx); -- return; -- } -- -- if (bpf_map_push_elem(cgq, &pid, 0)) { -- scx_bpf_error("cgroup[%llu] queue overflow", cgid); -- return; -- } -- -- /* bump q len, if going 0 -> 1, queue cgroup into the top_q */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len) { -- scx_bpf_error("MEMBER_VTPR malfunction"); -- return; -- } -- -- if (!__sync_fetch_and_add(cgq_len, 1) && -- bpf_map_push_elem(&top_q, &cgid, 0)) { -- scx_bpf_error("top_q overflow"); -- return; -- } --} -- --static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) --{ -- u32 *vptr; -- -- vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *pairc = bpf_map_lookup_elem(&pair_ctx, vptr); -- if (!(*pairc)) -- return -EINVAL; -- -- vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *mask = 1U << *vptr; -- -- return 0; --} -- --static int try_dispatch(s32 cpu) --{ -- struct pair_ctx *pairc; -- struct bpf_map *cgq_map; -- struct task_struct *p; -- u64 now = bpf_ktime_get_ns(); -- bool kick_pair = false; -- bool expired, pair_preempted; -- u32 *vptr, in_pair_mask; -- s32 pid, q_idx; -- u64 cgid; -- int ret; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) { -- scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]", -- cpu); -- return -ENOENT; -- } -- -- bpf_spin_lock(&pairc->lock); -- pairc->active_mask &= ~in_pair_mask; -- -- expired = time_before(pairc->started_at + pair_batch_dur_ns, now); -- if (expired || pairc->draining) { -- u64 new_cgid = 0; -- -- __sync_fetch_and_add(&nr_exps, 1); -- -- /* -- * We're done with the current cgid. An obvious optimization -- * would be not draining if the next cgroup is the current one. -- * For now, be dumb and always expire. -- */ -- pairc->draining = true; -- -- pair_preempted = pairc->preempted_mask; -- if (pairc->active_mask || pair_preempted) { -- /* -- * The other CPU is still active, or is no longer under -- * our control due to e.g. being preempted by a higher -- * priority sched_class. We want to wait until this -- * cgroup expires, or until control of our pair CPU has -- * been returned to us. -- * -- * If the pair controls its CPU, and the time already -- * expired, kick. When the other CPU arrives at -- * dispatch and clears its active mask, it'll push the -- * pair to the next cgroup and kick this CPU. -- */ -- __sync_fetch_and_add(&nr_exp_waits, 1); -- bpf_spin_unlock(&pairc->lock); -- if (expired && !pair_preempted) -- kick_pair = true; -- goto out_maybe_kick; -- } -- -- bpf_spin_unlock(&pairc->lock); -- -- /* -- * Pick the next cgroup. It'd be easier / cleaner to not drop -- * pairc->lock and use stronger synchronization here especially -- * given that we'll be switching cgroups significantly less -- * frequently than tasks. Unfortunately, bpf_spin_lock can't -- * really protect anything non-trivial. Let's do opportunistic -- * operations instead. -- */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u32 *q_idx; -- u64 *cgq_len; -- -- if (bpf_map_pop_elem(&top_q, &new_cgid)) { -- /* no active cgroup, go idle */ -- __sync_fetch_and_add(&nr_exp_empty, 1); -- return 0; -- } -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid); -- if (!q_idx) -- continue; -- -- /* -- * This is the only place where empty cgroups are taken -- * off the top_q. -- */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len || !*cgq_len) -- continue; -- -- /* -- * If it has any tasks, requeue as we may race and not -- * execute it. -- */ -- bpf_map_push_elem(&top_q, &new_cgid, 0); -- break; -- } -- -- bpf_spin_lock(&pairc->lock); -- -- /* -- * The other CPU may already have started on a new cgroup while -- * we dropped the lock. Make sure that we're still draining and -- * start on the new cgroup. -- */ -- if (pairc->draining && !pairc->active_mask) { -- __sync_fetch_and_add(&nr_cgrp_next, 1); -- pairc->cgid = new_cgid; -- pairc->started_at = now; -- pairc->draining = false; -- kick_pair = true; -- } else { -- __sync_fetch_and_add(&nr_cgrp_coll, 1); -- } -- } -- -- cgid = pairc->cgid; -- pairc->active_mask |= in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- -- /* again, it'd be better to do all these with the lock held, oh well */ -- vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!vptr) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return -ENOENT; -- } -- q_idx = *vptr; -- -- /* claim one task from cgrp_q w/ q_idx */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u64 *cgq_len, len; -- -- cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]); -- if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) { -- /* the cgroup must be empty, expire and repeat */ -- __sync_fetch_and_add(&nr_cgrp_empty, 1); -- bpf_spin_lock(&pairc->lock); -- pairc->draining = true; -- pairc->active_mask &= ~in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- return -EAGAIN; -- } -- -- if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len) -- continue; -- -- break; -- } -- -- cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx); -- if (!cgq_map) { -- scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- if (bpf_map_pop_elem(cgq_map, &pid)) { -- scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- p = bpf_task_from_pid(pid); -- if (p) { -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } else { -- /* we don't handle dequeues, retry on lost tasks */ -- __sync_fetch_and_add(&nr_missing, 1); -- return -EAGAIN; -- } -- --out_maybe_kick: -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } -- return 0; --} -- --void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) --{ -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_dispatch(cpu) != -EAGAIN) -- break; -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask &= ~in_pair_mask; -- /* Kick the pair CPU, unless it was also preempted. */ -- kick_pair = !pairc->preempted_mask; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask |= in_pair_mask; -- pairc->active_mask &= ~in_pair_mask; -- /* Kick the pair CPU if it's still running. */ -- kick_pair = pairc->active_mask; -- pairc->draining = true; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- } -- } -- __sync_fetch_and_add(&nr_preemptions, 1); --} -- --s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 i, q_idx; -- -- bpf_for(i, 0, MAX_CGRPS) { -- q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS; -- if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1)) -- break; -- } -- if (i == MAX_CGRPS) -- return -EBUSY; -- -- if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]); -- if (busy) -- *busy = 0; -- return -EBUSY; -- } -- -- return 0; --} -- --void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 *q_idx; -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (q_idx) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]); -- if (busy) -- *busy = 0; -- bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid); -- } --} -- --s32 BPF_STRUCT_OPS(pair_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops pair_ops = { -- .enqueue = (void *)pair_enqueue, -- .dispatch = (void *)pair_dispatch, -- .cpu_acquire = (void *)pair_cpu_acquire, -- .cpu_release = (void *)pair_cpu_release, -- .cgroup_init = (void *)pair_cgroup_init, -- .cgroup_exit = (void *)pair_cgroup_exit, -- .init = (void *)pair_init, -- .exit = (void *)pair_exit, -- .name = "pair", --}; -diff --git b/tools/sched_ext/scx_example_pair.c a/tools/sched_ext/scx_example_pair.c -deleted file mode 100644 -index 18e032bbc173..000000000000 ---- b/tools/sched_ext/scx_example_pair.c -+++ /dev/null -@@ -1,143 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_pair.h" --#include "scx_example_pair.skel.h" -- --const char help_fmt[] = --"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n" --"execute from the same CPU cgroup.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-S STRIDE] [-p]\n" --"\n" --" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_pair *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 stride, i, opt, outer_fd; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_pair__open(); -- assert(skel); -- -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- /* pair up the earlier half to the latter by default, override with -s */ -- stride = skel->rodata->nr_cpu_ids / 2; -- -- while ((opt = getopt(argc, argv, "S:ph")) != -1) { -- switch (opt) { -- case 'S': -- stride = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- for (i = 0; i < skel->rodata->nr_cpu_ids; i++) { -- if (skel->rodata->pair_cpu[i] < 0) { -- skel->rodata->pair_cpu[i] = i + stride; -- skel->rodata->pair_cpu[i + stride] = i; -- skel->rodata->pair_id[i] = i; -- skel->rodata->pair_id[i + stride] = i; -- skel->rodata->in_pair_idx[i] = 0; -- skel->rodata->in_pair_idx[i + stride] = 1; -- } -- } -- -- assert(!scx_example_pair__load(skel)); -- -- /* -- * Populate the cgrp_q_arr map which is an array containing per-cgroup -- * queues. It'd probably be better to do this from BPF but there are too -- * many to initialize statically and there's no way to dynamically -- * populate from BPF. -- */ -- outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr); -- assert(outer_fd >= 0); -- -- printf("Initializing"); -- for (i = 0; i < MAX_CGRPS; i++) { -- s32 inner_fd; -- -- if (exit_req) -- break; -- -- inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, -- sizeof(u32), MAX_QUEUED, NULL); -- assert(inner_fd >= 0); -- assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY)); -- close(inner_fd); -- -- if (!(i % 10)) -- printf("."); -- fflush(stdout); -- } -- printf("\n"); -- -- /* -- * Fully initialized, attach and run. -- */ -- link = bpf_map__attach_struct_ops(skel->maps.pair_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf(" total:%10lu dispatch:%10lu missing:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_dispatched, -- skel->bss->nr_missing); -- printf(" kicks:%10lu preemptions:%7lu\n", -- skel->bss->nr_kicks, -- skel->bss->nr_preemptions); -- printf(" exp:%10lu exp_wait:%10lu exp_empty:%10lu\n", -- skel->bss->nr_exps, -- skel->bss->nr_exp_waits, -- skel->bss->nr_exp_empty); -- printf("cgnext:%10lu cgcoll:%10lu cgempty:%10lu\n", -- skel->bss->nr_cgrp_next, -- skel->bss->nr_cgrp_coll, -- skel->bss->nr_cgrp_empty); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_pair__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_pair.h a/tools/sched_ext/scx_example_pair.h -deleted file mode 100644 -index f60b824272f7..000000000000 ---- b/tools/sched_ext/scx_example_pair.h -+++ /dev/null -@@ -1,10 +0,0 @@ --#ifndef __SCX_EXAMPLE_PAIR_H --#define __SCX_EXAMPLE_PAIR_H -- --enum { -- MAX_CPUS = 4096, -- MAX_QUEUED = 4096, -- MAX_CGRPS = 4096, --}; -- --#endif /* __SCX_EXAMPLE_PAIR_H */ -diff --git b/tools/sched_ext/scx_example_qmap.bpf.c a/tools/sched_ext/scx_example_qmap.bpf.c -deleted file mode 100644 -index 579ab21ae403..000000000000 ---- b/tools/sched_ext/scx_example_qmap.bpf.c -+++ /dev/null -@@ -1,401 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple five-level FIFO queue scheduler. -- * -- * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets -- * assigned to one depending on its compound weight. Each CPU round robins -- * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from -- * queue0, 2 from queue1, 4 from queue2 and so on. -- * -- * This scheduler demonstrates: -- * -- * - BPF-side queueing using PIDs. -- * - Sleepable per-task storage allocation using ops.prep_enable(). -- * - Using ops.cpu_release() to handle a higher priority scheduling class taking -- * the CPU away. -- * - Core-sched support. -- * -- * This scheduler is primarily for demonstration and testing of sched_ext -- * features and unlikely to be useful for actual workloads. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include -- --char _license[] SEC("license") = "GPL"; -- --const volatile u64 slice_ns = SCX_SLICE_DFL; --const volatile bool switch_partial; --const volatile u32 stall_user_nth; --const volatile u32 stall_kernel_nth; --const volatile u32 dsp_inf_loop_after; --const volatile s32 disallow_tgid; -- --u32 test_error_cnt; -- --struct user_exit_info uei; -- --struct qmap { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, u32); --} queue0 SEC(".maps"), -- queue1 SEC(".maps"), -- queue2 SEC(".maps"), -- queue3 SEC(".maps"), -- queue4 SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, 5); -- __type(key, int); -- __array(values, struct qmap); --} queue_arr SEC(".maps") = { -- .values = { -- [0] = &queue0, -- [1] = &queue1, -- [2] = &queue2, -- [3] = &queue3, -- [4] = &queue4, -- }, --}; -- --/* -- * Per-queue sequence numbers to implement core-sched ordering. -- * -- * Tail seq is assigned to each queued task and incremented. Head seq tracks the -- * sequence number of the latest dispatched task. The distance between the a -- * task's seq and the associated queue's head seq is called the queue distance -- * and used when comparing two tasks for ordering. See qmap_core_sched_before(). -- */ --static u64 core_sched_head_seqs[5]; --static u64 core_sched_tail_seqs[5]; -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local_dsq */ -- u64 core_sched_seq; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --/* Per-cpu dispatch index and remaining count */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(max_entries, 2); -- __type(key, u32); -- __type(value, u64); --} dispatch_idx_cnt SEC(".maps"); -- --/* Statistics */ --unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued; --unsigned long nr_core_sched_execed; -- --s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- struct task_ctx *tctx; -- s32 cpu; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- return cpu; -- -- return prev_cpu; --} -- --static int weight_to_idx(u32 weight) --{ -- /* Coarsely map the compound weight to a FIFO. */ -- if (weight <= 25) -- return 0; -- else if (weight <= 50) -- return 1; -- else if (weight < 200) -- return 2; -- else if (weight < 400) -- return 3; -- else -- return 4; --} -- --void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) --{ -- static u32 user_cnt, kernel_cnt; -- struct task_ctx *tctx; -- u32 pid = p->pid; -- int idx = weight_to_idx(p->scx.weight); -- void *ring; -- -- if (p->flags & PF_KTHREAD) { -- if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth)) -- return; -- } else { -- if (stall_user_nth && !(++user_cnt % stall_user_nth)) -- return; -- } -- -- if (test_error_cnt && !--test_error_cnt) -- scx_bpf_error("test triggering error"); -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * All enqueued tasks must have their core_sched_seq updated for correct -- * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in -- * qmap_ops.flags. -- */ -- tctx->core_sched_seq = core_sched_tail_seqs[idx]++; -- -- /* -- * If qmap_select_cpu() is telling us to or this is the last runnable -- * task on the CPU, enqueue locally. -- */ -- if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) { -- tctx->force_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); -- return; -- } -- -- /* -- * If the task was re-enqueued due to the CPU being preempted by a -- * higher priority scheduling class, just re-enqueue the task directly -- * on the global DSQ. As we want another CPU to pick it up, find and -- * kick an idle CPU. -- */ -- if (enq_flags & SCX_ENQ_REENQ) { -- s32 cpu; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags); -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- return; -- } -- -- ring = bpf_map_lookup_elem(&queue_arr, &idx); -- if (!ring) { -- scx_bpf_error("failed to find ring %d", idx); -- return; -- } -- -- /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */ -- if (bpf_map_push_elem(ring, &pid, 0)) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_enqueued, 1); --} -- --/* -- * The BPF queue map doesn't support removal and sched_ext can handle spurious -- * dispatches. qmap_dequeue() is only used to collect statistics. -- */ --void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags) --{ -- __sync_fetch_and_add(&nr_dequeued, 1); -- if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) -- __sync_fetch_and_add(&nr_core_sched_execed, 1); --} -- --static void update_core_sched_head_seq(struct task_struct *p) --{ -- struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- int idx = weight_to_idx(p->scx.weight); -- -- if (tctx) -- core_sched_head_seqs[idx] = tctx->core_sched_seq; -- else -- scx_bpf_error("task_ctx lookup failed"); --} -- --void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 zero = 0, one = 1; -- u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero); -- u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one); -- void *fifo; -- s32 pid; -- int i; -- -- if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) { -- struct task_struct *p; -- -- /* -- * PID 2 should be kthreadd which should mostly be idle and off -- * the scheduler. Let's keep dispatching it to force the kernel -- * to call this function over and over again. -- */ -- p = bpf_task_from_pid(2); -- if (p) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- if (!idx || !cnt) { -- scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt); -- return; -- } -- -- for (i = 0; i < 5; i++) { -- /* Advance the dispatch cursor and pick the fifo. */ -- if (!*cnt) { -- *idx = (*idx + 1) % 5; -- *cnt = 1 << *idx; -- } -- (*cnt)--; -- -- fifo = bpf_map_lookup_elem(&queue_arr, idx); -- if (!fifo) { -- scx_bpf_error("failed to find ring %llu", *idx); -- return; -- } -- -- /* Dispatch or advance. */ -- if (!bpf_map_pop_elem(fifo, &pid)) { -- struct task_struct *p; -- -- p = bpf_task_from_pid(pid); -- if (p) { -- update_core_sched_head_seq(p); -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- *cnt = 0; -- } --} -- --/* -- * The distance from the head of the queue scaled by the weight of the queue. -- * The lower the number, the older the task and the higher the priority. -- */ --static s64 task_qdist(struct task_struct *p) --{ -- int idx = weight_to_idx(p->scx.weight); -- struct task_ctx *tctx; -- s64 qdist; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return 0; -- } -- -- qdist = tctx->core_sched_seq - core_sched_head_seqs[idx]; -- -- /* -- * As queue index increments, the priority doubles. The queue w/ index 3 -- * is dispatched twice more frequently than 2. Reflect the difference by -- * scaling qdists accordingly. Note that the shift amount needs to be -- * flipped depending on the sign to avoid flipping priority direction. -- */ -- if (qdist >= 0) -- return qdist << (4 - idx); -- else -- return qdist << idx; --} -- --/* -- * This is called to determine the task ordering when core-sched is picking -- * tasks to execute on SMT siblings and should encode about the same ordering as -- * the regular scheduling path. Use the priority-scaled distances from the head -- * of the queues to compare the two tasks which should be consistent with the -- * dispatch path behavior. -- */ --bool BPF_STRUCT_OPS(qmap_core_sched_before, -- struct task_struct *a, struct task_struct *b) --{ -- return task_qdist(a) > task_qdist(b); --} -- --void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- u32 cnt; -- -- /* -- * Called when @cpu is taken by a higher priority scheduling class. This -- * makes @cpu no longer available for executing sched_ext tasks. As we -- * don't want the tasks in @cpu's local dsq to sit there until @cpu -- * becomes available again, re-enqueue them into the global dsq. See -- * %SCX_ENQ_REENQ handling in qmap_enqueue(). -- */ -- cnt = scx_bpf_reenqueue_local(); -- if (cnt) -- __sync_fetch_and_add(&nr_reenqueued, cnt); --} -- --s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (p->tgid == disallow_tgid) -- p->scx.disallow = true; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(qmap_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops qmap_ops = { -- .select_cpu = (void *)qmap_select_cpu, -- .enqueue = (void *)qmap_enqueue, -- .dequeue = (void *)qmap_dequeue, -- .dispatch = (void *)qmap_dispatch, -- .core_sched_before = (void *)qmap_core_sched_before, -- .cpu_release = (void *)qmap_cpu_release, -- .prep_enable = (void *)qmap_prep_enable, -- .init = (void *)qmap_init, -- .exit = (void *)qmap_exit, -- .flags = SCX_OPS_ENQ_LAST, -- .timeout_ms = 5000U, -- .name = "qmap", --}; -diff --git b/tools/sched_ext/scx_example_qmap.c a/tools/sched_ext/scx_example_qmap.c -deleted file mode 100644 -index ccb4814ee61b..000000000000 ---- b/tools/sched_ext/scx_example_qmap.c -+++ /dev/null -@@ -1,107 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_qmap.skel.h" -- --const char help_fmt[] = --"A simple five-level FIFO queue sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" --" -t COUNT Stall every COUNT'th user thread\n" --" -T COUNT Stall every COUNT'th kernel thread\n" --" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n" --" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_qmap *skel; -- struct bpf_link *link; -- int opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_qmap__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) { -- switch (opt) { -- case 's': -- skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; -- break; -- case 'e': -- skel->bss->test_error_cnt = strtoul(optarg, NULL, 0); -- break; -- case 't': -- skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0); -- break; -- case 'T': -- skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0); -- break; -- case 'l': -- skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0); -- break; -- case 'd': -- skel->rodata->disallow_tgid = strtol(optarg, NULL, 0); -- if (skel->rodata->disallow_tgid < 0) -- skel->rodata->disallow_tgid = getpid(); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_qmap__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.qmap_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- long nr_enqueued = skel->bss->nr_enqueued; -- long nr_dispatched = skel->bss->nr_dispatched; -- -- printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n", -- nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, -- skel->bss->nr_reenqueued, skel->bss->nr_dequeued, -- skel->bss->nr_core_sched_execed); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_qmap__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_simple.bpf.c a/tools/sched_ext/scx_example_simple.bpf.c -deleted file mode 100644 -index 4bccca3e2047..000000000000 ---- b/tools/sched_ext/scx_example_simple.bpf.c -+++ /dev/null -@@ -1,128 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple scheduler. -- * -- * By default, it operates as a simple global weighted vtime scheduler and can -- * be switched to FIFO scheduling. It also demonstrates the following niceties. -- * -- * - Statistics tracking how many tasks are queued to local and global dsq's. -- * - Termination notification for userspace. -- * -- * While very simple, this scheduler should work reasonably well on CPUs with a -- * uniform L3 cache topology. While preemption is not implemented, the fact that -- * the scheduling queue is shared across all CPUs means that whatever is at the -- * front of the queue is likely to be executed fairly quickly given enough -- * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads -- * but comes with the usual problems with FIFO scheduling where saturating -- * threads can easily drown out interactive ones. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --static u64 vtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, 2); /* [local, global] */ --} stats SEC(".maps"); -- --static void stat_inc(u32 idx) --{ -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); -- if (cnt_p) -- (*cnt_p)++; --} -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) --{ -- /* -- * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that -- * running @p on its CPU directly shouldn't affect fairness. Just queue -- * it on the local FIFO. -- */ -- if (enq_flags & SCX_ENQ_LOCAL) { -- stat_inc(0); /* count local queueing */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- return; -- } -- -- stat_inc(1); /* count global queueing */ -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) -- vtime = vtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --void BPF_STRUCT_OPS(simple_running, struct task_struct *p) --{ -- if (fifo_sched) -- return; -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(vtime_now, p->scx.dsq_vtime)) -- vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --s32 BPF_STRUCT_OPS(simple_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .running = (void *)simple_running, -- .stopping = (void *)simple_stopping, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", --}; -diff --git b/tools/sched_ext/scx_example_simple.c a/tools/sched_ext/scx_example_simple.c -deleted file mode 100644 -index 486b401f7c95..000000000000 ---- b/tools/sched_ext/scx_example_simple.c -+++ /dev/null -@@ -1,101 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_simple.skel.h" -- --const char help_fmt[] = --"A simple sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-f] [-p]\n" --"\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int simple) --{ -- exit_req = 1; --} -- --static void read_stats(struct scx_example_simple *skel, u64 *stats) --{ -- int nr_cpus = libbpf_num_possible_cpus(); -- u64 cnts[2][nr_cpus]; -- u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * 2); -- -- for (idx = 0; idx < 2; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_simple *skel; -- struct bpf_link *link; -- u32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_simple__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "fph")) != -1) { -- switch (opt) { -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_simple__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.simple_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- u64 stats[2]; -- -- read_stats(skel, stats); -- printf("local=%lu global=%lu\n", stats[0], stats[1]); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_simple__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_userland.bpf.c a/tools/sched_ext/scx_example_userland.bpf.c -deleted file mode 100644 -index a089bc6bbe86..000000000000 ---- b/tools/sched_ext/scx_example_userland.bpf.c -+++ /dev/null -@@ -1,269 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A minimal userland scheduler. -- * -- * In terms of scheduling, this provides two different types of behaviors: -- * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity. -- * All such tasks are direct-dispatched from the kernel, and are never -- * enqueued in user space. -- * 2. A primitive vruntime scheduler that is implemented in user space, for all -- * other tasks. -- * -- * Some parts of this example user space scheduler could be implemented more -- * efficiently using more complex and sophisticated data structures. For -- * example, rather than using BPF_MAP_TYPE_QUEUE's, -- * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between -- * user space and kernel space. Similarly, we use a simple vruntime-sorted list -- * in user space, but an rbtree could be used instead. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include --#include "scx_common.bpf.h" --#include "scx_example_userland_common.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; --const volatile s32 usersched_pid; -- --/* !0 for veristat, set during init */ --const volatile u32 num_possible_cpus = 64; -- --/* Stats that are printed by user space. */ --u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues; -- --struct user_exit_info uei; -- --/* -- * Whether the user space scheduler needs to be scheduled due to a task being -- * enqueued in user space. -- */ --static bool usersched_needed; -- --/* -- * The map containing tasks that are enqueued in user space from the kernel. -- * -- * This map is drained by the user space scheduler. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, struct scx_userland_enqueued_task); --} enqueued SEC(".maps"); -- --/* -- * The map containing tasks that are dispatched to the kernel from user space. -- * -- * Drained by the kernel in userland_dispatch(). -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, s32); --} dispatched SEC(".maps"); -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local DSQ */ --}; -- --/* Map that contains task-local storage. */ --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --static bool is_usersched_task(const struct task_struct *p) --{ -- return p->pid == usersched_pid; --} -- --static bool keep_in_kernel(const struct task_struct *p) --{ -- return p->nr_cpus_allowed < num_possible_cpus; --} -- --static struct task_struct *usersched_task(void) --{ -- struct task_struct *p; -- -- p = bpf_task_from_pid(usersched_pid); -- /* -- * Should never happen -- the usersched task should always be managed -- * by sched_ext. -- */ -- if (!p) { -- scx_bpf_error("Failed to find usersched task %d", usersched_pid); -- /* -- * We should never hit this path, and we error out of the -- * scheduler above just in case, so the scheduler will soon be -- * be evicted regardless. So as to simplify the logic in the -- * caller to not have to check for NULL, return an acquired -- * reference to the current task here rather than NULL. -- */ -- return bpf_task_acquire(bpf_get_current_task_btf()); -- } -- -- return p; --} -- --s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- if (keep_in_kernel(p)) { -- s32 cpu; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to look up task-local storage for %s", p->comm); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- tctx->force_local = true; -- return cpu; -- } -- } -- -- return prev_cpu; --} -- --static void dispatch_user_scheduler(void) --{ -- struct task_struct *p; -- -- usersched_needed = false; -- p = usersched_task(); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); --} -- --static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags) --{ -- struct scx_userland_enqueued_task task; -- -- memset(&task, 0, sizeof(task)); -- task.pid = p->pid; -- task.sum_exec_runtime = p->se.sum_exec_runtime; -- task.weight = p->scx.weight; -- -- if (bpf_map_push_elem(&enqueued, &task, 0)) { -- /* -- * If we fail to enqueue the task in user space, put it -- * directly on the global DSQ. -- */ -- __sync_fetch_and_add(&nr_failed_enqueues, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- __sync_fetch_and_add(&nr_user_enqueues, 1); -- usersched_needed = true; -- } --} -- --void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags) --{ -- if (keep_in_kernel(p)) { -- u64 dsq_id = SCX_DSQ_GLOBAL; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to lookup task ctx for %s", p->comm); -- return; -- } -- -- if (tctx->force_local) -- dsq_id = SCX_DSQ_LOCAL; -- tctx->force_local = false; -- scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags); -- __sync_fetch_and_add(&nr_kernel_enqueues, 1); -- return; -- } else if (!is_usersched_task(p)) { -- enqueue_task_in_user_space(p, enq_flags); -- } --} -- --void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (usersched_needed) -- dispatch_user_scheduler(); -- -- bpf_repeat(4096) { -- s32 pid; -- struct task_struct *p; -- -- if (bpf_map_pop_elem(&dispatched, &pid)) -- break; -- -- /* -- * The task could have exited by the time we get around to -- * dispatching it. Treat this as a normal occurrence, and simply -- * move onto the next iteration. -- */ -- p = bpf_task_from_pid(pid); -- if (!p) -- continue; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } --} -- --s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(userland_init) --{ -- if (num_possible_cpus == 0) { -- scx_bpf_error("User scheduler # CPUs uninitialized (%d)", -- num_possible_cpus); -- return -EINVAL; -- } -- -- if (usersched_pid <= 0) { -- scx_bpf_error("User scheduler pid uninitialized (%d)", -- usersched_pid); -- return -EINVAL; -- } -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops userland_ops = { -- .select_cpu = (void *)userland_select_cpu, -- .enqueue = (void *)userland_enqueue, -- .dispatch = (void *)userland_dispatch, -- .prep_enable = (void *)userland_prep_enable, -- .init = (void *)userland_init, -- .exit = (void *)userland_exit, -- .timeout_ms = 3000, -- .name = "userland", --}; -diff --git b/tools/sched_ext/scx_example_userland.c a/tools/sched_ext/scx_example_userland.c -deleted file mode 100644 -index 4152b1e65fe1..000000000000 ---- b/tools/sched_ext/scx_example_userland.c -+++ /dev/null -@@ -1,402 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext user space scheduler which provides vruntime semantics -- * using a simple ordered-list implementation. -- * -- * Each CPU in the system resides in a single, global domain. This precludes -- * the need to do any load balancing between domains. The scheduler could -- * easily be extended to support multiple domains, with load balancing -- * happening in user space. -- * -- * Any task which has any CPU affinity is scheduled entirely in BPF. This -- * program only schedules tasks which may run on any CPU. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include "user_exit_info.h" --#include "scx_example_userland_common.h" --#include "scx_example_userland.skel.h" -- --const char help_fmt[] = --"A minimal userland sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-b BATCH] [-p]\n" --"\n" --" -b BATCH The number of tasks to batch when dispatching (default: 8)\n" --" -p Don't switch all, switch only tasks on SCHED_EXT policy\n" --" -h Display this help and exit\n"; -- --/* Defined in UAPI */ --#define SCHED_EXT 7 -- --/* Number of tasks to batch when dispatching to user space. */ --static __u32 batch_size = 8; -- --static volatile int exit_req; --static int enqueued_fd, dispatched_fd; -- --static struct scx_example_userland *skel; --static struct bpf_link *ops_link; -- --/* Stats collected in user space. */ --static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches; -- --/* The data structure containing tasks that are enqueued in user space. */ --struct enqueued_task { -- LIST_ENTRY(enqueued_task) entries; -- __u64 sum_exec_runtime; -- double vruntime; --}; -- --/* -- * Use a vruntime-sorted list to store tasks. This could easily be extended to -- * a more optimal data structure, such as an rbtree as is done in CFS. We -- * currently elect to use a sorted list to simplify the example for -- * illustrative purposes. -- */ --LIST_HEAD(listhead, enqueued_task); -- --/* -- * A vruntime-sorted list of tasks. The head of the list contains the task with -- * the lowest vruntime. That is, the task that has the "highest" claim to be -- * scheduled. -- */ --static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head); -- --/* -- * The statically allocated array of tasks. We use a statically allocated list -- * here to avoid having to allocate on the enqueue path, which could cause a -- * deadlock. A more substantive user space scheduler could e.g. provide a hook -- * for newly enabled tasks that are passed to the scheduler from the -- * .prep_enable() callback to allows the scheduler to allocate on safe paths. -- */ --struct enqueued_task tasks[USERLAND_MAX_TASKS]; -- --static double min_vruntime; -- --static void sigint_handler(int userland) --{ -- exit_req = 1; --} -- --static __u32 task_pid(const struct enqueued_task *task) --{ -- return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task); --} -- --static int dispatch_task(s32 pid) --{ -- int err; -- -- err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d\n", pid); -- exit_req = 1; -- } else { -- nr_vruntime_dispatches++; -- } -- -- return err; --} -- --static struct enqueued_task *get_enqueued_task(__s32 pid) --{ -- if (pid >= USERLAND_MAX_TASKS) -- return NULL; -- -- return &tasks[pid]; --} -- --static double calc_vruntime_delta(__u64 weight, __u64 delta) --{ -- double weight_f = (double)weight / 100.0; -- double delta_f = (double)delta; -- -- return delta_f / weight_f; --} -- --static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task) --{ -- __u64 delta; -- -- delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime; -- -- enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta); -- if (min_vruntime > enqueued->vruntime) -- enqueued->vruntime = min_vruntime; -- enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime; --} -- --static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task) --{ -- struct enqueued_task *curr, *enqueued, *prev; -- -- curr = get_enqueued_task(bpf_task->pid); -- if (!curr) -- return ENOENT; -- -- update_enqueued(curr, bpf_task); -- nr_vruntime_enqueues++; -- -- /* -- * Enqueue the task in a vruntime-sorted list. A more optimal data -- * structure such as an rbtree could easily be used as well. We elect -- * to use a list here simply because it's less code, and thus the -- * example is less convoluted and better serves to illustrate what a -- * user space scheduler could look like. -- */ -- -- if (LIST_EMPTY(&vruntime_head)) { -- LIST_INSERT_HEAD(&vruntime_head, curr, entries); -- return 0; -- } -- -- LIST_FOREACH(enqueued, &vruntime_head, entries) { -- if (curr->vruntime <= enqueued->vruntime) { -- LIST_INSERT_BEFORE(enqueued, curr, entries); -- return 0; -- } -- prev = enqueued; -- } -- -- LIST_INSERT_AFTER(prev, curr, entries); -- -- return 0; --} -- --static void drain_enqueued_map(void) --{ -- while (1) { -- struct scx_userland_enqueued_task task; -- int err; -- -- if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) -- return; -- -- err = vruntime_enqueue(&task); -- if (err) { -- fprintf(stderr, "Failed to enqueue task %d: %s\n", -- task.pid, strerror(err)); -- exit_req = 1; -- return; -- } -- } --} -- --static void dispatch_batch(void) --{ -- __u32 i; -- -- for (i = 0; i < batch_size; i++) { -- struct enqueued_task *task; -- int err; -- __s32 pid; -- -- task = LIST_FIRST(&vruntime_head); -- if (!task) -- return; -- -- min_vruntime = task->vruntime; -- pid = task_pid(task); -- LIST_REMOVE(task, entries); -- err = dispatch_task(pid); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d in %u\n", -- pid, i); -- return; -- } -- } --} -- --static void *run_stats_printer(void *arg) --{ -- while (!exit_req) { -- __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; -- -- nr_failed_enqueues = skel->bss->nr_failed_enqueues; -- nr_kernel_enqueues = skel->bss->nr_kernel_enqueues; -- nr_user_enqueues = skel->bss->nr_user_enqueues; -- total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues; -- -- printf("o-----------------------o\n"); -- printf("| BPF ENQUEUES |\n"); -- printf("|-----------------------|\n"); -- printf("| kern: %10llu |\n", nr_kernel_enqueues); -- printf("| user: %10llu |\n", nr_user_enqueues); -- printf("| failed: %10llu |\n", nr_failed_enqueues); -- printf("| -------------------- |\n"); -- printf("| total: %10llu |\n", total); -- printf("| |\n"); -- printf("|-----------------------|\n"); -- printf("| VRUNTIME / USER |\n"); -- printf("|-----------------------|\n"); -- printf("| enq: %10llu |\n", nr_vruntime_enqueues); -- printf("| disp: %10llu |\n", nr_vruntime_dispatches); -- printf("o-----------------------o\n"); -- printf("\n\n"); -- sleep(1); -- } -- -- return NULL; --} -- --static int spawn_stats_thread(void) --{ -- pthread_t stats_printer; -- -- return pthread_create(&stats_printer, NULL, run_stats_printer, NULL); --} -- --static int bootstrap(int argc, char **argv) --{ -- int err; -- __u32 opt; -- struct sched_param sched_param = { -- .sched_priority = sched_get_priority_max(SCHED_EXT), -- }; -- bool switch_partial = false; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- /* -- * Enforce that the user scheduler task is managed by sched_ext. The -- * task eagerly drains the list of enqueued tasks in its main work -- * loop, and then yields the CPU. The BPF scheduler only schedules the -- * user space scheduler task when at least one other task in the system -- * needs to be scheduled. -- */ -- err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param); -- if (err) { -- fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err)); -- return err; -- } -- -- while ((opt = getopt(argc, argv, "b:ph")) != -1) { -- switch (opt) { -- case 'b': -- batch_size = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- exit(opt != 'h'); -- } -- } -- -- /* -- * It's not always safe to allocate in a user space scheduler, as an -- * enqueued task could hold a lock that we require in order to be able -- * to allocate. -- */ -- err = mlockall(MCL_CURRENT | MCL_FUTURE); -- if (err) { -- fprintf(stderr, "Failed to prefault and lock address space: %s\n", -- strerror(err)); -- return err; -- } -- -- skel = scx_example_userland__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno)); -- return errno; -- } -- skel->rodata->num_possible_cpus = libbpf_num_possible_cpus(); -- assert(skel->rodata->num_possible_cpus > 0); -- skel->rodata->usersched_pid = getpid(); -- assert(skel->rodata->usersched_pid > 0); -- skel->rodata->switch_partial = switch_partial; -- -- err = scx_example_userland__load(skel); -- if (err) { -- fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- enqueued_fd = bpf_map__fd(skel->maps.enqueued); -- dispatched_fd = bpf_map__fd(skel->maps.dispatched); -- assert(enqueued_fd > 0); -- assert(dispatched_fd > 0); -- -- err = spawn_stats_thread(); -- if (err) { -- fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops); -- if (!ops_link) { -- fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno)); -- err = errno; -- goto destroy_skel; -- } -- -- return 0; -- --destroy_skel: -- scx_example_userland__destroy(skel); -- exit_req = 1; -- return err; --} -- --static void sched_main_loop(void) --{ -- while (!exit_req) { -- /* -- * Perform the following work in the main user space scheduler -- * loop: -- * -- * 1. Drain all tasks from the enqueued map, and enqueue them -- * to the vruntime sorted list. -- * -- * 2. Dispatch a batch of tasks from the vruntime sorted list -- * down to the kernel. -- * -- * 3. Yield the CPU back to the system. The BPF scheduler will -- * reschedule the user space scheduler once another task has -- * been enqueued to user space. -- */ -- drain_enqueued_map(); -- dispatch_batch(); -- sched_yield(); -- } --} -- --int main(int argc, char **argv) --{ -- int err; -- -- err = bootstrap(argc, argv); -- if (err) { -- fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err)); -- return err; -- } -- -- sched_main_loop(); -- -- exit_req = 1; -- bpf_link__destroy(ops_link); -- uei_print(&skel->bss->uei); -- scx_example_userland__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_userland_common.h a/tools/sched_ext/scx_example_userland_common.h -deleted file mode 100644 -index 639c6809c5ff..000000000000 ---- b/tools/sched_ext/scx_example_userland_common.h -+++ /dev/null -@@ -1,19 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* Copyright (c) 2022 Meta, Inc */ -- --#ifndef __SCX_USERLAND_COMMON_H --#define __SCX_USERLAND_COMMON_H -- --#define USERLAND_MAX_TASKS 8192 -- --/* -- * An instance of a task that has been enqueued by the kernel for consumption -- * by a user space global scheduler thread. -- */ --struct scx_userland_enqueued_task { -- __s32 pid; -- u64 sum_exec_runtime; -- u64 weight; --}; -- --#endif // __SCX_USERLAND_COMMON_H -diff --git b/tools/sched_ext/user_exit_info.h a/tools/sched_ext/user_exit_info.h -deleted file mode 100644 -index e701ef0e0b86..000000000000 ---- b/tools/sched_ext/user_exit_info.h -+++ /dev/null -@@ -1,50 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Define struct user_exit_info which is shared between BPF and userspace parts -- * to communicate exit status and other information. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __USER_EXIT_INFO_H --#define __USER_EXIT_INFO_H -- --struct user_exit_info { -- int type; -- char reason[128]; -- char msg[1024]; --}; -- --#ifdef __bpf__ -- --#include "vmlinux.h" --#include -- --static inline void uei_record(struct user_exit_info *uei, -- const struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason); -- bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg); -- /* use __sync to force memory barrier */ -- __sync_val_compare_and_swap(&uei->type, uei->type, ei->type); --} -- --#else /* !__bpf__ */ -- --static inline bool uei_exited(struct user_exit_info *uei) --{ -- /* use __sync to force memory barrier */ -- return __sync_val_compare_and_swap(&uei->type, -1, -1); --} -- --static inline void uei_print(const struct user_exit_info *uei) --{ -- fprintf(stderr, "EXIT: %s", uei->reason); -- if (uei->msg[0] != '\0') -- fprintf(stderr, " (%s)", uei->msg); -- fputs("\n", stderr); --} -- --#endif /* __bpf__ */ --#endif /* __USER_EXIT_INFO_H */ diff --git a/oneplus/hmbird/fengchi_OP-ACE-5-ULTRA_A15.patch b/oneplus/hmbird/fengchi_OP-ACE-5-ULTRA_A15.patch deleted file mode 100644 index 42ddfa7e..00000000 --- a/oneplus/hmbird/fengchi_OP-ACE-5-ULTRA_A15.patch +++ /dev/null @@ -1,20969 +0,0 @@ -diff --git a/Documentation/scheduler/index.rst b/Documentation/scheduler/index.rst -index 0b650bb550e6..3170747226f6 100644 ---- a/Documentation/scheduler/index.rst -+++ b/Documentation/scheduler/index.rst -@@ -19,7 +19,6 @@ Scheduler - sched-nice-design - sched-rt-group - sched-stats -- sched-ext - sched-debug - - text_files -diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst -deleted file mode 100644 -index 84c30b44f104..000000000000 ---- a/Documentation/scheduler/sched-ext.rst -+++ /dev/null -@@ -1,230 +0,0 @@ --========================== --Extensible Scheduler Class --========================== -- --sched_ext is a scheduler class whose behavior can be defined by a set of BPF --programs - the BPF scheduler. -- --* sched_ext exports a full scheduling interface so that any scheduling -- algorithm can be implemented on top. -- --* The BPF scheduler can group CPUs however it sees fit and schedule them -- together, as tasks aren't tied to specific CPUs at the time of wakeup. -- --* The BPF scheduler can be turned on and off dynamically anytime. -- --* The system integrity is maintained no matter what the BPF scheduler does. -- The default scheduling behavior is restored anytime an error is detected, -- a runnable task stalls, or on invoking the SysRq key sequence -- :kbd:`SysRq-S`. -- --Switching to and from sched_ext --=============================== -- --``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and --``tools/sched_ext`` contains the example schedulers. -- --sched_ext is used only when the BPF scheduler is loaded and running. -- --If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be --treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is --loaded. On load, such tasks will be switched to and scheduled by sched_ext. -- --The BPF scheduler can choose to schedule all normal and lower class tasks by --calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this --case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and --``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers, --this mode can be selected with the ``-a`` option. -- --Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or --detection of any internal error including stalled runnable tasks aborts the --BPF scheduler and reverts all tasks back to CFS. -- --.. code-block:: none -- -- # make -j16 -C tools/sched_ext -- # tools/sched_ext/scx_example_simple -- local=0 global=3 -- local=5 global=24 -- local=9 global=44 -- local=13 global=56 -- local=17 global=72 -- ^CEXIT: BPF scheduler unregistered -- --If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler --and whether a given task is on sched_ext can be determined as follows: -- --.. code-block:: none -- -- # cat /sys/kernel/debug/sched/ext -- ops : simple -- enabled : 1 -- switching_all : 1 -- switched_all : 1 -- enable_state : enabled -- -- # grep ext /proc/self/sched -- ext.enabled : 1 -- --The Basics --========== -- --Userspace can implement an arbitrary BPF scheduler by loading a set of BPF --programs that implement ``struct sched_ext_ops``. The only mandatory field --is ``ops.name`` which must be a valid BPF object name. All operations are --optional. The following modified excerpt is from --``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO --scheduler. -- --.. code-block:: c -- -- s32 BPF_STRUCT_OPS(simple_init) -- { -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; -- } -- -- void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) -- { -- if (enq_flags & SCX_ENQ_LOCAL) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags); -- } -- -- void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) -- { -- exit_type = ei->type; -- } -- -- SEC(".struct_ops") -- struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", -- }; -- --Dispatch Queues ----------------- -- --To match the impedance between the scheduler core and the BPF scheduler, --sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a --priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``), --and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage --an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and --``scx_bpf_destroy_dsq()``. -- --A CPU always executes a task from its local DSQ. A task is "dispatched" to a --DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's --local DSQ. -- --When a CPU is looking for the next task to run, if the local DSQ is not --empty, the first task is picked. Otherwise, the CPU tries to consume the --global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()`` --is invoked. -- --Scheduling Cycle ------------------ -- --The following briefly shows how a waking task is scheduled and executed. -- --1. When a task is waking up, ``ops.select_cpu()`` is the first operation -- invoked. This serves two purposes. First, CPU selection optimization -- hint. Second, waking up the selected CPU if idle. -- -- The CPU selected by ``ops.select_cpu()`` is an optimization hint and not -- binding. The actual decision is made at the last step of scheduling. -- However, there is a small performance gain if the CPU -- ``ops.select_cpu()`` returns matches the CPU the task eventually runs on. -- -- A side-effect of selecting a CPU is waking it up from idle. While a BPF -- scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper, -- using ``ops.select_cpu()`` judiciously can be simpler and more efficient. -- -- Note that the scheduler core will ignore an invalid CPU selection, for -- example, if it's outside the allowed cpumask of the task. -- --2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can -- make one of the following decisions: -- -- * Immediately dispatch the task to either the global or local DSQ by -- calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or -- ``SCX_DSQ_LOCAL``, respectively. -- -- * Immediately dispatch the task to a custom DSQ by calling -- ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63. -- -- * Queue the task on the BPF side. -- --3. When a CPU is ready to schedule, it first looks at its local DSQ. If -- empty, it then looks at the global DSQ. If there still isn't a task to -- run, ``ops.dispatch()`` is invoked which can use the following two -- functions to populate the local DSQ. -- -- * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can -- be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``, -- ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()`` -- currently can't be called with BPF locks held, this is being worked on -- and will be supported. ``scx_bpf_dispatch()`` schedules dispatching -- rather than performing them immediately. There can be up to -- ``ops.dispatch_max_batch`` pending tasks. -- -- * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ -- to the dispatching DSQ. This function cannot be called with any BPF -- locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks -- before trying to consume the specified DSQ. -- --4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ, -- the CPU runs the first one. If empty, the following steps are taken: -- -- * Try to consume the global DSQ. If successful, run the task. -- -- * If ``ops.dispatch()`` has dispatched any tasks, retry #3. -- -- * If the previous task is an SCX task and still runnable, keep executing -- it (see ``SCX_OPS_ENQ_LAST``). -- -- * Go idle. -- --Note that the BPF scheduler can always choose to dispatch tasks immediately --in ``ops.enqueue()`` as illustrated in the above simple example. If only the --built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as --a task is never queued on the BPF scheduler and both the local and global --DSQs are consumed automatically. -- --``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use --``scx_bpf_dispatch_vtime()`` for the priority queue. See the function --documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for --more information. -- --Where to Look --============= -- --* ``include/linux/sched/ext.h`` defines the core data structures, ops table -- and constants. -- --* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. -- The functions prefixed with ``scx_bpf_`` can be called from the BPF -- scheduler. -- --* ``tools/sched_ext/`` hosts example BPF scheduler implementations. -- -- * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example -- using a custom DSQ. -- -- * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting -- five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. -- --ABI Instability --=============== -- --The APIs provided by sched_ext to BPF schedulers programs have no stability --guarantees. This includes the ops table callbacks and constants defined in --``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in --``kernel/sched/ext.c``. -- --While we will attempt to provide a relatively stable API surface when --possible, they are subject to change without warning between kernel --versions. -diff --git a/MAINTAINERS b/MAINTAINERS -index 9fc6108503c3..2384b55682ee 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -19132,8 +19132,6 @@ R: Ben Segall (CONFIG_CFS_BANDWIDTH) - R: Mel Gorman (CONFIG_NUMA_BALANCING) - R: Daniel Bristot de Oliveira (SCHED_DEADLINE) - R: Valentin Schneider (TOPOLOGY) --R: Tejun Heo (SCHED_EXT) --R: David Vernet (SCHED_EXT) - L: linux-kernel@vger.kernel.org - S: Maintained - T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core -@@ -19142,7 +19140,6 @@ F: include/linux/sched.h - F: include/linux/wait.h - F: include/uapi/linux/sched.h - F: kernel/sched/ --F: tools/sched_ext/ - - SCSI LIBSAS SUBSYSTEM - R: John Garry -diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig -index f93980c09d7f..5ba590235a8c 100644 ---- a/arch/arm64/configs/defconfig -+++ b/arch/arm64/configs/defconfig -@@ -6,8 +6,7 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_BPF_SYSCALL=y - CONFIG_BPF_JIT=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_BSD_PROCESS_ACCT=y - CONFIG_BSD_PROCESS_ACCT_V3=y -diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig -index 4f756dca5e05..6c1bb94f875d 100644 ---- a/arch/arm64/configs/gki_defconfig -+++ b/arch/arm64/configs/gki_defconfig -@@ -10,8 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y - # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set - CONFIG_BPF_LSM=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_TASKSTATS=y - CONFIG_TASK_DELAY_ACCT=y -diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c -index 4bd028fa7500..5825ce9d6d7b 100644 ---- a/drivers/gpu/drm/msm/msm_drv.c -+++ b/drivers/gpu/drm/msm/msm_drv.c -@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) - } - - sched_set_fifo(ev_thread->worker->task); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3); -+#endif - } - - ret = drm_vblank_init(ddev, priv->num_crtcs); -diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c -index bd001445815e..dcf2e1ebf9c5 100644 ---- a/drivers/tty/sysrq.c -+++ b/drivers/tty/sysrq.c -@@ -524,7 +524,6 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { - NULL, /* P */ - NULL, /* Q */ - NULL, /* R */ -- /* S: May be registered by sched_ext for resetting */ - NULL, /* S */ - NULL, /* T */ - NULL, /* U */ -diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h -index c45078a445c8..6c15659f874e 100644 ---- a/include/asm-generic/vmlinux.lds.h -+++ b/include/asm-generic/vmlinux.lds.h -@@ -131,7 +131,7 @@ - *(__dl_sched_class) \ - *(__rt_sched_class) \ - *(__fair_sched_class) \ -- *(__ext_sched_class) \ -+ *(__hmbird_sched_class) \ - *(__idle_sched_class) \ - __sched_class_lowest = .; - -diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h -index 63452733a2f9..3a2f906ed3ce 100644 ---- a/include/linux/cpufreq.h -+++ b/include/linux/cpufreq.h -@@ -44,6 +44,10 @@ enum cpufreq_table_sorting { - CPUFREQ_TABLE_SORTED_DESCENDING - }; - -+ssize_t store_scaling_governor(struct cpufreq_policy *policy, -+ const char *buf, size_t count); -+ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf); -+ - struct cpufreq_cpuinfo { - unsigned int max_freq; - unsigned int min_freq; -diff --git a/include/linux/sched.h b/include/linux/sched.h -index 2cbc342db12d..dd6b1d58af01 100644 ---- a/include/linux/sched.h -+++ b/include/linux/sched.h -@@ -73,7 +73,9 @@ struct task_delay_info; - struct task_group; - struct user_event_mm; - --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - - /* - * Task state bitmask. NOTE! These bits are also -@@ -298,11 +300,6 @@ enum { - - extern void scheduler_tick(void); - --#ifdef CONFIG_SLIM_SCHED --extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer); --extern void stop_shadow_tick_timer(void); --#endif -- - #define MAX_SCHEDULE_TIMEOUT LONG_MAX - - extern long schedule_timeout(long timeout); -@@ -1523,13 +1520,8 @@ struct task_struct { - */ - struct callback_head l1d_flush_kill; - #endif --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, unsigned long sched_prop); -- ANDROID_KABI_USE(2, struct sched_ext_entity *scx); --#else - ANDROID_KABI_RESERVE(1); - ANDROID_KABI_RESERVE(2); --#endif - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); - ANDROID_KABI_RESERVE(5); -@@ -1933,6 +1925,92 @@ static inline int task_nice(const struct task_struct *p) - return PRIO_TO_NICE((p)->static_prio); - } - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline bool task_is_top_task(struct task_struct *p) -+{ -+ return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ ->top_task_prop & TOP_TASK_BITS_MASK); -+} -+ -+static inline int get_top_task_prop(struct task_struct *p) -+{ -+ return get_hmbird_ts(p)->top_task_prop; -+} -+ -+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear) -+{ -+ if (set) -+ get_hmbird_ts(p)->top_task_prop |= set; -+ if (clear) -+ get_hmbird_ts(p)->top_task_prop &= ~clear; -+ return 0; -+} -+ -+static inline void reset_top_task_prop(struct task_struct *p) -+{ -+ get_hmbird_ts(p)->top_task_prop = 0; -+} -+ -+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ entity->sched_prop = sp; -+ } -+ return 0; -+} -+ -+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->sched_prop; -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq) -+{ -+ unsigned long new_dsq; -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq; -+ entity->sched_prop = new_dsq; -+ } -+} -+ -+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK); -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ entity->dsq_sync_ux = val; -+} -+ -+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->dsq_sync_ux; -+ else -+ return 0; -+} -+#endif -+ - extern int can_nice(const struct task_struct *p, const int nice); - extern int task_curr(const struct task_struct *p); - extern int idle_cpu(int cpu); -diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h -deleted file mode 100755 -index b737a00c1373..000000000000 ---- a/include/linux/sched/ext.h -+++ /dev/null -@@ -1,647 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef _LINUX_SCHED_EXT_H --#define _LINUX_SCHED_EXT_H -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --#include -- --enum scx_consts { -- SCX_OPS_NAME_LEN = 128, -- SCX_EXIT_REASON_LEN = 128, -- SCX_EXIT_BT_LEN = 64, -- SCX_EXIT_MSG_LEN = 1024, -- -- SCX_SLICE_DFL = 20 * NSEC_PER_MSEC, -- SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */ --}; -- --/* -- * DSQ (dispatch queue) IDs are 64bit of the format: -- * -- * Bits: [63] [62 .. 0] -- * [ B] [ ID ] -- * -- * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -- * ID: 63 bit ID -- * -- * Built-in IDs: -- * -- * Bits: [63] [62] [61..32] [31 .. 0] -- * [ 1] [ L] [ R ] [ V ] -- * -- * 1: 1 for built-in DSQs. -- * L: 1 for LOCAL_ON DSQ IDs, 0 for others -- * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -- */ --enum scx_dsq_id_flags { -- SCX_DSQ_FLAG_BUILTIN = 1LLU << 63, -- SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -- -- SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0, -- SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, -- SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, -- SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, -- SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, --}; -- --enum scx_exit_type { -- SCX_EXIT_NONE, -- SCX_EXIT_DONE, -- -- SCX_EXIT_UNREG = 64, /* BPF unregistration */ -- SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -- SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ -- SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ --}; -- --/* -- * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is -- * being disabled. -- */ --struct scx_exit_info { -- /* %SCX_EXIT_* - broad category of the exit reason */ -- enum scx_exit_type type; -- /* textual representation of the above */ -- char reason[SCX_EXIT_REASON_LEN]; -- /* number of entries in the backtrace */ -- u32 bt_len; -- /* backtrace if exiting due to an error */ -- unsigned long bt[SCX_EXIT_BT_LEN]; -- /* extra message */ -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --/* sched_ext_ops.flags */ --enum scx_ops_flags { -- /* -- * Keep built-in idle tracking even if ops.update_idle() is implemented. -- */ -- SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, -- -- /* -- * By default, if there are no other task to run on the CPU, ext core -- * keeps running the current task even after its slice expires. If this -- * flag is specified, such tasks are passed to ops.enqueue() with -- * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. -- */ -- SCX_OPS_ENQ_LAST = 1LLU << 1, -- -- /* -- * An exiting task may schedule after PF_EXITING is set. In such cases, -- * bpf_task_from_pid() may not be able to find the task and if the BPF -- * scheduler depends on pid lookup for dispatching, the task will be -- * lost leading to various issues including RCU grace period stalls. -- * -- * To mask this problem, by default, unhashed tasks are automatically -- * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't -- * depend on pid lookups and wants to handle these tasks directly, the -- * following flag can be used. -- */ -- SCX_OPS_ENQ_EXITING = 1LLU << 2, -- -- /* -- * CPU cgroup knob enable flags -- */ -- SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16, /* cpu.weight */ -- -- SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | -- SCX_OPS_ENQ_LAST | -- SCX_OPS_ENQ_EXITING | -- SCX_OPS_CGROUP_KNOB_WEIGHT, --}; -- --/* argument container for ops.enable() and friends */ --struct scx_enable_args { --}; -- --/* argument container for ops->cgroup_init() */ --struct scx_cgroup_init_args { -- /* the weight of the cgroup [1..10000] */ -- u32 weight; --}; -- --enum scx_cpu_preempt_reason { -- /* next task is being scheduled by &sched_class_rt */ -- SCX_CPU_PREEMPT_RT, -- /* next task is being scheduled by &sched_class_dl */ -- SCX_CPU_PREEMPT_DL, -- /* next task is being scheduled by &sched_class_stop */ -- SCX_CPU_PREEMPT_STOP, -- /* unknown reason for SCX being preempted */ -- SCX_CPU_PREEMPT_UNKNOWN, --}; -- --/* -- * Argument container for ops->cpu_acquire(). Currently empty, but may be -- * expanded in the future. -- */ --struct scx_cpu_acquire_args {}; -- --/* argument container for ops->cpu_release() */ --struct scx_cpu_release_args { -- /* the reason the CPU was preempted */ -- enum scx_cpu_preempt_reason reason; -- -- /* the task that's going to be scheduled on the CPU */ -- struct task_struct *task; --}; -- --/** -- * struct sched_ext_ops - Operation table for BPF scheduler implementation -- * -- * Userland can implement an arbitrary scheduling policy by implementing and -- * loading operations in this table. -- */ --struct sched_ext_ops { -- /** -- * select_cpu - Pick the target CPU for a task which is being woken up -- * @p: task being woken up -- * @prev_cpu: the cpu @p was on before sleeping -- * @wake_flags: SCX_WAKE_* -- * -- * Decision made here isn't final. @p may be moved to any CPU while it -- * is getting dispatched for execution later. However, as @p is not on -- * the rq at this point, getting the eventual execution CPU right here -- * saves a small bit of overhead down the line. -- * -- * If an idle CPU is returned, the CPU is kicked and will try to -- * dispatch. While an explicit custom mechanism can be added, -- * select_cpu() serves as the default way to wake up idle CPUs. -- */ -- s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); -- -- /** -- * enqueue - Enqueue a task on the BPF scheduler -- * @p: task being enqueued -- * @enq_flags: %SCX_ENQ_* -- * -- * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() -- * or enqueue on the BPF scheduler. If not directly dispatched, the bpf -- * scheduler owns @p and if it fails to dispatch @p, the task will -- * stall. -- */ -- void (*enqueue)(struct task_struct *p, u64 enq_flags); -- -- /** -- * dequeue - Remove a task from the BPF scheduler -- * @p: task being dequeued -- * @deq_flags: %SCX_DEQ_* -- * -- * Remove @p from the BPF scheduler. This is usually called to isolate -- * the task while updating its scheduling properties (e.g. priority). -- * -- * The ext core keeps track of whether the BPF side owns a given task or -- * not and can gracefully ignore spurious dispatches from BPF side, -- * which makes it safe to not implement this method. However, depending -- * on the scheduling logic, this can lead to confusing behaviors - e.g. -- * scheduling position not being updated across a priority change. -- */ -- void (*dequeue)(struct task_struct *p, u64 deq_flags); -- -- /** -- * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs -- * @cpu: CPU to dispatch tasks for -- * @prev: previous task being switched out -- * -- * Called when a CPU's local dsq is empty. The operation should dispatch -- * one or more tasks from the BPF scheduler into the DSQs using -- * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using -- * scx_bpf_consume(). -- * -- * The maximum number of times scx_bpf_dispatch() can be called without -- * an intervening scx_bpf_consume() is specified by -- * ops.dispatch_max_batch. See the comments on top of the two functions -- * for more details. -- * -- * When not %NULL, @prev is an SCX task with its slice depleted. If -- * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in -- * @prev->scx.flags, it is not enqueued yet and will be enqueued after -- * ops.dispatch() returns. To keep executing @prev, return without -- * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. -- */ -- void (*dispatch)(s32 cpu, struct task_struct *prev); -- -- /** -- * runnable - A task is becoming runnable on its associated CPU -- * @p: task becoming runnable -- * @enq_flags: %SCX_ENQ_* -- * -- * This and the following three functions can be used to track a task's -- * execution state transitions. A task becomes ->runnable() on a CPU, -- * and then goes through one or more ->running() and ->stopping() pairs -- * as it runs on the CPU, and eventually becomes ->quiescent() when it's -- * done running on the CPU. -- * -- * @p is becoming runnable on the CPU because it's -- * -- * - waking up (%SCX_ENQ_WAKEUP) -- * - being moved from another CPU -- * - being restored after temporarily taken off the queue for an -- * attribute change. -- * -- * This and ->enqueue() are related but not coupled. This operation -- * notifies @p's state transition and may not be followed by ->enqueue() -- * e.g. when @p is being dispatched to a remote CPU. Likewise, a task -- * may be ->enqueue()'d without being preceded by this operation e.g. -- * after exhausting its slice. -- */ -- void (*runnable)(struct task_struct *p, u64 enq_flags); -- -- /** -- * running - A task is starting to run on its associated CPU -- * @p: task starting to run -- * -- * See ->runnable() for explanation on the task state notifiers. -- */ -- void (*running)(struct task_struct *p); -- -- /** -- * stopping - A task is stopping execution -- * @p: task stopping to run -- * @runnable: is task @p still runnable? -- * -- * See ->runnable() for explanation on the task state notifiers. If -- * !@runnable, ->quiescent() will be invoked after this operation -- * returns. -- */ -- void (*stopping)(struct task_struct *p, bool runnable); -- -- /** -- * quiescent - A task is becoming not runnable on its associated CPU -- * @p: task becoming not runnable -- * @deq_flags: %SCX_DEQ_* -- * -- * See ->runnable() for explanation on the task state notifiers. -- * -- * @p is becoming quiescent on the CPU because it's -- * -- * - sleeping (%SCX_DEQ_SLEEP) -- * - being moved to another CPU -- * - being temporarily taken off the queue for an attribute change -- * (%SCX_DEQ_SAVE) -- * -- * This and ->dequeue() are related but not coupled. This operation -- * notifies @p's state transition and may not be preceded by ->dequeue() -- * e.g. when @p is being dispatched to a remote CPU. -- */ -- void (*quiescent)(struct task_struct *p, u64 deq_flags); -- -- /** -- * yield - Yield CPU -- * @from: yielding task -- * @to: optional yield target task -- * -- * If @to is NULL, @from is yielding the CPU to other runnable tasks. -- * The BPF scheduler should ensure that other available tasks are -- * dispatched before the yielding task. Return value is ignored in this -- * case. -- * -- * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf -- * scheduler can implement the request, return %true; otherwise, %false. -- */ -- bool (*yield)(struct task_struct *from, struct task_struct *to); -- -- /** -- * core_sched_before - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Used by core-sched to determine the ordering between two tasks. See -- * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on -- * core-sched. -- * -- * Both @a and @b are runnable and may or may not currently be queued on -- * the BPF scheduler. Should return %true if @a should run before @b. -- * %false if there's no required ordering or @b should run before @a. -- * -- * If not specified, the default is ordering them according to when they -- * became runnable. -- */ -- bool (*core_sched_before)(struct task_struct *a,struct task_struct *b); -- -- /** -- * set_weight - Set task weight -- * @p: task to set weight for -- * @weight: new eight [1..10000] -- * -- * Update @p's weight to @weight. -- */ -- void (*set_weight)(struct task_struct *p, u32 weight); -- -- /** -- * set_cpumask - Set CPU affinity -- * @p: task to set CPU affinity for -- * @cpumask: cpumask of cpus that @p can run on -- * -- * Update @p's CPU affinity to @cpumask. -- */ -- void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask); -- -- /** -- * update_idle - Update the idle state of a CPU -- * @cpu: CPU to udpate the idle state for -- * @idle: whether entering or exiting the idle state -- * -- * This operation is called when @rq's CPU goes or leaves the idle -- * state. By default, implementing this operation disables the built-in -- * idle CPU tracking and the following helpers become unavailable: -- * -- * - scx_bpf_select_cpu_dfl() -- * - scx_bpf_test_and_clear_cpu_idle() -- * - scx_bpf_pick_idle_cpu() -- * - scx_bpf_any_idle_cpu() -- * -- * The user also must implement ops.select_cpu() as the default -- * implementation relies on scx_bpf_select_cpu_dfl(). -- * -- * If you keep the built-in idle tracking, specify the -- * %SCX_OPS_KEEP_BUILTIN_IDLE flag. -- */ -- void (*update_idle)(s32 cpu, bool idle); -- -- /** -- * cpu_acquire - A CPU is becoming available to the BPF scheduler -- * @cpu: The CPU being acquired by the BPF scheduler. -- * @args: Acquire arguments, see the struct definition. -- * -- * A CPU that was previously released from the BPF scheduler is now once -- * again under its control. -- */ -- void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); -- -- /** -- * cpu_release - A CPU is taken away from the BPF scheduler -- * @cpu: The CPU being released by the BPF scheduler. -- * @args: Release arguments, see the struct definition. -- * -- * The specified CPU is no longer under the control of the BPF -- * scheduler. This could be because it was preempted by a higher -- * priority sched_class, though there may be other reasons as well. The -- * caller should consult @args->reason to determine the cause. -- */ -- void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -- -- /** -- * cpu_online - A CPU became online -- * @cpu: CPU which just came up -- * -- * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs beforehand. -- */ -- void (*cpu_online)(s32 cpu); -- -- /** -- * cpu_offline - A CPU is going offline -- * @cpu: CPU which is going offline -- * -- * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs afterwards. -- */ -- void (*cpu_offline)(s32 cpu); -- -- /** -- * prep_enable - Prepare to enable BPF scheduling for a task -- * @p: task to prepare BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Either we're loading a BPF scheduler or a new task is being forked. -- * Prepare BPF scheduling for @p. This operation may block and can be -- * used for allocations. -- * -- * Return 0 for success, -errno for failure. An error return while -- * loading will abort loading of the BPF scheduler. During a fork, will -- * abort the specific fork. -- */ -- s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * enable - Enable BPF scheduling for a task -- * @p: task to enable BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Enable @p for BPF scheduling. @p is now in the cgroup specified for -- * the preceding prep_enable() and will start running soon. -- */ -- void (*enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * cancel_enable - Cancel prep_enable() -- * @p: task being canceled -- * @args: enable arguments, see the struct definition -- * -- * @p was prep_enable()'d but failed before reaching enable(). Undo the -- * preparation. -- */ -- void (*cancel_enable)(struct task_struct *p, -- struct scx_enable_args *args); -- -- /** -- * disable - Disable BPF scheduling for a task -- * @p: task to disable BPF scheduling for -- * -- * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. -- * Disable BPF scheduling for @p. -- */ -- void (*disable)(struct task_struct *p); -- -- /* -- * All online ops must come before ops.init(). -- */ -- -- /** -- * init - Initialize the BPF scheduler -- */ -- s32 (*init)(void); -- -- /** -- * exit - Clean up after the BPF scheduler -- * @info: Exit info -- */ -- void (*exit)(struct scx_exit_info *info); -- -- /** -- * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch -- */ -- u32 dispatch_max_batch; -- -- /** -- * flags - %SCX_OPS_* flags -- */ -- u64 flags; -- -- /** -- * timeout_ms - The maximum amount of time, in milliseconds, that a -- * runnable task should be able to wait before being scheduled. The -- * maximum timeout may not exceed the default timeout of 30 seconds. -- * -- * Defaults to the maximum allowed timeout value of 30 seconds. -- */ -- u32 timeout_ms; -- -- /** -- * name - BPF scheduler's name -- * -- * Must be a non-zero valid BPF object name including only isalnum(), -- * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the -- * BPF scheduler is enabled. -- */ -- char name[SCX_OPS_NAME_LEN]; --}; -- --/* -- * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -- * scheduler core and the BPF scheduler. See the documentation for more details. -- */ --struct scx_dispatch_q { -- raw_spinlock_t lock; -- struct list_head fifo; /* processed in dispatching order */ -- struct rb_root_cached priq; /* processed in p->scx.dsq_vtime order */ -- u32 nr; -- u64 id; -- struct llist_node free_node; -- struct rcu_head rcu; --}; -- --/* scx_entity.flags */ --enum scx_ent_flags { -- SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ -- SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -- SCX_TASK_ENQ_LOCAL = 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */ -- -- SCX_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -- SCX_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -- -- SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -- SCX_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -- -- SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ --}; -- --/* scx_entity.dsq_flags */ --enum scx_ent_dsq_flags { -- SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ --}; -- --/* -- * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from -- * everywhere and the following bits track which kfunc sets are currently -- * allowed for %current. This simple per-task tracking works because SCX ops -- * nest in a limited way. BPF will likely implement a way to allow and disallow -- * kfuncs depending on the calling context which will replace this manual -- * mechanism. See scx_kf_allow(). -- */ --enum scx_kf_mask { -- SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ -- /* all non-sleepables may be nested inside INIT and SLEEPABLE */ -- SCX_KF_INIT = 1 << 0, /* running ops.init() */ -- SCX_KF_SLEEPABLE = 1 << 1, /* other sleepable init operations */ -- /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ -- SCX_KF_CPU_RELEASE = 1 << 2, /* ops.cpu_release() */ -- /* ops.dequeue (in REST) may be nested inside DISPATCH */ -- SCX_KF_DISPATCH = 1 << 3, /* ops.dispatch() */ -- SCX_KF_ENQUEUE = 1 << 4, /* ops.enqueue() */ -- SCX_KF_REST = 1 << 5, /* other rq-locked operations */ -- -- __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | -- SCX_KF_ENQUEUE | SCX_KF_REST, -- __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_REST, --}; -- --#define RAVG_HIST_SIZE 5 --struct scx_sched_task_stats { -- u64 mark_start; -- u64 window_start; -- u32 sum; -- u32 sum_history[RAVG_HIST_SIZE]; -- int cidx; -- -- u32 demand; -- u16 demand_scaled; -- void *sdsq; --}; -- -- -- --/* -- * The following is embedded in task_struct and contains all fields necessary -- * for a task to be scheduled by SCX. -- */ --struct sched_ext_entity { -- struct scx_dispatch_q *dsq; -- struct { -- struct list_head fifo; /* dispatch order */ -- struct rb_node priq; /* p->scx.dsq_vtime order */ -- } dsq_node; -- struct list_head watchdog_node; -- u32 flags; /* protected by rq lock */ -- u32 dsq_flags; /* protected by dsq lock */ -- u32 weight; -- s32 sticky_cpu; -- s32 holding_cpu; -- u32 kf_mask; /* see scx_kf_mask above */ -- struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ -- atomic64_t ops_state; -- unsigned long runnable_at; --#ifdef CONFIG_SCHED_CORE -- u64 core_sched_at; /* see scx_prio_less() */ --#endif -- -- /* BPF scheduler modifiable fields */ -- -- /* -- * Runtime budget in nsecs. This is usually set through -- * scx_bpf_dispatch() but can also be modified directly by the BPF -- * scheduler. Automatically decreased by SCX as the task executes. On -- * depletion, a scheduling event is triggered. -- * -- * This value is cleared to zero if the task is preempted by -- * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the -- * task ran. Use p->se.sum_exec_runtime instead. -- */ -- u64 slice; -- -- /* -- * Used to order tasks when dispatching to the vtime-ordered priority -- * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime() -- * but can also be modified directly by the BPF scheduler. Modifying it -- * while a task is queued on a dsq may mangle the ordering and is not -- * recommended. -- */ -- u64 dsq_vtime; -- -- /* -- * If set, reject future sched_setscheduler(2) calls updating the policy -- * to %SCHED_EXT with -%EACCES. -- * -- * If set from ops.prep_enable() and the task's policy is already -- * %SCHED_EXT, which can happen while the BPF scheduler is being loaded -- * or by inhering the parent's policy during fork, the task's policy is -- * rejected and forcefully reverted to %SCHED_NORMAL. The number of such -- * events are reported through /sys/kernel/debug/sched_ext::nr_rejected. -- */ -- bool disallow; /* reject switching into SCX */ -- -- /* cold fields */ -- struct list_head tasks_node; -- struct task_struct *task; -- struct scx_sched_task_stats sts; --}; -- --void sched_ext_free(struct task_struct *p); -- --#else /* !CONFIG_SCHED_CLASS_EXT */ -- --static inline void sched_ext_free(struct task_struct *p) {} -- --#endif /* CONFIG_SCHED_CLASS_EXT */ --#endif /* _LINUX_SCHED_EXT_H */ -diff --git a/include/linux/sched/hmbird_proc_val.h b/include/linux/sched/hmbird_proc_val.h -new file mode 100755 -index 000000000000..e59708b16ea3 ---- /dev/null -+++ b/include/linux/sched/hmbird_proc_val.h -@@ -0,0 +1,22 @@ -+#ifndef __HMBORD_PROC_VAL_H__ -+#define __HMBORD_PROC_VAL_H__ -+ -+extern int scx_enable; -+extern int partial_enable; -+extern int cpuctrl_high_ratio; -+extern int cpuctrl_low_ratio; -+extern int slim_stats; -+extern int hmbirdcore_debug; -+extern int slim_for_app; -+extern int misfit_ds; -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+extern int cpu7_tl; -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int slim_gov_debug; -+extern int scx_gov_ctrl; -+extern int sched_ravg_window_frame_per_sec; -+ -+#endif -\ No newline at end of file -diff --git a/include/linux/sched/hmbird_version.h b/include/linux/sched/hmbird_version.h -new file mode 100755 -index 000000000000..b2843881c26f ---- /dev/null -+++ b/include/linux/sched/hmbird_version.h -@@ -0,0 +1,52 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#ifndef _OPLUS_HMBIRD_VERSION_H_ -+#define _OPLUS_HMBIRD_VERSION_H_ -+#include -+#include -+#include -+ -+enum hmbird_version { -+ HMBIRD_UNINIT, -+ HMBIRD_GKI_VERSION, -+ HMBIRD_OGKI_VERSION, -+ HMBIRD_UNKNOW_VERSION, -+}; -+ -+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT; -+ -+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type" -+ -+static inline enum hmbird_version get_hmbird_version_type(void) -+{ -+ struct device_node *np = NULL; -+ const char *hmbird_version_str = NULL; -+ if (HMBIRD_UNINIT != hmbird_version_type) -+ return hmbird_version_type; -+ np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH); -+ if (np) { -+ of_property_read_string(np, "type", &hmbird_version_str); -+ if (NULL != hmbird_version_str) { -+ if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) { -+ hmbird_version_type = HMBIRD_OGKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi"); -+ } else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) { -+ hmbird_version_type = HMBIRD_GKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi"); -+ } else { -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi"); -+ } -+ return hmbird_version_type; -+ } -+ } -+ -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION"); -+ return hmbird_version_type; -+} -+ -+#endif /*_OPLUS_HMBIRD_VERSION_H_ */ -diff --git a/include/linux/sched/sa_common.h b/include/linux/sched/sa_common.h -new file mode 100755 -index 000000000000..6f76d7cfd7bf ---- /dev/null -+++ b/include/linux/sched/sa_common.h -@@ -0,0 +1,797 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_COMMON_H_ -+#define _OPLUS_SA_COMMON_H_ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+#include -+#endif -+#include -+#include -+#include -+#include -+#include -+ -+#include "sa_oemdata.h" -+#include "sa_common_struct.h" -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) -+#define OPLUS_UX_EEVDF_COMPATIBLE 1 -+#endif -+ -+#define SA_DEBUG_ON 0 -+ -+#if (SA_DEBUG_ON >= 1) -+#define DEBUG_BUG_ON(x) BUG_ON(x) -+#define DEBUG_WARN_ON(x) WARN_ON(x) -+#else -+#define DEBUG_BUG_ON(x) -+#define DEBUG_WARN_ON(x) -+#endif -+ -+#define ux_err(fmt, ...) \ -+ pr_err("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_warn(fmt, ...) \ -+ pr_warn("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_debug(fmt, ...) \ -+ pr_info("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+ -+#define UX_MSG_LEN 64 -+#define UX_DEPTH_MAX 5 -+ -+/* define for debug */ -+#define DEBUG_SYSTRACE (1 << 0) -+#define DEBUG_FTRACE (1 << 1) -+#define DEBUG_KMSG (1 << 2) /* used for frameboost */ -+#define DEBUG_PIPELINE (1 << 3) -+#define DEBUG_DYNAMIC_HZ (1 << 4) -+#define DEBUG_DYNAMIC_PREEMPT (1 << 5) -+#define DEBUG_AMU_INSTRUCTION (1 << 6) -+#define DEBUG_VERBOSE (1 << 10) /* used for frameboost */ -+#define DEBUG_SET_DSQ_ID (1 << 11) /* used for hmbird */ -+ -+/* define for sched assist feature */ -+#define FEATURE_COMMON (1 << 0) -+#define FEATURE_SPREAD (1 << 1) -+ -+#define UX_EXEC_SLICE (4000000U) -+ -+/* define for sched assist thread type, keep same as the define in java file */ -+#define SA_OPT_CLEAR (0) -+#define SA_TYPE_LIGHT (1 << 0) -+#define SA_TYPE_HEAVY (1 << 1) -+#define SA_TYPE_ANIMATOR (1 << 2) -+/* SA_TYPE_LISTPICK for camera */ -+#define SA_TYPE_LISTPICK (1 << 3) -+#define SA_TYPE_MQ_VIP (1 << 4) -+#define SA_OPT_SET (1 << 7) -+#define SA_OPT_RESET (1 << 8) -+#define SA_OPT_SET_PRIORITY (1 << 9) -+ -+/* The following ux value only used in kernel */ -+#define SA_TYPE_SWIFT (1 << 14) -+/* clear ux type when dequeue */ -+#define SA_TYPE_ONCE (1 << 15) -+#define SA_TYPE_INHERIT (1 << 16) -+/* SA_TYPE_COMBINED isn't marked in ux_state, it only exists in return value of function */ -+#define SA_TYPE_COMBINED (1 << 17) -+#define SA_TYPE_URGENT_MASK (SA_TYPE_LIGHT|SA_TYPE_ANIMATOR|SA_TYPE_SWIFT) -+#define SCHED_ASSIST_UX_MASK (SA_TYPE_LIGHT|SA_TYPE_HEAVY|SA_TYPE_ANIMATOR|SA_TYPE_LISTPICK|SA_TYPE_SWIFT) -+ -+/* load balance operation is performed on the following ux types */ -+#define SCHED_ASSIST_LB_UX (SA_TYPE_SWIFT | SA_TYPE_ANIMATOR | SA_TYPE_LIGHT | SA_TYPE_HEAVY) -+#define POSSIBLE_UX_MASK (SA_TYPE_SWIFT | \ -+ SA_TYPE_LIGHT | \ -+ SA_TYPE_HEAVY | \ -+ SA_TYPE_ANIMATOR | \ -+ SA_TYPE_LISTPICK | \ -+ SA_TYPE_ONCE | \ -+ SA_TYPE_INHERIT) -+ -+#define SCHED_ASSIST_UX_PRIORITY_MASK (0xFF000000) -+#define SCHED_ASSIST_UX_PRIORITY_SHIFT 24 -+ -+#define UX_PRIORITY_TOP_APP 0x0A000000 -+#define UX_PRIORITY_AUDIO 0x0A000000 -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+#define UX_PRIORITY_PIPELINE_UI 0x06000000 -+#define UX_PRIORITY_PIPELINE 0x05000000 -+#endif -+ -+/* define for sched assist scene type, keep same as the define in java file */ -+#define SA_SCENE_OPT_CLEAR (0) -+#define SA_LAUNCH (1 << 0) -+#define SA_SLIDE (1 << 1) -+#define SA_CAMERA (1 << 2) -+#define SA_ANIM_START (1 << 3) /* we care about both launcher and top app */ -+#define SA_ANIM (1 << 4) /* we only care about launcher */ -+#define SA_INPUT (1 << 5) -+#define SA_LAUNCHER_SI (1 << 6) -+#define SA_SCENE_OPT_SET (1 << 7) -+ -+#define ROOT_UID 0 -+#define SYSTEM_UID 1000 -+#define FIRST_APPLICATION_UID 10000 -+#define LAST_APPLICATION_UID 19999 -+#define PER_USER_RANGE 100000 -+/* define for clear IM_FLAG -+if im_flag is 70, it should clear im_flag_audio (70 - 64 = 6) -+eg: gerrit patchset "30438485" -+ */ -+#define IM_FLAG_CLEAR 64 -+ -+extern pid_t save_audio_tgid; -+extern pid_t save_top_app_tgid; -+extern unsigned int top_app_type; -+extern int global_lowend_plat_opt; -+ -+#ifdef CONFIG_OPLUS_SCHED_HALT_MASK_PRT -+/* This must be the same as the definition of pause_type in walt_halt.c */ -+enum oplus_pause_type { -+ OPLUS_HALT, -+ OPLUS_PARTIAL_HALT, -+ -+ OPLUS_MAX_PAUSE_TYPE -+}; -+extern cpumask_t cur_cpus_halt_mask; -+extern cpumask_t cur_cpus_phalt_mask; -+DECLARE_PER_CPU(int[OPLUS_MAX_PAUSE_TYPE], oplus_cur_pause_client); -+extern void sa_corectl_systrace_c(void); -+#endif /* CONFIG_OPLUS_SCHED_HALT_MASK_PRT */ -+ -+/* define for boost threshold unit */ -+#define BOOST_THRESHOLD_UNIT (51) -+ -+ -+enum UX_STATE_TYPE { -+ UX_STATE_INVALID = 0, -+ UX_STATE_NONE, -+ UX_STATE_STATIC, -+ UX_STATE_INHERIT, -+ UX_STATE_COMBINED, -+ MAX_UX_STATE_TYPE, -+}; -+ -+enum INHERIT_UX_TYPE { -+ INHERIT_UX_BINDER = 0, -+ INHERIT_UX_RWSEM, -+ INHERIT_UX_MUTEX, -+ INHERIT_UX_FUTEX, -+ INHERIT_UX_PIFUTEX, -+ INHERIT_UX_MAX, -+}; -+ -+/* -+ * WANNING: -+ * new flag should be add before MAX_IM_FLAG_TYPE, never change -+ * the value of those existed flag type. -+ */ -+enum IM_FLAG_TYPE { -+ IM_FLAG_NONE = 0, -+ IM_FLAG_SURFACEFLINGER, -+ IM_FLAG_HWC, /* Discarded */ -+ IM_FLAG_RENDERENGINE, -+ IM_FLAG_WEBVIEW, -+ IM_FLAG_CAMERA_HAL, -+ IM_FLAG_AUDIO, -+ IM_FLAG_HWBINDER, -+ IM_FLAG_LAUNCHER, -+ IM_FLAG_LAUNCHER_NON_UX_RENDER, -+ IM_FLAG_SS_LOCK_OWNER, -+ IM_FLAG_FORBID_SET_CPU_AFFINITY = 11, /* forbid setting cpu affinity from app */ -+ IM_FLAG_SYSTEMSERVER_PID, -+ IM_FLAG_MIDASD, -+ IM_FLAG_AUDIO_CAMERA_HAL, /* audio mode disable camera hal ux */ -+ IM_FLAG_AFFINITY_THREAD, -+ IM_FLAG_TPD_SET_CPU_AFFINITY = 16, -+ IM_FLAG_COMPRESS_THREAD = 17, /* compress thread skips locking protect */ -+ IM_FLAG_RENDER_THREAD = 18, -+ MAX_IM_FLAG_TYPE, -+}; -+ -+#define MAX_IM_FLAG_PRIO MAX_IM_FLAG_TYPE -+ -+enum ots_state { -+ OTS_STATE_SET_AFFINITY, -+ OTS_STATE_DDL_ACTIVE, -+ OTS_STATE_DDL_ACTIVE_PREEMPTED, -+ OTS_STATE_MAX, -+}; -+ -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+DECLARE_PER_CPU(u64, retired_instrs); -+DECLARE_PER_CPU(u64, nvcsw); -+DECLARE_PER_CPU(u64, nivcsw); -+#endif -+ -+struct ux_sched_cluster { -+ struct cpumask cpus; -+ unsigned long capacity; -+}; -+ -+#define OPLUS_NR_CPUS (8) -+#define OPLUS_MAX_CLS (5) -+struct ux_sched_cputopo { -+ int cls_nr; -+ struct ux_sched_cluster sched_cls[OPLUS_NR_CPUS]; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ cpumask_t oplus_cpu_array[2*OPLUS_MAX_CLS][OPLUS_MAX_CLS]; -+#endif -+}; -+ -+struct oplus_rq { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_root_cached ux_list; -+ /* a tree to track minimum exec vruntime */ -+ struct rb_root_cached exec_timeline; -+ /* malloc this spinlock_t instead of built-in to shrank size of oplus_rq */ -+ spinlock_t *ux_list_lock; -+ int nr_running; -+ u64 min_vruntime; -+ u64 load_weight; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) -+ struct rb_root_cached ddl_root; -+ spinlock_t *ddl_lock; -+#endif -+ -+#ifdef CONFIG_LOCKING_PROTECT -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ struct list_head locking_thread_list; -+ spinlock_t *locking_list_lock; -+ int rq_locking_task; -+#else -+ struct sched_entity *last_entity; -+#endif -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ struct oplus_lb lb; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+ struct hrtimer *resched_timer; -+ int cpu; -+#endif -+}; -+ -+extern int global_debug_enabled; -+extern int global_sched_assist_enabled; -+extern int global_sched_assist_scene; -+extern int global_silver_perf_core; -+extern int global_sched_group_enabled; -+ -+struct rq; -+ -+#ifdef CONFIG_LOCKING_PROTECT -+struct sched_assist_locking_ops { -+ void (*check_preempt_tick)(struct task_struct *p, -+ unsigned long *ideal_runtime, bool *skip_preempt, -+ unsigned long delta_exec, struct cfs_rq *cfs_rq, -+ struct sched_entity *curr, unsigned int granularity); -+ void (*check_preempt_wakeup)(struct rq *rq, struct task_struct *p, bool *preempt, bool *nopreempt); -+ void (*state_systrace_c)(unsigned int cpu, struct task_struct *p); -+ void (*locking_tick_hit)(struct task_struct *prev, struct task_struct *next); -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ void (*replace_next_task_fair)(struct rq *rq, -+ struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*enqueue_entity)(struct rq *rq, struct task_struct *p); -+ void (*dequeue_entity)(struct rq *rq, struct task_struct *p); -+#else -+ void (*set_last_entity)(struct task_struct *prev, struct task_struct *next, struct rq *rq); -+ void (*pick_last_entity)(struct rq *rq, struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*clear_last_entity)(struct rq *rq, struct task_struct *p); -+#endif -+ void (*opt_ss_lock_contention)(struct task_struct *p, int old_im, int new_im); -+}; -+ -+extern struct sched_assist_locking_ops *locking_ops; -+ -+ -+#define LOCKING_CALL_OP(op, args...) \ -+do { \ -+ if (locking_ops && locking_ops->op) { \ -+ locking_ops->op(args); \ -+ } \ -+} while (0) -+ -+#define LOCKING_CALL_OP_RET(op, args...) \ -+({ \ -+ __typeof__(locking_ops->op(args)) __ret = 0; \ -+ if (locking_ops && locking_ops->op) \ -+ __ret = locking_ops->op(args); \ -+ __ret; \ -+}) -+ -+void register_sched_assist_locking_ops(struct sched_assist_locking_ops *ops); -+#endif -+ -+/* attention: before insert .ko, task's list->prev/next is init with 0 */ -+static inline bool oplus_list_empty(struct list_head *list) -+{ -+ return list_empty(list) || (list->prev == 0 && list->next == 0); -+} -+ -+static inline bool oplus_rbtree_empty(struct rb_root_cached *list) -+{ -+ return (rb_first_cached(list) == NULL); -+} -+ -+/** -+ * Check if there are ux tasks waiting to run on the specified cpu -+ */ -+static inline bool orq_has_ux_tasks(struct oplus_rq *orq) -+{ -+ return !oplus_rbtree_empty(&orq->ux_list); -+} -+ -+/* attention: before insert .ko, task's node->__rb_parent_color is init with 0 */ -+static inline bool oplus_rbnode_empty(struct rb_node *node) -+{ -+ return RB_EMPTY_NODE(node) || (node->__rb_parent_color == 0); -+} -+ -+static inline struct oplus_task_struct *ux_list_first_entry(struct rb_root_cached *list) -+{ -+ struct rb_node *leftmost = rb_first_cached(list); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, ux_entry); -+} -+ -+static inline struct oplus_task_struct *exec_timeline_first_entry(struct rb_root_cached *tree) -+{ -+ struct rb_node *leftmost = rb_first_cached(tree); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, exec_time_node); -+} -+ -+typedef bool (*migrate_task_callback_t)(struct task_struct *tsk, int src_cpu, int dst_cpu); -+extern migrate_task_callback_t fbg_migrate_task_callback; -+typedef void (*android_rvh_schedule_handler_t)(struct task_struct *prev, -+ struct task_struct *next, struct rq *rq); -+extern android_rvh_schedule_handler_t fbg_android_rvh_schedule_callback; -+ -+extern struct kmem_cache *oplus_task_struct_cachep; -+ -+#define ots_to_ts(ots) (ots->task) -+#define OTS_IDX 0 -+#define ORQ_IDX 0 -+ -+static inline bool test_task_is_fair(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid CFS priority is MAX_RT_PRIO..MAX_PRIO-1 */ -+ if ((task->prio >= MAX_RT_PRIO) && (task->prio <= MAX_PRIO-1)) -+ return true; -+ return false; -+} -+ -+static inline bool test_task_is_rt(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid RT priority is 0..MAX_RT_PRIO-1 */ -+ if (rt_prio(task->prio)) -+ return true; -+ -+ return false; -+} -+ -+static inline struct oplus_task_struct *get_oplus_task_struct(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = NULL; -+ -+ /* not Skip idle thread */ -+ if (!t) -+ return NULL; -+ -+ ots = (struct oplus_task_struct *) READ_ONCE(t->android_oem_data1[OTS_IDX]); -+ if (IS_ERR_OR_NULL(ots)) -+ return NULL; -+ -+ return ots; -+} -+ -+static inline unsigned long oplus_get_im_flag(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return IM_FLAG_NONE; -+ -+ return ots->im_flag; -+} -+ -+static inline bool is_optimized_audio_thread(struct task_struct *t) -+{ -+ unsigned long im_flag; -+ -+ im_flag = oplus_get_im_flag(t); -+ if (test_bit(IM_FLAG_AUDIO, &im_flag)) -+ return true; -+ -+ return false; -+} -+ -+static inline void oplus_set_im_flag(struct task_struct *t, int im_flag) -+{ -+ struct oplus_task_struct *ots = NULL; -+ ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ set_bit(im_flag, &ots->im_flag); -+} -+ -+static inline int get_ots_ux_state(struct oplus_task_struct *ots) -+{ -+ int ux_state; -+ int sub_ux_state; -+ int ux_prio; -+ int ret; -+ -+ ux_prio = ots->ux_state & SCHED_ASSIST_UX_PRIORITY_MASK; -+ ux_state = ots->ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ sub_ux_state = ots->sub_ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ -+ if (sub_ux_state) { -+ ret = ux_state | (sub_ux_state & ~SA_TYPE_INHERIT) | SA_TYPE_COMBINED; -+ } else { -+ ret = ux_state; -+ } -+ -+ if (ret & SCHED_ASSIST_UX_MASK) { -+ return ux_prio | ret; -+ } -+ -+ return 0; -+} -+ -+bool is_multiple_ux(struct oplus_task_struct *ots); -+ -+static inline int oplus_get_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return get_ots_ux_state(ots); -+} -+ -+static inline int oplus_get_static_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return 0; -+ } -+ return ots->ux_state; -+} -+ -+static inline int oplus_get_sub_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->sub_ux_state; -+} -+ -+static inline int oplus_get_inherited_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return ots->ux_state; -+ } -+ return ots->sub_ux_state; -+} -+ -+void oplus_set_ux_state_lock(struct task_struct *t, int ux_state, int inherit_type, bool need_lock_rq); -+ -+static inline s64 oplus_get_inherit_ux(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return atomic64_read(&ots->inherit_ux); -+} -+ -+static inline void oplus_set_inherit_ux(struct task_struct *t, s64 inherit_ux) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ atomic64_set(&ots->inherit_ux, inherit_ux); -+} -+ -+static inline int oplus_get_ux_depth(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->ux_depth; -+} -+ -+static inline void oplus_set_ux_depth(struct task_struct *t, int ux_depth) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->ux_depth = ux_depth; -+} -+ -+static inline u64 oplus_get_enqueue_time(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->enqueue_time; -+} -+ -+static inline void oplus_set_enqueue_time(struct task_struct *t, u64 enqueue_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->enqueue_time = enqueue_time; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* -+ * Record the number of task context switches -+ * and scheduling delays when enqueueing a task. -+ */ -+ ots->snap_pcount = t->sched_info.pcount; -+ ots->snap_run_delay = t->sched_info.run_delay; -+#endif -+} -+ -+static inline void oplus_set_inherit_ux_start(struct task_struct *t, u64 start_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->inherit_ux_start = t->se.sum_exec_runtime; -+} -+ -+static inline void init_task_ux_info(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ RB_CLEAR_NODE(&ots->ux_entry); -+ RB_CLEAR_NODE(&ots->exec_time_node); -+ ots->ux_state = 0; -+ ots->sub_ux_state = 0; -+ atomic64_set(&ots->inherit_ux, 0); -+ ots->ux_depth = 0; -+ ots->enqueue_time = 0; -+ ots->inherit_ux_start = 0; -+ ots->ux_priority = -1; -+ ots->ux_nice = -1; -+ ots->vruntime = 0; -+ ots->preset_vruntime = 0; -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG) -+ ots->abnormal_flag = 0; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_SCHED_SPREAD -+ ots->lb_state = 0; -+ ots->ld_flag = 0; -+#endif -+ ots->exec_calc_runtime = 0; -+ ots->is_update_runtime = 0; -+ ots->target_process = -1; -+ ots->wake_tid = 0; -+ ots->running_start_time = 0; -+ ots->update_running_start_time = false; -+ ots->last_wake_ts = 0; -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ memset(&ots->lkinfo, 0, sizeof(struct locking_info)); -+ INIT_LIST_HEAD(&ots->lkinfo.node); -+/*#endif*/ -+ ots->block_start_time = 0; -+#ifdef CONFIG_LOCKING_PROTECT -+ INIT_LIST_HEAD(&ots->locking_entry); -+ ots->locking_start_time = 0; -+ ots->locking_depth = 0; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ ots->snap_pcount = 0; -+ ots->snap_run_delay = 0; -+ plist_node_init(&ots->rtb, MAX_IM_FLAG_PRIO); -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+ atomic_set(&ots->pipeline_cpu, -1); -+#endif -+ -+#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO) -+ ots->amu_cycle = 0; -+ ots->amu_instruct = 0; -+#endif -+}; -+ -+static inline bool test_ux_type(struct task_struct *task, int ux_type) -+{ -+ return oplus_get_ux_state(task) & ux_type; -+} -+ -+static inline bool is_heavy_ux_task(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return false; -+ -+ return get_ots_ux_state(ots) & SA_TYPE_HEAVY; -+} -+ -+static inline bool sched_assist_scene(unsigned int scene) -+{ -+ if (unlikely(!global_sched_assist_enabled)) -+ return false; -+ -+ return global_sched_assist_scene & scene; -+} -+ -+static inline unsigned long oplus_task_util(struct task_struct *p) -+{ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+ struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; -+ -+ return wts->demand_scaled; -+#else -+ return READ_ONCE(p->se.avg.util_avg); -+#endif -+} -+ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+static inline u32 task_wts_sum(struct task_struct *tsk) -+{ -+ struct walt_task_struct *wts = (struct walt_task_struct *) tsk->android_vendor_data1; -+ -+ return wts->sum; -+} -+#endif -+ -+bool is_min_cluster(int cpu); -+bool is_max_cluster(int cpu); -+bool is_mid_cluster(int cpu); -+bool im_mali(const char *comm); -+bool is_top(struct task_struct *p); -+bool task_is_runnable(struct task_struct *task); -+struct oplus_rq *get_oplus_rq(struct rq *rq); -+ -+typedef unsigned long (*kallsyms_lookup_name_t)(const char *name); -+extern kallsyms_lookup_name_t _kallsyms_lookup_name; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+int ux_mask_to_prio(int ux_mask); -+int ux_prio_to_mask(int prio); -+#endif -+ -+noinline int tracing_mark_write(const char *buf); -+void hwbinder_systrace_c(unsigned int cpu, int flag); -+void sched_assist_init_oplus_rq(void); -+void queue_ux_thread(struct rq *rq, struct task_struct *p, int enqueue); -+ -+void inherit_ux_inc(struct task_struct *task, int type); -+void inherit_ux_sub(struct task_struct *task, int type, int value); -+void set_inherit_ux(struct task_struct *task, int type, int depth, int inherit_val); -+void reset_inherit_ux(struct task_struct *inherit_task, struct task_struct *ux_task, int reset_type); -+void unset_inherit_ux(struct task_struct *task, int type); -+void unset_inherit_ux_value(struct task_struct *task, int type, int value); -+void inc_inherit_ux_refs(struct task_struct *task, int type); -+void clear_all_inherit_type(struct task_struct *p); -+int get_max_inherit_gran(struct task_struct *p); -+ -+bool is_heavy_load_top_task(struct task_struct *p); -+bool test_task_is_fair(struct task_struct *task); -+bool test_task_is_rt(struct task_struct *task); -+ -+bool test_task_ux(struct task_struct *task); -+bool test_task_ux_depth(int ux_depth); -+bool test_inherit_ux(struct task_struct *task, int type); -+bool test_set_inherit_ux(struct task_struct *task); -+int get_ux_state_type(struct task_struct *task); -+void sched_assist_target_comm(struct task_struct *task, const char *comm); -+unsigned int ux_task_exec_limit(struct task_struct *p); -+ -+void update_ux_sched_cputopo(void); -+bool is_task_util_over(struct task_struct *tsk, int threshold); -+bool oplus_task_misfit(struct task_struct *tsk, int cpu); -+ssize_t oplus_show_cpus(const struct cpumask *mask, char *buf); -+void adjust_rt_lowest_mask(struct task_struct *p, struct cpumask *local_cpu_mask, int ret, bool force_adjust); -+bool sa_skip_rt_sync(struct rq *rq, struct task_struct *p, bool *sync); -+void ux_state_systrace_c(unsigned int cpu, struct task_struct *p); -+bool sa_rt_skip_ux_cpu(int cpu); -+int is_vip_mvp(struct task_struct *p); -+ -+/* s64 account_ux_runtime(struct rq *rq, struct task_struct *curr); */ -+void opt_ss_lock_contention(struct task_struct *p, unsigned long old_im, int new_im); -+ -+/* register vender hook in kernel/sched/topology.c */ -+void android_vh_build_sched_domains_handler(void *unused, bool has_asym); -+ -+/* register vender hook in kernel/sched/rt.c */ -+void android_rvh_select_task_rq_rt_handler(void *unused, struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags, int *new_cpu); -+void android_rvh_find_lowest_rq_handler(void *unused, struct task_struct *p, struct cpumask *local_cpu_mask, int ret, int *best_cpu); -+ -+/* register vender hook in kernel/sched/core.c */ -+void android_rvh_sched_fork_handler(void *unused, struct task_struct *p); -+void android_rvh_after_enqueue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_dequeue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_schedule_handler(void *unused, struct task_struct *prev, struct task_struct *next, struct rq *rq); -+void android_vh_scheduler_tick_handler(void *unused, struct rq *rq); -+ -+void android_vh_account_process_tick_gran_handler(void *unused, struct task_struct *p, struct rq *rq, int user_tick, int *ticks); -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+void sa_sched_switch_handler(void *unused, bool preempt, struct task_struct *prev, struct task_struct *next, unsigned int prev_state); -+#endif -+ -+void set_im_flag_with_bit(int im_flag, struct task_struct *task); -+ -+/* register vendor hook in kernel/cgroup/cgroup-v1.c */ -+void android_vh_cgroup_set_task_handler(void *unused, int ret, struct task_struct *task); -+/* register vendor hook in kernel/signal.c */ -+void android_vh_exit_signal_handler(void *unused, struct task_struct *p); -+void android_rvh_set_cpus_allowed_by_task_handler(void *unused, const struct cpumask *cpu_valid_mask, -+ const struct cpumask *new_mask, struct task_struct *task, unsigned int *dest_cpu); -+void android_rvh_setscheduler_handler(void *unused, struct task_struct *p); -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_BAN_APP_SET_AFFINITY) -+void android_vh_sched_setaffinity_early_handler(void *unused, struct task_struct *task, const struct cpumask *new_mask, bool *skip); -+#endif -+ -+#ifdef CONFIG_OPLUS_SCHED_GROUP_OPT -+void android_vh_reweight_entity_handler(void *unused, struct sched_entity *se); -+#endif -+ -+#ifdef CONFIG_BLOCKIO_UX_OPT -+int sa_blockio_init(void); -+void sa_blockio_exit(void); -+#endif -+extern struct notifier_block process_exit_notifier_block; -+#endif /* _OPLUS_SA_COMMON_H_ */ -diff --git a/include/linux/sched/sa_common_struct.h b/include/linux/sched/sa_common_struct.h -new file mode 100755 -index 000000000000..876feff48b36 ---- /dev/null -+++ b/include/linux/sched/sa_common_struct.h -@@ -0,0 +1,277 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+/* -+this file is splited from the sa_common.h to adapt the OKI, -+IS_ENABLED is not allowed in here, because the macro will not work in OKI. -+*/ -+#ifndef _OPLUS_SA_COMMON_STRUCT_H_ -+#define _OPLUS_SA_COMMON_STRUCT_H_ -+ -+#define MAX_CLUSTER (4) -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+/* hot-thread */ -+struct task_record { -+#define RECOED_WINSIZE (1 << 8) -+#define RECOED_WINIDX_MASK (RECOED_WINSIZE - 1) -+ u8 winidx; -+ u8 count; -+}; -+ -+#define MAX_TASK_COMM_LEN 256 -+struct uid_struct { -+ uid_t uid; -+ u64 uid_total_cycle; -+ u64 uid_total_inst; -+ spinlock_t lock; -+ char leader_comm[TASK_COMM_LEN]; -+ char cmdline[MAX_TASK_COMM_LEN]; -+}; -+ -+struct amu_uid_entry { -+ uid_t uid; -+ struct uid_struct *uid_struct; -+ struct hlist_node node; -+}; -+ -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+struct locking_info { -+ u64 waittime_stamp; -+ u64 holdtime_stamp; -+ /* Used in torture acquire latency statistic.*/ -+ u64 acquire_stamp; -+ /* -+ * mutex or rwsem optimistic spin start time. Because a task -+ * can't spin both on mutex and rwsem at one time, use one common -+ * threshold time is OK. -+ */ -+ u64 opt_spin_start_time; -+ struct task_struct *holder; -+ u32 waittype; -+ bool ux_contrib; -+ /* -+ * Whether task is ux when it's going to be added to mutex or -+ * rwsem waiter list. It helps us check whether there is ux -+ * task on mutex or rwsem waiter list. Also, a task can't be -+ * added to both mutex and rwsem at one time, so use one common -+ * field is OK. -+ */ -+ bool is_block_ux; -+ u32 kill_flag; -+ /* for cfs enqueue smoothly.*/ -+ struct list_head node; -+ struct task_struct *owner; -+ struct list_head lock_head; -+ u64 clear_seq; -+ atomic_t lock_depth; -+}; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+#define RAVG_HIST_SIZE 5 -+#define SCX_SLICE_DFL (1 * NSEC_PER_MSEC) -+#define SCX_SLICE_INF U64_MAX -+#define DEFAULT_CGROUP_DL_IDX (8) -+#define EXT_FLAG_RT_CHANGED (1 << 0) -+#define EXT_FLAG_CFS_CHANGED (1 << 1) -+struct scx_task_stats { -+ u64 mark_start; -+ u64 window_start; -+ u32 sum; -+ u32 sum_history[RAVG_HIST_SIZE]; -+ int cidx; -+ u32 demand; -+ u16 demand_scaled; -+ void *sdsq; -+}; -+/* -+ * The following is embedded in task_struct and contains all fields necessary -+ * for a task to be scheduled by SCX. -+ */ -+struct scx_entity { -+ struct scx_dispatch_q *dsq; -+ struct { -+ struct list_head fifo; /* dispatch order */ -+ struct rb_node priq; /* p->scx.dsq_vtime order */ -+ } dsq_node; -+ u32 flags; /* protected by rq lock */ -+ u32 dsq_flags; /* protected by dsq lock */ -+ s32 sticky_cpu; -+ unsigned long runnable_at; -+ u64 slice; -+ u64 dsq_vtime; -+ int gdsq_idx; -+ int ext_flags; -+ int prio_backup; -+ unsigned long sched_prop; -+ struct scx_task_stats sts; -+}; -+/*#endif*/ -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ -+#define MAX_CPU_FREQ_STATE 32 -+#define MAX_CPU_CNT 8 -+ -+#define STATE_IN_POOL 0 -+#define STATE_ACTIVE 1 -+ -+struct powermodel_freq_task_state { -+ u8 powermodel_enqueued:1; -+ u8 powermodel_index:7; -+}; -+ -+struct powermodel_cpu_task_state { -+ struct oplus_task_struct *ots; -+ struct powermodel_freq_task_state powermodel_freq_task_states[MAX_CPU_FREQ_STATE]; -+ u64 powermodel_last_seq; -+ u64 last_seq; -+ struct list_head node; -+ int state; -+ int pool_id; -+}; -+ -+#endif -+ -+ -+/* Please add your own members of task_struct here :) */ -+struct oplus_task_struct { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_node ux_entry; -+ struct rb_node exec_time_node; -+ struct task_struct *task; -+ atomic64_t inherit_ux; -+ u64 enqueue_time; -+ u64 inherit_ux_start; -+ /* u64 sum_exec_baseline; */ -+ u64 total_exec; -+ u64 vruntime; -+ u64 preset_vruntime; -+ /* contains ux state -+ 1. if static and inherited ux both exist, static ux stores in ux_state, inherited ux in sub_ux_state. -+ 2. if only static ux exists, static ux stores in ux_state. -+ 2. if only inherited ux exists, inherited ux stores in ux_state */ -+ int ux_state; -+ int sub_ux_state; -+ u8 ux_depth; -+ s8 ux_priority; -+ s8 ux_nice; -+ pid_t affinity_pid; -+ pid_t affinity_tgid; -+ unsigned long state; -+ unsigned long im_flag; -+ atomic_t is_vip_mvp; -+ -+/* #if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) */ -+ u64 ddl; -+ u64 ddl_active_ts; -+ u64 runnable_ts; -+ struct rb_node ddl_node; -+/* #endif */ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG)*/ -+ int abnormal_flag; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_SCHED_SPREAD */ -+ int lb_state; -+ int ld_flag:1; -+ /* CONFIG_OPLUS_FEATURE_TASK_LOAD */ -+ int is_update_runtime:1; -+ int target_process; -+ u64 wake_tid; -+ u64 running_start_time; -+ bool update_running_start_time; -+ u64 exec_calc_runtime; -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct task_record record[MAX_CLUSTER]; /* 2*u64 */ -+ u64 block_start_time; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_FRAME_BOOST */ -+ struct list_head fbg_list; -+ raw_spinlock_t fbg_list_entry_lock; -+ bool fbg_running; /* task belongs to a group, and in running */ -+ u16 fbg_state; -+ s8 preferred_cluster_id; -+ s8 fbg_depth; -+ u64 last_wake_ts; -+ int fbg_cur_group; -+/*#ifdef CONFIG_LOCKING_PROTECT*/ -+ unsigned long locking_start_time; -+ unsigned long last_jiffies; -+ struct list_head locking_entry; -+ int locking_depth; -+ int lk_tick_hit; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ struct locking_info lkinfo; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+ struct scx_entity scx; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_FDLEAK_CHECK)*/ -+ u8 fdleak_flag; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+ /* for loadbalance */ -+ struct plist_node rtb; /* rt boost task */ -+ -+ /* -+ * The following variables are used to calculate the time -+ * a task spends in the running/runnable state. -+ */ -+ u64 snap_run_delay; -+ unsigned long snap_pcount; -+/*#endif*/ -+#if IS_ENABLED(CONFIG_OPLUS_SCHED_TUNE) -+ int stune_idx; -+#endif -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE)*/ -+ atomic_t pipeline_cpu; -+/*#endif*/ -+ -+ /* for oplus secure guard */ -+ int sg_flag; -+ int sg_scno; -+ uid_t sg_uid; -+ uid_t sg_euid; -+ gid_t sg_gid; -+ gid_t sg_egid; -+/*#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct uid_struct *uid_struct; -+ u64 amu_instruct; -+ u64 amu_cycle; -+/*#endif*/ -+ /* for binder ux */ -+ int binder_async_ux_enable; -+ bool binder_async_ux_sts; -+ int binder_thread_mode; -+ struct binder_node *binder_thread_node; -+ -+/* for powermodel */ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ struct powermodel_cpu_task_state *powermodel_cpu_task_states[MAX_CPU_CNT]; -+ u64 exec_runtime; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_CFBT) -+ int cfbt_cur_group; -+ bool cfbt_running; -+#endif /* CONFIG_OPLUS_FEATURE_SCHED_CFBT */ -+} ____cacheline_aligned; -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+#define INVALID_PID (-1) -+struct oplus_lb { -+ /* used for active_balance to record the running task. */ -+ pid_t pid; -+}; -+/*#endif*/ -+ -+#endif /* _OPLUS_SA_COMMON_STRUCT_H_ */ -+ -diff --git a/include/linux/sched/sa_oemdata.h b/include/linux/sched/sa_oemdata.h -new file mode 100755 -index 000000000000..ebbbc754e391 ---- /dev/null -+++ b/include/linux/sched/sa_oemdata.h -@@ -0,0 +1,13 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_OEMDATA_H_ -+#define _OPLUS_SA_OEMDATA_H_ -+ -+int sa_oemdata_init(void); -+void sa_oemdata_deinit(void); -+ -+#endif /* _OPLUS_SA_OEMDATA_H_ */ -diff --git a/include/linux/sched/sched_ext.h b/include/linux/sched/sched_ext.h -new file mode 100755 -index 000000000000..9f1afdb8a04b ---- /dev/null -+++ b/include/linux/sched/sched_ext.h -@@ -0,0 +1,57 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef _OPLUS_SCHED_EXT_H -+#define _OPLUS_SCHED_EXT_H -+#include "sa_common.h" -+ -+#define SCHED_PROP_TOP_THREAD_SHIFT (8) -+#define SCHED_PROP_TOP_THREAD_MASK (0xf << SCHED_PROP_TOP_THREAD_SHIFT) -+#define SCHED_PROP_DEADLINE_MASK (0xFF) /* deadline for ext sched class */ -+#define SCHED_PROP_DEADLINE_LEVEL1 (1) /* 1ms for user-aware audio tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL2 (2) /* 2ms for user-aware touch tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL3 (3) /* 4ms for user aware dispaly tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL4 (4) /* 6ms */ -+#define SCHED_PROP_DEADLINE_LEVEL5 (5) /* 8ms */ -+#define SCHED_PROP_DEADLINE_LEVEL6 (6) /* 16ms */ -+#define SCHED_PROP_DEADLINE_LEVEL7 (7) /* 32ms */ -+#define SCHED_PROP_DEADLINE_LEVEL8 (8) /* 64ms */ -+#define SCHED_PROP_DEADLINE_LEVEL9 (9) /* 128ms */ -+ -+static inline int sched_prop_get_top_thread_id(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ return -EPERM; -+ } -+ -+ return ((ots->scx.sched_prop & SCHED_PROP_TOP_THREAD_MASK) >> SCHED_PROP_TOP_THREAD_SHIFT); -+} -+ -+static inline int sched_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_set_sched_prop failed! fn=%s\n", __func__); -+ return -EPERM; -+ } -+ -+ ots->scx.sched_prop = sp; -+ return 0; -+} -+ -+static inline unsigned long sched_get_sched_prop(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_get_sched_prop failed! fn=%s\n", __func__); -+ return (unsigned long)-1; -+ } -+ return ots->scx.sched_prop; -+} -+ -+#endif /*_OPLUS_SCHED_EXT_H */ -diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h -index 03d35e3eda3c..6a5f0c0d74bd 100644 ---- a/include/linux/sched/task.h -+++ b/include/linux/sched/task.h -@@ -61,8 +61,10 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); - extern void init_idle(struct task_struct *idle, int cpu); - - extern int sched_fork(unsigned long clone_flags, struct task_struct *p); --extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); --extern void sched_cancel_fork(struct task_struct *p); -+extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); -+#ifdef CONFIG_HMBIRD_SCHED -+extern void hmbird_cancel_fork(struct task_struct *p); -+#endif - extern void sched_post_fork(struct task_struct *p); - extern void sched_dead(struct task_struct *p); - -diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h -index 359a14cc76a4..969716ab4320 100644 ---- a/include/uapi/linux/sched.h -+++ b/include/uapi/linux/sched.h -@@ -118,7 +118,7 @@ struct clone_args { - /* SCHED_ISO: reserved but not implemented yet */ - #define SCHED_IDLE 5 - #define SCHED_DEADLINE 6 --#define SCHED_EXT 7 -+#define SCHED_HMBIRD 7 - - /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ - #define SCHED_RESET_ON_FORK 0x40000000 -diff --git a/init/Kconfig b/init/Kconfig -index 0ead65803ecf..fbbc4a970b00 100644 ---- a/init/Kconfig -+++ b/init/Kconfig -@@ -1030,10 +1030,6 @@ config RT_GROUP_SCHED - realtime bandwidth for them. - See Documentation/scheduler/sched-rt-group.rst for more information. - --config EXT_GROUP_SCHED -- bool -- depends on SCHED_CLASS_EXT && CGROUP_SCHED -- default y - endif #CGROUP_SCHED - - config SCHED_MM_CID -diff --git a/init/init_task.c b/init/init_task.c -index 69a046388995..31ceb0e469f7 100644 ---- a/init/init_task.c -+++ b/init/init_task.c -@@ -6,7 +6,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -102,9 +101,6 @@ struct task_struct init_task - #endif - #ifdef CONFIG_CGROUP_SCHED - .sched_task_group = &root_task_group, --#endif --#ifdef CONFIG_SLIM_SCHED -- .scx = NULL, - #endif - .ptraced = LIST_HEAD_INIT(init_task.ptraced), - .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), -diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt -index cf22ef6107b8..b131d5662b48 100644 ---- a/kernel/Kconfig.preempt -+++ b/kernel/Kconfig.preempt -@@ -133,12 +133,8 @@ config SCHED_CORE - which is the likely usage by Linux distributions, there should - be no measurable impact on performance. - --config SLIM_SCHED -- bool "slim sched" -- default n --config SCHED_CLASS_EXT -- bool "Extensible Scheduling Class" -- depends on BPF_SYSCALL && BPF_JIT -+config HMBIRD_SCHED -+ bool "hmbird Scheduling Class(base on ext scheduler)" - help - This option enables a new scheduler class sched_ext (SCX), which - allows scheduling policies to be implemented as BPF programs to -@@ -159,3 +155,10 @@ config SCHED_CLASS_EXT - similar to struct sched_class. - - See Documentation/scheduler/sched-ext.rst for more details. -+ -+config DYNAMIC_WALT_SUPPORT -+ bool "Dynamic WALT Support for Extensible Scheduling Class" -+ depends on HMBIRD_SCHED -+ default n -+ help -+ Enable dynamic WALT support for Extensible Scheduling Class. -diff --git a/kernel/bpf/bpf_struct_ops_types.h b/kernel/bpf/bpf_struct_ops_types.h -index 3618769d853d..5678a9ddf817 100644 ---- a/kernel/bpf/bpf_struct_ops_types.h -+++ b/kernel/bpf/bpf_struct_ops_types.h -@@ -9,8 +9,4 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) - #include - BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) - #endif --#ifdef CONFIG_SCHED_CLASS_EXT --#include --BPF_STRUCT_OPS_TYPE(sched_ext_ops) --#endif - #endif -diff --git a/kernel/fork.c b/kernel/fork.c -index d0a46a152428..7a91505b9378 100644 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -23,7 +23,9 @@ - #include - #include - #include --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - #include - #include - #include -@@ -999,7 +1001,9 @@ void __put_task_struct(struct task_struct *tsk) - WARN_ON(!tsk->exit_state); - WARN_ON(refcount_read(&tsk->usage)); - WARN_ON(tsk == current); -- sched_ext_free(tsk); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_free(tsk); -+#endif - io_uring_free(tsk); - cgroup_free(tsk); - task_numa_free(tsk, true); -@@ -2516,7 +2520,11 @@ __latent_entropy struct task_struct *copy_process( - - retval = perf_event_init_task(p, clone_flags); - if (retval) -- goto bad_fork_sched_cancel_fork; -+#ifdef CONFIG_HMBIRD_SCHED -+ goto bad_fork_hmbird_cancel_fork; -+#else -+ goto bad_fork_cleanup_policy; -+#endif - retval = audit_alloc(p); - if (retval) - goto bad_fork_cleanup_perf; -@@ -2648,9 +2656,7 @@ __latent_entropy struct task_struct *copy_process( - * cgroup specific, it unconditionally needs to place the task on a - * runqueue. - */ -- retval = sched_cgroup_fork(p, args); -- if (retval) -- goto bad_fork_cancel_cgroup; -+ sched_cgroup_fork(p, args); - - /* - * From this point on we must avoid any synchronous user-space -@@ -2696,13 +2702,13 @@ __latent_entropy struct task_struct *copy_process( - /* Don't start children in a dying pid namespace */ - if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { - retval = -ENOMEM; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* Let kill terminate clone/fork in the middle */ - if (fatal_signal_pending(current)) { - retval = -EINTR; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* No more failure paths after this point. */ -@@ -2779,11 +2785,10 @@ __latent_entropy struct task_struct *copy_process( - - return p; - --bad_fork_core_free: -+bad_fork_cancel_cgroup: - sched_core_free(p); - spin_unlock(¤t->sighand->siglock); - write_unlock_irq(&tasklist_lock); --bad_fork_cancel_cgroup: - cgroup_cancel_fork(p, args); - bad_fork_put_pidfd: - if (clone_flags & CLONE_PIDFD) { -@@ -2822,8 +2827,10 @@ __latent_entropy struct task_struct *copy_process( - audit_free(p); - bad_fork_cleanup_perf: - perf_event_free_task(p); --bad_fork_sched_cancel_fork: -- sched_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+bad_fork_hmbird_cancel_fork: -+ hmbird_cancel_fork(p); -+#endif - bad_fork_cleanup_policy: - lockdep_free_task(p); - #ifdef CONFIG_NUMA -diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c -index 005025f55bea..c091539a0f60 100644 ---- a/kernel/sched/build_policy.c -+++ b/kernel/sched/build_policy.c -@@ -28,8 +28,10 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED - #include - #include -+#endif - - #include - -@@ -54,6 +56,10 @@ - #include "cputime.c" - #include "deadline.c" - --#ifdef CONFIG_SCHED_CLASS_EXT --# include "ext.c" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/hmbird_util_track.c" -+#include "hmbird/hmbird_sched_proc.c" -+#include "hmbird/hmbird_shadow_tick.c" -+#include "hmbird/hmbird.c" -+#include "hmbird/hmbird_misc.c" - #endif -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index d486b2dd8240..692c9aa0ffa6 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -96,6 +96,12 @@ - #include "../../io_uring/io-wq.h" - #include "../smpboot.h" - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/slim.h" -+#include "hmbird/hmbird_shadow_tick.h" -+#include "hmbird.h" -+#endif -+ - #include - #include - #include -@@ -170,7 +176,6 @@ __read_mostly int scheduler_running; - - DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); - -- - /* kernel prio, less is more */ - static inline int __task_prio(const struct task_struct *p) - { -@@ -183,8 +188,8 @@ static inline int __task_prio(const struct task_struct *p) - if (p->sched_class == &idle_sched_class) - return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - --#ifdef CONFIG_SCHED_CLASS_EXT -- if (p->sched_class == &ext_sched_class) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (p->sched_class == &hmbird_sched_class) - return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ - #endif - -@@ -217,9 +222,9 @@ static inline bool prio_less(const struct task_struct *a, - if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ - return cfs_prio_less(a, b, in_fi); - --#ifdef CONFIG_SCHED_CLASS_EXT -+#ifdef CONFIG_HMBIRD_SCHED - if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ -- return scx_prio_less(a, b, in_fi); -+ return hmbird_prio_less(a, b, in_fi); - #endif - - return false; -@@ -1279,17 +1284,26 @@ bool sched_can_stop_tick(struct rq *rq) - if (fifo_nr_running) - return true; - -+#ifdef CONFIG_HMBIRD_SCHED - /* -- * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks -+ * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks - * left. For CFS, if there's more than one we need the tick for -- * involuntary preemption. For SCX, ask. -+ * involuntary preemption. For HMBIRD, ask. - */ -- if (!scx_switched_all() && rq->nr_running > 1) -+ if (!hmbird_enabled() && rq->nr_running > 1) - return false; - -- if (scx_enabled() && !scx_can_stop_tick(rq)) -+ if (hmbird_enabled() && !hmbird_can_stop_tick(rq)) - return false; -- -+#else -+ /* -+ * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; -+ * if there's more than one we need the tick for involuntary -+ * preemption. -+ */ -+ if (rq->nr_running > 1) -+ return false; -+#endif - /* - * If there is one task and it has CFS runtime bandwidth constraints - * and it's on the cpu now we don't want to stop the tick. -@@ -2222,10 +2236,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) - } - EXPORT_SYMBOL_GPL(deactivate_task); - --struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) -+#ifdef CONFIG_HMBIRD_SCHED -+struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - { -- struct sched_change_guard cg = { -+ struct hmbird_sched_change_guard cg = { - .rq = rq, - .p = p, - .queued = task_on_rq_queued(p), -@@ -2247,7 +2262,7 @@ sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - return cg; - } - --void sched_change_guard_fini(struct sched_change_guard *cg, int flags) -+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags) - { - if (cg->queued) - enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); -@@ -2255,6 +2270,7 @@ void sched_change_guard_fini(struct sched_change_guard *cg, int flags) - set_next_task(cg->rq, cg->p); - cg->done = true; - } -+#endif - - static inline int __normal_prio(int policy, int rt_prio, int nice) - { -@@ -2320,9 +2336,9 @@ inline int task_curr(const struct task_struct *p) - * this means any call to check_class_changed() must be followed by a call to - * balance_callback(). - */ --void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio) -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) - { - if (prev_class != p->sched_class) { - if (prev_class->switched_from) -@@ -2885,6 +2901,7 @@ static void - __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - { - struct rq *rq = task_rq(p); -+ bool queued, running; - - /* - * This here violates the locking rules for affinity, since we're only -@@ -2903,9 +2920,26 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - else - lockdep_assert_held(&p->pi_lock); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->sched_class->set_cpus_allowed(p, ctx); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ -+ if (queued) { -+ /* -+ * Because __kthread_bind() calls this on blocked tasks without -+ * holding rq->lock. -+ */ -+ lockdep_assert_rq_held(rq); -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); - } -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->sched_class->set_cpus_allowed(p, ctx); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - } - - /* -@@ -3772,7 +3806,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class)) -+#else - if (unlikely(!is_cpu_allowed(p, cpu))) -+#endif - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -@@ -4891,7 +4929,9 @@ late_initcall(sched_core_sysctl_init); - */ - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { -+#ifdef CONFIG_HMBIRD_SCHED - int ret; -+#endif - - trace_android_rvh_sched_fork(p); - -@@ -4932,21 +4972,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_reset_on_fork = 0; - } - -- scx_pre_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ ret = hmbird_pre_fork(p); -+ if (ret) -+ goto out_cancel; - - if (dl_prio(p->prio)) { - ret = -EAGAIN; - goto out_cancel; --#ifdef CONFIG_SCHED_CLASS_EXT -- } else if (task_on_scx(p)) { -- p->sched_class = &ext_sched_class; --#endif -+ } else if (task_on_hmbird(p)) { -+ p->sched_class = &hmbird_sched_class; - } else if (rt_prio(p->prio)) { - p->sched_class = &rt_sched_class; - } else { - p->sched_class = &fair_sched_class; - } -- -+#else -+ if (dl_prio(p->prio)) -+ return -EAGAIN; -+ else if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - init_entity_runnable_average(&p->se); - trace_android_rvh_finish_prio_fork(p); - -@@ -4965,12 +5013,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - #endif - return 0; - -+#ifdef CONFIG_HMBIRD_SCHED - out_cancel: -- scx_cancel_fork(p); -+ hmbird_cancel_fork(p); - return ret; -+#endif - } - --int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) -+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - { - unsigned long flags; - -@@ -4997,19 +5047,17 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -- -- return scx_fork(p); --} -- --void sched_cancel_fork(struct task_struct *p) --{ -- scx_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_fork(p); -+#endif - } - - void sched_post_fork(struct task_struct *p) - { - uclamp_post_fork(p); -- scx_post_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_post_fork(p); -+#endif - } - - unsigned long to_ratio(u64 period, u64 runtime) -@@ -5874,19 +5922,28 @@ void scheduler_tick(void) - if (sched_feat(LATENCY_WARN) && resched_latency) - resched_latency_warn(cpu, resched_latency); - -- scx_notify_sched_tick(); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_notify_sched_tick(); -+#endif - perf_event_task_tick(); - - if (curr->flags & PF_WQ_WORKER) - wq_worker_tick(curr); - - #ifdef CONFIG_SMP -- if (!scx_switched_all()) { -+#ifdef CONFIG_HMBIRD_SCHED -+ if (!hmbird_enabled()) { -+#endif - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); -+#ifdef CONFIG_HMBIRD_SCHED - } -+#endif - #endif - trace_android_vh_scheduler_tick(rq); -+#ifdef CONFIG_HMBIRD_SCHED -+ scheduler_tick_handler(NULL, NULL); -+#endif - } - - #ifdef CONFIG_NO_HZ_FULL -@@ -6188,7 +6245,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ -+#ifdef CONFIG_HMBIRD_SCHED - for_balance_class_range(class, prev->sched_class, &idle_sched_class) { -+#else -+ for_class_range(class, prev->sched_class, &idle_sched_class) { -+#endif - if (class->balance(rq, prev, rf)) - break; - } -@@ -6206,9 +6267,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - const struct sched_class *class; - struct task_struct *p; - -- if (scx_enabled()) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (hmbird_enabled()) - goto restart; -- -+#endif - /* - * Optimization: we know that if all tasks are in the fair class we can - * call that function directly, but only if the @prev task wasn't of a -@@ -6234,13 +6296,21 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - restart: - put_prev_task_balance(rq, prev, rf); - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { - p = class->pick_next_task(rq); - if (p) { -- scx_notify_pick_next_task(rq, p, class); -+ hmbird_notify_pick_next_task(rq, p, class); - return p; - } - } -+#else -+ for_each_class(class) { -+ p = class->pick_next_task(rq); -+ if (p) -+ return p; -+ } -+#endif - - BUG(); /* The idle class should always have a runnable task. */ - } -@@ -6269,7 +6339,11 @@ static inline struct task_struct *pick_task(struct rq *rq) - const struct sched_class *class; - struct task_struct *p; - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { -+#else -+ for_each_class(class) { -+#endif - p = class->pick_task(rq); - if (p) - return p; -@@ -6913,6 +6987,9 @@ static void __sched notrace __schedule(unsigned int sched_mode) - psi_sched_switch(prev, next, !task_on_rq_queued(prev)); - - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#ifdef CONFIG_HMBIRD_SCHED -+ sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#endif - - /* Also unlocks the rq: */ - rq = context_switch(rq, prev, next, &rf); -@@ -7241,24 +7318,31 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag - } - EXPORT_SYMBOL(default_wake_function); - --void __setscheduler_prio(struct task_struct *p, int prio) -+static void __setscheduler_prio(struct task_struct *p, int prio) - { -- /* -- * After switching all rt and fair class to ext, -- * stop class is switched to ext class too. Just -- * skip it. */ -+#ifdef CONFIG_HMBIRD_SCHED -+ bool on_hmbird = task_on_hmbird(p); -+ - if (p->sched_class == &stop_sched_class) -- ;/* do nothing */ -+ ; - else if (dl_prio(prio)) - p->sched_class = &dl_sched_class; --#ifdef CONFIG_SCHED_CLASS_EXT -- else if (task_on_scx(p)) -- p->sched_class = &ext_sched_class; --#endif -+ else if (rt_prio(prio) && on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#else -+ if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; - else if (rt_prio(prio)) - p->sched_class = &rt_sched_class; - else - p->sched_class = &fair_sched_class; -+#endif - - p->prio = prio; - trace_android_rvh_setscheduler_prio(p); -@@ -7294,7 +7378,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - */ - void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - { -- int prio, oldprio, queue_flag = -+ int prio, oldprio, queued, running, queue_flag = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *prev_class; - struct rq_flags rf; -@@ -7357,42 +7441,52 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - queue_flag &= ~DEQUEUE_MOVE; - - prev_class = p->sched_class; -- SCHED_CHANGE_BLOCK(rq, p, queue_flag) { -- /* -- * Boosting condition are: -- * 1. -rt task is running and holds mutex A -- * --> -dl task blocks on mutex A -- * -- * 2. -dl task is running and holds mutex A -- * --> -dl task blocks on mutex A and could preempt the -- * running task -- */ -- if (dl_prio(prio)) { -- if (!dl_prio(p->normal_prio) || -- (pi_task && dl_prio(pi_task->prio) && -- dl_entity_preempt(&pi_task->dl, &p->dl))) { -- p->dl.pi_se = pi_task->dl.pi_se; -- queue_flag |= ENQUEUE_REPLENISH; -- } else { -- p->dl.pi_se = &p->dl; -- } -- } else if (rt_prio(prio)) { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- if (oldprio < prio) -- queue_flag |= ENQUEUE_HEAD; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flag); -+ if (running) -+ put_prev_task(rq, p); -+ -+ /* -+ * Boosting condition are: -+ * 1. -rt task is running and holds mutex A -+ * --> -dl task blocks on mutex A -+ * -+ * 2. -dl task is running and holds mutex A -+ * --> -dl task blocks on mutex A and could preempt the -+ * running task -+ */ -+ if (dl_prio(prio)) { -+ if (!dl_prio(p->normal_prio) || -+ (pi_task && dl_prio(pi_task->prio) && -+ dl_entity_preempt(&pi_task->dl, &p->dl))) { -+ p->dl.pi_se = pi_task->dl.pi_se; -+ queue_flag |= ENQUEUE_REPLENISH; - } else { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- else if (rt_prio(oldprio)) -- p->rt.timeout = 0; -- else if (!task_has_idle_policy(p)) -- reweight_task(p, prio - MAX_RT_PRIO); -+ p->dl.pi_se = &p->dl; - } -- -- __setscheduler_prio(p, prio); -+ } else if (rt_prio(prio)) { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ if (oldprio < prio) -+ queue_flag |= ENQUEUE_HEAD; -+ } else { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ else if (rt_prio(oldprio)) -+ p->rt.timeout = 0; -+ else if (!task_has_idle_policy(p)) -+ reweight_task(p, prio - MAX_RT_PRIO); - } - -+ __setscheduler_prio(p, prio); -+ -+ if (queued) -+ enqueue_task(rq, p, queue_flag); -+ if (running) -+ set_next_task(rq, p); -+ - check_class_changed(rq, p, prev_class, oldprio); - out_unlock: - /* Avoid rq from going away on us: */ -@@ -7413,7 +7507,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - - void set_user_nice(struct task_struct *p, long nice) - { -- bool allowed = false; -+ bool queued, running, allowed = false; - int old_prio; - struct rq_flags rf; - struct rq *rq; -@@ -7442,13 +7536,22 @@ void set_user_nice(struct task_struct *p, long nice) - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); -+ if (running) -+ put_prev_task(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->static_prio = NICE_TO_PRIO(nice); -- set_load_weight(p, true); -- old_prio = p->prio; -- p->prio = effective_prio(p); -- } -+ p->static_prio = NICE_TO_PRIO(nice); -+ set_load_weight(p, true); -+ old_prio = p->prio; -+ p->prio = effective_prio(p); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - - /* - * If the task increased its priority or is running and -@@ -7865,7 +7968,7 @@ static int __sched_setscheduler(struct task_struct *p, - bool user, bool pi) - { - int oldpolicy = -1, policy = attr->sched_policy; -- int retval, oldprio, newprio; -+ int retval, oldprio, newprio, queued, running; - const struct sched_class *prev_class; - struct balance_callback *head; - struct rq_flags rf; -@@ -7873,6 +7976,9 @@ static int __sched_setscheduler(struct task_struct *p, - int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct rq *rq; - bool cpuset_locked = false; -+#ifdef CONFIG_HMBIRD_SCHED -+ unsigned long flags; -+#endif - - /* The pi code expects interrupts enabled */ - BUG_ON(pi && in_interrupt()); -@@ -7938,6 +8044,9 @@ static int __sched_setscheduler(struct task_struct *p, - * To be able to change p->policy safely, the appropriate - * runqueue lock must be held. - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+#endif - rq = task_rq_lock(p, &rf); - update_rq_clock(rq); - -@@ -7949,10 +8058,11 @@ static int __sched_setscheduler(struct task_struct *p, - goto unlock; - } - -- retval = scx_check_setscheduler(p, policy); -+#ifdef CONFIG_HMBIRD_SCHED -+ retval = hmbird_check_setscheduler(p, policy); - if (retval) - goto unlock; -- -+#endif - /* - * If not changing anything there's no need to proceed further, - * but store a possible modification of reset_on_fork. -@@ -8009,6 +8119,9 @@ static int __sched_setscheduler(struct task_struct *p, - if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { - policy = oldpolicy = -1; - task_rq_unlock(rq, p, &rf); -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - goto recheck; -@@ -8041,16 +8154,23 @@ static int __sched_setscheduler(struct task_struct *p, - queue_flags &= ~DEQUEUE_MOVE; - } - -- SCHED_CHANGE_BLOCK(rq, p, queue_flags) { -- prev_class = p->sched_class; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flags); -+ if (running) -+ put_prev_task(rq, p); - -- if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -- __setscheduler_params(p, attr); -- __setscheduler_prio(p, newprio); -- trace_android_rvh_setscheduler(p); -- } -- __setscheduler_uclamp(p, attr); -+ prev_class = p->sched_class; - -+ if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -+ __setscheduler_params(p, attr); -+ __setscheduler_prio(p, newprio); -+ trace_android_rvh_setscheduler(p); -+ } -+ __setscheduler_uclamp(p, attr); -+ -+ if (queued) { - /* - * We enqueue to tail when the priority of a task is - * increased (user space view). -@@ -8058,7 +8178,10 @@ static int __sched_setscheduler(struct task_struct *p, - if (oldprio < p->prio) - queue_flags |= ENQUEUE_HEAD; - -+ enqueue_task(rq, p, queue_flags); - } -+ if (running) -+ set_next_task(rq, p); - - check_class_changed(rq, p, prev_class, oldprio); - -@@ -8066,7 +8189,9 @@ static int __sched_setscheduler(struct task_struct *p, - preempt_disable(); - head = splice_balance_callbacks(rq); - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (pi) { - if (cpuset_locked) - cpuset_unlock(); -@@ -8081,6 +8206,9 @@ static int __sched_setscheduler(struct task_struct *p, - - unlock: - task_rq_unlock(rq, p, &rf); -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - return retval; -@@ -9302,7 +9430,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - break; - } -@@ -9330,7 +9460,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - } - return ret; -@@ -9631,15 +9763,25 @@ int migrate_task_to(struct task_struct *p, int target_cpu) - */ - void sched_setnuma(struct task_struct *p, int nid) - { -+ bool queued, running; - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) { -- p->numa_preferred_nid = nid; -- } -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE); -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->numa_preferred_nid = nid; - -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - task_rq_unlock(rq, p, &rf); - } - #endif /* CONFIG_NUMA_BALANCING */ -@@ -10205,15 +10347,22 @@ void __init sched_init(void) - int i; - - /* Make sure the linker didn't screw up */ -+#ifdef CONFIG_HMBIRD_SCHED - #ifdef CONFIG_SMP -- BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+#endif -+ WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class)); -+#else -+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 || -+ &fair_sched_class != &rt_sched_class + 1 || -+ &rt_sched_class != &dl_sched_class + 1); -+#ifdef CONFIG_SMP -+ BUG_ON(&dl_sched_class != &stop_sched_class + 1); - #endif -- BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); -- BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); -- BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); --#ifdef CONFIG_SCHED_CLASS_EXT -- BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); -- BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); - #endif - - wait_bit_init(); -@@ -10385,8 +10534,9 @@ void __init sched_init(void) - balance_push_set(smp_processor_id(), false); - #endif - init_sched_fair_class(); -- init_sched_ext_class(); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ init_sched_hmbird_class(); -+#endif - psi_init(); - - init_uclamp(); -@@ -10790,6 +10940,9 @@ static void sched_change_group(struct task_struct *tsk) - */ - void sched_move_task(struct task_struct *tsk) - { -+ int queued, running, queue_flags = -+ DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; -+ struct task_group *group; - struct rq_flags rf; - struct rq *rq; - -@@ -10797,18 +10950,27 @@ void sched_move_task(struct task_struct *tsk) - rq = task_rq_lock(tsk, &rf); - update_rq_clock(rq); - -- SCHED_CHANGE_BLOCK(rq, tsk, -- DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { -- sched_change_group(tsk); -- } -+ running = task_current(rq, tsk); -+ queued = task_on_rq_queued(tsk); - -- /* -- * After changing group, the running task may have joined a throttled -- * one but it's still the running task. Trigger a resched to make sure -- * that task can still run. -- */ -- if (task_current(rq, tsk)) -+ if (queued) -+ dequeue_task(rq, tsk, queue_flags); -+ if (running) -+ put_prev_task(rq, tsk); -+ -+ sched_change_group(tsk, group); -+ -+ if (queued) -+ enqueue_task(rq, tsk, queue_flags); -+ if (running) { -+ set_next_task(rq, tsk); -+ /* -+ * After changing group, the running task may have joined a -+ * throttled one but it's still the running task. Trigger a -+ * resched to make sure that task can still run. -+ */ - resched_curr(rq); -+ } - - task_rq_unlock(rq, tsk, &rf); - } -@@ -10845,6 +11007,10 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) - struct task_group *tg = css_tg(css); - struct task_group *parent = css_tg(css->parent); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_tg_online(tg); -+#endif -+ - if (parent) - sched_online_group(tg, parent); - -@@ -10894,13 +11060,79 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) - } - #endif - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 dl) -+{ -+ int i; -+ -+ for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) { -+ if (dl < HMBIRD_BPF_DSQS_DEADLINE[i]) -+ break; -+ } -+ -+ i = max_t(int, i-1, MIN_CGROUP_DL_IDX); -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return 0; -+ update_cgroup_ids_table(css->cgroup->kn->id, i); -+ -+ return 0; -+} -+ -+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ u8 i; -+ -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX]; -+ i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1); -+ if (i < 0) { -+ pr_err(" <%s> i is %d, less than 0, name is %s\n", -+ __func__, i, css->cgroup->kn->name); -+ i = DEFAULT_CGROUP_DL_IDX; -+ } -+ -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[i]; -+} -+ -+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css, -+ struct task_struct *p, int prio) -+{ -+ if (!css || !rt_prio(prio)) -+ return; -+ -+ if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) { -+ if (!strcmp(css->cgroup->kn->name, "display")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3); -+ else if (!strcmp(css->cgroup->kn->name, "touch")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2); -+ else if (!strcmp(css->cgroup->kn->name, "multimedia")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+ } -+} -+#endif -+ - static void cpu_cgroup_attach(struct cgroup_taskset *tset) - { - struct task_struct *task; - struct cgroup_subsys_state *css; - -- cgroup_taskset_for_each(task, css, tset) -+ cgroup_taskset_for_each(task, css, tset) { -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_change_rt_sched_prop(css, task, task->prio); -+#endif - sched_move_task(task); -+ } - - trace_android_rvh_cpu_cgroup_attach(tset); - } -@@ -11579,6 +11811,13 @@ static struct cftype cpu_legacy_files[] = { - .read_u64 = cpu_uclamp_ls_read_u64, - .write_u64 = cpu_uclamp_ls_write_u64, - }, -+#endif -+#ifdef CONFIG_HMBIRD_SCHED -+ { -+ .name = "hmbird.deadline", -+ .read_u64 = cgroup_read_hmbird_deadline, -+ .write_u64 = cgroup_write_hmbird_deadline, -+ }, - #endif - { } /* Terminate */ - }; -@@ -11628,7 +11867,6 @@ static int cpu_local_stat_show(struct seq_file *sf, - } - - #ifdef CONFIG_FAIR_GROUP_SCHED -- - static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, - struct cftype *cft) - { -diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c -index a6c99df9edf9..f647dacc22e2 100644 ---- a/kernel/sched/debug.c -+++ b/kernel/sched/debug.c -@@ -376,8 +376,8 @@ static __init int sched_init_debug(void) - - debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); - --#ifdef CONFIG_SCHED_CLASS_EXT -- debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops); -+#ifdef CONFIG_HMBIRD_SCHED -+ debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops); - #endif - return 0; - } -@@ -1006,9 +1006,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - SEQ_printf(m, - "---------------------------------------------------------" - "----------\n"); --#ifdef CONFIG_SLIM_SCHED -- SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop); --#endif - - #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) - #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) -@@ -1104,8 +1101,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - } else if (fair_policy(p->policy)) { - P(se.slice); - } --#ifdef CONFIG_SCHED_CLASS_EXT -- __PS("ext.enabled", p->sched_class == &ext_sched_class); -+#ifdef CONFIG_HMBIRD_SCHED -+ __PS("hmbird.enabled", p->sched_class == &hmbird_sched_class); -+ __PS("sched_prop", get_hmbird_ts(p)->sched_prop); -+ __PS("top_task_prop", get_hmbird_ts(p)->top_task_prop); - #endif - #undef PN_SCHEDSTAT - #undef P_SCHEDSTAT -diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c -deleted file mode 100755 -index 4845d441df2b..000000000000 ---- a/kernel/sched/ext.c -+++ /dev/null -@@ -1,3978 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -- --enum scx_internal_consts { -- SCX_NR_ONLINE_OPS = SCX_OP_IDX(init), -- SCX_DSP_DFL_MAX_BATCH = 32, -- SCX_DSP_MAX_LOOPS = 32, -- SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, --}; -- --enum scx_ops_enable_state { -- SCX_OPS_PREPPING, -- SCX_OPS_ENABLING, -- SCX_OPS_ENABLED, -- SCX_OPS_DISABLING, -- SCX_OPS_DISABLED, --}; -- --static inline struct task_group *css_tg(struct cgroup_subsys_state *css) --{ -- return css ? container_of(css, struct task_group, css) : NULL; --} -- --/* -- * sched_ext_entity->ops_state -- * -- * Used to track the task ownership between the SCX core and the BPF scheduler. -- * State transitions look as follows: -- * -- * NONE -> QUEUEING -> QUEUED -> DISPATCHING -- * ^ | | -- * | v v -- * \-------------------------------/ -- * -- * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -- * sites for explanations on the conditions being waited upon and why they are -- * safe. Transitions out of them into NONE or QUEUED must store_release and the -- * waiters should load_acquire. -- * -- * Tracking scx_ops_state enables sched_ext core to reliably determine whether -- * any given task can be dispatched by the BPF scheduler at all times and thus -- * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -- * to try to dispatch any task anytime regardless of its state as the SCX core -- * can safely reject invalid dispatches. -- */ --enum scx_ops_state { -- SCX_OPSS_NONE, /* owned by the SCX core */ -- SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -- SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ -- SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ -- -- /* -- * QSEQ brands each QUEUED instance so that, when dispatch races -- * dequeue/requeue, the dispatcher can tell whether it still has a claim -- * on the task being dispatched. -- */ -- SCX_OPSS_QSEQ_SHIFT = 2, -- SCX_OPSS_STATE_MASK = (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1, -- SCX_OPSS_QSEQ_MASK = ~SCX_OPSS_STATE_MASK, --}; -- --/* -- * During exit, a task may schedule after losing its PIDs. When disabling the -- * BPF scheduler, we need to be able to iterate tasks in every state to -- * guarantee system safety. Maintain a dedicated task list which contains every -- * task between its fork and eventual free. -- */ --static DEFINE_SPINLOCK(scx_tasks_lock); --static LIST_HEAD(scx_tasks); -- --/* ops enable/disable */ --static struct kthread_worker *scx_ops_helper; --static DEFINE_MUTEX(scx_ops_enable_mutex); --DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); --DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); --static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); --static bool scx_switch_all_req; --static bool scx_switching_all; --DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -- --static struct sched_ext_ops scx_ops; --static bool scx_warned_zero_slice; -- --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); --DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); --static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); -- --struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] = -- { [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT }; -- --static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE); --static struct scx_exit_info scx_exit_info; -- --static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0); -- --/* -- * The maximum amount of time in jiffies that a task may be runnable without -- * being scheduled on a CPU. If this timeout is exceeded, it will trigger -- * scx_ops_error(). -- */ --unsigned long scx_watchdog_timeout; -- --/* -- * The last time the delayed work was run. This delayed work relies on -- * ksoftirqd being able to run to service timer interrupts, so it's possible -- * that this work itself could get wedged. To account for this, we check that -- * it's not stalled in the timer tick, and trigger an error if it is. -- */ --unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; -- --static struct delayed_work scx_watchdog_work; -- --/* idle tracking */ --#ifdef CONFIG_SMP --#ifdef CONFIG_CPUMASK_OFFSTACK --#define CL_ALIGNED_IF_ONSTACK --#else --#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp --#endif -- --static struct { -- cpumask_var_t cpu; -- cpumask_var_t smt; --} idle_masks CL_ALIGNED_IF_ONSTACK; -- --static bool __cacheline_aligned_in_smp scx_has_idle_cpus; --#endif /* CONFIG_SMP */ -- --/* for %SCX_KICK_WAIT */ --static u64 __percpu *scx_kick_cpus_pnt_seqs; -- --/* -- * Direct dispatch marker. -- * -- * Non-NULL values are used for direct dispatch from enqueue path. A valid -- * pointer points to the task currently being enqueued. An ERR_PTR value is used -- * to indicate that direct dispatch has already happened. -- */ --static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); -- --/* dispatch queues */ --static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; -- --static LLIST_HEAD(dsqs_to_free); -- --/* dispatch buf */ --struct scx_dsp_buf_ent { -- struct task_struct *task; -- u64 qseq; -- u64 dsq_id; -- u64 enq_flags; --}; -- --static u32 scx_dsp_max_batch; --static struct scx_dsp_buf_ent __percpu *scx_dsp_buf; -- --struct scx_dsp_ctx { -- struct rq *rq; -- struct rq_flags *rf; -- u32 buf_cursor; -- u32 nr_tasks; --}; -- --static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx); -- --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags); --void scx_bpf_kick_cpu(s32 cpu, u64 flags); -- --struct scx_task_iter { -- struct sched_ext_entity cursor; -- struct task_struct *locked; -- struct rq *rq; -- struct rq_flags rf; --}; -- --#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) -- --/* if the highest set bit is N, return a mask with bits [N+1, 31] set */ --static u32 higher_bits(u32 flags) --{ -- return ~((1 << fls(flags)) - 1); --} -- --/* return the mask with only the highest bit set */ --static u32 highest_bit(u32 flags) --{ -- int bit = fls(flags); -- return bit ? 1 << (bit - 1) : 0; --} -- --/* -- * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX -- * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate -- * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check -- * whether it's running from an allowed context. -- * -- * @mask is constant, always inline to cull the mask calculations. -- */ --static __always_inline void scx_kf_allow(u32 mask) --{ -- /* nesting is allowed only in increasing scx_kf_mask order */ -- WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask, -- "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n", -- current->scx->kf_mask, mask); -- current->scx->kf_mask |= mask; --} -- --static void scx_kf_disallow(u32 mask) --{ -- current->scx->kf_mask &= ~mask; --} -- --#define SCX_CALL_OP(mask, op, args...) \ --do { \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- scx_ops.op(args); \ -- } \ --} while (0) -- --#define SCX_CALL_OP_RET(mask, op, args...) \ --({ \ -- __typeof__(scx_ops.op(args)) __ret; \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- __ret = scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- __ret = scx_ops.op(args); \ -- } \ -- __ret; \ --}) -- --/* -- * Some kfuncs are allowed only on the tasks that are subjects of the -- * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such -- * restrictions, the following SCX_CALL_OP_*() variants should be used when -- * invoking scx_ops operations that take task arguments. These can only be used -- * for non-nesting operations due to the way the tasks are tracked. -- * -- * kfuncs which can only operate on such tasks can in turn use -- * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on -- * the specific task. -- */ --#define SCX_CALL_OP_TASK(mask, op, task, args...) \ --do { \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- SCX_CALL_OP(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ --} while (0) -- --#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ --({ \ -- __typeof__(scx_ops.op(task, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- __ret; \ --}) -- --#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ --({ \ -- __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task0; \ -- current->scx->kf_tasks[1] = task1; \ -- __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- current->scx->kf_tasks[1] = NULL; \ -- __ret; \ --}) -- --//#include "./slim_walt.c" -- --/* @mask is constant, always inline to cull unnecessary branches */ --static __always_inline bool scx_kf_allowed(u32 mask) --{ -- if (unlikely(!(current->scx->kf_mask & mask))) { -- scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", -- mask, current->scx->kf_mask); -- return false; -- } -- -- if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) && -- in_interrupt())) { -- scx_ops_error("sleepable kfunc called from non-sleepable context"); -- return false; -- } -- -- /* -- * Enforce nesting boundaries. e.g. A kfunc which can be called from -- * DISPATCH must not be called if we're running DEQUEUE which is nested -- * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE -- * boundary thanks to the above in_interrupt() check. -- */ -- if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && -- (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { -- scx_ops_error("cpu_release kfunc called from a nested operation"); -- return false; -- } -- -- if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && -- (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) { -- scx_ops_error("dispatch kfunc called from a nested operation"); -- return false; -- } -- -- return true; --} -- --/* see SCX_CALL_OP_TASK() */ --static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, -- struct task_struct *p) --{ -- if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED)) -- return false; -- -- if (unlikely((p != current->scx->kf_tasks[0] && -- p != current->scx->kf_tasks[1]))) { -- scx_ops_error("called on a task not being operated on"); -- return false; -- } -- -- return true; --} -- --/** -- * scx_task_iter_init - Initialize a task iterator -- * @iter: iterator to init -- * -- * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized, -- * @iter must eventually be exited with scx_task_iter_exit(). -- * -- * scx_tasks_lock may be released between this and the first next() call or -- * between any two next() calls. If scx_tasks_lock is released between two -- * next() calls, the caller is responsible for ensuring that the task being -- * iterated remains accessible either through RCU read lock or obtaining a -- * reference count. -- * -- * All tasks which existed when the iteration started are guaranteed to be -- * visited as long as they still exist. -- */ --static void scx_task_iter_init(struct scx_task_iter *iter) --{ -- lockdep_assert_held(&scx_tasks_lock); -- -- iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; -- list_add(&iter->cursor.tasks_node, &scx_tasks); -- iter->locked = NULL; --} -- --/** -- * scx_task_iter_exit - Exit a task iterator -- * @iter: iterator to exit -- * -- * Exit a previously initialized @iter. Must be called with scx_tasks_lock held. -- * If the iterator holds a task's rq lock, that rq lock is released. See -- * scx_task_iter_init() for details. -- */ --static void scx_task_iter_exit(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- if (list_empty(cursor)) -- return; -- -- list_del_init(cursor); --} -- --/** -- * scx_task_iter_next - Next task -- * @iter: iterator to walk -- * -- * Visit the next task. See scx_task_iter_init() for details. -- */ --static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- struct sched_ext_entity *pos; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- list_for_each_entry(pos, cursor, tasks_node) { -- if (&pos->tasks_node == &scx_tasks) -- return NULL; -- if (!(pos->flags & SCX_TASK_CURSOR)) { -- list_move(cursor, &pos->tasks_node); -- return pos->task; -- } -- } -- -- /* can't happen, should always terminate at scx_tasks above */ -- BUG(); --} -- --/** -- * scx_task_iter_next_filtered - Next non-idle task -- * @iter: iterator to walk -- * -- * Visit the next non-idle task. See scx_task_iter_init() for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- while ((p = scx_task_iter_next(iter))) { -- if (!is_idle_task(p)) -- return p; -- } -- return NULL; --} -- --/** -- * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked -- * @iter: iterator to walk -- * -- * Visit the next non-idle task with its rq lock held. See scx_task_iter_init() -- * for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered_locked(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- p = scx_task_iter_next_filtered(iter); -- if (!p) -- return NULL; -- -- iter->rq = task_rq_lock(p, &iter->rf); -- iter->locked = p; -- return p; --} -- --static enum scx_ops_enable_state scx_ops_enable_state(void) --{ -- return atomic_read(&scx_ops_enable_state_var); --} -- --static enum scx_ops_enable_state --scx_ops_set_enable_state(enum scx_ops_enable_state to) --{ -- return atomic_xchg(&scx_ops_enable_state_var, to); --} -- --static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, -- enum scx_ops_enable_state from) --{ -- int from_v = from; -- -- return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); --} -- --static bool scx_ops_disabling(void) --{ -- return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING); --} -- --/** -- * wait_ops_state - Busy-wait the specified ops state to end -- * @p: target task -- * @opss: state to wait the end of -- * -- * Busy-wait for @p to transition out of @opss. This can only be used when the -- * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also -- * has load_acquire semantics to ensure that the caller can see the updates made -- * in the enqueueing and dispatching paths. -- */ --static void wait_ops_state(struct task_struct *p, u64 opss) --{ -- do { -- cpu_relax(); -- } while (atomic64_read_acquire(&p->scx->ops_state) == opss); --} -- --/** -- * ops_cpu_valid - Verify a cpu number -- * @cpu: cpu number which came from a BPF ops -- * -- * @cpu is a cpu number which came from the BPF scheduler and can be any value. -- * Verify that it is in range and one of the possible cpus. -- */ --static bool ops_cpu_valid(s32 cpu) --{ -- return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); --} -- --/** -- * ops_sanitize_err - Sanitize a -errno value -- * @ops_name: operation to blame on failure -- * @err: -errno value to sanitize -- * -- * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return -- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can -- * cause misbehaviors. For an example, a large negative return from -- * ops.prep_enable() triggers an oops when passed up the call chain because the -- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is -- * handled as a pointer. -- */ --static int ops_sanitize_err(const char *ops_name, s32 err) --{ -- if (err < 0 && err >= -MAX_ERRNO) -- return err; -- -- scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); -- return -EPROTO; --} -- --/** -- * touch_core_sched - Update timestamp used for core-sched task ordering -- * @rq: rq to read clock from, must be locked -- * @p: task to update the timestamp for -- * -- * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to -- * implement global or local-DSQ FIFO ordering for core-sched. Should be called -- * when a task becomes runnable and its turn on the CPU ends (e.g. slice -- * exhaustion). -- */ --static void touch_core_sched(struct rq *rq, struct task_struct *p) --{ --#ifdef CONFIG_SCHED_CORE -- /* -- * It's okay to update the timestamp spuriously. Use -- * sched_core_disabled() which is cheaper than enabled(). -- */ -- if (!sched_core_disabled()) -- p->scx->core_sched_at = rq_clock_task(rq); --#endif --} -- --/** -- * touch_core_sched_dispatch - Update core-sched timestamp on dispatch -- * @rq: rq to read clock from, must be locked -- * @p: task being dispatched -- * -- * If the BPF scheduler implements custom core-sched ordering via -- * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO -- * ordering within each local DSQ. This function is called from dispatch paths -- * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect. -- */ --static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- assert_clock_updated(rq); -- --#ifdef CONFIG_SCHED_CORE -- if (SCX_HAS_OP(core_sched_before)) -- touch_core_sched(rq, p); --#endif --} -- --static void update_curr_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- u64 now = rq_clock_task(rq); -- u64 delta_exec; -- -- if (time_before_eq64(now, curr->se.exec_start)) -- return; -- -- delta_exec = now - curr->se.exec_start; -- curr->se.exec_start = now; -- curr->se.sum_exec_runtime += delta_exec; -- account_group_exec_runtime(curr, delta_exec); -- cgroup_account_cputime(curr, delta_exec); -- -- if (curr->scx->slice != SCX_SLICE_INF) { -- curr->scx->slice -= min(curr->scx->slice, delta_exec); -- if (!curr->scx->slice) -- touch_core_sched(rq, curr); -- } --} -- --static bool scx_dsq_priq_less(struct rb_node *node_a, -- const struct rb_node *node_b) --{ -- const struct sched_ext_entity *a = -- container_of(node_a, struct sched_ext_entity, dsq_node.priq); -- const struct sched_ext_entity *b = -- container_of(node_b, struct sched_ext_entity, dsq_node.priq); -- -- return time_before64(a->dsq_vtime, b->dsq_vtime); --} -- --static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, -- u64 enq_flags) --{ -- bool is_local = dsq->id == SCX_DSQ_LOCAL; -- -- WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo)); -- WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq)); -- -- if (!is_local) { -- raw_spin_lock(&dsq->lock); -- if (unlikely(dsq->id == SCX_DSQ_INVALID)) { -- scx_ops_error("attempting to dispatch to a destroyed dsq"); -- /* fall back to the global dsq */ -- raw_spin_unlock(&dsq->lock); -- dsq = &scx_dsq_global; -- raw_spin_lock(&dsq->lock); -- } -- } -- -- if (enq_flags & SCX_ENQ_DSQ_PRIQ) { -- p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; -- rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq, -- scx_dsq_priq_less); -- } else { -- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) -- list_add(&p->scx->dsq_node.fifo, &dsq->fifo); -- else -- list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo); -- } -- dsq->nr++; -- p->scx->dsq = dsq; -- -- /* -- * We're transitioning out of QUEUEING or DISPATCHING. store_release to -- * match waiters' load_acquire. -- */ -- if (enq_flags & SCX_ENQ_CLEAR_OPSS) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- if (is_local) { -- struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq); -- struct rq *rq = scx->rq; -- bool preempt = false; -- -- if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && -- rq->curr->sched_class == &ext_sched_class) { -- rq->curr->scx->slice = 0; -- preempt = true; -- } -- -- if (preempt || sched_class_above(&ext_sched_class, -- rq->curr->sched_class)) -- resched_curr(rq); -- } else { -- raw_spin_unlock(&dsq->lock); -- } --} -- --static void task_unlink_from_dsq(struct task_struct *p, -- struct scx_dispatch_q *dsq) --{ -- if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { -- rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; -- } else { -- list_del_init(&p->scx->dsq_node.fifo); -- } -- --} -- --static bool task_linked_on_dsq(struct task_struct *p) --{ -- return !list_empty(&p->scx->dsq_node.fifo) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq); --} -- --static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p) --{ -- struct scx_dispatch_q *dsq = p->scx->dsq; -- bool is_local = dsq == &scx_rq->local_dsq; -- -- if (!dsq) { -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- /* -- * When dispatching directly from the BPF scheduler to a local -- * DSQ, the task isn't associated with any DSQ but -- * @p->scx->holding_cpu may be set under the protection of -- * %SCX_OPSS_DISPATCHING. -- */ -- if (p->scx->holding_cpu >= 0) -- p->scx->holding_cpu = -1; -- return; -- } -- -- if (!is_local) -- raw_spin_lock(&dsq->lock); -- -- /* -- * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node -- * can't change underneath us. -- */ -- if (p->scx->holding_cpu < 0) { -- /* @p must still be on @dsq, dequeue */ -- WARN_ON_ONCE(!task_linked_on_dsq(p)); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- } else { -- /* -- * We're racing against dispatch_to_local_dsq() which already -- * removed @p from @dsq and set @p->scx->holding_cpu. Clear the -- * holding_cpu which tells dispatch_to_local_dsq() that it lost -- * the race. -- */ -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- p->scx->holding_cpu = -1; -- } -- p->scx->dsq = NULL; -- -- if (!is_local) -- raw_spin_unlock(&dsq->lock); --} -- --static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) --{ -- lockdep_assert(rcu_read_lock_any_held()); -- -- return &scx_dsq_global; --} -- --static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, -- struct task_struct *p) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id == SCX_DSQ_LOCAL) -- return &rq->scx->local_dsq; -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", -- dsq_id, p->comm, p->pid); -- return &scx_dsq_global; -- } -- -- return dsq; --} -- --static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- -- /* @p must match the task which is being enqueued */ -- if (unlikely(p != ddsp_task)) { -- if (IS_ERR(ddsp_task)) -- scx_ops_error("%s[%d] already direct-dispatched", -- p->comm, p->pid); -- else -- scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]", -- ddsp_task->comm, ddsp_task->pid, -- p->comm, p->pid); -- return; -- } -- -- /* -- * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because -- * dispatching to the local DSQ of a different CPU requires unlocking -- * the current rq which isn't allowed in the enqueue path. Use -- * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL. -- */ -- if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) { -- scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch"); -- return; -- } -- -- touch_core_sched_dispatch(task_rq(p), p); -- -- dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- -- /* -- * Mark that dispatch already happened by spoiling direct_dispatch_task -- * with a non-NULL value which can never match a valid task pointer. -- */ -- __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); --} -- --static bool test_rq_online(struct rq *rq) --{ --#ifdef CONFIG_SMP -- return rq->online; --#else -- return true; --#endif --} -- --static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- int sticky_cpu) --{ -- struct task_struct **ddsp_taskp; -- u64 qseq; -- -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- if (p->scx->flags & SCX_TASK_ENQ_LOCAL) { -- enq_flags |= SCX_ENQ_LOCAL; -- p->scx->flags &= ~SCX_TASK_ENQ_LOCAL; -- } -- -- /* rq migration */ -- if (sticky_cpu == cpu_of(rq)) -- goto local_norefill; -- -- /* -- * If !rq->online, we already told the BPF scheduler that the CPU is -- * offline. We're just trying to on/offline the CPU. Don't bother the -- * BPF scheduler. -- */ -- if (unlikely(!test_rq_online(rq))) -- goto local; -- -- /* see %SCX_OPS_ENQ_EXITING */ -- if (!static_branch_unlikely(&scx_ops_enq_exiting) && -- unlikely(p->flags & PF_EXITING)) -- goto local; -- -- /* see %SCX_OPS_ENQ_LAST */ -- if (!static_branch_unlikely(&scx_ops_enq_last) && -- (enq_flags & SCX_ENQ_LAST)) -- goto local; -- -- if (!SCX_HAS_OP(enqueue)) { -- if (enq_flags & SCX_ENQ_LOCAL) -- goto local; -- else -- goto global; -- } -- -- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ -- qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; -- -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq); -- -- ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); -- WARN_ON_ONCE(*ddsp_taskp); -- *ddsp_taskp = p; -- -- SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); -- -- /* -- * If not directly dispatched, QUEUEING isn't clear yet and dispatch or -- * dequeue may be waiting. The store_release matches their load_acquire. -- */ -- if (*ddsp_taskp == p) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq); -- *ddsp_taskp = NULL; -- return; -- --local: -- /* -- * For task-ordering, slice refill must be treated as implying the end -- * of the current slice. Otherwise, the longer @p stays on the CPU, the -- * higher priority it becomes from scx_prio_less()'s POV. -- */ -- touch_core_sched(rq, p); -- p->scx->slice = SCX_SLICE_DFL; --local_norefill: -- dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags); -- return; -- --global: -- touch_core_sched(rq, p); /* see the comment in local: */ -- p->scx->slice = SCX_SLICE_DFL; -- dispatch_enqueue(&scx_dsq_global, p, enq_flags); --} -- --static bool watchdog_task_watched(const struct task_struct *p) --{ -- return !list_empty(&p->scx->watchdog_node); --} -- --static void watchdog_watch_task(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- if (p->scx->flags & SCX_TASK_WATCHDOG_RESET) -- p->scx->runnable_at = jiffies; -- p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET; -- list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list); --} -- --static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) --{ -- list_del_init(&p->scx->watchdog_node); -- if (reset_timeout) -- p->scx->flags |= SCX_TASK_WATCHDOG_RESET; --} -- --static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) --{ -- int sticky_cpu = p->scx->sticky_cpu; -- -- enq_flags |= rq->scx->extra_enq_flags; -- -- if (sticky_cpu >= 0) -- p->scx->sticky_cpu = -1; -- -- /* -- * Restoring a running task will be immediately followed by -- * set_next_task_scx() which expects the task to not be on the BPF -- * scheduler as tasks can only start running through local DSQs. Force -- * direct-dispatch into the local DSQ by setting the sticky_cpu. -- */ -- if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -- sticky_cpu = cpu_of(rq); -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- WARN_ON_ONCE(!watchdog_task_watched(p)); -- return; -- } -- -- watchdog_watch_task(rq, p); -- p->scx->flags |= SCX_TASK_QUEUED; -- rq->scx->nr_running++; -- add_nr_running(rq, 1); -- -- if (SCX_HAS_OP(runnable)) -- SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); -- -- if (enq_flags & SCX_ENQ_WAKEUP) -- touch_core_sched(rq, p); -- -- do_enqueue_task(rq, p, enq_flags, sticky_cpu); --} -- --static void ops_dequeue(struct task_struct *p, u64 deq_flags) --{ -- u64 opss; -- -- watchdog_unwatch_task(p, false); -- -- /* acquire ensures that we see the preceding updates on QUEUED */ -- opss = atomic64_read_acquire(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_NONE: -- break; -- case SCX_OPSS_QUEUEING: -- /* -- * QUEUEING is started and finished while holding @p's rq lock. -- * As we're holding the rq lock now, we shouldn't see QUEUEING. -- */ -- BUG(); -- case SCX_OPSS_QUEUED: -- if (SCX_HAS_OP(dequeue)) -- SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); -- -- if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_NONE)) -- break; -- fallthrough; -- case SCX_OPSS_DISPATCHING: -- /* -- * If @p is being dispatched from the BPF scheduler to a DSQ, -- * wait for the transfer to complete so that @p doesn't get -- * added to its DSQ after dequeueing is complete. -- * -- * As we're waiting on DISPATCHING with the rq locked, the -- * dispatching side shouldn't try to lock the rq while -- * DISPATCHING is set. See dispatch_to_local_dsq(). -- * -- * DISPATCHING shouldn't have qseq set and control can reach -- * here with NONE @opss from the above QUEUED case block. -- * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. -- */ -- wait_ops_state(p, SCX_OPSS_DISPATCHING); -- BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- break; -- } --} -- --static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) --{ -- struct scx_rq *scx_rq = rq->scx; -- -- if (!(p->scx->flags & SCX_TASK_QUEUED)) { -- WARN_ON_ONCE(watchdog_task_watched(p)); -- return; -- } -- -- ops_dequeue(p, deq_flags); -- -- /* -- * A currently running task which is going off @rq first gets dequeued -- * and then stops running. As we want running <-> stopping transitions -- * to be contained within runnable <-> quiescent transitions, trigger -- * ->stopping() early here instead of in put_prev_task_scx(). -- * -- * @p may go through multiple stopping <-> running transitions between -- * here and put_prev_task_scx() if task attribute changes occur while -- * balance_scx() leaves @rq unlocked. However, they don't contain any -- * information meaningful to the BPF scheduler and can be suppressed by -- * skipping the callbacks if the task is !QUEUED. -- */ -- if (SCX_HAS_OP(stopping) && task_current(rq, p)) { -- update_curr_scx(rq); -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); -- } -- -- //if(task_current(rq, p)) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- if (SCX_HAS_OP(quiescent)) -- SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); -- -- if (deq_flags & SCX_DEQ_SLEEP) -- p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP; -- else -- p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP; -- -- p->scx->flags &= ~SCX_TASK_QUEUED; -- BUG_ON(!scx_rq->nr_running); -- scx_rq->nr_running--; -- sub_nr_running(rq, 1); -- -- dispatch_dequeue(scx_rq, p); --} -- --static void yield_task_scx(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); -- else -- p->scx->slice = 0; --} -- --static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) --{ -- struct task_struct *from = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); -- else -- return false; --} -- --#ifdef CONFIG_SMP --/** -- * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -- * @rq: rq to move the task into, currently locked -- * @p: task to move -- * @enq_flags: %SCX_ENQ_* -- * -- * Move @p which is currently on a different rq to @rq's local DSQ. The caller -- * must: -- * -- * 1. Start with exclusive access to @p either through its DSQ lock or -- * %SCX_OPSS_DISPATCHING flag. -- * -- * 2. Set @p->scx->holding_cpu to raw_smp_processor_id(). -- * -- * 3. Remember task_rq(@p). Release the exclusive access so that we don't -- * deadlock with dequeue. -- * -- * 4. Lock @rq and the task_rq from #3. -- * -- * 5. Call this function. -- * -- * Returns %true if @p was successfully moved. %false after racing dequeue and -- * losing. -- */ --static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -- u64 enq_flags) --{ -- struct rq *task_rq; -- -- lockdep_assert_rq_held(rq); -- -- /* -- * If dequeue got to @p while we were trying to lock both rq's, it'd -- * have cleared @p->scx->holding_cpu to -1. While other cpus may have -- * updated it to different values afterwards, as this operation can't be -- * preempted or recurse, @p->scx->holding_cpu can never become -- * raw_smp_processor_id() again before we're done. Thus, we can tell -- * whether we lost to dequeue by testing whether @p->scx->holding_cpu is -- * still raw_smp_processor_id(). -- * -- * See dispatch_dequeue() for the counterpart. -- */ -- if (unlikely(p->scx->holding_cpu != raw_smp_processor_id())) -- return false; -- -- /* @p->rq couldn't have changed if we're still the holding cpu */ -- task_rq = task_rq(p); -- lockdep_assert_rq_held(task_rq); -- -- WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)); -- deactivate_task(task_rq, p, 0); -- set_task_cpu(p, cpu_of(rq)); -- p->scx->sticky_cpu = cpu_of(rq); -- -- /* -- * We want to pass scx-specific enq_flags but activate_task() will -- * truncate the upper 32 bit. As we own @rq, we can pass them through -- * @rq->scx->extra_enq_flags instead. -- */ -- WARN_ON_ONCE(rq->scx->extra_enq_flags); -- rq->scx->extra_enq_flags = enq_flags; -- activate_task(rq, p, 0); -- rq->scx->extra_enq_flags = 0; -- -- return true; --} -- --/** -- * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * We're holding @rq lock and trying to dispatch a task from @src_rq to -- * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether -- * @rq stays locked isn't important as long as the state is restored after -- * dispatch_to_local_dsq_unlock(). -- */ --static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- rq_unpin_lock(rq, rf); -- -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(rq); -- raw_spin_rq_lock(dst_rq); -- } else if (rq == src_rq) { -- double_lock_balance(rq, dst_rq); -- rq_repin_lock(rq, rf); -- } else if (rq == dst_rq) { -- double_lock_balance(rq, src_rq); -- rq_repin_lock(rq, rf); -- } else { -- raw_spin_rq_unlock(rq); -- double_rq_lock(src_rq, dst_rq); -- } --} -- --/** -- * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock() -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return. -- */ --static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } else if (rq == src_rq) { -- double_unlock_balance(rq, dst_rq); -- } else if (rq == dst_rq) { -- double_unlock_balance(rq, src_rq); -- } else { -- double_rq_unlock(src_rq, dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } --} --#endif /* CONFIG_SMP */ -- -- --static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq) --{ -- return likely(test_rq_online(rq)) && !is_migration_disabled(p) && -- cpumask_test_cpu(cpu_of(rq), p->cpus_ptr); --} -- --static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -- struct scx_dispatch_q *dsq) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rb_node *rb_node; -- struct rq *task_rq; -- bool moved = false; --retry: -- if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -- return false; -- -- raw_spin_lock(&dsq->lock); -- -- list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- for (rb_node = rb_first_cached(&dsq->priq); rb_node; -- rb_node = rb_next(rb_node)) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- raw_spin_unlock(&dsq->lock); -- return false; -- --this_rq: -- /* @dsq is locked and @p is on this rq */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo); -- dsq->nr--; -- scx_rq->local_dsq.nr++; -- p->scx->dsq = &scx_rq->local_dsq; -- raw_spin_unlock(&dsq->lock); -- return true; -- --remote_rq: --#ifdef CONFIG_SMP -- /* -- * @dsq is locked and @p is on a remote rq. @p is currently protected by -- * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -- * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -- * rq lock or fail, do a little dancing from our side. See -- * move_task_to_local_dsq(). -- */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- p->scx->holding_cpu = raw_smp_processor_id(); -- raw_spin_unlock(&dsq->lock); -- -- rq_unpin_lock(rq, rf); -- double_lock_balance(rq, task_rq); -- rq_repin_lock(rq, rf); -- -- moved = move_task_to_local_dsq(rq, p, 0); -- -- double_unlock_balance(rq, task_rq); --#endif /* CONFIG_SMP */ -- if (likely(moved)) -- return true; -- goto retry; --} -- --enum dispatch_to_local_dsq_ret { -- DTL_DISPATCHED, /* successfully dispatched */ -- DTL_LOST, /* lost race to dequeue */ -- DTL_NOT_LOCAL, /* destination is not a local DSQ */ -- DTL_INVALID, /* invalid local dsq_id */ --}; -- --/** -- * dispatch_to_local_dsq - Dispatch a task to a local dsq -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @dsq_id: destination dsq ID -- * @p: task to dispatch -- * @enq_flags: %SCX_ENQ_* -- * -- * We're holding @rq lock and want to dispatch @p to the local DSQ identified by -- * @dsq_id. This function performs all the synchronization dancing needed -- * because local DSQs are protected with rq locks. -- * -- * The caller must have exclusive ownership of @p (e.g. through -- * %SCX_OPSS_DISPATCHING). -- */ --static enum dispatch_to_local_dsq_ret --dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id, -- struct task_struct *p, u64 enq_flags) --{ -- struct rq *src_rq = task_rq(p); -- struct rq *dst_rq; -- -- /* -- * We're synchronized against dequeue through DISPATCHING. As @p can't -- * be dequeued, its task_rq and cpus_allowed are stable too. -- */ -- if (dsq_id == SCX_DSQ_LOCAL) { -- dst_rq = rq; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]", -- cpu, p->comm, p->pid); -- return DTL_INVALID; -- } -- dst_rq = cpu_rq(cpu); -- } else { -- return DTL_NOT_LOCAL; -- } -- -- /* if dispatching to @rq that @p is already on, no lock dancing needed */ -- if (rq == src_rq && rq == dst_rq) { -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags | SCX_ENQ_CLEAR_OPSS); -- return DTL_DISPATCHED; -- } -- --#ifdef CONFIG_SMP -- if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { -- struct rq *locked_dst_rq = dst_rq; -- bool dsp; -- -- /* -- * @p is on a possibly remote @src_rq which we need to lock to -- * move the task. If dequeue is in progress, it'd be locking -- * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq -- * lock while holding DISPATCHING. -- * -- * As DISPATCHING guarantees that @p is wholly ours, we can -- * pretend that we're moving from a DSQ and use the same -- * mechanism - mark the task under transfer with holding_cpu, -- * release DISPATCHING and then follow the same protocol. -- */ -- p->scx->holding_cpu = raw_smp_processor_id(); -- -- /* store_release ensures that dequeue sees the above */ -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq); -- -- /* -- * We don't require the BPF scheduler to avoid dispatching to -- * offline CPUs mostly for convenience but also because CPUs can -- * go offline between scx_bpf_dispatch() calls and here. If @p -- * is destined to an offline CPU, queue it on its current CPU -- * instead, which should always be safe. As this is an allowed -- * behavior, don't trigger an ops error. -- */ -- if (unlikely(!test_rq_online(dst_rq))) -- dst_rq = src_rq; -- -- if (src_rq == dst_rq) { -- /* -- * As @p is staying on the same rq, there's no need to -- * go through the full deactivate/activate cycle. -- * Optimize by abbreviating the operations in -- * move_task_to_local_dsq(). -- */ -- dsp = p->scx->holding_cpu == raw_smp_processor_id(); -- if (likely(dsp)) { -- p->scx->holding_cpu = -1; -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags); -- } -- } else { -- dsp = move_task_to_local_dsq(dst_rq, p, enq_flags); -- } -- -- /* if the destination CPU is idle, wake it up */ -- if (dsp && p->sched_class > dst_rq->curr->sched_class) -- resched_curr(dst_rq); -- -- dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq); -- -- return dsp ? DTL_DISPATCHED : DTL_LOST; -- } --#endif /* CONFIG_SMP */ -- -- scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", -- cpu_of(dst_rq), p->comm, p->pid); -- return DTL_INVALID; --} -- --/** -- * finish_dispatch - Asynchronously finish dispatching a task -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @p: task to finish dispatching -- * @qseq_at_dispatch: qseq when @p started getting dispatched -- * @dsq_id: destination DSQ ID -- * @enq_flags: %SCX_ENQ_* -- * -- * Dispatching to local DSQs may need to wait for queueing to complete or -- * require rq lock dancing. As we don't wanna do either while inside -- * ops.dispatch() to avoid locking order inversion, we split dispatching into -- * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the -- * task and its qseq. Once ops.dispatch() returns, this function is called to -- * finish up. -- * -- * There is no guarantee that @p is still valid for dispatching or even that it -- * was valid in the first place. Make sure that the task is still owned by the -- * BPF scheduler and claim the ownership before dispatching. -- */ --static void finish_dispatch(struct rq *rq, struct rq_flags *rf, -- struct task_struct *p, u64 qseq_at_dispatch, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- u64 opss; -- -- touch_core_sched_dispatch(rq, p); --retry: -- /* -- * No need for _acquire here. @p is accessed only after a successful -- * try_cmpxchg to DISPATCHING. -- */ -- opss = atomic64_read(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_DISPATCHING: -- case SCX_OPSS_NONE: -- /* someone else already got to it */ -- return; -- case SCX_OPSS_QUEUED: -- /* -- * If qseq doesn't match, @p has gone through at least one -- * dispatch/dequeue and re-enqueue cycle between -- * scx_bpf_dispatch() and here and we have no claim on it. -- */ -- if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) -- return; -- -- /* -- * While we know @p is accessible, we don't yet have a claim on -- * it - the BPF scheduler is allowed to dispatch tasks -- * spuriously and there can be a racing dequeue attempt. Let's -- * claim @p by atomically transitioning it from QUEUED to -- * DISPATCHING. -- */ -- if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_DISPATCHING))) -- break; -- goto retry; -- case SCX_OPSS_QUEUEING: -- /* -- * do_enqueue_task() is in the process of transferring the task -- * to the BPF scheduler while holding @p's rq lock. As we aren't -- * holding any kernel or BPF resource that the enqueue path may -- * depend upon, it's safe to wait. -- */ -- wait_ops_state(p, opss); -- goto retry; -- } -- -- BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) { -- case DTL_DISPATCHED: -- break; -- case DTL_LOST: -- break; -- case DTL_INVALID: -- dsq_id = SCX_DSQ_GLOBAL; -- fallthrough; -- case DTL_NOT_LOCAL: -- dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()), -- dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- break; -- } --} -- --static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- u32 u; -- -- for (u = 0; u < dspc->buf_cursor; u++) { -- struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u]; -- -- finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id, -- ent->enq_flags); -- } -- -- dspc->nr_tasks += dspc->buf_cursor; -- dspc->buf_cursor = 0; --} -- --static int balance_one(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf, bool local) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- bool prev_on_scx = prev->sched_class == &ext_sched_class; -- int nr_loops = SCX_DSP_MAX_LOOPS; -- -- lockdep_assert_rq_held(rq); -- -- if (static_branch_unlikely(&scx_ops_cpu_preempt) && -- unlikely(rq->scx->cpu_released)) { -- /* -- * If the previous sched_class for the current CPU was not SCX, -- * notify the BPF scheduler that it again has control of the -- * core. This callback complements ->cpu_release(), which is -- * emitted in scx_notify_pick_next_task(). -- */ -- if (SCX_HAS_OP(cpu_acquire)) -- SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq), -- NULL); -- rq->scx->cpu_released = false; -- } -- -- if (prev_on_scx) { -- WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP)); -- update_curr_scx(rq); -- -- /* -- * If @prev is runnable & has slice left, it has priority and -- * fetching more just increases latency for the fetched tasks. -- * Tell put_prev_task_scx() to put @prev on local_dsq. If the -- * BPF scheduler wants to handle this explicitly, it should -- * implement ->cpu_released(). -- * -- * See scx_ops_disable_workfn() for the explanation on the -- * disabling() test. -- * -- * When balancing a remote CPU for core-sched, there won't be a -- * following put_prev_task_scx() call and we don't own -- * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the -- * same conditions later and pick @rq->curr accordingly. -- */ -- if ((prev->scx->flags & SCX_TASK_QUEUED) && -- prev->scx->slice && !scx_ops_disabling()) { -- if (local) -- prev->scx->flags |= SCX_TASK_BAL_KEEP; -- return 1; -- } -- } -- -- /* if there already are tasks to run, nothing to do */ -- if (scx_rq->local_dsq.nr) -- return 1; -- -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- if (!SCX_HAS_OP(dispatch)) -- return 0; -- -- dspc->rq = rq; -- dspc->rf = rf; -- -- /* -- * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, -- * the local DSQ might still end up empty after a successful -- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() -- * produced some tasks, retry. The BPF scheduler may depend on this -- * looping behavior to simplify its implementation. -- */ -- do { -- dspc->nr_tasks = 0; -- -- SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), -- prev_on_scx ? prev : NULL); -- -- flush_dispatch_buf(rq, rf); -- -- if (scx_rq->local_dsq.nr) -- return 1; -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- /* -- * ops.dispatch() can trap us in this loop by repeatedly -- * dispatching ineligible tasks. Break out once in a while to -- * allow the watchdog to run. As IRQ can't be enabled in -- * balance(), we want to complete this scheduling cycle and then -- * start a new one. IOW, we want to call resched_curr() on the -- * next, most likely idle, task, not the current one. Use -- * scx_bpf_kick_cpu() for deferred kicking. -- */ -- if (unlikely(!--nr_loops)) { -- scx_bpf_kick_cpu(cpu_of(rq), 0); -- break; -- } -- } while (dspc->nr_tasks); -- -- return 0; --} -- --static int balance_scx(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf) --{ -- int ret; -- -- ret = balance_one(rq, prev, rf, true); --#ifdef CONFIG_SCHED_SMT -- /* -- * When core-sched is enabled, this ops.balance() call will be followed -- * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx() -- * on the SMT siblings. Balance the siblings too. -- */ -- if (sched_core_enabled(rq)) { -- const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); -- int scpu; -- -- for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { -- struct rq *srq = cpu_rq(scpu); -- struct rq_flags srf; -- struct task_struct *sprev = srq->curr; -- -- /* -- * While core-scheduling, rq lock is shared among -- * siblings but the debug annotations and rq clock -- * aren't. Do pinning dance to transfer the ownership. -- */ -- WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); -- rq_unpin_lock(rq, rf); -- rq_pin_lock(srq, &srf); -- -- update_rq_clock(srq); -- balance_one(srq, sprev, &srf, false); -- -- rq_unpin_lock(srq, &srf); -- rq_repin_lock(rq, rf); -- } -- } --#endif -- return ret; --} -- --static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) --{ -- if (p->scx->flags & SCX_TASK_QUEUED) { -- /* -- * Core-sched might decide to execute @p before it is -- * dispatched. Call ops_dequeue() to notify the BPF scheduler. -- */ -- ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); -- dispatch_dequeue(rq->scx, p); -- } -- -- p->se.exec_start = rq_clock_task(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, running, p); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock); -- -- watchdog_unwatch_task(p, true); -- -- /* -- * @p is getting newly scheduled or got kicked after someone updated its -- * slice. Refresh whether tick can be stopped. See can_stop_tick_scx(). -- */ -- if ((p->scx->slice == SCX_SLICE_INF) != -- (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) { -- if (p->scx->slice == SCX_SLICE_INF) -- rq->scx->flags |= SCX_RQ_CAN_STOP_TICK; -- else -- rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK; -- -- sched_update_tick_dependency(rq); -- } --} -- --static void put_prev_task_scx(struct rq *rq, struct task_struct *p) --{ --#ifndef CONFIG_SMP -- /* -- * UP workaround. -- * -- * Because SCX may transfer tasks across CPUs during dispatch, dispatch -- * is performed from its balance operation which isn't called in UP. -- * Let's work around by calling it from the operations which come right -- * after. -- * -- * 1. If the prev task is on SCX, pick_next_task() calls -- * .put_prev_task() right after. As .put_prev_task() is also called -- * from other places, we need to distinguish the calls which can be -- * done by looking at the previous task's state - if still queued or -- * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task(). -- * This case is handled here. -- * -- * 2. If the prev task is not on SCX, the first following call into SCX -- * will be .pick_next_task(), which is covered by calling -- * balance_scx() from pick_next_task_scx(). -- * -- * Note that we can't merge the first case into the second as -- * balance_scx() must be called before the previous SCX task goes -- * through put_prev_task_scx(). -- * -- * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf. -- * Pass in %NULL. -- */ -- if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP)) -- balance_scx(rq, p, NULL); --#endif -- -- update_curr_scx(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- -- /* -- * If we're being called from put_prev_task_balance(), balance_scx() may -- * have decided that @p should keep running. -- */ -- if (p->scx->flags & SCX_TASK_BAL_KEEP) { -- p->scx->flags &= ~SCX_TASK_BAL_KEEP; -- watchdog_watch_task(rq, p); -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- watchdog_watch_task(rq, p); -- -- /* -- * If @p has slice left and balance_scx() didn't tag it for -- * keeping, @p is getting preempted by a higher priority -- * scheduler class or core-sched forcing a different task. Leave -- * it at the head of the local DSQ. -- */ -- if (p->scx->slice && !scx_ops_disabling()) { -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- /* -- * If we're in the pick_next_task path, balance_scx() should -- * have already populated the local DSQ if there are any other -- * available tasks. If empty, tell ops.enqueue() that @p is the -- * only one available for this cpu. ops.enqueue() should put it -- * on the local DSQ so that the subsequent pick_next_task_scx() -- * can find the task unless it wants to trigger a separate -- * follow-up scheduling event. -- */ -- if (list_empty(&rq->scx->local_dsq.fifo)) -- do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1); -- else -- do_enqueue_task(rq, p, 0, -1); -- } --} -- --static struct task_struct *first_local_task(struct rq *rq) --{ -- struct rb_node *rb_node; -- struct sched_ext_entity *entity; -- -- if (!list_empty(&rq->scx->local_dsq.fifo)) { -- entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo); -- return entity->task; -- } -- -- rb_node = rb_first_cached(&rq->scx->local_dsq.priq); -- if (rb_node) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- return entity->task; -- } -- -- return NULL; --} -- --static struct task_struct *pick_next_task_scx(struct rq *rq) --{ -- struct task_struct *p; -- --#ifndef CONFIG_SMP -- /* UP workaround - see the comment at the head of put_prev_task_scx() */ -- if (unlikely(rq->curr->sched_class != &ext_sched_class)) -- balance_scx(rq, rq->curr, NULL); --#endif -- -- p = first_local_task(rq); -- if (!p) -- return NULL; -- -- if (unlikely(!p->scx->slice)) { -- if (!scx_ops_disabling() && !scx_warned_zero_slice) { -- printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n", -- p->comm, p->pid); -- scx_warned_zero_slice = true; -- } -- p->scx->slice = SCX_SLICE_DFL; -- } -- -- set_next_task_scx(rq, p, true); -- -- return p; --} -- --#ifdef CONFIG_SCHED_CORE --/** -- * scx_prio_less - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Core-sched is implemented as an additional scheduling layer on top of the -- * usual sched_class'es and needs to find out the expected task ordering. For -- * SCX, core-sched calls this function to interrogate the task ordering. -- * -- * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used -- * to implement the default task ordering. The older the timestamp, the higher -- * prority the task - the global FIFO ordering matching the default scheduling -- * behavior. -- * -- * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to -- * implement FIFO ordering within each local DSQ. See pick_task_scx(). -- */ --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi) --{ -- /* -- * The const qualifiers are dropped from task_struct pointers when -- * calling ops.core_sched_before(). Accesses are controlled by the -- * verifier. -- */ -- if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling()) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, -- (struct task_struct *)a, -- (struct task_struct *)b); -- else -- return time_after64(a->scx->core_sched_at, b->scx->core_sched_at); --} -- --/** -- * pick_task_scx - Pick a candidate task for core-sched -- * @rq: rq to pick the candidate task from -- * -- * Core-sched calls this function on each SMT sibling to determine the next -- * tasks to run on the SMT siblings. balance_one() has been called on all -- * siblings and put_prev_task_scx() has been called only for the current CPU. -- * -- * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look -- * at the first task in the local dsq. @rq->curr has to be considered explicitly -- * to mimic %SCX_TASK_BAL_KEEP. -- */ --static struct task_struct *pick_task_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- struct task_struct *first = first_local_task(rq); -- -- if (curr->scx->flags & SCX_TASK_QUEUED) { -- /* is curr the only runnable task? */ -- if (!first) -- return curr; -- -- /* -- * Does curr trump first? We can always go by core_sched_at for -- * this comparison as it represents global FIFO ordering when -- * the default core-sched ordering is used and local-DSQ FIFO -- * ordering otherwise. -- * -- * We can have a task with an earlier timestamp on the DSQ. For -- * example, when a current task is preempted by a sibling -- * picking a different cookie, the task would be requeued at the -- * head of the local DSQ with an earlier timestamp than the -- * core-sched picked next task. Besides, the BPF scheduler may -- * dispatch any tasks to the local DSQ anytime. -- */ -- if (curr->scx->slice && time_before64(curr->scx->core_sched_at, -- first->scx->core_sched_at)) -- return curr; -- } -- -- return first; /* this may be %NULL */ --} --#endif /* CONFIG_SCHED_CORE */ -- --static enum scx_cpu_preempt_reason --preempt_reason_from_class(const struct sched_class *class) --{ --#ifdef CONFIG_SMP -- if (class == &stop_sched_class) -- return SCX_CPU_PREEMPT_STOP; --#endif -- if (class == &dl_sched_class) -- return SCX_CPU_PREEMPT_DL; -- if (class == &rt_sched_class) -- return SCX_CPU_PREEMPT_RT; -- return SCX_CPU_PREEMPT_UNKNOWN; --} -- --void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task, -- const struct sched_class *active) --{ -- lockdep_assert_rq_held(rq); -- -- /* -- * The callback is conceptually meant to convey that the CPU is no -- * longer under the control of SCX. Therefore, don't invoke the -- * callback if the CPU is is staying on SCX, or going idle (in which -- * case the SCX scheduler has actively decided not to schedule any -- * tasks on the CPU). -- */ -- if (likely(active >= &ext_sched_class)) -- return; -- -- /* -- * At this point we know that SCX was preempted by a higher priority -- * sched_class, so invoke the ->cpu_release() callback if we have not -- * done so already. We only send the callback once between SCX being -- * preempted, and it regaining control of the CPU. -- * -- * ->cpu_release() complements ->cpu_acquire(), which is emitted the -- * next time that balance_scx() is invoked. -- */ -- if (!rq->scx->cpu_released) { -- if (SCX_HAS_OP(cpu_release)) { -- struct scx_cpu_release_args args = { -- .reason = preempt_reason_from_class(active), -- .task = task, -- }; -- -- SCX_CALL_OP(SCX_KF_CPU_RELEASE, -- cpu_release, cpu_of(rq), &args); -- } -- rq->scx->cpu_released = true; -- } --} -- --#ifdef CONFIG_SMP -- --static bool test_and_clear_cpu_idle(int cpu) --{ -- if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -- if (cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- return true; -- } else { -- return false; -- } --} -- --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- int cpu; -- -- do { -- cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); -- if (cpu < nr_cpu_ids) { -- const struct cpumask *sbm = topology_sibling_cpumask(cpu); -- -- /* -- * If offline, @cpu is not its own sibling and we can -- * get caught in an infinite loop as @cpu is never -- * cleared from idle_masks.smt. Clear @cpu directly in -- * such cases. -- */ -- if (likely(cpumask_test_cpu(cpu, sbm))) -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm); -- else -- cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu)); -- } else { -- cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -- if (cpu >= nr_cpu_ids) -- return -EBUSY; -- } -- } while (!test_and_clear_cpu_idle(cpu)); -- -- return cpu; --} -- --static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) --{ -- s32 cpu; -- -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return prev_cpu; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local DSQ of the waker. -- */ -- if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 && -- scx_has_idle_cpus && !(current->flags & PF_EXITING)) { -- cpu = smp_processor_id(); -- if (cpumask_test_cpu(cpu, p->cpus_ptr)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (test_and_clear_cpu_idle(prev_cpu)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return prev_cpu; -- } -- -- if (p->nr_cpus_allowed == 1) -- return prev_cpu; -- -- cpu = scx_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- -- return prev_cpu; --} -- --static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) --{ -- if (SCX_HAS_OP(select_cpu)) { -- s32 cpu; -- -- cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu, -- wake_flags); -- if (ops_cpu_valid(cpu)) { -- return cpu; -- } else { -- scx_ops_error("select_cpu returned invalid cpu %d", cpu); -- return prev_cpu; -- } -- } else { -- return scx_select_cpu_dfl(p, prev_cpu, wake_flags); -- } --} -- --static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx) --{ -- set_cpus_allowed_common(p, ctx); -- -- /* -- * The effective cpumask is stored in @p->cpus_ptr which may temporarily -- * differ from the configured one in @p->cpus_mask. Always tell the bpf -- * scheduler the effective one. -- * -- * Fine-grained memory write control is enforced by BPF making the const -- * designation pointless. Cast it away when calling the operation. -- */ -- if (SCX_HAS_OP(set_cpumask)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, -- (struct cpumask *)p->cpus_ptr); --} -- --static void reset_idle_masks(void) --{ -- /* consider all cpus idle, should converge to the actual state quickly */ -- cpumask_setall(idle_masks.cpu); -- cpumask_setall(idle_masks.smt); -- scx_has_idle_cpus = true; --} -- --void __scx_update_idle(struct rq *rq, bool idle) --{ -- int cpu = cpu_of(rq); -- struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -- -- if (SCX_HAS_OP(update_idle)) { -- SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); -- if (!static_branch_unlikely(&scx_builtin_idle_enabled)) -- return; -- } -- -- if (idle) { -- cpumask_set_cpu(cpu, idle_masks.cpu); -- if (!scx_has_idle_cpus) -- scx_has_idle_cpus = true; -- -- /* -- * idle_masks.smt handling is racy but that's fine as it's only -- * for optimization and self-correcting. -- */ -- for_each_cpu(cpu, sib_mask) { -- if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -- return; -- } -- cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -- } else { -- cpumask_clear_cpu(cpu, idle_masks.cpu); -- if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -- } --} -- --#else /* !CONFIG_SMP */ -- --static bool test_and_clear_cpu_idle(int cpu) { return false; } --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } --static void reset_idle_masks(void) {} -- --#endif /* CONFIG_SMP */ -- --static bool check_rq_for_timeouts(struct rq *rq) --{ -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rq_flags rf; -- bool timed_out = false; -- -- rq_lock_irqsave(rq, &rf); -- list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) { -- unsigned long last_runnable; -- -- p = entity->task; -- last_runnable = p->scx->runnable_at; -- -- if (unlikely(time_after(jiffies, -- last_runnable + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "%s[%d] failed to run for %u.%03us", -- p->comm, p->pid, -- dur_ms / 1000, dur_ms % 1000); -- timed_out = true; -- break; -- } -- } -- rq_unlock_irqrestore(rq, &rf); -- -- return timed_out; --} -- --static void scx_watchdog_workfn(struct work_struct *work) --{ -- int cpu; -- -- scx_watchdog_timestamp = jiffies; -- -- for_each_online_cpu(cpu) { -- if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) -- break; -- -- cond_resched(); -- } -- queue_delayed_work(system_unbound_wq, to_delayed_work(work), -- scx_watchdog_timeout / 2); --} -- --static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) --{ -- update_curr_scx(rq); -- //scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock); -- /* -- * While disabling, always resched and refresh core-sched timestamp as -- * we can't trust the slice management or ops.core_sched_before(). -- */ -- if (scx_ops_disabling()) { -- curr->scx->slice = 0; -- touch_core_sched(rq, curr); -- } -- -- if (!curr->scx->slice) -- resched_curr(rq); --} -- --#define SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- --static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg) --{ -- int ret; -- -- WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED); -- -- p->scx->disallow = false; -- -- if (SCX_HAS_OP(prep_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- }; -- -- ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args); -- if (unlikely(ret)) { -- ret = ops_sanitize_err("prep_enable", ret); -- return ret; -- } -- } -- //scx_sched_init_task(p); -- -- if (p->scx->disallow) { -- struct rq *rq; -- struct rq_flags rf; -- -- rq = task_rq_lock(p, &rf); -- -- /* -- * We're either in fork or load path and @p->policy will be -- * applied right after. Reverting @p->policy here and rejecting -- * %SCHED_EXT transitions from scx_check_setscheduler() -- * guarantees that if ops.prep_enable() sets @p->disallow, @p -- * can never be in SCX. -- */ -- if (p->policy == SCHED_EXT) { -- p->policy = SCHED_NORMAL; -- atomic64_inc(&scx_nr_rejected); -- } -- -- task_rq_unlock(rq, p, &rf); -- } -- -- p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET); -- return 0; --} -- --static void scx_ops_enable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED)); -- -- if (SCX_HAS_OP(enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- p->scx->flags |= SCX_TASK_OPS_ENABLED; --} -- --static void scx_ops_disable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- if (p->scx->flags & SCX_TASK_OPS_PREPPED) { -- if (SCX_HAS_OP(cancel_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- } else if (p->scx->flags & SCX_TASK_OPS_ENABLED) { -- if (SCX_HAS_OP(disable)) -- SCX_CALL_OP(SCX_KF_REST, disable, p); -- p->scx->flags &= ~SCX_TASK_OPS_ENABLED; -- } --} -- --static void set_task_scx_weight(struct task_struct *p) --{ -- u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -- -- p->scx->weight = sched_weight_to_cgroup(weight); --} -- --/** -- * refresh_scx_weight - Refresh a task's ext weight -- * @p: task to refresh ext weight for -- * -- * @p->scx->weight carries the task's static priority in cgroup weight scale to -- * enable easy access from the BPF scheduler. To keep it synchronized with the -- * current task priority, this function should be called when a new task is -- * created, priority is changed for a task on sched_ext, and a task is switched -- * to sched_ext from other classes. -- */ --static void refresh_scx_weight(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- set_task_scx_weight(p); -- if (SCX_HAS_OP(set_weight)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight); --} -- --void scx_pre_fork(struct task_struct *p) --{ -- p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL); -- if (!p->scx) { -- goto lock; -- } -- -- p->scx->dsq = NULL; -- INIT_LIST_HEAD(&p->scx->dsq_node.fifo); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- INIT_LIST_HEAD(&p->scx->watchdog_node); -- p->scx->flags = 0; -- p->scx->weight = 0; -- p->scx->sticky_cpu = -1; -- p->scx->holding_cpu = -1; -- p->scx->kf_mask = 0; -- atomic64_set(&p->scx->ops_state, 0); -- p->scx->runnable_at = INITIAL_JIFFIES; -- p->scx->slice = SCX_SLICE_DFL; -- p->scx->task = p; -- p->sched_prop = 0; -- -- /* -- * BPF scheduler enable/disable paths want to be able to iterate and -- * update all tasks which can become complex when racing forks. As -- * enable/disable are very cold paths, let's use a percpu_rwsem to -- * exclude forks. -- */ --lock: -- percpu_down_read(&scx_fork_rwsem); --} -- --int scx_fork(struct task_struct *p) --{ -- percpu_rwsem_assert_held(&scx_fork_rwsem); -- -- if (scx_enabled()) -- return scx_ops_prepare_task(p, task_group(p)); -- else -- return 0; --} -- --void scx_post_fork(struct task_struct *p) --{ -- if (scx_enabled()) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- /* -- * Set the weight manually before calling ops.enable() so that -- * the scheduler doesn't see a stale value if they inspect the -- * task struct. We'll invoke ops.set_weight() afterwards, as it -- * would be odd to receive a callback on the task before we -- * tell the scheduler that it's been fully enabled. -- */ -- set_task_scx_weight(p); -- scx_ops_enable_task(p); -- refresh_scx_weight(p); -- task_rq_unlock(rq, p, &rf); -- } -- -- spin_lock_irq(&scx_tasks_lock); -- list_add_tail(&p->scx->tasks_node, &scx_tasks); -- spin_unlock_irq(&scx_tasks_lock); -- -- percpu_up_read(&scx_fork_rwsem); --} -- --void scx_cancel_fork(struct task_struct *p) --{ -- if (scx_enabled()) -- scx_ops_disable_task(p); -- percpu_up_read(&scx_fork_rwsem); --} -- --void sched_ext_free(struct task_struct *p) --{ -- unsigned long flags; -- -- spin_lock_irqsave(&scx_tasks_lock, flags); -- list_del_init(&p->scx->tasks_node); -- spin_unlock_irqrestore(&scx_tasks_lock, flags); -- -- /* -- * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED -> -- * ENABLED transitions can't race us. Disable ops for @p. -- */ -- if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- scx_ops_disable_task(p); -- task_rq_unlock(rq, p, &rf); -- } --} -- --static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) --{ --} -- --static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} --static void switched_to_scx(struct rq *rq, struct task_struct *p) {} -- --int scx_check_setscheduler(struct task_struct *p, int policy) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- /* if disallow, reject transitioning into SCX */ -- if (scx_enabled() && READ_ONCE(p->scx->disallow) && -- p->policy != policy && policy == SCHED_EXT) -- return -EACCES; -- -- return 0; --} -- --#ifdef CONFIG_NO_HZ_FULL --bool scx_can_stop_tick(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (scx_ops_disabling()) -- return false; -- -- if (p->sched_class != &ext_sched_class) -- return true; -- -- /* -- * @rq can dispatch from different DSQs, so we can't tell whether it -- * needs the tick or not by looking at nr_running. Allow stopping ticks -- * iff the BPF scheduler indicated so. See set_next_task_scx(). -- */ -- return rq->scx->flags & SCX_RQ_CAN_STOP_TICK; --} --#endif -- --static inline void scx_cgroup_lock(void) {} --static inline void scx_cgroup_unlock(void) {} -- --/* -- * Omitted operations: -- * -- * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -- * task isn't tied to the CPU at that point. Preemption is implemented by -- * resetting the victim task's slice to 0 and triggering reschedule on the -- * target CPU. -- * -- * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -- * -- * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -- * their current sched_class. Call them directly from sched core instead. -- * -- * - task_woken, switched_from: Unnecessary. -- */ --DEFINE_SCHED_CLASS(ext) = { -- .enqueue_task = enqueue_task_scx, -- .dequeue_task = dequeue_task_scx, -- .yield_task = yield_task_scx, -- .yield_to_task = yield_to_task_scx, -- -- .check_preempt_curr = check_preempt_curr_scx, -- -- .pick_next_task = pick_next_task_scx, -- -- .put_prev_task = put_prev_task_scx, -- .set_next_task = set_next_task_scx, -- --#ifdef CONFIG_SMP -- .balance = balance_scx, -- .select_task_rq = select_task_rq_scx, -- .set_cpus_allowed = set_cpus_allowed_scx, --#endif -- --#ifdef CONFIG_SCHED_CORE -- .pick_task = pick_task_scx, --#endif -- -- .task_tick = task_tick_scx, -- -- .switched_to = switched_to_scx, -- .prio_changed = prio_changed_scx, -- -- .update_curr = update_curr_scx, -- --#ifdef CONFIG_UCLAMP_TASK -- .uclamp_enabled = 0, --#endif --}; -- --static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) --{ -- memset(dsq, 0, sizeof(*dsq)); -- -- raw_spin_lock_init(&dsq->lock); -- INIT_LIST_HEAD(&dsq->fifo); -- dsq->id = dsq_id; --} -- --static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id & SCX_DSQ_FLAG_BUILTIN) -- return ERR_PTR(-EINVAL); -- -- dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node); -- if (!dsq) -- return ERR_PTR(-ENOMEM); -- -- init_dsq(dsq, dsq_id); -- -- return dsq; --} -- --static void free_dsq_irq_workfn(struct irq_work *irq_work) --{ -- struct llist_node *to_free = llist_del_all(&dsqs_to_free); -- struct scx_dispatch_q *dsq, *tmp_dsq; -- -- llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) -- kfree_rcu(dsq, rcu); --} -- --static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); -- --static void destroy_dsq(u64 dsq_id) --{ --} -- --static void scx_cgroup_exit(void) {} --static int scx_cgroup_init(void) { return 0; } --static void scx_cgroup_config_knobs(void) {} -- --/* -- * Used by sched_fork() and __setscheduler_prio() to pick the matching -- * sched_class. dl/rt are already handled. -- */ --bool task_on_scx(struct task_struct *p) --{ -- if (!scx_enabled() || scx_ops_disabling()) -- return false; -- if (READ_ONCE(scx_switching_all)) -- return true; -- return p->policy == SCHED_EXT; --} -- --static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags) --{ -- if (enq_flags & SCX_ENQ_LAST) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); --} -- --static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {} -- --static void scx_ops_disable_workfn(struct kthread_work *work) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- struct scx_task_iter sti; -- struct task_struct *p; -- const char *reason; -- int i, cpu, type; -- -- type = atomic_read(&scx_exit_type); -- while (true) { -- /* -- * NONE indicates that a new scx_ops has been registered since -- * disable was scheduled - don't kill the new ops. DONE -- * indicates that the ops has already been disabled. -- */ -- if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE) -- return; -- if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE)) -- break; -- } -- -- cancel_delayed_work_sync(&scx_watchdog_work); -- -- switch (type) { -- case SCX_EXIT_UNREG: -- reason = "BPF scheduler unregistered"; -- break; -- case SCX_EXIT_SYSRQ: -- reason = "disabled by sysrq-S"; -- break; -- case SCX_EXIT_ERROR: -- reason = "runtime error"; -- break; -- case SCX_EXIT_ERROR_BPF: -- reason = "scx_bpf_error"; -- break; -- case SCX_EXIT_ERROR_STALL: -- reason = "runnable task stall"; -- break; -- default: -- reason = ""; -- } -- -- ei->type = type; -- strlcpy(ei->reason, reason, sizeof(ei->reason)); -- -- switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { -- case SCX_OPS_DISABLED: -- pr_warn("sched_ext: ops error detected without ops (%s)\n", -- scx_exit_info.msg); -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- return; -- case SCX_OPS_PREPPING: -- goto forward_progress_guaranteed; -- case SCX_OPS_DISABLING: -- /* shouldn't happen but handle it like ENABLING if it does */ -- WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); -- fallthrough; -- case SCX_OPS_ENABLING: -- case SCX_OPS_ENABLED: -- break; -- } -- -- /* -- * DISABLING is set and ops was either ENABLING or ENABLED indicating -- * that the ops and static branches are set. -- * -- * We must guarantee that all runnable tasks make forward progress -- * without trusting the BPF scheduler. We can't grab any mutexes or -- * rwsems as they might be held by tasks that the BPF scheduler is -- * forgetting to run, which unfortunately also excludes toggling the -- * static branches. -- * -- * Let's work around by overriding a couple ops and modifying behaviors -- * based on the DISABLING state and then cycling the tasks through -- * dequeue/enqueue to force global FIFO scheduling. -- * -- * a. ops.enqueue() and .dispatch() are overridden for simple global -- * FIFO scheduling. -- * -- * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value -- * can't be trusted. Whenever a tick triggers, the running task is -- * rotated to the tail of the queue with core_sched_at touched. -- * -- * c. pick_next_task() suppresses zero slice warning. -- * -- * d. scx_prio_less() reverts to the default core_sched_at order. -- */ -- scx_ops.enqueue = scx_ops_fallback_enqueue; -- scx_ops.dispatch = scx_ops_fallback_dispatch; -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- SCHED_CHANGE_BLOCK(task_rq(p), p, -- DEQUEUE_SAVE | DEQUEUE_MOVE) { -- /* cycling deq/enq is enough, see above */ -- } -- } -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* kick all CPUs to restore ticks */ -- for_each_possible_cpu(cpu) -- resched_cpu(cpu); -- --forward_progress_guaranteed: -- /* -- * Here, every runnable task is guaranteed to make forward progress and -- * we can safely use blocking synchronization constructs. Actually -- * disable ops. -- */ -- mutex_lock(&scx_ops_enable_mutex); -- -- static_branch_disable(&__scx_switched_all); -- WRITE_ONCE(scx_switching_all, false); -- -- /* avoid racing against fork and cgroup changes */ -- cpus_read_lock(); -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- bool alive = READ_ONCE(p->__state) != TASK_DEAD; -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL); -- -- __setscheduler_prio(p, p->prio); -- } -- -- if (alive) -- check_class_changed(task_rq(p), p, old_class, p->prio); -- -- scx_ops_disable_task(p); -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* no task is on scx, turn off all the switches and flush in-progress calls */ -- static_branch_disable_cpuslocked(&__scx_ops_enabled); -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- static_branch_disable_cpuslocked(&scx_has_op[i]); -- static_branch_disable_cpuslocked(&scx_ops_enq_last); -- static_branch_disable_cpuslocked(&scx_ops_enq_exiting); -- static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- synchronize_rcu(); -- -- scx_cgroup_exit(); -- -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- cpus_read_unlock(); -- -- if (ei->type >= SCX_EXIT_ERROR) { -- printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name); -- -- if (ei->msg[0] == '\0') -- printk(KERN_ERR "sched_ext: %s\n", ei->reason); -- else -- printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg); -- -- stack_trace_print(ei->bt, ei->bt_len, 2); -- } -- -- if (scx_ops.exit) -- SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); -- -- memset(&scx_ops, 0, sizeof(scx_ops)); -- -- free_percpu(scx_dsp_buf); -- scx_dsp_buf = NULL; -- scx_dsp_max_batch = 0; -- -- mutex_unlock(&scx_ops_enable_mutex); -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- -- scx_cgroup_config_knobs(); --} -- --static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); -- --static void schedule_scx_ops_disable_work(void) --{ -- struct kthread_worker *helper = READ_ONCE(scx_ops_helper); -- -- /* -- * We may be called spuriously before the first bpf_sched_ext_reg(). If -- * scx_ops_helper isn't set up yet, there's nothing to do. -- */ -- if (helper) -- kthread_queue_work(helper, &scx_ops_disable_work); --} -- --static void scx_ops_disable(enum scx_exit_type type) --{ -- int none = SCX_EXIT_NONE; -- -- if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)) -- type = SCX_EXIT_ERROR; -- -- atomic_try_cmpxchg(&scx_exit_type, &none, type); -- -- schedule_scx_ops_disable_work(); --} -- --static void scx_ops_error_irq_workfn(struct irq_work *irq_work) --{ -- schedule_scx_ops_disable_work(); --} -- --static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- int none = SCX_EXIT_NONE; -- va_list args; -- -- if (!atomic_try_cmpxchg(&scx_exit_type, &none, type)) -- return; -- -- ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1); -- -- va_start(args, fmt); -- vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args); -- va_end(args); -- -- irq_work_queue(&scx_ops_error_irq_work); --} -- --static struct kthread_worker *scx_create_rt_helper(const char *name) --{ -- struct kthread_worker *helper; -- -- helper = kthread_create_worker(0, name); -- if (helper) -- sched_set_fifo(helper->task); -- return helper; --} -- --static int scx_ops_enable(struct sched_ext_ops *ops) --{ -- struct scx_task_iter sti; -- struct task_struct *p; -- int i, ret; -- int tcnt = 0; -- unsigned long long start = 0; -- unsigned long long total_start = sched_clock(); -- -- mutex_lock(&scx_ops_enable_mutex); -- -- if (!scx_ops_helper) { -- WRITE_ONCE(scx_ops_helper, -- scx_create_rt_helper("sched_ext_ops_helper")); -- if (!scx_ops_helper) { -- ret = -ENOMEM; -- goto err_unlock; -- } -- } -- -- if (scx_ops_enable_state() != SCX_OPS_DISABLED) { -- ret = -EBUSY; -- goto err_unlock; -- } -- -- //slim_walt_enable(true); -- -- /* -- * Set scx_ops, transition to PREPPING and clear exit info to arm the -- * disable path. Failure triggers full disabling from here on. -- */ -- scx_ops = *ops; -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != -- SCX_OPS_DISABLED); -- -- memset(&scx_exit_info, 0, sizeof(scx_exit_info)); -- atomic_set(&scx_exit_type, SCX_EXIT_NONE); -- scx_warned_zero_slice = false; -- -- atomic64_set(&scx_nr_rejected, 0); -- -- /* -- * Keep CPUs stable during enable so that the BPF scheduler can track -- * online CPUs by watching ->on/offline_cpu() after ->init(). -- */ -- cpus_read_lock(); -- -- scx_switch_all_req = true; -- if (scx_ops.init) { -- ret = SCX_CALL_OP_RET(SCX_KF_INIT, init); -- if (ret) { -- ret = ops_sanitize_err("init", ret); -- goto err_disable; -- } -- -- /* -- * Exit early if ops.init() triggered scx_bpf_error(). Not -- * strictly necessary as we'll fail transitioning into ENABLING -- * later but that'd be after calling ops.prep_enable() on all -- * tasks and with -EBUSY which isn't very intuitive. Let's exit -- * early with success so that the condition is notified through -- * ops.exit() like other scx_bpf_error() invocations. -- */ -- if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE) -- goto err_disable; -- } -- -- WARN_ON_ONCE(scx_dsp_buf); -- scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; -- scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch, -- __alignof__(scx_dsp_buf[0])); -- if (!scx_dsp_buf) { -- ret = -ENOMEM; -- goto err_disable; -- } -- -- scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; -- if (ops->timeout_ms) -- scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); -- -- scx_watchdog_timestamp = jiffies; -- queue_delayed_work(system_unbound_wq, &scx_watchdog_work, -- scx_watchdog_timeout / 2); -- -- /* -- * Lock out forks, cgroup on/offlining and moves before opening the -- * floodgate so that they don't wander into the operations prematurely. -- */ -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- if (((void (**)(void))ops)[i]) -- static_branch_enable_cpuslocked(&scx_has_op[i]); -- -- if (ops->flags & SCX_OPS_ENQ_LAST) -- static_branch_enable_cpuslocked(&scx_ops_enq_last); -- -- if (ops->flags & SCX_OPS_ENQ_EXITING) -- static_branch_enable_cpuslocked(&scx_ops_enq_exiting); -- if (scx_ops.cpu_acquire || scx_ops.cpu_release) -- static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); -- -- if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { -- reset_idle_masks(); -- static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); -- } else { -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- } -- -- /* -- * All cgroups should be initialized before letting in tasks. cgroup -- * on/offlining and task migrations are already locked out. -- */ -- ret = scx_cgroup_init(); -- if (ret) -- goto err_disable_unlock; -- -- static_branch_enable_cpuslocked(&__scx_ops_enabled); -- -- /* -- * Enable ops for every task. Fork is excluded by scx_fork_rwsem -- * preventing new tasks from being added. No need to exclude tasks -- * leaving as sched_ext_free() can handle both prepped and enabled -- * tasks. Prep all tasks first and then enable them with preemption -- * disabled. -- */ -- spin_lock_irq(&scx_tasks_lock); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered(&sti))) { -- get_task_struct(p); -- spin_unlock_irq(&scx_tasks_lock); -- -- ret = scx_ops_prepare_task(p, task_group(p)); -- if (ret) { -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n", -- ret, p->comm, p->pid); -- goto err_disable_unlock; -- } -- -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- } -- scx_task_iter_exit(&sti); -- -- /* -- * All tasks are prepped but are still ops-disabled. Ensure that -- * %current can't be scheduled out and switch everyone. -- * preempt_disable() is necessary because we can't guarantee that -- * %current won't be starved if scheduled out while switching. -- */ -- preempt_disable(); -- -- /* -- * From here on, the disable path must assume that tasks have ops -- * enabled and need to be recovered. -- */ -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { -- preempt_enable(); -- spin_unlock_irq(&scx_tasks_lock); -- ret = -EBUSY; -- goto err_disable_unlock; -- } -- -- /* -- * We're fully committed and can't fail. The PREPPED -> ENABLED -- * transitions here are synchronized against sched_ext_free() through -- * scx_tasks_lock. -- */ -- WRITE_ONCE(scx_switching_all, scx_switch_all_req); -- start = sched_clock(); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- tcnt++; -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- scx_ops_enable_task(p); -- __setscheduler_prio(p, p->prio); -- } -- -- check_class_changed(task_rq(p), p, old_class, p->prio); -- } else { -- scx_ops_disable_task(p); -- } -- } -- printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id()); -- scx_task_iter_exit(&sti); -- -- spin_unlock_irq(&scx_tasks_lock); -- preempt_enable(); -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { -- ret = -EBUSY; -- goto err_disable; -- } -- -- if (scx_switch_all_req) -- static_branch_enable_cpuslocked(&__scx_switched_all); -- -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- -- scx_cgroup_config_knobs(); -- printk("\n total duration = %llu , current cpu = %d\n", sched_clock() - total_start, smp_processor_id()); -- -- return 0; -- --err_unlock: -- mutex_unlock(&scx_ops_enable_mutex); -- return ret; -- --err_disable_unlock: -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); --err_disable: -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- /* must be fully disabled before returning */ -- scx_ops_disable(SCX_EXIT_ERROR); -- kthread_flush_work(&scx_ops_disable_work); -- return ret; --} -- --#ifdef CONFIG_SCHED_DEBUG --static const char *scx_ops_enable_state_str[] = { -- [SCX_OPS_PREPPING] = "prepping", -- [SCX_OPS_ENABLING] = "enabling", -- [SCX_OPS_ENABLED] = "enabled", -- [SCX_OPS_DISABLING] = "disabling", -- [SCX_OPS_DISABLED] = "disabled", --}; -- --static int scx_debug_show(struct seq_file *m, void *v) --{ -- mutex_lock(&scx_ops_enable_mutex); -- seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name); -- seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled()); -- seq_printf(m, "%-30s: %d\n", "switching_all", -- READ_ONCE(scx_switching_all)); -- seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all()); -- seq_printf(m, "%-30s: %s\n", "enable_state", -- scx_ops_enable_state_str[scx_ops_enable_state()]); -- seq_printf(m, "%-30s: %llu\n", "nr_rejected", -- atomic64_read(&scx_nr_rejected)); -- mutex_unlock(&scx_ops_enable_mutex); -- return 0; --} -- --static int scx_debug_open(struct inode *inode, struct file *file) --{ -- return single_open(file, scx_debug_show, NULL); --} -- --const struct file_operations sched_ext_fops = { -- .open = scx_debug_open, -- .read = seq_read, -- .llseek = seq_lseek, -- .release = single_release, --}; --#endif -- --/******************************************************************************** -- * bpf_struct_ops plumbing. -- */ --#include --#include --#include -- --extern struct btf *btf_vmlinux; --static const struct btf_type *task_struct_type; -- --static bool bpf_scx_is_valid_access(int off, int size, -- enum bpf_access_type type, -- const struct bpf_prog *prog, -- struct bpf_insn_access_aux *info) --{ -- if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) -- return false; -- if (type != BPF_READ) -- return false; -- if (off % size != 0) -- return false; -- -- return btf_ctx_access(off, size, type, prog, info); --} -- --static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, -- const struct bpf_reg_state *reg, -- int off, int size) --{ -- return 0; --} -- --static const struct bpf_func_proto * --bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) --{ -- switch (func_id) { -- case BPF_FUNC_task_storage_get: -- return &bpf_task_storage_get_proto; -- case BPF_FUNC_task_storage_delete: -- return &bpf_task_storage_delete_proto; -- default: -- return bpf_base_func_proto(func_id); -- } --} -- --const struct bpf_verifier_ops bpf_scx_verifier_ops = { -- .get_func_proto = bpf_scx_get_func_proto, -- .is_valid_access = bpf_scx_is_valid_access, -- .btf_struct_access = bpf_scx_btf_struct_access, --}; -- --static int bpf_scx_init_member(const struct btf_type *t, -- const struct btf_member *member, -- void *kdata, const void *udata) --{ -- const struct sched_ext_ops *uops = udata; -- struct sched_ext_ops *ops = kdata; -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- int ret; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, dispatch_max_batch): -- if (*(u32 *)(udata + moff) > INT_MAX) -- return -E2BIG; -- ops->dispatch_max_batch = *(u32 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, flags): -- if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) -- return -EINVAL; -- ops->flags = *(u64 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, name): -- ret = bpf_obj_name_cpy(ops->name, uops->name, -- sizeof(ops->name)); -- if (ret < 0) -- return ret; -- if (ret == 0) -- return -EINVAL; -- return 1; -- case offsetof(struct sched_ext_ops, timeout_ms): -- if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT) -- return -E2BIG; -- ops->timeout_ms = *(u32 *)(udata + moff); -- return 1; -- } -- -- return 0; --} -- --static int bpf_scx_check_member(const struct btf_type *t, -- const struct btf_member *member, -- const struct bpf_prog *prog) --{ -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, prep_enable): -- case offsetof(struct sched_ext_ops, init): -- case offsetof(struct sched_ext_ops, exit): -- break; -- default: -- /* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */ -- /* if (prog->aux->sleepable) -- return -EINVAL; */ -- break; -- } -- -- return 0; --} -- --static int bpf_scx_reg(void *kdata) --{ -- return scx_ops_enable(kdata); --} -- --static void bpf_scx_unreg(void *kdata) --{ -- scx_ops_disable(SCX_EXIT_UNREG); -- kthread_flush_work(&scx_ops_disable_work); --} -- --static int bpf_scx_init(struct btf *btf) --{ -- u32 type_id; -- -- type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); -- if (type_id < 0) -- return -EINVAL; -- task_struct_type = btf_type_by_id(btf, type_id); -- -- return 0; --} -- --/* "extern" to avoid sparse warning, only used in this file */ --extern struct bpf_struct_ops bpf_sched_ext_ops; -- --struct bpf_struct_ops bpf_sched_ext_ops = { -- .verifier_ops = &bpf_scx_verifier_ops, -- .reg = bpf_scx_reg, -- .unreg = bpf_scx_unreg, -- .check_member = bpf_scx_check_member, -- .init_member = bpf_scx_init_member, -- .init = bpf_scx_init, -- .name = "sched_ext_ops", --}; -- --static void sysrq_handle_sched_ext_reset(u8 key) --{ -- if (scx_ops_helper) -- scx_ops_disable(SCX_EXIT_SYSRQ); -- else -- pr_info("sched_ext: BPF scheduler not yet used\n"); --} -- --static const struct sysrq_key_op sysrq_sched_ext_reset_op = { -- .handler = sysrq_handle_sched_ext_reset, -- .help_msg = "reset-sched-ext(S)", -- .action_msg = "Disable sched_ext and revert all tasks to CFS", -- .enable_mask = SYSRQ_ENABLE_RTNICE, --}; -- --static void kick_cpus_irq_workfn(struct irq_work *irq_work) --{ -- struct rq *this_rq = this_rq(); -- u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); -- int this_cpu = cpu_of(this_rq); -- int cpu; -- -- for_each_cpu(cpu, this_rq->scx->cpus_to_kick) { -- struct rq *rq = cpu_rq(cpu); -- unsigned long flags; -- -- raw_spin_rq_lock_irqsave(rq, flags); -- -- if (cpu_online(cpu) || cpu == this_cpu) { -- if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) && -- rq->curr->sched_class == &ext_sched_class) -- rq->curr->scx->slice = 0; -- pseqs[cpu] = rq->scx->pnt_seq; -- resched_curr(rq); -- } else { -- cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait); -- } -- -- raw_spin_rq_unlock_irqrestore(rq, flags); -- } -- -- for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait, -- cpumask_of(this_cpu)) { -- /* -- * Pairs with smp_store_release() issued by this CPU in -- * scx_notify_pick_next_task() on the resched path. -- * -- * We busy-wait here to guarantee that no other task can be -- * scheduled on our core before the target CPU has entered the -- * resched path. -- */ -- while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu]) -- cpu_relax(); -- } -- -- cpumask_clear(this_rq->scx->cpus_to_kick); -- cpumask_clear(this_rq->scx->cpus_to_preempt); -- cpumask_clear(this_rq->scx->cpus_to_wait); --} -- --void __init init_sched_ext_class(void) --{ -- int cpu; -- u32 v; -- -- /* -- * The following is to prevent the compiler from optimizing out the enum -- * definitions so that BPF scheduler implementations can use them -- * through the generated vmlinux.h. -- */ -- WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | -- SCX_TG_ONLINE | SCX_KICK_PREEMPT); -- -- init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); --#ifdef CONFIG_SMP -- BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -- BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); --#endif -- scx_kick_cpus_pnt_seqs = -- __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * -- num_possible_cpus(), -- __alignof__(scx_kick_cpus_pnt_seqs[0])); -- BUG_ON(!scx_kick_cpus_pnt_seqs); -- -- for_each_possible_cpu(cpu) { -- struct rq *rq = cpu_rq(cpu); -- -- rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL); -- if (rq->scx) -- rq->scx->rq = rq; -- else -- pr_err("fatal error : alloc rq->scx failed!!!\n"); -- -- init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL); -- INIT_LIST_HEAD(&rq->scx->watchdog_list); -- -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL)); -- init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn); -- } -- -- register_sysrq_key('S', &sysrq_sched_ext_reset_op); -- INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); -- scx_cgroup_config_knobs(); --} -- -- --/******************************************************************************** -- * Helpers that can be called from the BPF scheduler. -- */ --#include -- --/* Disables missing prototype warnings for kfuncs */ --__diag_push(); --__diag_ignore_all("-Wmissing-prototypes", -- "Global functions as their definitions will be in vmlinux BTF"); -- --/** -- * scx_bpf_switch_all - Switch all tasks into SCX -- * @into_scx: switch direction -- * -- * If @into_scx is %true, all existing and future non-dl/rt tasks are switched -- * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on -- * SCX. The actual switching is asynchronous. Can be called from ops.init(). -- */ --void scx_bpf_switch_all(void) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return; -- -- scx_switch_all_req = true; --} -- --BTF_SET8_START(scx_kfunc_ids_init) --BTF_ID_FLAGS(func, scx_bpf_switch_all) --BTF_SET8_END(scx_kfunc_ids_init) -- --static const struct btf_kfunc_id_set scx_kfunc_set_init = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_init, --}; -- --/** -- * scx_bpf_create_dsq - Create a custom DSQ -- * @dsq_id: DSQ to create -- * @node: NUMA node to allocate from -- * -- * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(), -- * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move(). -- */ --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return -EINVAL; -- -- if (unlikely(node >= (int)nr_node_ids || -- (node < 0 && node != NUMA_NO_NODE))) -- return -EINVAL; -- return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); --} -- --BTF_SET8_START(scx_kfunc_ids_sleepable) --BTF_ID_FLAGS(func, scx_bpf_create_dsq) --BTF_SET8_END(scx_kfunc_ids_sleepable) -- --static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_sleepable, --}; -- --static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) --{ -- if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) -- return false; -- -- lockdep_assert_irqs_disabled(); -- -- if (unlikely(!p)) { -- scx_ops_error("called with NULL task"); -- return false; -- } -- -- if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { -- scx_ops_error("invalid enq_flags 0x%llx", enq_flags); -- return false; -- } -- -- return true; --} -- --static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) --{ -- struct task_struct *ddsp_task; -- int idx; -- -- ddsp_task = __this_cpu_read(direct_dispatch_task); -- if (ddsp_task) { -- direct_dispatch(ddsp_task, p, dsq_id, enq_flags); -- return; -- } -- -- idx = __this_cpu_read(scx_dsp_ctx.buf_cursor); -- if (unlikely(idx >= scx_dsp_max_batch)) { -- scx_ops_error("dispatch buffer overflow"); -- return; -- } -- -- this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){ -- .task = p, -- .qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK, -- .dsq_id = dsq_id, -- .enq_flags = enq_flags, -- }; -- __this_cpu_inc(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe -- * to call this function spuriously. Can be called from ops.enqueue() and -- * ops.dispatch(). -- * -- * When called from ops.enqueue(), it's for direct dispatch and @p must match -- * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the -- * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be -- * on the target CPU in the first place. -- * -- * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id -- * and this function can be called upto ops.dispatch_max_batch times to dispatch -- * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the -- * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. -- * -- * This function doesn't have any locking restrictions and may be called under -- * BPF locks (in the future when BPF introduces more flexible locking). -- * -- * @p is allowed to run for @slice. The scheduling path is triggered on slice -- * exhaustion. If zero, the current residual slice is maintained. If -- * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with -- * scx_bpf_kick_cpu() to trigger scheduling. -- */ --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- scx_dispatch_commit(p, dsq_id, enq_flags); --} -- --/** -- * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. -- * Tasks queued into the priority queue are ordered by @vtime and always -- * consumed after the tasks in the FIFO queue. All other aspects are identical -- * to scx_bpf_dispatch(). -- * -- * @vtime ordering is according to time_before64() which considers wrapping. A -- * numerically larger vtime may indicate an earlier position in the ordering and -- * vice-versa. -- */ --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 vtime, u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- p->scx->dsq_vtime = vtime; -- -- scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); --} -- --BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime) --BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_enqueue_dispatch, --}; -- --/** -- * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots -- * -- * Can only be called from ops.dispatch(). -- */ --u32 scx_bpf_dispatch_nr_slots(void) --{ -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return 0; -- -- return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ -- * @dsq_id: DSQ to consume -- * -- * Consume a task from the non-local DSQ identified by @dsq_id and transfer it -- * to the current CPU's local DSQ for execution. Can only be called from -- * ops.dispatch(). -- * -- * This function flushes the in-flight dispatches from scx_bpf_dispatch() before -- * trying to consume the specified DSQ. It may also grab rq locks and thus can't -- * be called under any BPF locks. -- * -- * Returns %true if a task has been consumed, %false if there isn't any task to -- * consume. -- */ --bool scx_bpf_consume(u64 dsq_id) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- struct scx_dispatch_q *dsq; -- -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return false; -- -- flush_dispatch_buf(dspc->rq, dspc->rf); -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); -- return false; -- } -- -- if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) { -- /* -- * A successfully consumed task can be dequeued before it starts -- * running while the CPU is trying to migrate other dispatched -- * tasks. Bump nr_tasks to tell balance_scx() to retry on empty -- * local DSQ. -- */ -- dspc->nr_tasks++; -- return true; -- } else { -- return false; -- } --} -- --BTF_SET8_START(scx_kfunc_ids_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) --BTF_ID_FLAGS(func, scx_bpf_consume) --BTF_SET8_END(scx_kfunc_ids_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_dispatch, --}; -- --/** -- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ -- * -- * Iterate over all of the tasks currently enqueued on the local DSQ of the -- * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of -- * processed tasks. Can only be called from ops.cpu_release(). -- */ --u32 scx_bpf_reenqueue_local(void) --{ -- u32 nr_enqueued, i; -- struct rq *rq; -- struct scx_rq *scx_rq; -- -- if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) -- return 0; -- -- rq = cpu_rq(smp_processor_id()); -- lockdep_assert_rq_held(rq); -- scx_rq = rq->scx; -- -- /* -- * Get the number of tasks on the local DSQ before iterating over it to -- * pull off tasks. The enqueue callback below can signal that it wants -- * the task to stay on the local DSQ, and we want to prevent the BPF -- * scheduler from causing us to loop indefinitely. -- */ -- nr_enqueued = scx_rq->local_dsq.nr; -- for (i = 0; i < nr_enqueued; i++) { -- struct task_struct *p; -- -- p = first_local_task(rq); -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- WARN_ON_ONCE(p->scx->holding_cpu != -1); -- dispatch_dequeue(scx_rq, p); -- do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); -- } -- -- return nr_enqueued; --} -- --BTF_SET8_START(scx_kfunc_ids_cpu_release) --BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) --BTF_SET8_END(scx_kfunc_ids_cpu_release) -- --static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_cpu_release, --}; -- --/** -- * scx_bpf_kick_cpu - Trigger reschedule on a CPU -- * @cpu: cpu to kick -- * @flags: SCX_KICK_* flags -- * -- * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or -- * trigger rescheduling on a busy CPU. This can be called from any online -- * scx_ops operation and the actual kicking is performed asynchronously through -- * an irq work. -- */ --void scx_bpf_kick_cpu(s32 cpu, u64 flags) --{ -- struct rq *rq; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d", cpu); -- return; -- } -- -- preempt_disable(); -- rq = this_rq(); -- -- /* -- * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting -- * rq locks. We can probably be smarter and avoid bouncing if called -- * from ops which don't hold a rq lock. -- */ -- cpumask_set_cpu(cpu, rq->scx->cpus_to_kick); -- if (flags & SCX_KICK_PREEMPT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt); -- if (flags & SCX_KICK_WAIT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_wait); -- -- irq_work_queue(&rq->scx->kick_cpus_irq_work); -- preempt_enable(); --} -- --/** -- * scx_bpf_dsq_nr_queued - Return the number of queued tasks -- * @dsq_id: id of the DSQ -- * -- * Return the number of tasks in the DSQ matching @dsq_id. If not found, -- * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops -- * operations. -- */ --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) --{ -- struct scx_dispatch_q *dsq; -- -- lockdep_assert(rcu_read_lock_any_held()); -- -- if (dsq_id == SCX_DSQ_LOCAL) { -- return this_rq()->scx->local_dsq.nr; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (ops_cpu_valid(cpu)) -- return cpu_rq(cpu)->scx->local_dsq.nr; -- } else { -- dsq = find_non_local_dsq(dsq_id); -- if (dsq) -- return dsq->nr; -- } -- return -ENOENT; --} -- --/** -- * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state -- * @cpu: cpu to test and clear idle for -- * -- * Returns %true if @cpu was idle and its idle state was successfully cleared. -- * %false otherwise. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return false; -- } -- -- if (ops_cpu_valid(cpu)) -- return test_and_clear_cpu_idle(cpu); -- else -- return false; --} -- --/** -- * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu -- * @cpus_allowed: Allowed cpumask -- * -- * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked -- * idle cpu number on success. -%EBUSY if no matching cpu was found. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return -EBUSY; -- } -- -- return scx_pick_idle_cpu(cpus_allowed); --} -- --/** -- * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking -- * per-CPU cpumask. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_cpumask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.cpu; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, -- * per-physical-core cpumask. Can be used to determine if an entire physical -- * core is free. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_smtmask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.smt; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to -- * either the percpu, or SMT idle-tracking cpumask. -- */ --void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) --{ -- /* -- * Empty function body because we aren't actually acquiring or -- * releasing a reference to a global idle cpumask, which is read-only -- * in the caller and is never released. The acquire / release semantics -- * here are just used to make the cpumask is a trusted pointer in the -- * caller. -- */ --} -- --struct scx_bpf_error_bstr_bufs { -- u64 data[MAX_BPRINTF_VARARGS]; -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs); -- --/** -- * scx_bpf_error_bstr - Indicate fatal error -- * @fmt: error message format string -- * @data: format string parameters packaged using ___bpf_fill() macro -- * @data__sz: @data len, must end in '__sz' for the verifier -- * -- * Indicate that the BPF scheduler encountered a fatal error and initiate ops -- * disabling. -- */ --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) --{ -- struct scx_bpf_error_bstr_bufs *bufs; -- unsigned long flags; -- struct bpf_bprintf_data args = {}; -- int ret; -- -- local_irq_save(flags); -- bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs); -- -- if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || -- (data__sz && !data)) { -- scx_ops_error("invalid data=%p and data__sz=%u", -- (void *)data, data__sz); -- goto out_restore; -- } -- -- ret = copy_from_kernel_nofault(bufs->data, data, data__sz); -- if (ret) { -- scx_ops_error("failed to read data fields (%d)", ret); -- goto out_restore; -- } -- -- ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args); -- if (ret < 0) { -- scx_ops_error("failed to format prepration (%d)", ret); -- goto out_restore; -- } -- -- ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt, -- args.bin_args); -- bpf_bprintf_cleanup(&args); -- if (ret < 0) { -- scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format", -- fmt, data, data__sz); -- goto out_restore; -- } -- -- scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg); --out_restore: -- local_irq_restore(flags); --} -- --/** -- * scx_bpf_destroy_dsq - Destroy a custom DSQ -- * @dsq_id: DSQ to destroy -- * -- * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with -- * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is -- * empty and no further tasks are dispatched to it. Ignored if called on a DSQ -- * which doesn't exist. Can be called from any online scx_ops operations. -- */ --void scx_bpf_destroy_dsq(u64 dsq_id) --{ -- destroy_dsq(dsq_id); --} -- --/** -- * scx_bpf_task_running - Is task currently running? -- * @p: task of interest -- */ --bool scx_bpf_task_running(const struct task_struct *p) --{ -- return task_rq(p)->curr == p; --} -- --/** -- * scx_bpf_task_cpu - CPU a task is currently associated with -- * @p: task of interest -- */ --s32 scx_bpf_task_cpu(const struct task_struct *p) --{ -- return task_cpu(p); --} -- --/** -- * scx_bpf_task_cgroup - Return the sched cgroup of a task -- * @p: task of interest -- * -- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with -- * from the scheduler's POV. SCX operations should use this function to -- * determine @p's current cgroup as, unlike following @p->cgroups, -- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all -- * rq-locked operations. Can be called on the parameter tasks of rq-locked -- * operations. The restriction guarantees that @p's rq is locked by the caller. -- */ --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) --{ -- struct task_group *tg = p->sched_task_group; -- struct cgroup *cgrp = &cgrp_dfl_root.cgrp; -- -- if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) -- goto out; -- -- /* -- * A task_group may either be a cgroup or an autogroup. In the latter -- * case, @tg->css.cgroup is %NULL. A task_group can't become the other -- * kind once created. -- */ -- if (tg && tg->css.cgroup) -- cgrp = tg->css.cgroup; -- else -- cgrp = &cgrp_dfl_root.cgrp; --out: -- cgroup_get(cgrp); -- return cgrp; --} -- --BTF_SET8_START(scx_kfunc_ids_any) --BTF_ID_FLAGS(func, scx_bpf_kick_cpu) --BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) --BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) --BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu) --BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) --BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) --BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) --BTF_ID_FLAGS(func, scx_bpf_task_running) --BTF_ID_FLAGS(func, scx_bpf_task_cpu) --BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE) --BTF_SET8_END(scx_kfunc_ids_any) -- --static const struct btf_kfunc_id_set scx_kfunc_set_any = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_any, --}; -- --__diag_pop(); -- --/* -- * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set() -- * needs most of the system to be up. -- */ --static int __init register_ext_kfuncs(void) --{ -- int ret; -- -- /* -- * Some kfuncs are context-sensitive and can only be called from -- * specific SCX ops. They are grouped into BTF sets accordingly. -- * Unfortunately, BPF currently doesn't have a way of enforcing such -- * restrictions. Eventually, the verifier should be able to enforce -- * them. For now, register them the same and make each kfunc explicitly -- * check using scx_kf_allowed(). -- */ -- if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_init)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_sleepable)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_enqueue_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_cpu_release)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_any))) { -- pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret); -- return ret; -- } -- -- return 0; --} --__initcall(register_ext_kfuncs); -diff --git a/kernel/sched/ext.h b/kernel/sched/ext.h -deleted file mode 100755 -index fba8ed845f99..000000000000 ---- a/kernel/sched/ext.h -+++ /dev/null -@@ -1,257 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ -- -- --/* -- * Tag marking a kernel function as a kfunc. This is meant to minimize the -- * amount of copy-paste that kfunc authors have to include for correctness so -- * as to avoid issues such as the compiler inlining or eliding either a static -- * kfunc, or a global kfunc in an LTO build. -- */ --#define __bpf_kfunc __used noinline -- --enum scx_wake_flags { -- /* expose select WF_* flags as enums */ -- SCX_WAKE_EXEC = WF_EXEC, -- SCX_WAKE_FORK = WF_FORK, -- SCX_WAKE_TTWU = WF_TTWU, -- SCX_WAKE_SYNC = WF_SYNC, --}; -- --enum scx_enq_flags { -- /* expose select ENQUEUE_* flags as enums */ -- SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, -- SCX_ENQ_HEAD = ENQUEUE_HEAD, -- -- /* high 32bits are SCX specific */ -- -- /* -- * Set the following to trigger preemption when calling -- * scx_bpf_dispatch() with a local dsq as the target. The slice of the -- * current task is cleared to zero and the CPU is kicked into the -- * scheduling path. Implies %SCX_ENQ_HEAD. -- */ -- SCX_ENQ_PREEMPT = 1LLU << 32, -- -- /* -- * The task being enqueued was previously enqueued on the current CPU's -- * %SCX_DSQ_LOCAL, but was removed from it in a call to the -- * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was -- * invoked in a ->cpu_release() callback, and the task is again -- * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the -- * task will not be scheduled on the CPU until at least the next invocation -- * of the ->cpu_acquire() callback. -- */ -- SCX_ENQ_REENQ = 1LLU << 40, -- -- /* -- * The task being enqueued is the only task available for the cpu. By -- * default, ext core keeps executing such tasks but when -- * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with -- * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set. -- * -- * If the BPF scheduler wants to continue executing the task, -- * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately. -- * If the task gets queued on a different dsq or the BPF side, the BPF -- * scheduler is responsible for triggering a follow-up scheduling event. -- * Otherwise, Execution may stall. -- */ -- SCX_ENQ_LAST = 1LLU << 41, -- -- /* -- * A hint indicating that it's advisable to enqueue the task on the -- * local dsq of the currently selected CPU. Currently used by -- * select_cpu_dfl() and together with %SCX_ENQ_LAST. -- */ -- SCX_ENQ_LOCAL = 1LLU << 42, -- -- /* high 8 bits are internal */ -- __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, -- -- SCX_ENQ_CLEAR_OPSS = 1LLU << 56, -- SCX_ENQ_DSQ_PRIQ = 1LLU << 57, --}; -- --enum scx_deq_flags { -- /* expose select DEQUEUE_* flags as enums */ -- SCX_DEQ_SLEEP = DEQUEUE_SLEEP, -- -- /* high 32bits are SCX specific */ -- -- /* -- * The generic core-sched layer decided to execute the task even though -- * it hasn't been dispatched yet. Dequeue from the BPF side. -- */ -- SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, --}; -- --enum scx_tg_flags { -- SCX_TG_ONLINE = 1U << 0, -- SCX_TG_INITED = 1U << 1, --}; -- --enum scx_kick_flags { -- SCX_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -- SCX_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ --}; -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --extern const struct sched_class ext_sched_class; --extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops; --extern const struct file_operations sched_ext_fops; --extern unsigned long scx_watchdog_timeout; --extern unsigned long scx_watchdog_timestamp; -- --DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); --DECLARE_STATIC_KEY_FALSE(__scx_switched_all); --#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) --#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) -- --DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); -- --bool task_on_scx(struct task_struct *p); --void scx_pre_fork(struct task_struct *p); --int scx_fork(struct task_struct *p); --void scx_post_fork(struct task_struct *p); --void scx_cancel_fork(struct task_struct *p); --int scx_check_setscheduler(struct task_struct *p, int policy); --bool scx_can_stop_tick(struct rq *rq); --void init_sched_ext_class(void); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...); --#define scx_ops_error(fmt, args...) \ -- scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args) -- --void __scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active); -- --static inline void scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active) --{ -- if (!scx_enabled()) -- return; --#ifdef CONFIG_SMP -- /* -- * Pairs with the smp_load_acquire() issued by a CPU in -- * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -- * resched. -- */ -- smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1); --#endif -- if (!static_branch_unlikely(&scx_ops_cpu_preempt)) -- return; -- __scx_notify_pick_next_task(rq, p, active); --} -- --//extern void scx_scheduler_tick(void); --static inline void scx_notify_sched_tick(void) --{ -- unsigned long last_check; -- -- if (!scx_enabled()) -- return; -- //scx_scheduler_tick(); -- -- last_check = scx_watchdog_timestamp; -- if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "watchdog failed to check in for %u.%03us", -- dur_ms / 1000, dur_ms % 1000); -- } --} -- --static inline const struct sched_class *next_active_class(const struct sched_class *class) --{ -- class++; -- if (scx_switched_all() && class == &fair_sched_class) -- class++; -- if (!scx_enabled() && class == &ext_sched_class) -- class++; -- return class; --} -- --#define for_active_class_range(class, _from, _to) \ -- for (class = (_from); class != (_to); class = next_active_class(class)) -- --#define for_each_active_class(class) \ -- for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -- --/* -- * SCX requires a balance() call before every pick_next_task() call including -- * when waking up from idle. -- */ --#define for_balance_class_range(class, prev_class, end_class) \ -- for_active_class_range(class, (prev_class) > &ext_sched_class ? \ -- &ext_sched_class : (prev_class), (end_class)) -- --#ifdef CONFIG_SCHED_CORE --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi); --#endif -- --#else /* CONFIG_SCHED_CLASS_EXT */ -- --#define scx_enabled() false --#define scx_switched_all() false -- --static inline void scx_pre_fork(struct task_struct *p) {} --static inline int scx_fork(struct task_struct *p) { return 0; } --static inline void scx_post_fork(struct task_struct *p) {} --static inline void scx_cancel_fork(struct task_struct *p) {} --static inline int scx_check_setscheduler(struct task_struct *p, -- int policy) { return 0; } --static inline bool scx_can_stop_tick(struct rq *rq) { return true; } --static inline void init_sched_ext_class(void) {} --static inline void scx_notify_pick_next_task(struct rq *rq, -- const struct task_struct *p, -- const struct sched_class *active) {} --static inline void scx_notify_sched_tick(void) {} -- --#define for_each_active_class for_each_class --#define for_balance_class_range for_class_range -- --#endif /* CONFIG_SCHED_CLASS_EXT */ -- --#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) --void __scx_update_idle(struct rq *rq, bool idle); -- --static inline void scx_update_idle(struct rq *rq, bool idle) --{ -- if (scx_enabled()) -- __scx_update_idle(rq, idle); --} --#else --static inline void scx_update_idle(struct rq *rq, bool idle) {} --#endif -- --#ifdef CONFIG_CGROUP_SCHED --#ifdef CONFIG_EXT_GROUP_SCHED --int scx_tg_online(struct task_group *tg); --void scx_tg_offline(struct task_group *tg); --int scx_cgroup_can_attach(struct cgroup_taskset *tset); --void scx_move_task(struct task_struct *p); --void scx_cgroup_finish_attach(void); --void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); --void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); --#else /* CONFIG_EXT_GROUP_SCHED */ --static inline int scx_tg_online(struct task_group *tg) { return 0; } --static inline void scx_tg_offline(struct task_group *tg) {} --static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } --static inline void scx_move_task(struct task_struct *p) {} --static inline void scx_cgroup_finish_attach(void) {} --static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} --static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} --#endif /* CONFIG_EXT_GROUP_SCHED */ --#endif /* CONFIG_CGROUP_SCHED */ -diff --git a/kernel/sched/hmbird.h b/kernel/sched/hmbird.h -new file mode 100755 -index 000000000000..d0d84ddee13d ---- /dev/null -+++ b/kernel/sched/hmbird.h -@@ -0,0 +1,343 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#ifndef _HMBIRD_H_ -+#define _HMBIRD_H_ -+ -+/* -+ * Tag marking a kernel function as a kfunc. This is meant to minimize the -+ * amount of copy-paste that kfunc authors have to include for correctness so -+ * as to avoid issues such as the compiler inlining or eliding either a static -+ * kfunc, or a global kfunc in an LTO build. -+ */ -+ -+#define DEBUG_INTERNAL (1 << 0) -+#define DEBUG_INFO_TRACE (1 << 1) -+#define DEBUG_INFO_SYSTRACE (1 << 2) -+ -+#define NUMS_CGROUP_KINDS (512) -+#define SLIM_FOR_SGAME (1 << 0) -+#define SLIM_FOR_GENSHIN (1 << 1) -+ -+#ifdef MAX_TASK_NR -+#define MAX_KEY_THREAD_RECORD ((MAX_TASK_NR + 1) >> 1) -+#else -+#define MAX_KEY_THREAD_RECORD (1 << 3) -+#endif /* MAX_TASK_NR */ -+ -+#define TOP_TASK_SHIFT (8) -+#define TOP_TASK_MAX (1 << TOP_TASK_SHIFT) -+#ifndef TOP_TASK_BITS_MASK -+#define TOP_TASK_BITS_MASK (TOP_TASK_MAX - 1) -+#endif /* TOP_TASK_BITS_MASK */ -+ -+extern struct md_info_t *md_info; -+ -+#define debug_enabled() \ -+ (unlikely(hmbirdcore_debug)) -+ -+#define hmbird_debug(fmt, ...) \ -+ pr_info(":"fmt, ##__VA_ARGS__) -+ -+#define hmbird_err(id, fmt, ...) \ -+do { \ -+ pr_err(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_deferred_err(id, fmt, ...) \ -+do { \ -+ printk_deferred(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \ -+do { \ -+ if (unlikely(cond)) { \ -+ hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+ } \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_info_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_info_trace(fmt, ...) -+#endif -+ -+#define hmbird_info_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+ } \ -+} while (0) -+ -+#define hmbird_output_systrace(fmt, ...) \ -+do { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_internal_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_internal_trace(fmt, ...) -+#endif -+ -+#define hmbird_internal_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) { \ -+ hmbird_output_systrace(fmt, ##__VA_ARGS__); \ -+ } \ -+} while (0) -+ -+enum hmbird_wake_flags { -+ /* expose select WF_* flags as enums */ -+ HMBIRD_WAKE_EXEC = WF_EXEC, -+ HMBIRD_WAKE_FORK = WF_FORK, -+ HMBIRD_WAKE_TTWU = WF_TTWU, -+ HMBIRD_WAKE_SYNC = WF_SYNC, -+}; -+ -+enum hmbird_enq_flags { -+ /* expose select ENQUEUE_* flags as enums */ -+ HMBIRD_ENQ_WAKEUP = ENQUEUE_WAKEUP, -+ HMBIRD_ENQ_HEAD = ENQUEUE_HEAD, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * Set the following to trigger preemption when calling -+ * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the -+ * current task is cleared to zero and the CPU is kicked into the -+ * scheduling path. Implies %HMBIRD_ENQ_HEAD. -+ */ -+ HMBIRD_ENQ_PREEMPT = 1LLU << 32, -+ HMBIRD_ENQ_REENQ = 1LLU << 40, -+ HMBIRD_ENQ_LAST = 1LLU << 41, -+ -+ /* -+ * A hint indicating that it's advisable to enqueue the task on the -+ * local dsq of the currently selected CPU. Currently used by -+ * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST. -+ */ -+ HMBIRD_ENQ_LOCAL = 1LLU << 42, -+ -+ /* high 8 bits are internal */ -+ __HMBIRD_ENQ_INTERNAL_MASK = 0xffLLU << 56, -+ -+ HMBIRD_ENQ_CLEAR_OPSS = 1LLU << 56, -+ HMBIRD_ENQ_DSQ_PRIQ = 1LLU << 57, -+}; -+ -+enum hmbird_deq_flags { -+ /* expose select DEQUEUE_* flags as enums */ -+ HMBIRD_DEQ_SLEEP = DEQUEUE_SLEEP, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * The generic core-sched layer decided to execute the task even though -+ * it hasn't been dispatched yet. Dequeue from the BPF side. -+ */ -+ HMBIRD_DEQ_CORE_SCHED_EXEC = 1LLU << 32, -+}; -+ -+enum hmbird_kick_flags { -+ HMBIRD_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -+ HMBIRD_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ -+}; -+ -+enum hmbird_task_prop_type { -+ HMBIRD_TASK_PROP_COMMON, -+ HMBIRD_TASK_PROP_PIPELINE, -+ HMBIRD_TASK_PROP_DEBUG_OR_LOG, -+ HMBIRD_TASK_PROP_HIGH_LOAD, -+ HMBIRD_TASK_PROP_IO, -+ HMBIRD_TASK_PROP_NETWORK, -+ HMBIRD_TASK_PROP_PERIODIC, -+ HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL, -+ HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL, -+ HMBIRD_TASK_PROP_ISOLATE, -+ HMBIRD_TASK_PROP_MAX, -+}; -+ -+enum hmbird_preempt_policy_id { -+ HMBIRD_PREEMPT_POLICY_NONE, -+ HMBIRD_PREEMPT_POLICY_PRIO_BASED, -+}; -+ -+extern const struct sched_class hmbird_sched_class; -+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops; -+extern const struct file_operations sched_hmbird_fops; -+extern unsigned long hmbird_watchdog_timeout; -+extern unsigned long hmbird_watchdog_timestamp; -+ -+enum scx_rq_flags { -+ HMBIRD_RQ_CAN_STOP_TICK = 1 << 0, -+}; -+ -+struct hmbird_rq { -+ struct hmbird_dispatch_q local_dsq; -+ struct list_head watchdog_list; -+ u64 ops_qseq; -+ u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -+ u32 nr_running; -+ u32 flags; -+ bool cpu_released; -+ cpumask_var_t cpus_to_kick; -+ cpumask_var_t cpus_to_preempt; -+ cpumask_var_t cpus_to_wait; -+ u64 pnt_seq; -+ u64 *prev_runnable_sum_fixed; -+ u32 *prev_window_size; -+ struct irq_work kick_cpus_irq_work; -+ bool pipeline; -+ bool exclusive; -+ bool period_disallow; -+ bool nonperiod_disallow; -+ struct rq *rq; -+ struct hmbird_sched_rq_stats *srq; -+}; -+ -+struct hmbird_sched_change_guard { -+ struct task_struct *p; -+ struct rq *rq; -+ bool queued; -+ bool running; -+ bool done; -+}; -+ -+extern struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -+ -+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags); -+ -+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -+ for (struct hmbird_sched_change_guard __cg = \ -+ hmbird_sched_change_guard_init(__rq, __p, __flags); \ -+ !__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags)) -+ -+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+bool task_on_hmbird(struct task_struct *p); -+int hmbird_pre_fork(struct task_struct *p); -+int hmbird_fork(struct task_struct *p); -+void hmbird_post_fork(struct task_struct *p); -+void hmbird_cancel_fork(struct task_struct *p); -+int hmbird_check_setscheduler(struct task_struct *p, int policy); -+bool hmbird_can_stop_tick(struct rq *rq); -+void init_sched_hmbird_class(void); -+ -+void hmbird_ops_exit(void); -+#define hmbird_ops_error(fmt, ...) \ -+do { \ -+ hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__); \ -+ hmbird_ops_exit(); \ -+} while (0) -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active); -+ -+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight) -+{ -+ return clamp_t(unsigned long, -+ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -+ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); -+} -+ -+static inline void hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active) -+{ -+ if (!hmbird_enabled()) -+ return; -+#ifdef CONFIG_SMP -+ /* -+ * Pairs with the smp_load_acquire() issued by a CPU in -+ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -+ * resched. -+ */ -+ smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1); -+#endif -+ if (!static_branch_unlikely(&hmbird_ops_cpu_preempt)) -+ return; -+ __hmbird_notify_pick_next_task(rq, p, active); -+} -+ -+extern void hmbird_scheduler_tick(void); -+void scan_timeout(struct rq *rq); -+void hmbird_notify_sched_tick(void); -+ -+static inline const struct sched_class *next_active_class(const struct sched_class *class) -+{ -+ class++; -+ -+ if (hmbird_enabled() && class == &fair_sched_class) -+ class++; -+ if (!hmbird_enabled() && class == &hmbird_sched_class) -+ class++; -+ return class; -+} -+ -+#define for_active_class_range(class, _from, _to) \ -+ for (class = (_from); class != (_to); class = next_active_class(class)) -+ -+#define for_each_active_class(class) \ -+ for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -+ -+/* -+ * HMBIRD requires a balance() call before every pick_next_task() call including -+ * when waking up from idle. -+ */ -+#define for_balance_class_range(class, prev_class, end_class) \ -+ for_active_class_range(class, (prev_class) > &hmbird_sched_class ? \ -+ &hmbird_sched_class : (prev_class), (end_class)) -+ -+#define MIN_CGROUP_DL_IDX (5) /* 8ms */ -+#define DEFAULT_CGROUP_DL_IDX (8) /* 64ms */ -+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS]; -+ -+ -+void __hmbird_update_idle(struct rq *rq, bool idle); -+ -+static inline void hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ if (hmbird_enabled()) -+ __hmbird_update_idle(rq, idle); -+} -+ -+int hmbird_ctrl(bool enable); -+ -+int hmbird_tg_online(struct task_group *tg); -+ -+ -+static inline u16 hmbird_task_util(struct task_struct *p) -+{ -+ return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0; -+} -+u16 hmbird_cpu_util(int cpu); -+ -+bool get_hmbird_ops_enabled(void); -+bool get_non_hmbird_task(void); -+void set_cpu_cluster(u64 cpu_cluster); -+#endif /*_EXT_H_*/ -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -new file mode 100755 -index 000000000000..ce05928392bf ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird.c -@@ -0,0 +1,4313 @@ -+// SPDX-License-Identifier: GPL-2.0 -+ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#include -+#include -+ -+#include "slim.h" -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+#include -+ -+#define CLUSTER_SEPARATE -+ -+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0); -+atomic_t non_hmbird_task; -+atomic_t hmbird_module_loaded = ATOMIC_INIT(0); -+int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0}; -+ -+enum hmbird_internal_consts { -+ HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ, -+}; -+ -+enum hmbird_ops_enable_state { -+ HMBIRD_OPS_PREPPING, -+ HMBIRD_OPS_ENABLING, -+ HMBIRD_OPS_ENABLED, -+ HMBIRD_OPS_DISABLING, -+ HMBIRD_OPS_DISABLED, -+}; -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+/* -+ * hmbird_entity->ops_state -+ * -+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler. -+ * State transitions look as follows: -+ * -+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING -+ * ^ | | -+ * | v v -+ * \-------------------------------/ -+ * -+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -+ * sites for explanations on the conditions being waited upon and why they are -+ * safe. Transitions out of them into NONE or QUEUED must store_release and the -+ * waiters should load_acquire. -+ * -+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether -+ * any given task can be dispatched by the BPF scheduler at all times and thus -+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core -+ * can safely reject invalid dispatches. -+ */ -+enum hmbird_ops_state { -+ HMBIRD_OPSS_NONE, /* owned by the HMBIRD core */ -+ HMBIRD_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -+ HMBIRD_OPSS_QUEUED, /* owned by the BPF scheduler */ -+ HMBIRD_OPSS_DISPATCHING, /* in transit back to the HMBIRD core */ -+ -+ /* -+ * QSEQ brands each QUEUED instance so that, when dispatch races -+ * dequeue/requeue, the dispatcher can tell whether it still has a claim -+ * on the task being dispatched. -+ */ -+ HMBIRD_OPSS_QSEQ_SHIFT = 2, -+ HMBIRD_OPSS_STATE_MASK = (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1, -+ HMBIRD_OPSS_QSEQ_MASK = ~HMBIRD_OPSS_STATE_MASK, -+}; -+ -+enum switch_stat { -+ HMBIRD_DISABLED, -+ HMBIRD_SWITCH_PREP, -+ HMBIRD_RQ_SWITCH_BEGIN, -+ HMBIRD_RQ_SWITCH_DONE, -+ HMBIRD_ENABLED, -+}; -+enum switch_stat curr_ss; -+ -+/* -+ * During exit, a task may schedule after losing its PIDs. When disabling the -+ * BPF scheduler, we need to be able to iterate tasks in every state to -+ * guarantee system safety. Maintain a dedicated task list which contains every -+ * task between its fork and eventual free. -+ */ -+DEFINE_SPINLOCK(hmbird_tasks_lock); -+static LIST_HEAD(hmbird_tasks); -+ -+/* ops enable/disable */ -+static struct kthread_worker *hmbird_ops_helper; -+static DEFINE_MUTEX(hmbird_ops_enable_mutex); -+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem); -+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED); -+ -+static bool hmbird_warned_zero_slice; -+enum hmbird_switch_type sw_type; -+static int skip_num[MAX_GLOBAL_DSQS]; -+static int big_distribute_mask_prev; -+static int little_distribute_mask_prev; -+ -+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0); -+ -+/* -+ * The maximum amount of time in jiffies that a task may be runnable without -+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger -+ * hmbird_ops_error(). -+ */ -+unsigned long hmbird_watchdog_timeout; -+ -+/* -+ * The last time the delayed work was run. This delayed work relies on -+ * ksoftirqd being able to run to service timer interrupts, so it's possible -+ * that this work itself could get wedged. To account for this, we check that -+ * it's not stalled in the timer tick, and trigger an error if it is. -+ */ -+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES; -+ -+static struct delayed_work hmbird_watchdog_work; -+static struct work_struct hmbird_err_exit_work; -+ -+/* idle tracking */ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_CPUMASK_OFFSTACK -+#define CL_ALIGNED_IF_ONSTACK -+#else -+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp -+#endif -+ -+static struct { -+ cpumask_var_t cpu; -+ cpumask_var_t smt; -+} idle_masks CL_ALIGNED_IF_ONSTACK; -+ -+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus; -+#endif /* CONFIG_SMP */ -+ -+/* dispatch queues */ -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global; -+ -+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128}; -+u32 pcp_dsq_deadline = 20; -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS]; -+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq); -+ -+static u64 max_hmbird_dsq_internal_id; -+ -+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/ -+#define CLUSTER_SEPARATE_IDX (8) -+#define GDSQS_ID_BASE (3) -+#define UX_COMPATIBLE_IDX (4) -+#define NON_PERIOD_START (5) -+#define NON_PERIOD_END (MAX_GLOBAL_DSQS) -+#define CREATE_DSQ_LEVEL_WITHIN (1) -+ -+struct hmbird_sched_info { -+ spinlock_t lock; -+ int curr_idx[2]; -+ int rtime[MAX_GLOBAL_DSQS]; -+}; -+ -+struct pcp_sched_info { -+ s64 pcp_seq; -+ int rtime; -+ bool pcp_round; -+}; -+ -+/* -+ * pcp_info may rw by another cpu. -+ * protected by rq->lock. -+ */ -+atomic64_t pcp_dsq_round; -+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info); -+ -+struct md_info_t *md_info; -+ -+static int b_rescue_l, l_rescue_b; -+static struct hmbird_sched_info sinfo; -+ -+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC; -+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = { -+ 0, 0, 0, 0, 0, -+ 32 * NSEC_PER_MSEC, -+ 20 * NSEC_PER_MSEC, -+ 14 * NSEC_PER_MSEC, -+ 8 * NSEC_PER_MSEC, -+ 6 * NSEC_PER_MSEC -+}; -+ -+struct cluster_ctx { -+ /* cpu-dsq map must within [lower, upper) */ -+ int upper; -+ int lower; -+ int tidx; -+}; -+ -+enum stat_items { -+ GLOBAL_STAT, -+ CPU_ALLOW_FAIL, -+ RT_CNT, -+ KEY_TASK_CNT, -+ SWITCH_IDX, -+ TIMEOUT_CNT, -+ -+ TOTAL_DSP_CNT, -+ MOVE_RQ_CNT, -+ SELECT_CPU, -+ -+ DWORD_STAT_END = SELECT_CPU, -+ -+ GDSQ_CNT, -+ ERR_IDX, -+ PCP_TIMEOUT_CNT, -+ PCP_LDSQ_CNT, -+ PCP_ENQL_CNT, -+ -+ MAX_ITEMS, -+}; -+static DEFINE_SPINLOCK(stats_lock); -+static char *stats_str[MAX_ITEMS] = { -+ "global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt", -+ "switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt", -+ "select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt", -+ "pcp_ldsq_cnt", "pcp_enql_cnt" -+}; -+ -+ -+struct stats_struct { -+ u64 global_stat[2]; -+ u64 cpu_allow_fail[2]; -+ u64 rt_cnt[2]; -+ u64 key_task_cnt[2]; -+ u64 switch_idx[2]; -+ u64 timeout_cnt[2]; -+ -+ u64 total_dsp_cnt[2]; -+ u64 move_rq_cnt[2]; -+ u64 select_cpu[2]; -+ -+ u64 gdsq_count[MAX_GLOBAL_DSQS][2]; -+ u64 err_idx[5]; -+ u64 pcp_timeout_cnt[NR_CPUS]; -+ u64 pcp_ldsq_count[NR_CPUS][2]; -+ u64 pcp_enql_cnt[NR_CPUS]; -+} stats_data; -+ -+static struct { -+ cpumask_var_t ex_free; -+ cpumask_var_t exclusive; -+ cpumask_var_t partial; -+ cpumask_var_t big; -+ cpumask_var_t little; -+} iso_masks __read_mostly; -+ -+/* -+ * Need more synchronization for these two variables? -+ * I choose not to. -+ */ -+static int l_need_rescue, b_need_rescue; -+ -+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...) \ -+{ \ -+ char buf[MAX_FATAL_INFO]; \ -+ \ -+ scnprintf(buf, MAX_FATAL_INFO, fmt, ##args); \ -+ hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf); \ -+ trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable), \ -+ READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf); \ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); \ -+} \ -+ -+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2) -+{ -+ int c1, c2; -+ struct cpumask mask = {.bits = {0}}; -+ struct cpumask tmp = {.bits = {0}}; -+ -+ if (!slim_stats) -+ return false; -+ -+ if (!rq1 || !rq2) -+ return false; -+ -+ c1 = cpu_of(rq1); -+ c2 = cpu_of(rq2); -+ cpumask_set_cpu(c1, &mask); -+ cpumask_set_cpu(c2, &mask); -+ -+ if (cpumask_and(&tmp, iso_masks.little, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.big, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.partial, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ return false; -+} -+ -+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu) -+{ -+ unsigned long flags; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ if (!slim_stats) -+ return; -+ -+ switch (item) { -+ case GLOBAL_STAT: -+ fallthrough; -+ case CPU_ALLOW_FAIL: -+ fallthrough; -+ case RT_CNT: -+ fallthrough; -+ case KEY_TASK_CNT: -+ fallthrough; -+ case SWITCH_IDX: -+ fallthrough; -+ case TIMEOUT_CNT: -+ fallthrough; -+ case TOTAL_DSP_CNT: -+ fallthrough; -+ case MOVE_RQ_CNT: -+ fallthrough; -+ case SELECT_CPU: -+ pval = pbase + item * 2 + idx; -+ break; -+ case GDSQ_CNT: -+ pval = &stats_data.gdsq_count[dsq_id][idx]; -+ break; -+ case ERR_IDX: -+ pval = &stats_data.err_idx[idx]; -+ break; -+ case PCP_TIMEOUT_CNT: -+ pval = &stats_data.pcp_timeout_cnt[cpu]; -+ break; -+ case PCP_LDSQ_CNT: -+ pval = &stats_data.pcp_ldsq_count[cpu][idx]; -+ break; -+ case PCP_ENQL_CNT: -+ pval = &stats_data.pcp_enql_cnt[cpu]; -+ break; -+ default: -+ return; -+ } -+ -+ spin_lock_irqsave(&stats_lock, flags); -+ *pval += 1; -+ spin_unlock_irqrestore(&stats_lock, flags); -+} -+ -+static inline bool handle_ret(int ret, int *idx, int len) -+{ -+ if (ret < 0 || ret >= len - *idx) -+ return true; -+ *idx += ret; -+ return false; -+} -+ -+#define PRINT_INTV (5 * HZ) -+void stats_print(char *buf, int len) -+{ -+ int idx = 0, i, j, ret; -+ int item = 0; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ for (item = 0; item < MAX_ITEMS; item++) { -+ if (item <= DWORD_STAT_END) { -+ pval = pbase + item * 2; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n", -+ stats_str[item], pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == GDSQ_CNT) { -+ for (j = 0; j < MAX_GLOBAL_DSQS; j++) { -+ pval = (u64 *)&stats_data.gdsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == ERR_IDX) { -+ pval = (u64 *)&stats_data.err_idx; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n", -+ stats_str[item], pval[0], -+ pval[1], pval[2], pval[3], pval[4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == PCP_TIMEOUT_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_timeout_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_LDSQ_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_ldsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_ENQL_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_enql_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } -+ } -+ -+ if (!md_info) { -+ buf[idx] = '\0'; -+ return; -+ } -+ -+ ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_SWITCHS; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "sw_rec[%d] = %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.sw_rec[i].switch_at, -+ md_info->kern_dump.sw_rec[i].is_success, -+ md_info->kern_dump.sw_rec[i].end_state, -+ md_info->kern_dump.sw_rec[i].switch_reason); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_EXCEP_ID; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "excep[%d] = %llu %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.excep_rec[i][0], -+ md_info->kern_dump.excep_rec[i][1], -+ md_info->kern_dump.excep_rec[i][2], -+ md_info->kern_dump.excep_rec[i][3], -+ md_info->kern_dump.excep_rec[i][4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n", -+ i, md_info->kern_dump.excep_idx[i]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ -+ buf[idx] = '\0'; -+} -+ -+enum cpu_type { -+ LITTLE, -+ BIG, -+ PARTIAL, -+ EXCLUSIVE, -+ EX_FREE, -+ INVALID -+}; -+ -+enum dsq_type { -+ GLOBAL_DSQ, -+ PCP_DSQ, -+ OTHER, -+ MAX_DSQ_TYPE, -+}; -+ -+static enum cpu_type cpu_cluster(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (get_hmbird_rq(rq)->exclusive) { -+ return EXCLUSIVE; -+ } else { -+ if (cpumask_test_cpu(cpu, iso_masks.little)) -+ return LITTLE; -+ else if (cpumask_test_cpu(cpu, iso_masks.big)) -+ return BIG; -+ else if (cpumask_test_cpu(cpu, iso_masks.partial)) -+ return PARTIAL; -+ else if (cpumask_test_cpu(cpu, iso_masks.exclusive)) -+ return EXCLUSIVE; -+ else if (cpumask_test_cpu(cpu, iso_masks.ex_free)) -+ return EX_FREE; -+ } -+ return INVALID; -+} -+ -+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq) -+{ -+ if (!dsq) -+ return OTHER; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= GDSQS_ID_BASE) && -+ ((dsq->id & 0xff) < MAX_GLOBAL_DSQS)) -+ return GLOBAL_DSQ; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) && -+ ((dsq->id & 0xff) < max_hmbird_dsq_internal_id)) -+ return PCP_DSQ; -+ -+ return OTHER; -+} -+ -+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq) -+{ -+ enum dsq_type type; -+ -+ type = get_dsq_type(dsq); -+ switch (type) { -+ case GLOBAL_DSQ: -+ case PCP_DSQ: -+ return (dsq->id & 0xff) - GDSQS_ID_BASE; -+ default: -+ return -1; -+ } -+ return -1; -+} -+ -+static void update_cpus_idle(bool set, struct cpumask *mask) -+{ -+ int cpu; -+ struct rq *rq; -+ -+ if (set) { -+ for_each_cpu(cpu, mask) { -+ rq = cpu_rq(cpu); -+ if (is_idle_task(rq->curr)) -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ } -+ } else -+ cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask); -+} -+ -+static void set_partial_status(bool enable, bool little, bool big) -+{ -+ WRITE_ONCE(partial_enable, enable); -+ WRITE_ONCE(l_need_rescue, little); -+ WRITE_ONCE(b_need_rescue, big); -+} -+ -+static bool is_little_need_rescue(void) -+{ -+ return READ_ONCE(l_need_rescue); -+} -+ -+static bool is_big_need_rescue(void) -+{ -+ return READ_ONCE(b_need_rescue); -+} -+ -+static bool is_partial_enabled(void) -+{ -+ return READ_ONCE(partial_enable); -+} -+ -+static bool is_partial_cpu(int cpu) -+{ -+ return cpumask_test_cpu(cpu, iso_masks.partial); -+} -+ -+static void set_iso_par_free(bool enable) -+{ -+ WRITE_ONCE(isolate_ctrl, enable); -+} -+ -+static bool is_iso_par_free(void) -+{ -+ return READ_ONCE(isolate_ctrl); -+} -+ -+static bool skip_update_idle(void) -+{ -+ int cpu = smp_processor_id(); -+ enum cpu_type type = cpu_cluster(cpu); -+ -+ if ((type == EXCLUSIVE && !is_iso_par_free()) || -+ /* partial enable may changed during idle, it doesn't matter. */ -+ (!is_partial_enabled() && type == PARTIAL)) -+ return true; -+ -+ return false; -+} -+ -+static void init_isolate_cpus(void) -+{ -+ WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -+ cpumask_set_cpu(0, iso_masks.big); -+ cpumask_set_cpu(1, iso_masks.big); -+ cpumask_set_cpu(2, iso_masks.big); -+ cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(4, iso_masks.big); -+ cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(6, iso_masks.partial); -+ cpumask_set_cpu(7, iso_masks.ex_free); -+} -+ -+extern spinlock_t css_set_lock; -+ -+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp) -+{ -+ int i; -+ struct cgroup *anc; -+ -+ spin_lock_irq(&css_set_lock); -+ for (i = 0; i < cgrp->level; i++) { -+ anc = cgrp->ancestors[i]; -+ if (anc->level != CREATE_DSQ_LEVEL_WITHIN) -+ continue; -+ cgroup_get(anc); -+ spin_unlock_irq(&css_set_lock); -+ return anc; -+ } -+ spin_unlock_irq(&css_set_lock); -+ hmbird_err(NO_CGROUP_L1, ":error cgroup = %s\n", cgrp->kn->name); -+ return NULL; -+} -+ -+#define PCP_IDX_BIT (1 << 31) -+ -+static bool is_pcp_rt(struct task_struct *p) -+{ -+ return rt_prio(p->prio) && (p->nr_cpus_allowed == 1); -+} -+ -+static bool is_pcp_idx(int idx) -+{ -+ return idx >= MAX_GLOBAL_DSQS; -+} -+ -+static bool is_critical_system_task(struct task_struct *p) -+{ -+ int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK; -+ -+ return (p->prio < (MAX_RT_PRIO >> 1) && -+ (sp_dl < SCHED_PROP_DEADLINE_LEVEL3)); -+} -+ -+#define ISOLATE_TASK_TOP_BIT (1 << 17) -+#define PIPELINE_TASK_TOP_BIT (1 << 9) -+static bool is_pipeline_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT); -+} -+ -+static bool is_isolate_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT); -+} -+ -+static bool is_critical_app_task_without_isolate(struct task_struct *p) -+{ -+ return task_is_top_task(p) && !is_isolate_task(p); -+} -+ -+static int find_idx_from_task(struct task_struct *p) -+{ -+ int idx, cpu; -+ int sp_dl; -+ struct task_group *tg = p->sched_task_group; -+ -+ if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) { -+ cpu = cpumask_any(p->cpus_ptr); -+ idx = cpu + MAX_GLOBAL_DSQS; -+ return idx; -+ } -+ -+ if (is_critical_system_task(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL0; -+ goto done; -+ } -+ -+ if (is_critical_app_task_without_isolate(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL1; -+ goto done; -+ } -+ -+ sp_dl = hmbird_get_dsq_id(p); -+ if (sp_dl) { -+ idx = sp_dl; -+ goto done; -+ } -+ -+ if (rt_prio(p->prio)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL3; -+ goto done; -+ } -+ -+ if (tg && tg->css.cgroup && tg->css.cgroup->kn) { -+ if (likely(tg->css.cgroup->kn->id >= 0 && -+ tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS)) -+ idx = cgroup_ids_table[tg->css.cgroup->kn->id]; -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ -+done: -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) { -+ hmbird_err(DSQ_ID_ERR, " : idx error, idx = %d-----\n", idx); -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } -+ return idx; -+} -+ -+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p) -+{ -+ int idx; -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq; -+ -+ if (!p) -+ return NULL; -+ -+ idx = find_idx_from_task(p); -+ if (is_pcp_idx(idx)) { -+ idx -= MAX_GLOBAL_DSQS; -+ dsq = &per_cpu(pcp_ldsq, idx); -+ get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq); -+ slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx); -+ } else { -+ dsq = &gdsqs[idx]; -+ get_hmbird_ts(p)->gdsq_idx = idx; -+ slim_stats_record(GDSQ_CNT, 0, idx, 0); -+ } -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ dsq->last_consume_at = jiffies; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ return dsq; -+} -+ -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq); -+ -+static void set_partial_rescue(bool p_state, bool l_over, bool b_over) -+{ -+ set_partial_status(p_state, l_over, b_over); -+ update_cpus_idle(p_state, iso_masks.partial); -+ hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled()); -+ hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue()); -+ hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue()); -+} -+ -+static void free_isocpu(bool enable) -+{ -+ set_iso_par_free(enable); -+ update_cpus_idle(enable, iso_masks.ex_free); -+ hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free()); -+} -+ -+inline u64 get_hmbird_cpu_util(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed) -+ return 0; -+ u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed); -+ u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size); -+ -+ do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT); -+ -+ return prev_runnable_sum_fixed; -+} -+ -+static inline unsigned int get_scaling_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->max; -+} -+ -+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static u64 get_cpus_max_util(struct cpumask *mask) -+{ -+ int cpu; -+ u64 max = 0; -+ u64 util, ratio; -+ unsigned long effective_cap = 0; -+ for_each_cpu(cpu, mask) { -+ if (slim_walt_ctrl) -+ slim_get_cpu_util(cpu, &util); -+ else -+ util = get_hmbird_cpu_util(cpu); -+ -+ /* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/ -+ if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu))) -+ effective_cap = arch_scale_cpu_capacity(cpu); -+ else -+ effective_cap = arch_scale_cpu_capacity(cpu) * -+ get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu); -+ -+ ratio = util * 100 / effective_cap; -+ hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util); -+ hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n", -+ cpu, (u64)arch_scale_cpu_capacity(cpu)); -+ hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n", -+ cpu, (u64)effective_cap); -+ -+ if (ratio > 100) -+ ratio = 100; -+ -+ if (ratio > max) -+ max = ratio; -+ } -+ return max; -+} -+ -+static bool cluster_need_rescue(struct cpumask *mask, int hres) -+{ -+ return get_cpus_max_util(mask) > hres; -+} -+ -+void partial_dynamic_ctrl(void) -+{ -+ u64 lmax = 0, bmax = 0; -+ bool l_over, l_under, b_over, b_under; -+ static bool last_l_over, last_b_over; -+ static unsigned long last_check; -+ -+ /* Check partial every jiffies. need lock? */ -+ if (time_before_eq(jiffies, READ_ONCE(last_check))) -+ return; -+ -+ WRITE_ONCE(last_check, jiffies); -+ -+ lmax = get_cpus_max_util(iso_masks.little); -+ l_over = lmax > parctrl_high_ratio_l; -+ l_under = lmax < parctrl_low_ratio_l; -+ -+ bmax = get_cpus_max_util(iso_masks.big); -+ b_over = bmax > parctrl_high_ratio; -+ b_under = bmax < parctrl_low_ratio; -+ -+ if (is_partial_enabled() && (l_over || b_over)) { -+ if (last_l_over != l_over || last_b_over != b_over) -+ set_partial_rescue(true, l_over, b_over); -+ } else if (!is_partial_enabled() && (l_over || b_over)) { -+ set_partial_rescue(true, l_over, b_over); -+ } else if (is_partial_enabled() && l_under && b_under) { -+ set_partial_rescue(false, false, false); -+ } -+ last_l_over = l_over; -+ last_b_over = b_over; -+ -+ if (is_partial_enabled()) { -+ bmax = get_cpus_max_util(iso_masks.partial); -+ if (!is_iso_par_free() && bmax > isoctrl_high_ratio) { -+ hmbird_info_trace("partial max = %llu\n", bmax); -+ free_isocpu(true); -+ } else if (is_iso_par_free() && bmax < isoctrl_low_ratio) -+ free_isocpu(false); -+ } else if (is_iso_par_free()) { -+ free_isocpu(false); -+ } -+} -+ -+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx) -+{ -+ hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx); -+} -+ -+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx) -+{ -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[idx])) { -+ slim_stats_record(GDSQ_CNT, 1, idx, 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ int i; -+ -+ for (i = 0; i < UX_COMPATIBLE_IDX; i++) { -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ slim_stats_record(GDSQ_CNT, 1, i, 0); -+ return 1; -+ } -+ } -+ return 0; -+} -+ -+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) { -+ slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ goto clear_timeout; -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) -+ goto clear_timeout; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ hmbird_info_trace( -+ "dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n", -+ dsq_id_to_internal(dsq), entity->task->comm, -+ jiffies, msecs_to_jiffies(deadline), entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1); -+ return; -+ -+clear_timeout: -+ hmbird_info_trace("dsq[%d] clear timeout\n", -+ dsq_id_to_internal(dsq)); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0); -+ dsq->is_timeout = false; -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq)); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime) -+{ -+ hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime); -+} -+ -+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any) -+{ -+ bool is_timeout; -+ int cpu = cpu_of(rq); -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu); -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ is_timeout = dsq->is_timeout; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ /* -+ * dsq->is_timeout may change here, let it be. -+ * it won't cause serious problems. -+ * the same for consume_dispatch_q later. -+ */ -+ if (!is_timeout && !any) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, dsq)) { -+ if (is_timeout) { -+ hmbird_info_trace("dsq[%d] consume a pcp timeout task\n", -+ dsq_id_to_internal(dsq)); -+ update_timeout_stats(rq, dsq, pcp_dsq_deadline); -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu); -+ } -+ slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu); -+ return 1; -+ } -+ /* -+ * No pcp task, clear quota. -+ */ -+ if (any) { -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+ } -+ return 0; -+} -+ -+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf) -+{ -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ } -+ return 0; -+} -+ -+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf, -+ int tmp, int cidx, int tidx) -+{ -+ unsigned long flags; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) { -+ if (tmp != cidx) { -+ spin_lock(&sinfo.lock); -+ sinfo.curr_idx[tidx] = tmp; -+ spin_unlock(&sinfo.lock); -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]); -+ slim_stats_record(SWITCH_IDX, 0, 0, 0); -+ } -+ slim_stats_record(GDSQ_CNT, 1, tmp, 0); -+ -+ raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags); -+ gdsqs[tmp].last_consume_at = jiffies; -+ raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags); -+ return 1; -+ } -+ return 0; -+ -+} -+ -+static int get_cidx(struct cluster_ctx *ctx) -+{ -+ int cidx; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ if (cidx < ctx->lower || cidx >= ctx->upper) { -+ sinfo.curr_idx[ctx->tidx] = ctx->lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]); -+ slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ } -+ spin_unlock(&sinfo.lock); -+ -+ return cidx; -+} -+ -+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = CLUSTER_SEPARATE_IDX; -+ if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) { -+ pr_debug(" %s iso_masks.little first is %d\n", -+ __func__, cpumask_first(iso_masks.little)); -+ ctx->upper = NON_PERIOD_END; -+ } -+ ctx->tidx = 0; -+ break; -+ case LITTLE: -+ ctx->lower = CLUSTER_SEPARATE_IDX; -+ ctx->upper = NON_PERIOD_END; -+ if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) { -+ pr_debug(" %s iso_masks.big first is %d", -+ __func__, cpumask_first(iso_masks.big)); -+ ctx->lower = NON_PERIOD_START; -+ } -+ ctx->tidx = 1; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ case LITTLE: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = NON_PERIOD_END; -+ ctx->tidx = 0; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ if (cluster_separate) { -+ return gen_cluster_ctx_separate(ctx, type); -+ } else { -+ return gen_cluster_ctx_common(ctx, type); -+ } -+} -+ -+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ int i; -+ bool is_timeout; -+ unsigned long flags; -+ struct cluster_ctx ctx; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ /* Third param means only consume timeout pcp dsq. */ -+ if (consume_pcp_dsq(rq, rf, false)) -+ return 1; -+ -+ for (i = ctx.lower; i < ctx.upper; i++) { -+ raw_spin_lock_irqsave(&gdsqs[i].lock, flags); -+ is_timeout = gdsqs[i].is_timeout; -+ raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags); -+ /* gdsqs[i].is_timeout may change here, let it be... */ -+ if (is_timeout) { -+ /* -+ * consume_dispatch_q will acquire dsq-lock, -+ * So cannot keep lock here, annoy enough. -+ * may rewrite a consume_dispatch_q_locked, TODO. -+ */ -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ hmbird_info_trace("dsq[%d] consume a timeout task\n", i); -+ slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0); -+ update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]); -+ return 1; -+ } -+ } -+ } -+ return 0; -+} -+ -+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ struct cluster_ctx ctx; -+ int cidx; -+ int tmp; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ cidx = get_cidx(&ctx); -+ tmp = cidx; -+ do { -+ if (check_pcp_dsq_round(rq, rf)) -+ return 1; -+ if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx)) -+ return 1; -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[tmp] = 0; -+ systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]); -+ tmp++; -+ if (tmp >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ tmp = ctx.lower; -+ } -+ spin_unlock(&sinfo.lock); -+ } while (tmp != cidx); -+ -+ return consume_pcp_dsq(rq, rf, true); -+} -+ -+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ enum cpu_type type = cpu_cluster(cpu_of(rq)); -+ bool period_allowed = !get_hmbird_rq(rq)->period_disallow; -+ bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow; -+ -+ switch (type) { -+ case EX_FREE: -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ break; -+ case EXCLUSIVE: -+ if (!is_iso_par_free()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ } -+ /* Only non-period task can run on isolate cpus */ -+ if (!READ_ONCE(iso_free_rescue)) { -+ if (is_big_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ } else { -+ /* Free run, can run on any task. */ -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ case PARTIAL: -+ if (!is_partial_enabled()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ return 0; -+ } -+ if (is_big_need_rescue()) { -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ -+ case BIG: -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ if (is_iso_par_free() || (is_little_need_rescue() && -+ !cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) { -+ hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++); -+ return 1; -+ } -+ } -+ break; -+ -+ case LITTLE: -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ -+ if (is_iso_par_free() || (is_big_need_rescue() && -+ !cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) { -+ hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++); -+ return 1; -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return 0; -+} -+ -+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf) -+{ -+ return consume_hmbird_global_dsq(rq, rf); -+} -+ -+ -+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time) -+{ -+ int idx; -+ -+ /* which dsq belongs to while task enqueue, task will consume its running time. */ -+ idx = get_hmbird_ts(p)->gdsq_idx; -+ /* Only non-period dsq share running time between each other. */ -+ if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id) -+ return; -+ -+ if (idx >= MAX_GLOBAL_DSQS) { -+ per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } else { -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[idx] += exec_time; -+ spin_unlock(&sinfo.lock); -+ systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]); -+ } -+} -+ -+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type) -+{ -+ int cidx; -+ struct cluster_ctx ctx; -+ int cpu = cpu_of(rq); -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ if (cidx < ctx.lower || cidx >= ctx.upper) { -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ slim_stats_record(ERR_IDX, ctx.tidx, 0, 0); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ } -+ -+ while (1) { -+ if (per_cpu(pcp_info, cpu).pcp_round) { -+ if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) { -+ hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n", -+ cpu, per_cpu(pcp_info, cpu).rtime); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } -+ } -+ if (sinfo.rtime[cidx] < dsq_quota[cidx]) -+ break; -+ -+ /* clear current dsq rtime */ -+ sinfo.rtime[cidx] = 0; -+ systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]); -+ -+ sinfo.curr_idx[ctx.tidx]++; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", -+ ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ } -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ slim_stats_record(SWITCH_IDX, 1, 0, 0); -+ } -+ spin_unlock(&sinfo.lock); -+} -+ -+ -+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p) -+{ -+ enum cpu_type type; -+ -+ if (!rq || !p) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ switch (type) { -+ case PARTIAL: -+ return; -+ case EXCLUSIVE: -+ return; -+ case EX_FREE: -+ return; -+ default: -+ break; -+ } -+ update_dsq_idx(rq, p, type); -+} -+ -+ -+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ int dsq_id; -+ -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo) || dsq->is_timeout) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ hmbird_deferred_err(SCAN_ENTITY_NULL, -+ " : entity is NULL, dsq->id = %llu\n", dsq->id); -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ dsq->is_timeout = true; -+ dsq_id = dsq_id_to_internal(dsq); -+ hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n", -+ dsq_id, entity->task->comm, jiffies, entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1); -+ raw_spin_unlock(&dsq->lock); -+ -+ return true; -+} -+ -+void scan_timeout(struct rq *rq) -+{ -+ int i; -+ int cpu = cpu_of(rq); -+ struct hmbird_dispatch_q *dsq; -+ static u64 last_scan_at; -+ static DEFINE_PER_CPU(u64, pcp_last_scan_at); -+ -+ if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu))) -+ return; -+ per_cpu(pcp_last_scan_at, cpu) = jiffies; -+ -+ dsq = &per_cpu(pcp_ldsq, cpu); -+ scan_dsq_timeout(rq, dsq, pcp_dsq_deadline); -+ -+ if (time_before_eq(jiffies, (unsigned long)last_scan_at)) -+ return; -+ last_scan_at = jiffies; -+ -+ for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) { -+ dsq = &gdsqs[i]; -+ scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]); -+ } -+} -+ -+/*******************************Initialize***********************************/ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id); -+static void init_dsq_at_boot(void) -+{ -+ int i, cpu; -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i)); -+ } -+ for_each_possible_cpu(cpu) -+ init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i + cpu)); -+ -+ max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu; -+ spin_lock_init(&sinfo.lock); -+} -+ -+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgrp_name_to_idx(struct cgroup *cgrp) -+{ -+ int idx; -+ -+ if (!cgrp) -+ return -1; -+ -+ if (!strcmp(cgrp->kn->name, "display") -+ || !strcmp(cgrp->kn->name, "multimedia")) -+ idx = 5; /* 8ms */ -+ else if (!strcmp(cgrp->kn->name, "top-app") -+ || !strcmp(cgrp->kn->name, "ss-top")) -+ idx = 6; /* 16ms */ -+ else if (!strcmp(cgrp->kn->name, "ssfg") -+ || !strcmp(cgrp->kn->name, "foreground")) -+ idx = 7; /* 32ms */ -+ else if (!strcmp(cgrp->kn->name, "bg") -+ || !strcmp(cgrp->kn->name, "log") -+ || !strcmp(cgrp->kn->name, "dex2oat") -+ || !strcmp(cgrp->kn->name, "background")) -+ idx = 9; /* 128ms */ -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */ -+ -+ return idx; -+} -+ -+static void init_root_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX); -+} -+ -+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) { -+ pr_err("%s idx err!\n", __func__); -+ return; -+ } -+ -+ if (cgroup_ids_table[cgrp->kn->id] == -1) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp)); -+} -+ -+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ struct cgroup *l1cgrp; -+ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ l1cgrp = cgroup_ancestor_l1(cgrp); -+ if (l1cgrp) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp)); -+ cgroup_put(l1cgrp); -+} -+ -+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg) -+{ -+ switch (cgrp->level) { -+ case 0: -+ init_root_tg(cgrp, tg); -+ break; -+ case 1: -+ init_level1_tg(cgrp, tg); -+ break; -+ default: -+ init_child_tg(cgrp, tg); -+ break; -+ } -+} -+ -+/**************************************************************************/ -+ -+struct hmbird_task_iter { -+ struct hmbird_entity cursor; -+ struct task_struct *locked; -+ struct rq *rq; -+ struct rq_flags rf; -+}; -+ -+/** -+ * hmbird_task_iter_init - Initialize a task iterator -+ * @iter: iterator to init -+ * -+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized, -+ * @iter must eventually be exited with hmbird_task_iter_exit(). -+ * -+ * hmbird_tasks_lock may be released between this and the first next() call or -+ * between any two next() calls. If hmbird_tasks_lock is released between two -+ * next() calls, the caller is responsible for ensuring that the task being -+ * iterated remains accessible either through RCU read lock or obtaining a -+ * reference count. -+ * -+ * All tasks which existed when the iteration started are guaranteed to be -+ * visited as long as they still exist. -+ */ -+static void hmbird_task_iter_init(struct hmbird_task_iter *iter) -+{ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR }; -+ list_add(&iter->cursor.tasks_node, &hmbird_tasks); -+ iter->locked = NULL; -+} -+ -+/** -+ * hmbird_task_iter_exit - Exit a task iterator -+ * @iter: iterator to exit -+ * -+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held. -+ * If the iterator holds a task's rq lock, that rq lock is released. See -+ * hmbird_task_iter_init() for details. -+ */ -+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ if (list_empty(cursor)) -+ return; -+ -+ list_del_init(cursor); -+} -+ -+/** -+ * hmbird_task_iter_next - Next task -+ * @iter: iterator to walk -+ * -+ * Visit the next task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ struct hmbird_entity *pos; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ list_for_each_entry(pos, cursor, tasks_node) { -+ if (&pos->tasks_node == &hmbird_tasks) -+ return NULL; -+ if (!(pos->flags & HMBIRD_TASK_CURSOR)) { -+ list_move(cursor, &pos->tasks_node); -+ return pos->task; -+ } -+ } -+ -+ /* can't happen, should always terminate at hmbird_tasks above */ -+ hmbird_deferred_err(ITER_RET_NULL, " : unreachable path in scx_task_iter_next\n"); -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered - Next non-idle task -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ while ((p = hmbird_task_iter_next(iter))) { -+ if (!is_idle_task(p)) -+ return p; -+ } -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init() -+ * for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ p = hmbird_task_iter_next_filtered(iter); -+ if (!p) -+ return NULL; -+ -+ iter->rq = task_rq_lock(p, &iter->rf); -+ iter->locked = p; -+ return p; -+} -+ -+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void) -+{ -+ return atomic_read(&hmbird_ops_enable_state_var); -+} -+ -+static enum hmbird_ops_enable_state -+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to) -+{ -+ return atomic_xchg(&hmbird_ops_enable_state_var, to); -+} -+ -+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to, -+ enum hmbird_ops_enable_state from) -+{ -+ int from_v = from; -+ -+ return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to); -+} -+ -+static bool hmbird_ops_disabling(void) -+{ -+ return false; -+} -+ -+/** -+ * wait_ops_state - Busy-wait the specified ops state to end -+ * @p: target task -+ * @opss: state to wait the end of -+ * -+ * Busy-wait for @p to transition out of @opss. This can only be used when the -+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also -+ * has load_acquire semantics to ensure that the caller can see the updates made -+ * in the enqueueing and dispatching paths. -+ */ -+static void wait_ops_state(struct task_struct *p, u64 opss) -+{ -+ do { -+ cpu_relax(); -+ } while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss); -+} -+ -+ -+static void update_curr_hmbird(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ u64 now = rq_clock_task(rq); -+ u64 delta_exec; -+ -+ if (time_before_eq64(now, curr->se.exec_start)) -+ return; -+ -+ delta_exec = now - curr->se.exec_start; -+ curr->se.exec_start = now; -+ update_runningtime(rq, curr, delta_exec); -+ curr->se.sum_exec_runtime += delta_exec; -+ account_group_exec_runtime(curr, delta_exec); -+ cgroup_account_cputime(curr, delta_exec); -+ -+ if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF) -+ get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec); -+ -+ trace_sched_stat_runtime(curr, delta_exec, 0); -+} -+ -+static bool hmbird_dsq_priq_less(struct rb_node *node_a, -+ const struct rb_node *node_b) -+{ -+ const struct hmbird_entity *a = -+ container_of(node_a, struct hmbird_entity, dsq_node.priq); -+ const struct hmbird_entity *b = -+ container_of(node_b, struct hmbird_entity, dsq_node.priq); -+ -+ return time_before64(a->dsq_vtime, b->dsq_vtime); -+} -+ -+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ bool is_local = dsq->id == HMBIRD_DSQ_LOCAL; -+ unsigned long flags; -+ -+ hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq || -+ !list_empty(&get_hmbird_ts(p)->dsq_node.fifo), -+ "task = %s, dsq->id = %llu\n", p->comm, dsq->id); -+ hmbird_cond_deferred_err(ENQ_EXIST2, -+ (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq), -+ "task = %s\n", p->comm); -+ -+ if (!is_local) { -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ); -+ hmbird_ops_error(": %s\n", -+ "attempting to dispatch to a destroyed dsq"); -+ /* fall back to the global dsq */ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ dsq = &hmbird_dsq_global; -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ } -+ } -+ -+ if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) { -+ get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ; -+ rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq, -+ hmbird_dsq_priq_less); -+ } else { -+ if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT)) -+ list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ else -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ } -+ dsq->nr++; -+ get_hmbird_ts(p)->dsq = dsq; -+ -+ /* -+ * We're transitioning out of QUEUEING or DISPATCHING. store_release to -+ * match waiters' load_acquire. -+ */ -+ if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS) -+ atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE); -+ -+ if (is_local) { -+ struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq); -+ struct rq *rq = hmbird->rq; -+ bool preempt = false; -+ -+ if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr && -+ rq->curr->sched_class == &hmbird_sched_class) { -+ get_hmbird_ts(rq->curr)->slice = 0; -+ preempt = true; -+ } -+ -+ if (preempt || sched_class_above(&hmbird_sched_class, -+ rq->curr->sched_class)) -+ resched_curr(rq); -+ } else { -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ } -+} -+ -+static void task_unlink_from_dsq(struct task_struct *p, -+ struct hmbird_dispatch_q *dsq) -+{ -+ if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) { -+ rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ; -+ } else { -+ list_del_init(&get_hmbird_ts(p)->dsq_node.fifo); -+ } -+} -+ -+static bool task_linked_on_dsq(struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+} -+ -+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq; -+ bool is_local = dsq == &hmbird_rq->local_dsq; -+ -+ if (!dsq) { -+ hmbird_cond_deferred_err(TASK_LINKED1, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ /* -+ * When dispatching directly from the BPF scheduler to a local -+ * DSQ, the task isn't associated with any DSQ but -+ * @get_hmbird_ts(p)->holding_cpu may be set under the protection of -+ * %HMBIRD_OPSS_DISPATCHING. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu >= 0) -+ get_hmbird_ts(p)->holding_cpu = -1; -+ return; -+ } -+ -+ if (!is_local) -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ /* -+ * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node -+ * can't change underneath us. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu < 0) { -+ /* @p must still be on @dsq, dequeue */ -+ hmbird_cond_deferred_err(TASK_UNLINKED, -+ !task_linked_on_dsq(p), "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ } else { -+ /* -+ * We're racing against dispatch_to_local_dsq() which already -+ * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the -+ * holding_cpu which tells dispatch_to_local_dsq() that it lost -+ * the race. -+ */ -+ hmbird_cond_deferred_err(TASK_LINKED2, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ get_hmbird_ts(p)->holding_cpu = -1; -+ } -+ get_hmbird_ts(p)->dsq = NULL; -+ -+ if (!is_local) -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+ -+static bool test_rq_online(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ return rq->online; -+#else -+ return true; -+#endif -+} -+ -+static void refill_task_slice(struct task_struct *p) -+{ -+ if (is_isolate_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO; -+ else if (is_pipeline_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2; -+ else -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+} -+ -+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -+ int sticky_cpu) -+{ -+ struct hmbird_dispatch_q *d; -+ s32 cpu = -1; -+ -+ hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), -+ "task = %s\n", p->comm); -+ -+ if (is_pcp_rt(p)) { -+ /* Enqueue percpu rt task to local directly. */ -+ /* Or cause a bug when disable dispatch. */ -+ if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ } -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (cpu >= 0) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ -+ if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ /* rq migration */ -+ if (sticky_cpu == cpu_of(rq)) -+ goto local_norefill; -+ /* -+ * If !rq->online, we already told the BPF scheduler that the CPU is -+ * offline. We're just trying to on/offline the CPU. Don't bother the -+ * BPF scheduler. -+ */ -+ if (unlikely(!test_rq_online(rq))) -+ goto local; -+ -+ /* see %HMBIRD_OPS_ENQ_LAST */ -+ if (enq_flags & HMBIRD_ENQ_LAST) -+ goto local; -+ -+ if (enq_flags & HMBIRD_ENQ_LOCAL) -+ goto local; -+ else -+ goto global; -+local: -+ /* -+ * For task-ordering, slice refill must be treated as implying the end -+ * of the current slice. Otherwise, the longer @p stays on the CPU, the -+ * higher priority it becomes from hmbird_prio_less()'s POV. -+ */ -+ refill_task_slice(p); -+local_norefill: -+ dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags); -+ slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq)); -+ return; -+ -+global: -+ d = find_dsq_from_task(p); -+ if (d) { -+ refill_task_slice(p); -+ dispatch_enqueue(d, p, enq_flags); -+ return; -+ } -+ slim_stats_record(GLOBAL_STAT, 0, 0, 0); -+ refill_task_slice(p); -+ dispatch_enqueue(&hmbird_dsq_global, p, enq_flags); -+} -+ -+static bool watchdog_task_watched(const struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->watchdog_node); -+} -+ -+static void watchdog_watch_task(struct rq *rq, struct task_struct *p) -+{ -+ lockdep_assert_rq_held(rq); -+ if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ get_hmbird_ts(p)->runnable_at = jiffies; -+ clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list); -+} -+ -+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) -+{ -+ list_del_init(&get_hmbird_ts(p)->watchdog_node); -+ if (reset_timeout) -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags) -+{ -+ int sticky_cpu = get_hmbird_ts(p)->sticky_cpu; -+ -+ enq_flags |= get_hmbird_rq(rq)->extra_enq_flags; -+ -+ if (sticky_cpu >= 0) -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ -+ /* -+ * Restoring a running task will be immediately followed by -+ * set_next_task_hmbird() which expects the task to not be on the BPF -+ * scheduler as tasks can only start running through local DSQs. Force -+ * direct-dispatch into the local DSQ by setting the sticky_cpu. -+ */ -+ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -+ sticky_cpu = cpu_of(rq); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_UNWATCHED, -+ !watchdog_task_watched(p), "task = %s\n", p->comm); -+ return; -+ } -+ -+ watchdog_watch_task(rq, p); -+ set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ get_hmbird_rq(rq)->nr_running++; -+ add_nr_running(rq, 1); -+ -+ do_enqueue_task(rq, p, enq_flags, sticky_cpu); -+} -+ -+static void ops_dequeue(struct task_struct *p, u64 deq_flags) -+{ -+ u64 opss; -+ -+ watchdog_unwatch_task(p, false); -+ -+ /* acquire ensures that we see the preceding updates on QUEUED */ -+ opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state); -+ -+ switch (opss & HMBIRD_OPSS_STATE_MASK) { -+ case HMBIRD_OPSS_NONE: -+ break; -+ case HMBIRD_OPSS_QUEUEING: -+ /* -+ * QUEUEING is started and finished while holding @p's rq lock. -+ * As we're holding the rq lock now, we shouldn't see QUEUEING. -+ */ -+ hmbird_deferred_err(DEQ_DEQING, " : unreachable path in %s\n", __func__); -+ break; -+ case HMBIRD_OPSS_QUEUED: -+ if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss, -+ HMBIRD_OPSS_NONE)) -+ break; -+ fallthrough; -+ case HMBIRD_OPSS_DISPATCHING: -+ /* -+ * If @p is being dispatched from the BPF scheduler to a DSQ, -+ * wait for the transfer to complete so that @p doesn't get -+ * added to its DSQ after dequeueing is complete. -+ * -+ * As we're waiting on DISPATCHING with the rq locked, the -+ * dispatching side shouldn't try to lock the rq while -+ * DISPATCHING is set. See dispatch_to_local_dsq(). -+ * -+ * DISPATCHING shouldn't have qseq set and control can reach -+ * here with NONE @opss from the above QUEUED case block. -+ * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss. -+ */ -+ wait_ops_state(p, HMBIRD_OPSS_DISPATCHING); -+ hmbird_cond_deferred_err(HMBIRD_OPN, -+ atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE, -+ "task = %s\n", p->comm); -+ break; -+ } -+} -+ -+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ -+ if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p), -+ "task = %s\n", p->comm); -+ return; -+ } -+ -+ ops_dequeue(p, deq_flags); -+ -+ if (slim_walt_ctrl) { -+ if (task_current(rq, p)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ if (deq_flags & HMBIRD_DEQ_SLEEP) -+ set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else -+ clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), -+ (unsigned long *)&get_hmbird_ts(p)->flags); -+ -+ clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm); -+ hmbird_rq->nr_running--; -+ sub_nr_running(rq, 1); -+ dispatch_dequeue(hmbird_rq, p); -+} -+ -+static void yield_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ get_hmbird_ts(p)->slice = 0; -+} -+ -+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to) -+{ -+ return false; -+} -+ -+#ifdef CONFIG_SMP -+/** -+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -+ * @rq: rq to move the task into, currently locked -+ * @p: task to move -+ * @enq_flags: %HMBIRD_ENQ_* -+ * -+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller -+ * must: -+ * -+ * 1. Start with exclusive access to @p either through its DSQ lock or -+ * %HMBIRD_OPSS_DISPATCHING flag. -+ * -+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id(). -+ * -+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't -+ * deadlock with dequeue. -+ * -+ * 4. Lock @rq and the task_rq from #3. -+ * -+ * 5. Call this function. -+ * -+ * Returns %true if @p was successfully moved. %false after racing dequeue and -+ * losing. -+ */ -+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ struct rq *task_rq; -+ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * If dequeue got to @p while we were trying to lock both rq's, it'd -+ * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have -+ * updated it to different values afterwards, as this operation can't be -+ * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become -+ * raw_smp_processor_id() again before we're done. Thus, we can tell -+ * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is -+ * still raw_smp_processor_id(). -+ * -+ * See dispatch_dequeue() for the counterpart. -+ */ -+ if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id())) -+ return false; -+ -+ /* @p->rq couldn't have changed if we're still the holding cpu */ -+ task_rq = task_rq(p); -+ lockdep_assert_rq_held(task_rq); -+ deactivate_task(task_rq, p, 0); -+ set_task_cpu(p, cpu_of(rq)); -+ get_hmbird_ts(p)->sticky_cpu = cpu_of(rq); -+ -+ /* -+ * We want to pass hmbird-specific enq_flags but activate_task() will -+ * truncate the upper 32 bit. As we own @rq, we can pass them through -+ * @get_hmbird_rq(rq)->extra_enq_flags instead. -+ */ -+ hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags, -+ "task = %s\n", p->comm); -+ get_hmbird_rq(rq)->extra_enq_flags = enq_flags; -+ activate_task(rq, p, 0); -+ get_hmbird_rq(rq)->extra_enq_flags = 0; -+ -+ return true; -+} -+ -+#endif /* CONFIG_SMP */ -+ -+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu) -+{ -+ int fitable = 1; -+ -+ return fitable; -+} -+ -+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq, -+ struct hmbird_dispatch_q *dsq) -+{ -+ bool dsq_misfit; -+ int cpu = cpu_of(rq); -+ u64 task_util = 0; -+ struct cluster_ctx ctx; -+ int dsq_int = dsq_id_to_internal(dsq); -+ -+ if (!cpumask_test_cpu(cpu, iso_masks.little)) -+ return false; -+ -+ gen_cluster_ctx(&ctx, BIG); -+ dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 && -+ dsq_int <= SCHED_PROP_DEADLINE_LEVEL4); -+#ifdef CLUSTER_SEPARATE -+ /* In rescue mode, little will consume big cluster's dsq.*/ -+ dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper); -+#endif -+ if (!dsq_misfit) -+ return false; -+ -+ if (p) { -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &task_util); -+ else -+ task_util = get_hmbird_ts(p)->demand_scaled; -+ } -+ -+ if (task_util <= misfit_ds) -+ return false; -+ -+ hmbird_info_trace(":task %s can't run on cpu%d, util = %llu\n", -+ p->comm, cpu, task_util); -+ return true; -+} -+ -+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq) -+{ -+ if (!task_fits_cpu_hmbird(p, cpu_of(rq))) -+ return false; -+ -+ if (check_misfit_task_on_little(p, rq, dsq)) -+ return false; -+ -+ return likely(test_rq_online(rq)); -+} -+ -+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return; -+ -+ if (add) -+ skipn[idx]++; -+ else -+ skipn[idx] = 0; -+} -+ -+static bool skip_too_much(struct hmbird_dispatch_q *dsq) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return false; -+ -+ if (skip_num[idx] > 3) { -+ skip_num[idx] = 0; -+ return true; -+ } -+ -+ return false; -+} -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rb_node *rb_node; -+ struct rq *task_rq; -+ unsigned long flags; -+ bool moved = false; -+ struct task_struct *may_fit = NULL; -+ int skip = 0; -+ -+retry: -+ if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -+ return false; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) { -+ set_skip_num(dsq, skip_num, (bool)may_fit); -+ goto this_rq; -+ } -+ if (skip_too_much(dsq)) -+ goto remote_rq; -+ if (!may_fit) -+ may_fit = p; -+ if (++skip <= 3) -+ continue; -+ /* -+ * If the recent 3 tasks not fit, use the first one. -+ * and clear the skip, because the first one is consumed. -+ */ -+ set_skip_num(dsq, skip_num, false); -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ /* No more task, use the first may fit task.*/ -+ if (may_fit) { -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ -+ for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) -+ goto this_rq; -+ goto remote_rq; -+ } -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ return false; -+ -+this_rq: -+ /* @dsq is locked and @p is on this rq */ -+ hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo); -+ dsq->nr--; -+ hmbird_rq->local_dsq.nr++; -+ get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ -+remote_rq: -+#ifdef CONFIG_SMP -+ if (cpu_same_cluster_stat(p, rq, task_rq)) -+ slim_stats_record(MOVE_RQ_CNT, 0, 0, 0); -+ else -+ slim_stats_record(MOVE_RQ_CNT, 1, 0, 0); -+ /* -+ * @dsq is locked and @p is on a remote rq. @p is currently protected by -+ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -+ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -+ * rq lock or fail, do a little dancing from our side. See -+ * move_task_to_local_dsq(). -+ */ -+ hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id(); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ rq_unpin_lock(rq, rf); -+ double_lock_balance(rq, task_rq); -+ rq_repin_lock(rq, rf); -+ -+ moved = move_task_to_local_dsq(rq, p, 0); -+ -+ double_unlock_balance(rq, task_rq); -+#endif /* CONFIG_SMP */ -+ if (likely(moved)) { -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ } -+ may_fit = NULL; -+ goto retry; -+} -+ -+ -+static int balance_one(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf, bool local) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class; -+ -+ if (!hmbird_rq) -+ return 1; -+ -+ lockdep_assert_rq_held(rq); -+ -+ if (static_branch_unlikely(&hmbird_ops_cpu_preempt) && -+ unlikely(get_hmbird_rq(rq)->cpu_released)) { -+ /* -+ * If the previous sched_class for the current CPU was not HMBIRD, -+ * notify the BPF scheduler that it again has control of the -+ * core. This callback complements ->cpu_release(), which is -+ * emitted in hmbird_notify_pick_next_task(). -+ */ -+ get_hmbird_rq(rq)->cpu_released = false; -+ } -+ -+ if (prev_on_hmbird) -+ update_curr_hmbird(rq); -+ /* if there already are tasks to run, nothing to do */ -+ if (hmbird_rq->local_dsq.nr) -+ return 1; -+ -+ if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) { -+ slim_stats_record(GLOBAL_STAT, 1, 0, 0); -+ return 1; -+ } -+ -+ if (consume_dispatch_global(rq, rf)) -+ return 1; -+ -+ return 0; -+} -+ -+static int balance_hmbird(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf) -+{ -+ return balance_one(rq, prev, rf, true); -+} -+ -+/* -+ * output task util to systrace, only for debug mode. -+ * we can not output too many logs to systrace buffer even in debug mode -+ * only output debug-info while it exceed misfit_ds. -+ */ -+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p) -+{ -+ static DEFINE_PER_CPU(int, is_last_exceed); -+ int cpu = cpu_of(rq); -+ u64 util = 0; -+ -+ if (likely(!debug_enabled())) -+ return; -+ -+ if (!p) -+ return; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ util = uclamp_rq_util_with(rq, util, p); -+ -+ if (util >= misfit_ds) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util); -+ per_cpu(is_last_exceed, cpu) = true; -+ } else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0); -+ per_cpu(is_last_exceed, cpu) = false; -+ } else { -+ } -+} -+ -+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first) -+{ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ /* -+ * Core-sched might decide to execute @p before it is -+ * dispatched. Call ops_dequeue() to notify the BPF scheduler. -+ */ -+ ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC); -+ dispatch_dequeue(get_hmbird_rq(rq), p); -+ } -+ -+ p->se.exec_start = rq_clock_task(rq); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK); -+ } -+ -+ watchdog_unwatch_task(p, true); -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx); -+ systrace_output_cpu_ds(rq, p); -+ /* -+ * @p is getting newly scheduled or got kicked after someone updated its -+ * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird(). -+ */ -+ if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) != -+ (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) { -+ if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) -+ get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK; -+ else -+ get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK; -+ -+ sched_update_tick_dependency(rq); -+ } -+ -+ p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime; -+} -+ -+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p) -+{ -+ update_curr_hmbird(rq); -+ -+ update_dispatch_dsq_info(rq, p); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ watchdog_watch_task(rq, p); -+ -+ if (is_pipeline_task(p)) { -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1); -+ return; -+ } -+ -+ /* -+ * If we're in the pick_next_task path, balance_hmbird() should -+ * have already populated the local DSQ if there are any other -+ * available tasks. If empty, tell ops.enqueue() that @p is the -+ * only one available for this cpu. ops.enqueue() should put it -+ * on the local DSQ so that the subsequent pick_next_task_hmbird() -+ * can find the task unless it wants to trigger a separate -+ * follow-up scheduling event. -+ */ -+ if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1); -+ else -+ do_enqueue_task(rq, p, 0, -1); -+ } -+} -+ -+static struct task_struct *first_local_task(struct rq *rq) -+{ -+ struct rb_node *rb_node; -+ struct hmbird_entity *entity; -+ -+ if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) { -+ entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo, -+ struct hmbird_entity, dsq_node.fifo); -+ return entity->task; -+ } -+ -+ rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq); -+ if (rb_node) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ return entity->task; -+ } -+ return NULL; -+} -+ -+static struct task_struct *pick_next_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p; -+ -+ p = first_local_task(rq); -+ if (!p) -+ return NULL; -+ -+ if (unlikely(!get_hmbird_ts(p)->slice)) { -+ if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice) -+ hmbird_warned_zero_slice = true; -+ -+ refill_task_slice(p); -+ } -+ -+ set_next_task_hmbird(rq, p, true); -+ -+ return p; -+} -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task, -+ const struct sched_class *active) -+{ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * The callback is conceptually meant to convey that the CPU is no -+ * longer under the control of HMBIRD. Therefore, don't invoke the -+ * callback if the CPU is staying on HMBIRD, or going idle (in which -+ * case the HMBIRD scheduler has actively decided not to schedule any -+ * tasks on the CPU). -+ */ -+ if (likely(active >= &hmbird_sched_class)) -+ return; -+ -+ /* -+ * At this point we know that HMBIRD was preempted by a higher priority -+ * sched_class, so invoke the ->cpu_release() callback if we have not -+ * done so already. We only send the callback once between HMBIRD being -+ * preempted, and it regaining control of the CPU. -+ * -+ * ->cpu_release() complements ->cpu_acquire(), which is emitted the -+ * next time that balance_hmbird() is invoked. -+ */ -+ if (!get_hmbird_rq(rq)->cpu_released) -+ get_hmbird_rq(rq)->cpu_released = true; -+} -+ -+#ifdef CONFIG_SMP -+ -+static bool test_and_clear_cpu_idle(int cpu) -+{ -+ if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -+ if (cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ return true; -+ } else { -+ return false; -+ } -+} -+ -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) -+{ -+ int cpu; -+ -+ do { -+ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -+ if (cpu >= nr_cpu_ids) -+ return -EBUSY; -+ } while (!test_and_clear_cpu_idle(cpu)); -+ -+ return cpu; -+} -+ -+ -+static bool prev_cpu_misfit(int prev) -+{ -+ if (!is_partial_enabled() && is_partial_cpu(prev)) -+ return true; -+ -+ return false; -+} -+ -+static int heavy_rt_placement(struct task_struct *p, int prev) -+{ -+ struct cpumask tmp = {.bits = {0}}; -+ int cpu; -+ u64 util = 0; -+ -+ if (!rt_prio(p->prio)) -+ return -EFAULT; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ -+ if (util < misfit_ds) -+ return -EFAULT; -+ -+ if (is_partial_enabled()) -+ cpumask_or(&tmp, iso_masks.big, iso_masks.partial); -+ else -+ cpumask_copy(&tmp, iso_masks.big); -+ -+ cpu = hmbird_pick_idle_cpu(&tmp); -+ if (cpu >= 0) -+ return cpu; -+ -+ if (cpumask_test_cpu(prev, iso_masks.big) || -+ (is_partial_enabled() && is_partial_cpu(prev))) -+ return prev; -+ -+ return cpumask_first(&tmp); -+} -+ -+static int spec_task_before_pick_idle(struct task_struct *p, int prev) -+{ -+ int cpu; -+ -+ cpu = heavy_rt_placement(p, prev); -+ if (cpu >= 0) -+ return cpu; -+ return -EFAULT; -+} -+ -+static int cpumask_distribute_next(struct cpumask *mask, int *prev) -+{ -+ int p = *prev, n; -+ -+ n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1); -+ if (n < nr_cpu_ids) -+ WRITE_ONCE(*prev, n); -+ return n; -+} -+ -+static int repick_fallback_cpu(void) -+{ -+ /* -+ * partial cpu follow big cluster's Scheduling policy, -+ * simply return first bit cpu. -+ */ -+ return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev); -+} -+ -+/* -+ * Must return a valid cpu num, as this task's cpu. -+ */ -+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu, -+ struct cpumask *mask, int *prev_mask) -+{ -+ int cpu; -+ -+ cpu = hmbird_pick_idle_cpu(mask); -+ if (cpu >= 0) -+ return cpu; -+ return cpumask_distribute_next(mask, prev_mask); -+} -+ -+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type) -+{ -+ int idx; -+ struct cluster_ctx ctx; -+ -+ idx = find_idx_from_task(p); -+ if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS) -+ return false; -+ -+ gen_cluster_ctx(&ctx, type); -+ if (idx >= ctx.lower && idx < ctx.upper) -+ return true; -+ return false; -+} -+ -+static bool is_valid_cpu(int cpu) -+{ -+ return (cpu >= 0) && (cpu < nr_cpu_ids); -+} -+ -+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) -+{ -+ s32 cpu = -1; -+ struct cpumask mask = {.bits = {0}}; -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (is_valid_cpu(cpu)) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ partial_dynamic_ctrl(); -+ -+ if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ iso_masks.big, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ if (p->nr_cpus_allowed == 1) { -+ cpu = cpumask_any(p->cpus_ptr); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* For non-period global dsq, not contain pcp task. */ -+ if (task_only_blongs_to_cluster(p, LITTLE)) { -+ cpumask_copy(&mask, iso_masks.little); -+ if (unlikely(l_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &little_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ if (task_only_blongs_to_cluster(p, BIG)) { -+ cpumask_copy(&mask, iso_masks.big); -+ if (unlikely(b_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ cpu = spec_task_before_pick_idle(p, prev_cpu); -+ if (is_valid_cpu(cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* if the previous CPU is idle, dispatch directly to it */ -+ if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+ } -+ -+ cpu = hmbird_pick_idle_cpu(cpu_possible_mask); -+ if (is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ if (prev_cpu_misfit(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ cpu = repick_fallback_cpu(); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+} -+ -+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags) -+{ -+ return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags); -+} -+ -+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx) -+{ -+ set_cpus_allowed_common(p, ctx); -+} -+ -+static void reset_idle_masks(void) -+{ -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little); -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big); -+ if (is_partial_enabled()) -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial); -+ hmbird_has_idle_cpus = true; -+} -+ -+void __hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ int cpu = cpu_of(rq); -+ struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -+ -+ if (skip_update_idle()) -+ return; -+ -+ if (idle) { -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ if (!hmbird_has_idle_cpus) -+ hmbird_has_idle_cpus = true; -+ -+ /* -+ * idle_masks.smt handling is racy but that's fine as it's only -+ * for optimization and self-correcting. -+ */ -+ for_each_cpu(cpu, sib_mask) { -+ if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -+ return; -+ } -+ cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -+ } else { -+ cpumask_clear_cpu(cpu, idle_masks.cpu); -+ if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ -+ cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -+ } -+} -+ -+#else /* !CONFIG_SMP */ -+ -+static bool test_and_clear_cpu_idle(int cpu) { return false; } -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } -+static void reset_idle_masks(void) {} -+ -+#endif /* CONFIG_SMP */ -+ -+static bool check_rq_for_timeouts(struct rq *rq) -+{ -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rq_flags rf; -+ -+ rq_lock_irqsave(rq, &rf); -+ list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) { -+ unsigned long last_runnable; -+ -+ p = entity->task; -+ last_runnable = get_hmbird_ts(p)->runnable_at; -+ -+ if (unlikely(time_after(jiffies, -+ last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -+ -+ rq_unlock_irqrestore(rq, &rf); -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb", -+ p->comm, p->pid, -+ dur_ms / 1000, dur_ms % 1000, -+ get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0, -+ cpumask_pr_args(&p->cpus_mask)); -+ return 1; -+ } -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ return 0; -+} -+ -+static void hmbird_watchdog_workfn(struct work_struct *work) -+{ -+ int cpu; -+ bool timeout; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ -+ for_each_online_cpu(cpu) { -+ timeout = check_rq_for_timeouts(cpu_rq(cpu)); -+ if (unlikely(timeout)) -+ break; -+ -+ cond_resched(); -+ } -+ if (!timeout) -+ queue_delayed_work(system_unbound_wq, to_delayed_work(work), -+ hmbird_watchdog_timeout / 2); -+} -+ -+static void set_pcp_round(struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) { -+ per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round); -+ per_cpu(pcp_info, cpu).pcp_round = true; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+} -+ -+ -+/* -+ * Just for debug: output hmbird on/off state per 10s. -+ */ -+#define OUTPUT_INTVAL (msecs_to_jiffies(10 * 1000)) -+static void inform_hmbird_onoff_from_systrace(void) -+{ -+ static unsigned long __read_mostly next_print; -+ -+ if (time_before(jiffies, READ_ONCE(next_print))) -+ return; -+ -+ WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL); -+ hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio); -+} -+ -+void hmbird_notify_sched_tick(void) -+{ -+ unsigned long last_check; -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ hmbird_scheduler_tick(); -+ -+ if (!hmbird_enabled()) -+ return; -+ -+ last_check = hmbird_watchdog_timestamp; -+ if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -+ -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "watchdog failed to check in for %u.%03us", -+ dur_ms / 1000, dur_ms % 1000); -+ } -+ scan_timeout(rq); -+} -+ -+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued) -+{ -+ update_curr_hmbird(rq); -+ -+ set_pcp_round(rq); -+ -+ if (slim_walt_ctrl) -+ hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE); -+ /* -+ * While disabling, always resched and refresh core-sched timestamp as -+ * we can't trust the slice management or ops.core_sched_before(). -+ */ -+ if (hmbird_ops_disabling()) -+ get_hmbird_ts(curr)->slice = 0; -+ -+ if (!get_hmbird_ts(curr)->slice) -+ resched_curr(rq); -+ -+ inform_hmbird_onoff_from_systrace(); -+} -+ -+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg) -+{ -+ hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ get_hmbird_ts(p)->disallow = false; -+ -+ hmbird_sched_init_task(p); -+ -+ set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ return 0; -+} -+ -+static void hmbird_ops_enable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void hmbird_ops_disable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void set_task_hmbird_weight(struct task_struct *p) -+{ -+ u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -+ -+ get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight); -+} -+ -+/** -+ * refresh_hmbird_weight - Refresh a task's hmbird weight -+ * @p: task to refresh hmbird weight for -+ * -+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to -+ * enable easy access from the BPF scheduler. To keep it synchronized with the -+ * current task priority, this function should be called when a new task is -+ * created, priority is changed for a task on hmbird, and a task is switched -+ * to hmbird from other classes. -+ */ -+static void refresh_hmbird_weight(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ set_task_hmbird_weight(p); -+} -+ -+int hmbird_pre_fork(struct task_struct *p) -+{ -+ int ret = 0; -+ -+ p->android_oem_data1[HMBIRD_TS_IDX] = -+ (u64)(kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL)); -+ if (!get_hmbird_ts(p)) { -+ ret = -1; -+ goto lock; -+ } -+ -+ get_hmbird_ts(p)->dsq = NULL; -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node); -+ get_hmbird_ts(p)->flags = 0; -+ get_hmbird_ts(p)->weight = 0; -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ get_hmbird_ts(p)->holding_cpu = -1; -+ get_hmbird_ts(p)->kf_mask = 0; -+ atomic64_set(&get_hmbird_ts(p)->ops_state, 0); -+ get_hmbird_ts(p)->runnable_at = INITIAL_JIFFIES; -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+ get_hmbird_ts(p)->task = p; -+ hmbird_set_sched_prop(p, 0); -+ -+ get_hmbird_ts(p)->critical_affinity_cpu = -1; -+ get_hmbird_ts(p)->sched_class = &hmbird_sched_class; -+ -+ /* -+ * BPF scheduler enable/disable paths want to be able to iterate and -+ * update all tasks which can become complex when racing forks. As -+ * enable/disable are very cold paths, let's use a percpu_rwsem to -+ * exclude forks. -+ */ -+lock: -+ percpu_down_read(&hmbird_fork_rwsem); -+ -+ return ret; -+} -+ -+int hmbird_fork(struct task_struct *p) -+{ -+ percpu_rwsem_assert_held(&hmbird_fork_rwsem); -+ -+ if (hmbird_enabled()) -+ return hmbird_ops_prepare_task(p, task_group(p)); -+ else -+ return 0; -+} -+ -+void hmbird_post_fork(struct task_struct *p) -+{ -+ if (hmbird_enabled()) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ /* -+ * Set the weight manually before calling ops.enable() so that -+ * the scheduler doesn't see a stale value if they inspect the -+ * task struct. We'll invoke ops.set_weight() afterwards, as it -+ * would be odd to receive a callback on the task before we -+ * tell the scheduler that it's been fully enabled. -+ */ -+ set_task_hmbird_weight(p); -+ hmbird_ops_enable_task(p); -+ refresh_hmbird_weight(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ -+ spin_lock_irq(&hmbird_tasks_lock); -+ list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_cancel_fork(struct task_struct *p) -+{ -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ if (hmbird_enabled()) -+ hmbird_ops_disable_task(p); -+ -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_free(struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+ list_del_init(&get_hmbird_ts(p)->tasks_node); -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+ -+ /* -+ * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED -> -+ * ENABLED transitions can't race us. Disable ops for @p. -+ */ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) || -+ test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ hmbird_ops_disable_task(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+} -+ -+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio) -+{ -+} -+ -+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type) -+{ -+ return (prop >> TOP_TASK_SHIFT) & (1 << type); -+} -+ -+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p) -+{ -+ uint32_t prop = get_top_task_prop(p); -+ -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE)) -+ return HMBIRD_TASK_PROP_PIPELINE; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) || -+ !task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) { -+ return HMBIRD_TASK_PROP_COMMON; -+ } -+ return HMBIRD_TASK_PROP_DEBUG_OR_LOG; -+} -+ -+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b) -+{ -+ int type_a = hmbird_get_task_type(a); -+ int type_b = hmbird_get_task_type(b); -+ -+ return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b]; -+} -+ -+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ enum cpu_type type; -+ int sp_dl; -+ struct task_struct *curr = NULL; -+ -+ switch (hmbird_preempt_policy) { -+ case HMBIRD_PREEMPT_POLICY_PRIO_BASED: -+ curr = rq->curr; -+ if (curr && hmbird_prio_higher(p, curr)) -+ goto preempt; -+ break; -+ default: -+ break; -+ } -+ if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) || -+ (is_critical_system_task(p) && !is_critical_system_task(rq->curr))) -+ goto preempt; -+ -+ sp_dl = find_idx_from_task(p); -+ if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled())) -+ return; -+ -+ if (rq->curr->prio > p->prio) -+ goto preempt; -+ -+ return; -+ -+preempt: -+ resched_curr(rq); -+} -+ -+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {} -+ -+int hmbird_check_setscheduler(struct task_struct *p, int policy) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ /* if disallow, reject transitioning into HMBIRD */ -+ if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) && -+ p->policy != policy && policy == SCHED_HMBIRD) -+ return -EACCES; -+ -+ return 0; -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool hmbird_can_stop_tick(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ if (hmbird_ops_disabling()) -+ return false; -+ -+ if (p->sched_class != &hmbird_sched_class) -+ return true; -+ -+ return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK; -+} -+#endif -+ -+int hmbird_tg_online(struct task_group *tg) -+{ -+ struct cgroup *cgrp; -+ -+ if (!tg) -+ return 0; -+ cgrp = tg->css.cgroup; -+ if (!cgrp || !(cgrp->kn)) -+ return 0; -+ update_cgroup_ids_table(cgrp->kn->id, -1); -+ return 0; -+} -+ -+/* -+ * Omitted operations: -+ * -+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -+ * task isn't tied to the CPU at that point. Preemption is implemented by -+ * resetting the victim task's slice to 0 and triggering reschedule on the -+ * target CPU. -+ * -+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -+ * -+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -+ * their current sched_class. Call them directly from sched core instead. -+ * -+ * - task_woken, switched_from: Unnecessary. -+ */ -+DEFINE_SCHED_CLASS(hmbird) = { -+ .enqueue_task = enqueue_task_hmbird, -+ .dequeue_task = dequeue_task_hmbird, -+ .yield_task = yield_task_hmbird, -+ .yield_to_task = yield_to_task_hmbird, -+ -+ .check_preempt_curr = check_preempt_curr_hmbird, -+ -+ .pick_next_task = pick_next_task_hmbird, -+ -+ .put_prev_task = put_prev_task_hmbird, -+ .set_next_task = set_next_task_hmbird, -+ -+#ifdef CONFIG_SMP -+ .balance = balance_hmbird, -+ .select_task_rq = select_task_rq_hmbird, -+ .set_cpus_allowed = set_cpus_allowed_hmbird, -+#endif -+ .task_tick = task_tick_hmbird, -+ -+ .switched_to = switched_to_hmbird, -+ .prio_changed = prio_changed_hmbird, -+ -+ .update_curr = update_curr_hmbird, -+ -+#ifdef CONFIG_UCLAMP_TASK -+ .uclamp_enabled = 0, -+#endif -+}; -+ -+/* -+ * Must with rq lock held. -+ */ -+bool task_is_hmbird(struct task_struct *p) -+{ -+ return p->sched_class == &hmbird_sched_class; -+} -+ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id) -+{ -+ memset(dsq, 0, sizeof(*dsq)); -+ -+ raw_spin_lock_init(&dsq->lock); -+ INIT_LIST_HEAD(&dsq->fifo); -+ dsq->id = dsq_id; -+} -+ -+static int hmbird_cgroup_init(void) -+{ -+ struct cgroup_subsys_state *css; -+ -+ css_for_each_descendant_pre(css, &root_task_group.css) { -+ struct task_group *tg = css_tg(css); -+ -+ cgrp_dsq_idx_init(css->cgroup, tg); -+ } -+ return 0; -+} -+ -+/* -+ * Used by sched_fork() and __setscheduler_prio() to pick the matching -+ * sched_class. dl/rt are already handled. -+ */ -+bool task_on_hmbird(struct task_struct *p) -+{ -+ return hmbird_enabled(); -+} -+ -+static void __setscheduler_prio(struct task_struct *p, int prio) -+{ -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) { -+ p->prio = prio; -+ return; -+ } else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (on_hmbird && rt_prio(prio)) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ -+ p->prio = prio; -+} -+ -+/* -+ * Heartbeat, avoid humbird keep running while APP already exit. -+ * Check whether APP send alive-signal periodly. -+ */ -+#define HEARTBEAT_TIMEOUT (msecs_to_jiffies(2500)) -+#define HEARTBEAT_CHECK_INTERVAL (msecs_to_jiffies(1000)) -+static struct timer_list hb_timer; -+static unsigned long next_hb; -+ -+void hb_timer_handler(struct timer_list *timer) -+{ -+ if (!heartbeat_enable) -+ goto refill; -+ -+ pr_err(": enter timer.\n"); -+ if (heartbeat) { -+ heartbeat = 0; -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ } -+ -+ /* can't detect heartbeat, disable ext. */ -+ if (time_after(jiffies, READ_ONCE(next_hb))) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT, -+ "can't detect heartbeat, disable ext\n"); -+ } -+refill: -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_start(void) -+{ -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_init(void) -+{ -+ timer_setup(&hb_timer, hb_timer_handler, 0); -+} -+ -+static void hb_timer_exit(void) -+{ -+ del_timer(&hb_timer); -+} -+ -+static bool check_and_disable_cpuhp(void) -+{ -+ struct rq *rq; -+ struct rq_flags rf; -+ int cpu; -+ -+ cpu_hotplug_disable(); -+ cpus_read_lock(); -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ rq_lock_irqsave(rq, &rf); -+ if (!rq->online) { -+ rq_unlock_irqrestore(rq, &rf); -+ goto err_offline; -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ cpus_read_unlock(); -+ return true; -+ -+err_offline: -+ cpus_read_unlock(); -+ cpu_hotplug_enable(); -+ return false; -+} -+ -+static void reenable_cpuhp(void) -+{ -+ cpu_hotplug_enable(); -+} -+ -+static void scheduler_switch_done(bool final_state) -+{ -+ /* set scx_enable again, switch may fail. */ -+ scx_enable = final_state; -+ /* reset heartbeat*/ -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ -+ if (final_state) -+ hb_timer_start(); -+ else -+ hb_timer_exit(); -+} -+ -+/** -+ * ss : curr switch state -+ * finish : success or fail -+ * enable : curr operation is diabling or enabling? -+ * fail_reason : string output when failed, empty string when success. -+ */ -+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason) -+{ -+ char *s1 = finish ? "finished" : "failed"; -+ char *s2 = enable ? "enabled" : "disabled"; -+ -+ hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss); -+ if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) { -+ sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type)); -+ hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n", -+ s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason); -+ } -+ curr_ss = ss; -+} -+ -+bool get_hmbird_ops_enabled(void) -+{ -+ return atomic_read(&__hmbird_ops_enabled); -+} -+ -+bool get_non_hmbird_task(void) -+{ -+ return atomic_read(&non_hmbird_task); -+} -+ -+static void hmbird_ops_disable_workfn(struct kthread_work *work) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int cpu; -+ -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, ""); -+ cancel_delayed_work_sync(&hmbird_watchdog_work); -+ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) { -+ case HMBIRD_OPS_DISABLED: -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled"); -+ scheduler_switch_done(false); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return; -+ case HMBIRD_OPS_PREPPING: -+ fallthrough; -+ case HMBIRD_OPS_DISABLING: -+ /* shouldn't happen but handle it like ENABLING if it does */ -+ WARN_ONCE(true, "hmbird: duplicate disabling instance?"); -+ fallthrough; -+ case HMBIRD_OPS_ENABLING: -+ case HMBIRD_OPS_ENABLED: -+ break; -+ } -+ -+ /* kick all CPUs to restore ticks */ -+ for_each_possible_cpu(cpu) -+ resched_cpu(cpu); -+ -+ /* avoid racing against fork and cgroup changes */ -+ cpus_read_lock(); -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, ""); -+ spin_lock_irq(&hmbird_tasks_lock); -+ atomic_set(&__hmbird_ops_enabled, false); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ bool alive = READ_ONCE(p->__state) != TASK_DEAD; -+ -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ get_hmbird_ts(p)->slice = min_t(u64, -+ get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL); -+ -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ if (alive) -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ -+ hmbird_ops_disable_task(p); -+ } -+ hmbird_task_iter_exit(&sti); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, ""); -+ -+ atomic_set(&non_hmbird_task, true); -+ /* no task is on hmbird, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt); -+ synchronize_rcu(); -+ -+ percpu_up_write(&hmbird_fork_rwsem); -+ cpus_read_unlock(); -+ -+ if (slim_walt_ctrl) -+ slim_walt_enable(false); -+ -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 1, 0, ""); -+ scheduler_switch_done(false); -+ reenable_cpuhp(); -+} -+ -+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn); -+ -+static void schedule_hmbird_ops_disable_work(void) -+{ -+ struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper); -+ -+ /* -+ * We may be called spuriously before the first bpf_hmbird_reg(). If -+ * hmbird_ops_helper isn't set up yet, there's nothing to do. -+ */ -+ if (helper) -+ kthread_queue_work(helper, &hmbird_ops_disable_work); -+} -+ -+static void hmbird_ops_disable(void) -+{ -+ schedule_hmbird_ops_disable_work(); -+} -+ -+static void hmbird_err_exit_workfn(struct work_struct *work) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ hmbird_ctrl(false); -+ -+ for_each_present_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy) -+ continue; -+ if (cpu != policy->cpu) -+ goto put; -+ down_write(&policy->rwsem); -+ WARN_ON(store_scaling_governor(policy, -+ saved_gov[cpu], strlen(saved_gov[cpu])) <= 0); -+ up_write(&policy->rwsem); -+ hmbird_info_trace(":restore origin gov : %s\n", saved_gov[cpu]); -+put: -+ cpufreq_cpu_put(policy); -+ } -+ memset((char *)saved_gov, 0, sizeof(saved_gov)); -+} -+ -+void hmbird_ops_exit(void) -+{ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); -+} -+ -+static struct kthread_worker *hmbird_create_rt_helper(const char *name) -+{ -+ struct kthread_worker *helper; -+ -+ helper = kthread_create_worker(0, name); -+ if (helper) -+ sched_set_fifo(helper->task); -+ return helper; -+} -+ -+static inline void set_audio_thread_sched_prop(struct task_struct *p) -+{ -+ struct cgroup_subsys_state *css; -+ -+ if (likely(p->prio >= MAX_RT_PRIO)) -+ return; -+ -+ rcu_read_lock(); -+ css = task_css(p, cpuset_cgrp_id); -+ if (!css) { -+ rcu_read_unlock(); -+ return; -+ } -+ rcu_read_unlock(); -+ -+ if (!strcmp(css->cgroup->kn->name, "audio-app")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+} -+ -+static int hmbird_ops_enable(void *unused) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int ret; -+ int tcnt = 0; -+ unsigned long long start = 0; -+ -+ if (!check_and_disable_cpuhp()) { -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline"); -+ return -EBUSY; -+ } -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, ""); -+ mutex_lock(&hmbird_ops_enable_mutex); -+ -+ if (!hmbird_ops_helper) { -+ WRITE_ONCE(hmbird_ops_helper, -+ hmbird_create_rt_helper("hmbird_ops_helper")); -+ if (!hmbird_ops_helper) { -+ ret = -ENOMEM; -+ goto err_unlock; -+ } -+ } -+ -+ if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) { -+ ret = -EBUSY; -+ goto err_unlock; -+ } -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) != -+ HMBIRD_OPS_DISABLED); -+ -+ hmbird_warned_zero_slice = false; -+ -+ atomic64_set(&hmbird_nr_rejected, 0); -+ -+ /* -+ * Keep CPUs stable during enable so that the BPF scheduler can track -+ * online CPUs by watching ->on/offline_cpu() after ->init(). -+ */ -+ cpus_read_lock(); -+ -+ hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work, -+ hmbird_watchdog_timeout / 2); -+ -+ /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ */ -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ reset_idle_masks(); -+ -+ /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. -+ */ -+ ret = hmbird_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock; -+ -+ /* -+ * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem -+ * preventing new tasks from being added. No need to exclude tasks -+ * leaving as hmbird_free() can handle both prepped and enabled -+ * tasks. Prep all tasks first and then enable them with preemption -+ * disabled. -+ */ -+ spin_lock_irq(&hmbird_tasks_lock); -+ -+ atomic_set(&non_hmbird_task, false); -+ atomic_set(&__hmbird_ops_enabled, true); -+ -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered(&sti))) -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ hmbird_task_iter_exit(&sti); -+ -+ /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. -+ */ -+ preempt_disable(); -+ -+ /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ */ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) { -+ atomic_set(&non_hmbird_task, true); -+ atomic_set(&__hmbird_ops_enabled, false); -+ preempt_enable(); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ ret = -EBUSY; -+ goto err_disable_unlock; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED -+ * transitions here are synchronized against hmbird_free() through -+ * hmbird_tasks_lock. -+ */ -+ start = sched_clock(); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, ""); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ tcnt++; -+ if (READ_ONCE(p->__state) != TASK_DEAD) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ -+ set_audio_thread_sched_prop(p); -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ hmbird_ops_enable_task(p); -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ } else { -+ hmbird_ops_disable_task(p); -+ } -+ } -+ hmbird_task_iter_exit(&sti); -+ -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, ""); -+ preempt_enable(); -+ percpu_up_write(&hmbird_fork_rwsem); -+ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) { -+ ret = -EBUSY; -+ goto err_disable; -+ } -+ -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_ENABLED, 1, 1, ""); -+ scheduler_switch_done(true); -+ -+ return 0; -+ -+err_unlock: -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock"); -+ scheduler_switch_done(false); -+ return ret; -+ -+err_disable_unlock: -+ percpu_up_write(&hmbird_fork_rwsem); -+err_disable: -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ /* must be fully disabled before returning */ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable"); -+ scheduler_switch_done(false); -+ return ret; -+} -+ -+#ifdef CONFIG_SCHED_DEBUG -+static const char *hmbird_ops_enable_state_str[] = { -+ [HMBIRD_OPS_PREPPING] = "prepping", -+ [HMBIRD_OPS_ENABLING] = "enabling", -+ [HMBIRD_OPS_ENABLED] = "enabled", -+ [HMBIRD_OPS_DISABLING] = "disabling", -+ [HMBIRD_OPS_DISABLED] = "disabled", -+}; -+ -+static int hmbird_debug_show(struct seq_file *m, void *v) -+{ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled()); -+ seq_printf(m, "%-30s: %s\n", "enable_state", -+ hmbird_ops_enable_state_str[hmbird_ops_enable_state()]); -+ seq_printf(m, "%-30s: %llu\n", "nr_rejected", -+ atomic64_read(&hmbird_nr_rejected)); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return 0; -+} -+ -+static int hmbird_debug_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_debug_show, NULL); -+} -+ -+const struct file_operations sched_hmbird_fops = { -+ .open = hmbird_debug_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif -+ -+ -+static int bpf_hmbird_reg(void *kdata) -+{ -+ return hmbird_ops_enable(kdata); -+} -+ -+static int bpf_hmbird_unreg(void *kdata) -+{ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ return 0; -+} -+ -+void set_hmbird_module_loaded(int is_loaded) -+{ -+ atomic_set(&hmbird_module_loaded, is_loaded); -+} -+ -+/* -+ * MUST load hmbird module before enable hmbird scheduler -+ * load track & hmbird gover implemented in hmbird module -+ */ -+int hmbird_ctrl(bool enable) -+{ -+ if (!atomic_read(&hmbird_module_loaded)) { -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "ext module unloaded\n"); -+ return -EINVAL; -+ } -+ -+ if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED -+ || hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING -+ || hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already enabled(ing)\n"); -+ hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING || -+ hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already disabled(ing)\n"); -+ hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (enable) -+ return bpf_hmbird_reg(NULL); -+ else -+ return bpf_hmbird_unreg(NULL); -+} -+ -+void set_cpu_isomask(int cpu, cpumask_var_t *mask) -+{ -+ cpumask_clear_cpu(cpu, iso_masks.ex_free); -+ cpumask_clear_cpu(cpu, iso_masks.exclusive); -+ cpumask_clear_cpu(cpu, iso_masks.partial); -+ cpumask_clear_cpu(cpu, iso_masks.big); -+ cpumask_clear_cpu(cpu, iso_masks.little); -+ cpumask_set_cpu(cpu, *mask); -+} -+ -+void set_cpu_cluster(u64 cpu_cluster) -+{ -+ int cpu; -+ -+ for_each_present_cpu(cpu) { -+ u64 pos = 1 << cpu; -+ -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.ex_free)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.exclusive)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.partial)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.big)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) -+ set_cpu_isomask(cpu, &(iso_masks.little)); -+ } -+} -+ -+static void get_hmbird_snapshot(struct panic_snapshot_t *p) -+{ -+ struct hmbird_dispatch_q *dsq; -+ struct rq *rq; -+ int cpu, i; -+ -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ p->rq_nr[cpu] = rq->nr_running; -+ p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running; -+ } -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ struct hmbird_entity *entity; -+ -+ dsq = &gdsqs[i]; -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo)) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ p->runnable_at[i] = entity->runnable_at; -+ raw_spin_unlock(&dsq->lock); -+ -+ } -+ -+ p->snap_misc.hmbird_enabled = hmbird_enabled(); -+ p->snap_misc.curr_ss = curr_ss; -+ p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state(); -+ p->snap_misc.parctrl_high_ratio = parctrl_high_ratio; -+ p->snap_misc.parctrl_low_ratio = parctrl_low_ratio; -+ p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l; -+ p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l; -+ p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio; -+ p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio; -+ p->snap_misc.misfit_ds = misfit_ds; -+ p->snap_misc.partial_enable = partial_enable; -+ p->snap_misc.iso_free_rescue = iso_free_rescue; -+ p->snap_misc.isolate_ctrl = isolate_ctrl; -+ p->snap_misc.snap_jiffies = jiffies; -+ p->snap_misc.snap_time = local_clock(); -+} -+ -+// MTK minidump begin -+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3) -+{ -+ strscpy(m->desc_str, str, DESC_STR_LEN); -+ m->len = d1 * d2 * d3; -+ m->parse[0] = d1; -+ m->parse[1] = d2; -+ m->parse[2] = d3; -+} -+ -+static void init_desc_metas(struct md_info_t *m) -+{ -+ init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :", -+ 1, MAX_SWITCHS, SWITCH_ITEMS); -+ -+ init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1); -+ -+ init_desc_meta(&m->kern_dump.excep_rec_meta, -+ "excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS); -+ -+ init_desc_meta(&m->kern_dump.excep_idx_meta, -+ "excep idx :", 1, 1, MAX_EXCEP_ID); -+ -+ init_desc_meta(&m->kern_dump.snap.runnable_at_meta, -+ "each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS); -+ -+ init_desc_meta(&m->kern_dump.snap.rq_nr_meta, -+ "rq runnable task nr:", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta, -+ "scxrq runnable task nr :", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.snap_misc_meta, -+ "misc snap params :", 1, 1, SNAP_ITEMS); -+} -+ -+static void init_md_meta(struct md_info_t *m) -+{ -+ m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64); -+ m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64); -+ m->meta.unit_size = sizeof(u64); -+ m->meta.switches = MAX_SWITCHS; -+ m->meta.exceps = MAX_EXCEPS; -+ m->meta.global_dsqs = MAX_GLOBAL_DSQS; -+ m->meta.parse_dimens = PARSE_DIMENS; -+ m->meta.nr_cpus = num_possible_cpus(); -+ m->meta.real_cpus = nr_cpu_ids; -+ m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64); -+ m->meta.nr_meta_desc = 8; -+ m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64); -+ -+ init_desc_metas(m); -+} -+ -+#define MINIDUMP_DFL_SIZE (4 * 1024) -+ -+struct notifier_block hmbird_panic_blk; -+static int hmbird_panic_handler(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ if (!md_info) -+ return NOTIFY_DONE; -+ -+ get_hmbird_snapshot(&md_info->kern_dump.snap); -+ -+ return NOTIFY_DONE; -+} -+ -+static void panic_blk_init(void) -+{ -+ int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE); -+ -+ md_info = kzalloc(dump_size, GFP_KERNEL); -+ if (!md_info) -+ return; -+ init_md_meta(md_info); -+ -+ hmbird_panic_blk.notifier_call = hmbird_panic_handler; -+ /* make sure to execute before minidump. */ -+ hmbird_panic_blk.priority = INT_MAX; -+ atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk); -+ -+ hmbird_debug("register minidump.\n"); -+} -+ -+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size) -+{ -+ *vaddr = (unsigned long)md_info; -+ *size = sizeof(struct md_info_t); -+} -+// MTK minidump end -+ -+static inline void init_sched_prop_to_preempt_prio(void) -+{ -+ for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) { -+ switch (i) { -+ case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 5; -+ break; -+ -+ case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 4; -+ break; -+ -+ case HMBIRD_TASK_PROP_PIPELINE: -+ case HMBIRD_TASK_PROP_ISOLATE: -+ sched_prop_to_preempt_prio[i] = 3; -+ break; -+ -+ case HMBIRD_TASK_PROP_COMMON: -+ sched_prop_to_preempt_prio[i] = 1; -+ break; -+ -+ case HMBIRD_TASK_PROP_DEBUG_OR_LOG: -+ sched_prop_to_preempt_prio[i] = 0; -+ break; -+ -+ default: -+ sched_prop_to_preempt_prio[i] = 2; -+ break; -+ } -+ } -+} -+ -+void __init init_sched_hmbird_class(void) -+{ -+ int cpu; -+ u32 v; -+ struct hmbird_entity *init_hmbird; -+ -+ /* -+ * The following is to prevent the compiler from optimizing out the enum -+ * definitions so that BPF scheduler implementations can use them -+ * through the generated vmlinux.h. -+ */ -+ WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP | -+ HMBIRD_KICK_PREEMPT); -+ -+ init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL); -+ init_dsq_at_boot(); -+ init_isolate_cpus(); -+ hb_timer_init(); -+ init_sched_prop_to_preempt_prio(); -+#ifdef CONFIG_SMP -+ WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); -+#endif -+ -+ /* -+ * we can't static init init_task's hmbird struct, init here. -+ * init_task->hmbird would not use during boot. -+ */ -+ init_hmbird = kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL); -+ init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird; -+ if (init_hmbird) { -+ INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo); -+ INIT_LIST_HEAD(&init_hmbird->watchdog_node); -+ init_hmbird->sticky_cpu = -1; -+ init_hmbird->holding_cpu = -1; -+ atomic64_set(&init_hmbird->ops_state, 0); -+ init_hmbird->runnable_at = jiffies; -+ init_hmbird->slice = HMBIRD_SLICE_DFL; -+ init_hmbird->task = &init_task; -+ hmbird_set_sched_prop(&init_task, 0); -+ } else { -+ hmbird_err(INIT_TASK_FAIL, ":alloc init_task.scx failed!!!\n"); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ /* -+ * exec during boot phase, no need to care about alloc failed. -+ * lifecycle same to rq, no need to free. -+ */ -+ rq->android_oem_data1[HMBIRD_RQ_IDX] = -+ (u64)kmalloc(sizeof(struct hmbird_rq), GFP_KERNEL); -+ if (get_hmbird_rq(rq)) { -+ get_hmbird_rq(rq)->rq = rq; -+ get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window; -+ } else { -+ hmbird_err(ALLOC_RQSCX_FAIL, ":alloc rq->scx failed!!!\n"); -+ } -+ -+ rq->android_oem_data1[HMBIRD_OPS_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL); -+ if (!get_hmbird_ops(rq)) -+ pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n"); -+ init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL); -+ INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list); -+ -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL)); -+ -+ hmbird_ops_init(get_hmbird_ops(rq)); -+ } -+ -+ INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn); -+ INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn); -+ hmbird_misc_init(); -+ -+ panic_blk_init(); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_misc.c b/kernel/sched/hmbird/hmbird_misc.c -new file mode 100755 -index 000000000000..52582c22052c ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_misc.c -@@ -0,0 +1,124 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * @Description: -+ * @Version: -+ * @Author: wangrui8 -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include "hmbird_sched.h" -+#include -+#include -+#include "slim.h" -+ -+noinline int tracing_mark_write(const char *buf) -+{ -+ trace_printk(buf); -+ return 0; -+} -+ -+struct yield_opt_params yield_opt_params = { -+ .enable = 0, -+ .frame_per_sec = 120, -+ .frame_time_ns = NSEC_PER_SEC / 120, -+ .yield_headroom = 10, -+}; -+ -+DEFINE_PER_CPU(struct sched_yield_state, ystate); -+ -+static inline void hmbird_yield_state_update(struct sched_yield_state *ys) -+{ -+ if (!raw_spin_is_locked(&ys->lock)) -+ return; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ -+ if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1 -+ || ys->yield_cnt_after_sleep > yield_headroom) { -+ ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP); -+ } else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) { -+ ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP); -+ } -+ ys->yield_cnt = 0; -+ ys->sleep_times = 0; -+ ys->yield_cnt_after_sleep = 0; -+} -+ -+void hmbird_skip_yield(long *skip) -+{ -+ if (!get_hmbird_ops_enabled() || !yield_opt_params.enable) -+ return; -+ unsigned long flags, sleep_now = 0; -+ struct sched_yield_state *ys; -+ int cpu = raw_smp_processor_id(), cont_yield, new_frame; -+ int frame_time_ns = yield_opt_params.frame_time_ns; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ u64 wc; -+ -+ if (!(*skip)) { -+ wc = sched_clock(); -+ ys = &per_cpu(ystate, cpu); -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ -+ cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP; -+ new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1); -+ -+ if (!cont_yield && new_frame) { -+ hmbird_yield_state_update(ys); -+ ys->last_update_time = wc; -+ ys->sleep_end = ys->last_yield_time + frame_time_ns -+ - yield_headroom * YIELD_DURATION; -+ } -+ -+ if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) { -+ *skip = true; -+ -+ sleep_now = ys->sleep_times ? -+ max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep; -+ if (wc + sleep_now > ys->sleep_end) { -+ u64 delta = ys->sleep_end - wc; -+ -+ if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION) -+ sleep_now = delta; -+ else -+ sleep_now = 0; -+ } -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ if (sleep_now) { -+ sleep_now = div64_u64(sleep_now, 1000); -+ usleep_range_state(sleep_now, sleep_now, TASK_IDLE); -+ } -+ ys->sleep_times++; -+ ys->last_yield_time = sched_clock(); -+ return; -+ } -+ if (ys->sleep_times) -+ ys->yield_cnt_after_sleep++; -+ else -+ (ys->yield_cnt)++; -+ ys->last_yield_time = wc; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+} -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops) -+{ -+ hmbird_ops->scx_enable = get_hmbird_ops_enabled; -+ hmbird_ops->check_non_task = get_non_hmbird_task; -+ hmbird_ops->hmbird_get_md_info = hmbird_get_md_info; -+} -+ -+void hmbird_misc_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_init(&ys->lock); -+ } -+ -+ set_hmbird_module_loaded(1); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_sched.h b/kernel/sched/hmbird/hmbird_sched.h -new file mode 100755 -index 000000000000..6b5ef43cb827 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef __HMBIRD_SCHED__ -+#define __HMBIRD_SCHED__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include "../../time/tick-sched.h" -+#include "../../sched/sched.h" -+#include -+#include -+#include -+ -+#define REGISTER_TRACE_VH(vender_hook, handler) \ -+{ \ -+ ret = register_trace_##vender_hook(handler, NULL); \ -+ if (ret) { \ -+ pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \ -+ } \ -+} -+#define REGISTER_TRACE(vendor_hook, handler, data, err) \ -+do { \ -+ ret = register_trace_##vendor_hook(handler, data); \ -+ if (ret) { \ -+ pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret); \ -+ goto err; \ -+ } \ -+} while (0) -+ -+#define UNREGISTER_TRACE(vendor_hook, handler, data) \ -+ unregister_trace_##vendor_hook(handler, data) \ -+ -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+ -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int sched_ravg_window_frame_per_sec; -+extern int slim_gov_debug; -+extern int cpu7_tl; -+extern int scx_gov_ctrl; -+extern spinlock_t new_sched_ravg_window_lock; -+extern int cluster_separate; -+ -+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER BIT(31) -+#define MAX_YIELD_SLEEP (2000000ULL) -+#define MIN_YIELD_SLEEP (200000ULL) -+#define YIELD_DURATION (5000ULL) -+#define DEFAULT_YIELD_SLEEP_TH (10) -+ -+struct sched_yield_state { -+ raw_spinlock_t lock; -+ u64 last_yield_time; -+ u64 last_update_time; -+ u64 sleep_end; -+ unsigned long yield_cnt; -+ unsigned long yield_cnt_after_sleep; -+ unsigned long sleep; -+ int sleep_times; -+}; -+ -+DECLARE_PER_CPU(struct sched_yield_state, ystate); -+ -+void hmbird_window_rollover_run_once(struct rq *rq); -+void hmbird_yield_state_update_per_frame(void); -+void hmbird_misc_init(void); -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops); -+#endif /*__HMBIRD_SCHED__*/ -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.c b/kernel/sched/hmbird/hmbird_sched_proc.c -new file mode 100755 -index 000000000000..234768fc767a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.c -@@ -0,0 +1,527 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "hmbird_sched_proc.h" -+#include -+ -+#include "hmbird_util_track.h" -+#include "slim.h" -+ -+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched" -+#define SLIM_FREQ_GOV_DIR "slim_freq_gov" -+#define LOAD_TRACK_DIR "slim_walt" -+#define HMBIRD_PROC_PERMISSION 0666 -+ -+int scx_enable; -+int partial_enable; -+int cpuctrl_high_ratio = 55; -+int cpuctrl_low_ratio = 40; -+int slim_stats; -+int hmbirdcore_debug; -+int slim_for_app; -+int misfit_ds = 90; -+unsigned int highres_tick_ctrl; -+unsigned int highres_tick_ctrl_dbg; -+int cpu7_tl = 70; -+int slim_walt_ctrl; -+int slim_walt_dump; -+int slim_walt_policy; -+int slim_gov_debug; -+int scx_gov_ctrl = 1; -+int sched_ravg_window_frame_per_sec = 125; -+int parctrl_high_ratio = 55; -+int parctrl_low_ratio = 40; -+int parctrl_high_ratio_l = 65; -+int parctrl_low_ratio_l = 50; -+int isoctrl_high_ratio = 75; -+int isoctrl_low_ratio = 60; -+int isolate_ctrl; -+int iso_free_rescue; -+int heartbeat; -+int heartbeat_enable = 1; -+int watchdog_enable; -+int save_gov; -+u64 cpu_cluster_masks; -+int hmbird_preempt_policy; -+int cluster_separate; -+ -+char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= 32) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtoint(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoint\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+static int set_proc_buf_val_u64(struct file *file, const char __user *buf, -+ size_t count, u64 *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= sizeof(kbuf)) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtou64(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoul\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/* common ops begin */ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ return count; -+} -+ -+static int hmbird_common_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%d\n", *(int *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_show, pde_data(inode)); -+} -+ -+static int hmbird_common_ul_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%lu\n", *(unsigned long *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_ul_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_ul_show, pde_data(inode)); -+} -+ -+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write); -+/* common ops end */ -+ -+/* scx_enable ops begin */ -+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC); -+ if (hmbird_ctrl(*pval)) -+ return -EFAULT; -+ -+ return count; -+} -+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write); -+/* scx_enable ops end */ -+ -+/* hmbird_stats ops begin */ -+#define MAX_STATS_BUF (4096) -+static int hmbird_stats_proc_show(struct seq_file *m, void *v) -+{ -+ char *buf; -+ -+ buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ stats_print(buf, MAX_STATS_BUF); -+ -+ seq_printf(m, "%s\n", buf); -+ -+ kfree(buf); -+ return 0; -+} -+ -+static int hmbird_stats_proc_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_stats_proc_show, inode); -+} -+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL); -+/* hmbird_stats ops end */ -+ -+/* sched_ravg_window_frame_per_sec ops begin */ -+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file, -+ const char __user *buf, size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ sched_ravg_window_change(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open, -+ sched_ravg_window_frame_per_sec_proc_write); -+/* sched_ravg_window_frame_per_sec ops end */ -+ -+static ssize_t save_gov_str(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ for_each_possible_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy || (cpu != policy->cpu)) -+ continue; -+ WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0); -+ hmbird_info_systrace(":save origin gov : %s\n", saved_gov[cpu]); -+ } -+ return count; -+} -+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str); -+ -+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ u64 *pval = (u64 *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val_u64(file, buf, count, pval)) -+ return -EFAULT; -+ -+ if (scx_enable == 0) -+ set_cpu_cluster(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write); -+ -+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ int tmp_val; -+ -+ if (set_proc_buf_val(file, buf, count, &tmp_val)) -+ return -EFAULT; -+ -+ slim_walt_enable(tmp_val); -+ *pval = tmp_val; -+ -+ return count; -+} -+ -+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write); -+ -+/* yield_opt ops begin */ -+static int yield_opt_show(struct seq_file *m, void *v) -+{ -+ struct yield_opt_params *data = m->private; -+ -+ seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n", -+ data->enable, data->frame_per_sec, data->yield_headroom); -+ return 0; -+} -+ -+static int yield_opt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, yield_opt_show, pde_data(inode)); -+} -+ -+static ssize_t yield_opt_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu; -+ unsigned long flags; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if (copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp -+ != 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) || -+ (yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp; -+ yield_opt_params.frame_per_sec = frame_per_sec_tmp; -+ yield_opt_params.yield_headroom = yield_headroom_tmp; -+ yield_opt_params.enable = enable_tmp; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ ys->last_yield_time = 0; -+ ys->last_update_time = 0; -+ ys->sleep_end = 0; -+ ys->yield_cnt = 0; -+ ys->yield_cnt_after_sleep = 0; -+ ys->sleep = 0; -+ ys->sleep_times = 0; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write); -+ -+ -+static int __init hmbird_proc_init(void) -+{ -+ struct proc_dir_entry *hmbird_dir; -+ struct proc_dir_entry *load_track_dir; -+ struct proc_dir_entry *freq_gov_dir; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ -+ /* mkdir /proc/hmbird_sched */ -+ hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL); -+ if (!hmbird_dir) { -+ pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &scx_enable_proc_ops, -+ &scx_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &partial_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_stats); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbirdcore_debug); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_for_app); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &misfit_ds); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl_dbg); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpu7_tl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &cpu_cluster_masks_proc_ops, -+ &cpu_cluster_masks); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &save_gov_proc_ops, -+ &save_gov); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &watchdog_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isolate_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &iso_free_rescue); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_stats_proc_ops); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &yield_opt_proc_ops, -+ &yield_opt_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbird_preempt_policy); -+ /* /proc/hmbird_sched--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_walt */ -+ load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir); -+ if (!load_track_dir) { -+ pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_walt--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &slim_walt_ctrl_proc_ops, -+ &slim_walt_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_dump); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_policy); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &sched_ravg_window_frame_per_sec_proc_ops, -+ &sched_ravg_window_frame_per_sec); -+ /* /proc/hmbird_sched/slim_walt--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_freq_gov */ -+ freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir); -+ if (!freq_gov_dir) { -+ pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_freq_gov--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &slim_gov_debug); -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &scx_gov_ctrl); -+ /* /proc/hmbird_sched/slim_freq_gov--end */ -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cluster_separate); -+ -+ return 0; -+} -+ -+device_initcall(hmbird_proc_init); -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.h b/kernel/sched/hmbird/hmbird_sched_proc.h -new file mode 100755 -index 000000000000..e22bc75f1dd7 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SCHED_PROC_H__ -+#define __HMBIRD_SCHED_PROC_H__ -+ -+#include -+#include -+ -+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \ -+ do { \ -+ if (!proc_create(name, mode, parent, proc_ops)) { \ -+ pr_err("Error creating proc entry %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \ -+ do { \ -+ if (!proc_create_data(name, mode, parent, proc_ops, data)) { \ -+ pr_err("Error creating proc entry with data %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_PROC_OPS(name, open_func, write_func) \ -+ static const struct proc_ops name##_proc_ops = { \ -+ .proc_open = open_func, \ -+ .proc_write = write_func, \ -+ .proc_read = seq_read, \ -+ .proc_lseek = seq_lseek, \ -+ .proc_release = single_release, \ -+ } -+ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos); -+static int hmbird_common_show(struct seq_file *m, void *v); -+static int hmbird_common_open(struct inode *inode, struct file *file); -+ -+#endif -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.c b/kernel/sched/hmbird/hmbird_shadow_tick.c -new file mode 100755 -index 000000000000..21de551c5132 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.c -@@ -0,0 +1,159 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include -+#include "../../time/tick-sched.h" -+#include -+ -+#include "hmbird_shadow_tick.h" -+ -+#define HIGHRES_WATCH_CPU 0 -+ -+#include -+static bool shadow_tick_enable(void) {return highres_tick_ctrl; } -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; } -+#endif -+static bool shadow_tick_timer_init_flag; -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define shadow_tick_printk(fmt, args...) \ -+do { \ -+ int cpu = smp_processor_id(); \ -+ if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU) \ -+ trace_printk("hmbird shadow tick :"fmt, args); \ -+} while (0) -+#else -+#define shadow_tick_printk(fmt, args...) -+#endif -+ -+DEFINE_PER_CPU(struct hrtimer, stt); -+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu))) -+#define STOP_IDLE_TRIGGER (1) -+#define PERIODIC_TICK_TRIGGER (2) -+#define TICK_INTVAL (1000000ULL) -+/* -+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms, -+ * so we can't restart hrtimer in scheduler tick. -+ */ -+static DEFINE_PER_CPU(u8, trigger_event); -+ -+/* -+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick. -+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler. -+ */ -+static DEFINE_PER_CPU(u8, tick_phase); -+ -+static inline void highres_timer_ctrl(bool enable, int cpu) -+{ -+ if (enable && hmbird_enabled()) { -+ if (!hrtimer_active(shadow_tick_timer(cpu))) -+ hrtimer_start(shadow_tick_timer(cpu), -+ ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED); -+ } else { -+ if (!enable) -+ hrtimer_cancel(shadow_tick_timer(cpu)); -+ } -+} -+ -+static inline void high_res_clear_phase(int cpu) -+{ -+ per_cpu(tick_phase, cpu) = 0; -+} -+ -+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer) -+{ -+ per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3; -+ if (per_cpu(tick_phase, cpu)) { -+ hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) { -+ per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ struct rq_flags rf; -+ -+ rq_lock(rq, &rf); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ rq_unlock(rq, &rf); -+ -+ return highres_next_phase(cpu, timer); -+} -+ -+void shadow_tick_timer_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance; -+ } -+} -+ -+void start_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable()) { -+ if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER) -+ highres_timer_ctrl(false, cpu); -+ per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static void stop_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ per_cpu(trigger_event, cpu) = 0; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(false, cpu); -+} -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ stop_shadow_tick_timer(); -+} -+ -+void scheduler_tick_handler(void *unused, struct rq *rq) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ start_shadow_tick_timer(); -+} -+ -+static int __init hmbird_shadow_tick_init(void) -+{ -+ int ret = 0; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ shadow_tick_timer_init(); -+ shadow_tick_timer_init_flag = true; -+ return ret; -+} -+ -+device_initcall(hmbird_shadow_tick_init); -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.h b/kernel/sched/hmbird/hmbird_shadow_tick.h -new file mode 100755 -index 000000000000..0c26fd984f0a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.h -@@ -0,0 +1,11 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SHADOW_TICK_H__ -+#define __HMBIRD_SHADOW_TICK_H__ -+ -+#include -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+void scheduler_tick_handler(void *unused, struct rq *rq); -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state); -+#endif -diff --git a/kernel/sched/hmbird/hmbird_trace.h b/kernel/sched/hmbird/hmbird_trace.h -new file mode 100755 -index 000000000000..8468b406f347 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_trace.h -@@ -0,0 +1,92 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hmbird -+ -+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HMBIRD_H -+ -+#include -+#include -+#include -+ -+#define MAX_FATAL_INFO (64) -+ -+TRACE_EVENT(hmbird_fatal_info, -+ -+ TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info), -+ -+ TP_ARGS(type, pe, lnr, bnr, info), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, type) -+ __field(int, pe) -+ __field(int, lnr) -+ __field(int, bnr) -+ __array(char, info, MAX_FATAL_INFO)), -+ -+ TP_fast_assign( -+ __entry->type = type; -+ __entry->pe = pe; -+ __entry->lnr = lnr; -+ __entry->bnr = bnr; -+ memcpy(__entry->info, info, MAX_FATAL_INFO);), -+ -+ TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s", -+ __entry->type, __entry->pe, __entry->lnr, __entry->bnr, -+ __entry->info) -+); -+ -+TRACE_EVENT(hmbird_update_history, -+ -+ TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq, -+ struct task_struct *p, u32 runtime, int samples, int event), -+ -+ TP_ARGS(hmbird, rq, p, runtime, samples, event), -+ -+ TP_STRUCT__entry( -+ __array(char, comm, TASK_COMM_LEN) -+ __field(pid_t, pid) -+ __field(unsigned int, runtime) -+ __field(int, samples) -+ __field(int, event) -+ __field(unsigned int, demand) -+ __array(u32, hist, RAVG_HIST_SIZE) -+ __field(u16, task_util) -+ __field(int, cpu)), -+ -+ TP_fast_assign( -+ memcpy(__entry->comm, p->comm, TASK_COMM_LEN); -+ __entry->pid = p->pid; -+ __entry->runtime = runtime; -+ __entry->samples = samples; -+ __entry->event = event; -+ __entry->demand = hmbird->sts.demand; -+ memcpy(__entry->hist, hmbird->sts.sum_history, -+ RAVG_HIST_SIZE * sizeof(u32)); -+ __entry->task_util = hmbird->sts.demand_scaled; -+ __entry->cpu = rq->cpu;), -+ -+ TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d", -+ __entry->comm, __entry->pid, -+ __entry->runtime, __entry->samples, -+ __entry->event, -+ __entry->demand, -+ __entry->hist[0], __entry->hist[1], -+ __entry->hist[2], __entry->hist[3], -+ __entry->hist[4], -+ __entry->task_util, -+ __entry->cpu) -+); -+ -+#endif /*_TRACE_HMBIRD_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird -+ -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE hmbird_trace -+/* This part must be outside protection */ -+#include -diff --git a/kernel/sched/hmbird/hmbird_util_track.c b/kernel/sched/hmbird/hmbird_util_track.c -new file mode 100755 -index 000000000000..d520a8403401 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.c -@@ -0,0 +1,626 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+ -+#define CREATE_TRACE_POINTS -+#include "hmbird_trace.h" -+#undef CREATE_TRACE_POINTS -+ -+#define HMBIRD_DEBUG_PANIC (1 << 3) -+static bool init_irq_work_inited; -+ -+extern noinline int tracing_mark_write(const char *buf); -+ -+int hmbird_sched_ravg_window = 8000000; -+int new_hmbird_sched_ravg_window = 8000000; -+DEFINE_SPINLOCK(new_sched_ravg_window_lock); -+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+ -+static struct irq_work hmbird_slim_walt_irq_work; -+ -+/*Sysctl related interface*/ -+#define WINDOW_STATS_RECENT 0 -+#define WINDOW_STATS_MAX 1 -+#define WINDOW_STATS_MAX_RECENT_AVG 2 -+#define WINDOW_STATS_AVG 3 -+#define WINDOW_STATS_INVALID_POLICY 4 -+ -+ -+#define SCX_SCHED_CAPACITY_SHIFT 10 -+#define SCHED_ACCOUNT_WAIT_TIME 0 -+ -+atomic64_t hmbird_irq_work_lastq_ws; -+static u64 tick_sched_clock; -+ -+static inline u64 scale_exec_time(u64 delta, struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT; -+} -+ -+static inline u64 scale_time_to_util(u64 d) -+{ -+ do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT); -+ return d; -+} -+ -+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ delta = scale_exec_time(delta, rq); -+ sts->sum += delta; -+ if (unlikely(sts->sum > hmbird_sched_ravg_window)) -+ sts->sum = hmbird_sched_ravg_window; -+ -+ return delta; -+} -+ -+ -+static int -+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) -+{ -+ /* -+ * No need to bother updating task demand for the idle task. -+ */ -+ if (is_idle_task(p)) -+ return 0; -+ -+ /* -+ * When a task is waking up it is completing a segment of non-busy -+ * time. Likewise, if wait time is not treated as busy time, then -+ * when a task begins to run or is migrated, it is not running and -+ * is completing a segment of non-busy time. -+ */ -+ if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && -+ (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) -+ return 0; -+ -+ /* -+ * The idle exit time is not accounted for the first task _picked_ up to -+ * run on the idle CPU. -+ */ -+ if (event == PICK_NEXT_TASK && rq->curr == rq->idle) -+ return 0; -+ -+ /* -+ * TASK_UPDATE can be called on sleeping task, when its moved between -+ * related groups -+ */ -+ if (event == TASK_UPDATE) { -+ if (rq->curr == p) -+ return 1; -+ -+ return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; -+ } -+ -+ return 1; -+} -+ -+static void rollover_cpu_window(struct rq *rq, bool full_window) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 curr_sum = srq->curr_runnable_sum; -+ -+ if (unlikely(full_window)) -+ curr_sum = 0; -+ -+ srq->prev_runnable_sum = curr_sum; -+ srq->curr_runnable_sum = 0; -+} -+ -+static u64 -+update_window_start(struct rq *rq, u64 wallclock, int event) -+{ -+ s64 delta; -+ int nr_windows; -+ bool full_window; -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start = srq->window_start; -+ -+ if (wallclock < srq->latest_clock) -+ wallclock = srq->latest_clock; -+ delta = wallclock - srq->window_start; -+ if (delta < 0) { -+ delta = 0; -+ wallclock = srq->window_start; -+ } -+ srq->latest_clock = wallclock; -+ if (delta < hmbird_sched_ravg_window) -+ return old_window_start; -+ -+ nr_windows = div64_u64(delta, hmbird_sched_ravg_window); -+ srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window; -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ full_window = nr_windows > 1; -+ rollover_cpu_window(rq, full_window); -+ -+ return old_window_start; -+} -+ -+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -+ -+static inline unsigned int get_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static inline unsigned int cpu_cur_freq(int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cur; -+} -+ -+static void -+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, -+ u64 wallclock) -+{ -+ int cpu = cpu_of(rq); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * -+ arch_scale_cpu_capacity(cpu), get_max_freq(cpu)); -+} -+ -+/* -+ * Called when new window is starting for a task, to record cpu usage over -+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 -+ * when, say, a real-time task runs without preemption for several windows at a -+ * stretch. -+ */ -+static void update_history(struct rq *rq, struct task_struct *p, -+ u32 runtime, int samples, int event) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u32 *hist = &sts->sum_history[0]; -+ int i; -+ u32 max = 0, avg, demand; -+ u64 sum = 0; -+ u16 demand_scaled; -+ -+ /* Ignore windows where task had no activity */ -+ if (!runtime || is_idle_task(p) || !samples) -+ goto done; -+ -+ /* Push new 'runtime' value onto stack */ -+ for (; samples > 0; samples--) { -+ hist[sts->cidx] = runtime; -+ sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE; -+ } -+ -+ for (i = 0; i < RAVG_HIST_SIZE; i++) { -+ sum += hist[i]; -+ if (hist[i] > max) -+ max = hist[i]; -+ } -+ -+ sts->sum = 0; -+ avg = div64_u64(sum, RAVG_HIST_SIZE); -+ -+ switch (slim_walt_policy) { -+ case WINDOW_STATS_RECENT: -+ demand = runtime; -+ break; -+ case WINDOW_STATS_MAX: -+ demand = max; -+ break; -+ case WINDOW_STATS_AVG: -+ demand = avg; -+ break; -+ default: -+ demand = max(avg, runtime); -+ } -+ -+ demand_scaled = scale_time_to_util(demand); -+ -+ sts->demand = demand; -+ sts->demand_scaled = demand_scaled; -+ -+done: -+ return; -+} -+ -+ -+static u64 update_task_demand(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ -+ u64 delta, window_start = srq->window_start; -+ int new_window, nr_full_windows; -+ u32 window_size = hmbird_sched_ravg_window; -+ u64 runtime; -+ -+ new_window = mark_start < window_start; -+ if (!account_busy_for_task_demand(rq, p, event)) { -+ if (new_window) -+ /* -+ * If the time accounted isn't being accounted as -+ * busy time, and a new window started, only the -+ * previous window need be closed out with the -+ * pre-existing demand. Multiple windows may have -+ * elapsed, but since empty windows are dropped, -+ * it is not necessary to account those. -+ */ -+ update_history(rq, p, sts->sum, 1, event); -+ return 0; -+ } -+ -+ if (!new_window) { -+ /* -+ * The simple case - busy time contained within the existing -+ * window. -+ */ -+ return add_to_task_demand(rq, p, wallclock - mark_start); -+ } -+ -+ /* -+ * Busy time spans at least two windows. Temporarily rewind -+ * window_start to first window boundary after mark_start. -+ */ -+ delta = window_start - mark_start; -+ nr_full_windows = div64_u64(delta, window_size); -+ window_start -= (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (window_start - mark_start) first */ -+ runtime = add_to_task_demand(rq, p, window_start - mark_start); -+ -+ /* Push new sample(s) into task's demand history */ -+ update_history(rq, p, sts->sum, 1, event); -+ if (nr_full_windows) { -+ u64 scaled_window = scale_exec_time(window_size, rq); -+ -+ update_history(rq, p, scaled_window, nr_full_windows, event); -+ runtime += nr_full_windows * scaled_window; -+ } -+ -+ /* -+ * Roll window_start back to current to process any remainder -+ * in current window. -+ */ -+ window_start += (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (wallclock - window_start) next */ -+ mark_start = window_start; -+ runtime += add_to_task_demand(rq, p, wallclock - mark_start); -+ -+ return runtime; -+} -+ -+u16 slim_walt_cpu_util(int cpu) -+{ -+ u64 prev_runnable_sum; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ prev_runnable_sum = srq->prev_runnable_sum; -+ do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT); -+ -+ return (u16)prev_runnable_sum; -+} -+ -+static DEFINE_PER_CPU(u16, prev_cpu_util); -+static inline void cpu_util_update_systrace_c(int cpu) -+{ -+ char buf[256]; -+ u16 cpu_util = slim_walt_cpu_util(cpu); -+ -+ if (cpu_util != per_cpu(prev_cpu_util, cpu)) { -+ snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n", -+ cpu, cpu_util); -+ tracing_mark_write(buf); -+ per_cpu(prev_cpu_util, cpu) = cpu_util; -+ } -+} -+ -+ -+static inline int account_busy_for_cpu_time(struct rq *rq, -+ struct task_struct *p, -+ int event) -+{ -+ return !is_idle_task(p) && (event == PUT_PREV_TASK -+ || event == TASK_UPDATE); -+} -+ -+ -+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ int new_window, full_window = 0; -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 window_start = srq->window_start; -+ u32 window_size = srq->prev_window_size; -+ u64 delta; -+ u64 *curr_runnable_sum = &srq->curr_runnable_sum; -+ u64 *prev_runnable_sum = &srq->prev_runnable_sum; -+ -+ new_window = mark_start < window_start; -+ if (new_window) -+ full_window = (window_start - mark_start) >= window_size; -+ -+ -+ if (!account_busy_for_cpu_time(rq, p, event)) -+ goto done; -+ -+ -+ if (!new_window) { -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. No rollover -+ * since we didn't start a new window. An example of this is -+ * when a task starts execution and then sleeps within the -+ * same window. -+ */ -+ delta = wallclock - mark_start; -+ -+ delta = scale_exec_time(delta, rq); -+ *curr_runnable_sum += delta; -+ -+ goto done; -+ } -+ -+ /* -+ * situations below this need window rollover, -+ * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done -+ * in update_window_start() -+ * -+ * For task counters curr/prev_window[_cpu] are rolled over in the early part of -+ * this function. If full_window(s) have expired and time since last update needs -+ * to be accounted as busy time, set the prev to a complete window size time, else -+ * add the prev window portion. -+ * -+ * For task curr counters a new window has begun, always assign -+ */ -+ -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. A new window -+ * must have been started in udpate_window_start() -+ * If any of these three above conditions are true -+ * then this busy time can't be accounted as irqtime. -+ * -+ * Busy time for the idle task need not be accounted. -+ * -+ * An example of this would be a task that starts execution -+ * and then sleeps once a new window has begun. -+ */ -+ -+ /* -+ * A full window hasn't elapsed, account partial -+ * contribution to previous completed window. -+ */ -+ -+ -+ delta = full_window ? scale_exec_time(window_size, rq) : -+ scale_exec_time(window_start - mark_start, rq); -+ -+ *prev_runnable_sum += delta; -+ -+ /* Account piece of busy time in the current window. */ -+ delta = scale_exec_time(wallclock - window_start, rq); -+ *curr_runnable_sum += delta; -+ -+done: -+ if (slim_walt_dump && new_window) -+ cpu_util_update_systrace_c(rq->cpu); -+} -+ -+ -+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq) -+{ -+ u64 result; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 new_window_start = srq->window_start; -+ -+ if (old_window_start == new_window_start) -+ return; -+ -+ result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws, -+ old_window_start, new_window_start); -+ if (result != old_window_start) -+ return; -+ -+ if (likely(cpu_online(raw_smp_processor_id()))) -+ irq_work_queue(&hmbird_slim_walt_irq_work); -+ else -+ irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask)); -+} -+ -+/* -+ * In the core scheduler, most of the load update points update the rq_clock after -+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of -+ * obtaining the time, but to prevent the subsequent migration of the load update -+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock -+ * update here. -+ */ -+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p, -+ struct rq *rq, int event) -+{ -+ if (!(rq->clock_update_flags & RQCF_UPDATED)) -+ update_rq_clock(rq); -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock)); -+} -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start; -+ -+ if (!slim_walt_ctrl) -+ return; -+ -+ if (!srq->window_start || sts->mark_start == wallclock) -+ return; -+ -+ old_window_start = update_window_start(rq, wallclock, event); -+ -+ if (!sts->window_start) -+ sts->window_start = srq->window_start; -+ -+ if (!sts->mark_start) -+ goto done; -+ -+ update_task_rq_cpu_cycles(p, rq, event, wallclock); -+ update_task_demand(p, rq, event, wallclock); -+ update_cpu_busy_time(p, rq, event, wallclock); -+ -+ sts->window_start = srq->window_start; -+ -+done: -+ sts->mark_start = wallclock; -+ slim_walt_window_rollover_run_once(old_window_start, rq); -+} -+ -+static void slim_walt_irq_work(struct irq_work *irq_work) -+{ -+ cpumask_t lock_cpus; -+ struct hmbird_sched_rq_stats *srq; -+ struct rq *rq; -+ int cpu; -+ int level = 0; -+ u64 wc; -+ unsigned long flags; -+ -+ cpumask_copy(&lock_cpus, cpu_possible_mask); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ if (level == 0) -+ raw_spin_lock(&cpu_rq(cpu)->__lock); -+ else -+ raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); -+ level++; -+ } -+ -+ wc = sched_clock(); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ rq = cpu_rq(cpu); -+ hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc); -+ } -+ -+ cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER); -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) { -+ srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id()); -+ if (wc < srq->window_start + new_hmbird_sched_ravg_window) -+ hmbird_sched_ravg_window = new_hmbird_sched_ravg_window; -+ } -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ raw_spin_unlock(&cpu_rq(cpu)->__lock); -+ } -+} -+static void hmbird_sched_init_rq(struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ srq->task_exec_scale = 1024; -+ srq->window_start = 0; -+} -+ -+void hmbird_sched_init_task(struct task_struct *p) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ memset(sts, 0, sizeof(struct hmbird_sched_task_stats)); -+} -+ -+static void hmbird_sched_stats_init(void) -+{ -+ unsigned long flags; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ raw_spin_lock_irqsave(&rq->__lock, flags); -+ hmbird_sched_init_rq(rq); -+ raw_spin_unlock_irqrestore(&rq->__lock, flags); -+ } -+ slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG; -+ -+ if (false == init_irq_work_inited) { -+ init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work); -+ init_irq_work_inited = true; -+ } -+} -+ -+void hmbird_scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (unlikely(!tick_sched_clock)) { -+ /* -+ * Let the window begin 20us prior to the tick, -+ * that way we are guaranteed a rollover when the tick occurs. -+ * Use rq->clock directly instead of rq_clock() since -+ * we do not have the rq lock and -+ * rq->clock was updated in the tick callpath. -+ */ -+ if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000)) -+ return; -+ for_each_possible_cpu(cpu) { -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ srq->window_start = tick_sched_clock; -+ } -+ atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock); -+ } -+} -+ -+void slim_walt_enable(int enable) -+{ -+ if (1 == !!enable) { -+ hmbird_sched_stats_init(); -+ WRITE_ONCE(tick_sched_clock, 0); -+ } else -+ slim_walt_ctrl = 0; -+} -+ -+void slim_get_cpu_util(int cpu, u64 *util) -+{ -+ if (cpu < 0) -+ return; -+ -+ *util = slim_walt_cpu_util(cpu); -+} -+ -+void slim_get_task_util(struct task_struct *p, u64 *util) -+{ -+ if (p == NULL) -+ return; -+ -+ *util = get_hmbird_ts(p)->sts.demand_scaled; -+} -+ -+void sched_ravg_window_change(int frame_per_sec) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec; -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+} -diff --git a/kernel/sched/hmbird/hmbird_util_track.h b/kernel/sched/hmbird/hmbird_util_track.h -new file mode 100755 -index 000000000000..17b85df7f83b ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.h -@@ -0,0 +1,33 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef __HMBIRD_UTIL_TRACK_H__ -+#define __HMBIRD_UTIL_TRACK_H__ -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock); -+void hmbird_sched_init_task(struct task_struct *p); -+void slim_walt_enable(int enable); -+void slim_get_cpu_util(int cpu, u64 *util); -+void slim_get_task_util(struct task_struct *p, u64 *util); -+ -+extern atomic64_t hmbird_irq_work_lastq_ws; -+ -+enum task_event { -+ PUT_PREV_TASK = 0, -+ PICK_NEXT_TASK = 1, -+ TASK_WAKE = 2, -+ TASK_MIGRATE = 3, -+ TASK_UPDATE = 4, -+ IRQ_UPDATE = 5, -+}; -+ -+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+#endif /* __HMBIRD_UTIL_TRACK_H__ */ -diff --git a/kernel/sched/hmbird/slim.h b/kernel/sched/hmbird/slim.h -new file mode 100755 -index 000000000000..eb386260e950 ---- /dev/null -+++ b/kernel/sched/hmbird/slim.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+ -+#ifndef __SLIM_H -+#define __SLIM_H -+ -+extern atomic_t __hmbird_ops_enabled; -+extern atomic_t non_hmbird_task; -+extern int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+extern int heartbeat; -+extern int heartbeat_enable; -+extern int watchdog_enable; -+extern int isolate_ctrl; -+extern int parctrl_high_ratio; -+extern int parctrl_low_ratio; -+extern int isoctrl_high_ratio; -+extern int isoctrl_low_ratio; -+extern int iso_free_rescue; -+extern int yield_opt; -+ -+extern enum hmbird_switch_type sw_type; -+extern noinline int tracing_mark_write(const char *buf); -+int task_top_id(struct task_struct *p); -+void stats_print(char *buf, int len); -+void hmbird_skip_yield(long *skip); -+extern spinlock_t hmbird_tasks_lock; -+ -+struct yield_opt_params { -+ int enable; -+ int frame_per_sec; -+ u64 frame_time_ns; -+ int yield_headroom; -+}; -+ -+extern struct yield_opt_params yield_opt_params; -+ -+#define MAX_GOV_LEN (16) -+extern char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+#endif -diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c -index b33cefeb4188..487255ac6832 100644 ---- a/kernel/sched/idle.c -+++ b/kernel/sched/idle.c -@@ -408,13 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl - - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) - { -- scx_update_idle(rq, false); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, false); -+#endif - } - - static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) - { - update_idle_core(rq); -- scx_update_idle(rq, true); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, true); -+#endif - schedstat_inc(rq->sched_goidle); - } - -diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h -index cbc478992cc4..e35473a1f0ea 100644 ---- a/kernel/sched/sched.h -+++ b/kernel/sched/sched.h -@@ -187,18 +187,13 @@ static inline int idle_policy(int policy) - return policy == SCHED_IDLE; - } - --static inline int normal_policy(int policy) --{ --#ifdef CONFIG_SCHED_CLASS_EXT -- if (policy == SCHED_EXT) -- return true; --#endif -- return policy == SCHED_NORMAL; --} -- - static inline int fair_policy(int policy) - { -- return normal_policy(policy) || policy == SCHED_BATCH; -+#ifdef CONFIG_HMBIRD_SCHED -+ return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#else -+ return policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#endif - } - - static inline int rt_policy(int policy) -@@ -246,24 +241,6 @@ static inline void update_avg(u64 *avg, u64 sample) - #define shr_bound(val, shift) \ - (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) - --/* -- * cgroup weight knobs should use the common MIN, DFL and MAX values which are -- * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it -- * maps pretty well onto the shares value used by scheduler and the round-trip -- * conversions preserve the original value over the entire range. -- */ --static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) --{ -- return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); --} -- --static inline unsigned long sched_weight_to_cgroup(unsigned long weight) --{ -- return clamp_t(unsigned long, -- DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -- CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); --} -- - /* - * !! For sched_setattr_nocheck() (kernel) only !! - * -@@ -531,10 +508,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) { } - #endif /* CONFIG_SMP */ - #else /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_HMBIRD_SCHED - static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) - { - return 0; - } -+#endif - #endif /* CONFIG_FAIR_GROUP_SCHED */ - - #else /* CONFIG_CGROUP_SCHED */ -@@ -699,29 +678,6 @@ struct cfs_rq { - #endif /* CONFIG_FAIR_GROUP_SCHED */ - }; - --#ifdef CONFIG_SCHED_CLASS_EXT --/* scx_rq->flags, protected by the rq lock */ --enum scx_rq_flags { -- SCX_RQ_CAN_STOP_TICK = 1 << 0, --}; -- --struct scx_rq { -- struct scx_dispatch_q local_dsq; -- struct list_head watchdog_list; -- u64 ops_qseq; -- u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -- u32 nr_running; -- u32 flags; -- bool cpu_released; -- cpumask_var_t cpus_to_kick; -- cpumask_var_t cpus_to_preempt; -- cpumask_var_t cpus_to_wait; -- u64 pnt_seq; -- struct irq_work kick_cpus_irq_work; -- struct rq *rq; --}; --#endif /* CONFIG_SCHED_CLASS_EXT */ -- - static inline int rt_bandwidth_enabled(void) - { - return sysctl_sched_rt_runtime >= 0; -@@ -1257,11 +1213,7 @@ struct rq { - - ANDROID_OEM_DATA_ARRAY(1, 16); - --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, struct scx_rq *scx); --#else - ANDROID_KABI_RESERVE(1); --#endif - ANDROID_KABI_RESERVE(2); - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); -@@ -2339,11 +2291,6 @@ struct affinity_context { - unsigned int flags; - }; - --enum rq_onoff_reason { -- RQ_ONOFF_HOTPLUG, /* CPU is going on/offline */ -- RQ_ONOFF_TOPOLOGY, /* sched domain topology update */ --}; -- - struct sched_class { - - #ifdef CONFIG_UCLAMP_TASK -@@ -2550,7 +2497,6 @@ extern void init_sched_rt_class(void); - extern void init_sched_fair_class(void); - - extern void reweight_task(struct task_struct *p, int prio); --extern void __setscheduler_prio(struct task_struct *p, int prio); - - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); -@@ -2631,53 +2577,6 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) - extern void activate_task(struct rq *rq, struct task_struct *p, int flags); - extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); - --struct sched_change_guard { -- struct task_struct *p; -- struct rq *rq; -- bool queued; -- bool running; -- bool done; --}; -- --extern struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -- --extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags); -- --/** -- * SCHED_CHANGE_BLOCK - Nested block for task attribute updates -- * @__rq: Runqueue the target task belongs to -- * @__p: Target task -- * @__flags: DEQUEUE/ENQUEUE_* flags -- * -- * A task may need to be dequeued and put_prev_task'd for attribute updates and -- * set_next_task'd and re-enqueued afterwards. This helper defines a nested -- * block which automatically handles these preparation and cleanup operations. -- * -- * SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- * update_attribute(p); -- * ... -- * } -- * -- * If @__flags is a variable, the variable may be updated in the block body and -- * the updated value will be used when re-enqueueing @p. -- * -- * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling -- * update_rq_clock() beforehand. Otherwise, the rq clock is automatically -- * updated iff the task needs to be dequeued and re-enqueued. Only the former -- * case guarantees that the rq clock is up-to-date inside and after the block. -- */ --#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -- for (struct sched_change_guard __cg = \ -- sched_change_guard_init(__rq, __p, __flags); \ -- !__cg.done; sched_change_guard_fini(&__cg, __flags)) -- --extern void check_class_changing(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class); --extern void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio); -- - extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); - - #ifdef CONFIG_PREEMPT_RT -@@ -3709,6 +3608,8 @@ static inline bool cpu_busy_with_softirqs(int cpu) - } - #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ - --#include "ext.h" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird.h" -+#endif - - #endif /* _KERNEL_SCHED_SCHED_H */ -diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c -index c197c61d6ea9..919fed4ff08b 100644 ---- a/kernel/time/tick-sched.c -+++ b/kernel/time/tick-sched.c -@@ -34,6 +34,10 @@ - - #include - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "../sched/hmbird/hmbird_shadow_tick.h" -+#endif -+ - /* - * Per-CPU nohz control structure - */ -@@ -1112,6 +1116,9 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) - return true; - } - -+#ifdef CONFIG_HMBIRD_SCHED -+extern void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+#endif - /** - * tick_nohz_idle_stop_tick - stop the idle tick from the idle task - * -@@ -1124,7 +1131,9 @@ void tick_nohz_idle_stop_tick(void) - ktime_t expires; - - trace_android_vh_tick_nohz_idle_stop_tick(NULL); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL); -+#endif - /* - * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the - * tick timer expiration time is known already. -diff --git a/tools/sched_ext/.gitignore b/tools/sched_ext/.gitignore -deleted file mode 100644 -index a3240f9f7eba..000000000000 ---- a/tools/sched_ext/.gitignore -+++ /dev/null -@@ -1,9 +0,0 @@ --scx_example_simple --scx_example_qmap --scx_example_central --scx_example_pair --scx_example_flatcg --scx_example_userland --*.skel.h --*.subskel.h --/tools/ -diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile -deleted file mode 100644 -index 73c43782837d..000000000000 ---- a/tools/sched_ext/Makefile -+++ /dev/null -@@ -1,216 +0,0 @@ --# SPDX-License-Identifier: GPL-2.0 --# Copyright (c) 2022 Meta Platforms, Inc. and affiliates. --include ../build/Build.include --include ../scripts/Makefile.arch --include ../scripts/Makefile.include -- --ifneq ($(LLVM),) --ifneq ($(filter %/,$(LLVM)),) --LLVM_PREFIX := $(LLVM) --else ifneq ($(filter -%,$(LLVM)),) --LLVM_SUFFIX := $(LLVM) --endif -- --CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi --CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu --CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl --CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu --CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu --CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu --CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu --CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu --CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu --CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) -- --ifeq ($(CROSS_COMPILE),) --ifeq ($(CLANG_TARGET_FLAGS),) --$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk --else --CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) --endif # CLANG_TARGET_FLAGS --else --CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) --endif # CROSS_COMPILE -- --CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as --else --CC := $(CROSS_COMPILE)gcc --endif # LLVM -- --CURDIR := $(abspath .) --TOOLSDIR := $(abspath ..) --LIBDIR := $(TOOLSDIR)/lib --BPFDIR := $(LIBDIR)/bpf --TOOLSINCDIR := $(TOOLSDIR)/include --BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool --APIDIR := $(TOOLSINCDIR)/uapi --GENDIR := $(abspath ../../include/generated) --GENHDR := $(GENDIR)/autoconf.h -- --SCRATCH_DIR := $(CURDIR)/tools --BUILD_DIR := $(SCRATCH_DIR)/build --INCLUDE_DIR := $(SCRATCH_DIR)/include --BPFOBJ_DIR := $(BUILD_DIR)/libbpf --BPFOBJ := $(BPFOBJ_DIR)/libbpf.a --ifneq ($(CROSS_COMPILE),) --HOST_BUILD_DIR := $(BUILD_DIR)/host --HOST_SCRATCH_DIR := host-tools --HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include --else --HOST_BUILD_DIR := $(BUILD_DIR) --HOST_SCRATCH_DIR := $(SCRATCH_DIR) --HOST_INCLUDE_DIR := $(INCLUDE_DIR) --endif --HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a --RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids --DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -- --VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ -- $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ -- ../../vmlinux \ -- /sys/kernel/btf/vmlinux \ -- /boot/vmlinux-$(shell uname -r) --VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) --ifeq ($(VMLINUX_BTF),) --$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") --endif -- --BPFTOOL ?= $(DEFAULT_BPFTOOL) -- --ifneq ($(wildcard $(GENHDR)),) -- GENFLAGS := -DHAVE_GENHDR --endif -- --CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ -- -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -- -I$(TOOLSINCDIR) -I$(APIDIR) -- --CARGOFLAGS := --release -- --# Silence some warnings when compiled with clang --ifneq ($(LLVM),) --CFLAGS += -Wno-unused-command-line-argument --endif -- --LDFLAGS = -lelf -lz -lpthread -- --IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - &1 \ -- | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ --$(shell $(1) -dM -E - $@ --else -- $(call msg,CP,,$@) -- $(Q)cp "$(VMLINUX_H)" $@ --endif -- --%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \ -- | $(BPFOBJ) -- $(call msg,CLNG-BPF,,$@) -- $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ -- --%.skel.h: %.bpf.o $(BPFTOOL) -- $(call msg,GEN-SKEL,,$@) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) -- $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) -- $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@ -- $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h) -- --scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_userland: scx_example_userland.c scx_example_userland.skel.h \ -- scx_example_userland_common.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR) --atropos: export ATROPOS_CLANG = $(CLANG) --atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS) --atropos: $(INCLUDE_DIR)/vmlinux.h -- cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS) -- --clean: -- cargo clean --manifest-path=atropos/Cargo.toml -- rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) -- rm -f *.o *.bpf.o *.skel.h *.subskel.h -- rm -f scx_example_simple scx_example_qmap scx_example_central \ -- scx_example_pair scx_example_flatcg scx_example_userland -- --.PHONY: all atropos clean -- --# delete failed targets --.DELETE_ON_ERROR: -- --# keep intermediate (.skel.h, .bpf.o, etc) targets --.SECONDARY: -diff --git a/tools/sched_ext/atropos/.gitignore b/tools/sched_ext/atropos/.gitignore -deleted file mode 100644 -index 186dba259ec2..000000000000 ---- a/tools/sched_ext/atropos/.gitignore -+++ /dev/null -@@ -1,3 +0,0 @@ --src/bpf/.output --Cargo.lock --target -diff --git a/tools/sched_ext/atropos/Cargo.toml b/tools/sched_ext/atropos/Cargo.toml -deleted file mode 100644 -index 7462a836d53d..000000000000 ---- a/tools/sched_ext/atropos/Cargo.toml -+++ /dev/null -@@ -1,28 +0,0 @@ --[package] --name = "scx_atropos" --version = "0.5.0" --authors = ["Dan Schatzberg ", "Meta"] --edition = "2021" --description = "Userspace scheduling with BPF" --license = "GPL-2.0-only" -- --[dependencies] --anyhow = "1.0.65" --bitvec = { version = "1.0", features = ["serde"] } --clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } --ctrlc = { version = "3.1", features = ["termination"] } --fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"} --hex = "0.4.3" --libbpf-rs = "0.19.1" --libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] } --libc = "0.2.137" --log = "0.4.17" --ordered-float = "3.4.0" --simplelog = "0.12.0" -- --[build-dependencies] --bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false } --libbpf-cargo = "0.13.0" -- --[features] --enable_backtrace = [] -diff --git a/tools/sched_ext/atropos/build.rs b/tools/sched_ext/atropos/build.rs -deleted file mode 100644 -index 26e792c5e17e..000000000000 ---- a/tools/sched_ext/atropos/build.rs -+++ /dev/null -@@ -1,70 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --extern crate bindgen; -- --use std::env; --use std::fs::create_dir_all; --use std::path::Path; --use std::path::PathBuf; -- --use libbpf_cargo::SkeletonBuilder; -- --const HEADER_PATH: &str = "src/bpf/atropos.h"; -- --fn bindgen_atropos() { -- // Tell cargo to invalidate the built crate whenever the wrapper changes -- println!("cargo:rerun-if-changed={}", HEADER_PATH); -- -- // The bindgen::Builder is the main entry point -- // to bindgen, and lets you build up options for -- // the resulting bindings. -- let bindings = bindgen::Builder::default() -- // The input header we would like to generate -- // bindings for. -- .header(HEADER_PATH) -- // Tell cargo to invalidate the built crate whenever any of the -- // included header files changed. -- .parse_callbacks(Box::new(bindgen::CargoCallbacks)) -- // Finish the builder and generate the bindings. -- .generate() -- // Unwrap the Result and panic on failure. -- .expect("Unable to generate bindings"); -- -- // Write the bindings to the $OUT_DIR/bindings.rs file. -- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); -- bindings -- .write_to_file(out_path.join("atropos-sys.rs")) -- .expect("Couldn't write bindings!"); --} -- --fn gen_bpf_sched(name: &str) { -- let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap(); -- let clang = env::var("ATROPOS_CLANG").unwrap(); -- eprintln!("{}", clang); -- let outpath = format!("./src/bpf/.output/{}.skel.rs", name); -- let skel = Path::new(&outpath); -- let src = format!("./src/bpf/{}.bpf.c", name); -- SkeletonBuilder::new() -- .source(src.clone()) -- .clang(clang) -- .clang_args(bpf_cflags) -- .build_and_generate(&skel) -- .unwrap(); -- println!("cargo:rerun-if-changed={}", src); --} -- --fn main() { -- bindgen_atropos(); -- // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton. -- // Reasons are because the generated skeleton contains compiler attributes -- // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]` -- // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside -- // the path attribute either (see https://github.com/rust-lang/rust/pull/83366). -- // -- // However, there is hope! When the above feature stabilizes we can clean this -- // all up. -- create_dir_all("./src/bpf/.output").unwrap(); -- gen_bpf_sched("atropos"); --} -diff --git a/tools/sched_ext/atropos/rustfmt.toml b/tools/sched_ext/atropos/rustfmt.toml -deleted file mode 100644 -index b7258ed0a8d8..000000000000 ---- a/tools/sched_ext/atropos/rustfmt.toml -+++ /dev/null -@@ -1,8 +0,0 @@ --# Get help on options with `rustfmt --help=config` --# Please keep these in alphabetical order. --edition = "2021" --group_imports = "StdExternalCrate" --imports_granularity = "Item" --merge_derives = false --use_field_init_shorthand = true --version = "Two" -diff --git a/tools/sched_ext/atropos/src/atropos_sys.rs b/tools/sched_ext/atropos/src/atropos_sys.rs -deleted file mode 100644 -index bbeaf856d40e..000000000000 ---- a/tools/sched_ext/atropos/src/atropos_sys.rs -+++ /dev/null -@@ -1,10 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#![allow(non_upper_case_globals)] --#![allow(non_camel_case_types)] --#![allow(non_snake_case)] --#![allow(dead_code)] -- --include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs")); -diff --git a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -deleted file mode 100644 -index 3905a403e940..000000000000 ---- a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -+++ /dev/null -@@ -1,743 +0,0 @@ --/* Copyright (c) Meta Platforms, Inc. and affiliates. */ --/* -- * This software may be used and distributed according to the terms of the -- * GNU General Public License version 2. -- * -- * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF -- * part does simple round robin in each domain and the userspace part -- * calculates the load factor of each domain and tells the BPF part how to load -- * balance the domains. -- * -- * Every task has an entry in the task_data map which lists which domain the -- * task belongs to. When a task first enters the system (atropos_prep_enable), -- * they are round-robined to a domain. -- * -- * atropos_select_cpu is the primary scheduling logic, invoked when a task -- * becomes runnable. The lb_data map is populated by userspace to inform the BPF -- * scheduler that a task should be migrated to a new domain. Otherwise, the task -- * is scheduled in priority order as follows: -- * * The current core if the task was woken up synchronously and there are idle -- * cpus in the system -- * * The previous core, if idle -- * * The pinned-to core if the task is pinned to a specific core -- * * Any idle cpu in the domain -- * -- * If none of the above conditions are met, then the task is enqueued to a -- * dispatch queue corresponding to the domain (atropos_enqueue). -- * -- * atropos_dispatch will attempt to consume a task from its domain's -- * corresponding dispatch queue (this occurs after scheduling any tasks directly -- * assigned to it due to the logic in atropos_select_cpu). If no task is found, -- * then greedy load stealing will attempt to find a task on another dispatch -- * queue to run. -- * -- * Load balancing is almost entirely handled by userspace. BPF populates the -- * task weight, dom mask and current dom in the task_data map and executes the -- * load balance based on userspace populating the lb_data map. -- */ --#include "../../../scx_common.bpf.h" --#include "atropos.h" -- --#include --#include --#include --#include --#include --#include -- --char _license[] SEC("license") = "GPL"; -- --/* -- * const volatiles are set during initialization and treated as consts by the -- * jit compiler. -- */ -- --/* -- * Domains and cpus -- */ --const volatile __u32 nr_doms = 32; /* !0 for veristat, set during init */ --const volatile __u32 nr_cpus = 64; /* !0 for veristat, set during init */ --const volatile __u32 cpu_dom_id_map[MAX_CPUS]; --const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; -- --const volatile bool kthreads_local; --const volatile bool fifo_sched; --const volatile bool switch_partial; --const volatile __u32 greedy_threshold; -- --/* base slice duration */ --const volatile __u64 slice_us = 20000; -- --/* -- * Exit info -- */ --int exit_type = SCX_EXIT_NONE; --char exit_msg[SCX_EXIT_MSG_LEN]; -- --struct pcpu_ctx { -- __u32 dom_rr_cur; /* used when scanning other doms */ -- -- /* libbpf-rs does not respect the alignment, so pad out the struct explicitly */ -- __u8 _padding[CACHELINE_SIZE - sizeof(u64)]; --} __attribute__((aligned(CACHELINE_SIZE))); -- --struct pcpu_ctx pcpu_ctx[MAX_CPUS]; -- --/* -- * Domain context -- */ --struct dom_ctx { -- struct bpf_cpumask __kptr *cpumask; -- u64 vtime_now; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __type(key, u32); -- __type(value, struct dom_ctx); -- __uint(max_entries, MAX_DOMS); -- __uint(map_flags, 0); --} dom_ctx SEC(".maps"); -- --/* -- * Statistics -- */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, ATROPOS_NR_STATS); --} stats SEC(".maps"); -- --static inline void stat_add(enum stat_idx idx, u64 addend) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p) += addend; --} -- --/* Map pid -> task_ctx */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, struct task_ctx); -- __uint(max_entries, 1000000); -- __uint(map_flags, 0); --} task_data SEC(".maps"); -- --/* -- * This is populated from userspace to indicate which pids should be reassigned -- * to new doms. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, u32); -- __uint(max_entries, 1000); -- __uint(map_flags, 0); --} lb_data SEC(".maps"); -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p, -- u32 new_dom_id) --{ -- struct dom_ctx *old_domc, *new_domc; -- struct bpf_cpumask *d_cpumask, *t_cpumask; -- u32 old_dom_id = task_ctx->dom_id; -- s64 vtime_delta; -- -- old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id); -- if (!old_domc) { -- scx_bpf_error("No dom%u", old_dom_id); -- return false; -- } -- -- vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now; -- -- new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id); -- if (!new_domc) { -- scx_bpf_error("No dom%u", new_dom_id); -- return false; -- } -- -- d_cpumask = new_domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to get domain %u cpumask kptr", -- new_dom_id); -- return false; -- } -- -- t_cpumask = task_ctx->cpumask; -- if (!t_cpumask) { -- scx_bpf_error("Failed to look up task cpumask"); -- return false; -- } -- -- /* -- * set_cpumask might have happened between userspace requesting LB and -- * here and @p might not be able to run in @dom_id anymore. Verify. -- */ -- if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- p->cpus_ptr)) { -- p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta; -- task_ctx->dom_id = new_dom_id; -- bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask, -- p->cpus_ptr); -- } -- -- return task_ctx->dom_id == new_dom_id; --} -- --s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu, -- u32 wake_flags) --{ -- s32 cpu; -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- struct bpf_cpumask *p_cpumask; -- -- if (!task_ctx) -- return -ENOENT; -- -- if (kthreads_local && -- (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- cpu = prev_cpu; -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local dsq of the waker. -- */ -- if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) { -- struct task_struct *current = (void *)bpf_get_current_task(); -- -- if (!(BPF_CORE_READ(current, flags) & PF_EXITING) && -- task_ctx->dom_id < MAX_DOMS) { -- struct dom_ctx *domc; -- struct bpf_cpumask *d_cpumask; -- const struct cpumask *idle_cpumask; -- bool has_idle; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id); -- if (!domc) { -- scx_bpf_error("Failed to find dom%u", -- task_ctx->dom_id); -- return prev_cpu; -- } -- d_cpumask = domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to acquire domain %u cpumask kptr", -- task_ctx->dom_id); -- return prev_cpu; -- } -- -- idle_cpumask = scx_bpf_get_idle_cpumask(); -- -- has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- idle_cpumask); -- -- scx_bpf_put_idle_cpumask(idle_cpumask); -- -- if (has_idle) { -- cpu = bpf_get_smp_processor_id(); -- if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- stat_add(ATROPOS_STAT_WAKE_SYNC, 1); -- goto local; -- } -- } -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- stat_add(ATROPOS_STAT_PREV_IDLE, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- /* If only one core is allowed, dispatch */ -- if (p->nr_cpus_allowed == 1) { -- stat_add(ATROPOS_STAT_PINNED, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) -- return -ENOENT; -- -- /* If there is an eligible idle CPU, dispatch directly */ -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- if (cpu >= 0) { -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * @prev_cpu may be in a different domain. Returning an out-of-domain -- * CPU can lead to stalls as all in-domain CPUs may be idle by the time -- * @p gets enqueued. -- */ -- if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask)) -- cpu = prev_cpu; -- else -- cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask); -- -- return cpu; -- --local: -- task_ctx->dispatch_local = true; -- return cpu; --} -- --void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- u32 *new_dom; -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- new_dom = bpf_map_lookup_elem(&lb_data, &pid); -- if (new_dom && *new_dom != task_ctx->dom_id && -- task_set_dsq(task_ctx, p, *new_dom)) { -- struct bpf_cpumask *p_cpumask; -- s32 cpu; -- -- stat_add(ATROPOS_STAT_LOAD_BALANCE, 1); -- -- /* -- * If dispatch_local is set, We own @p's idle state but we are -- * not gonna put the task in the associated local dsq which can -- * cause the CPU to stall. Kick it. -- */ -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0); -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) { -- scx_bpf_error("Failed to get task_ctx->cpumask"); -- return; -- } -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- } -- -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags); -- return; -- } -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000, -- enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- u32 dom_id = task_ctx->dom_id; -- struct dom_ctx *domc; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, domc->vtime_now - slice_us * 1000)) -- vtime = domc->vtime_now - slice_us * 1000; -- -- scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --static u32 cpu_to_dom_id(s32 cpu) --{ -- const volatile u32 *dom_idp; -- -- if (nr_doms <= 1) -- return 0; -- -- dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]); -- if (!dom_idp) -- return MAX_DOMS; -- -- return *dom_idp; --} -- --static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) --{ -- s32 cpu; -- -- if (dom_id >= MAX_DOMS) -- return false; -- -- bpf_for(cpu, 0, nr_cpus) { -- if (bpf_cpumask_test_cpu(cpu, cpumask) && -- (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64)))) -- return true; -- } -- return false; --} -- --static u32 dom_rr_next(s32 cpu) --{ -- struct pcpu_ctx *pcpuc; -- u32 dom_id; -- -- pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]); -- if (!pcpuc) -- return 0; -- -- dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms; -- -- if (dom_id == cpu_to_dom_id(cpu)) -- dom_id = (dom_id + 1) % nr_doms; -- -- pcpuc->dom_rr_cur = dom_id; -- return dom_id; --} -- --void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 dom = cpu_to_dom_id(cpu); -- -- if (scx_bpf_consume(dom)) { -- stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1); -- return; -- } -- -- if (!greedy_threshold) -- return; -- -- bpf_repeat(nr_doms - 1) { -- u32 dom_id = dom_rr_next(cpu); -- -- if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold && -- scx_bpf_consume(dom_id)) { -- stat_add(ATROPOS_STAT_GREEDY, 1); -- break; -- } -- } --} -- --void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_at = bpf_ktime_get_ns(); --} -- --void BPF_STRUCT_OPS(atropos_running, struct task_struct *p) --{ -- struct task_ctx *taskc; -- struct dom_ctx *domc; -- pid_t pid = p->pid; -- u32 dom_id; -- -- if (fifo_sched) -- return; -- -- taskc = bpf_map_lookup_elem(&task_data, &pid); -- if (!taskc) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- dom_id = taskc->dom_id; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(domc->vtime_now, p->scx.dsq_vtime)) -- domc->vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at; -- task_ctx->runnable_at = 0; --} -- --void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->weight = weight; --} -- --struct pick_task_domain_loop_ctx { -- struct task_struct *p; -- const struct cpumask *cpumask; -- u64 dom_mask; -- u32 dom_rr_base; -- u32 dom_id; --}; -- --static int pick_task_domain_loopfn(u32 idx, void *data) --{ -- struct pick_task_domain_loop_ctx *lctx = data; -- u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms; -- -- if (dom_id >= MAX_DOMS) -- return 1; -- -- if (cpumask_intersects_domain(lctx->cpumask, dom_id)) { -- lctx->dom_mask |= 1LLU << dom_id; -- if (lctx->dom_id == MAX_DOMS) -- lctx->dom_id = dom_id; -- } -- return 0; --} -- --static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- struct pick_task_domain_loop_ctx lctx = { -- .p = p, -- .cpumask = cpumask, -- .dom_id = MAX_DOMS, -- }; -- s32 cpu = bpf_get_smp_processor_id(); -- -- if (cpu < 0 || cpu >= MAX_CPUS) -- return MAX_DOMS; -- -- lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur); -- -- bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0); -- task_ctx->dom_mask = lctx.dom_mask; -- -- return lctx.dom_id; --} -- --static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- u32 dom_id = 0; -- -- if (nr_doms > 1) -- dom_id = pick_task_domain(task_ctx, p, cpumask); -- -- if (!task_set_dsq(task_ctx, p, dom_id)) -- scx_bpf_error("Failed to set domain %d for %s[%d]", -- dom_id, p->comm, p->pid); --} -- --void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_set_domain(task_ctx, p, cpumask); --} -- --s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct bpf_cpumask *cpumask; -- struct task_ctx task_ctx, *map_value; -- long ret; -- pid_t pid; -- -- memset(&task_ctx, 0, sizeof(task_ctx)); -- -- pid = p->pid; -- ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return ret; -- } -- -- /* -- * Read the entry from the map immediately so we can add the cpumask -- * with bpf_kptr_xchg(). -- */ -- map_value = bpf_map_lookup_elem(&task_data, &pid); -- if (!map_value) -- /* Should never happen -- it was just inserted above. */ -- return -EINVAL; -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- bpf_map_delete_elem(&task_data, &pid); -- return -ENOMEM; -- } -- -- cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask); -- if (cpumask) { -- /* Should never happen as we just inserted it above. */ -- bpf_cpumask_release(cpumask); -- bpf_map_delete_elem(&task_data, &pid); -- return -EINVAL; -- } -- -- task_set_domain(map_value, p, p->cpus_ptr); -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p) --{ -- pid_t pid = p->pid; -- long ret = bpf_map_delete_elem(&task_data, &pid); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return; -- } --} -- --static int create_dom_dsq(u32 idx, void *data) --{ -- struct dom_ctx domc_init = {}, *domc; -- struct bpf_cpumask *cpumask; -- u32 cpu, dom_id = idx; -- s32 ret; -- -- ret = scx_bpf_create_dsq(dom_id, -1); -- if (ret < 0) { -- scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret); -- return 1; -- } -- -- ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0); -- if (ret) { -- scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret); -- return 1; -- } -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- /* Should never happen, we just inserted it above. */ -- scx_bpf_error("No dom%u", dom_id); -- return 1; -- } -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id); -- return 1; -- } -- -- for (cpu = 0; cpu < MAX_CPUS; cpu++) { -- const volatile __u64 *dmask; -- -- dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]); -- if (!dmask) { -- scx_bpf_error("array index error"); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- if (*dmask & (1LLU << (cpu % 64))) -- bpf_cpumask_set_cpu(cpu, cpumask); -- } -- -- cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask); -- if (cpumask) { -- scx_bpf_error("Domain %u was already present", dom_id); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(atropos_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- bpf_loop(nr_doms, create_dom_dsq, NULL, 0); -- -- for (u32 i = 0; i < nr_cpus; i++) -- pcpu_ctx[i].dom_rr_cur = i; -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg); -- exit_type = ei->type; --} -- --SEC(".struct_ops") --struct sched_ext_ops atropos = { -- .select_cpu = (void *)atropos_select_cpu, -- .enqueue = (void *)atropos_enqueue, -- .dispatch = (void *)atropos_dispatch, -- .runnable = (void *)atropos_runnable, -- .running = (void *)atropos_running, -- .stopping = (void *)atropos_stopping, -- .quiescent = (void *)atropos_quiescent, -- .set_weight = (void *)atropos_set_weight, -- .set_cpumask = (void *)atropos_set_cpumask, -- .prep_enable = (void *)atropos_prep_enable, -- .disable = (void *)atropos_disable, -- .init = (void *)atropos_init, -- .exit = (void *)atropos_exit, -- .flags = 0, -- .name = "atropos", --}; -diff --git a/tools/sched_ext/atropos/src/bpf/atropos.h b/tools/sched_ext/atropos/src/bpf/atropos.h -deleted file mode 100644 -index addf29ca104a..000000000000 ---- a/tools/sched_ext/atropos/src/bpf/atropos.h -+++ /dev/null -@@ -1,44 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#ifndef __ATROPOS_H --#define __ATROPOS_H -- --#include --#ifndef __kptr --#ifdef __KERNEL__ --#error "__kptr_ref not defined in the kernel" --#endif --#define __kptr --#endif -- --#define MAX_CPUS 512 --#define MAX_DOMS 64 /* limited to avoid complex bitmask ops */ --#define CACHELINE_SIZE 64 -- --/* Statistics */ --enum stat_idx { -- ATROPOS_STAT_TASK_GET_ERR, -- ATROPOS_STAT_WAKE_SYNC, -- ATROPOS_STAT_PREV_IDLE, -- ATROPOS_STAT_PINNED, -- ATROPOS_STAT_DIRECT_DISPATCH, -- ATROPOS_STAT_DSQ_DISPATCH, -- ATROPOS_STAT_GREEDY, -- ATROPOS_STAT_LOAD_BALANCE, -- ATROPOS_STAT_LAST_TASK, -- ATROPOS_NR_STATS, --}; -- --struct task_ctx { -- unsigned long long dom_mask; /* the domains this task can run on */ -- struct bpf_cpumask __kptr *cpumask; -- unsigned int dom_id; -- unsigned int weight; -- unsigned long long runnable_at; -- unsigned long long runnable_for; -- bool dispatch_local; --}; -- --#endif /* __ATROPOS_H */ -diff --git a/tools/sched_ext/atropos/src/main.rs b/tools/sched_ext/atropos/src/main.rs -deleted file mode 100644 -index 0d313662f713..000000000000 ---- a/tools/sched_ext/atropos/src/main.rs -+++ /dev/null -@@ -1,942 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#[path = "bpf/.output/atropos.skel.rs"] --mod atropos; --pub use atropos::*; --pub mod atropos_sys; -- --use std::cell::Cell; --use std::collections::{BTreeMap, BTreeSet}; --use std::ffi::CStr; --use std::ops::Bound::{Included, Unbounded}; --use std::sync::atomic::{AtomicBool, Ordering}; --use std::sync::Arc; --use std::time::{Duration, SystemTime}; -- --use ::fb_procfs as procfs; --use anyhow::{anyhow, bail, Context, Result}; --use bitvec::prelude::*; --use clap::Parser; --use log::{info, trace, warn}; --use ordered_float::OrderedFloat; -- --/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF --/// part does simple round robin in each domain and the userspace part --/// calculates the load factor of each domain and tells the BPF part how to load --/// balance the domains. -- --/// This scheduler demonstrates dividing scheduling logic between BPF and --/// userspace and using rust to build the userspace part. An earlier variant of --/// this scheduler was used to balance across six domains, each representing a --/// chiplet in a six-chiplet AMD processor, and could match the performance of --/// production setup using CFS. --#[derive(Debug, Parser)] --struct Opts { -- /// Scheduling slice duration in microseconds. -- #[clap(short, long, default_value = "20000")] -- slice_us: u64, -- -- /// Monitoring and load balance interval in seconds. -- #[clap(short, long, default_value = "2.0")] -- interval: f64, -- -- /// Build domains according to how CPUs are grouped at this cache level -- /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id. -- #[clap(short = 'c', long, default_value = "3")] -- cache_level: u32, -- -- /// Instead of using cache locality, set the cpumask for each domain -- /// manually, provide multiple --cpumasks, one for each domain. E.g. -- /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with -- /// the corresponding CPUs belonging to each domain. Each CPU must -- /// belong to precisely one domain. -- #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")] -- cpumasks: Vec, -- -- /// When non-zero, enable greedy task stealing. When a domain is idle, a -- /// cpu will attempt to steal tasks from a domain with at least -- /// greedy_threshold tasks enqueued. These tasks aren't permanently -- /// stolen from the domain. -- #[clap(short, long, default_value = "4")] -- greedy_threshold: u32, -- -- /// The load decay factor. Every interval, the existing load is decayed -- /// by this factor and new load is added. Must be in the range [0.0, -- /// 0.99]. The smaller the value, the more sensitive load calculation -- /// is to recent changes. When 0.0, history is ignored and the load -- /// value from the latest period is used directly. -- #[clap(short, long, default_value = "0.5")] -- load_decay_factor: f64, -- -- /// Disable load balancing. Unless disabled, periodically userspace will -- /// calculate the load factor of each domain and instruct BPF which -- /// processes to move. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- no_load_balance: bool, -- -- /// Put per-cpu kthreads directly into local dsq's. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- kthreads_local: bool, -- -- /// Use FIFO scheduling instead of weighted vtime scheduling. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- fifo_sched: bool, -- -- /// If specified, only tasks which have their scheduling policy set to -- /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all -- /// tasks are switched. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- partial: bool, -- -- /// Enable verbose output including libbpf details. Specify multiple -- /// times to increase verbosity. -- #[clap(short, long, action = clap::ArgAction::Count)] -- verbose: u8, --} -- --fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result { -- Ok(reader -- .read_stat() -- .context("Failed to read procfs")? -- .total_cpu -- .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?) --} -- --fn now_monotonic() -> u64 { -- let mut time = libc::timespec { -- tv_sec: 0, -- tv_nsec: 0, -- }; -- let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) }; -- assert!(ret == 0); -- time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 --} -- --fn clear_map(map: &mut libbpf_rs::Map) { -- // XXX: libbpf_rs has some design flaw that make it impossible to -- // delete while iterating despite it being safe so we alias it here -- let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) }; -- for key in map.keys() { -- let _ = deleter.delete(&key); -- } --} -- --#[derive(Debug)] --struct TaskLoad { -- runnable_for: u64, -- load: f64, --} -- --#[derive(Debug)] --struct TaskInfo { -- pid: i32, -- dom_mask: u64, -- migrated: Cell, --} -- --struct LoadBalancer<'a, 'b, 'c> { -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- -- tasks_by_load: Vec, TaskInfo>>, -- load_avg: f64, -- dom_loads: Vec, -- -- imbal: Vec, -- doms_to_push: BTreeMap, u32>, -- doms_to_pull: BTreeMap, u32>, -- -- nr_lb_data_errors: &'c mut u64, --} -- --impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> { -- const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10; -- const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1; -- const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50; -- -- fn new( -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- nr_lb_data_errors: &'c mut u64, -- ) -> Self { -- Self { -- maps, -- task_loads, -- nr_doms, -- load_decay_factor, -- -- tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(), -- load_avg: 0f64, -- dom_loads: vec![0.0; nr_doms], -- -- imbal: vec![0.0; nr_doms], -- doms_to_pull: BTreeMap::new(), -- doms_to_push: BTreeMap::new(), -- -- nr_lb_data_errors, -- } -- } -- -- fn read_task_loads(&mut self, period: Duration) -> Result<()> { -- let now_mono = now_monotonic(); -- let task_data = self.maps.task_data(); -- let mut this_task_loads = BTreeMap::::new(); -- let mut load_sum = 0.0f64; -- self.dom_loads = vec![0f64; self.nr_doms]; -- -- for key in task_data.keys() { -- if let Some(task_ctx_vec) = task_data -- .lookup(&key, libbpf_rs::MapFlags::ANY) -- .context("Failed to lookup task_data")? -- { -- let task_ctx = -- unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) }; -- let pid = i32::from_ne_bytes( -- key.as_slice() -- .try_into() -- .context("Invalid key length in task_data map")?, -- ); -- -- let (this_at, this_for, weight) = unsafe { -- ( -- std::ptr::read_volatile(&task_ctx.runnable_at as *const u64), -- std::ptr::read_volatile(&task_ctx.runnable_for as *const u64), -- std::ptr::read_volatile(&task_ctx.weight as *const u32), -- ) -- }; -- -- let (mut delta, prev_load) = match self.task_loads.get(&pid) { -- Some(prev) => (this_for - prev.runnable_for, Some(prev.load)), -- None => (this_for, None), -- }; -- -- // Non-zero this_at indicates that the task is currently -- // runnable. Note that we read runnable_at and runnable_for -- // without any synchronization and there is a small window -- // where we end up misaccounting. While this can cause -- // temporary error, it's unlikely to cause any noticeable -- // misbehavior especially given the load value clamping. -- if this_at > 0 && this_at < now_mono { -- delta += now_mono - this_at; -- } -- -- delta = delta.min(period.as_nanos() as u64); -- let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64) -- .clamp(0.0, weight as f64); -- -- let this_load = match prev_load { -- Some(prev_load) => { -- prev_load * self.load_decay_factor -- + this_load * (1.0 - self.load_decay_factor) -- } -- None => this_load, -- }; -- -- this_task_loads.insert( -- pid, -- TaskLoad { -- runnable_for: this_for, -- load: this_load, -- }, -- ); -- -- load_sum += this_load; -- self.dom_loads[task_ctx.dom_id as usize] += this_load; -- // Only record pids that are eligible for load balancing -- if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) { -- continue; -- } -- self.tasks_by_load[task_ctx.dom_id as usize].insert( -- OrderedFloat(this_load), -- TaskInfo { -- pid, -- dom_mask: task_ctx.dom_mask, -- migrated: Cell::new(false), -- }, -- ); -- } -- } -- -- self.load_avg = load_sum / self.nr_doms as f64; -- *self.task_loads = this_task_loads; -- Ok(()) -- } -- -- // To balance dom loads we identify doms with lower and higher load than average -- fn calculate_dom_load_balance(&mut self) -> Result<()> { -- for (dom, dom_load) in self.dom_loads.iter().enumerate() { -- let imbal = dom_load - self.load_avg; -- if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO { -- if imbal > 0f64 { -- self.doms_to_push.insert(OrderedFloat(imbal), dom as u32); -- } else { -- self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32); -- } -- self.imbal[dom] = imbal; -- } -- } -- Ok(()) -- } -- -- // Find the first candidate pid which hasn't already been migrated and -- // can run in @pull_dom. -- fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)> -- where -- I: IntoIterator, &'d TaskInfo)>, -- { -- match tasks_by_load -- .into_iter() -- .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0) -- .next() -- { -- Some((OrderedFloat(load), task)) => Some((*load, task)), -- None => None, -- } -- } -- -- fn pick_victim( -- &self, -- (push_dom, to_push): (u32, f64), -- (pull_dom, to_pull): (u32, f64), -- ) -> Option<(&TaskInfo, f64)> { -- let to_xfer = to_pull.min(to_push); -- -- trace!( -- "considering dom {}@{:.2} -> {}@{:.2}", -- push_dom, -- to_push, -- pull_dom, -- to_pull -- ); -- -- let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs(); -- -- trace!( -- "to_xfer={:.2} tasks_by_load={:?}", -- to_xfer, -- &self.tasks_by_load[push_dom as usize] -- ); -- -- // We want to pick a task to transfer from push_dom to pull_dom to -- // maximize the reduction of load imbalance between the two. IOW, -- // pick a task which has the closest load value to $to_xfer that can -- // be migrated. Find such task by locating the first migratable task -- // while scanning left from $to_xfer and the counterpart while -- // scanning right and picking the better of the two. -- let (load, task, new_imbal) = match ( -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Unbounded, Included(&OrderedFloat(to_xfer)))) -- .rev(), -- pull_dom, -- ), -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Included(&OrderedFloat(to_xfer)), Unbounded)), -- pull_dom, -- ), -- ) { -- (None, None) => return None, -- (Some((load, task)), None) | (None, Some((load, task))) => { -- (load, task, calc_new_imbal(load)) -- } -- (Some((load0, task0)), Some((load1, task1))) => { -- let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1)); -- if new_imbal0 <= new_imbal1 { -- (load0, task0, new_imbal0) -- } else { -- (load1, task1, new_imbal1) -- } -- } -- }; -- -- // If the best candidate can't reduce the imbalance, there's nothing -- // to do for this pair. -- let old_imbal = to_push + to_pull; -- if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal { -- trace!( -- "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal -- ); -- return None; -- } -- -- trace!( -- "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal, -- ); -- -- Some((task, load)) -- } -- -- // Actually execute the load balancing. Concretely this writes pid -> dom -- // entries into the lb_data map for bpf side to consume. -- fn load_balance(&mut self) -> Result<()> { -- clear_map(self.maps.lb_data()); -- -- trace!("imbal={:?}", &self.imbal); -- trace!("doms_to_push={:?}", &self.doms_to_push); -- trace!("doms_to_pull={:?}", &self.doms_to_pull); -- -- // Push from the most imbalanced to least. -- while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() { -- let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO; -- let mut pushed = 0f64; -- -- // Transfer tasks from push_dom to reduce imbalance. -- loop { -- let last_pushed = pushed; -- -- // Pull from the most imbalaned to least. -- let mut doms_to_pull = BTreeMap::<_, _>::new(); -- std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull); -- let mut pull_doms = doms_to_pull.into_iter().rev().collect::>(); -- -- for (to_pull, pull_dom) in pull_doms.iter_mut() { -- if let Some((task, load)) = -- self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull))) -- { -- // Execute migration. -- task.migrated.set(true); -- to_push -= load; -- *to_pull -= load; -- pushed += load; -- -- // Ask BPF code to execute the migration. -- let pid = task.pid; -- let cpid = (pid as libc::pid_t).to_ne_bytes(); -- if let Err(e) = self.maps.lb_data().update( -- &cpid, -- &pull_dom.to_ne_bytes(), -- libbpf_rs::MapFlags::NO_EXIST, -- ) { -- warn!( -- "Failed to update lb_data map for pid={} error={:?}", -- pid, &e -- ); -- *self.nr_lb_data_errors += 1; -- } -- -- // Always break after a successful migration so that -- // the pulling domains are always considered in the -- // descending imbalance order. -- break; -- } -- } -- -- pull_doms -- .into_iter() -- .map(|(k, v)| self.doms_to_pull.insert(k, v)) -- .count(); -- -- // Stop repeating if nothing got transferred or pushed enough. -- if pushed == last_pushed || pushed >= push_max { -- break; -- } -- } -- } -- Ok(()) -- } --} -- --struct Scheduler<'a> { -- skel: AtroposSkel<'a>, -- struct_ops: Option, -- -- nr_cpus: usize, -- nr_doms: usize, -- load_decay_factor: f64, -- balance_load: bool, -- -- proc_reader: procfs::ProcReader, -- -- prev_at: SystemTime, -- prev_total_cpu: procfs::CpuStat, -- task_loads: BTreeMap, -- -- nr_lb_data_errors: u64, --} -- --impl<'a> Scheduler<'a> { -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn parse_cpusets( -- cpumasks: &[String], -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- if cpumasks.len() > atropos_sys::MAX_DOMS as usize { -- bail!( -- "Number of requested DSQs ({}) is greater than MAX_DOMS ({})", -- cpumasks.len(), -- atropos_sys::MAX_DOMS -- ); -- } -- let mut cpus = vec![-1i32; nr_cpus]; -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()]; -- for (dq, cpumask) in cpumasks.iter().enumerate() { -- let hex_str = { -- let mut tmp_str = cpumask -- .strip_prefix("0x") -- .unwrap_or(cpumask) -- .replace('_', ""); -- if tmp_str.len() % 2 != 0 { -- tmp_str = "0".to_string() + &tmp_str; -- } -- tmp_str -- }; -- let byte_vec = hex::decode(&hex_str) -- .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?; -- -- for (index, &val) in byte_vec.iter().rev().enumerate() { -- let mut v = val; -- while v != 0 { -- let lsb = v.trailing_zeros() as usize; -- v &= !(1 << lsb); -- let cpu = index * 8 + lsb; -- if cpu > nr_cpus { -- bail!( -- concat!( -- "Found cpu ({}) in cpumask ({}) which is larger", -- " than the number of cpus on the machine ({})" -- ), -- cpu, -- cpumask, -- nr_cpus -- ); -- } -- if cpus[cpu] != -1 { -- bail!( -- "Found cpu ({}) with dq ({}) but also in cpumask ({})", -- cpu, -- cpus[cpu], -- cpumask -- ); -- } -- cpus[cpu] = dq as i32; -- cpusets[dq].set(cpu, true); -- } -- } -- cpusets[dq].set_uninitialized(false); -- } -- -- for (cpu, &dq) in cpus.iter().enumerate() { -- if dq < 0 { -- bail!( -- "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.", -- cpu -- ); -- } -- } -- -- Ok((cpusets, cpus)) -- } -- -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn cpusets_from_cache( -- level: u32, -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- let mut cpu_to_cache = vec![]; // (cpu_id, cache_id) -- let mut cache_ids = BTreeSet::::new(); -- let mut nr_not_found = 0; -- -- // Build cpu -> cache ID mapping. -- for cpu in 0..nr_cpus { -- let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level); -- let id = match std::fs::read_to_string(&path) { -- Ok(val) => val -- .trim() -- .parse::() -- .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?, -- Err(e) if e.kind() == std::io::ErrorKind::NotFound => { -- nr_not_found += 1; -- 0 -- } -- Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)), -- }; -- -- cpu_to_cache.push(id); -- cache_ids.insert(id); -- } -- -- if nr_not_found > 1 { -- warn!( -- "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0", -- level, nr_not_found, nr_cpus -- ); -- } -- -- // Cache IDs may have holes. Assign consecutive domain IDs to -- // existing cache IDs. -- let mut cache_to_dom = BTreeMap::::new(); -- let mut nr_doms = 0; -- for cache_id in cache_ids.iter() { -- cache_to_dom.insert(*cache_id, nr_doms); -- nr_doms += 1; -- } -- -- if nr_doms > atropos_sys::MAX_DOMS { -- bail!( -- "Total number of doms {} is greater than MAX_DOMS ({})", -- nr_doms, -- atropos_sys::MAX_DOMS -- ); -- } -- -- // Build and return dom -> cpumask and cpu -> dom mappings. -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize]; -- let mut cpu_to_dom = vec![]; -- -- for cpu in 0..nr_cpus { -- let dom_id = cache_to_dom[&cpu_to_cache[cpu]]; -- cpusets[dom_id as usize].set(cpu, true); -- cpu_to_dom.push(dom_id as i32); -- } -- -- Ok((cpusets, cpu_to_dom)) -- } -- -- fn init(opts: &Opts) -> Result { -- // Open the BPF prog first for verification. -- let mut skel_builder = AtroposSkelBuilder::default(); -- skel_builder.obj_builder.debug(opts.verbose > 0); -- let mut skel = skel_builder.open().context("Failed to open BPF program")?; -- -- let nr_cpus = libbpf_rs::num_possible_cpus().unwrap(); -- if nr_cpus > atropos_sys::MAX_CPUS as usize { -- bail!( -- "nr_cpus ({}) is greater than MAX_CPUS ({})", -- nr_cpus, -- atropos_sys::MAX_CPUS -- ); -- } -- -- // Initialize skel according to @opts. -- let (cpusets, cpus) = if opts.cpumasks.len() > 0 { -- Self::parse_cpusets(&opts.cpumasks, nr_cpus)? -- } else { -- Self::cpusets_from_cache(opts.cache_level, nr_cpus)? -- }; -- let nr_doms = cpusets.len(); -- skel.rodata().nr_doms = nr_doms as u32; -- skel.rodata().nr_cpus = nr_cpus as u32; -- -- for (cpu, dom) in cpus.iter().enumerate() { -- skel.rodata().cpu_dom_id_map[cpu] = *dom as u32; -- } -- -- for (dom, cpuset) in cpusets.iter().enumerate() { -- let raw_cpuset_slice = cpuset.as_raw_slice(); -- let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom]; -- let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len()); -- left.clone_from_slice(cpuset.as_raw_slice()); -- let cpumask_str = dom_cpumask_slice -- .iter() -- .take((nr_cpus + 63) / 64) -- .rev() -- .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x)); -- info!( -- "DOM[{:02}] cpumask{} ({} cpus)", -- dom, -- &cpumask_str, -- cpuset.count_ones() -- ); -- } -- -- skel.rodata().slice_us = opts.slice_us; -- skel.rodata().kthreads_local = opts.kthreads_local; -- skel.rodata().fifo_sched = opts.fifo_sched; -- skel.rodata().switch_partial = opts.partial; -- skel.rodata().greedy_threshold = opts.greedy_threshold; -- -- // Attach. -- let mut skel = skel.load().context("Failed to load BPF program")?; -- skel.attach().context("Failed to attach BPF program")?; -- let struct_ops = Some( -- skel.maps_mut() -- .atropos() -- .attach_struct_ops() -- .context("Failed to attach atropos struct ops")?, -- ); -- info!("Atropos Scheduler Attached"); -- -- // Other stuff. -- let mut proc_reader = procfs::ProcReader::new(); -- let prev_total_cpu = read_total_cpu(&mut proc_reader)?; -- -- Ok(Self { -- skel, -- struct_ops, // should be held to keep it attached -- -- nr_cpus, -- nr_doms, -- load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99), -- balance_load: !opts.no_load_balance, -- -- proc_reader, -- -- prev_at: SystemTime::now(), -- prev_total_cpu, -- task_loads: BTreeMap::new(), -- -- nr_lb_data_errors: 0, -- }) -- } -- -- fn get_cpu_busy(&mut self) -> Result { -- let total_cpu = read_total_cpu(&mut self.proc_reader)?; -- let busy = match (&self.prev_total_cpu, &total_cpu) { -- ( -- procfs::CpuStat { -- user_usec: Some(prev_user), -- nice_usec: Some(prev_nice), -- system_usec: Some(prev_system), -- idle_usec: Some(prev_idle), -- iowait_usec: Some(prev_iowait), -- irq_usec: Some(prev_irq), -- softirq_usec: Some(prev_softirq), -- stolen_usec: Some(prev_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- procfs::CpuStat { -- user_usec: Some(curr_user), -- nice_usec: Some(curr_nice), -- system_usec: Some(curr_system), -- idle_usec: Some(curr_idle), -- iowait_usec: Some(curr_iowait), -- irq_usec: Some(curr_irq), -- softirq_usec: Some(curr_softirq), -- stolen_usec: Some(curr_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- ) => { -- let idle_usec = curr_idle - prev_idle; -- let iowait_usec = curr_iowait - prev_iowait; -- let user_usec = curr_user - prev_user; -- let system_usec = curr_system - prev_system; -- let nice_usec = curr_nice - prev_nice; -- let irq_usec = curr_irq - prev_irq; -- let softirq_usec = curr_softirq - prev_softirq; -- let stolen_usec = curr_stolen - prev_stolen; -- -- let busy_usec = -- user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec; -- let total_usec = idle_usec + busy_usec + iowait_usec; -- busy_usec as f64 / total_usec as f64 -- } -- _ => { -- bail!("Some procfs stats are not populated!"); -- } -- }; -- -- self.prev_total_cpu = total_cpu; -- Ok(busy) -- } -- -- fn read_bpf_stats(&mut self) -> Result> { -- let mut maps = self.skel.maps_mut(); -- let stats_map = maps.stats(); -- let mut stats: Vec = Vec::new(); -- let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus]; -- -- for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS { -- let cpu_stat_vec = stats_map -- .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY) -- .with_context(|| format!("Failed to lookup stat {}", stat))? -- .expect("per-cpu stat should exist"); -- let sum = cpu_stat_vec -- .iter() -- .map(|val| { -- u64::from_ne_bytes( -- val.as_slice() -- .try_into() -- .expect("Invalid value length in stat map"), -- ) -- }) -- .sum(); -- stats_map -- .update_percpu( -- &(stat as u32).to_ne_bytes(), -- &zero_vec, -- libbpf_rs::MapFlags::ANY, -- ) -- .context("Failed to zero stat")?; -- stats.push(sum); -- } -- Ok(stats) -- } -- -- fn report( -- &self, -- stats: &Vec, -- cpu_busy: f64, -- processing_dur: Duration, -- load_avg: f64, -- dom_loads: &Vec, -- imbal: &Vec, -- ) { -- let stat = |idx| stats[idx as usize]; -- let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK); -- -- info!( -- "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms", -- cpu_busy * 100.0, -- load_avg, -- stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize], -- stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize], -- self.nr_lb_data_errors, -- processing_dur.as_millis(), -- ); -- -- let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0; -- -- info!( -- "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}", -- total, -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY), -- ); -- -- for i in 0..self.nr_doms { -- info!( -- "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}", -- i, -- dom_loads[i], -- if imbal[i] < 0.0 { -imbal[i] } else { 0.0 }, -- if imbal[i] > 0.0 { imbal[i] } else { 0.0 }, -- ); -- } -- } -- -- fn step(&mut self) -> Result<()> { -- let started_at = std::time::SystemTime::now(); -- let bpf_stats = self.read_bpf_stats()?; -- let cpu_busy = self.get_cpu_busy()?; -- -- let mut lb = LoadBalancer::new( -- self.skel.maps_mut(), -- &mut self.task_loads, -- self.nr_doms, -- self.load_decay_factor, -- &mut self.nr_lb_data_errors, -- ); -- -- lb.read_task_loads(started_at.duration_since(self.prev_at)?)?; -- lb.calculate_dom_load_balance()?; -- -- if self.balance_load { -- lb.load_balance()?; -- } -- -- // Extract fields needed for reporting and drop lb to release -- // mutable borrows. -- let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal); -- -- self.report( -- &bpf_stats, -- cpu_busy, -- std::time::SystemTime::now().duration_since(started_at)?, -- load_avg, -- &dom_loads, -- &imbal, -- ); -- -- self.prev_at = started_at; -- Ok(()) -- } -- -- fn read_bpf_exit_type(&mut self) -> i32 { -- unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) } -- } -- -- fn report_bpf_exit_type(&mut self) -> Result<()> { -- // Report msg if EXT_OPS_EXIT_ERROR. -- match self.read_bpf_exit_type() { -- 0 => Ok(()), -- etype if etype == 2 => { -- let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) }; -- let msg = cstr -- .to_str() -- .context("Failed to convert exit msg to string") -- .unwrap(); -- bail!("BPF exit_type={} msg={}", etype, msg); -- } -- etype => { -- info!("BPF exit_type={}", etype); -- Ok(()) -- } -- } -- } --} -- --impl<'a> Drop for Scheduler<'a> { -- fn drop(&mut self) { -- if let Some(struct_ops) = self.struct_ops.take() { -- drop(struct_ops); -- } -- } --} -- --fn main() -> Result<()> { -- let opts = Opts::parse(); -- -- let llv = match opts.verbose { -- 0 => simplelog::LevelFilter::Info, -- 1 => simplelog::LevelFilter::Debug, -- _ => simplelog::LevelFilter::Trace, -- }; -- let mut lcfg = simplelog::ConfigBuilder::new(); -- lcfg.set_time_level(simplelog::LevelFilter::Error) -- .set_location_level(simplelog::LevelFilter::Off) -- .set_target_level(simplelog::LevelFilter::Off) -- .set_thread_level(simplelog::LevelFilter::Off); -- simplelog::TermLogger::init( -- llv, -- lcfg.build(), -- simplelog::TerminalMode::Stderr, -- simplelog::ColorChoice::Auto, -- )?; -- -- let shutdown = Arc::new(AtomicBool::new(false)); -- let shutdown_clone = shutdown.clone(); -- ctrlc::set_handler(move || { -- shutdown_clone.store(true, Ordering::Relaxed); -- }) -- .context("Error setting Ctrl-C handler")?; -- -- let mut sched = Scheduler::init(&opts)?; -- -- while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 { -- std::thread::sleep(Duration::from_secs_f64(opts.interval)); -- sched.step()?; -- } -- -- sched.report_bpf_exit_type() --} -diff --git a/tools/sched_ext/gnu/stubs.h b/tools/sched_ext/gnu/stubs.h -deleted file mode 100644 -index 719225b16626..000000000000 ---- a/tools/sched_ext/gnu/stubs.h -+++ /dev/null -@@ -1 +0,0 @@ --/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ -diff --git a/tools/sched_ext/scx_common.bpf.h b/tools/sched_ext/scx_common.bpf.h -deleted file mode 100644 -index e56de9dc86f2..000000000000 ---- a/tools/sched_ext/scx_common.bpf.h -+++ /dev/null -@@ -1,288 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __SCHED_EXT_COMMON_BPF_H --#define __SCHED_EXT_COMMON_BPF_H -- --#include "vmlinux.h" --#include --#include --#include --#include "user_exit_info.h" -- --#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ --#define PF_EXITING 0x00000004 --#define CLOCK_MONOTONIC 1 -- --/* -- * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can -- * lead to really confusing misbehaviors. Let's trigger a build failure. -- */ --static inline void ___vmlinux_h_sanity_check___(void) --{ -- _Static_assert(SCX_DSQ_FLAG_BUILTIN, -- "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); --} -- --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; -- --static inline __attribute__((format(printf, 1, 2))) --void ___scx_bpf_error_format_checker(const char *fmt, ...) {} -- --/* -- * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments -- * instead of an array of u64. Note that __param[] must have at least one -- * element to keep the verifier happy. -- */ --#define scx_bpf_error(fmt, args...) \ --({ \ -- static char ___fmt[] = fmt; \ -- unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ -- \ -- _Pragma("GCC diagnostic push") \ -- _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ -- ___bpf_fill(___param, args); \ -- _Pragma("GCC diagnostic pop") \ -- \ -- scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ -- \ -- ___scx_bpf_error_format_checker(fmt, ##args); \ --}) -- --void scx_bpf_switch_all(void) __ksym; --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; --bool scx_bpf_consume(u64 dsq_id) __ksym; --u32 scx_bpf_dispatch_nr_slots(void) __ksym; --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym; --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym; --void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; --s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym; --const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; --const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; --void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; --void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; --bool scx_bpf_task_running(const struct task_struct *p) __ksym; --s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym; --u32 scx_bpf_reenqueue_local(void) __ksym; -- --#define BPF_STRUCT_OPS(name, args...) \ --SEC("struct_ops/"#name) \ --BPF_PROG(name, ##args) -- --#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ --SEC("struct_ops.s/"#name) \ --BPF_PROG(name, ##args) -- --/** -- * MEMBER_VPTR - Obtain the verified pointer to a struct or array member -- * @base: struct or array to index -- * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...) -- * -- * The verifier often gets confused by the instruction sequence the compiler -- * generates for indexing struct fields or arrays. This macro forces the -- * compiler to generate a code sequence which first calculates the byte offset, -- * checks it against the struct or array size and add that byte offset to -- * generate the pointer to the member to help the verifier. -- * -- * Ideally, we want to abort if the calculated offset is out-of-bounds. However, -- * BPF currently doesn't support abort, so evaluate to NULL instead. The caller -- * must check for NULL and take appropriate action to appease the verifier. To -- * avoid confusing the verifier, it's best to check for NULL and dereference -- * immediately. -- * -- * vptr = MEMBER_VPTR(my_array, [i][j]); -- * if (!vptr) -- * return error; -- * *vptr = new_value; -- */ --#define MEMBER_VPTR(base, member) (typeof(base member) *)({ \ -- u64 __base = (u64)base; \ -- u64 __addr = (u64)&(base member) - __base; \ -- asm volatile ( \ -- "if %0 <= %[max] goto +2\n" \ -- "%0 = 0\n" \ -- "goto +1\n" \ -- "%0 += %1\n" \ -- : "+r"(__addr) \ -- : "r"(__base), \ -- [max]"i"(sizeof(base) - sizeof(base member))); \ -- __addr; \ --}) -- --/* -- * BPF core and other generic helpers -- */ -- --/* list and rbtree */ --#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) --#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) -- --void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; --void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; -- --#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) --#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) -- --void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; --struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; --struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, -- struct bpf_rb_node *node) __ksym; --void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, -- bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym; --struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; -- --/* task */ --struct task_struct *bpf_task_from_pid(s32 pid) __ksym; --struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; --void bpf_task_release(struct task_struct *p) __ksym; -- --/* cgroup */ --struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; --void bpf_cgroup_release(struct cgroup *cgrp) __ksym; --struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; -- --/* cpumask */ --struct bpf_cpumask *bpf_cpumask_create(void) __ksym; --struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; --u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; --u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym; -- --/* rcu */ --void bpf_rcu_read_lock(void) __ksym; --void bpf_rcu_read_unlock(void) __ksym; -- --/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */ --struct bpf_iter_num; -- --extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym; --extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym; --extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym; -- --#ifndef bpf_for_each --/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for -- * using BPF open-coded iterators without having to write mundane explicit -- * low-level loop logic. Instead, it provides for()-like generic construct -- * that can be used pretty naturally. E.g., for some hypothetical cgroup -- * iterator, you'd write: -- * -- * struct cgroup *cg, *parent_cg = <...>; -- * -- * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { -- * bpf_printk("Child cgroup id = %d", cg->cgroup_id); -- * if (cg->cgroup_id == 123) -- * break; -- * } -- * -- * I.e., it looks almost like high-level for each loop in other languages, -- * supports continue/break, and is verifiable by BPF verifier. -- * -- * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) -- * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to -- * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int -- * *`, not just `int`. So for integers bpf_for() is more convenient. -- * -- * Note: this macro relies on C99 feature of allowing to declare variables -- * inside for() loop, bound to for() loop lifetime. It also utilizes GCC -- * extension: __attribute__((cleanup())), supported by both GCC and -- * Clang. -- */ --#define bpf_for_each(type, cur, args...) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ -- cleanup(bpf_iter_##type##_destroy))), \ -- /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_##type##_new(&___it, ##args), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_##type##_destroy, (void *)0); \ -- /* iteration and termination check */ \ -- (((cur) = bpf_iter_##type##_next(&___it))); \ --) --#endif /* bpf_for_each */ -- --#ifndef bpf_for --/* bpf_for(i, start, end) implements a for()-like looping construct that sets -- * provided integer variable *i* to values starting from *start* through, -- * but not including, *end*. It also proves to BPF verifier that *i* belongs -- * to range [start, end), so this can be used for accessing arrays without -- * extra checks. -- * -- * Note: *start* and *end* are assumed to be expressions with no side effects -- * and whose values do not change throughout bpf_for() loop execution. They do -- * not have to be statically known or constant, though. -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_for(i, start, end) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, (start), (end)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- ({ \ -- /* iteration step */ \ -- int *___t = bpf_iter_num_next(&___it); \ -- /* termination and bounds check */ \ -- (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ -- }); \ --) --#endif /* bpf_for */ -- --#ifndef bpf_repeat --/* bpf_repeat(N) performs N iterations without exposing iteration number -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_repeat(N) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, 0, (N)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- bpf_iter_num_next(&___it); \ -- /* nothing here */ \ --) --#endif /* bpf_repeat */ -- --#endif /* __SCHED_EXT_COMMON_BPF_H */ -diff --git a/tools/sched_ext/scx_example_central.bpf.c b/tools/sched_ext/scx_example_central.bpf.c -deleted file mode 100644 -index 4cec04b4c2ed..000000000000 ---- a/tools/sched_ext/scx_example_central.bpf.c -+++ /dev/null -@@ -1,334 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A central FIFO sched_ext scheduler which demonstrates the followings: -- * -- * a. Making all scheduling decisions from one CPU: -- * -- * The central CPU is the only one making scheduling decisions. All other -- * CPUs kick the central CPU when they run out of tasks to run. -- * -- * There is one global BPF queue and the central CPU schedules all CPUs by -- * dispatching from the global queue to each CPU's local dsq from dispatch(). -- * This isn't the most straightforward. e.g. It'd be easier to bounce -- * through per-CPU BPF queues. The current design is chosen to maximally -- * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching. -- * -- * b. Tickless operation -- * -- * All tasks are dispatched with the infinite slice which allows stopping the -- * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full -- * parameter. The tickless operation can be observed through -- * /proc/interrupts. -- * -- * Periodic switching is enforced by a periodic timer checking all CPUs and -- * preempting them as necessary. Unfortunately, BPF timer currently doesn't -- * have a way to pin to a specific CPU, so the periodic timer isn't pinned to -- * the central CPU. -- * -- * c. Preemption -- * -- * Kthreads are unconditionally queued to the head of a matching local dsq -- * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always -- * prioritized over user threads, which is required for ensuring forward -- * progress as e.g. the periodic timer may run on a ksoftirqd and if the -- * ksoftirqd gets starved by a user thread, there may not be anything else to -- * vacate that user thread. -- * -- * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the -- * next tasks. -- * -- * This scheduler is designed to maximize usage of various SCX mechanisms. A -- * more practical implementation would likely put the scheduling loop outside -- * the central CPU's dispatch() path and add some form of priority mechanism. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --enum { -- FALLBACK_DSQ_ID = 0, -- MAX_CPUS = 4096, -- MS_TO_NS = 1000LLU * 1000, -- TIMER_INTERVAL_NS = 1 * MS_TO_NS, --}; -- --const volatile bool switch_partial; --const volatile s32 central_cpu; --const volatile u32 nr_cpu_ids = 64; /* !0 for veristat, set during init */ -- --u64 nr_total, nr_locals, nr_queued, nr_lost_pids; --u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries; --u64 nr_overflows; -- --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, s32); --} central_q SEC(".maps"); -- --/* can't use percpu map due to bad lookups */ --static bool cpu_gimme_task[MAX_CPUS]; --static u64 cpu_started_at[MAX_CPUS]; -- --struct central_timer { -- struct bpf_timer timer; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, 1); -- __type(key, u32); -- __type(value, struct central_timer); --} central_timer SEC(".maps"); -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- /* -- * Steer wakeups to the central CPU as much as possible to avoid -- * disturbing other CPUs. It's safe to blindly return the central cpu as -- * select_cpu() is a hint and if @p can't be on it, the kernel will -- * automatically pick a fallback CPU. -- */ -- return central_cpu; --} -- --void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags) --{ -- s32 pid = p->pid; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- /* -- * Push per-cpu kthreads at the head of local dsq's and preempt the -- * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked -- * behind other threads which is necessary for forward progress -- * guarantee as we depend on the BPF timer which may run from ksoftirqd. -- */ -- if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- __sync_fetch_and_add(&nr_locals, 1); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF, -- enq_flags | SCX_ENQ_PREEMPT); -- return; -- } -- -- if (bpf_map_push_elem(¢ral_q, &pid, 0)) { -- __sync_fetch_and_add(&nr_overflows, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_queued, 1); -- -- if (!scx_bpf_task_running(p)) -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); --} -- --static bool dispatch_to_cpu(s32 cpu) --{ -- struct task_struct *p; -- s32 pid; -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (bpf_map_pop_elem(¢ral_q, &pid)) -- break; -- -- __sync_fetch_and_sub(&nr_queued, 1); -- -- p = bpf_task_from_pid(pid); -- if (!p) { -- __sync_fetch_and_add(&nr_lost_pids, 1); -- continue; -- } -- -- /* -- * If we can't run the task at the top, do the dumb thing and -- * bounce it to the fallback dsq. -- */ -- if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- __sync_fetch_and_add(&nr_mismatches, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0); -- bpf_task_release(p); -- continue; -- } -- -- /* dispatch to local and mark that @cpu doesn't need more */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0); -- -- if (cpu != central_cpu) -- scx_bpf_kick_cpu(cpu, 0); -- -- bpf_task_release(p); -- return true; -- } -- -- return false; --} -- --void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (cpu == central_cpu) { -- /* dispatch for all other CPUs first */ -- __sync_fetch_and_add(&nr_dispatches, 1); -- -- bpf_for(cpu, 0, nr_cpu_ids) { -- bool *gimme; -- -- if (!scx_bpf_dispatch_nr_slots()) -- break; -- -- /* central's gimme is never set */ -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme && !*gimme) -- continue; -- -- if (dispatch_to_cpu(cpu)) -- *gimme = false; -- } -- -- /* -- * Retry if we ran out of dispatch buffer slots as we might have -- * skipped some CPUs and also need to dispatch for self. The ext -- * core automatically retries if the local dsq is empty but we -- * can't rely on that as we're dispatching for other CPUs too. -- * Kick self explicitly to retry. -- */ -- if (!scx_bpf_dispatch_nr_slots()) { -- __sync_fetch_and_add(&nr_retries, 1); -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- return; -- } -- -- /* look for a task to run on the central CPU */ -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- dispatch_to_cpu(central_cpu); -- } else { -- bool *gimme; -- -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme) -- *gimme = true; -- -- /* -- * Force dispatch on the scheduling CPU so that it finds a task -- * to run for us. -- */ -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- } --} -- --void BPF_STRUCT_OPS(central_running, struct task_struct *p) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ --} -- --void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = 0; --} -- --static int central_timerfn(void *map, int *key, struct bpf_timer *timer) --{ -- u64 now = bpf_ktime_get_ns(); -- u64 nr_to_kick = nr_queued; -- s32 i; -- -- bpf_for(i, 0, nr_cpu_ids) { -- s32 cpu = (nr_timers + i) % nr_cpu_ids; -- u64 *started_at; -- -- if (cpu == central_cpu) -- continue; -- -- /* kick iff the current one exhausted its slice */ -- started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at && *started_at && -- vtime_before(now, *started_at + SCX_SLICE_DFL)) -- continue; -- -- /* and there's something pending */ -- if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) || -- scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu)) -- ; -- else if (nr_to_kick) -- nr_to_kick--; -- else -- continue; -- -- scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); -- } -- -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- -- bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- __sync_fetch_and_add(&nr_timers, 1); -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(central_init) --{ -- u32 key = 0; -- struct bpf_timer *timer; -- int ret; -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1); -- if (ret) -- return ret; -- -- timer = bpf_map_lookup_elem(¢ral_timer, &key); -- if (!timer) -- return -ESRCH; -- -- bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); -- bpf_timer_set_callback(timer, central_timerfn); -- ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- return ret; --} -- --void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops central_ops = { -- /* -- * We are offloading all scheduling decisions to the central CPU and -- * thus being the last task on a given CPU doesn't mean anything -- * special. Enqueue the last tasks like any other tasks. -- */ -- .flags = SCX_OPS_ENQ_LAST, -- -- .select_cpu = (void *)central_select_cpu, -- .enqueue = (void *)central_enqueue, -- .dispatch = (void *)central_dispatch, -- .running = (void *)central_running, -- .stopping = (void *)central_stopping, -- .init = (void *)central_init, -- .exit = (void *)central_exit, -- .name = "central", --}; -diff --git a/tools/sched_ext/scx_example_central.c b/tools/sched_ext/scx_example_central.c -deleted file mode 100644 -index 7ad591cbdc65..000000000000 ---- a/tools/sched_ext/scx_example_central.c -+++ /dev/null -@@ -1,94 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_central.skel.h" -- --const char help_fmt[] = --"A central FIFO sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-c CPU] [-p]\n" --"\n" --" -c CPU Override the central CPU (default: 0)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_central *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_central__open(); -- assert(skel); -- -- skel->rodata->central_cpu = 0; -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "c:ph")) != -1) { -- switch (opt) { -- case 'c': -- skel->rodata->central_cpu = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_central__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.central_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf("total :%10lu local:%10lu queued:%10lu lost:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_locals, -- skel->bss->nr_queued, -- skel->bss->nr_lost_pids); -- printf("timer :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n", -- skel->bss->nr_timers, -- skel->bss->nr_dispatches, -- skel->bss->nr_mismatches, -- skel->bss->nr_retries); -- printf("overflow:%10lu\n", -- skel->bss->nr_overflows); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_central__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_flatcg.bpf.c b/tools/sched_ext/scx_example_flatcg.bpf.c -deleted file mode 100644 -index f6078b9a681f..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.bpf.c -+++ /dev/null -@@ -1,872 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext flattened cgroup hierarchy scheduler. It implements -- * hierarchical weight-based cgroup CPU control by flattening the cgroup -- * hierarchy into a single layer by compounding the active weight share at each -- * level. Consider the following hierarchy with weights in parentheses: -- * -- * R + A (100) + B (100) -- * | \ C (100) -- * \ D (200) -- * -- * Ignoring the root and threaded cgroups, only B, C and D can contain tasks. -- * Let's say all three have runnable tasks. The total share that each of these -- * three cgroups is entitled to can be calculated by compounding its share at -- * each level. -- * -- * For example, B is competing against C and in that competition its share is -- * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's -- * share in that competition is 200/(200+100) == 1/3. B's eventual share in the -- * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's -- * eventual shaer is the same at 1/6. D is only competing at the top level and -- * its share is 200/(100+200) == 2/3. -- * -- * So, instead of hierarchically scheduling level-by-level, we can consider it -- * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3 -- * and keep updating the eventual shares as the cgroups' runnable states change. -- * -- * This flattening of hierarchy can bring a substantial performance gain when -- * the cgroup hierarchy is nested multiple levels. in a simple benchmark using -- * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it -- * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two -- * apache instances competing with 2:1 weight ratio nested four level deep. -- * -- * However, the gain comes at the cost of not being able to properly handle -- * thundering herd of cgroups. For example, if many cgroups which are nested -- * behind a low priority parent cgroup wake up around the same time, they may be -- * able to consume more CPU cycles than they are entitled to. In many use cases, -- * this isn't a real concern especially given the performance gain. Also, there -- * are ways to mitigate the problem further by e.g. introducing an extra -- * scheduling layer on cgroup delegation boundaries. -- * -- * The scheduler first picks the cgroup to run and then schedule the tasks -- * within by using nested weighted vtime scheduling by default. The -- * cgroup-internal scheduling can be switched to FIFO with the -f option. -- */ --#include "scx_common.bpf.h" --#include "user_exit_info.h" --#include "scx_example_flatcg.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */ --const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL; --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --u64 cvtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, u64); -- __uint(max_entries, FCG_NR_STATS); --} stats SEC(".maps"); -- --static void stat_inc(enum fcg_stat_idx idx) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p)++; --} -- --struct fcg_cpu_ctx { -- u64 cur_cgid; -- u64 cur_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, struct fcg_cpu_ctx); -- __uint(max_entries, 1); --} cpu_ctx SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_cgrp_ctx); --} cgrp_ctx SEC(".maps"); -- --struct cgv_node { -- struct bpf_rb_node rb_node; -- __u64 cvtime; -- __u64 cgid; --}; -- --private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock; --private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node); -- --struct cgv_node_stash { -- struct cgv_node __kptr *node; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, 16384); -- __type(key, __u64); -- __type(value, struct cgv_node_stash); --} cgv_node_stash SEC(".maps"); -- --struct fcg_task_ctx { -- u64 bypassed_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_task_ctx); --} task_ctx SEC(".maps"); -- --/* gets inc'd on weight tree changes to expire the cached hweights */ --unsigned long hweight_gen = 1; -- --static u64 div_round_up(u64 dividend, u64 divisor) --{ -- return (dividend + divisor - 1) / divisor; --} -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) --{ -- struct cgv_node *cgc_a, *cgc_b; -- -- cgc_a = container_of(a, struct cgv_node, rb_node); -- cgc_b = container_of(b, struct cgv_node, rb_node); -- -- return cgc_a->cvtime < cgc_b->cvtime; --} -- --static struct fcg_cpu_ctx *find_cpu_ctx(void) --{ -- struct fcg_cpu_ctx *cpuc; -- u32 idx = 0; -- -- cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx); -- if (!cpuc) { -- scx_bpf_error("cpu_ctx lookup failed"); -- return NULL; -- } -- return cpuc; --} -- --static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id); -- return NULL; -- } -- return cgc; --} -- --static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgrp = bpf_cgroup_ancestor(cgrp, level); -- if (!cgrp) { -- scx_bpf_error("ancestor cgroup lookup failed"); -- return NULL; -- } -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- scx_bpf_error("ancestor cgrp_ctx lookup failed"); -- bpf_cgroup_release(cgrp); -- return cgc; --} -- --static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- int level; -- -- if (!cgc->nr_active) { -- stat_inc(FCG_STAT_HWT_SKIP); -- return; -- } -- -- if (cgc->hweight_gen == hweight_gen) { -- stat_inc(FCG_STAT_HWT_CACHE); -- return; -- } -- -- stat_inc(FCG_STAT_HWT_UPDATES); -- bpf_for(level, 0, cgrp->level + 1) { -- struct fcg_cgrp_ctx *cgc; -- bool is_active; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- -- if (!level) { -- cgc->hweight = FCG_HWEIGHT_ONE; -- cgc->hweight_gen = hweight_gen; -- } else { -- struct fcg_cgrp_ctx *pcgc; -- -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- -- /* -- * We can be oppotunistic here and not grab the -- * cgv_tree_lock and deal with the occasional races. -- * However, hweight updates are already cached and -- * relatively low-frequency. Let's just do the -- * straightforward thing. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- is_active = cgc->nr_active; -- if (is_active) { -- cgc->hweight_gen = pcgc->hweight_gen; -- cgc->hweight = -- div_round_up(pcgc->hweight * cgc->weight, -- pcgc->child_weight_sum); -- } -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!is_active) { -- stat_inc(FCG_STAT_HWT_RACE); -- break; -- } -- } -- } --} -- --static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) --{ -- u64 delta, cvtime, max_budget; -- -- /* -- * A node which is on the rbtree can't be pointed to from elsewhere yet -- * and thus can't be updated and repositioned. Instead, we collect the -- * vtime deltas separately and apply it asynchronously here. -- */ -- delta = cgc->cvtime_delta; -- __sync_fetch_and_sub(&cgc->cvtime_delta, delta); -- cvtime = cgv_node->cvtime + delta; -- -- /* -- * Allow a cgroup to carry the maximum budget proportional to its -- * hweight such that a full-hweight cgroup can immediately take up half -- * of the CPUs at the most while staying at the front of the rbtree. -- */ -- max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / -- (2 * FCG_HWEIGHT_ONE); -- if (vtime_before(cvtime, cvtime_now - max_budget)) -- cvtime = cvtime_now - max_budget; -- -- cgv_node->cvtime = cvtime; --} -- --static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- u64 cgid = cgrp->kn->id; -- -- /* paired with cmpxchg in try_pick_next_cgroup() */ -- if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { -- stat_inc(FCG_STAT_ENQ_SKIP); -- return; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid); -- return; -- } -- -- /* NULL if the node is already on the rbtree */ -- cgv_node = bpf_kptr_xchg(&stash->node, NULL); -- if (!cgv_node) { -- stat_inc(FCG_STAT_ENQ_RACE); -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); --} -- --void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * If select_cpu_dfl() is recommending local enqueue, the target CPU is -- * idle. Follow it and charge the cgroup later in fcg_stopping() after -- * the fact. Use the same mechanism to deal with tasks with custom -- * affinities so that we don't have to worry about per-cgroup dq's -- * containing tasks that can't be executed from some CPUs. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) { -- /* -- * Tell fcg_stopping() that this bypassed the regular scheduling -- * path and should be force charged to the cgroup. 0 is used to -- * indicate that the task isn't bypassing, so if the current -- * runtime is 0, go back by one nanosecond. -- */ -- taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1; -- -- /* -- * The global dq is deprioritized as we don't want to let tasks -- * to boost themselves by constraining its cpumask. The -- * deprioritization is rather severe, so let's not apply that to -- * per-cpu kernel threads. This is ham-fisted. We probably wanna -- * implement per-cgroup fallback dq's instead so that we have -- * more control over when tasks with custom cpumask get issued. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || -- (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) { -- stat_inc(FCG_STAT_LOCAL); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- } else { -- stat_inc(FCG_STAT_GLOBAL); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } -- return; -- } -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- goto out_release; -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 tvtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) -- tvtime = cgc->tvtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, -- tvtime, enq_flags); -- } -- -- cgrp_enqueued(cgrp, cgc); --out_release: -- bpf_cgroup_release(cgrp); --} -- --/* -- * Walk the cgroup tree to update the active weight sums as tasks wake up and -- * sleep. The weight sums are used as the base when calculating the proportion a -- * given cgroup or task is entitled to at each level. -- */ --static void update_active_weight_sums(struct cgroup *cgrp, bool runnable) --{ -- struct fcg_cgrp_ctx *cgc; -- bool updated = false; -- int idx; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- /* -- * In most cases, a hot cgroup would have multiple threads going to -- * sleep and waking up while the whole cgroup stays active. In leaf -- * cgroups, ->nr_runnable which is updated with __sync operations gates -- * ->nr_active updates, so that we don't have to grab the cgv_tree_lock -- * repeatedly for a busy cgroup which is staying active. -- */ -- if (runnable) { -- if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_ACT); -- } else { -- if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_DEACT); -- } -- -- /* -- * If @cgrp is becoming runnable, its hweight should be refreshed after -- * it's added to the weight tree so that enqueue has the up-to-date -- * value. If @cgrp is becoming quiescent, the hweight should be -- * refreshed before it's removed from the weight tree so that the usage -- * charging which happens afterwards has access to the latest value. -- */ -- if (!runnable) -- cgrp_refresh_hweight(cgrp, cgc); -- -- /* propagate upwards */ -- bpf_for(idx, 0, cgrp->level) { -- int level = cgrp->level - idx; -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- bool propagate = false; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- if (level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- } -- -- /* -- * We need the propagation protected by a lock to synchronize -- * against weight changes. There's no reason to drop the lock at -- * each level but bpf_spin_lock() doesn't want any function -- * calls while locked. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- -- if (runnable) { -- if (!cgc->nr_active++) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum += cgc->weight; -- } -- } -- } else { -- if (!--cgc->nr_active) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum -= cgc->weight; -- } -- } -- } -- -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!propagate) -- break; -- } -- -- if (updated) -- __sync_fetch_and_add(&hweight_gen, 1); -- -- if (runnable) -- cgrp_refresh_hweight(cgrp, cgc); --} -- --void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, true); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) --{ -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- if (fifo_sched) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- /* -- * @cgc->tvtime_now always progresses forward as tasks start -- * executing. The test and update can be performed concurrently -- * from multiple CPUs and thus racy. Any error should be -- * contained and temporary. Let's just live with it. -- */ -- if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) -- cgc->tvtime_now = p->scx.dsq_vtime; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- /* scale the execution time by the inverse of the weight and charge */ -- if (!fifo_sched) -- p->scx.dsq_vtime += -- (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- if (!taskc->bypassed_at) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- __sync_fetch_and_add(&cgc->cvtime_delta, -- p->se.sum_exec_runtime - taskc->bypassed_at); -- taskc->bypassed_at = 0; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, false); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) --{ -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- if (cgrp->level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1); -- if (!pcgc) -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- if (pcgc && cgc->nr_active) -- pcgc->child_weight_sum += (s64)weight - cgc->weight; -- cgc->weight = weight; -- bpf_spin_unlock(&cgv_tree_lock); --} -- --static bool try_pick_next_cgroup(u64 *cgidp) --{ -- struct bpf_rb_node *rb_node; -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 cgid; -- -- /* pop the front cgroup and wind cvtime_now accordingly */ -- bpf_spin_lock(&cgv_tree_lock); -- -- rb_node = bpf_rbtree_first(&cgv_tree); -- if (!rb_node) { -- bpf_spin_unlock(&cgv_tree_lock); -- stat_inc(FCG_STAT_PNC_NO_CGRP); -- *cgidp = 0; -- return true; -- } -- -- rb_node = bpf_rbtree_remove(&cgv_tree, rb_node); -- bpf_spin_unlock(&cgv_tree_lock); -- -- cgv_node = container_of(rb_node, struct cgv_node, rb_node); -- cgid = cgv_node->cgid; -- -- if (vtime_before(cvtime_now, cgv_node->cvtime)) -- cvtime_now = cgv_node->cvtime; -- -- /* -- * If lookup fails, the cgroup's gone. Free and move on. See -- * fcg_cgroup_exit(). -- */ -- cgrp = bpf_cgroup_from_id(cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- if (!scx_bpf_consume(cgid)) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_EMPTY); -- goto out_stash; -- } -- -- /* -- * Successfully consumed from the cgroup. This will be our current -- * cgroup for the new slice. Refresh its hweight. -- */ -- cgrp_refresh_hweight(cgrp, cgc); -- -- bpf_cgroup_release(cgrp); -- -- /* -- * As the cgroup may have more tasks, add it back to the rbtree. Note -- * that here we charge the full slice upfront and then exact later -- * according to the actual consumption. This prevents lowpri thundering -- * herd from saturating the machine. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- -- *cgidp = cgid; -- stat_inc(FCG_STAT_PNC_NEXT); -- return true; -- --out_stash: -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- /* -- * Paired with cmpxchg in cgrp_enqueued(). If they see the following -- * transition, they'll enqueue the cgroup. If they are earlier, we'll -- * see their task in the dq below and requeue the cgroup. -- */ -- __sync_val_compare_and_swap(&cgc->queued, 1, 0); -- -- if (scx_bpf_dsq_nr_queued(cgid)) { -- bpf_spin_lock(&cgv_tree_lock); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- goto out_free; -- } -- } -- -- return false; -- --out_free: -- bpf_obj_drop(cgv_node); -- return false; --} -- --void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) --{ -- struct fcg_cpu_ctx *cpuc; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 now = bpf_ktime_get_ns(); -- -- cpuc = find_cpu_ctx(); -- if (!cpuc) -- return; -- -- if (!cpuc->cur_cgid) -- goto pick_next_cgroup; -- -- if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { -- if (scx_bpf_consume(cpuc->cur_cgid)) { -- stat_inc(FCG_STAT_CNS_KEEP); -- return; -- } -- stat_inc(FCG_STAT_CNS_EMPTY); -- } else { -- stat_inc(FCG_STAT_CNS_EXPIRE); -- } -- -- /* -- * The current cgroup is expiring. It was already charged a full slice. -- * Calculate the actual usage and accumulate the delta. -- */ -- cgrp = bpf_cgroup_from_id(cpuc->cur_cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_CNS_GONE); -- goto pick_next_cgroup; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (cgc) { -- /* -- * We want to update the vtime delta and then look for the next -- * cgroup to execute but the latter needs to be done in a loop -- * and we can't keep the lock held. Oh well... -- */ -- bpf_spin_lock(&cgv_tree_lock); -- __sync_fetch_and_add(&cgc->cvtime_delta, -- (cpuc->cur_at + cgrp_slice_ns - now) * -- FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- stat_inc(FCG_STAT_CNS_GONE); -- } -- -- bpf_cgroup_release(cgrp); -- --pick_next_cgroup: -- cpuc->cur_at = now; -- -- if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { -- cpuc->cur_cgid = 0; -- return; -- } -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_pick_next_cgroup(&cpuc->cur_cgid)) -- break; -- } --} -- --s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct fcg_task_ctx *taskc; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- taskc = bpf_task_storage_get(&task_ctx, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!taskc) -- return -ENOMEM; -- -- taskc->bypassed_at = 0; -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp, -- struct scx_cgroup_init_args *args) --{ -- struct fcg_cgrp_ctx *cgc; -- struct cgv_node *cgv_node; -- struct cgv_node_stash empty_stash = {}, *stash; -- u64 cgid = cgrp->kn->id; -- int ret; -- -- /* -- * Technically incorrect as cgroup ID is full 64bit while dq ID is -- * 63bit. Should not be a problem in practice and easy to spot in the -- * unlikely case that it breaks. -- */ -- ret = scx_bpf_create_dsq(cgid, -1); -- if (ret) -- return ret; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!cgc) { -- ret = -ENOMEM; -- goto err_destroy_dsq; -- } -- -- cgc->weight = args->weight; -- cgc->hweight = FCG_HWEIGHT_ONE; -- -- ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash, -- BPF_NOEXIST); -- if (ret) { -- if (ret != -ENOMEM) -- scx_bpf_error("unexpected stash creation error (%d)", -- ret); -- goto err_destroy_dsq; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("unexpected cgv_node stash lookup failure"); -- ret = -ENOENT; -- goto err_destroy_dsq; -- } -- -- cgv_node = bpf_obj_new(struct cgv_node); -- if (!cgv_node) { -- ret = -ENOMEM; -- goto err_del_cgv_node; -- } -- -- cgv_node->cgid = cgid; -- cgv_node->cvtime = cvtime_now; -- -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- ret = -EBUSY; -- goto err_drop; -- } -- -- return 0; -- --err_drop: -- bpf_obj_drop(cgv_node); --err_del_cgv_node: -- bpf_map_delete_elem(&cgv_node_stash, &cgid); --err_destroy_dsq: -- scx_bpf_destroy_dsq(cgid); -- return ret; --} -- --void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- -- /* -- * For now, there's no way find and remove the cgv_node if it's on the -- * cgv_tree. Let's drain them in the dispatch path as they get popped -- * off the front of the tree. -- */ -- bpf_map_delete_elem(&cgv_node_stash, &cgid); -- scx_bpf_destroy_dsq(cgid); --} -- --s32 BPF_STRUCT_OPS(fcg_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops flatcg_ops = { -- .enqueue = (void *)fcg_enqueue, -- .dispatch = (void *)fcg_dispatch, -- .runnable = (void *)fcg_runnable, -- .running = (void *)fcg_running, -- .stopping = (void *)fcg_stopping, -- .quiescent = (void *)fcg_quiescent, -- .prep_enable = (void *)fcg_prep_enable, -- .cgroup_set_weight = (void *)fcg_cgroup_set_weight, -- .cgroup_init = (void *)fcg_cgroup_init, -- .cgroup_exit = (void *)fcg_cgroup_exit, -- .init = (void *)fcg_init, -- .exit = (void *)fcg_exit, -- .flags = SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING, -- .name = "flatcg", --}; -diff --git a/tools/sched_ext/scx_example_flatcg.c b/tools/sched_ext/scx_example_flatcg.c -deleted file mode 100644 -index f9c8a5b84a70..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.c -+++ /dev/null -@@ -1,232 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2023 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2023 Tejun Heo -- * Copyright (c) 2023 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_flatcg.h" --#include "scx_example_flatcg.skel.h" -- --#ifndef FILEID_KERNFS --#define FILEID_KERNFS 0xfe --#endif -- --const char help_fmt[] = --"A flattened cgroup hierarchy sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -i INTERVAL Report interval\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --static float read_cpu_util(__u64 *last_sum, __u64 *last_idle) --{ -- FILE *fp; -- char buf[4096]; -- char *line, *cur = NULL, *tok; -- __u64 sum = 0, idle = 0; -- __u64 delta_sum, delta_idle; -- int idx; -- -- fp = fopen("/proc/stat", "r"); -- if (!fp) { -- perror("fopen(\"/proc/stat\")"); -- return 0.0; -- } -- -- if (!fgets(buf, sizeof(buf), fp)) { -- perror("fgets(\"/proc/stat\")"); -- fclose(fp); -- return 0.0; -- } -- fclose(fp); -- -- line = buf; -- for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) { -- char *endp = NULL; -- __u64 v; -- -- if (idx == 0) { -- line = NULL; -- continue; -- } -- v = strtoull(tok, &endp, 0); -- if (!endp || *endp != '\0') { -- fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n", -- idx, tok); -- continue; -- } -- sum += v; -- if (idx == 4) -- idle = v; -- } -- -- delta_sum = sum - *last_sum; -- delta_idle = idle - *last_idle; -- *last_sum = sum; -- *last_idle = idle; -- -- return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0; --} -- --static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats) --{ -- __u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus]; -- __u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); -- -- for (idx = 0; idx < FCG_NR_STATS; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_flatcg *skel; -- struct bpf_link *link; -- struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 }; -- bool dump_cgrps = false; -- __u64 last_cpu_sum = 0, last_cpu_idle = 0; -- __u64 last_stats[FCG_NR_STATS] = {}; -- unsigned long seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_flatcg__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open: %s\n", strerror(errno)); -- return 1; -- } -- -- skel->rodata->nr_cpus = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) { -- double v; -- -- switch (opt) { -- case 's': -- v = strtod(optarg, NULL); -- skel->rodata->cgrp_slice_ns = v * 1000; -- break; -- case 'i': -- v = strtod(optarg, NULL); -- intv_ts.tv_sec = v; -- intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000; -- break; -- case 'd': -- dump_cgrps = true; -- break; -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- case 'h': -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d", -- (double)skel->rodata->cgrp_slice_ns / 1000000.0, -- (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0, -- dump_cgrps); -- -- if (scx_example_flatcg__load(skel)) { -- fprintf(stderr, "Failed to load: %s\n", strerror(errno)); -- return 1; -- } -- -- link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops); -- if (!link) { -- fprintf(stderr, "Failed to attach_struct_ops: %s\n", -- strerror(errno)); -- return 1; -- } -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- __u64 acc_stats[FCG_NR_STATS]; -- __u64 stats[FCG_NR_STATS]; -- float cpu_util; -- int i; -- -- cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle); -- -- fcg_read_stats(skel, acc_stats); -- for (i = 0; i < FCG_NR_STATS; i++) -- stats[i] = acc_stats[i] - last_stats[i]; -- -- memcpy(last_stats, acc_stats, sizeof(acc_stats)); -- -- printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n", -- seq++, cpu_util * 100.0, skel->data->hweight_gen); -- printf(" act:%6llu deact:%6llu local:%6llu global:%6llu\n", -- stats[FCG_STAT_ACT], -- stats[FCG_STAT_DEACT], -- stats[FCG_STAT_LOCAL], -- stats[FCG_STAT_GLOBAL]); -- printf("HWT skip:%6llu race:%6llu cache:%6llu update:%6llu\n", -- stats[FCG_STAT_HWT_SKIP], -- stats[FCG_STAT_HWT_RACE], -- stats[FCG_STAT_HWT_CACHE], -- stats[FCG_STAT_HWT_UPDATES]); -- printf("ENQ skip:%6llu race:%6llu\n", -- stats[FCG_STAT_ENQ_SKIP], -- stats[FCG_STAT_ENQ_RACE]); -- printf("CNS keep:%6llu expire:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_CNS_KEEP], -- stats[FCG_STAT_CNS_EXPIRE], -- stats[FCG_STAT_CNS_EMPTY], -- stats[FCG_STAT_CNS_GONE]); -- printf("PNC nocgrp:%6llu next:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_PNC_NO_CGRP], -- stats[FCG_STAT_PNC_NEXT], -- stats[FCG_STAT_PNC_EMPTY], -- stats[FCG_STAT_PNC_GONE]); -- printf("BAD remove:%6llu\n", -- acc_stats[FCG_STAT_BAD_REMOVAL]); -- -- nanosleep(&intv_ts, NULL); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_flatcg__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_flatcg.h b/tools/sched_ext/scx_example_flatcg.h -deleted file mode 100644 -index 490758ed41f0..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.h -+++ /dev/null -@@ -1,49 +0,0 @@ --#ifndef __SCX_EXAMPLE_FLATCG_H --#define __SCX_EXAMPLE_FLATCG_H -- --enum { -- FCG_HWEIGHT_ONE = 1LLU << 16, --}; -- --enum fcg_stat_idx { -- FCG_STAT_ACT, -- FCG_STAT_DEACT, -- FCG_STAT_LOCAL, -- FCG_STAT_GLOBAL, -- -- FCG_STAT_HWT_UPDATES, -- FCG_STAT_HWT_CACHE, -- FCG_STAT_HWT_SKIP, -- FCG_STAT_HWT_RACE, -- -- FCG_STAT_ENQ_SKIP, -- FCG_STAT_ENQ_RACE, -- -- FCG_STAT_CNS_KEEP, -- FCG_STAT_CNS_EXPIRE, -- FCG_STAT_CNS_EMPTY, -- FCG_STAT_CNS_GONE, -- -- FCG_STAT_PNC_NO_CGRP, -- FCG_STAT_PNC_NEXT, -- FCG_STAT_PNC_EMPTY, -- FCG_STAT_PNC_GONE, -- -- FCG_STAT_BAD_REMOVAL, -- -- FCG_NR_STATS, --}; -- --struct fcg_cgrp_ctx { -- u32 nr_active; -- u32 nr_runnable; -- u32 queued; -- u32 weight; -- u32 hweight; -- u64 child_weight_sum; -- u64 hweight_gen; -- s64 cvtime_delta; -- u64 tvtime_now; --}; -- --#endif /* __SCX_EXAMPLE_FLATCG_H */ -diff --git a/tools/sched_ext/scx_example_pair.bpf.c b/tools/sched_ext/scx_example_pair.bpf.c -deleted file mode 100644 -index 279efe58b777..000000000000 ---- a/tools/sched_ext/scx_example_pair.bpf.c -+++ /dev/null -@@ -1,627 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext core-scheduler which always makes every sibling CPU pair -- * execute from the same CPU cgroup. -- * -- * This scheduler is a minimal implementation and would need some form of -- * priority handling both inside each cgroup and across the cgroups to be -- * practically useful. -- * -- * Each CPU in the system is paired with exactly one other CPU, according to a -- * "stride" value that can be specified when the BPF scheduler program is first -- * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee -- * that they will only ever schedule tasks that belong to the same CPU cgroup. -- * -- * Scheduler Initialization -- * ------------------------ -- * -- * The scheduler BPF program is first initialized from user space, before it is -- * enabled. During this initialization process, each CPU on the system is -- * assigned several values that are constant throughout its runtime: -- * -- * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling -- * decisions. Paired CPUs always schedule tasks from the same -- * CPU cgroup, and synchronize with each other to guarantee -- * that this constraint is not violated. -- * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access -- * a struct pair_ctx object that is shared between the pair. -- * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the -- * pair. Each struct pair_ctx has an active_mask field, -- * which is a bitmap used to indicate whether each core -- * in the pair currently has an actively running task. -- * This index specifies which entry in the bitmap corresponds -- * to each CPU in the pair. -- * -- * During this initialization, the CPUs are paired according to a "stride" that -- * may be specified when invoking the user space program that initializes and -- * loads the scheduler. By default, the stride is 1/2 the total number of CPUs. -- * -- * Tasks and cgroups -- * ----------------- -- * -- * Every cgroup in the system is registered with the scheduler using the -- * pair_cgroup_init() callback, and every task in the system is associated with -- * exactly one cgroup. At a high level, the idea with the pair scheduler is to -- * always schedule tasks from the same cgroup within a given CPU pair. When a -- * task is enqueued (i.e. passed to the pair_enqueue() callback function), its -- * cgroup ID is read from its task struct, and then a corresponding queue map -- * is used to FIFO-enqueue the task for that cgroup. -- * -- * If you look through the implementation of the scheduler, you'll notice that -- * there is quite a bit of complexity involved with looking up the per-cgroup -- * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash -- * BPF hash map that is used to map a cgroup ID to a globally unique ID that's -- * allocated in the BPF program. This is done because we use separate maps to -- * store the FIFO queue of tasks, and the length of that map, per cgroup. This -- * complexity is only present because of current deficiencies in BPF that will -- * soon be addressed. The main point to keep in mind is that newly enqueued -- * tasks are added to their cgroup's FIFO queue. -- * -- * Dispatching tasks -- * ----------------- -- * -- * This section will describe how enqueued tasks are dispatched and scheduled. -- * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is -- * as follows: -- * -- * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is -- * the structure that's used to synchronize amongst the two pair CPUs in their -- * scheduling decisions. After any of the following events have occurred: -- * -- * - The cgroup's slice run has expired, or -- * - The cgroup becomes empty, or -- * - Either CPU in the pair is preempted by a higher priority scheduling class -- * -- * The cgroup transitions to the draining state and stops executing new tasks -- * from the cgroup. -- * -- * 2. If the pair is still executing a task, mark the pair_ctx as draining, and -- * wait for the pair CPU to be preempted. -- * -- * 3. Otherwise, if the pair CPU is not running a task, we can move onto -- * scheduling new tasks. Pop the next cgroup id from the top_q queue. -- * -- * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it. -- * -- * Note again that this scheduling behavior is simple, but the implementation -- * is complex mostly because this it hits several BPF shortcomings and has to -- * work around in often awkward ways. Most of the shortcomings are expected to -- * be resolved in the near future which should allow greatly simplifying this -- * scheduler. -- * -- * Dealing with preemption -- * ----------------------- -- * -- * SCX is the lowest priority sched_class, and could be preempted by them at -- * any time. To address this, the scheduler implements pair_cpu_release() and -- * pair_cpu_acquire() callbacks which are invoked by the core scheduler when -- * the scheduler loses and gains control of the CPU respectively. -- * -- * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and -- * then invoke: -- * -- * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- * -- * This preempts the pair CPU, and waits until it has re-entered the scheduler -- * before returning. This is necessary to ensure that the higher priority -- * sched_class that preempted our scheduler does not schedule a task -- * concurrently with our pair CPU. -- * -- * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption -- * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable -- * pair scheduling. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include "scx_example_pair.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; -- --/* !0 for veristat, set during init */ --const volatile u32 nr_cpu_ids = 64; -- --/* a pair of CPUs stay on a cgroup for this duration */ --const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL; -- --/* cpu ID -> pair cpu ID */ --const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 }; -- --/* cpu ID -> pair_id */ --const volatile u32 pair_id[MAX_CPUS]; -- --/* CPU ID -> CPU # in the pair (0 or 1) */ --const volatile u32 in_pair_idx[MAX_CPUS]; -- --struct pair_ctx { -- struct bpf_spin_lock lock; -- -- /* the cgroup the pair is currently executing */ -- u64 cgid; -- -- /* the pair started executing the current cgroup at */ -- u64 started_at; -- -- /* whether the current cgroup is draining */ -- bool draining; -- -- /* the CPUs that are currently active on the cgroup */ -- u32 active_mask; -- -- /* -- * the CPUs that are currently preempted and running tasks in a -- * different scheduler. -- */ -- u32 preempted_mask; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, MAX_CPUS / 2); -- __type(key, u32); -- __type(value, struct pair_ctx); --} pair_ctx SEC(".maps"); -- --/* queue of cgrp_q's possibly with tasks on them */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- /* -- * Because it's difficult to build strong synchronization encompassing -- * multiple non-trivial operations in BPF, this queue is managed in an -- * opportunistic way so that we guarantee that a cgroup w/ active tasks -- * is always on it but possibly multiple times. Once we have more robust -- * synchronization constructs and e.g. linked list, we should be able to -- * do this in a prettier way but for now just size it big enough. -- */ -- __uint(max_entries, 4 * MAX_CGRPS); -- __type(value, u64); --} top_q SEC(".maps"); -- --/* per-cgroup q which FIFOs the tasks from the cgroup */ --struct cgrp_q { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, MAX_QUEUED); -- __type(value, u32); --}; -- --/* -- * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local -- * storage; however, a cgroup local storage can only be accessed from the BPF -- * progs attached to the cgroup. For now, work around by allocating array of -- * cgrp_q's and then allocating per-cgroup indices. -- * -- * Another caveat: It's difficult to populate a large array of maps statically -- * or from BPF. Initialize it from userland. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, MAX_CGRPS); -- __type(key, s32); -- __array(values, struct cgrp_q); --} cgrp_q_arr SEC(".maps"); -- --static u64 cgrp_q_len[MAX_CGRPS]; -- --/* -- * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be -- * useful to have as a map type. -- */ --static u32 cgrp_q_idx_cursor; --static u64 cgrp_q_idx_busy[MAX_CGRPS]; -- --/* -- * All added up, the following is what we do: -- * -- * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking -- * for a free ID. If not found, fail cgroup creation with -EBUSY. -- * -- * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following -- * cgrp_q_idx_hash. -- * -- * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from -- * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr. -- * -- * This is sadly complicated for something pretty simple. Hopefully, we should -- * be able to simplify in the future. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, MAX_CGRPS); -- __uint(key_size, sizeof(u64)); /* cgrp ID */ -- __uint(value_size, sizeof(s32)); /* cgrp_q idx */ --} cgrp_q_idx_hash SEC(".maps"); -- --/* statistics */ --u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions; --u64 nr_exps, nr_exp_waits, nr_exp_empty; --u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty; -- --struct user_exit_info uei; -- --static bool time_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- struct cgrp_q *cgq; -- s32 pid = p->pid; -- u64 cgid; -- u32 *q_idx; -- u64 *cgq_len; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- cgrp = scx_bpf_task_cgroup(p); -- cgid = cgrp->kn->id; -- bpf_cgroup_release(cgrp); -- -- /* find the cgroup's q and push @p into it */ -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!q_idx) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return; -- } -- -- cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx); -- if (!cgq) { -- scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]", -- cgid, *q_idx); -- return; -- } -- -- if (bpf_map_push_elem(cgq, &pid, 0)) { -- scx_bpf_error("cgroup[%llu] queue overflow", cgid); -- return; -- } -- -- /* bump q len, if going 0 -> 1, queue cgroup into the top_q */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len) { -- scx_bpf_error("MEMBER_VTPR malfunction"); -- return; -- } -- -- if (!__sync_fetch_and_add(cgq_len, 1) && -- bpf_map_push_elem(&top_q, &cgid, 0)) { -- scx_bpf_error("top_q overflow"); -- return; -- } --} -- --static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) --{ -- u32 *vptr; -- -- vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *pairc = bpf_map_lookup_elem(&pair_ctx, vptr); -- if (!(*pairc)) -- return -EINVAL; -- -- vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *mask = 1U << *vptr; -- -- return 0; --} -- --static int try_dispatch(s32 cpu) --{ -- struct pair_ctx *pairc; -- struct bpf_map *cgq_map; -- struct task_struct *p; -- u64 now = bpf_ktime_get_ns(); -- bool kick_pair = false; -- bool expired, pair_preempted; -- u32 *vptr, in_pair_mask; -- s32 pid, q_idx; -- u64 cgid; -- int ret; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) { -- scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]", -- cpu); -- return -ENOENT; -- } -- -- bpf_spin_lock(&pairc->lock); -- pairc->active_mask &= ~in_pair_mask; -- -- expired = time_before(pairc->started_at + pair_batch_dur_ns, now); -- if (expired || pairc->draining) { -- u64 new_cgid = 0; -- -- __sync_fetch_and_add(&nr_exps, 1); -- -- /* -- * We're done with the current cgid. An obvious optimization -- * would be not draining if the next cgroup is the current one. -- * For now, be dumb and always expire. -- */ -- pairc->draining = true; -- -- pair_preempted = pairc->preempted_mask; -- if (pairc->active_mask || pair_preempted) { -- /* -- * The other CPU is still active, or is no longer under -- * our control due to e.g. being preempted by a higher -- * priority sched_class. We want to wait until this -- * cgroup expires, or until control of our pair CPU has -- * been returned to us. -- * -- * If the pair controls its CPU, and the time already -- * expired, kick. When the other CPU arrives at -- * dispatch and clears its active mask, it'll push the -- * pair to the next cgroup and kick this CPU. -- */ -- __sync_fetch_and_add(&nr_exp_waits, 1); -- bpf_spin_unlock(&pairc->lock); -- if (expired && !pair_preempted) -- kick_pair = true; -- goto out_maybe_kick; -- } -- -- bpf_spin_unlock(&pairc->lock); -- -- /* -- * Pick the next cgroup. It'd be easier / cleaner to not drop -- * pairc->lock and use stronger synchronization here especially -- * given that we'll be switching cgroups significantly less -- * frequently than tasks. Unfortunately, bpf_spin_lock can't -- * really protect anything non-trivial. Let's do opportunistic -- * operations instead. -- */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u32 *q_idx; -- u64 *cgq_len; -- -- if (bpf_map_pop_elem(&top_q, &new_cgid)) { -- /* no active cgroup, go idle */ -- __sync_fetch_and_add(&nr_exp_empty, 1); -- return 0; -- } -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid); -- if (!q_idx) -- continue; -- -- /* -- * This is the only place where empty cgroups are taken -- * off the top_q. -- */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len || !*cgq_len) -- continue; -- -- /* -- * If it has any tasks, requeue as we may race and not -- * execute it. -- */ -- bpf_map_push_elem(&top_q, &new_cgid, 0); -- break; -- } -- -- bpf_spin_lock(&pairc->lock); -- -- /* -- * The other CPU may already have started on a new cgroup while -- * we dropped the lock. Make sure that we're still draining and -- * start on the new cgroup. -- */ -- if (pairc->draining && !pairc->active_mask) { -- __sync_fetch_and_add(&nr_cgrp_next, 1); -- pairc->cgid = new_cgid; -- pairc->started_at = now; -- pairc->draining = false; -- kick_pair = true; -- } else { -- __sync_fetch_and_add(&nr_cgrp_coll, 1); -- } -- } -- -- cgid = pairc->cgid; -- pairc->active_mask |= in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- -- /* again, it'd be better to do all these with the lock held, oh well */ -- vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!vptr) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return -ENOENT; -- } -- q_idx = *vptr; -- -- /* claim one task from cgrp_q w/ q_idx */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u64 *cgq_len, len; -- -- cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]); -- if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) { -- /* the cgroup must be empty, expire and repeat */ -- __sync_fetch_and_add(&nr_cgrp_empty, 1); -- bpf_spin_lock(&pairc->lock); -- pairc->draining = true; -- pairc->active_mask &= ~in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- return -EAGAIN; -- } -- -- if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len) -- continue; -- -- break; -- } -- -- cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx); -- if (!cgq_map) { -- scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- if (bpf_map_pop_elem(cgq_map, &pid)) { -- scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- p = bpf_task_from_pid(pid); -- if (p) { -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } else { -- /* we don't handle dequeues, retry on lost tasks */ -- __sync_fetch_and_add(&nr_missing, 1); -- return -EAGAIN; -- } -- --out_maybe_kick: -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } -- return 0; --} -- --void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) --{ -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_dispatch(cpu) != -EAGAIN) -- break; -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask &= ~in_pair_mask; -- /* Kick the pair CPU, unless it was also preempted. */ -- kick_pair = !pairc->preempted_mask; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask |= in_pair_mask; -- pairc->active_mask &= ~in_pair_mask; -- /* Kick the pair CPU if it's still running. */ -- kick_pair = pairc->active_mask; -- pairc->draining = true; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- } -- } -- __sync_fetch_and_add(&nr_preemptions, 1); --} -- --s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 i, q_idx; -- -- bpf_for(i, 0, MAX_CGRPS) { -- q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS; -- if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1)) -- break; -- } -- if (i == MAX_CGRPS) -- return -EBUSY; -- -- if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]); -- if (busy) -- *busy = 0; -- return -EBUSY; -- } -- -- return 0; --} -- --void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 *q_idx; -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (q_idx) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]); -- if (busy) -- *busy = 0; -- bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid); -- } --} -- --s32 BPF_STRUCT_OPS(pair_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops pair_ops = { -- .enqueue = (void *)pair_enqueue, -- .dispatch = (void *)pair_dispatch, -- .cpu_acquire = (void *)pair_cpu_acquire, -- .cpu_release = (void *)pair_cpu_release, -- .cgroup_init = (void *)pair_cgroup_init, -- .cgroup_exit = (void *)pair_cgroup_exit, -- .init = (void *)pair_init, -- .exit = (void *)pair_exit, -- .name = "pair", --}; -diff --git a/tools/sched_ext/scx_example_pair.c b/tools/sched_ext/scx_example_pair.c -deleted file mode 100644 -index 18e032bbc173..000000000000 ---- a/tools/sched_ext/scx_example_pair.c -+++ /dev/null -@@ -1,143 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_pair.h" --#include "scx_example_pair.skel.h" -- --const char help_fmt[] = --"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n" --"execute from the same CPU cgroup.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-S STRIDE] [-p]\n" --"\n" --" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_pair *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 stride, i, opt, outer_fd; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_pair__open(); -- assert(skel); -- -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- /* pair up the earlier half to the latter by default, override with -s */ -- stride = skel->rodata->nr_cpu_ids / 2; -- -- while ((opt = getopt(argc, argv, "S:ph")) != -1) { -- switch (opt) { -- case 'S': -- stride = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- for (i = 0; i < skel->rodata->nr_cpu_ids; i++) { -- if (skel->rodata->pair_cpu[i] < 0) { -- skel->rodata->pair_cpu[i] = i + stride; -- skel->rodata->pair_cpu[i + stride] = i; -- skel->rodata->pair_id[i] = i; -- skel->rodata->pair_id[i + stride] = i; -- skel->rodata->in_pair_idx[i] = 0; -- skel->rodata->in_pair_idx[i + stride] = 1; -- } -- } -- -- assert(!scx_example_pair__load(skel)); -- -- /* -- * Populate the cgrp_q_arr map which is an array containing per-cgroup -- * queues. It'd probably be better to do this from BPF but there are too -- * many to initialize statically and there's no way to dynamically -- * populate from BPF. -- */ -- outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr); -- assert(outer_fd >= 0); -- -- printf("Initializing"); -- for (i = 0; i < MAX_CGRPS; i++) { -- s32 inner_fd; -- -- if (exit_req) -- break; -- -- inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, -- sizeof(u32), MAX_QUEUED, NULL); -- assert(inner_fd >= 0); -- assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY)); -- close(inner_fd); -- -- if (!(i % 10)) -- printf("."); -- fflush(stdout); -- } -- printf("\n"); -- -- /* -- * Fully initialized, attach and run. -- */ -- link = bpf_map__attach_struct_ops(skel->maps.pair_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf(" total:%10lu dispatch:%10lu missing:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_dispatched, -- skel->bss->nr_missing); -- printf(" kicks:%10lu preemptions:%7lu\n", -- skel->bss->nr_kicks, -- skel->bss->nr_preemptions); -- printf(" exp:%10lu exp_wait:%10lu exp_empty:%10lu\n", -- skel->bss->nr_exps, -- skel->bss->nr_exp_waits, -- skel->bss->nr_exp_empty); -- printf("cgnext:%10lu cgcoll:%10lu cgempty:%10lu\n", -- skel->bss->nr_cgrp_next, -- skel->bss->nr_cgrp_coll, -- skel->bss->nr_cgrp_empty); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_pair__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_pair.h b/tools/sched_ext/scx_example_pair.h -deleted file mode 100644 -index f60b824272f7..000000000000 ---- a/tools/sched_ext/scx_example_pair.h -+++ /dev/null -@@ -1,10 +0,0 @@ --#ifndef __SCX_EXAMPLE_PAIR_H --#define __SCX_EXAMPLE_PAIR_H -- --enum { -- MAX_CPUS = 4096, -- MAX_QUEUED = 4096, -- MAX_CGRPS = 4096, --}; -- --#endif /* __SCX_EXAMPLE_PAIR_H */ -diff --git a/tools/sched_ext/scx_example_qmap.bpf.c b/tools/sched_ext/scx_example_qmap.bpf.c -deleted file mode 100644 -index 579ab21ae403..000000000000 ---- a/tools/sched_ext/scx_example_qmap.bpf.c -+++ /dev/null -@@ -1,401 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple five-level FIFO queue scheduler. -- * -- * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets -- * assigned to one depending on its compound weight. Each CPU round robins -- * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from -- * queue0, 2 from queue1, 4 from queue2 and so on. -- * -- * This scheduler demonstrates: -- * -- * - BPF-side queueing using PIDs. -- * - Sleepable per-task storage allocation using ops.prep_enable(). -- * - Using ops.cpu_release() to handle a higher priority scheduling class taking -- * the CPU away. -- * - Core-sched support. -- * -- * This scheduler is primarily for demonstration and testing of sched_ext -- * features and unlikely to be useful for actual workloads. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include -- --char _license[] SEC("license") = "GPL"; -- --const volatile u64 slice_ns = SCX_SLICE_DFL; --const volatile bool switch_partial; --const volatile u32 stall_user_nth; --const volatile u32 stall_kernel_nth; --const volatile u32 dsp_inf_loop_after; --const volatile s32 disallow_tgid; -- --u32 test_error_cnt; -- --struct user_exit_info uei; -- --struct qmap { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, u32); --} queue0 SEC(".maps"), -- queue1 SEC(".maps"), -- queue2 SEC(".maps"), -- queue3 SEC(".maps"), -- queue4 SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, 5); -- __type(key, int); -- __array(values, struct qmap); --} queue_arr SEC(".maps") = { -- .values = { -- [0] = &queue0, -- [1] = &queue1, -- [2] = &queue2, -- [3] = &queue3, -- [4] = &queue4, -- }, --}; -- --/* -- * Per-queue sequence numbers to implement core-sched ordering. -- * -- * Tail seq is assigned to each queued task and incremented. Head seq tracks the -- * sequence number of the latest dispatched task. The distance between the a -- * task's seq and the associated queue's head seq is called the queue distance -- * and used when comparing two tasks for ordering. See qmap_core_sched_before(). -- */ --static u64 core_sched_head_seqs[5]; --static u64 core_sched_tail_seqs[5]; -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local_dsq */ -- u64 core_sched_seq; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --/* Per-cpu dispatch index and remaining count */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(max_entries, 2); -- __type(key, u32); -- __type(value, u64); --} dispatch_idx_cnt SEC(".maps"); -- --/* Statistics */ --unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued; --unsigned long nr_core_sched_execed; -- --s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- struct task_ctx *tctx; -- s32 cpu; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- return cpu; -- -- return prev_cpu; --} -- --static int weight_to_idx(u32 weight) --{ -- /* Coarsely map the compound weight to a FIFO. */ -- if (weight <= 25) -- return 0; -- else if (weight <= 50) -- return 1; -- else if (weight < 200) -- return 2; -- else if (weight < 400) -- return 3; -- else -- return 4; --} -- --void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) --{ -- static u32 user_cnt, kernel_cnt; -- struct task_ctx *tctx; -- u32 pid = p->pid; -- int idx = weight_to_idx(p->scx.weight); -- void *ring; -- -- if (p->flags & PF_KTHREAD) { -- if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth)) -- return; -- } else { -- if (stall_user_nth && !(++user_cnt % stall_user_nth)) -- return; -- } -- -- if (test_error_cnt && !--test_error_cnt) -- scx_bpf_error("test triggering error"); -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * All enqueued tasks must have their core_sched_seq updated for correct -- * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in -- * qmap_ops.flags. -- */ -- tctx->core_sched_seq = core_sched_tail_seqs[idx]++; -- -- /* -- * If qmap_select_cpu() is telling us to or this is the last runnable -- * task on the CPU, enqueue locally. -- */ -- if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) { -- tctx->force_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); -- return; -- } -- -- /* -- * If the task was re-enqueued due to the CPU being preempted by a -- * higher priority scheduling class, just re-enqueue the task directly -- * on the global DSQ. As we want another CPU to pick it up, find and -- * kick an idle CPU. -- */ -- if (enq_flags & SCX_ENQ_REENQ) { -- s32 cpu; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags); -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- return; -- } -- -- ring = bpf_map_lookup_elem(&queue_arr, &idx); -- if (!ring) { -- scx_bpf_error("failed to find ring %d", idx); -- return; -- } -- -- /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */ -- if (bpf_map_push_elem(ring, &pid, 0)) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_enqueued, 1); --} -- --/* -- * The BPF queue map doesn't support removal and sched_ext can handle spurious -- * dispatches. qmap_dequeue() is only used to collect statistics. -- */ --void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags) --{ -- __sync_fetch_and_add(&nr_dequeued, 1); -- if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) -- __sync_fetch_and_add(&nr_core_sched_execed, 1); --} -- --static void update_core_sched_head_seq(struct task_struct *p) --{ -- struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- int idx = weight_to_idx(p->scx.weight); -- -- if (tctx) -- core_sched_head_seqs[idx] = tctx->core_sched_seq; -- else -- scx_bpf_error("task_ctx lookup failed"); --} -- --void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 zero = 0, one = 1; -- u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero); -- u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one); -- void *fifo; -- s32 pid; -- int i; -- -- if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) { -- struct task_struct *p; -- -- /* -- * PID 2 should be kthreadd which should mostly be idle and off -- * the scheduler. Let's keep dispatching it to force the kernel -- * to call this function over and over again. -- */ -- p = bpf_task_from_pid(2); -- if (p) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- if (!idx || !cnt) { -- scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt); -- return; -- } -- -- for (i = 0; i < 5; i++) { -- /* Advance the dispatch cursor and pick the fifo. */ -- if (!*cnt) { -- *idx = (*idx + 1) % 5; -- *cnt = 1 << *idx; -- } -- (*cnt)--; -- -- fifo = bpf_map_lookup_elem(&queue_arr, idx); -- if (!fifo) { -- scx_bpf_error("failed to find ring %llu", *idx); -- return; -- } -- -- /* Dispatch or advance. */ -- if (!bpf_map_pop_elem(fifo, &pid)) { -- struct task_struct *p; -- -- p = bpf_task_from_pid(pid); -- if (p) { -- update_core_sched_head_seq(p); -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- *cnt = 0; -- } --} -- --/* -- * The distance from the head of the queue scaled by the weight of the queue. -- * The lower the number, the older the task and the higher the priority. -- */ --static s64 task_qdist(struct task_struct *p) --{ -- int idx = weight_to_idx(p->scx.weight); -- struct task_ctx *tctx; -- s64 qdist; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return 0; -- } -- -- qdist = tctx->core_sched_seq - core_sched_head_seqs[idx]; -- -- /* -- * As queue index increments, the priority doubles. The queue w/ index 3 -- * is dispatched twice more frequently than 2. Reflect the difference by -- * scaling qdists accordingly. Note that the shift amount needs to be -- * flipped depending on the sign to avoid flipping priority direction. -- */ -- if (qdist >= 0) -- return qdist << (4 - idx); -- else -- return qdist << idx; --} -- --/* -- * This is called to determine the task ordering when core-sched is picking -- * tasks to execute on SMT siblings and should encode about the same ordering as -- * the regular scheduling path. Use the priority-scaled distances from the head -- * of the queues to compare the two tasks which should be consistent with the -- * dispatch path behavior. -- */ --bool BPF_STRUCT_OPS(qmap_core_sched_before, -- struct task_struct *a, struct task_struct *b) --{ -- return task_qdist(a) > task_qdist(b); --} -- --void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- u32 cnt; -- -- /* -- * Called when @cpu is taken by a higher priority scheduling class. This -- * makes @cpu no longer available for executing sched_ext tasks. As we -- * don't want the tasks in @cpu's local dsq to sit there until @cpu -- * becomes available again, re-enqueue them into the global dsq. See -- * %SCX_ENQ_REENQ handling in qmap_enqueue(). -- */ -- cnt = scx_bpf_reenqueue_local(); -- if (cnt) -- __sync_fetch_and_add(&nr_reenqueued, cnt); --} -- --s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (p->tgid == disallow_tgid) -- p->scx.disallow = true; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(qmap_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops qmap_ops = { -- .select_cpu = (void *)qmap_select_cpu, -- .enqueue = (void *)qmap_enqueue, -- .dequeue = (void *)qmap_dequeue, -- .dispatch = (void *)qmap_dispatch, -- .core_sched_before = (void *)qmap_core_sched_before, -- .cpu_release = (void *)qmap_cpu_release, -- .prep_enable = (void *)qmap_prep_enable, -- .init = (void *)qmap_init, -- .exit = (void *)qmap_exit, -- .flags = SCX_OPS_ENQ_LAST, -- .timeout_ms = 5000U, -- .name = "qmap", --}; -diff --git a/tools/sched_ext/scx_example_qmap.c b/tools/sched_ext/scx_example_qmap.c -deleted file mode 100644 -index ccb4814ee61b..000000000000 ---- a/tools/sched_ext/scx_example_qmap.c -+++ /dev/null -@@ -1,107 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_qmap.skel.h" -- --const char help_fmt[] = --"A simple five-level FIFO queue sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" --" -t COUNT Stall every COUNT'th user thread\n" --" -T COUNT Stall every COUNT'th kernel thread\n" --" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n" --" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_qmap *skel; -- struct bpf_link *link; -- int opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_qmap__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) { -- switch (opt) { -- case 's': -- skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; -- break; -- case 'e': -- skel->bss->test_error_cnt = strtoul(optarg, NULL, 0); -- break; -- case 't': -- skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0); -- break; -- case 'T': -- skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0); -- break; -- case 'l': -- skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0); -- break; -- case 'd': -- skel->rodata->disallow_tgid = strtol(optarg, NULL, 0); -- if (skel->rodata->disallow_tgid < 0) -- skel->rodata->disallow_tgid = getpid(); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_qmap__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.qmap_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- long nr_enqueued = skel->bss->nr_enqueued; -- long nr_dispatched = skel->bss->nr_dispatched; -- -- printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n", -- nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, -- skel->bss->nr_reenqueued, skel->bss->nr_dequeued, -- skel->bss->nr_core_sched_execed); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_qmap__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_simple.bpf.c b/tools/sched_ext/scx_example_simple.bpf.c -deleted file mode 100644 -index 4bccca3e2047..000000000000 ---- a/tools/sched_ext/scx_example_simple.bpf.c -+++ /dev/null -@@ -1,128 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple scheduler. -- * -- * By default, it operates as a simple global weighted vtime scheduler and can -- * be switched to FIFO scheduling. It also demonstrates the following niceties. -- * -- * - Statistics tracking how many tasks are queued to local and global dsq's. -- * - Termination notification for userspace. -- * -- * While very simple, this scheduler should work reasonably well on CPUs with a -- * uniform L3 cache topology. While preemption is not implemented, the fact that -- * the scheduling queue is shared across all CPUs means that whatever is at the -- * front of the queue is likely to be executed fairly quickly given enough -- * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads -- * but comes with the usual problems with FIFO scheduling where saturating -- * threads can easily drown out interactive ones. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --static u64 vtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, 2); /* [local, global] */ --} stats SEC(".maps"); -- --static void stat_inc(u32 idx) --{ -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); -- if (cnt_p) -- (*cnt_p)++; --} -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) --{ -- /* -- * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that -- * running @p on its CPU directly shouldn't affect fairness. Just queue -- * it on the local FIFO. -- */ -- if (enq_flags & SCX_ENQ_LOCAL) { -- stat_inc(0); /* count local queueing */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- return; -- } -- -- stat_inc(1); /* count global queueing */ -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) -- vtime = vtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --void BPF_STRUCT_OPS(simple_running, struct task_struct *p) --{ -- if (fifo_sched) -- return; -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(vtime_now, p->scx.dsq_vtime)) -- vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --s32 BPF_STRUCT_OPS(simple_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .running = (void *)simple_running, -- .stopping = (void *)simple_stopping, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", --}; -diff --git a/tools/sched_ext/scx_example_simple.c b/tools/sched_ext/scx_example_simple.c -deleted file mode 100644 -index 486b401f7c95..000000000000 ---- a/tools/sched_ext/scx_example_simple.c -+++ /dev/null -@@ -1,101 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_simple.skel.h" -- --const char help_fmt[] = --"A simple sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-f] [-p]\n" --"\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int simple) --{ -- exit_req = 1; --} -- --static void read_stats(struct scx_example_simple *skel, u64 *stats) --{ -- int nr_cpus = libbpf_num_possible_cpus(); -- u64 cnts[2][nr_cpus]; -- u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * 2); -- -- for (idx = 0; idx < 2; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_simple *skel; -- struct bpf_link *link; -- u32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_simple__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "fph")) != -1) { -- switch (opt) { -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_simple__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.simple_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- u64 stats[2]; -- -- read_stats(skel, stats); -- printf("local=%lu global=%lu\n", stats[0], stats[1]); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_simple__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_userland.bpf.c b/tools/sched_ext/scx_example_userland.bpf.c -deleted file mode 100644 -index a089bc6bbe86..000000000000 ---- a/tools/sched_ext/scx_example_userland.bpf.c -+++ /dev/null -@@ -1,269 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A minimal userland scheduler. -- * -- * In terms of scheduling, this provides two different types of behaviors: -- * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity. -- * All such tasks are direct-dispatched from the kernel, and are never -- * enqueued in user space. -- * 2. A primitive vruntime scheduler that is implemented in user space, for all -- * other tasks. -- * -- * Some parts of this example user space scheduler could be implemented more -- * efficiently using more complex and sophisticated data structures. For -- * example, rather than using BPF_MAP_TYPE_QUEUE's, -- * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between -- * user space and kernel space. Similarly, we use a simple vruntime-sorted list -- * in user space, but an rbtree could be used instead. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include --#include "scx_common.bpf.h" --#include "scx_example_userland_common.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; --const volatile s32 usersched_pid; -- --/* !0 for veristat, set during init */ --const volatile u32 num_possible_cpus = 64; -- --/* Stats that are printed by user space. */ --u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues; -- --struct user_exit_info uei; -- --/* -- * Whether the user space scheduler needs to be scheduled due to a task being -- * enqueued in user space. -- */ --static bool usersched_needed; -- --/* -- * The map containing tasks that are enqueued in user space from the kernel. -- * -- * This map is drained by the user space scheduler. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, struct scx_userland_enqueued_task); --} enqueued SEC(".maps"); -- --/* -- * The map containing tasks that are dispatched to the kernel from user space. -- * -- * Drained by the kernel in userland_dispatch(). -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, s32); --} dispatched SEC(".maps"); -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local DSQ */ --}; -- --/* Map that contains task-local storage. */ --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --static bool is_usersched_task(const struct task_struct *p) --{ -- return p->pid == usersched_pid; --} -- --static bool keep_in_kernel(const struct task_struct *p) --{ -- return p->nr_cpus_allowed < num_possible_cpus; --} -- --static struct task_struct *usersched_task(void) --{ -- struct task_struct *p; -- -- p = bpf_task_from_pid(usersched_pid); -- /* -- * Should never happen -- the usersched task should always be managed -- * by sched_ext. -- */ -- if (!p) { -- scx_bpf_error("Failed to find usersched task %d", usersched_pid); -- /* -- * We should never hit this path, and we error out of the -- * scheduler above just in case, so the scheduler will soon be -- * be evicted regardless. So as to simplify the logic in the -- * caller to not have to check for NULL, return an acquired -- * reference to the current task here rather than NULL. -- */ -- return bpf_task_acquire(bpf_get_current_task_btf()); -- } -- -- return p; --} -- --s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- if (keep_in_kernel(p)) { -- s32 cpu; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to look up task-local storage for %s", p->comm); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- tctx->force_local = true; -- return cpu; -- } -- } -- -- return prev_cpu; --} -- --static void dispatch_user_scheduler(void) --{ -- struct task_struct *p; -- -- usersched_needed = false; -- p = usersched_task(); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); --} -- --static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags) --{ -- struct scx_userland_enqueued_task task; -- -- memset(&task, 0, sizeof(task)); -- task.pid = p->pid; -- task.sum_exec_runtime = p->se.sum_exec_runtime; -- task.weight = p->scx.weight; -- -- if (bpf_map_push_elem(&enqueued, &task, 0)) { -- /* -- * If we fail to enqueue the task in user space, put it -- * directly on the global DSQ. -- */ -- __sync_fetch_and_add(&nr_failed_enqueues, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- __sync_fetch_and_add(&nr_user_enqueues, 1); -- usersched_needed = true; -- } --} -- --void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags) --{ -- if (keep_in_kernel(p)) { -- u64 dsq_id = SCX_DSQ_GLOBAL; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to lookup task ctx for %s", p->comm); -- return; -- } -- -- if (tctx->force_local) -- dsq_id = SCX_DSQ_LOCAL; -- tctx->force_local = false; -- scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags); -- __sync_fetch_and_add(&nr_kernel_enqueues, 1); -- return; -- } else if (!is_usersched_task(p)) { -- enqueue_task_in_user_space(p, enq_flags); -- } --} -- --void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (usersched_needed) -- dispatch_user_scheduler(); -- -- bpf_repeat(4096) { -- s32 pid; -- struct task_struct *p; -- -- if (bpf_map_pop_elem(&dispatched, &pid)) -- break; -- -- /* -- * The task could have exited by the time we get around to -- * dispatching it. Treat this as a normal occurrence, and simply -- * move onto the next iteration. -- */ -- p = bpf_task_from_pid(pid); -- if (!p) -- continue; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } --} -- --s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(userland_init) --{ -- if (num_possible_cpus == 0) { -- scx_bpf_error("User scheduler # CPUs uninitialized (%d)", -- num_possible_cpus); -- return -EINVAL; -- } -- -- if (usersched_pid <= 0) { -- scx_bpf_error("User scheduler pid uninitialized (%d)", -- usersched_pid); -- return -EINVAL; -- } -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops userland_ops = { -- .select_cpu = (void *)userland_select_cpu, -- .enqueue = (void *)userland_enqueue, -- .dispatch = (void *)userland_dispatch, -- .prep_enable = (void *)userland_prep_enable, -- .init = (void *)userland_init, -- .exit = (void *)userland_exit, -- .timeout_ms = 3000, -- .name = "userland", --}; -diff --git a/tools/sched_ext/scx_example_userland.c b/tools/sched_ext/scx_example_userland.c -deleted file mode 100644 -index 4152b1e65fe1..000000000000 ---- a/tools/sched_ext/scx_example_userland.c -+++ /dev/null -@@ -1,402 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext user space scheduler which provides vruntime semantics -- * using a simple ordered-list implementation. -- * -- * Each CPU in the system resides in a single, global domain. This precludes -- * the need to do any load balancing between domains. The scheduler could -- * easily be extended to support multiple domains, with load balancing -- * happening in user space. -- * -- * Any task which has any CPU affinity is scheduled entirely in BPF. This -- * program only schedules tasks which may run on any CPU. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include "user_exit_info.h" --#include "scx_example_userland_common.h" --#include "scx_example_userland.skel.h" -- --const char help_fmt[] = --"A minimal userland sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-b BATCH] [-p]\n" --"\n" --" -b BATCH The number of tasks to batch when dispatching (default: 8)\n" --" -p Don't switch all, switch only tasks on SCHED_EXT policy\n" --" -h Display this help and exit\n"; -- --/* Defined in UAPI */ --#define SCHED_EXT 7 -- --/* Number of tasks to batch when dispatching to user space. */ --static __u32 batch_size = 8; -- --static volatile int exit_req; --static int enqueued_fd, dispatched_fd; -- --static struct scx_example_userland *skel; --static struct bpf_link *ops_link; -- --/* Stats collected in user space. */ --static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches; -- --/* The data structure containing tasks that are enqueued in user space. */ --struct enqueued_task { -- LIST_ENTRY(enqueued_task) entries; -- __u64 sum_exec_runtime; -- double vruntime; --}; -- --/* -- * Use a vruntime-sorted list to store tasks. This could easily be extended to -- * a more optimal data structure, such as an rbtree as is done in CFS. We -- * currently elect to use a sorted list to simplify the example for -- * illustrative purposes. -- */ --LIST_HEAD(listhead, enqueued_task); -- --/* -- * A vruntime-sorted list of tasks. The head of the list contains the task with -- * the lowest vruntime. That is, the task that has the "highest" claim to be -- * scheduled. -- */ --static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head); -- --/* -- * The statically allocated array of tasks. We use a statically allocated list -- * here to avoid having to allocate on the enqueue path, which could cause a -- * deadlock. A more substantive user space scheduler could e.g. provide a hook -- * for newly enabled tasks that are passed to the scheduler from the -- * .prep_enable() callback to allows the scheduler to allocate on safe paths. -- */ --struct enqueued_task tasks[USERLAND_MAX_TASKS]; -- --static double min_vruntime; -- --static void sigint_handler(int userland) --{ -- exit_req = 1; --} -- --static __u32 task_pid(const struct enqueued_task *task) --{ -- return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task); --} -- --static int dispatch_task(s32 pid) --{ -- int err; -- -- err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d\n", pid); -- exit_req = 1; -- } else { -- nr_vruntime_dispatches++; -- } -- -- return err; --} -- --static struct enqueued_task *get_enqueued_task(__s32 pid) --{ -- if (pid >= USERLAND_MAX_TASKS) -- return NULL; -- -- return &tasks[pid]; --} -- --static double calc_vruntime_delta(__u64 weight, __u64 delta) --{ -- double weight_f = (double)weight / 100.0; -- double delta_f = (double)delta; -- -- return delta_f / weight_f; --} -- --static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task) --{ -- __u64 delta; -- -- delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime; -- -- enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta); -- if (min_vruntime > enqueued->vruntime) -- enqueued->vruntime = min_vruntime; -- enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime; --} -- --static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task) --{ -- struct enqueued_task *curr, *enqueued, *prev; -- -- curr = get_enqueued_task(bpf_task->pid); -- if (!curr) -- return ENOENT; -- -- update_enqueued(curr, bpf_task); -- nr_vruntime_enqueues++; -- -- /* -- * Enqueue the task in a vruntime-sorted list. A more optimal data -- * structure such as an rbtree could easily be used as well. We elect -- * to use a list here simply because it's less code, and thus the -- * example is less convoluted and better serves to illustrate what a -- * user space scheduler could look like. -- */ -- -- if (LIST_EMPTY(&vruntime_head)) { -- LIST_INSERT_HEAD(&vruntime_head, curr, entries); -- return 0; -- } -- -- LIST_FOREACH(enqueued, &vruntime_head, entries) { -- if (curr->vruntime <= enqueued->vruntime) { -- LIST_INSERT_BEFORE(enqueued, curr, entries); -- return 0; -- } -- prev = enqueued; -- } -- -- LIST_INSERT_AFTER(prev, curr, entries); -- -- return 0; --} -- --static void drain_enqueued_map(void) --{ -- while (1) { -- struct scx_userland_enqueued_task task; -- int err; -- -- if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) -- return; -- -- err = vruntime_enqueue(&task); -- if (err) { -- fprintf(stderr, "Failed to enqueue task %d: %s\n", -- task.pid, strerror(err)); -- exit_req = 1; -- return; -- } -- } --} -- --static void dispatch_batch(void) --{ -- __u32 i; -- -- for (i = 0; i < batch_size; i++) { -- struct enqueued_task *task; -- int err; -- __s32 pid; -- -- task = LIST_FIRST(&vruntime_head); -- if (!task) -- return; -- -- min_vruntime = task->vruntime; -- pid = task_pid(task); -- LIST_REMOVE(task, entries); -- err = dispatch_task(pid); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d in %u\n", -- pid, i); -- return; -- } -- } --} -- --static void *run_stats_printer(void *arg) --{ -- while (!exit_req) { -- __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; -- -- nr_failed_enqueues = skel->bss->nr_failed_enqueues; -- nr_kernel_enqueues = skel->bss->nr_kernel_enqueues; -- nr_user_enqueues = skel->bss->nr_user_enqueues; -- total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues; -- -- printf("o-----------------------o\n"); -- printf("| BPF ENQUEUES |\n"); -- printf("|-----------------------|\n"); -- printf("| kern: %10llu |\n", nr_kernel_enqueues); -- printf("| user: %10llu |\n", nr_user_enqueues); -- printf("| failed: %10llu |\n", nr_failed_enqueues); -- printf("| -------------------- |\n"); -- printf("| total: %10llu |\n", total); -- printf("| |\n"); -- printf("|-----------------------|\n"); -- printf("| VRUNTIME / USER |\n"); -- printf("|-----------------------|\n"); -- printf("| enq: %10llu |\n", nr_vruntime_enqueues); -- printf("| disp: %10llu |\n", nr_vruntime_dispatches); -- printf("o-----------------------o\n"); -- printf("\n\n"); -- sleep(1); -- } -- -- return NULL; --} -- --static int spawn_stats_thread(void) --{ -- pthread_t stats_printer; -- -- return pthread_create(&stats_printer, NULL, run_stats_printer, NULL); --} -- --static int bootstrap(int argc, char **argv) --{ -- int err; -- __u32 opt; -- struct sched_param sched_param = { -- .sched_priority = sched_get_priority_max(SCHED_EXT), -- }; -- bool switch_partial = false; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- /* -- * Enforce that the user scheduler task is managed by sched_ext. The -- * task eagerly drains the list of enqueued tasks in its main work -- * loop, and then yields the CPU. The BPF scheduler only schedules the -- * user space scheduler task when at least one other task in the system -- * needs to be scheduled. -- */ -- err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param); -- if (err) { -- fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err)); -- return err; -- } -- -- while ((opt = getopt(argc, argv, "b:ph")) != -1) { -- switch (opt) { -- case 'b': -- batch_size = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- exit(opt != 'h'); -- } -- } -- -- /* -- * It's not always safe to allocate in a user space scheduler, as an -- * enqueued task could hold a lock that we require in order to be able -- * to allocate. -- */ -- err = mlockall(MCL_CURRENT | MCL_FUTURE); -- if (err) { -- fprintf(stderr, "Failed to prefault and lock address space: %s\n", -- strerror(err)); -- return err; -- } -- -- skel = scx_example_userland__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno)); -- return errno; -- } -- skel->rodata->num_possible_cpus = libbpf_num_possible_cpus(); -- assert(skel->rodata->num_possible_cpus > 0); -- skel->rodata->usersched_pid = getpid(); -- assert(skel->rodata->usersched_pid > 0); -- skel->rodata->switch_partial = switch_partial; -- -- err = scx_example_userland__load(skel); -- if (err) { -- fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- enqueued_fd = bpf_map__fd(skel->maps.enqueued); -- dispatched_fd = bpf_map__fd(skel->maps.dispatched); -- assert(enqueued_fd > 0); -- assert(dispatched_fd > 0); -- -- err = spawn_stats_thread(); -- if (err) { -- fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops); -- if (!ops_link) { -- fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno)); -- err = errno; -- goto destroy_skel; -- } -- -- return 0; -- --destroy_skel: -- scx_example_userland__destroy(skel); -- exit_req = 1; -- return err; --} -- --static void sched_main_loop(void) --{ -- while (!exit_req) { -- /* -- * Perform the following work in the main user space scheduler -- * loop: -- * -- * 1. Drain all tasks from the enqueued map, and enqueue them -- * to the vruntime sorted list. -- * -- * 2. Dispatch a batch of tasks from the vruntime sorted list -- * down to the kernel. -- * -- * 3. Yield the CPU back to the system. The BPF scheduler will -- * reschedule the user space scheduler once another task has -- * been enqueued to user space. -- */ -- drain_enqueued_map(); -- dispatch_batch(); -- sched_yield(); -- } --} -- --int main(int argc, char **argv) --{ -- int err; -- -- err = bootstrap(argc, argv); -- if (err) { -- fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err)); -- return err; -- } -- -- sched_main_loop(); -- -- exit_req = 1; -- bpf_link__destroy(ops_link); -- uei_print(&skel->bss->uei); -- scx_example_userland__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_userland_common.h b/tools/sched_ext/scx_example_userland_common.h -deleted file mode 100644 -index 639c6809c5ff..000000000000 ---- a/tools/sched_ext/scx_example_userland_common.h -+++ /dev/null -@@ -1,19 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* Copyright (c) 2022 Meta, Inc */ -- --#ifndef __SCX_USERLAND_COMMON_H --#define __SCX_USERLAND_COMMON_H -- --#define USERLAND_MAX_TASKS 8192 -- --/* -- * An instance of a task that has been enqueued by the kernel for consumption -- * by a user space global scheduler thread. -- */ --struct scx_userland_enqueued_task { -- __s32 pid; -- u64 sum_exec_runtime; -- u64 weight; --}; -- --#endif // __SCX_USERLAND_COMMON_H -diff --git a/tools/sched_ext/user_exit_info.h b/tools/sched_ext/user_exit_info.h -deleted file mode 100644 -index e701ef0e0b86..000000000000 ---- a/tools/sched_ext/user_exit_info.h -+++ /dev/null -@@ -1,50 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Define struct user_exit_info which is shared between BPF and userspace parts -- * to communicate exit status and other information. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __USER_EXIT_INFO_H --#define __USER_EXIT_INFO_H -- --struct user_exit_info { -- int type; -- char reason[128]; -- char msg[1024]; --}; -- --#ifdef __bpf__ -- --#include "vmlinux.h" --#include -- --static inline void uei_record(struct user_exit_info *uei, -- const struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason); -- bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg); -- /* use __sync to force memory barrier */ -- __sync_val_compare_and_swap(&uei->type, uei->type, ei->type); --} -- --#else /* !__bpf__ */ -- --static inline bool uei_exited(struct user_exit_info *uei) --{ -- /* use __sync to force memory barrier */ -- return __sync_val_compare_and_swap(&uei->type, -1, -1); --} -- --static inline void uei_print(const struct user_exit_info *uei) --{ -- fprintf(stderr, "EXIT: %s", uei->reason); -- if (uei->msg[0] != '\0') -- fprintf(stderr, " (%s)", uei->msg); -- fputs("\n", stderr); --} -- --#endif /* __bpf__ */ --#endif /* __USER_EXIT_INFO_H */ -diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c -index 3a7bd62ef6b7..e8c98ab86084 100644 ---- a/drivers/cpufreq/cpufreq.c -+++ b/drivers/cpufreq/cpufreq.c -@@ -810,7 +810,7 @@ static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy, - /* - * show_scaling_governor - show the current policy for the specified CPU - */ --static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) -+ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) - { - if (policy->policy == CPUFREQ_POLICY_POWERSAVE) - return sprintf(buf, "powersave\n"); -@@ -825,7 +825,7 @@ static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) - /* - * store_scaling_governor - store policy for the specified CPU - */ --static ssize_t store_scaling_governor(struct cpufreq_policy *policy, -+ssize_t store_scaling_governor(struct cpufreq_policy *policy, - const char *buf, size_t count) - { - char str_governor[16]; -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -index ce05928392bf..467061a1ffa4 100755 ---- a/kernel/sched/hmbird/hmbird.c -+++ b/kernel/sched/hmbird/hmbird.c -@@ -633,14 +633,14 @@ static void init_isolate_cpus(void) - WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -- cpumask_set_cpu(0, iso_masks.big); -- cpumask_set_cpu(1, iso_masks.big); -- cpumask_set_cpu(2, iso_masks.big); -- cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(0, iso_masks.little); -+ cpumask_set_cpu(1, iso_masks.little); -+ cpumask_set_cpu(2, iso_masks.little); -+ cpumask_set_cpu(3, iso_masks.little); - cpumask_set_cpu(4, iso_masks.big); -- cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(5, iso_masks.big); - cpumask_set_cpu(6, iso_masks.partial); -- cpumask_set_cpu(7, iso_masks.ex_free); -+ cpumask_set_cpu(7, iso_masks.exclusive); - } - - extern spinlock_t css_set_lock; -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index 692c9aa0ffa6..c536364f4736 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -10942,7 +10942,6 @@ void sched_move_task(struct task_struct *tsk) - { - int queued, running, queue_flags = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; -- struct task_group *group; - struct rq_flags rf; - struct rq *rq; - -@@ -10958,7 +10957,7 @@ void sched_move_task(struct task_struct *tsk) - if (running) - put_prev_task(rq, tsk); - -- sched_change_group(tsk, group); -+ sched_change_group(tsk); - - if (queued) - enqueue_task(rq, tsk, queue_flags); diff --git a/oneplus/hmbird/fengchi_OP-ACE-5-ULTRA_A16.patch b/oneplus/hmbird/fengchi_OP-ACE-5-ULTRA_A16.patch deleted file mode 100644 index 42ddfa7e..00000000 --- a/oneplus/hmbird/fengchi_OP-ACE-5-ULTRA_A16.patch +++ /dev/null @@ -1,20969 +0,0 @@ -diff --git a/Documentation/scheduler/index.rst b/Documentation/scheduler/index.rst -index 0b650bb550e6..3170747226f6 100644 ---- a/Documentation/scheduler/index.rst -+++ b/Documentation/scheduler/index.rst -@@ -19,7 +19,6 @@ Scheduler - sched-nice-design - sched-rt-group - sched-stats -- sched-ext - sched-debug - - text_files -diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst -deleted file mode 100644 -index 84c30b44f104..000000000000 ---- a/Documentation/scheduler/sched-ext.rst -+++ /dev/null -@@ -1,230 +0,0 @@ --========================== --Extensible Scheduler Class --========================== -- --sched_ext is a scheduler class whose behavior can be defined by a set of BPF --programs - the BPF scheduler. -- --* sched_ext exports a full scheduling interface so that any scheduling -- algorithm can be implemented on top. -- --* The BPF scheduler can group CPUs however it sees fit and schedule them -- together, as tasks aren't tied to specific CPUs at the time of wakeup. -- --* The BPF scheduler can be turned on and off dynamically anytime. -- --* The system integrity is maintained no matter what the BPF scheduler does. -- The default scheduling behavior is restored anytime an error is detected, -- a runnable task stalls, or on invoking the SysRq key sequence -- :kbd:`SysRq-S`. -- --Switching to and from sched_ext --=============================== -- --``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and --``tools/sched_ext`` contains the example schedulers. -- --sched_ext is used only when the BPF scheduler is loaded and running. -- --If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be --treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is --loaded. On load, such tasks will be switched to and scheduled by sched_ext. -- --The BPF scheduler can choose to schedule all normal and lower class tasks by --calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this --case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and --``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers, --this mode can be selected with the ``-a`` option. -- --Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or --detection of any internal error including stalled runnable tasks aborts the --BPF scheduler and reverts all tasks back to CFS. -- --.. code-block:: none -- -- # make -j16 -C tools/sched_ext -- # tools/sched_ext/scx_example_simple -- local=0 global=3 -- local=5 global=24 -- local=9 global=44 -- local=13 global=56 -- local=17 global=72 -- ^CEXIT: BPF scheduler unregistered -- --If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler --and whether a given task is on sched_ext can be determined as follows: -- --.. code-block:: none -- -- # cat /sys/kernel/debug/sched/ext -- ops : simple -- enabled : 1 -- switching_all : 1 -- switched_all : 1 -- enable_state : enabled -- -- # grep ext /proc/self/sched -- ext.enabled : 1 -- --The Basics --========== -- --Userspace can implement an arbitrary BPF scheduler by loading a set of BPF --programs that implement ``struct sched_ext_ops``. The only mandatory field --is ``ops.name`` which must be a valid BPF object name. All operations are --optional. The following modified excerpt is from --``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO --scheduler. -- --.. code-block:: c -- -- s32 BPF_STRUCT_OPS(simple_init) -- { -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; -- } -- -- void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) -- { -- if (enq_flags & SCX_ENQ_LOCAL) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags); -- } -- -- void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) -- { -- exit_type = ei->type; -- } -- -- SEC(".struct_ops") -- struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", -- }; -- --Dispatch Queues ----------------- -- --To match the impedance between the scheduler core and the BPF scheduler, --sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a --priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``), --and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage --an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and --``scx_bpf_destroy_dsq()``. -- --A CPU always executes a task from its local DSQ. A task is "dispatched" to a --DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's --local DSQ. -- --When a CPU is looking for the next task to run, if the local DSQ is not --empty, the first task is picked. Otherwise, the CPU tries to consume the --global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()`` --is invoked. -- --Scheduling Cycle ------------------ -- --The following briefly shows how a waking task is scheduled and executed. -- --1. When a task is waking up, ``ops.select_cpu()`` is the first operation -- invoked. This serves two purposes. First, CPU selection optimization -- hint. Second, waking up the selected CPU if idle. -- -- The CPU selected by ``ops.select_cpu()`` is an optimization hint and not -- binding. The actual decision is made at the last step of scheduling. -- However, there is a small performance gain if the CPU -- ``ops.select_cpu()`` returns matches the CPU the task eventually runs on. -- -- A side-effect of selecting a CPU is waking it up from idle. While a BPF -- scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper, -- using ``ops.select_cpu()`` judiciously can be simpler and more efficient. -- -- Note that the scheduler core will ignore an invalid CPU selection, for -- example, if it's outside the allowed cpumask of the task. -- --2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can -- make one of the following decisions: -- -- * Immediately dispatch the task to either the global or local DSQ by -- calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or -- ``SCX_DSQ_LOCAL``, respectively. -- -- * Immediately dispatch the task to a custom DSQ by calling -- ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63. -- -- * Queue the task on the BPF side. -- --3. When a CPU is ready to schedule, it first looks at its local DSQ. If -- empty, it then looks at the global DSQ. If there still isn't a task to -- run, ``ops.dispatch()`` is invoked which can use the following two -- functions to populate the local DSQ. -- -- * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can -- be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``, -- ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()`` -- currently can't be called with BPF locks held, this is being worked on -- and will be supported. ``scx_bpf_dispatch()`` schedules dispatching -- rather than performing them immediately. There can be up to -- ``ops.dispatch_max_batch`` pending tasks. -- -- * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ -- to the dispatching DSQ. This function cannot be called with any BPF -- locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks -- before trying to consume the specified DSQ. -- --4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ, -- the CPU runs the first one. If empty, the following steps are taken: -- -- * Try to consume the global DSQ. If successful, run the task. -- -- * If ``ops.dispatch()`` has dispatched any tasks, retry #3. -- -- * If the previous task is an SCX task and still runnable, keep executing -- it (see ``SCX_OPS_ENQ_LAST``). -- -- * Go idle. -- --Note that the BPF scheduler can always choose to dispatch tasks immediately --in ``ops.enqueue()`` as illustrated in the above simple example. If only the --built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as --a task is never queued on the BPF scheduler and both the local and global --DSQs are consumed automatically. -- --``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use --``scx_bpf_dispatch_vtime()`` for the priority queue. See the function --documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for --more information. -- --Where to Look --============= -- --* ``include/linux/sched/ext.h`` defines the core data structures, ops table -- and constants. -- --* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. -- The functions prefixed with ``scx_bpf_`` can be called from the BPF -- scheduler. -- --* ``tools/sched_ext/`` hosts example BPF scheduler implementations. -- -- * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example -- using a custom DSQ. -- -- * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting -- five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. -- --ABI Instability --=============== -- --The APIs provided by sched_ext to BPF schedulers programs have no stability --guarantees. This includes the ops table callbacks and constants defined in --``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in --``kernel/sched/ext.c``. -- --While we will attempt to provide a relatively stable API surface when --possible, they are subject to change without warning between kernel --versions. -diff --git a/MAINTAINERS b/MAINTAINERS -index 9fc6108503c3..2384b55682ee 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -19132,8 +19132,6 @@ R: Ben Segall (CONFIG_CFS_BANDWIDTH) - R: Mel Gorman (CONFIG_NUMA_BALANCING) - R: Daniel Bristot de Oliveira (SCHED_DEADLINE) - R: Valentin Schneider (TOPOLOGY) --R: Tejun Heo (SCHED_EXT) --R: David Vernet (SCHED_EXT) - L: linux-kernel@vger.kernel.org - S: Maintained - T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core -@@ -19142,7 +19140,6 @@ F: include/linux/sched.h - F: include/linux/wait.h - F: include/uapi/linux/sched.h - F: kernel/sched/ --F: tools/sched_ext/ - - SCSI LIBSAS SUBSYSTEM - R: John Garry -diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig -index f93980c09d7f..5ba590235a8c 100644 ---- a/arch/arm64/configs/defconfig -+++ b/arch/arm64/configs/defconfig -@@ -6,8 +6,7 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_BPF_SYSCALL=y - CONFIG_BPF_JIT=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_BSD_PROCESS_ACCT=y - CONFIG_BSD_PROCESS_ACCT_V3=y -diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig -index 4f756dca5e05..6c1bb94f875d 100644 ---- a/arch/arm64/configs/gki_defconfig -+++ b/arch/arm64/configs/gki_defconfig -@@ -10,8 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y - # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set - CONFIG_BPF_LSM=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_TASKSTATS=y - CONFIG_TASK_DELAY_ACCT=y -diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c -index 4bd028fa7500..5825ce9d6d7b 100644 ---- a/drivers/gpu/drm/msm/msm_drv.c -+++ b/drivers/gpu/drm/msm/msm_drv.c -@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) - } - - sched_set_fifo(ev_thread->worker->task); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3); -+#endif - } - - ret = drm_vblank_init(ddev, priv->num_crtcs); -diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c -index bd001445815e..dcf2e1ebf9c5 100644 ---- a/drivers/tty/sysrq.c -+++ b/drivers/tty/sysrq.c -@@ -524,7 +524,6 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { - NULL, /* P */ - NULL, /* Q */ - NULL, /* R */ -- /* S: May be registered by sched_ext for resetting */ - NULL, /* S */ - NULL, /* T */ - NULL, /* U */ -diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h -index c45078a445c8..6c15659f874e 100644 ---- a/include/asm-generic/vmlinux.lds.h -+++ b/include/asm-generic/vmlinux.lds.h -@@ -131,7 +131,7 @@ - *(__dl_sched_class) \ - *(__rt_sched_class) \ - *(__fair_sched_class) \ -- *(__ext_sched_class) \ -+ *(__hmbird_sched_class) \ - *(__idle_sched_class) \ - __sched_class_lowest = .; - -diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h -index 63452733a2f9..3a2f906ed3ce 100644 ---- a/include/linux/cpufreq.h -+++ b/include/linux/cpufreq.h -@@ -44,6 +44,10 @@ enum cpufreq_table_sorting { - CPUFREQ_TABLE_SORTED_DESCENDING - }; - -+ssize_t store_scaling_governor(struct cpufreq_policy *policy, -+ const char *buf, size_t count); -+ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf); -+ - struct cpufreq_cpuinfo { - unsigned int max_freq; - unsigned int min_freq; -diff --git a/include/linux/sched.h b/include/linux/sched.h -index 2cbc342db12d..dd6b1d58af01 100644 ---- a/include/linux/sched.h -+++ b/include/linux/sched.h -@@ -73,7 +73,9 @@ struct task_delay_info; - struct task_group; - struct user_event_mm; - --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - - /* - * Task state bitmask. NOTE! These bits are also -@@ -298,11 +300,6 @@ enum { - - extern void scheduler_tick(void); - --#ifdef CONFIG_SLIM_SCHED --extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer); --extern void stop_shadow_tick_timer(void); --#endif -- - #define MAX_SCHEDULE_TIMEOUT LONG_MAX - - extern long schedule_timeout(long timeout); -@@ -1523,13 +1520,8 @@ struct task_struct { - */ - struct callback_head l1d_flush_kill; - #endif --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, unsigned long sched_prop); -- ANDROID_KABI_USE(2, struct sched_ext_entity *scx); --#else - ANDROID_KABI_RESERVE(1); - ANDROID_KABI_RESERVE(2); --#endif - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); - ANDROID_KABI_RESERVE(5); -@@ -1933,6 +1925,92 @@ static inline int task_nice(const struct task_struct *p) - return PRIO_TO_NICE((p)->static_prio); - } - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline bool task_is_top_task(struct task_struct *p) -+{ -+ return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ ->top_task_prop & TOP_TASK_BITS_MASK); -+} -+ -+static inline int get_top_task_prop(struct task_struct *p) -+{ -+ return get_hmbird_ts(p)->top_task_prop; -+} -+ -+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear) -+{ -+ if (set) -+ get_hmbird_ts(p)->top_task_prop |= set; -+ if (clear) -+ get_hmbird_ts(p)->top_task_prop &= ~clear; -+ return 0; -+} -+ -+static inline void reset_top_task_prop(struct task_struct *p) -+{ -+ get_hmbird_ts(p)->top_task_prop = 0; -+} -+ -+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ entity->sched_prop = sp; -+ } -+ return 0; -+} -+ -+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->sched_prop; -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq) -+{ -+ unsigned long new_dsq; -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq; -+ entity->sched_prop = new_dsq; -+ } -+} -+ -+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK); -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ entity->dsq_sync_ux = val; -+} -+ -+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->dsq_sync_ux; -+ else -+ return 0; -+} -+#endif -+ - extern int can_nice(const struct task_struct *p, const int nice); - extern int task_curr(const struct task_struct *p); - extern int idle_cpu(int cpu); -diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h -deleted file mode 100755 -index b737a00c1373..000000000000 ---- a/include/linux/sched/ext.h -+++ /dev/null -@@ -1,647 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef _LINUX_SCHED_EXT_H --#define _LINUX_SCHED_EXT_H -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --#include -- --enum scx_consts { -- SCX_OPS_NAME_LEN = 128, -- SCX_EXIT_REASON_LEN = 128, -- SCX_EXIT_BT_LEN = 64, -- SCX_EXIT_MSG_LEN = 1024, -- -- SCX_SLICE_DFL = 20 * NSEC_PER_MSEC, -- SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */ --}; -- --/* -- * DSQ (dispatch queue) IDs are 64bit of the format: -- * -- * Bits: [63] [62 .. 0] -- * [ B] [ ID ] -- * -- * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -- * ID: 63 bit ID -- * -- * Built-in IDs: -- * -- * Bits: [63] [62] [61..32] [31 .. 0] -- * [ 1] [ L] [ R ] [ V ] -- * -- * 1: 1 for built-in DSQs. -- * L: 1 for LOCAL_ON DSQ IDs, 0 for others -- * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -- */ --enum scx_dsq_id_flags { -- SCX_DSQ_FLAG_BUILTIN = 1LLU << 63, -- SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -- -- SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0, -- SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, -- SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, -- SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, -- SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, --}; -- --enum scx_exit_type { -- SCX_EXIT_NONE, -- SCX_EXIT_DONE, -- -- SCX_EXIT_UNREG = 64, /* BPF unregistration */ -- SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -- SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ -- SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ --}; -- --/* -- * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is -- * being disabled. -- */ --struct scx_exit_info { -- /* %SCX_EXIT_* - broad category of the exit reason */ -- enum scx_exit_type type; -- /* textual representation of the above */ -- char reason[SCX_EXIT_REASON_LEN]; -- /* number of entries in the backtrace */ -- u32 bt_len; -- /* backtrace if exiting due to an error */ -- unsigned long bt[SCX_EXIT_BT_LEN]; -- /* extra message */ -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --/* sched_ext_ops.flags */ --enum scx_ops_flags { -- /* -- * Keep built-in idle tracking even if ops.update_idle() is implemented. -- */ -- SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, -- -- /* -- * By default, if there are no other task to run on the CPU, ext core -- * keeps running the current task even after its slice expires. If this -- * flag is specified, such tasks are passed to ops.enqueue() with -- * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. -- */ -- SCX_OPS_ENQ_LAST = 1LLU << 1, -- -- /* -- * An exiting task may schedule after PF_EXITING is set. In such cases, -- * bpf_task_from_pid() may not be able to find the task and if the BPF -- * scheduler depends on pid lookup for dispatching, the task will be -- * lost leading to various issues including RCU grace period stalls. -- * -- * To mask this problem, by default, unhashed tasks are automatically -- * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't -- * depend on pid lookups and wants to handle these tasks directly, the -- * following flag can be used. -- */ -- SCX_OPS_ENQ_EXITING = 1LLU << 2, -- -- /* -- * CPU cgroup knob enable flags -- */ -- SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16, /* cpu.weight */ -- -- SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | -- SCX_OPS_ENQ_LAST | -- SCX_OPS_ENQ_EXITING | -- SCX_OPS_CGROUP_KNOB_WEIGHT, --}; -- --/* argument container for ops.enable() and friends */ --struct scx_enable_args { --}; -- --/* argument container for ops->cgroup_init() */ --struct scx_cgroup_init_args { -- /* the weight of the cgroup [1..10000] */ -- u32 weight; --}; -- --enum scx_cpu_preempt_reason { -- /* next task is being scheduled by &sched_class_rt */ -- SCX_CPU_PREEMPT_RT, -- /* next task is being scheduled by &sched_class_dl */ -- SCX_CPU_PREEMPT_DL, -- /* next task is being scheduled by &sched_class_stop */ -- SCX_CPU_PREEMPT_STOP, -- /* unknown reason for SCX being preempted */ -- SCX_CPU_PREEMPT_UNKNOWN, --}; -- --/* -- * Argument container for ops->cpu_acquire(). Currently empty, but may be -- * expanded in the future. -- */ --struct scx_cpu_acquire_args {}; -- --/* argument container for ops->cpu_release() */ --struct scx_cpu_release_args { -- /* the reason the CPU was preempted */ -- enum scx_cpu_preempt_reason reason; -- -- /* the task that's going to be scheduled on the CPU */ -- struct task_struct *task; --}; -- --/** -- * struct sched_ext_ops - Operation table for BPF scheduler implementation -- * -- * Userland can implement an arbitrary scheduling policy by implementing and -- * loading operations in this table. -- */ --struct sched_ext_ops { -- /** -- * select_cpu - Pick the target CPU for a task which is being woken up -- * @p: task being woken up -- * @prev_cpu: the cpu @p was on before sleeping -- * @wake_flags: SCX_WAKE_* -- * -- * Decision made here isn't final. @p may be moved to any CPU while it -- * is getting dispatched for execution later. However, as @p is not on -- * the rq at this point, getting the eventual execution CPU right here -- * saves a small bit of overhead down the line. -- * -- * If an idle CPU is returned, the CPU is kicked and will try to -- * dispatch. While an explicit custom mechanism can be added, -- * select_cpu() serves as the default way to wake up idle CPUs. -- */ -- s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); -- -- /** -- * enqueue - Enqueue a task on the BPF scheduler -- * @p: task being enqueued -- * @enq_flags: %SCX_ENQ_* -- * -- * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() -- * or enqueue on the BPF scheduler. If not directly dispatched, the bpf -- * scheduler owns @p and if it fails to dispatch @p, the task will -- * stall. -- */ -- void (*enqueue)(struct task_struct *p, u64 enq_flags); -- -- /** -- * dequeue - Remove a task from the BPF scheduler -- * @p: task being dequeued -- * @deq_flags: %SCX_DEQ_* -- * -- * Remove @p from the BPF scheduler. This is usually called to isolate -- * the task while updating its scheduling properties (e.g. priority). -- * -- * The ext core keeps track of whether the BPF side owns a given task or -- * not and can gracefully ignore spurious dispatches from BPF side, -- * which makes it safe to not implement this method. However, depending -- * on the scheduling logic, this can lead to confusing behaviors - e.g. -- * scheduling position not being updated across a priority change. -- */ -- void (*dequeue)(struct task_struct *p, u64 deq_flags); -- -- /** -- * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs -- * @cpu: CPU to dispatch tasks for -- * @prev: previous task being switched out -- * -- * Called when a CPU's local dsq is empty. The operation should dispatch -- * one or more tasks from the BPF scheduler into the DSQs using -- * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using -- * scx_bpf_consume(). -- * -- * The maximum number of times scx_bpf_dispatch() can be called without -- * an intervening scx_bpf_consume() is specified by -- * ops.dispatch_max_batch. See the comments on top of the two functions -- * for more details. -- * -- * When not %NULL, @prev is an SCX task with its slice depleted. If -- * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in -- * @prev->scx.flags, it is not enqueued yet and will be enqueued after -- * ops.dispatch() returns. To keep executing @prev, return without -- * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. -- */ -- void (*dispatch)(s32 cpu, struct task_struct *prev); -- -- /** -- * runnable - A task is becoming runnable on its associated CPU -- * @p: task becoming runnable -- * @enq_flags: %SCX_ENQ_* -- * -- * This and the following three functions can be used to track a task's -- * execution state transitions. A task becomes ->runnable() on a CPU, -- * and then goes through one or more ->running() and ->stopping() pairs -- * as it runs on the CPU, and eventually becomes ->quiescent() when it's -- * done running on the CPU. -- * -- * @p is becoming runnable on the CPU because it's -- * -- * - waking up (%SCX_ENQ_WAKEUP) -- * - being moved from another CPU -- * - being restored after temporarily taken off the queue for an -- * attribute change. -- * -- * This and ->enqueue() are related but not coupled. This operation -- * notifies @p's state transition and may not be followed by ->enqueue() -- * e.g. when @p is being dispatched to a remote CPU. Likewise, a task -- * may be ->enqueue()'d without being preceded by this operation e.g. -- * after exhausting its slice. -- */ -- void (*runnable)(struct task_struct *p, u64 enq_flags); -- -- /** -- * running - A task is starting to run on its associated CPU -- * @p: task starting to run -- * -- * See ->runnable() for explanation on the task state notifiers. -- */ -- void (*running)(struct task_struct *p); -- -- /** -- * stopping - A task is stopping execution -- * @p: task stopping to run -- * @runnable: is task @p still runnable? -- * -- * See ->runnable() for explanation on the task state notifiers. If -- * !@runnable, ->quiescent() will be invoked after this operation -- * returns. -- */ -- void (*stopping)(struct task_struct *p, bool runnable); -- -- /** -- * quiescent - A task is becoming not runnable on its associated CPU -- * @p: task becoming not runnable -- * @deq_flags: %SCX_DEQ_* -- * -- * See ->runnable() for explanation on the task state notifiers. -- * -- * @p is becoming quiescent on the CPU because it's -- * -- * - sleeping (%SCX_DEQ_SLEEP) -- * - being moved to another CPU -- * - being temporarily taken off the queue for an attribute change -- * (%SCX_DEQ_SAVE) -- * -- * This and ->dequeue() are related but not coupled. This operation -- * notifies @p's state transition and may not be preceded by ->dequeue() -- * e.g. when @p is being dispatched to a remote CPU. -- */ -- void (*quiescent)(struct task_struct *p, u64 deq_flags); -- -- /** -- * yield - Yield CPU -- * @from: yielding task -- * @to: optional yield target task -- * -- * If @to is NULL, @from is yielding the CPU to other runnable tasks. -- * The BPF scheduler should ensure that other available tasks are -- * dispatched before the yielding task. Return value is ignored in this -- * case. -- * -- * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf -- * scheduler can implement the request, return %true; otherwise, %false. -- */ -- bool (*yield)(struct task_struct *from, struct task_struct *to); -- -- /** -- * core_sched_before - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Used by core-sched to determine the ordering between two tasks. See -- * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on -- * core-sched. -- * -- * Both @a and @b are runnable and may or may not currently be queued on -- * the BPF scheduler. Should return %true if @a should run before @b. -- * %false if there's no required ordering or @b should run before @a. -- * -- * If not specified, the default is ordering them according to when they -- * became runnable. -- */ -- bool (*core_sched_before)(struct task_struct *a,struct task_struct *b); -- -- /** -- * set_weight - Set task weight -- * @p: task to set weight for -- * @weight: new eight [1..10000] -- * -- * Update @p's weight to @weight. -- */ -- void (*set_weight)(struct task_struct *p, u32 weight); -- -- /** -- * set_cpumask - Set CPU affinity -- * @p: task to set CPU affinity for -- * @cpumask: cpumask of cpus that @p can run on -- * -- * Update @p's CPU affinity to @cpumask. -- */ -- void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask); -- -- /** -- * update_idle - Update the idle state of a CPU -- * @cpu: CPU to udpate the idle state for -- * @idle: whether entering or exiting the idle state -- * -- * This operation is called when @rq's CPU goes or leaves the idle -- * state. By default, implementing this operation disables the built-in -- * idle CPU tracking and the following helpers become unavailable: -- * -- * - scx_bpf_select_cpu_dfl() -- * - scx_bpf_test_and_clear_cpu_idle() -- * - scx_bpf_pick_idle_cpu() -- * - scx_bpf_any_idle_cpu() -- * -- * The user also must implement ops.select_cpu() as the default -- * implementation relies on scx_bpf_select_cpu_dfl(). -- * -- * If you keep the built-in idle tracking, specify the -- * %SCX_OPS_KEEP_BUILTIN_IDLE flag. -- */ -- void (*update_idle)(s32 cpu, bool idle); -- -- /** -- * cpu_acquire - A CPU is becoming available to the BPF scheduler -- * @cpu: The CPU being acquired by the BPF scheduler. -- * @args: Acquire arguments, see the struct definition. -- * -- * A CPU that was previously released from the BPF scheduler is now once -- * again under its control. -- */ -- void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); -- -- /** -- * cpu_release - A CPU is taken away from the BPF scheduler -- * @cpu: The CPU being released by the BPF scheduler. -- * @args: Release arguments, see the struct definition. -- * -- * The specified CPU is no longer under the control of the BPF -- * scheduler. This could be because it was preempted by a higher -- * priority sched_class, though there may be other reasons as well. The -- * caller should consult @args->reason to determine the cause. -- */ -- void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -- -- /** -- * cpu_online - A CPU became online -- * @cpu: CPU which just came up -- * -- * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs beforehand. -- */ -- void (*cpu_online)(s32 cpu); -- -- /** -- * cpu_offline - A CPU is going offline -- * @cpu: CPU which is going offline -- * -- * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs afterwards. -- */ -- void (*cpu_offline)(s32 cpu); -- -- /** -- * prep_enable - Prepare to enable BPF scheduling for a task -- * @p: task to prepare BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Either we're loading a BPF scheduler or a new task is being forked. -- * Prepare BPF scheduling for @p. This operation may block and can be -- * used for allocations. -- * -- * Return 0 for success, -errno for failure. An error return while -- * loading will abort loading of the BPF scheduler. During a fork, will -- * abort the specific fork. -- */ -- s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * enable - Enable BPF scheduling for a task -- * @p: task to enable BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Enable @p for BPF scheduling. @p is now in the cgroup specified for -- * the preceding prep_enable() and will start running soon. -- */ -- void (*enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * cancel_enable - Cancel prep_enable() -- * @p: task being canceled -- * @args: enable arguments, see the struct definition -- * -- * @p was prep_enable()'d but failed before reaching enable(). Undo the -- * preparation. -- */ -- void (*cancel_enable)(struct task_struct *p, -- struct scx_enable_args *args); -- -- /** -- * disable - Disable BPF scheduling for a task -- * @p: task to disable BPF scheduling for -- * -- * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. -- * Disable BPF scheduling for @p. -- */ -- void (*disable)(struct task_struct *p); -- -- /* -- * All online ops must come before ops.init(). -- */ -- -- /** -- * init - Initialize the BPF scheduler -- */ -- s32 (*init)(void); -- -- /** -- * exit - Clean up after the BPF scheduler -- * @info: Exit info -- */ -- void (*exit)(struct scx_exit_info *info); -- -- /** -- * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch -- */ -- u32 dispatch_max_batch; -- -- /** -- * flags - %SCX_OPS_* flags -- */ -- u64 flags; -- -- /** -- * timeout_ms - The maximum amount of time, in milliseconds, that a -- * runnable task should be able to wait before being scheduled. The -- * maximum timeout may not exceed the default timeout of 30 seconds. -- * -- * Defaults to the maximum allowed timeout value of 30 seconds. -- */ -- u32 timeout_ms; -- -- /** -- * name - BPF scheduler's name -- * -- * Must be a non-zero valid BPF object name including only isalnum(), -- * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the -- * BPF scheduler is enabled. -- */ -- char name[SCX_OPS_NAME_LEN]; --}; -- --/* -- * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -- * scheduler core and the BPF scheduler. See the documentation for more details. -- */ --struct scx_dispatch_q { -- raw_spinlock_t lock; -- struct list_head fifo; /* processed in dispatching order */ -- struct rb_root_cached priq; /* processed in p->scx.dsq_vtime order */ -- u32 nr; -- u64 id; -- struct llist_node free_node; -- struct rcu_head rcu; --}; -- --/* scx_entity.flags */ --enum scx_ent_flags { -- SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ -- SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -- SCX_TASK_ENQ_LOCAL = 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */ -- -- SCX_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -- SCX_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -- -- SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -- SCX_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -- -- SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ --}; -- --/* scx_entity.dsq_flags */ --enum scx_ent_dsq_flags { -- SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ --}; -- --/* -- * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from -- * everywhere and the following bits track which kfunc sets are currently -- * allowed for %current. This simple per-task tracking works because SCX ops -- * nest in a limited way. BPF will likely implement a way to allow and disallow -- * kfuncs depending on the calling context which will replace this manual -- * mechanism. See scx_kf_allow(). -- */ --enum scx_kf_mask { -- SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ -- /* all non-sleepables may be nested inside INIT and SLEEPABLE */ -- SCX_KF_INIT = 1 << 0, /* running ops.init() */ -- SCX_KF_SLEEPABLE = 1 << 1, /* other sleepable init operations */ -- /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ -- SCX_KF_CPU_RELEASE = 1 << 2, /* ops.cpu_release() */ -- /* ops.dequeue (in REST) may be nested inside DISPATCH */ -- SCX_KF_DISPATCH = 1 << 3, /* ops.dispatch() */ -- SCX_KF_ENQUEUE = 1 << 4, /* ops.enqueue() */ -- SCX_KF_REST = 1 << 5, /* other rq-locked operations */ -- -- __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | -- SCX_KF_ENQUEUE | SCX_KF_REST, -- __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_REST, --}; -- --#define RAVG_HIST_SIZE 5 --struct scx_sched_task_stats { -- u64 mark_start; -- u64 window_start; -- u32 sum; -- u32 sum_history[RAVG_HIST_SIZE]; -- int cidx; -- -- u32 demand; -- u16 demand_scaled; -- void *sdsq; --}; -- -- -- --/* -- * The following is embedded in task_struct and contains all fields necessary -- * for a task to be scheduled by SCX. -- */ --struct sched_ext_entity { -- struct scx_dispatch_q *dsq; -- struct { -- struct list_head fifo; /* dispatch order */ -- struct rb_node priq; /* p->scx.dsq_vtime order */ -- } dsq_node; -- struct list_head watchdog_node; -- u32 flags; /* protected by rq lock */ -- u32 dsq_flags; /* protected by dsq lock */ -- u32 weight; -- s32 sticky_cpu; -- s32 holding_cpu; -- u32 kf_mask; /* see scx_kf_mask above */ -- struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ -- atomic64_t ops_state; -- unsigned long runnable_at; --#ifdef CONFIG_SCHED_CORE -- u64 core_sched_at; /* see scx_prio_less() */ --#endif -- -- /* BPF scheduler modifiable fields */ -- -- /* -- * Runtime budget in nsecs. This is usually set through -- * scx_bpf_dispatch() but can also be modified directly by the BPF -- * scheduler. Automatically decreased by SCX as the task executes. On -- * depletion, a scheduling event is triggered. -- * -- * This value is cleared to zero if the task is preempted by -- * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the -- * task ran. Use p->se.sum_exec_runtime instead. -- */ -- u64 slice; -- -- /* -- * Used to order tasks when dispatching to the vtime-ordered priority -- * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime() -- * but can also be modified directly by the BPF scheduler. Modifying it -- * while a task is queued on a dsq may mangle the ordering and is not -- * recommended. -- */ -- u64 dsq_vtime; -- -- /* -- * If set, reject future sched_setscheduler(2) calls updating the policy -- * to %SCHED_EXT with -%EACCES. -- * -- * If set from ops.prep_enable() and the task's policy is already -- * %SCHED_EXT, which can happen while the BPF scheduler is being loaded -- * or by inhering the parent's policy during fork, the task's policy is -- * rejected and forcefully reverted to %SCHED_NORMAL. The number of such -- * events are reported through /sys/kernel/debug/sched_ext::nr_rejected. -- */ -- bool disallow; /* reject switching into SCX */ -- -- /* cold fields */ -- struct list_head tasks_node; -- struct task_struct *task; -- struct scx_sched_task_stats sts; --}; -- --void sched_ext_free(struct task_struct *p); -- --#else /* !CONFIG_SCHED_CLASS_EXT */ -- --static inline void sched_ext_free(struct task_struct *p) {} -- --#endif /* CONFIG_SCHED_CLASS_EXT */ --#endif /* _LINUX_SCHED_EXT_H */ -diff --git a/include/linux/sched/hmbird_proc_val.h b/include/linux/sched/hmbird_proc_val.h -new file mode 100755 -index 000000000000..e59708b16ea3 ---- /dev/null -+++ b/include/linux/sched/hmbird_proc_val.h -@@ -0,0 +1,22 @@ -+#ifndef __HMBORD_PROC_VAL_H__ -+#define __HMBORD_PROC_VAL_H__ -+ -+extern int scx_enable; -+extern int partial_enable; -+extern int cpuctrl_high_ratio; -+extern int cpuctrl_low_ratio; -+extern int slim_stats; -+extern int hmbirdcore_debug; -+extern int slim_for_app; -+extern int misfit_ds; -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+extern int cpu7_tl; -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int slim_gov_debug; -+extern int scx_gov_ctrl; -+extern int sched_ravg_window_frame_per_sec; -+ -+#endif -\ No newline at end of file -diff --git a/include/linux/sched/hmbird_version.h b/include/linux/sched/hmbird_version.h -new file mode 100755 -index 000000000000..b2843881c26f ---- /dev/null -+++ b/include/linux/sched/hmbird_version.h -@@ -0,0 +1,52 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#ifndef _OPLUS_HMBIRD_VERSION_H_ -+#define _OPLUS_HMBIRD_VERSION_H_ -+#include -+#include -+#include -+ -+enum hmbird_version { -+ HMBIRD_UNINIT, -+ HMBIRD_GKI_VERSION, -+ HMBIRD_OGKI_VERSION, -+ HMBIRD_UNKNOW_VERSION, -+}; -+ -+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT; -+ -+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type" -+ -+static inline enum hmbird_version get_hmbird_version_type(void) -+{ -+ struct device_node *np = NULL; -+ const char *hmbird_version_str = NULL; -+ if (HMBIRD_UNINIT != hmbird_version_type) -+ return hmbird_version_type; -+ np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH); -+ if (np) { -+ of_property_read_string(np, "type", &hmbird_version_str); -+ if (NULL != hmbird_version_str) { -+ if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) { -+ hmbird_version_type = HMBIRD_OGKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi"); -+ } else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) { -+ hmbird_version_type = HMBIRD_GKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi"); -+ } else { -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi"); -+ } -+ return hmbird_version_type; -+ } -+ } -+ -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION"); -+ return hmbird_version_type; -+} -+ -+#endif /*_OPLUS_HMBIRD_VERSION_H_ */ -diff --git a/include/linux/sched/sa_common.h b/include/linux/sched/sa_common.h -new file mode 100755 -index 000000000000..6f76d7cfd7bf ---- /dev/null -+++ b/include/linux/sched/sa_common.h -@@ -0,0 +1,797 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_COMMON_H_ -+#define _OPLUS_SA_COMMON_H_ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+#include -+#endif -+#include -+#include -+#include -+#include -+#include -+ -+#include "sa_oemdata.h" -+#include "sa_common_struct.h" -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) -+#define OPLUS_UX_EEVDF_COMPATIBLE 1 -+#endif -+ -+#define SA_DEBUG_ON 0 -+ -+#if (SA_DEBUG_ON >= 1) -+#define DEBUG_BUG_ON(x) BUG_ON(x) -+#define DEBUG_WARN_ON(x) WARN_ON(x) -+#else -+#define DEBUG_BUG_ON(x) -+#define DEBUG_WARN_ON(x) -+#endif -+ -+#define ux_err(fmt, ...) \ -+ pr_err("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_warn(fmt, ...) \ -+ pr_warn("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_debug(fmt, ...) \ -+ pr_info("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+ -+#define UX_MSG_LEN 64 -+#define UX_DEPTH_MAX 5 -+ -+/* define for debug */ -+#define DEBUG_SYSTRACE (1 << 0) -+#define DEBUG_FTRACE (1 << 1) -+#define DEBUG_KMSG (1 << 2) /* used for frameboost */ -+#define DEBUG_PIPELINE (1 << 3) -+#define DEBUG_DYNAMIC_HZ (1 << 4) -+#define DEBUG_DYNAMIC_PREEMPT (1 << 5) -+#define DEBUG_AMU_INSTRUCTION (1 << 6) -+#define DEBUG_VERBOSE (1 << 10) /* used for frameboost */ -+#define DEBUG_SET_DSQ_ID (1 << 11) /* used for hmbird */ -+ -+/* define for sched assist feature */ -+#define FEATURE_COMMON (1 << 0) -+#define FEATURE_SPREAD (1 << 1) -+ -+#define UX_EXEC_SLICE (4000000U) -+ -+/* define for sched assist thread type, keep same as the define in java file */ -+#define SA_OPT_CLEAR (0) -+#define SA_TYPE_LIGHT (1 << 0) -+#define SA_TYPE_HEAVY (1 << 1) -+#define SA_TYPE_ANIMATOR (1 << 2) -+/* SA_TYPE_LISTPICK for camera */ -+#define SA_TYPE_LISTPICK (1 << 3) -+#define SA_TYPE_MQ_VIP (1 << 4) -+#define SA_OPT_SET (1 << 7) -+#define SA_OPT_RESET (1 << 8) -+#define SA_OPT_SET_PRIORITY (1 << 9) -+ -+/* The following ux value only used in kernel */ -+#define SA_TYPE_SWIFT (1 << 14) -+/* clear ux type when dequeue */ -+#define SA_TYPE_ONCE (1 << 15) -+#define SA_TYPE_INHERIT (1 << 16) -+/* SA_TYPE_COMBINED isn't marked in ux_state, it only exists in return value of function */ -+#define SA_TYPE_COMBINED (1 << 17) -+#define SA_TYPE_URGENT_MASK (SA_TYPE_LIGHT|SA_TYPE_ANIMATOR|SA_TYPE_SWIFT) -+#define SCHED_ASSIST_UX_MASK (SA_TYPE_LIGHT|SA_TYPE_HEAVY|SA_TYPE_ANIMATOR|SA_TYPE_LISTPICK|SA_TYPE_SWIFT) -+ -+/* load balance operation is performed on the following ux types */ -+#define SCHED_ASSIST_LB_UX (SA_TYPE_SWIFT | SA_TYPE_ANIMATOR | SA_TYPE_LIGHT | SA_TYPE_HEAVY) -+#define POSSIBLE_UX_MASK (SA_TYPE_SWIFT | \ -+ SA_TYPE_LIGHT | \ -+ SA_TYPE_HEAVY | \ -+ SA_TYPE_ANIMATOR | \ -+ SA_TYPE_LISTPICK | \ -+ SA_TYPE_ONCE | \ -+ SA_TYPE_INHERIT) -+ -+#define SCHED_ASSIST_UX_PRIORITY_MASK (0xFF000000) -+#define SCHED_ASSIST_UX_PRIORITY_SHIFT 24 -+ -+#define UX_PRIORITY_TOP_APP 0x0A000000 -+#define UX_PRIORITY_AUDIO 0x0A000000 -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+#define UX_PRIORITY_PIPELINE_UI 0x06000000 -+#define UX_PRIORITY_PIPELINE 0x05000000 -+#endif -+ -+/* define for sched assist scene type, keep same as the define in java file */ -+#define SA_SCENE_OPT_CLEAR (0) -+#define SA_LAUNCH (1 << 0) -+#define SA_SLIDE (1 << 1) -+#define SA_CAMERA (1 << 2) -+#define SA_ANIM_START (1 << 3) /* we care about both launcher and top app */ -+#define SA_ANIM (1 << 4) /* we only care about launcher */ -+#define SA_INPUT (1 << 5) -+#define SA_LAUNCHER_SI (1 << 6) -+#define SA_SCENE_OPT_SET (1 << 7) -+ -+#define ROOT_UID 0 -+#define SYSTEM_UID 1000 -+#define FIRST_APPLICATION_UID 10000 -+#define LAST_APPLICATION_UID 19999 -+#define PER_USER_RANGE 100000 -+/* define for clear IM_FLAG -+if im_flag is 70, it should clear im_flag_audio (70 - 64 = 6) -+eg: gerrit patchset "30438485" -+ */ -+#define IM_FLAG_CLEAR 64 -+ -+extern pid_t save_audio_tgid; -+extern pid_t save_top_app_tgid; -+extern unsigned int top_app_type; -+extern int global_lowend_plat_opt; -+ -+#ifdef CONFIG_OPLUS_SCHED_HALT_MASK_PRT -+/* This must be the same as the definition of pause_type in walt_halt.c */ -+enum oplus_pause_type { -+ OPLUS_HALT, -+ OPLUS_PARTIAL_HALT, -+ -+ OPLUS_MAX_PAUSE_TYPE -+}; -+extern cpumask_t cur_cpus_halt_mask; -+extern cpumask_t cur_cpus_phalt_mask; -+DECLARE_PER_CPU(int[OPLUS_MAX_PAUSE_TYPE], oplus_cur_pause_client); -+extern void sa_corectl_systrace_c(void); -+#endif /* CONFIG_OPLUS_SCHED_HALT_MASK_PRT */ -+ -+/* define for boost threshold unit */ -+#define BOOST_THRESHOLD_UNIT (51) -+ -+ -+enum UX_STATE_TYPE { -+ UX_STATE_INVALID = 0, -+ UX_STATE_NONE, -+ UX_STATE_STATIC, -+ UX_STATE_INHERIT, -+ UX_STATE_COMBINED, -+ MAX_UX_STATE_TYPE, -+}; -+ -+enum INHERIT_UX_TYPE { -+ INHERIT_UX_BINDER = 0, -+ INHERIT_UX_RWSEM, -+ INHERIT_UX_MUTEX, -+ INHERIT_UX_FUTEX, -+ INHERIT_UX_PIFUTEX, -+ INHERIT_UX_MAX, -+}; -+ -+/* -+ * WANNING: -+ * new flag should be add before MAX_IM_FLAG_TYPE, never change -+ * the value of those existed flag type. -+ */ -+enum IM_FLAG_TYPE { -+ IM_FLAG_NONE = 0, -+ IM_FLAG_SURFACEFLINGER, -+ IM_FLAG_HWC, /* Discarded */ -+ IM_FLAG_RENDERENGINE, -+ IM_FLAG_WEBVIEW, -+ IM_FLAG_CAMERA_HAL, -+ IM_FLAG_AUDIO, -+ IM_FLAG_HWBINDER, -+ IM_FLAG_LAUNCHER, -+ IM_FLAG_LAUNCHER_NON_UX_RENDER, -+ IM_FLAG_SS_LOCK_OWNER, -+ IM_FLAG_FORBID_SET_CPU_AFFINITY = 11, /* forbid setting cpu affinity from app */ -+ IM_FLAG_SYSTEMSERVER_PID, -+ IM_FLAG_MIDASD, -+ IM_FLAG_AUDIO_CAMERA_HAL, /* audio mode disable camera hal ux */ -+ IM_FLAG_AFFINITY_THREAD, -+ IM_FLAG_TPD_SET_CPU_AFFINITY = 16, -+ IM_FLAG_COMPRESS_THREAD = 17, /* compress thread skips locking protect */ -+ IM_FLAG_RENDER_THREAD = 18, -+ MAX_IM_FLAG_TYPE, -+}; -+ -+#define MAX_IM_FLAG_PRIO MAX_IM_FLAG_TYPE -+ -+enum ots_state { -+ OTS_STATE_SET_AFFINITY, -+ OTS_STATE_DDL_ACTIVE, -+ OTS_STATE_DDL_ACTIVE_PREEMPTED, -+ OTS_STATE_MAX, -+}; -+ -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+DECLARE_PER_CPU(u64, retired_instrs); -+DECLARE_PER_CPU(u64, nvcsw); -+DECLARE_PER_CPU(u64, nivcsw); -+#endif -+ -+struct ux_sched_cluster { -+ struct cpumask cpus; -+ unsigned long capacity; -+}; -+ -+#define OPLUS_NR_CPUS (8) -+#define OPLUS_MAX_CLS (5) -+struct ux_sched_cputopo { -+ int cls_nr; -+ struct ux_sched_cluster sched_cls[OPLUS_NR_CPUS]; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ cpumask_t oplus_cpu_array[2*OPLUS_MAX_CLS][OPLUS_MAX_CLS]; -+#endif -+}; -+ -+struct oplus_rq { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_root_cached ux_list; -+ /* a tree to track minimum exec vruntime */ -+ struct rb_root_cached exec_timeline; -+ /* malloc this spinlock_t instead of built-in to shrank size of oplus_rq */ -+ spinlock_t *ux_list_lock; -+ int nr_running; -+ u64 min_vruntime; -+ u64 load_weight; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) -+ struct rb_root_cached ddl_root; -+ spinlock_t *ddl_lock; -+#endif -+ -+#ifdef CONFIG_LOCKING_PROTECT -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ struct list_head locking_thread_list; -+ spinlock_t *locking_list_lock; -+ int rq_locking_task; -+#else -+ struct sched_entity *last_entity; -+#endif -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ struct oplus_lb lb; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+ struct hrtimer *resched_timer; -+ int cpu; -+#endif -+}; -+ -+extern int global_debug_enabled; -+extern int global_sched_assist_enabled; -+extern int global_sched_assist_scene; -+extern int global_silver_perf_core; -+extern int global_sched_group_enabled; -+ -+struct rq; -+ -+#ifdef CONFIG_LOCKING_PROTECT -+struct sched_assist_locking_ops { -+ void (*check_preempt_tick)(struct task_struct *p, -+ unsigned long *ideal_runtime, bool *skip_preempt, -+ unsigned long delta_exec, struct cfs_rq *cfs_rq, -+ struct sched_entity *curr, unsigned int granularity); -+ void (*check_preempt_wakeup)(struct rq *rq, struct task_struct *p, bool *preempt, bool *nopreempt); -+ void (*state_systrace_c)(unsigned int cpu, struct task_struct *p); -+ void (*locking_tick_hit)(struct task_struct *prev, struct task_struct *next); -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ void (*replace_next_task_fair)(struct rq *rq, -+ struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*enqueue_entity)(struct rq *rq, struct task_struct *p); -+ void (*dequeue_entity)(struct rq *rq, struct task_struct *p); -+#else -+ void (*set_last_entity)(struct task_struct *prev, struct task_struct *next, struct rq *rq); -+ void (*pick_last_entity)(struct rq *rq, struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*clear_last_entity)(struct rq *rq, struct task_struct *p); -+#endif -+ void (*opt_ss_lock_contention)(struct task_struct *p, int old_im, int new_im); -+}; -+ -+extern struct sched_assist_locking_ops *locking_ops; -+ -+ -+#define LOCKING_CALL_OP(op, args...) \ -+do { \ -+ if (locking_ops && locking_ops->op) { \ -+ locking_ops->op(args); \ -+ } \ -+} while (0) -+ -+#define LOCKING_CALL_OP_RET(op, args...) \ -+({ \ -+ __typeof__(locking_ops->op(args)) __ret = 0; \ -+ if (locking_ops && locking_ops->op) \ -+ __ret = locking_ops->op(args); \ -+ __ret; \ -+}) -+ -+void register_sched_assist_locking_ops(struct sched_assist_locking_ops *ops); -+#endif -+ -+/* attention: before insert .ko, task's list->prev/next is init with 0 */ -+static inline bool oplus_list_empty(struct list_head *list) -+{ -+ return list_empty(list) || (list->prev == 0 && list->next == 0); -+} -+ -+static inline bool oplus_rbtree_empty(struct rb_root_cached *list) -+{ -+ return (rb_first_cached(list) == NULL); -+} -+ -+/** -+ * Check if there are ux tasks waiting to run on the specified cpu -+ */ -+static inline bool orq_has_ux_tasks(struct oplus_rq *orq) -+{ -+ return !oplus_rbtree_empty(&orq->ux_list); -+} -+ -+/* attention: before insert .ko, task's node->__rb_parent_color is init with 0 */ -+static inline bool oplus_rbnode_empty(struct rb_node *node) -+{ -+ return RB_EMPTY_NODE(node) || (node->__rb_parent_color == 0); -+} -+ -+static inline struct oplus_task_struct *ux_list_first_entry(struct rb_root_cached *list) -+{ -+ struct rb_node *leftmost = rb_first_cached(list); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, ux_entry); -+} -+ -+static inline struct oplus_task_struct *exec_timeline_first_entry(struct rb_root_cached *tree) -+{ -+ struct rb_node *leftmost = rb_first_cached(tree); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, exec_time_node); -+} -+ -+typedef bool (*migrate_task_callback_t)(struct task_struct *tsk, int src_cpu, int dst_cpu); -+extern migrate_task_callback_t fbg_migrate_task_callback; -+typedef void (*android_rvh_schedule_handler_t)(struct task_struct *prev, -+ struct task_struct *next, struct rq *rq); -+extern android_rvh_schedule_handler_t fbg_android_rvh_schedule_callback; -+ -+extern struct kmem_cache *oplus_task_struct_cachep; -+ -+#define ots_to_ts(ots) (ots->task) -+#define OTS_IDX 0 -+#define ORQ_IDX 0 -+ -+static inline bool test_task_is_fair(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid CFS priority is MAX_RT_PRIO..MAX_PRIO-1 */ -+ if ((task->prio >= MAX_RT_PRIO) && (task->prio <= MAX_PRIO-1)) -+ return true; -+ return false; -+} -+ -+static inline bool test_task_is_rt(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid RT priority is 0..MAX_RT_PRIO-1 */ -+ if (rt_prio(task->prio)) -+ return true; -+ -+ return false; -+} -+ -+static inline struct oplus_task_struct *get_oplus_task_struct(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = NULL; -+ -+ /* not Skip idle thread */ -+ if (!t) -+ return NULL; -+ -+ ots = (struct oplus_task_struct *) READ_ONCE(t->android_oem_data1[OTS_IDX]); -+ if (IS_ERR_OR_NULL(ots)) -+ return NULL; -+ -+ return ots; -+} -+ -+static inline unsigned long oplus_get_im_flag(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return IM_FLAG_NONE; -+ -+ return ots->im_flag; -+} -+ -+static inline bool is_optimized_audio_thread(struct task_struct *t) -+{ -+ unsigned long im_flag; -+ -+ im_flag = oplus_get_im_flag(t); -+ if (test_bit(IM_FLAG_AUDIO, &im_flag)) -+ return true; -+ -+ return false; -+} -+ -+static inline void oplus_set_im_flag(struct task_struct *t, int im_flag) -+{ -+ struct oplus_task_struct *ots = NULL; -+ ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ set_bit(im_flag, &ots->im_flag); -+} -+ -+static inline int get_ots_ux_state(struct oplus_task_struct *ots) -+{ -+ int ux_state; -+ int sub_ux_state; -+ int ux_prio; -+ int ret; -+ -+ ux_prio = ots->ux_state & SCHED_ASSIST_UX_PRIORITY_MASK; -+ ux_state = ots->ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ sub_ux_state = ots->sub_ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ -+ if (sub_ux_state) { -+ ret = ux_state | (sub_ux_state & ~SA_TYPE_INHERIT) | SA_TYPE_COMBINED; -+ } else { -+ ret = ux_state; -+ } -+ -+ if (ret & SCHED_ASSIST_UX_MASK) { -+ return ux_prio | ret; -+ } -+ -+ return 0; -+} -+ -+bool is_multiple_ux(struct oplus_task_struct *ots); -+ -+static inline int oplus_get_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return get_ots_ux_state(ots); -+} -+ -+static inline int oplus_get_static_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return 0; -+ } -+ return ots->ux_state; -+} -+ -+static inline int oplus_get_sub_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->sub_ux_state; -+} -+ -+static inline int oplus_get_inherited_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return ots->ux_state; -+ } -+ return ots->sub_ux_state; -+} -+ -+void oplus_set_ux_state_lock(struct task_struct *t, int ux_state, int inherit_type, bool need_lock_rq); -+ -+static inline s64 oplus_get_inherit_ux(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return atomic64_read(&ots->inherit_ux); -+} -+ -+static inline void oplus_set_inherit_ux(struct task_struct *t, s64 inherit_ux) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ atomic64_set(&ots->inherit_ux, inherit_ux); -+} -+ -+static inline int oplus_get_ux_depth(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->ux_depth; -+} -+ -+static inline void oplus_set_ux_depth(struct task_struct *t, int ux_depth) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->ux_depth = ux_depth; -+} -+ -+static inline u64 oplus_get_enqueue_time(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->enqueue_time; -+} -+ -+static inline void oplus_set_enqueue_time(struct task_struct *t, u64 enqueue_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->enqueue_time = enqueue_time; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* -+ * Record the number of task context switches -+ * and scheduling delays when enqueueing a task. -+ */ -+ ots->snap_pcount = t->sched_info.pcount; -+ ots->snap_run_delay = t->sched_info.run_delay; -+#endif -+} -+ -+static inline void oplus_set_inherit_ux_start(struct task_struct *t, u64 start_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->inherit_ux_start = t->se.sum_exec_runtime; -+} -+ -+static inline void init_task_ux_info(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ RB_CLEAR_NODE(&ots->ux_entry); -+ RB_CLEAR_NODE(&ots->exec_time_node); -+ ots->ux_state = 0; -+ ots->sub_ux_state = 0; -+ atomic64_set(&ots->inherit_ux, 0); -+ ots->ux_depth = 0; -+ ots->enqueue_time = 0; -+ ots->inherit_ux_start = 0; -+ ots->ux_priority = -1; -+ ots->ux_nice = -1; -+ ots->vruntime = 0; -+ ots->preset_vruntime = 0; -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG) -+ ots->abnormal_flag = 0; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_SCHED_SPREAD -+ ots->lb_state = 0; -+ ots->ld_flag = 0; -+#endif -+ ots->exec_calc_runtime = 0; -+ ots->is_update_runtime = 0; -+ ots->target_process = -1; -+ ots->wake_tid = 0; -+ ots->running_start_time = 0; -+ ots->update_running_start_time = false; -+ ots->last_wake_ts = 0; -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ memset(&ots->lkinfo, 0, sizeof(struct locking_info)); -+ INIT_LIST_HEAD(&ots->lkinfo.node); -+/*#endif*/ -+ ots->block_start_time = 0; -+#ifdef CONFIG_LOCKING_PROTECT -+ INIT_LIST_HEAD(&ots->locking_entry); -+ ots->locking_start_time = 0; -+ ots->locking_depth = 0; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ ots->snap_pcount = 0; -+ ots->snap_run_delay = 0; -+ plist_node_init(&ots->rtb, MAX_IM_FLAG_PRIO); -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+ atomic_set(&ots->pipeline_cpu, -1); -+#endif -+ -+#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO) -+ ots->amu_cycle = 0; -+ ots->amu_instruct = 0; -+#endif -+}; -+ -+static inline bool test_ux_type(struct task_struct *task, int ux_type) -+{ -+ return oplus_get_ux_state(task) & ux_type; -+} -+ -+static inline bool is_heavy_ux_task(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return false; -+ -+ return get_ots_ux_state(ots) & SA_TYPE_HEAVY; -+} -+ -+static inline bool sched_assist_scene(unsigned int scene) -+{ -+ if (unlikely(!global_sched_assist_enabled)) -+ return false; -+ -+ return global_sched_assist_scene & scene; -+} -+ -+static inline unsigned long oplus_task_util(struct task_struct *p) -+{ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+ struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; -+ -+ return wts->demand_scaled; -+#else -+ return READ_ONCE(p->se.avg.util_avg); -+#endif -+} -+ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+static inline u32 task_wts_sum(struct task_struct *tsk) -+{ -+ struct walt_task_struct *wts = (struct walt_task_struct *) tsk->android_vendor_data1; -+ -+ return wts->sum; -+} -+#endif -+ -+bool is_min_cluster(int cpu); -+bool is_max_cluster(int cpu); -+bool is_mid_cluster(int cpu); -+bool im_mali(const char *comm); -+bool is_top(struct task_struct *p); -+bool task_is_runnable(struct task_struct *task); -+struct oplus_rq *get_oplus_rq(struct rq *rq); -+ -+typedef unsigned long (*kallsyms_lookup_name_t)(const char *name); -+extern kallsyms_lookup_name_t _kallsyms_lookup_name; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+int ux_mask_to_prio(int ux_mask); -+int ux_prio_to_mask(int prio); -+#endif -+ -+noinline int tracing_mark_write(const char *buf); -+void hwbinder_systrace_c(unsigned int cpu, int flag); -+void sched_assist_init_oplus_rq(void); -+void queue_ux_thread(struct rq *rq, struct task_struct *p, int enqueue); -+ -+void inherit_ux_inc(struct task_struct *task, int type); -+void inherit_ux_sub(struct task_struct *task, int type, int value); -+void set_inherit_ux(struct task_struct *task, int type, int depth, int inherit_val); -+void reset_inherit_ux(struct task_struct *inherit_task, struct task_struct *ux_task, int reset_type); -+void unset_inherit_ux(struct task_struct *task, int type); -+void unset_inherit_ux_value(struct task_struct *task, int type, int value); -+void inc_inherit_ux_refs(struct task_struct *task, int type); -+void clear_all_inherit_type(struct task_struct *p); -+int get_max_inherit_gran(struct task_struct *p); -+ -+bool is_heavy_load_top_task(struct task_struct *p); -+bool test_task_is_fair(struct task_struct *task); -+bool test_task_is_rt(struct task_struct *task); -+ -+bool test_task_ux(struct task_struct *task); -+bool test_task_ux_depth(int ux_depth); -+bool test_inherit_ux(struct task_struct *task, int type); -+bool test_set_inherit_ux(struct task_struct *task); -+int get_ux_state_type(struct task_struct *task); -+void sched_assist_target_comm(struct task_struct *task, const char *comm); -+unsigned int ux_task_exec_limit(struct task_struct *p); -+ -+void update_ux_sched_cputopo(void); -+bool is_task_util_over(struct task_struct *tsk, int threshold); -+bool oplus_task_misfit(struct task_struct *tsk, int cpu); -+ssize_t oplus_show_cpus(const struct cpumask *mask, char *buf); -+void adjust_rt_lowest_mask(struct task_struct *p, struct cpumask *local_cpu_mask, int ret, bool force_adjust); -+bool sa_skip_rt_sync(struct rq *rq, struct task_struct *p, bool *sync); -+void ux_state_systrace_c(unsigned int cpu, struct task_struct *p); -+bool sa_rt_skip_ux_cpu(int cpu); -+int is_vip_mvp(struct task_struct *p); -+ -+/* s64 account_ux_runtime(struct rq *rq, struct task_struct *curr); */ -+void opt_ss_lock_contention(struct task_struct *p, unsigned long old_im, int new_im); -+ -+/* register vender hook in kernel/sched/topology.c */ -+void android_vh_build_sched_domains_handler(void *unused, bool has_asym); -+ -+/* register vender hook in kernel/sched/rt.c */ -+void android_rvh_select_task_rq_rt_handler(void *unused, struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags, int *new_cpu); -+void android_rvh_find_lowest_rq_handler(void *unused, struct task_struct *p, struct cpumask *local_cpu_mask, int ret, int *best_cpu); -+ -+/* register vender hook in kernel/sched/core.c */ -+void android_rvh_sched_fork_handler(void *unused, struct task_struct *p); -+void android_rvh_after_enqueue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_dequeue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_schedule_handler(void *unused, struct task_struct *prev, struct task_struct *next, struct rq *rq); -+void android_vh_scheduler_tick_handler(void *unused, struct rq *rq); -+ -+void android_vh_account_process_tick_gran_handler(void *unused, struct task_struct *p, struct rq *rq, int user_tick, int *ticks); -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+void sa_sched_switch_handler(void *unused, bool preempt, struct task_struct *prev, struct task_struct *next, unsigned int prev_state); -+#endif -+ -+void set_im_flag_with_bit(int im_flag, struct task_struct *task); -+ -+/* register vendor hook in kernel/cgroup/cgroup-v1.c */ -+void android_vh_cgroup_set_task_handler(void *unused, int ret, struct task_struct *task); -+/* register vendor hook in kernel/signal.c */ -+void android_vh_exit_signal_handler(void *unused, struct task_struct *p); -+void android_rvh_set_cpus_allowed_by_task_handler(void *unused, const struct cpumask *cpu_valid_mask, -+ const struct cpumask *new_mask, struct task_struct *task, unsigned int *dest_cpu); -+void android_rvh_setscheduler_handler(void *unused, struct task_struct *p); -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_BAN_APP_SET_AFFINITY) -+void android_vh_sched_setaffinity_early_handler(void *unused, struct task_struct *task, const struct cpumask *new_mask, bool *skip); -+#endif -+ -+#ifdef CONFIG_OPLUS_SCHED_GROUP_OPT -+void android_vh_reweight_entity_handler(void *unused, struct sched_entity *se); -+#endif -+ -+#ifdef CONFIG_BLOCKIO_UX_OPT -+int sa_blockio_init(void); -+void sa_blockio_exit(void); -+#endif -+extern struct notifier_block process_exit_notifier_block; -+#endif /* _OPLUS_SA_COMMON_H_ */ -diff --git a/include/linux/sched/sa_common_struct.h b/include/linux/sched/sa_common_struct.h -new file mode 100755 -index 000000000000..876feff48b36 ---- /dev/null -+++ b/include/linux/sched/sa_common_struct.h -@@ -0,0 +1,277 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+/* -+this file is splited from the sa_common.h to adapt the OKI, -+IS_ENABLED is not allowed in here, because the macro will not work in OKI. -+*/ -+#ifndef _OPLUS_SA_COMMON_STRUCT_H_ -+#define _OPLUS_SA_COMMON_STRUCT_H_ -+ -+#define MAX_CLUSTER (4) -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+/* hot-thread */ -+struct task_record { -+#define RECOED_WINSIZE (1 << 8) -+#define RECOED_WINIDX_MASK (RECOED_WINSIZE - 1) -+ u8 winidx; -+ u8 count; -+}; -+ -+#define MAX_TASK_COMM_LEN 256 -+struct uid_struct { -+ uid_t uid; -+ u64 uid_total_cycle; -+ u64 uid_total_inst; -+ spinlock_t lock; -+ char leader_comm[TASK_COMM_LEN]; -+ char cmdline[MAX_TASK_COMM_LEN]; -+}; -+ -+struct amu_uid_entry { -+ uid_t uid; -+ struct uid_struct *uid_struct; -+ struct hlist_node node; -+}; -+ -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+struct locking_info { -+ u64 waittime_stamp; -+ u64 holdtime_stamp; -+ /* Used in torture acquire latency statistic.*/ -+ u64 acquire_stamp; -+ /* -+ * mutex or rwsem optimistic spin start time. Because a task -+ * can't spin both on mutex and rwsem at one time, use one common -+ * threshold time is OK. -+ */ -+ u64 opt_spin_start_time; -+ struct task_struct *holder; -+ u32 waittype; -+ bool ux_contrib; -+ /* -+ * Whether task is ux when it's going to be added to mutex or -+ * rwsem waiter list. It helps us check whether there is ux -+ * task on mutex or rwsem waiter list. Also, a task can't be -+ * added to both mutex and rwsem at one time, so use one common -+ * field is OK. -+ */ -+ bool is_block_ux; -+ u32 kill_flag; -+ /* for cfs enqueue smoothly.*/ -+ struct list_head node; -+ struct task_struct *owner; -+ struct list_head lock_head; -+ u64 clear_seq; -+ atomic_t lock_depth; -+}; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+#define RAVG_HIST_SIZE 5 -+#define SCX_SLICE_DFL (1 * NSEC_PER_MSEC) -+#define SCX_SLICE_INF U64_MAX -+#define DEFAULT_CGROUP_DL_IDX (8) -+#define EXT_FLAG_RT_CHANGED (1 << 0) -+#define EXT_FLAG_CFS_CHANGED (1 << 1) -+struct scx_task_stats { -+ u64 mark_start; -+ u64 window_start; -+ u32 sum; -+ u32 sum_history[RAVG_HIST_SIZE]; -+ int cidx; -+ u32 demand; -+ u16 demand_scaled; -+ void *sdsq; -+}; -+/* -+ * The following is embedded in task_struct and contains all fields necessary -+ * for a task to be scheduled by SCX. -+ */ -+struct scx_entity { -+ struct scx_dispatch_q *dsq; -+ struct { -+ struct list_head fifo; /* dispatch order */ -+ struct rb_node priq; /* p->scx.dsq_vtime order */ -+ } dsq_node; -+ u32 flags; /* protected by rq lock */ -+ u32 dsq_flags; /* protected by dsq lock */ -+ s32 sticky_cpu; -+ unsigned long runnable_at; -+ u64 slice; -+ u64 dsq_vtime; -+ int gdsq_idx; -+ int ext_flags; -+ int prio_backup; -+ unsigned long sched_prop; -+ struct scx_task_stats sts; -+}; -+/*#endif*/ -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ -+#define MAX_CPU_FREQ_STATE 32 -+#define MAX_CPU_CNT 8 -+ -+#define STATE_IN_POOL 0 -+#define STATE_ACTIVE 1 -+ -+struct powermodel_freq_task_state { -+ u8 powermodel_enqueued:1; -+ u8 powermodel_index:7; -+}; -+ -+struct powermodel_cpu_task_state { -+ struct oplus_task_struct *ots; -+ struct powermodel_freq_task_state powermodel_freq_task_states[MAX_CPU_FREQ_STATE]; -+ u64 powermodel_last_seq; -+ u64 last_seq; -+ struct list_head node; -+ int state; -+ int pool_id; -+}; -+ -+#endif -+ -+ -+/* Please add your own members of task_struct here :) */ -+struct oplus_task_struct { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_node ux_entry; -+ struct rb_node exec_time_node; -+ struct task_struct *task; -+ atomic64_t inherit_ux; -+ u64 enqueue_time; -+ u64 inherit_ux_start; -+ /* u64 sum_exec_baseline; */ -+ u64 total_exec; -+ u64 vruntime; -+ u64 preset_vruntime; -+ /* contains ux state -+ 1. if static and inherited ux both exist, static ux stores in ux_state, inherited ux in sub_ux_state. -+ 2. if only static ux exists, static ux stores in ux_state. -+ 2. if only inherited ux exists, inherited ux stores in ux_state */ -+ int ux_state; -+ int sub_ux_state; -+ u8 ux_depth; -+ s8 ux_priority; -+ s8 ux_nice; -+ pid_t affinity_pid; -+ pid_t affinity_tgid; -+ unsigned long state; -+ unsigned long im_flag; -+ atomic_t is_vip_mvp; -+ -+/* #if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) */ -+ u64 ddl; -+ u64 ddl_active_ts; -+ u64 runnable_ts; -+ struct rb_node ddl_node; -+/* #endif */ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG)*/ -+ int abnormal_flag; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_SCHED_SPREAD */ -+ int lb_state; -+ int ld_flag:1; -+ /* CONFIG_OPLUS_FEATURE_TASK_LOAD */ -+ int is_update_runtime:1; -+ int target_process; -+ u64 wake_tid; -+ u64 running_start_time; -+ bool update_running_start_time; -+ u64 exec_calc_runtime; -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct task_record record[MAX_CLUSTER]; /* 2*u64 */ -+ u64 block_start_time; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_FRAME_BOOST */ -+ struct list_head fbg_list; -+ raw_spinlock_t fbg_list_entry_lock; -+ bool fbg_running; /* task belongs to a group, and in running */ -+ u16 fbg_state; -+ s8 preferred_cluster_id; -+ s8 fbg_depth; -+ u64 last_wake_ts; -+ int fbg_cur_group; -+/*#ifdef CONFIG_LOCKING_PROTECT*/ -+ unsigned long locking_start_time; -+ unsigned long last_jiffies; -+ struct list_head locking_entry; -+ int locking_depth; -+ int lk_tick_hit; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ struct locking_info lkinfo; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+ struct scx_entity scx; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_FDLEAK_CHECK)*/ -+ u8 fdleak_flag; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+ /* for loadbalance */ -+ struct plist_node rtb; /* rt boost task */ -+ -+ /* -+ * The following variables are used to calculate the time -+ * a task spends in the running/runnable state. -+ */ -+ u64 snap_run_delay; -+ unsigned long snap_pcount; -+/*#endif*/ -+#if IS_ENABLED(CONFIG_OPLUS_SCHED_TUNE) -+ int stune_idx; -+#endif -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE)*/ -+ atomic_t pipeline_cpu; -+/*#endif*/ -+ -+ /* for oplus secure guard */ -+ int sg_flag; -+ int sg_scno; -+ uid_t sg_uid; -+ uid_t sg_euid; -+ gid_t sg_gid; -+ gid_t sg_egid; -+/*#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct uid_struct *uid_struct; -+ u64 amu_instruct; -+ u64 amu_cycle; -+/*#endif*/ -+ /* for binder ux */ -+ int binder_async_ux_enable; -+ bool binder_async_ux_sts; -+ int binder_thread_mode; -+ struct binder_node *binder_thread_node; -+ -+/* for powermodel */ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ struct powermodel_cpu_task_state *powermodel_cpu_task_states[MAX_CPU_CNT]; -+ u64 exec_runtime; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_CFBT) -+ int cfbt_cur_group; -+ bool cfbt_running; -+#endif /* CONFIG_OPLUS_FEATURE_SCHED_CFBT */ -+} ____cacheline_aligned; -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+#define INVALID_PID (-1) -+struct oplus_lb { -+ /* used for active_balance to record the running task. */ -+ pid_t pid; -+}; -+/*#endif*/ -+ -+#endif /* _OPLUS_SA_COMMON_STRUCT_H_ */ -+ -diff --git a/include/linux/sched/sa_oemdata.h b/include/linux/sched/sa_oemdata.h -new file mode 100755 -index 000000000000..ebbbc754e391 ---- /dev/null -+++ b/include/linux/sched/sa_oemdata.h -@@ -0,0 +1,13 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_OEMDATA_H_ -+#define _OPLUS_SA_OEMDATA_H_ -+ -+int sa_oemdata_init(void); -+void sa_oemdata_deinit(void); -+ -+#endif /* _OPLUS_SA_OEMDATA_H_ */ -diff --git a/include/linux/sched/sched_ext.h b/include/linux/sched/sched_ext.h -new file mode 100755 -index 000000000000..9f1afdb8a04b ---- /dev/null -+++ b/include/linux/sched/sched_ext.h -@@ -0,0 +1,57 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef _OPLUS_SCHED_EXT_H -+#define _OPLUS_SCHED_EXT_H -+#include "sa_common.h" -+ -+#define SCHED_PROP_TOP_THREAD_SHIFT (8) -+#define SCHED_PROP_TOP_THREAD_MASK (0xf << SCHED_PROP_TOP_THREAD_SHIFT) -+#define SCHED_PROP_DEADLINE_MASK (0xFF) /* deadline for ext sched class */ -+#define SCHED_PROP_DEADLINE_LEVEL1 (1) /* 1ms for user-aware audio tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL2 (2) /* 2ms for user-aware touch tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL3 (3) /* 4ms for user aware dispaly tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL4 (4) /* 6ms */ -+#define SCHED_PROP_DEADLINE_LEVEL5 (5) /* 8ms */ -+#define SCHED_PROP_DEADLINE_LEVEL6 (6) /* 16ms */ -+#define SCHED_PROP_DEADLINE_LEVEL7 (7) /* 32ms */ -+#define SCHED_PROP_DEADLINE_LEVEL8 (8) /* 64ms */ -+#define SCHED_PROP_DEADLINE_LEVEL9 (9) /* 128ms */ -+ -+static inline int sched_prop_get_top_thread_id(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ return -EPERM; -+ } -+ -+ return ((ots->scx.sched_prop & SCHED_PROP_TOP_THREAD_MASK) >> SCHED_PROP_TOP_THREAD_SHIFT); -+} -+ -+static inline int sched_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_set_sched_prop failed! fn=%s\n", __func__); -+ return -EPERM; -+ } -+ -+ ots->scx.sched_prop = sp; -+ return 0; -+} -+ -+static inline unsigned long sched_get_sched_prop(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_get_sched_prop failed! fn=%s\n", __func__); -+ return (unsigned long)-1; -+ } -+ return ots->scx.sched_prop; -+} -+ -+#endif /*_OPLUS_SCHED_EXT_H */ -diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h -index 03d35e3eda3c..6a5f0c0d74bd 100644 ---- a/include/linux/sched/task.h -+++ b/include/linux/sched/task.h -@@ -61,8 +61,10 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); - extern void init_idle(struct task_struct *idle, int cpu); - - extern int sched_fork(unsigned long clone_flags, struct task_struct *p); --extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); --extern void sched_cancel_fork(struct task_struct *p); -+extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); -+#ifdef CONFIG_HMBIRD_SCHED -+extern void hmbird_cancel_fork(struct task_struct *p); -+#endif - extern void sched_post_fork(struct task_struct *p); - extern void sched_dead(struct task_struct *p); - -diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h -index 359a14cc76a4..969716ab4320 100644 ---- a/include/uapi/linux/sched.h -+++ b/include/uapi/linux/sched.h -@@ -118,7 +118,7 @@ struct clone_args { - /* SCHED_ISO: reserved but not implemented yet */ - #define SCHED_IDLE 5 - #define SCHED_DEADLINE 6 --#define SCHED_EXT 7 -+#define SCHED_HMBIRD 7 - - /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ - #define SCHED_RESET_ON_FORK 0x40000000 -diff --git a/init/Kconfig b/init/Kconfig -index 0ead65803ecf..fbbc4a970b00 100644 ---- a/init/Kconfig -+++ b/init/Kconfig -@@ -1030,10 +1030,6 @@ config RT_GROUP_SCHED - realtime bandwidth for them. - See Documentation/scheduler/sched-rt-group.rst for more information. - --config EXT_GROUP_SCHED -- bool -- depends on SCHED_CLASS_EXT && CGROUP_SCHED -- default y - endif #CGROUP_SCHED - - config SCHED_MM_CID -diff --git a/init/init_task.c b/init/init_task.c -index 69a046388995..31ceb0e469f7 100644 ---- a/init/init_task.c -+++ b/init/init_task.c -@@ -6,7 +6,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -102,9 +101,6 @@ struct task_struct init_task - #endif - #ifdef CONFIG_CGROUP_SCHED - .sched_task_group = &root_task_group, --#endif --#ifdef CONFIG_SLIM_SCHED -- .scx = NULL, - #endif - .ptraced = LIST_HEAD_INIT(init_task.ptraced), - .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), -diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt -index cf22ef6107b8..b131d5662b48 100644 ---- a/kernel/Kconfig.preempt -+++ b/kernel/Kconfig.preempt -@@ -133,12 +133,8 @@ config SCHED_CORE - which is the likely usage by Linux distributions, there should - be no measurable impact on performance. - --config SLIM_SCHED -- bool "slim sched" -- default n --config SCHED_CLASS_EXT -- bool "Extensible Scheduling Class" -- depends on BPF_SYSCALL && BPF_JIT -+config HMBIRD_SCHED -+ bool "hmbird Scheduling Class(base on ext scheduler)" - help - This option enables a new scheduler class sched_ext (SCX), which - allows scheduling policies to be implemented as BPF programs to -@@ -159,3 +155,10 @@ config SCHED_CLASS_EXT - similar to struct sched_class. - - See Documentation/scheduler/sched-ext.rst for more details. -+ -+config DYNAMIC_WALT_SUPPORT -+ bool "Dynamic WALT Support for Extensible Scheduling Class" -+ depends on HMBIRD_SCHED -+ default n -+ help -+ Enable dynamic WALT support for Extensible Scheduling Class. -diff --git a/kernel/bpf/bpf_struct_ops_types.h b/kernel/bpf/bpf_struct_ops_types.h -index 3618769d853d..5678a9ddf817 100644 ---- a/kernel/bpf/bpf_struct_ops_types.h -+++ b/kernel/bpf/bpf_struct_ops_types.h -@@ -9,8 +9,4 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) - #include - BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) - #endif --#ifdef CONFIG_SCHED_CLASS_EXT --#include --BPF_STRUCT_OPS_TYPE(sched_ext_ops) --#endif - #endif -diff --git a/kernel/fork.c b/kernel/fork.c -index d0a46a152428..7a91505b9378 100644 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -23,7 +23,9 @@ - #include - #include - #include --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - #include - #include - #include -@@ -999,7 +1001,9 @@ void __put_task_struct(struct task_struct *tsk) - WARN_ON(!tsk->exit_state); - WARN_ON(refcount_read(&tsk->usage)); - WARN_ON(tsk == current); -- sched_ext_free(tsk); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_free(tsk); -+#endif - io_uring_free(tsk); - cgroup_free(tsk); - task_numa_free(tsk, true); -@@ -2516,7 +2520,11 @@ __latent_entropy struct task_struct *copy_process( - - retval = perf_event_init_task(p, clone_flags); - if (retval) -- goto bad_fork_sched_cancel_fork; -+#ifdef CONFIG_HMBIRD_SCHED -+ goto bad_fork_hmbird_cancel_fork; -+#else -+ goto bad_fork_cleanup_policy; -+#endif - retval = audit_alloc(p); - if (retval) - goto bad_fork_cleanup_perf; -@@ -2648,9 +2656,7 @@ __latent_entropy struct task_struct *copy_process( - * cgroup specific, it unconditionally needs to place the task on a - * runqueue. - */ -- retval = sched_cgroup_fork(p, args); -- if (retval) -- goto bad_fork_cancel_cgroup; -+ sched_cgroup_fork(p, args); - - /* - * From this point on we must avoid any synchronous user-space -@@ -2696,13 +2702,13 @@ __latent_entropy struct task_struct *copy_process( - /* Don't start children in a dying pid namespace */ - if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { - retval = -ENOMEM; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* Let kill terminate clone/fork in the middle */ - if (fatal_signal_pending(current)) { - retval = -EINTR; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* No more failure paths after this point. */ -@@ -2779,11 +2785,10 @@ __latent_entropy struct task_struct *copy_process( - - return p; - --bad_fork_core_free: -+bad_fork_cancel_cgroup: - sched_core_free(p); - spin_unlock(¤t->sighand->siglock); - write_unlock_irq(&tasklist_lock); --bad_fork_cancel_cgroup: - cgroup_cancel_fork(p, args); - bad_fork_put_pidfd: - if (clone_flags & CLONE_PIDFD) { -@@ -2822,8 +2827,10 @@ __latent_entropy struct task_struct *copy_process( - audit_free(p); - bad_fork_cleanup_perf: - perf_event_free_task(p); --bad_fork_sched_cancel_fork: -- sched_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+bad_fork_hmbird_cancel_fork: -+ hmbird_cancel_fork(p); -+#endif - bad_fork_cleanup_policy: - lockdep_free_task(p); - #ifdef CONFIG_NUMA -diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c -index 005025f55bea..c091539a0f60 100644 ---- a/kernel/sched/build_policy.c -+++ b/kernel/sched/build_policy.c -@@ -28,8 +28,10 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED - #include - #include -+#endif - - #include - -@@ -54,6 +56,10 @@ - #include "cputime.c" - #include "deadline.c" - --#ifdef CONFIG_SCHED_CLASS_EXT --# include "ext.c" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/hmbird_util_track.c" -+#include "hmbird/hmbird_sched_proc.c" -+#include "hmbird/hmbird_shadow_tick.c" -+#include "hmbird/hmbird.c" -+#include "hmbird/hmbird_misc.c" - #endif -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index d486b2dd8240..692c9aa0ffa6 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -96,6 +96,12 @@ - #include "../../io_uring/io-wq.h" - #include "../smpboot.h" - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/slim.h" -+#include "hmbird/hmbird_shadow_tick.h" -+#include "hmbird.h" -+#endif -+ - #include - #include - #include -@@ -170,7 +176,6 @@ __read_mostly int scheduler_running; - - DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); - -- - /* kernel prio, less is more */ - static inline int __task_prio(const struct task_struct *p) - { -@@ -183,8 +188,8 @@ static inline int __task_prio(const struct task_struct *p) - if (p->sched_class == &idle_sched_class) - return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - --#ifdef CONFIG_SCHED_CLASS_EXT -- if (p->sched_class == &ext_sched_class) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (p->sched_class == &hmbird_sched_class) - return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ - #endif - -@@ -217,9 +222,9 @@ static inline bool prio_less(const struct task_struct *a, - if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ - return cfs_prio_less(a, b, in_fi); - --#ifdef CONFIG_SCHED_CLASS_EXT -+#ifdef CONFIG_HMBIRD_SCHED - if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ -- return scx_prio_less(a, b, in_fi); -+ return hmbird_prio_less(a, b, in_fi); - #endif - - return false; -@@ -1279,17 +1284,26 @@ bool sched_can_stop_tick(struct rq *rq) - if (fifo_nr_running) - return true; - -+#ifdef CONFIG_HMBIRD_SCHED - /* -- * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks -+ * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks - * left. For CFS, if there's more than one we need the tick for -- * involuntary preemption. For SCX, ask. -+ * involuntary preemption. For HMBIRD, ask. - */ -- if (!scx_switched_all() && rq->nr_running > 1) -+ if (!hmbird_enabled() && rq->nr_running > 1) - return false; - -- if (scx_enabled() && !scx_can_stop_tick(rq)) -+ if (hmbird_enabled() && !hmbird_can_stop_tick(rq)) - return false; -- -+#else -+ /* -+ * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; -+ * if there's more than one we need the tick for involuntary -+ * preemption. -+ */ -+ if (rq->nr_running > 1) -+ return false; -+#endif - /* - * If there is one task and it has CFS runtime bandwidth constraints - * and it's on the cpu now we don't want to stop the tick. -@@ -2222,10 +2236,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) - } - EXPORT_SYMBOL_GPL(deactivate_task); - --struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) -+#ifdef CONFIG_HMBIRD_SCHED -+struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - { -- struct sched_change_guard cg = { -+ struct hmbird_sched_change_guard cg = { - .rq = rq, - .p = p, - .queued = task_on_rq_queued(p), -@@ -2247,7 +2262,7 @@ sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - return cg; - } - --void sched_change_guard_fini(struct sched_change_guard *cg, int flags) -+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags) - { - if (cg->queued) - enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); -@@ -2255,6 +2270,7 @@ void sched_change_guard_fini(struct sched_change_guard *cg, int flags) - set_next_task(cg->rq, cg->p); - cg->done = true; - } -+#endif - - static inline int __normal_prio(int policy, int rt_prio, int nice) - { -@@ -2320,9 +2336,9 @@ inline int task_curr(const struct task_struct *p) - * this means any call to check_class_changed() must be followed by a call to - * balance_callback(). - */ --void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio) -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) - { - if (prev_class != p->sched_class) { - if (prev_class->switched_from) -@@ -2885,6 +2901,7 @@ static void - __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - { - struct rq *rq = task_rq(p); -+ bool queued, running; - - /* - * This here violates the locking rules for affinity, since we're only -@@ -2903,9 +2920,26 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - else - lockdep_assert_held(&p->pi_lock); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->sched_class->set_cpus_allowed(p, ctx); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ -+ if (queued) { -+ /* -+ * Because __kthread_bind() calls this on blocked tasks without -+ * holding rq->lock. -+ */ -+ lockdep_assert_rq_held(rq); -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); - } -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->sched_class->set_cpus_allowed(p, ctx); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - } - - /* -@@ -3772,7 +3806,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class)) -+#else - if (unlikely(!is_cpu_allowed(p, cpu))) -+#endif - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -@@ -4891,7 +4929,9 @@ late_initcall(sched_core_sysctl_init); - */ - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { -+#ifdef CONFIG_HMBIRD_SCHED - int ret; -+#endif - - trace_android_rvh_sched_fork(p); - -@@ -4932,21 +4972,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_reset_on_fork = 0; - } - -- scx_pre_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ ret = hmbird_pre_fork(p); -+ if (ret) -+ goto out_cancel; - - if (dl_prio(p->prio)) { - ret = -EAGAIN; - goto out_cancel; --#ifdef CONFIG_SCHED_CLASS_EXT -- } else if (task_on_scx(p)) { -- p->sched_class = &ext_sched_class; --#endif -+ } else if (task_on_hmbird(p)) { -+ p->sched_class = &hmbird_sched_class; - } else if (rt_prio(p->prio)) { - p->sched_class = &rt_sched_class; - } else { - p->sched_class = &fair_sched_class; - } -- -+#else -+ if (dl_prio(p->prio)) -+ return -EAGAIN; -+ else if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - init_entity_runnable_average(&p->se); - trace_android_rvh_finish_prio_fork(p); - -@@ -4965,12 +5013,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - #endif - return 0; - -+#ifdef CONFIG_HMBIRD_SCHED - out_cancel: -- scx_cancel_fork(p); -+ hmbird_cancel_fork(p); - return ret; -+#endif - } - --int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) -+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - { - unsigned long flags; - -@@ -4997,19 +5047,17 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -- -- return scx_fork(p); --} -- --void sched_cancel_fork(struct task_struct *p) --{ -- scx_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_fork(p); -+#endif - } - - void sched_post_fork(struct task_struct *p) - { - uclamp_post_fork(p); -- scx_post_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_post_fork(p); -+#endif - } - - unsigned long to_ratio(u64 period, u64 runtime) -@@ -5874,19 +5922,28 @@ void scheduler_tick(void) - if (sched_feat(LATENCY_WARN) && resched_latency) - resched_latency_warn(cpu, resched_latency); - -- scx_notify_sched_tick(); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_notify_sched_tick(); -+#endif - perf_event_task_tick(); - - if (curr->flags & PF_WQ_WORKER) - wq_worker_tick(curr); - - #ifdef CONFIG_SMP -- if (!scx_switched_all()) { -+#ifdef CONFIG_HMBIRD_SCHED -+ if (!hmbird_enabled()) { -+#endif - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); -+#ifdef CONFIG_HMBIRD_SCHED - } -+#endif - #endif - trace_android_vh_scheduler_tick(rq); -+#ifdef CONFIG_HMBIRD_SCHED -+ scheduler_tick_handler(NULL, NULL); -+#endif - } - - #ifdef CONFIG_NO_HZ_FULL -@@ -6188,7 +6245,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ -+#ifdef CONFIG_HMBIRD_SCHED - for_balance_class_range(class, prev->sched_class, &idle_sched_class) { -+#else -+ for_class_range(class, prev->sched_class, &idle_sched_class) { -+#endif - if (class->balance(rq, prev, rf)) - break; - } -@@ -6206,9 +6267,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - const struct sched_class *class; - struct task_struct *p; - -- if (scx_enabled()) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (hmbird_enabled()) - goto restart; -- -+#endif - /* - * Optimization: we know that if all tasks are in the fair class we can - * call that function directly, but only if the @prev task wasn't of a -@@ -6234,13 +6296,21 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - restart: - put_prev_task_balance(rq, prev, rf); - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { - p = class->pick_next_task(rq); - if (p) { -- scx_notify_pick_next_task(rq, p, class); -+ hmbird_notify_pick_next_task(rq, p, class); - return p; - } - } -+#else -+ for_each_class(class) { -+ p = class->pick_next_task(rq); -+ if (p) -+ return p; -+ } -+#endif - - BUG(); /* The idle class should always have a runnable task. */ - } -@@ -6269,7 +6339,11 @@ static inline struct task_struct *pick_task(struct rq *rq) - const struct sched_class *class; - struct task_struct *p; - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { -+#else -+ for_each_class(class) { -+#endif - p = class->pick_task(rq); - if (p) - return p; -@@ -6913,6 +6987,9 @@ static void __sched notrace __schedule(unsigned int sched_mode) - psi_sched_switch(prev, next, !task_on_rq_queued(prev)); - - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#ifdef CONFIG_HMBIRD_SCHED -+ sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#endif - - /* Also unlocks the rq: */ - rq = context_switch(rq, prev, next, &rf); -@@ -7241,24 +7318,31 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag - } - EXPORT_SYMBOL(default_wake_function); - --void __setscheduler_prio(struct task_struct *p, int prio) -+static void __setscheduler_prio(struct task_struct *p, int prio) - { -- /* -- * After switching all rt and fair class to ext, -- * stop class is switched to ext class too. Just -- * skip it. */ -+#ifdef CONFIG_HMBIRD_SCHED -+ bool on_hmbird = task_on_hmbird(p); -+ - if (p->sched_class == &stop_sched_class) -- ;/* do nothing */ -+ ; - else if (dl_prio(prio)) - p->sched_class = &dl_sched_class; --#ifdef CONFIG_SCHED_CLASS_EXT -- else if (task_on_scx(p)) -- p->sched_class = &ext_sched_class; --#endif -+ else if (rt_prio(prio) && on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#else -+ if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; - else if (rt_prio(prio)) - p->sched_class = &rt_sched_class; - else - p->sched_class = &fair_sched_class; -+#endif - - p->prio = prio; - trace_android_rvh_setscheduler_prio(p); -@@ -7294,7 +7378,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - */ - void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - { -- int prio, oldprio, queue_flag = -+ int prio, oldprio, queued, running, queue_flag = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *prev_class; - struct rq_flags rf; -@@ -7357,42 +7441,52 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - queue_flag &= ~DEQUEUE_MOVE; - - prev_class = p->sched_class; -- SCHED_CHANGE_BLOCK(rq, p, queue_flag) { -- /* -- * Boosting condition are: -- * 1. -rt task is running and holds mutex A -- * --> -dl task blocks on mutex A -- * -- * 2. -dl task is running and holds mutex A -- * --> -dl task blocks on mutex A and could preempt the -- * running task -- */ -- if (dl_prio(prio)) { -- if (!dl_prio(p->normal_prio) || -- (pi_task && dl_prio(pi_task->prio) && -- dl_entity_preempt(&pi_task->dl, &p->dl))) { -- p->dl.pi_se = pi_task->dl.pi_se; -- queue_flag |= ENQUEUE_REPLENISH; -- } else { -- p->dl.pi_se = &p->dl; -- } -- } else if (rt_prio(prio)) { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- if (oldprio < prio) -- queue_flag |= ENQUEUE_HEAD; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flag); -+ if (running) -+ put_prev_task(rq, p); -+ -+ /* -+ * Boosting condition are: -+ * 1. -rt task is running and holds mutex A -+ * --> -dl task blocks on mutex A -+ * -+ * 2. -dl task is running and holds mutex A -+ * --> -dl task blocks on mutex A and could preempt the -+ * running task -+ */ -+ if (dl_prio(prio)) { -+ if (!dl_prio(p->normal_prio) || -+ (pi_task && dl_prio(pi_task->prio) && -+ dl_entity_preempt(&pi_task->dl, &p->dl))) { -+ p->dl.pi_se = pi_task->dl.pi_se; -+ queue_flag |= ENQUEUE_REPLENISH; - } else { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- else if (rt_prio(oldprio)) -- p->rt.timeout = 0; -- else if (!task_has_idle_policy(p)) -- reweight_task(p, prio - MAX_RT_PRIO); -+ p->dl.pi_se = &p->dl; - } -- -- __setscheduler_prio(p, prio); -+ } else if (rt_prio(prio)) { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ if (oldprio < prio) -+ queue_flag |= ENQUEUE_HEAD; -+ } else { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ else if (rt_prio(oldprio)) -+ p->rt.timeout = 0; -+ else if (!task_has_idle_policy(p)) -+ reweight_task(p, prio - MAX_RT_PRIO); - } - -+ __setscheduler_prio(p, prio); -+ -+ if (queued) -+ enqueue_task(rq, p, queue_flag); -+ if (running) -+ set_next_task(rq, p); -+ - check_class_changed(rq, p, prev_class, oldprio); - out_unlock: - /* Avoid rq from going away on us: */ -@@ -7413,7 +7507,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - - void set_user_nice(struct task_struct *p, long nice) - { -- bool allowed = false; -+ bool queued, running, allowed = false; - int old_prio; - struct rq_flags rf; - struct rq *rq; -@@ -7442,13 +7536,22 @@ void set_user_nice(struct task_struct *p, long nice) - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); -+ if (running) -+ put_prev_task(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->static_prio = NICE_TO_PRIO(nice); -- set_load_weight(p, true); -- old_prio = p->prio; -- p->prio = effective_prio(p); -- } -+ p->static_prio = NICE_TO_PRIO(nice); -+ set_load_weight(p, true); -+ old_prio = p->prio; -+ p->prio = effective_prio(p); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - - /* - * If the task increased its priority or is running and -@@ -7865,7 +7968,7 @@ static int __sched_setscheduler(struct task_struct *p, - bool user, bool pi) - { - int oldpolicy = -1, policy = attr->sched_policy; -- int retval, oldprio, newprio; -+ int retval, oldprio, newprio, queued, running; - const struct sched_class *prev_class; - struct balance_callback *head; - struct rq_flags rf; -@@ -7873,6 +7976,9 @@ static int __sched_setscheduler(struct task_struct *p, - int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct rq *rq; - bool cpuset_locked = false; -+#ifdef CONFIG_HMBIRD_SCHED -+ unsigned long flags; -+#endif - - /* The pi code expects interrupts enabled */ - BUG_ON(pi && in_interrupt()); -@@ -7938,6 +8044,9 @@ static int __sched_setscheduler(struct task_struct *p, - * To be able to change p->policy safely, the appropriate - * runqueue lock must be held. - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+#endif - rq = task_rq_lock(p, &rf); - update_rq_clock(rq); - -@@ -7949,10 +8058,11 @@ static int __sched_setscheduler(struct task_struct *p, - goto unlock; - } - -- retval = scx_check_setscheduler(p, policy); -+#ifdef CONFIG_HMBIRD_SCHED -+ retval = hmbird_check_setscheduler(p, policy); - if (retval) - goto unlock; -- -+#endif - /* - * If not changing anything there's no need to proceed further, - * but store a possible modification of reset_on_fork. -@@ -8009,6 +8119,9 @@ static int __sched_setscheduler(struct task_struct *p, - if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { - policy = oldpolicy = -1; - task_rq_unlock(rq, p, &rf); -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - goto recheck; -@@ -8041,16 +8154,23 @@ static int __sched_setscheduler(struct task_struct *p, - queue_flags &= ~DEQUEUE_MOVE; - } - -- SCHED_CHANGE_BLOCK(rq, p, queue_flags) { -- prev_class = p->sched_class; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flags); -+ if (running) -+ put_prev_task(rq, p); - -- if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -- __setscheduler_params(p, attr); -- __setscheduler_prio(p, newprio); -- trace_android_rvh_setscheduler(p); -- } -- __setscheduler_uclamp(p, attr); -+ prev_class = p->sched_class; - -+ if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -+ __setscheduler_params(p, attr); -+ __setscheduler_prio(p, newprio); -+ trace_android_rvh_setscheduler(p); -+ } -+ __setscheduler_uclamp(p, attr); -+ -+ if (queued) { - /* - * We enqueue to tail when the priority of a task is - * increased (user space view). -@@ -8058,7 +8178,10 @@ static int __sched_setscheduler(struct task_struct *p, - if (oldprio < p->prio) - queue_flags |= ENQUEUE_HEAD; - -+ enqueue_task(rq, p, queue_flags); - } -+ if (running) -+ set_next_task(rq, p); - - check_class_changed(rq, p, prev_class, oldprio); - -@@ -8066,7 +8189,9 @@ static int __sched_setscheduler(struct task_struct *p, - preempt_disable(); - head = splice_balance_callbacks(rq); - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (pi) { - if (cpuset_locked) - cpuset_unlock(); -@@ -8081,6 +8206,9 @@ static int __sched_setscheduler(struct task_struct *p, - - unlock: - task_rq_unlock(rq, p, &rf); -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - return retval; -@@ -9302,7 +9430,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - break; - } -@@ -9330,7 +9460,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - } - return ret; -@@ -9631,15 +9763,25 @@ int migrate_task_to(struct task_struct *p, int target_cpu) - */ - void sched_setnuma(struct task_struct *p, int nid) - { -+ bool queued, running; - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) { -- p->numa_preferred_nid = nid; -- } -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE); -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->numa_preferred_nid = nid; - -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - task_rq_unlock(rq, p, &rf); - } - #endif /* CONFIG_NUMA_BALANCING */ -@@ -10205,15 +10347,22 @@ void __init sched_init(void) - int i; - - /* Make sure the linker didn't screw up */ -+#ifdef CONFIG_HMBIRD_SCHED - #ifdef CONFIG_SMP -- BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+#endif -+ WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class)); -+#else -+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 || -+ &fair_sched_class != &rt_sched_class + 1 || -+ &rt_sched_class != &dl_sched_class + 1); -+#ifdef CONFIG_SMP -+ BUG_ON(&dl_sched_class != &stop_sched_class + 1); - #endif -- BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); -- BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); -- BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); --#ifdef CONFIG_SCHED_CLASS_EXT -- BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); -- BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); - #endif - - wait_bit_init(); -@@ -10385,8 +10534,9 @@ void __init sched_init(void) - balance_push_set(smp_processor_id(), false); - #endif - init_sched_fair_class(); -- init_sched_ext_class(); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ init_sched_hmbird_class(); -+#endif - psi_init(); - - init_uclamp(); -@@ -10790,6 +10940,9 @@ static void sched_change_group(struct task_struct *tsk) - */ - void sched_move_task(struct task_struct *tsk) - { -+ int queued, running, queue_flags = -+ DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; -+ struct task_group *group; - struct rq_flags rf; - struct rq *rq; - -@@ -10797,18 +10950,27 @@ void sched_move_task(struct task_struct *tsk) - rq = task_rq_lock(tsk, &rf); - update_rq_clock(rq); - -- SCHED_CHANGE_BLOCK(rq, tsk, -- DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { -- sched_change_group(tsk); -- } -+ running = task_current(rq, tsk); -+ queued = task_on_rq_queued(tsk); - -- /* -- * After changing group, the running task may have joined a throttled -- * one but it's still the running task. Trigger a resched to make sure -- * that task can still run. -- */ -- if (task_current(rq, tsk)) -+ if (queued) -+ dequeue_task(rq, tsk, queue_flags); -+ if (running) -+ put_prev_task(rq, tsk); -+ -+ sched_change_group(tsk, group); -+ -+ if (queued) -+ enqueue_task(rq, tsk, queue_flags); -+ if (running) { -+ set_next_task(rq, tsk); -+ /* -+ * After changing group, the running task may have joined a -+ * throttled one but it's still the running task. Trigger a -+ * resched to make sure that task can still run. -+ */ - resched_curr(rq); -+ } - - task_rq_unlock(rq, tsk, &rf); - } -@@ -10845,6 +11007,10 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) - struct task_group *tg = css_tg(css); - struct task_group *parent = css_tg(css->parent); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_tg_online(tg); -+#endif -+ - if (parent) - sched_online_group(tg, parent); - -@@ -10894,13 +11060,79 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) - } - #endif - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 dl) -+{ -+ int i; -+ -+ for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) { -+ if (dl < HMBIRD_BPF_DSQS_DEADLINE[i]) -+ break; -+ } -+ -+ i = max_t(int, i-1, MIN_CGROUP_DL_IDX); -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return 0; -+ update_cgroup_ids_table(css->cgroup->kn->id, i); -+ -+ return 0; -+} -+ -+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ u8 i; -+ -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX]; -+ i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1); -+ if (i < 0) { -+ pr_err(" <%s> i is %d, less than 0, name is %s\n", -+ __func__, i, css->cgroup->kn->name); -+ i = DEFAULT_CGROUP_DL_IDX; -+ } -+ -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[i]; -+} -+ -+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css, -+ struct task_struct *p, int prio) -+{ -+ if (!css || !rt_prio(prio)) -+ return; -+ -+ if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) { -+ if (!strcmp(css->cgroup->kn->name, "display")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3); -+ else if (!strcmp(css->cgroup->kn->name, "touch")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2); -+ else if (!strcmp(css->cgroup->kn->name, "multimedia")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+ } -+} -+#endif -+ - static void cpu_cgroup_attach(struct cgroup_taskset *tset) - { - struct task_struct *task; - struct cgroup_subsys_state *css; - -- cgroup_taskset_for_each(task, css, tset) -+ cgroup_taskset_for_each(task, css, tset) { -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_change_rt_sched_prop(css, task, task->prio); -+#endif - sched_move_task(task); -+ } - - trace_android_rvh_cpu_cgroup_attach(tset); - } -@@ -11579,6 +11811,13 @@ static struct cftype cpu_legacy_files[] = { - .read_u64 = cpu_uclamp_ls_read_u64, - .write_u64 = cpu_uclamp_ls_write_u64, - }, -+#endif -+#ifdef CONFIG_HMBIRD_SCHED -+ { -+ .name = "hmbird.deadline", -+ .read_u64 = cgroup_read_hmbird_deadline, -+ .write_u64 = cgroup_write_hmbird_deadline, -+ }, - #endif - { } /* Terminate */ - }; -@@ -11628,7 +11867,6 @@ static int cpu_local_stat_show(struct seq_file *sf, - } - - #ifdef CONFIG_FAIR_GROUP_SCHED -- - static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, - struct cftype *cft) - { -diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c -index a6c99df9edf9..f647dacc22e2 100644 ---- a/kernel/sched/debug.c -+++ b/kernel/sched/debug.c -@@ -376,8 +376,8 @@ static __init int sched_init_debug(void) - - debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); - --#ifdef CONFIG_SCHED_CLASS_EXT -- debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops); -+#ifdef CONFIG_HMBIRD_SCHED -+ debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops); - #endif - return 0; - } -@@ -1006,9 +1006,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - SEQ_printf(m, - "---------------------------------------------------------" - "----------\n"); --#ifdef CONFIG_SLIM_SCHED -- SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop); --#endif - - #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) - #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) -@@ -1104,8 +1101,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - } else if (fair_policy(p->policy)) { - P(se.slice); - } --#ifdef CONFIG_SCHED_CLASS_EXT -- __PS("ext.enabled", p->sched_class == &ext_sched_class); -+#ifdef CONFIG_HMBIRD_SCHED -+ __PS("hmbird.enabled", p->sched_class == &hmbird_sched_class); -+ __PS("sched_prop", get_hmbird_ts(p)->sched_prop); -+ __PS("top_task_prop", get_hmbird_ts(p)->top_task_prop); - #endif - #undef PN_SCHEDSTAT - #undef P_SCHEDSTAT -diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c -deleted file mode 100755 -index 4845d441df2b..000000000000 ---- a/kernel/sched/ext.c -+++ /dev/null -@@ -1,3978 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -- --enum scx_internal_consts { -- SCX_NR_ONLINE_OPS = SCX_OP_IDX(init), -- SCX_DSP_DFL_MAX_BATCH = 32, -- SCX_DSP_MAX_LOOPS = 32, -- SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, --}; -- --enum scx_ops_enable_state { -- SCX_OPS_PREPPING, -- SCX_OPS_ENABLING, -- SCX_OPS_ENABLED, -- SCX_OPS_DISABLING, -- SCX_OPS_DISABLED, --}; -- --static inline struct task_group *css_tg(struct cgroup_subsys_state *css) --{ -- return css ? container_of(css, struct task_group, css) : NULL; --} -- --/* -- * sched_ext_entity->ops_state -- * -- * Used to track the task ownership between the SCX core and the BPF scheduler. -- * State transitions look as follows: -- * -- * NONE -> QUEUEING -> QUEUED -> DISPATCHING -- * ^ | | -- * | v v -- * \-------------------------------/ -- * -- * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -- * sites for explanations on the conditions being waited upon and why they are -- * safe. Transitions out of them into NONE or QUEUED must store_release and the -- * waiters should load_acquire. -- * -- * Tracking scx_ops_state enables sched_ext core to reliably determine whether -- * any given task can be dispatched by the BPF scheduler at all times and thus -- * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -- * to try to dispatch any task anytime regardless of its state as the SCX core -- * can safely reject invalid dispatches. -- */ --enum scx_ops_state { -- SCX_OPSS_NONE, /* owned by the SCX core */ -- SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -- SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ -- SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ -- -- /* -- * QSEQ brands each QUEUED instance so that, when dispatch races -- * dequeue/requeue, the dispatcher can tell whether it still has a claim -- * on the task being dispatched. -- */ -- SCX_OPSS_QSEQ_SHIFT = 2, -- SCX_OPSS_STATE_MASK = (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1, -- SCX_OPSS_QSEQ_MASK = ~SCX_OPSS_STATE_MASK, --}; -- --/* -- * During exit, a task may schedule after losing its PIDs. When disabling the -- * BPF scheduler, we need to be able to iterate tasks in every state to -- * guarantee system safety. Maintain a dedicated task list which contains every -- * task between its fork and eventual free. -- */ --static DEFINE_SPINLOCK(scx_tasks_lock); --static LIST_HEAD(scx_tasks); -- --/* ops enable/disable */ --static struct kthread_worker *scx_ops_helper; --static DEFINE_MUTEX(scx_ops_enable_mutex); --DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); --DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); --static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); --static bool scx_switch_all_req; --static bool scx_switching_all; --DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -- --static struct sched_ext_ops scx_ops; --static bool scx_warned_zero_slice; -- --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); --DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); --static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); -- --struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] = -- { [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT }; -- --static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE); --static struct scx_exit_info scx_exit_info; -- --static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0); -- --/* -- * The maximum amount of time in jiffies that a task may be runnable without -- * being scheduled on a CPU. If this timeout is exceeded, it will trigger -- * scx_ops_error(). -- */ --unsigned long scx_watchdog_timeout; -- --/* -- * The last time the delayed work was run. This delayed work relies on -- * ksoftirqd being able to run to service timer interrupts, so it's possible -- * that this work itself could get wedged. To account for this, we check that -- * it's not stalled in the timer tick, and trigger an error if it is. -- */ --unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; -- --static struct delayed_work scx_watchdog_work; -- --/* idle tracking */ --#ifdef CONFIG_SMP --#ifdef CONFIG_CPUMASK_OFFSTACK --#define CL_ALIGNED_IF_ONSTACK --#else --#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp --#endif -- --static struct { -- cpumask_var_t cpu; -- cpumask_var_t smt; --} idle_masks CL_ALIGNED_IF_ONSTACK; -- --static bool __cacheline_aligned_in_smp scx_has_idle_cpus; --#endif /* CONFIG_SMP */ -- --/* for %SCX_KICK_WAIT */ --static u64 __percpu *scx_kick_cpus_pnt_seqs; -- --/* -- * Direct dispatch marker. -- * -- * Non-NULL values are used for direct dispatch from enqueue path. A valid -- * pointer points to the task currently being enqueued. An ERR_PTR value is used -- * to indicate that direct dispatch has already happened. -- */ --static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); -- --/* dispatch queues */ --static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; -- --static LLIST_HEAD(dsqs_to_free); -- --/* dispatch buf */ --struct scx_dsp_buf_ent { -- struct task_struct *task; -- u64 qseq; -- u64 dsq_id; -- u64 enq_flags; --}; -- --static u32 scx_dsp_max_batch; --static struct scx_dsp_buf_ent __percpu *scx_dsp_buf; -- --struct scx_dsp_ctx { -- struct rq *rq; -- struct rq_flags *rf; -- u32 buf_cursor; -- u32 nr_tasks; --}; -- --static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx); -- --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags); --void scx_bpf_kick_cpu(s32 cpu, u64 flags); -- --struct scx_task_iter { -- struct sched_ext_entity cursor; -- struct task_struct *locked; -- struct rq *rq; -- struct rq_flags rf; --}; -- --#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) -- --/* if the highest set bit is N, return a mask with bits [N+1, 31] set */ --static u32 higher_bits(u32 flags) --{ -- return ~((1 << fls(flags)) - 1); --} -- --/* return the mask with only the highest bit set */ --static u32 highest_bit(u32 flags) --{ -- int bit = fls(flags); -- return bit ? 1 << (bit - 1) : 0; --} -- --/* -- * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX -- * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate -- * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check -- * whether it's running from an allowed context. -- * -- * @mask is constant, always inline to cull the mask calculations. -- */ --static __always_inline void scx_kf_allow(u32 mask) --{ -- /* nesting is allowed only in increasing scx_kf_mask order */ -- WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask, -- "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n", -- current->scx->kf_mask, mask); -- current->scx->kf_mask |= mask; --} -- --static void scx_kf_disallow(u32 mask) --{ -- current->scx->kf_mask &= ~mask; --} -- --#define SCX_CALL_OP(mask, op, args...) \ --do { \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- scx_ops.op(args); \ -- } \ --} while (0) -- --#define SCX_CALL_OP_RET(mask, op, args...) \ --({ \ -- __typeof__(scx_ops.op(args)) __ret; \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- __ret = scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- __ret = scx_ops.op(args); \ -- } \ -- __ret; \ --}) -- --/* -- * Some kfuncs are allowed only on the tasks that are subjects of the -- * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such -- * restrictions, the following SCX_CALL_OP_*() variants should be used when -- * invoking scx_ops operations that take task arguments. These can only be used -- * for non-nesting operations due to the way the tasks are tracked. -- * -- * kfuncs which can only operate on such tasks can in turn use -- * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on -- * the specific task. -- */ --#define SCX_CALL_OP_TASK(mask, op, task, args...) \ --do { \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- SCX_CALL_OP(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ --} while (0) -- --#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ --({ \ -- __typeof__(scx_ops.op(task, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- __ret; \ --}) -- --#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ --({ \ -- __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task0; \ -- current->scx->kf_tasks[1] = task1; \ -- __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- current->scx->kf_tasks[1] = NULL; \ -- __ret; \ --}) -- --//#include "./slim_walt.c" -- --/* @mask is constant, always inline to cull unnecessary branches */ --static __always_inline bool scx_kf_allowed(u32 mask) --{ -- if (unlikely(!(current->scx->kf_mask & mask))) { -- scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", -- mask, current->scx->kf_mask); -- return false; -- } -- -- if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) && -- in_interrupt())) { -- scx_ops_error("sleepable kfunc called from non-sleepable context"); -- return false; -- } -- -- /* -- * Enforce nesting boundaries. e.g. A kfunc which can be called from -- * DISPATCH must not be called if we're running DEQUEUE which is nested -- * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE -- * boundary thanks to the above in_interrupt() check. -- */ -- if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && -- (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { -- scx_ops_error("cpu_release kfunc called from a nested operation"); -- return false; -- } -- -- if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && -- (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) { -- scx_ops_error("dispatch kfunc called from a nested operation"); -- return false; -- } -- -- return true; --} -- --/* see SCX_CALL_OP_TASK() */ --static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, -- struct task_struct *p) --{ -- if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED)) -- return false; -- -- if (unlikely((p != current->scx->kf_tasks[0] && -- p != current->scx->kf_tasks[1]))) { -- scx_ops_error("called on a task not being operated on"); -- return false; -- } -- -- return true; --} -- --/** -- * scx_task_iter_init - Initialize a task iterator -- * @iter: iterator to init -- * -- * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized, -- * @iter must eventually be exited with scx_task_iter_exit(). -- * -- * scx_tasks_lock may be released between this and the first next() call or -- * between any two next() calls. If scx_tasks_lock is released between two -- * next() calls, the caller is responsible for ensuring that the task being -- * iterated remains accessible either through RCU read lock or obtaining a -- * reference count. -- * -- * All tasks which existed when the iteration started are guaranteed to be -- * visited as long as they still exist. -- */ --static void scx_task_iter_init(struct scx_task_iter *iter) --{ -- lockdep_assert_held(&scx_tasks_lock); -- -- iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; -- list_add(&iter->cursor.tasks_node, &scx_tasks); -- iter->locked = NULL; --} -- --/** -- * scx_task_iter_exit - Exit a task iterator -- * @iter: iterator to exit -- * -- * Exit a previously initialized @iter. Must be called with scx_tasks_lock held. -- * If the iterator holds a task's rq lock, that rq lock is released. See -- * scx_task_iter_init() for details. -- */ --static void scx_task_iter_exit(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- if (list_empty(cursor)) -- return; -- -- list_del_init(cursor); --} -- --/** -- * scx_task_iter_next - Next task -- * @iter: iterator to walk -- * -- * Visit the next task. See scx_task_iter_init() for details. -- */ --static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- struct sched_ext_entity *pos; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- list_for_each_entry(pos, cursor, tasks_node) { -- if (&pos->tasks_node == &scx_tasks) -- return NULL; -- if (!(pos->flags & SCX_TASK_CURSOR)) { -- list_move(cursor, &pos->tasks_node); -- return pos->task; -- } -- } -- -- /* can't happen, should always terminate at scx_tasks above */ -- BUG(); --} -- --/** -- * scx_task_iter_next_filtered - Next non-idle task -- * @iter: iterator to walk -- * -- * Visit the next non-idle task. See scx_task_iter_init() for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- while ((p = scx_task_iter_next(iter))) { -- if (!is_idle_task(p)) -- return p; -- } -- return NULL; --} -- --/** -- * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked -- * @iter: iterator to walk -- * -- * Visit the next non-idle task with its rq lock held. See scx_task_iter_init() -- * for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered_locked(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- p = scx_task_iter_next_filtered(iter); -- if (!p) -- return NULL; -- -- iter->rq = task_rq_lock(p, &iter->rf); -- iter->locked = p; -- return p; --} -- --static enum scx_ops_enable_state scx_ops_enable_state(void) --{ -- return atomic_read(&scx_ops_enable_state_var); --} -- --static enum scx_ops_enable_state --scx_ops_set_enable_state(enum scx_ops_enable_state to) --{ -- return atomic_xchg(&scx_ops_enable_state_var, to); --} -- --static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, -- enum scx_ops_enable_state from) --{ -- int from_v = from; -- -- return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); --} -- --static bool scx_ops_disabling(void) --{ -- return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING); --} -- --/** -- * wait_ops_state - Busy-wait the specified ops state to end -- * @p: target task -- * @opss: state to wait the end of -- * -- * Busy-wait for @p to transition out of @opss. This can only be used when the -- * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also -- * has load_acquire semantics to ensure that the caller can see the updates made -- * in the enqueueing and dispatching paths. -- */ --static void wait_ops_state(struct task_struct *p, u64 opss) --{ -- do { -- cpu_relax(); -- } while (atomic64_read_acquire(&p->scx->ops_state) == opss); --} -- --/** -- * ops_cpu_valid - Verify a cpu number -- * @cpu: cpu number which came from a BPF ops -- * -- * @cpu is a cpu number which came from the BPF scheduler and can be any value. -- * Verify that it is in range and one of the possible cpus. -- */ --static bool ops_cpu_valid(s32 cpu) --{ -- return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); --} -- --/** -- * ops_sanitize_err - Sanitize a -errno value -- * @ops_name: operation to blame on failure -- * @err: -errno value to sanitize -- * -- * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return -- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can -- * cause misbehaviors. For an example, a large negative return from -- * ops.prep_enable() triggers an oops when passed up the call chain because the -- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is -- * handled as a pointer. -- */ --static int ops_sanitize_err(const char *ops_name, s32 err) --{ -- if (err < 0 && err >= -MAX_ERRNO) -- return err; -- -- scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); -- return -EPROTO; --} -- --/** -- * touch_core_sched - Update timestamp used for core-sched task ordering -- * @rq: rq to read clock from, must be locked -- * @p: task to update the timestamp for -- * -- * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to -- * implement global or local-DSQ FIFO ordering for core-sched. Should be called -- * when a task becomes runnable and its turn on the CPU ends (e.g. slice -- * exhaustion). -- */ --static void touch_core_sched(struct rq *rq, struct task_struct *p) --{ --#ifdef CONFIG_SCHED_CORE -- /* -- * It's okay to update the timestamp spuriously. Use -- * sched_core_disabled() which is cheaper than enabled(). -- */ -- if (!sched_core_disabled()) -- p->scx->core_sched_at = rq_clock_task(rq); --#endif --} -- --/** -- * touch_core_sched_dispatch - Update core-sched timestamp on dispatch -- * @rq: rq to read clock from, must be locked -- * @p: task being dispatched -- * -- * If the BPF scheduler implements custom core-sched ordering via -- * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO -- * ordering within each local DSQ. This function is called from dispatch paths -- * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect. -- */ --static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- assert_clock_updated(rq); -- --#ifdef CONFIG_SCHED_CORE -- if (SCX_HAS_OP(core_sched_before)) -- touch_core_sched(rq, p); --#endif --} -- --static void update_curr_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- u64 now = rq_clock_task(rq); -- u64 delta_exec; -- -- if (time_before_eq64(now, curr->se.exec_start)) -- return; -- -- delta_exec = now - curr->se.exec_start; -- curr->se.exec_start = now; -- curr->se.sum_exec_runtime += delta_exec; -- account_group_exec_runtime(curr, delta_exec); -- cgroup_account_cputime(curr, delta_exec); -- -- if (curr->scx->slice != SCX_SLICE_INF) { -- curr->scx->slice -= min(curr->scx->slice, delta_exec); -- if (!curr->scx->slice) -- touch_core_sched(rq, curr); -- } --} -- --static bool scx_dsq_priq_less(struct rb_node *node_a, -- const struct rb_node *node_b) --{ -- const struct sched_ext_entity *a = -- container_of(node_a, struct sched_ext_entity, dsq_node.priq); -- const struct sched_ext_entity *b = -- container_of(node_b, struct sched_ext_entity, dsq_node.priq); -- -- return time_before64(a->dsq_vtime, b->dsq_vtime); --} -- --static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, -- u64 enq_flags) --{ -- bool is_local = dsq->id == SCX_DSQ_LOCAL; -- -- WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo)); -- WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq)); -- -- if (!is_local) { -- raw_spin_lock(&dsq->lock); -- if (unlikely(dsq->id == SCX_DSQ_INVALID)) { -- scx_ops_error("attempting to dispatch to a destroyed dsq"); -- /* fall back to the global dsq */ -- raw_spin_unlock(&dsq->lock); -- dsq = &scx_dsq_global; -- raw_spin_lock(&dsq->lock); -- } -- } -- -- if (enq_flags & SCX_ENQ_DSQ_PRIQ) { -- p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; -- rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq, -- scx_dsq_priq_less); -- } else { -- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) -- list_add(&p->scx->dsq_node.fifo, &dsq->fifo); -- else -- list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo); -- } -- dsq->nr++; -- p->scx->dsq = dsq; -- -- /* -- * We're transitioning out of QUEUEING or DISPATCHING. store_release to -- * match waiters' load_acquire. -- */ -- if (enq_flags & SCX_ENQ_CLEAR_OPSS) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- if (is_local) { -- struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq); -- struct rq *rq = scx->rq; -- bool preempt = false; -- -- if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && -- rq->curr->sched_class == &ext_sched_class) { -- rq->curr->scx->slice = 0; -- preempt = true; -- } -- -- if (preempt || sched_class_above(&ext_sched_class, -- rq->curr->sched_class)) -- resched_curr(rq); -- } else { -- raw_spin_unlock(&dsq->lock); -- } --} -- --static void task_unlink_from_dsq(struct task_struct *p, -- struct scx_dispatch_q *dsq) --{ -- if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { -- rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; -- } else { -- list_del_init(&p->scx->dsq_node.fifo); -- } -- --} -- --static bool task_linked_on_dsq(struct task_struct *p) --{ -- return !list_empty(&p->scx->dsq_node.fifo) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq); --} -- --static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p) --{ -- struct scx_dispatch_q *dsq = p->scx->dsq; -- bool is_local = dsq == &scx_rq->local_dsq; -- -- if (!dsq) { -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- /* -- * When dispatching directly from the BPF scheduler to a local -- * DSQ, the task isn't associated with any DSQ but -- * @p->scx->holding_cpu may be set under the protection of -- * %SCX_OPSS_DISPATCHING. -- */ -- if (p->scx->holding_cpu >= 0) -- p->scx->holding_cpu = -1; -- return; -- } -- -- if (!is_local) -- raw_spin_lock(&dsq->lock); -- -- /* -- * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node -- * can't change underneath us. -- */ -- if (p->scx->holding_cpu < 0) { -- /* @p must still be on @dsq, dequeue */ -- WARN_ON_ONCE(!task_linked_on_dsq(p)); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- } else { -- /* -- * We're racing against dispatch_to_local_dsq() which already -- * removed @p from @dsq and set @p->scx->holding_cpu. Clear the -- * holding_cpu which tells dispatch_to_local_dsq() that it lost -- * the race. -- */ -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- p->scx->holding_cpu = -1; -- } -- p->scx->dsq = NULL; -- -- if (!is_local) -- raw_spin_unlock(&dsq->lock); --} -- --static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) --{ -- lockdep_assert(rcu_read_lock_any_held()); -- -- return &scx_dsq_global; --} -- --static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, -- struct task_struct *p) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id == SCX_DSQ_LOCAL) -- return &rq->scx->local_dsq; -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", -- dsq_id, p->comm, p->pid); -- return &scx_dsq_global; -- } -- -- return dsq; --} -- --static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- -- /* @p must match the task which is being enqueued */ -- if (unlikely(p != ddsp_task)) { -- if (IS_ERR(ddsp_task)) -- scx_ops_error("%s[%d] already direct-dispatched", -- p->comm, p->pid); -- else -- scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]", -- ddsp_task->comm, ddsp_task->pid, -- p->comm, p->pid); -- return; -- } -- -- /* -- * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because -- * dispatching to the local DSQ of a different CPU requires unlocking -- * the current rq which isn't allowed in the enqueue path. Use -- * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL. -- */ -- if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) { -- scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch"); -- return; -- } -- -- touch_core_sched_dispatch(task_rq(p), p); -- -- dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- -- /* -- * Mark that dispatch already happened by spoiling direct_dispatch_task -- * with a non-NULL value which can never match a valid task pointer. -- */ -- __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); --} -- --static bool test_rq_online(struct rq *rq) --{ --#ifdef CONFIG_SMP -- return rq->online; --#else -- return true; --#endif --} -- --static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- int sticky_cpu) --{ -- struct task_struct **ddsp_taskp; -- u64 qseq; -- -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- if (p->scx->flags & SCX_TASK_ENQ_LOCAL) { -- enq_flags |= SCX_ENQ_LOCAL; -- p->scx->flags &= ~SCX_TASK_ENQ_LOCAL; -- } -- -- /* rq migration */ -- if (sticky_cpu == cpu_of(rq)) -- goto local_norefill; -- -- /* -- * If !rq->online, we already told the BPF scheduler that the CPU is -- * offline. We're just trying to on/offline the CPU. Don't bother the -- * BPF scheduler. -- */ -- if (unlikely(!test_rq_online(rq))) -- goto local; -- -- /* see %SCX_OPS_ENQ_EXITING */ -- if (!static_branch_unlikely(&scx_ops_enq_exiting) && -- unlikely(p->flags & PF_EXITING)) -- goto local; -- -- /* see %SCX_OPS_ENQ_LAST */ -- if (!static_branch_unlikely(&scx_ops_enq_last) && -- (enq_flags & SCX_ENQ_LAST)) -- goto local; -- -- if (!SCX_HAS_OP(enqueue)) { -- if (enq_flags & SCX_ENQ_LOCAL) -- goto local; -- else -- goto global; -- } -- -- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ -- qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; -- -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq); -- -- ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); -- WARN_ON_ONCE(*ddsp_taskp); -- *ddsp_taskp = p; -- -- SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); -- -- /* -- * If not directly dispatched, QUEUEING isn't clear yet and dispatch or -- * dequeue may be waiting. The store_release matches their load_acquire. -- */ -- if (*ddsp_taskp == p) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq); -- *ddsp_taskp = NULL; -- return; -- --local: -- /* -- * For task-ordering, slice refill must be treated as implying the end -- * of the current slice. Otherwise, the longer @p stays on the CPU, the -- * higher priority it becomes from scx_prio_less()'s POV. -- */ -- touch_core_sched(rq, p); -- p->scx->slice = SCX_SLICE_DFL; --local_norefill: -- dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags); -- return; -- --global: -- touch_core_sched(rq, p); /* see the comment in local: */ -- p->scx->slice = SCX_SLICE_DFL; -- dispatch_enqueue(&scx_dsq_global, p, enq_flags); --} -- --static bool watchdog_task_watched(const struct task_struct *p) --{ -- return !list_empty(&p->scx->watchdog_node); --} -- --static void watchdog_watch_task(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- if (p->scx->flags & SCX_TASK_WATCHDOG_RESET) -- p->scx->runnable_at = jiffies; -- p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET; -- list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list); --} -- --static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) --{ -- list_del_init(&p->scx->watchdog_node); -- if (reset_timeout) -- p->scx->flags |= SCX_TASK_WATCHDOG_RESET; --} -- --static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) --{ -- int sticky_cpu = p->scx->sticky_cpu; -- -- enq_flags |= rq->scx->extra_enq_flags; -- -- if (sticky_cpu >= 0) -- p->scx->sticky_cpu = -1; -- -- /* -- * Restoring a running task will be immediately followed by -- * set_next_task_scx() which expects the task to not be on the BPF -- * scheduler as tasks can only start running through local DSQs. Force -- * direct-dispatch into the local DSQ by setting the sticky_cpu. -- */ -- if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -- sticky_cpu = cpu_of(rq); -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- WARN_ON_ONCE(!watchdog_task_watched(p)); -- return; -- } -- -- watchdog_watch_task(rq, p); -- p->scx->flags |= SCX_TASK_QUEUED; -- rq->scx->nr_running++; -- add_nr_running(rq, 1); -- -- if (SCX_HAS_OP(runnable)) -- SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); -- -- if (enq_flags & SCX_ENQ_WAKEUP) -- touch_core_sched(rq, p); -- -- do_enqueue_task(rq, p, enq_flags, sticky_cpu); --} -- --static void ops_dequeue(struct task_struct *p, u64 deq_flags) --{ -- u64 opss; -- -- watchdog_unwatch_task(p, false); -- -- /* acquire ensures that we see the preceding updates on QUEUED */ -- opss = atomic64_read_acquire(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_NONE: -- break; -- case SCX_OPSS_QUEUEING: -- /* -- * QUEUEING is started and finished while holding @p's rq lock. -- * As we're holding the rq lock now, we shouldn't see QUEUEING. -- */ -- BUG(); -- case SCX_OPSS_QUEUED: -- if (SCX_HAS_OP(dequeue)) -- SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); -- -- if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_NONE)) -- break; -- fallthrough; -- case SCX_OPSS_DISPATCHING: -- /* -- * If @p is being dispatched from the BPF scheduler to a DSQ, -- * wait for the transfer to complete so that @p doesn't get -- * added to its DSQ after dequeueing is complete. -- * -- * As we're waiting on DISPATCHING with the rq locked, the -- * dispatching side shouldn't try to lock the rq while -- * DISPATCHING is set. See dispatch_to_local_dsq(). -- * -- * DISPATCHING shouldn't have qseq set and control can reach -- * here with NONE @opss from the above QUEUED case block. -- * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. -- */ -- wait_ops_state(p, SCX_OPSS_DISPATCHING); -- BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- break; -- } --} -- --static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) --{ -- struct scx_rq *scx_rq = rq->scx; -- -- if (!(p->scx->flags & SCX_TASK_QUEUED)) { -- WARN_ON_ONCE(watchdog_task_watched(p)); -- return; -- } -- -- ops_dequeue(p, deq_flags); -- -- /* -- * A currently running task which is going off @rq first gets dequeued -- * and then stops running. As we want running <-> stopping transitions -- * to be contained within runnable <-> quiescent transitions, trigger -- * ->stopping() early here instead of in put_prev_task_scx(). -- * -- * @p may go through multiple stopping <-> running transitions between -- * here and put_prev_task_scx() if task attribute changes occur while -- * balance_scx() leaves @rq unlocked. However, they don't contain any -- * information meaningful to the BPF scheduler and can be suppressed by -- * skipping the callbacks if the task is !QUEUED. -- */ -- if (SCX_HAS_OP(stopping) && task_current(rq, p)) { -- update_curr_scx(rq); -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); -- } -- -- //if(task_current(rq, p)) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- if (SCX_HAS_OP(quiescent)) -- SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); -- -- if (deq_flags & SCX_DEQ_SLEEP) -- p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP; -- else -- p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP; -- -- p->scx->flags &= ~SCX_TASK_QUEUED; -- BUG_ON(!scx_rq->nr_running); -- scx_rq->nr_running--; -- sub_nr_running(rq, 1); -- -- dispatch_dequeue(scx_rq, p); --} -- --static void yield_task_scx(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); -- else -- p->scx->slice = 0; --} -- --static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) --{ -- struct task_struct *from = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); -- else -- return false; --} -- --#ifdef CONFIG_SMP --/** -- * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -- * @rq: rq to move the task into, currently locked -- * @p: task to move -- * @enq_flags: %SCX_ENQ_* -- * -- * Move @p which is currently on a different rq to @rq's local DSQ. The caller -- * must: -- * -- * 1. Start with exclusive access to @p either through its DSQ lock or -- * %SCX_OPSS_DISPATCHING flag. -- * -- * 2. Set @p->scx->holding_cpu to raw_smp_processor_id(). -- * -- * 3. Remember task_rq(@p). Release the exclusive access so that we don't -- * deadlock with dequeue. -- * -- * 4. Lock @rq and the task_rq from #3. -- * -- * 5. Call this function. -- * -- * Returns %true if @p was successfully moved. %false after racing dequeue and -- * losing. -- */ --static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -- u64 enq_flags) --{ -- struct rq *task_rq; -- -- lockdep_assert_rq_held(rq); -- -- /* -- * If dequeue got to @p while we were trying to lock both rq's, it'd -- * have cleared @p->scx->holding_cpu to -1. While other cpus may have -- * updated it to different values afterwards, as this operation can't be -- * preempted or recurse, @p->scx->holding_cpu can never become -- * raw_smp_processor_id() again before we're done. Thus, we can tell -- * whether we lost to dequeue by testing whether @p->scx->holding_cpu is -- * still raw_smp_processor_id(). -- * -- * See dispatch_dequeue() for the counterpart. -- */ -- if (unlikely(p->scx->holding_cpu != raw_smp_processor_id())) -- return false; -- -- /* @p->rq couldn't have changed if we're still the holding cpu */ -- task_rq = task_rq(p); -- lockdep_assert_rq_held(task_rq); -- -- WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)); -- deactivate_task(task_rq, p, 0); -- set_task_cpu(p, cpu_of(rq)); -- p->scx->sticky_cpu = cpu_of(rq); -- -- /* -- * We want to pass scx-specific enq_flags but activate_task() will -- * truncate the upper 32 bit. As we own @rq, we can pass them through -- * @rq->scx->extra_enq_flags instead. -- */ -- WARN_ON_ONCE(rq->scx->extra_enq_flags); -- rq->scx->extra_enq_flags = enq_flags; -- activate_task(rq, p, 0); -- rq->scx->extra_enq_flags = 0; -- -- return true; --} -- --/** -- * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * We're holding @rq lock and trying to dispatch a task from @src_rq to -- * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether -- * @rq stays locked isn't important as long as the state is restored after -- * dispatch_to_local_dsq_unlock(). -- */ --static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- rq_unpin_lock(rq, rf); -- -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(rq); -- raw_spin_rq_lock(dst_rq); -- } else if (rq == src_rq) { -- double_lock_balance(rq, dst_rq); -- rq_repin_lock(rq, rf); -- } else if (rq == dst_rq) { -- double_lock_balance(rq, src_rq); -- rq_repin_lock(rq, rf); -- } else { -- raw_spin_rq_unlock(rq); -- double_rq_lock(src_rq, dst_rq); -- } --} -- --/** -- * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock() -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return. -- */ --static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } else if (rq == src_rq) { -- double_unlock_balance(rq, dst_rq); -- } else if (rq == dst_rq) { -- double_unlock_balance(rq, src_rq); -- } else { -- double_rq_unlock(src_rq, dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } --} --#endif /* CONFIG_SMP */ -- -- --static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq) --{ -- return likely(test_rq_online(rq)) && !is_migration_disabled(p) && -- cpumask_test_cpu(cpu_of(rq), p->cpus_ptr); --} -- --static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -- struct scx_dispatch_q *dsq) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rb_node *rb_node; -- struct rq *task_rq; -- bool moved = false; --retry: -- if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -- return false; -- -- raw_spin_lock(&dsq->lock); -- -- list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- for (rb_node = rb_first_cached(&dsq->priq); rb_node; -- rb_node = rb_next(rb_node)) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- raw_spin_unlock(&dsq->lock); -- return false; -- --this_rq: -- /* @dsq is locked and @p is on this rq */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo); -- dsq->nr--; -- scx_rq->local_dsq.nr++; -- p->scx->dsq = &scx_rq->local_dsq; -- raw_spin_unlock(&dsq->lock); -- return true; -- --remote_rq: --#ifdef CONFIG_SMP -- /* -- * @dsq is locked and @p is on a remote rq. @p is currently protected by -- * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -- * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -- * rq lock or fail, do a little dancing from our side. See -- * move_task_to_local_dsq(). -- */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- p->scx->holding_cpu = raw_smp_processor_id(); -- raw_spin_unlock(&dsq->lock); -- -- rq_unpin_lock(rq, rf); -- double_lock_balance(rq, task_rq); -- rq_repin_lock(rq, rf); -- -- moved = move_task_to_local_dsq(rq, p, 0); -- -- double_unlock_balance(rq, task_rq); --#endif /* CONFIG_SMP */ -- if (likely(moved)) -- return true; -- goto retry; --} -- --enum dispatch_to_local_dsq_ret { -- DTL_DISPATCHED, /* successfully dispatched */ -- DTL_LOST, /* lost race to dequeue */ -- DTL_NOT_LOCAL, /* destination is not a local DSQ */ -- DTL_INVALID, /* invalid local dsq_id */ --}; -- --/** -- * dispatch_to_local_dsq - Dispatch a task to a local dsq -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @dsq_id: destination dsq ID -- * @p: task to dispatch -- * @enq_flags: %SCX_ENQ_* -- * -- * We're holding @rq lock and want to dispatch @p to the local DSQ identified by -- * @dsq_id. This function performs all the synchronization dancing needed -- * because local DSQs are protected with rq locks. -- * -- * The caller must have exclusive ownership of @p (e.g. through -- * %SCX_OPSS_DISPATCHING). -- */ --static enum dispatch_to_local_dsq_ret --dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id, -- struct task_struct *p, u64 enq_flags) --{ -- struct rq *src_rq = task_rq(p); -- struct rq *dst_rq; -- -- /* -- * We're synchronized against dequeue through DISPATCHING. As @p can't -- * be dequeued, its task_rq and cpus_allowed are stable too. -- */ -- if (dsq_id == SCX_DSQ_LOCAL) { -- dst_rq = rq; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]", -- cpu, p->comm, p->pid); -- return DTL_INVALID; -- } -- dst_rq = cpu_rq(cpu); -- } else { -- return DTL_NOT_LOCAL; -- } -- -- /* if dispatching to @rq that @p is already on, no lock dancing needed */ -- if (rq == src_rq && rq == dst_rq) { -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags | SCX_ENQ_CLEAR_OPSS); -- return DTL_DISPATCHED; -- } -- --#ifdef CONFIG_SMP -- if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { -- struct rq *locked_dst_rq = dst_rq; -- bool dsp; -- -- /* -- * @p is on a possibly remote @src_rq which we need to lock to -- * move the task. If dequeue is in progress, it'd be locking -- * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq -- * lock while holding DISPATCHING. -- * -- * As DISPATCHING guarantees that @p is wholly ours, we can -- * pretend that we're moving from a DSQ and use the same -- * mechanism - mark the task under transfer with holding_cpu, -- * release DISPATCHING and then follow the same protocol. -- */ -- p->scx->holding_cpu = raw_smp_processor_id(); -- -- /* store_release ensures that dequeue sees the above */ -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq); -- -- /* -- * We don't require the BPF scheduler to avoid dispatching to -- * offline CPUs mostly for convenience but also because CPUs can -- * go offline between scx_bpf_dispatch() calls and here. If @p -- * is destined to an offline CPU, queue it on its current CPU -- * instead, which should always be safe. As this is an allowed -- * behavior, don't trigger an ops error. -- */ -- if (unlikely(!test_rq_online(dst_rq))) -- dst_rq = src_rq; -- -- if (src_rq == dst_rq) { -- /* -- * As @p is staying on the same rq, there's no need to -- * go through the full deactivate/activate cycle. -- * Optimize by abbreviating the operations in -- * move_task_to_local_dsq(). -- */ -- dsp = p->scx->holding_cpu == raw_smp_processor_id(); -- if (likely(dsp)) { -- p->scx->holding_cpu = -1; -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags); -- } -- } else { -- dsp = move_task_to_local_dsq(dst_rq, p, enq_flags); -- } -- -- /* if the destination CPU is idle, wake it up */ -- if (dsp && p->sched_class > dst_rq->curr->sched_class) -- resched_curr(dst_rq); -- -- dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq); -- -- return dsp ? DTL_DISPATCHED : DTL_LOST; -- } --#endif /* CONFIG_SMP */ -- -- scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", -- cpu_of(dst_rq), p->comm, p->pid); -- return DTL_INVALID; --} -- --/** -- * finish_dispatch - Asynchronously finish dispatching a task -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @p: task to finish dispatching -- * @qseq_at_dispatch: qseq when @p started getting dispatched -- * @dsq_id: destination DSQ ID -- * @enq_flags: %SCX_ENQ_* -- * -- * Dispatching to local DSQs may need to wait for queueing to complete or -- * require rq lock dancing. As we don't wanna do either while inside -- * ops.dispatch() to avoid locking order inversion, we split dispatching into -- * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the -- * task and its qseq. Once ops.dispatch() returns, this function is called to -- * finish up. -- * -- * There is no guarantee that @p is still valid for dispatching or even that it -- * was valid in the first place. Make sure that the task is still owned by the -- * BPF scheduler and claim the ownership before dispatching. -- */ --static void finish_dispatch(struct rq *rq, struct rq_flags *rf, -- struct task_struct *p, u64 qseq_at_dispatch, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- u64 opss; -- -- touch_core_sched_dispatch(rq, p); --retry: -- /* -- * No need for _acquire here. @p is accessed only after a successful -- * try_cmpxchg to DISPATCHING. -- */ -- opss = atomic64_read(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_DISPATCHING: -- case SCX_OPSS_NONE: -- /* someone else already got to it */ -- return; -- case SCX_OPSS_QUEUED: -- /* -- * If qseq doesn't match, @p has gone through at least one -- * dispatch/dequeue and re-enqueue cycle between -- * scx_bpf_dispatch() and here and we have no claim on it. -- */ -- if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) -- return; -- -- /* -- * While we know @p is accessible, we don't yet have a claim on -- * it - the BPF scheduler is allowed to dispatch tasks -- * spuriously and there can be a racing dequeue attempt. Let's -- * claim @p by atomically transitioning it from QUEUED to -- * DISPATCHING. -- */ -- if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_DISPATCHING))) -- break; -- goto retry; -- case SCX_OPSS_QUEUEING: -- /* -- * do_enqueue_task() is in the process of transferring the task -- * to the BPF scheduler while holding @p's rq lock. As we aren't -- * holding any kernel or BPF resource that the enqueue path may -- * depend upon, it's safe to wait. -- */ -- wait_ops_state(p, opss); -- goto retry; -- } -- -- BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) { -- case DTL_DISPATCHED: -- break; -- case DTL_LOST: -- break; -- case DTL_INVALID: -- dsq_id = SCX_DSQ_GLOBAL; -- fallthrough; -- case DTL_NOT_LOCAL: -- dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()), -- dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- break; -- } --} -- --static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- u32 u; -- -- for (u = 0; u < dspc->buf_cursor; u++) { -- struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u]; -- -- finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id, -- ent->enq_flags); -- } -- -- dspc->nr_tasks += dspc->buf_cursor; -- dspc->buf_cursor = 0; --} -- --static int balance_one(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf, bool local) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- bool prev_on_scx = prev->sched_class == &ext_sched_class; -- int nr_loops = SCX_DSP_MAX_LOOPS; -- -- lockdep_assert_rq_held(rq); -- -- if (static_branch_unlikely(&scx_ops_cpu_preempt) && -- unlikely(rq->scx->cpu_released)) { -- /* -- * If the previous sched_class for the current CPU was not SCX, -- * notify the BPF scheduler that it again has control of the -- * core. This callback complements ->cpu_release(), which is -- * emitted in scx_notify_pick_next_task(). -- */ -- if (SCX_HAS_OP(cpu_acquire)) -- SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq), -- NULL); -- rq->scx->cpu_released = false; -- } -- -- if (prev_on_scx) { -- WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP)); -- update_curr_scx(rq); -- -- /* -- * If @prev is runnable & has slice left, it has priority and -- * fetching more just increases latency for the fetched tasks. -- * Tell put_prev_task_scx() to put @prev on local_dsq. If the -- * BPF scheduler wants to handle this explicitly, it should -- * implement ->cpu_released(). -- * -- * See scx_ops_disable_workfn() for the explanation on the -- * disabling() test. -- * -- * When balancing a remote CPU for core-sched, there won't be a -- * following put_prev_task_scx() call and we don't own -- * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the -- * same conditions later and pick @rq->curr accordingly. -- */ -- if ((prev->scx->flags & SCX_TASK_QUEUED) && -- prev->scx->slice && !scx_ops_disabling()) { -- if (local) -- prev->scx->flags |= SCX_TASK_BAL_KEEP; -- return 1; -- } -- } -- -- /* if there already are tasks to run, nothing to do */ -- if (scx_rq->local_dsq.nr) -- return 1; -- -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- if (!SCX_HAS_OP(dispatch)) -- return 0; -- -- dspc->rq = rq; -- dspc->rf = rf; -- -- /* -- * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, -- * the local DSQ might still end up empty after a successful -- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() -- * produced some tasks, retry. The BPF scheduler may depend on this -- * looping behavior to simplify its implementation. -- */ -- do { -- dspc->nr_tasks = 0; -- -- SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), -- prev_on_scx ? prev : NULL); -- -- flush_dispatch_buf(rq, rf); -- -- if (scx_rq->local_dsq.nr) -- return 1; -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- /* -- * ops.dispatch() can trap us in this loop by repeatedly -- * dispatching ineligible tasks. Break out once in a while to -- * allow the watchdog to run. As IRQ can't be enabled in -- * balance(), we want to complete this scheduling cycle and then -- * start a new one. IOW, we want to call resched_curr() on the -- * next, most likely idle, task, not the current one. Use -- * scx_bpf_kick_cpu() for deferred kicking. -- */ -- if (unlikely(!--nr_loops)) { -- scx_bpf_kick_cpu(cpu_of(rq), 0); -- break; -- } -- } while (dspc->nr_tasks); -- -- return 0; --} -- --static int balance_scx(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf) --{ -- int ret; -- -- ret = balance_one(rq, prev, rf, true); --#ifdef CONFIG_SCHED_SMT -- /* -- * When core-sched is enabled, this ops.balance() call will be followed -- * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx() -- * on the SMT siblings. Balance the siblings too. -- */ -- if (sched_core_enabled(rq)) { -- const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); -- int scpu; -- -- for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { -- struct rq *srq = cpu_rq(scpu); -- struct rq_flags srf; -- struct task_struct *sprev = srq->curr; -- -- /* -- * While core-scheduling, rq lock is shared among -- * siblings but the debug annotations and rq clock -- * aren't. Do pinning dance to transfer the ownership. -- */ -- WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); -- rq_unpin_lock(rq, rf); -- rq_pin_lock(srq, &srf); -- -- update_rq_clock(srq); -- balance_one(srq, sprev, &srf, false); -- -- rq_unpin_lock(srq, &srf); -- rq_repin_lock(rq, rf); -- } -- } --#endif -- return ret; --} -- --static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) --{ -- if (p->scx->flags & SCX_TASK_QUEUED) { -- /* -- * Core-sched might decide to execute @p before it is -- * dispatched. Call ops_dequeue() to notify the BPF scheduler. -- */ -- ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); -- dispatch_dequeue(rq->scx, p); -- } -- -- p->se.exec_start = rq_clock_task(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, running, p); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock); -- -- watchdog_unwatch_task(p, true); -- -- /* -- * @p is getting newly scheduled or got kicked after someone updated its -- * slice. Refresh whether tick can be stopped. See can_stop_tick_scx(). -- */ -- if ((p->scx->slice == SCX_SLICE_INF) != -- (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) { -- if (p->scx->slice == SCX_SLICE_INF) -- rq->scx->flags |= SCX_RQ_CAN_STOP_TICK; -- else -- rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK; -- -- sched_update_tick_dependency(rq); -- } --} -- --static void put_prev_task_scx(struct rq *rq, struct task_struct *p) --{ --#ifndef CONFIG_SMP -- /* -- * UP workaround. -- * -- * Because SCX may transfer tasks across CPUs during dispatch, dispatch -- * is performed from its balance operation which isn't called in UP. -- * Let's work around by calling it from the operations which come right -- * after. -- * -- * 1. If the prev task is on SCX, pick_next_task() calls -- * .put_prev_task() right after. As .put_prev_task() is also called -- * from other places, we need to distinguish the calls which can be -- * done by looking at the previous task's state - if still queued or -- * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task(). -- * This case is handled here. -- * -- * 2. If the prev task is not on SCX, the first following call into SCX -- * will be .pick_next_task(), which is covered by calling -- * balance_scx() from pick_next_task_scx(). -- * -- * Note that we can't merge the first case into the second as -- * balance_scx() must be called before the previous SCX task goes -- * through put_prev_task_scx(). -- * -- * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf. -- * Pass in %NULL. -- */ -- if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP)) -- balance_scx(rq, p, NULL); --#endif -- -- update_curr_scx(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- -- /* -- * If we're being called from put_prev_task_balance(), balance_scx() may -- * have decided that @p should keep running. -- */ -- if (p->scx->flags & SCX_TASK_BAL_KEEP) { -- p->scx->flags &= ~SCX_TASK_BAL_KEEP; -- watchdog_watch_task(rq, p); -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- watchdog_watch_task(rq, p); -- -- /* -- * If @p has slice left and balance_scx() didn't tag it for -- * keeping, @p is getting preempted by a higher priority -- * scheduler class or core-sched forcing a different task. Leave -- * it at the head of the local DSQ. -- */ -- if (p->scx->slice && !scx_ops_disabling()) { -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- /* -- * If we're in the pick_next_task path, balance_scx() should -- * have already populated the local DSQ if there are any other -- * available tasks. If empty, tell ops.enqueue() that @p is the -- * only one available for this cpu. ops.enqueue() should put it -- * on the local DSQ so that the subsequent pick_next_task_scx() -- * can find the task unless it wants to trigger a separate -- * follow-up scheduling event. -- */ -- if (list_empty(&rq->scx->local_dsq.fifo)) -- do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1); -- else -- do_enqueue_task(rq, p, 0, -1); -- } --} -- --static struct task_struct *first_local_task(struct rq *rq) --{ -- struct rb_node *rb_node; -- struct sched_ext_entity *entity; -- -- if (!list_empty(&rq->scx->local_dsq.fifo)) { -- entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo); -- return entity->task; -- } -- -- rb_node = rb_first_cached(&rq->scx->local_dsq.priq); -- if (rb_node) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- return entity->task; -- } -- -- return NULL; --} -- --static struct task_struct *pick_next_task_scx(struct rq *rq) --{ -- struct task_struct *p; -- --#ifndef CONFIG_SMP -- /* UP workaround - see the comment at the head of put_prev_task_scx() */ -- if (unlikely(rq->curr->sched_class != &ext_sched_class)) -- balance_scx(rq, rq->curr, NULL); --#endif -- -- p = first_local_task(rq); -- if (!p) -- return NULL; -- -- if (unlikely(!p->scx->slice)) { -- if (!scx_ops_disabling() && !scx_warned_zero_slice) { -- printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n", -- p->comm, p->pid); -- scx_warned_zero_slice = true; -- } -- p->scx->slice = SCX_SLICE_DFL; -- } -- -- set_next_task_scx(rq, p, true); -- -- return p; --} -- --#ifdef CONFIG_SCHED_CORE --/** -- * scx_prio_less - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Core-sched is implemented as an additional scheduling layer on top of the -- * usual sched_class'es and needs to find out the expected task ordering. For -- * SCX, core-sched calls this function to interrogate the task ordering. -- * -- * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used -- * to implement the default task ordering. The older the timestamp, the higher -- * prority the task - the global FIFO ordering matching the default scheduling -- * behavior. -- * -- * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to -- * implement FIFO ordering within each local DSQ. See pick_task_scx(). -- */ --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi) --{ -- /* -- * The const qualifiers are dropped from task_struct pointers when -- * calling ops.core_sched_before(). Accesses are controlled by the -- * verifier. -- */ -- if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling()) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, -- (struct task_struct *)a, -- (struct task_struct *)b); -- else -- return time_after64(a->scx->core_sched_at, b->scx->core_sched_at); --} -- --/** -- * pick_task_scx - Pick a candidate task for core-sched -- * @rq: rq to pick the candidate task from -- * -- * Core-sched calls this function on each SMT sibling to determine the next -- * tasks to run on the SMT siblings. balance_one() has been called on all -- * siblings and put_prev_task_scx() has been called only for the current CPU. -- * -- * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look -- * at the first task in the local dsq. @rq->curr has to be considered explicitly -- * to mimic %SCX_TASK_BAL_KEEP. -- */ --static struct task_struct *pick_task_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- struct task_struct *first = first_local_task(rq); -- -- if (curr->scx->flags & SCX_TASK_QUEUED) { -- /* is curr the only runnable task? */ -- if (!first) -- return curr; -- -- /* -- * Does curr trump first? We can always go by core_sched_at for -- * this comparison as it represents global FIFO ordering when -- * the default core-sched ordering is used and local-DSQ FIFO -- * ordering otherwise. -- * -- * We can have a task with an earlier timestamp on the DSQ. For -- * example, when a current task is preempted by a sibling -- * picking a different cookie, the task would be requeued at the -- * head of the local DSQ with an earlier timestamp than the -- * core-sched picked next task. Besides, the BPF scheduler may -- * dispatch any tasks to the local DSQ anytime. -- */ -- if (curr->scx->slice && time_before64(curr->scx->core_sched_at, -- first->scx->core_sched_at)) -- return curr; -- } -- -- return first; /* this may be %NULL */ --} --#endif /* CONFIG_SCHED_CORE */ -- --static enum scx_cpu_preempt_reason --preempt_reason_from_class(const struct sched_class *class) --{ --#ifdef CONFIG_SMP -- if (class == &stop_sched_class) -- return SCX_CPU_PREEMPT_STOP; --#endif -- if (class == &dl_sched_class) -- return SCX_CPU_PREEMPT_DL; -- if (class == &rt_sched_class) -- return SCX_CPU_PREEMPT_RT; -- return SCX_CPU_PREEMPT_UNKNOWN; --} -- --void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task, -- const struct sched_class *active) --{ -- lockdep_assert_rq_held(rq); -- -- /* -- * The callback is conceptually meant to convey that the CPU is no -- * longer under the control of SCX. Therefore, don't invoke the -- * callback if the CPU is is staying on SCX, or going idle (in which -- * case the SCX scheduler has actively decided not to schedule any -- * tasks on the CPU). -- */ -- if (likely(active >= &ext_sched_class)) -- return; -- -- /* -- * At this point we know that SCX was preempted by a higher priority -- * sched_class, so invoke the ->cpu_release() callback if we have not -- * done so already. We only send the callback once between SCX being -- * preempted, and it regaining control of the CPU. -- * -- * ->cpu_release() complements ->cpu_acquire(), which is emitted the -- * next time that balance_scx() is invoked. -- */ -- if (!rq->scx->cpu_released) { -- if (SCX_HAS_OP(cpu_release)) { -- struct scx_cpu_release_args args = { -- .reason = preempt_reason_from_class(active), -- .task = task, -- }; -- -- SCX_CALL_OP(SCX_KF_CPU_RELEASE, -- cpu_release, cpu_of(rq), &args); -- } -- rq->scx->cpu_released = true; -- } --} -- --#ifdef CONFIG_SMP -- --static bool test_and_clear_cpu_idle(int cpu) --{ -- if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -- if (cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- return true; -- } else { -- return false; -- } --} -- --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- int cpu; -- -- do { -- cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); -- if (cpu < nr_cpu_ids) { -- const struct cpumask *sbm = topology_sibling_cpumask(cpu); -- -- /* -- * If offline, @cpu is not its own sibling and we can -- * get caught in an infinite loop as @cpu is never -- * cleared from idle_masks.smt. Clear @cpu directly in -- * such cases. -- */ -- if (likely(cpumask_test_cpu(cpu, sbm))) -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm); -- else -- cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu)); -- } else { -- cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -- if (cpu >= nr_cpu_ids) -- return -EBUSY; -- } -- } while (!test_and_clear_cpu_idle(cpu)); -- -- return cpu; --} -- --static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) --{ -- s32 cpu; -- -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return prev_cpu; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local DSQ of the waker. -- */ -- if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 && -- scx_has_idle_cpus && !(current->flags & PF_EXITING)) { -- cpu = smp_processor_id(); -- if (cpumask_test_cpu(cpu, p->cpus_ptr)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (test_and_clear_cpu_idle(prev_cpu)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return prev_cpu; -- } -- -- if (p->nr_cpus_allowed == 1) -- return prev_cpu; -- -- cpu = scx_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- -- return prev_cpu; --} -- --static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) --{ -- if (SCX_HAS_OP(select_cpu)) { -- s32 cpu; -- -- cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu, -- wake_flags); -- if (ops_cpu_valid(cpu)) { -- return cpu; -- } else { -- scx_ops_error("select_cpu returned invalid cpu %d", cpu); -- return prev_cpu; -- } -- } else { -- return scx_select_cpu_dfl(p, prev_cpu, wake_flags); -- } --} -- --static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx) --{ -- set_cpus_allowed_common(p, ctx); -- -- /* -- * The effective cpumask is stored in @p->cpus_ptr which may temporarily -- * differ from the configured one in @p->cpus_mask. Always tell the bpf -- * scheduler the effective one. -- * -- * Fine-grained memory write control is enforced by BPF making the const -- * designation pointless. Cast it away when calling the operation. -- */ -- if (SCX_HAS_OP(set_cpumask)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, -- (struct cpumask *)p->cpus_ptr); --} -- --static void reset_idle_masks(void) --{ -- /* consider all cpus idle, should converge to the actual state quickly */ -- cpumask_setall(idle_masks.cpu); -- cpumask_setall(idle_masks.smt); -- scx_has_idle_cpus = true; --} -- --void __scx_update_idle(struct rq *rq, bool idle) --{ -- int cpu = cpu_of(rq); -- struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -- -- if (SCX_HAS_OP(update_idle)) { -- SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); -- if (!static_branch_unlikely(&scx_builtin_idle_enabled)) -- return; -- } -- -- if (idle) { -- cpumask_set_cpu(cpu, idle_masks.cpu); -- if (!scx_has_idle_cpus) -- scx_has_idle_cpus = true; -- -- /* -- * idle_masks.smt handling is racy but that's fine as it's only -- * for optimization and self-correcting. -- */ -- for_each_cpu(cpu, sib_mask) { -- if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -- return; -- } -- cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -- } else { -- cpumask_clear_cpu(cpu, idle_masks.cpu); -- if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -- } --} -- --#else /* !CONFIG_SMP */ -- --static bool test_and_clear_cpu_idle(int cpu) { return false; } --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } --static void reset_idle_masks(void) {} -- --#endif /* CONFIG_SMP */ -- --static bool check_rq_for_timeouts(struct rq *rq) --{ -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rq_flags rf; -- bool timed_out = false; -- -- rq_lock_irqsave(rq, &rf); -- list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) { -- unsigned long last_runnable; -- -- p = entity->task; -- last_runnable = p->scx->runnable_at; -- -- if (unlikely(time_after(jiffies, -- last_runnable + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "%s[%d] failed to run for %u.%03us", -- p->comm, p->pid, -- dur_ms / 1000, dur_ms % 1000); -- timed_out = true; -- break; -- } -- } -- rq_unlock_irqrestore(rq, &rf); -- -- return timed_out; --} -- --static void scx_watchdog_workfn(struct work_struct *work) --{ -- int cpu; -- -- scx_watchdog_timestamp = jiffies; -- -- for_each_online_cpu(cpu) { -- if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) -- break; -- -- cond_resched(); -- } -- queue_delayed_work(system_unbound_wq, to_delayed_work(work), -- scx_watchdog_timeout / 2); --} -- --static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) --{ -- update_curr_scx(rq); -- //scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock); -- /* -- * While disabling, always resched and refresh core-sched timestamp as -- * we can't trust the slice management or ops.core_sched_before(). -- */ -- if (scx_ops_disabling()) { -- curr->scx->slice = 0; -- touch_core_sched(rq, curr); -- } -- -- if (!curr->scx->slice) -- resched_curr(rq); --} -- --#define SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- --static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg) --{ -- int ret; -- -- WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED); -- -- p->scx->disallow = false; -- -- if (SCX_HAS_OP(prep_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- }; -- -- ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args); -- if (unlikely(ret)) { -- ret = ops_sanitize_err("prep_enable", ret); -- return ret; -- } -- } -- //scx_sched_init_task(p); -- -- if (p->scx->disallow) { -- struct rq *rq; -- struct rq_flags rf; -- -- rq = task_rq_lock(p, &rf); -- -- /* -- * We're either in fork or load path and @p->policy will be -- * applied right after. Reverting @p->policy here and rejecting -- * %SCHED_EXT transitions from scx_check_setscheduler() -- * guarantees that if ops.prep_enable() sets @p->disallow, @p -- * can never be in SCX. -- */ -- if (p->policy == SCHED_EXT) { -- p->policy = SCHED_NORMAL; -- atomic64_inc(&scx_nr_rejected); -- } -- -- task_rq_unlock(rq, p, &rf); -- } -- -- p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET); -- return 0; --} -- --static void scx_ops_enable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED)); -- -- if (SCX_HAS_OP(enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- p->scx->flags |= SCX_TASK_OPS_ENABLED; --} -- --static void scx_ops_disable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- if (p->scx->flags & SCX_TASK_OPS_PREPPED) { -- if (SCX_HAS_OP(cancel_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- } else if (p->scx->flags & SCX_TASK_OPS_ENABLED) { -- if (SCX_HAS_OP(disable)) -- SCX_CALL_OP(SCX_KF_REST, disable, p); -- p->scx->flags &= ~SCX_TASK_OPS_ENABLED; -- } --} -- --static void set_task_scx_weight(struct task_struct *p) --{ -- u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -- -- p->scx->weight = sched_weight_to_cgroup(weight); --} -- --/** -- * refresh_scx_weight - Refresh a task's ext weight -- * @p: task to refresh ext weight for -- * -- * @p->scx->weight carries the task's static priority in cgroup weight scale to -- * enable easy access from the BPF scheduler. To keep it synchronized with the -- * current task priority, this function should be called when a new task is -- * created, priority is changed for a task on sched_ext, and a task is switched -- * to sched_ext from other classes. -- */ --static void refresh_scx_weight(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- set_task_scx_weight(p); -- if (SCX_HAS_OP(set_weight)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight); --} -- --void scx_pre_fork(struct task_struct *p) --{ -- p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL); -- if (!p->scx) { -- goto lock; -- } -- -- p->scx->dsq = NULL; -- INIT_LIST_HEAD(&p->scx->dsq_node.fifo); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- INIT_LIST_HEAD(&p->scx->watchdog_node); -- p->scx->flags = 0; -- p->scx->weight = 0; -- p->scx->sticky_cpu = -1; -- p->scx->holding_cpu = -1; -- p->scx->kf_mask = 0; -- atomic64_set(&p->scx->ops_state, 0); -- p->scx->runnable_at = INITIAL_JIFFIES; -- p->scx->slice = SCX_SLICE_DFL; -- p->scx->task = p; -- p->sched_prop = 0; -- -- /* -- * BPF scheduler enable/disable paths want to be able to iterate and -- * update all tasks which can become complex when racing forks. As -- * enable/disable are very cold paths, let's use a percpu_rwsem to -- * exclude forks. -- */ --lock: -- percpu_down_read(&scx_fork_rwsem); --} -- --int scx_fork(struct task_struct *p) --{ -- percpu_rwsem_assert_held(&scx_fork_rwsem); -- -- if (scx_enabled()) -- return scx_ops_prepare_task(p, task_group(p)); -- else -- return 0; --} -- --void scx_post_fork(struct task_struct *p) --{ -- if (scx_enabled()) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- /* -- * Set the weight manually before calling ops.enable() so that -- * the scheduler doesn't see a stale value if they inspect the -- * task struct. We'll invoke ops.set_weight() afterwards, as it -- * would be odd to receive a callback on the task before we -- * tell the scheduler that it's been fully enabled. -- */ -- set_task_scx_weight(p); -- scx_ops_enable_task(p); -- refresh_scx_weight(p); -- task_rq_unlock(rq, p, &rf); -- } -- -- spin_lock_irq(&scx_tasks_lock); -- list_add_tail(&p->scx->tasks_node, &scx_tasks); -- spin_unlock_irq(&scx_tasks_lock); -- -- percpu_up_read(&scx_fork_rwsem); --} -- --void scx_cancel_fork(struct task_struct *p) --{ -- if (scx_enabled()) -- scx_ops_disable_task(p); -- percpu_up_read(&scx_fork_rwsem); --} -- --void sched_ext_free(struct task_struct *p) --{ -- unsigned long flags; -- -- spin_lock_irqsave(&scx_tasks_lock, flags); -- list_del_init(&p->scx->tasks_node); -- spin_unlock_irqrestore(&scx_tasks_lock, flags); -- -- /* -- * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED -> -- * ENABLED transitions can't race us. Disable ops for @p. -- */ -- if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- scx_ops_disable_task(p); -- task_rq_unlock(rq, p, &rf); -- } --} -- --static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) --{ --} -- --static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} --static void switched_to_scx(struct rq *rq, struct task_struct *p) {} -- --int scx_check_setscheduler(struct task_struct *p, int policy) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- /* if disallow, reject transitioning into SCX */ -- if (scx_enabled() && READ_ONCE(p->scx->disallow) && -- p->policy != policy && policy == SCHED_EXT) -- return -EACCES; -- -- return 0; --} -- --#ifdef CONFIG_NO_HZ_FULL --bool scx_can_stop_tick(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (scx_ops_disabling()) -- return false; -- -- if (p->sched_class != &ext_sched_class) -- return true; -- -- /* -- * @rq can dispatch from different DSQs, so we can't tell whether it -- * needs the tick or not by looking at nr_running. Allow stopping ticks -- * iff the BPF scheduler indicated so. See set_next_task_scx(). -- */ -- return rq->scx->flags & SCX_RQ_CAN_STOP_TICK; --} --#endif -- --static inline void scx_cgroup_lock(void) {} --static inline void scx_cgroup_unlock(void) {} -- --/* -- * Omitted operations: -- * -- * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -- * task isn't tied to the CPU at that point. Preemption is implemented by -- * resetting the victim task's slice to 0 and triggering reschedule on the -- * target CPU. -- * -- * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -- * -- * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -- * their current sched_class. Call them directly from sched core instead. -- * -- * - task_woken, switched_from: Unnecessary. -- */ --DEFINE_SCHED_CLASS(ext) = { -- .enqueue_task = enqueue_task_scx, -- .dequeue_task = dequeue_task_scx, -- .yield_task = yield_task_scx, -- .yield_to_task = yield_to_task_scx, -- -- .check_preempt_curr = check_preempt_curr_scx, -- -- .pick_next_task = pick_next_task_scx, -- -- .put_prev_task = put_prev_task_scx, -- .set_next_task = set_next_task_scx, -- --#ifdef CONFIG_SMP -- .balance = balance_scx, -- .select_task_rq = select_task_rq_scx, -- .set_cpus_allowed = set_cpus_allowed_scx, --#endif -- --#ifdef CONFIG_SCHED_CORE -- .pick_task = pick_task_scx, --#endif -- -- .task_tick = task_tick_scx, -- -- .switched_to = switched_to_scx, -- .prio_changed = prio_changed_scx, -- -- .update_curr = update_curr_scx, -- --#ifdef CONFIG_UCLAMP_TASK -- .uclamp_enabled = 0, --#endif --}; -- --static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) --{ -- memset(dsq, 0, sizeof(*dsq)); -- -- raw_spin_lock_init(&dsq->lock); -- INIT_LIST_HEAD(&dsq->fifo); -- dsq->id = dsq_id; --} -- --static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id & SCX_DSQ_FLAG_BUILTIN) -- return ERR_PTR(-EINVAL); -- -- dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node); -- if (!dsq) -- return ERR_PTR(-ENOMEM); -- -- init_dsq(dsq, dsq_id); -- -- return dsq; --} -- --static void free_dsq_irq_workfn(struct irq_work *irq_work) --{ -- struct llist_node *to_free = llist_del_all(&dsqs_to_free); -- struct scx_dispatch_q *dsq, *tmp_dsq; -- -- llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) -- kfree_rcu(dsq, rcu); --} -- --static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); -- --static void destroy_dsq(u64 dsq_id) --{ --} -- --static void scx_cgroup_exit(void) {} --static int scx_cgroup_init(void) { return 0; } --static void scx_cgroup_config_knobs(void) {} -- --/* -- * Used by sched_fork() and __setscheduler_prio() to pick the matching -- * sched_class. dl/rt are already handled. -- */ --bool task_on_scx(struct task_struct *p) --{ -- if (!scx_enabled() || scx_ops_disabling()) -- return false; -- if (READ_ONCE(scx_switching_all)) -- return true; -- return p->policy == SCHED_EXT; --} -- --static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags) --{ -- if (enq_flags & SCX_ENQ_LAST) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); --} -- --static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {} -- --static void scx_ops_disable_workfn(struct kthread_work *work) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- struct scx_task_iter sti; -- struct task_struct *p; -- const char *reason; -- int i, cpu, type; -- -- type = atomic_read(&scx_exit_type); -- while (true) { -- /* -- * NONE indicates that a new scx_ops has been registered since -- * disable was scheduled - don't kill the new ops. DONE -- * indicates that the ops has already been disabled. -- */ -- if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE) -- return; -- if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE)) -- break; -- } -- -- cancel_delayed_work_sync(&scx_watchdog_work); -- -- switch (type) { -- case SCX_EXIT_UNREG: -- reason = "BPF scheduler unregistered"; -- break; -- case SCX_EXIT_SYSRQ: -- reason = "disabled by sysrq-S"; -- break; -- case SCX_EXIT_ERROR: -- reason = "runtime error"; -- break; -- case SCX_EXIT_ERROR_BPF: -- reason = "scx_bpf_error"; -- break; -- case SCX_EXIT_ERROR_STALL: -- reason = "runnable task stall"; -- break; -- default: -- reason = ""; -- } -- -- ei->type = type; -- strlcpy(ei->reason, reason, sizeof(ei->reason)); -- -- switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { -- case SCX_OPS_DISABLED: -- pr_warn("sched_ext: ops error detected without ops (%s)\n", -- scx_exit_info.msg); -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- return; -- case SCX_OPS_PREPPING: -- goto forward_progress_guaranteed; -- case SCX_OPS_DISABLING: -- /* shouldn't happen but handle it like ENABLING if it does */ -- WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); -- fallthrough; -- case SCX_OPS_ENABLING: -- case SCX_OPS_ENABLED: -- break; -- } -- -- /* -- * DISABLING is set and ops was either ENABLING or ENABLED indicating -- * that the ops and static branches are set. -- * -- * We must guarantee that all runnable tasks make forward progress -- * without trusting the BPF scheduler. We can't grab any mutexes or -- * rwsems as they might be held by tasks that the BPF scheduler is -- * forgetting to run, which unfortunately also excludes toggling the -- * static branches. -- * -- * Let's work around by overriding a couple ops and modifying behaviors -- * based on the DISABLING state and then cycling the tasks through -- * dequeue/enqueue to force global FIFO scheduling. -- * -- * a. ops.enqueue() and .dispatch() are overridden for simple global -- * FIFO scheduling. -- * -- * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value -- * can't be trusted. Whenever a tick triggers, the running task is -- * rotated to the tail of the queue with core_sched_at touched. -- * -- * c. pick_next_task() suppresses zero slice warning. -- * -- * d. scx_prio_less() reverts to the default core_sched_at order. -- */ -- scx_ops.enqueue = scx_ops_fallback_enqueue; -- scx_ops.dispatch = scx_ops_fallback_dispatch; -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- SCHED_CHANGE_BLOCK(task_rq(p), p, -- DEQUEUE_SAVE | DEQUEUE_MOVE) { -- /* cycling deq/enq is enough, see above */ -- } -- } -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* kick all CPUs to restore ticks */ -- for_each_possible_cpu(cpu) -- resched_cpu(cpu); -- --forward_progress_guaranteed: -- /* -- * Here, every runnable task is guaranteed to make forward progress and -- * we can safely use blocking synchronization constructs. Actually -- * disable ops. -- */ -- mutex_lock(&scx_ops_enable_mutex); -- -- static_branch_disable(&__scx_switched_all); -- WRITE_ONCE(scx_switching_all, false); -- -- /* avoid racing against fork and cgroup changes */ -- cpus_read_lock(); -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- bool alive = READ_ONCE(p->__state) != TASK_DEAD; -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL); -- -- __setscheduler_prio(p, p->prio); -- } -- -- if (alive) -- check_class_changed(task_rq(p), p, old_class, p->prio); -- -- scx_ops_disable_task(p); -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* no task is on scx, turn off all the switches and flush in-progress calls */ -- static_branch_disable_cpuslocked(&__scx_ops_enabled); -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- static_branch_disable_cpuslocked(&scx_has_op[i]); -- static_branch_disable_cpuslocked(&scx_ops_enq_last); -- static_branch_disable_cpuslocked(&scx_ops_enq_exiting); -- static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- synchronize_rcu(); -- -- scx_cgroup_exit(); -- -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- cpus_read_unlock(); -- -- if (ei->type >= SCX_EXIT_ERROR) { -- printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name); -- -- if (ei->msg[0] == '\0') -- printk(KERN_ERR "sched_ext: %s\n", ei->reason); -- else -- printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg); -- -- stack_trace_print(ei->bt, ei->bt_len, 2); -- } -- -- if (scx_ops.exit) -- SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); -- -- memset(&scx_ops, 0, sizeof(scx_ops)); -- -- free_percpu(scx_dsp_buf); -- scx_dsp_buf = NULL; -- scx_dsp_max_batch = 0; -- -- mutex_unlock(&scx_ops_enable_mutex); -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- -- scx_cgroup_config_knobs(); --} -- --static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); -- --static void schedule_scx_ops_disable_work(void) --{ -- struct kthread_worker *helper = READ_ONCE(scx_ops_helper); -- -- /* -- * We may be called spuriously before the first bpf_sched_ext_reg(). If -- * scx_ops_helper isn't set up yet, there's nothing to do. -- */ -- if (helper) -- kthread_queue_work(helper, &scx_ops_disable_work); --} -- --static void scx_ops_disable(enum scx_exit_type type) --{ -- int none = SCX_EXIT_NONE; -- -- if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)) -- type = SCX_EXIT_ERROR; -- -- atomic_try_cmpxchg(&scx_exit_type, &none, type); -- -- schedule_scx_ops_disable_work(); --} -- --static void scx_ops_error_irq_workfn(struct irq_work *irq_work) --{ -- schedule_scx_ops_disable_work(); --} -- --static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- int none = SCX_EXIT_NONE; -- va_list args; -- -- if (!atomic_try_cmpxchg(&scx_exit_type, &none, type)) -- return; -- -- ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1); -- -- va_start(args, fmt); -- vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args); -- va_end(args); -- -- irq_work_queue(&scx_ops_error_irq_work); --} -- --static struct kthread_worker *scx_create_rt_helper(const char *name) --{ -- struct kthread_worker *helper; -- -- helper = kthread_create_worker(0, name); -- if (helper) -- sched_set_fifo(helper->task); -- return helper; --} -- --static int scx_ops_enable(struct sched_ext_ops *ops) --{ -- struct scx_task_iter sti; -- struct task_struct *p; -- int i, ret; -- int tcnt = 0; -- unsigned long long start = 0; -- unsigned long long total_start = sched_clock(); -- -- mutex_lock(&scx_ops_enable_mutex); -- -- if (!scx_ops_helper) { -- WRITE_ONCE(scx_ops_helper, -- scx_create_rt_helper("sched_ext_ops_helper")); -- if (!scx_ops_helper) { -- ret = -ENOMEM; -- goto err_unlock; -- } -- } -- -- if (scx_ops_enable_state() != SCX_OPS_DISABLED) { -- ret = -EBUSY; -- goto err_unlock; -- } -- -- //slim_walt_enable(true); -- -- /* -- * Set scx_ops, transition to PREPPING and clear exit info to arm the -- * disable path. Failure triggers full disabling from here on. -- */ -- scx_ops = *ops; -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != -- SCX_OPS_DISABLED); -- -- memset(&scx_exit_info, 0, sizeof(scx_exit_info)); -- atomic_set(&scx_exit_type, SCX_EXIT_NONE); -- scx_warned_zero_slice = false; -- -- atomic64_set(&scx_nr_rejected, 0); -- -- /* -- * Keep CPUs stable during enable so that the BPF scheduler can track -- * online CPUs by watching ->on/offline_cpu() after ->init(). -- */ -- cpus_read_lock(); -- -- scx_switch_all_req = true; -- if (scx_ops.init) { -- ret = SCX_CALL_OP_RET(SCX_KF_INIT, init); -- if (ret) { -- ret = ops_sanitize_err("init", ret); -- goto err_disable; -- } -- -- /* -- * Exit early if ops.init() triggered scx_bpf_error(). Not -- * strictly necessary as we'll fail transitioning into ENABLING -- * later but that'd be after calling ops.prep_enable() on all -- * tasks and with -EBUSY which isn't very intuitive. Let's exit -- * early with success so that the condition is notified through -- * ops.exit() like other scx_bpf_error() invocations. -- */ -- if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE) -- goto err_disable; -- } -- -- WARN_ON_ONCE(scx_dsp_buf); -- scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; -- scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch, -- __alignof__(scx_dsp_buf[0])); -- if (!scx_dsp_buf) { -- ret = -ENOMEM; -- goto err_disable; -- } -- -- scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; -- if (ops->timeout_ms) -- scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); -- -- scx_watchdog_timestamp = jiffies; -- queue_delayed_work(system_unbound_wq, &scx_watchdog_work, -- scx_watchdog_timeout / 2); -- -- /* -- * Lock out forks, cgroup on/offlining and moves before opening the -- * floodgate so that they don't wander into the operations prematurely. -- */ -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- if (((void (**)(void))ops)[i]) -- static_branch_enable_cpuslocked(&scx_has_op[i]); -- -- if (ops->flags & SCX_OPS_ENQ_LAST) -- static_branch_enable_cpuslocked(&scx_ops_enq_last); -- -- if (ops->flags & SCX_OPS_ENQ_EXITING) -- static_branch_enable_cpuslocked(&scx_ops_enq_exiting); -- if (scx_ops.cpu_acquire || scx_ops.cpu_release) -- static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); -- -- if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { -- reset_idle_masks(); -- static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); -- } else { -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- } -- -- /* -- * All cgroups should be initialized before letting in tasks. cgroup -- * on/offlining and task migrations are already locked out. -- */ -- ret = scx_cgroup_init(); -- if (ret) -- goto err_disable_unlock; -- -- static_branch_enable_cpuslocked(&__scx_ops_enabled); -- -- /* -- * Enable ops for every task. Fork is excluded by scx_fork_rwsem -- * preventing new tasks from being added. No need to exclude tasks -- * leaving as sched_ext_free() can handle both prepped and enabled -- * tasks. Prep all tasks first and then enable them with preemption -- * disabled. -- */ -- spin_lock_irq(&scx_tasks_lock); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered(&sti))) { -- get_task_struct(p); -- spin_unlock_irq(&scx_tasks_lock); -- -- ret = scx_ops_prepare_task(p, task_group(p)); -- if (ret) { -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n", -- ret, p->comm, p->pid); -- goto err_disable_unlock; -- } -- -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- } -- scx_task_iter_exit(&sti); -- -- /* -- * All tasks are prepped but are still ops-disabled. Ensure that -- * %current can't be scheduled out and switch everyone. -- * preempt_disable() is necessary because we can't guarantee that -- * %current won't be starved if scheduled out while switching. -- */ -- preempt_disable(); -- -- /* -- * From here on, the disable path must assume that tasks have ops -- * enabled and need to be recovered. -- */ -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { -- preempt_enable(); -- spin_unlock_irq(&scx_tasks_lock); -- ret = -EBUSY; -- goto err_disable_unlock; -- } -- -- /* -- * We're fully committed and can't fail. The PREPPED -> ENABLED -- * transitions here are synchronized against sched_ext_free() through -- * scx_tasks_lock. -- */ -- WRITE_ONCE(scx_switching_all, scx_switch_all_req); -- start = sched_clock(); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- tcnt++; -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- scx_ops_enable_task(p); -- __setscheduler_prio(p, p->prio); -- } -- -- check_class_changed(task_rq(p), p, old_class, p->prio); -- } else { -- scx_ops_disable_task(p); -- } -- } -- printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id()); -- scx_task_iter_exit(&sti); -- -- spin_unlock_irq(&scx_tasks_lock); -- preempt_enable(); -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { -- ret = -EBUSY; -- goto err_disable; -- } -- -- if (scx_switch_all_req) -- static_branch_enable_cpuslocked(&__scx_switched_all); -- -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- -- scx_cgroup_config_knobs(); -- printk("\n total duration = %llu , current cpu = %d\n", sched_clock() - total_start, smp_processor_id()); -- -- return 0; -- --err_unlock: -- mutex_unlock(&scx_ops_enable_mutex); -- return ret; -- --err_disable_unlock: -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); --err_disable: -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- /* must be fully disabled before returning */ -- scx_ops_disable(SCX_EXIT_ERROR); -- kthread_flush_work(&scx_ops_disable_work); -- return ret; --} -- --#ifdef CONFIG_SCHED_DEBUG --static const char *scx_ops_enable_state_str[] = { -- [SCX_OPS_PREPPING] = "prepping", -- [SCX_OPS_ENABLING] = "enabling", -- [SCX_OPS_ENABLED] = "enabled", -- [SCX_OPS_DISABLING] = "disabling", -- [SCX_OPS_DISABLED] = "disabled", --}; -- --static int scx_debug_show(struct seq_file *m, void *v) --{ -- mutex_lock(&scx_ops_enable_mutex); -- seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name); -- seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled()); -- seq_printf(m, "%-30s: %d\n", "switching_all", -- READ_ONCE(scx_switching_all)); -- seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all()); -- seq_printf(m, "%-30s: %s\n", "enable_state", -- scx_ops_enable_state_str[scx_ops_enable_state()]); -- seq_printf(m, "%-30s: %llu\n", "nr_rejected", -- atomic64_read(&scx_nr_rejected)); -- mutex_unlock(&scx_ops_enable_mutex); -- return 0; --} -- --static int scx_debug_open(struct inode *inode, struct file *file) --{ -- return single_open(file, scx_debug_show, NULL); --} -- --const struct file_operations sched_ext_fops = { -- .open = scx_debug_open, -- .read = seq_read, -- .llseek = seq_lseek, -- .release = single_release, --}; --#endif -- --/******************************************************************************** -- * bpf_struct_ops plumbing. -- */ --#include --#include --#include -- --extern struct btf *btf_vmlinux; --static const struct btf_type *task_struct_type; -- --static bool bpf_scx_is_valid_access(int off, int size, -- enum bpf_access_type type, -- const struct bpf_prog *prog, -- struct bpf_insn_access_aux *info) --{ -- if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) -- return false; -- if (type != BPF_READ) -- return false; -- if (off % size != 0) -- return false; -- -- return btf_ctx_access(off, size, type, prog, info); --} -- --static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, -- const struct bpf_reg_state *reg, -- int off, int size) --{ -- return 0; --} -- --static const struct bpf_func_proto * --bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) --{ -- switch (func_id) { -- case BPF_FUNC_task_storage_get: -- return &bpf_task_storage_get_proto; -- case BPF_FUNC_task_storage_delete: -- return &bpf_task_storage_delete_proto; -- default: -- return bpf_base_func_proto(func_id); -- } --} -- --const struct bpf_verifier_ops bpf_scx_verifier_ops = { -- .get_func_proto = bpf_scx_get_func_proto, -- .is_valid_access = bpf_scx_is_valid_access, -- .btf_struct_access = bpf_scx_btf_struct_access, --}; -- --static int bpf_scx_init_member(const struct btf_type *t, -- const struct btf_member *member, -- void *kdata, const void *udata) --{ -- const struct sched_ext_ops *uops = udata; -- struct sched_ext_ops *ops = kdata; -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- int ret; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, dispatch_max_batch): -- if (*(u32 *)(udata + moff) > INT_MAX) -- return -E2BIG; -- ops->dispatch_max_batch = *(u32 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, flags): -- if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) -- return -EINVAL; -- ops->flags = *(u64 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, name): -- ret = bpf_obj_name_cpy(ops->name, uops->name, -- sizeof(ops->name)); -- if (ret < 0) -- return ret; -- if (ret == 0) -- return -EINVAL; -- return 1; -- case offsetof(struct sched_ext_ops, timeout_ms): -- if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT) -- return -E2BIG; -- ops->timeout_ms = *(u32 *)(udata + moff); -- return 1; -- } -- -- return 0; --} -- --static int bpf_scx_check_member(const struct btf_type *t, -- const struct btf_member *member, -- const struct bpf_prog *prog) --{ -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, prep_enable): -- case offsetof(struct sched_ext_ops, init): -- case offsetof(struct sched_ext_ops, exit): -- break; -- default: -- /* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */ -- /* if (prog->aux->sleepable) -- return -EINVAL; */ -- break; -- } -- -- return 0; --} -- --static int bpf_scx_reg(void *kdata) --{ -- return scx_ops_enable(kdata); --} -- --static void bpf_scx_unreg(void *kdata) --{ -- scx_ops_disable(SCX_EXIT_UNREG); -- kthread_flush_work(&scx_ops_disable_work); --} -- --static int bpf_scx_init(struct btf *btf) --{ -- u32 type_id; -- -- type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); -- if (type_id < 0) -- return -EINVAL; -- task_struct_type = btf_type_by_id(btf, type_id); -- -- return 0; --} -- --/* "extern" to avoid sparse warning, only used in this file */ --extern struct bpf_struct_ops bpf_sched_ext_ops; -- --struct bpf_struct_ops bpf_sched_ext_ops = { -- .verifier_ops = &bpf_scx_verifier_ops, -- .reg = bpf_scx_reg, -- .unreg = bpf_scx_unreg, -- .check_member = bpf_scx_check_member, -- .init_member = bpf_scx_init_member, -- .init = bpf_scx_init, -- .name = "sched_ext_ops", --}; -- --static void sysrq_handle_sched_ext_reset(u8 key) --{ -- if (scx_ops_helper) -- scx_ops_disable(SCX_EXIT_SYSRQ); -- else -- pr_info("sched_ext: BPF scheduler not yet used\n"); --} -- --static const struct sysrq_key_op sysrq_sched_ext_reset_op = { -- .handler = sysrq_handle_sched_ext_reset, -- .help_msg = "reset-sched-ext(S)", -- .action_msg = "Disable sched_ext and revert all tasks to CFS", -- .enable_mask = SYSRQ_ENABLE_RTNICE, --}; -- --static void kick_cpus_irq_workfn(struct irq_work *irq_work) --{ -- struct rq *this_rq = this_rq(); -- u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); -- int this_cpu = cpu_of(this_rq); -- int cpu; -- -- for_each_cpu(cpu, this_rq->scx->cpus_to_kick) { -- struct rq *rq = cpu_rq(cpu); -- unsigned long flags; -- -- raw_spin_rq_lock_irqsave(rq, flags); -- -- if (cpu_online(cpu) || cpu == this_cpu) { -- if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) && -- rq->curr->sched_class == &ext_sched_class) -- rq->curr->scx->slice = 0; -- pseqs[cpu] = rq->scx->pnt_seq; -- resched_curr(rq); -- } else { -- cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait); -- } -- -- raw_spin_rq_unlock_irqrestore(rq, flags); -- } -- -- for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait, -- cpumask_of(this_cpu)) { -- /* -- * Pairs with smp_store_release() issued by this CPU in -- * scx_notify_pick_next_task() on the resched path. -- * -- * We busy-wait here to guarantee that no other task can be -- * scheduled on our core before the target CPU has entered the -- * resched path. -- */ -- while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu]) -- cpu_relax(); -- } -- -- cpumask_clear(this_rq->scx->cpus_to_kick); -- cpumask_clear(this_rq->scx->cpus_to_preempt); -- cpumask_clear(this_rq->scx->cpus_to_wait); --} -- --void __init init_sched_ext_class(void) --{ -- int cpu; -- u32 v; -- -- /* -- * The following is to prevent the compiler from optimizing out the enum -- * definitions so that BPF scheduler implementations can use them -- * through the generated vmlinux.h. -- */ -- WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | -- SCX_TG_ONLINE | SCX_KICK_PREEMPT); -- -- init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); --#ifdef CONFIG_SMP -- BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -- BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); --#endif -- scx_kick_cpus_pnt_seqs = -- __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * -- num_possible_cpus(), -- __alignof__(scx_kick_cpus_pnt_seqs[0])); -- BUG_ON(!scx_kick_cpus_pnt_seqs); -- -- for_each_possible_cpu(cpu) { -- struct rq *rq = cpu_rq(cpu); -- -- rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL); -- if (rq->scx) -- rq->scx->rq = rq; -- else -- pr_err("fatal error : alloc rq->scx failed!!!\n"); -- -- init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL); -- INIT_LIST_HEAD(&rq->scx->watchdog_list); -- -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL)); -- init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn); -- } -- -- register_sysrq_key('S', &sysrq_sched_ext_reset_op); -- INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); -- scx_cgroup_config_knobs(); --} -- -- --/******************************************************************************** -- * Helpers that can be called from the BPF scheduler. -- */ --#include -- --/* Disables missing prototype warnings for kfuncs */ --__diag_push(); --__diag_ignore_all("-Wmissing-prototypes", -- "Global functions as their definitions will be in vmlinux BTF"); -- --/** -- * scx_bpf_switch_all - Switch all tasks into SCX -- * @into_scx: switch direction -- * -- * If @into_scx is %true, all existing and future non-dl/rt tasks are switched -- * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on -- * SCX. The actual switching is asynchronous. Can be called from ops.init(). -- */ --void scx_bpf_switch_all(void) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return; -- -- scx_switch_all_req = true; --} -- --BTF_SET8_START(scx_kfunc_ids_init) --BTF_ID_FLAGS(func, scx_bpf_switch_all) --BTF_SET8_END(scx_kfunc_ids_init) -- --static const struct btf_kfunc_id_set scx_kfunc_set_init = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_init, --}; -- --/** -- * scx_bpf_create_dsq - Create a custom DSQ -- * @dsq_id: DSQ to create -- * @node: NUMA node to allocate from -- * -- * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(), -- * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move(). -- */ --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return -EINVAL; -- -- if (unlikely(node >= (int)nr_node_ids || -- (node < 0 && node != NUMA_NO_NODE))) -- return -EINVAL; -- return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); --} -- --BTF_SET8_START(scx_kfunc_ids_sleepable) --BTF_ID_FLAGS(func, scx_bpf_create_dsq) --BTF_SET8_END(scx_kfunc_ids_sleepable) -- --static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_sleepable, --}; -- --static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) --{ -- if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) -- return false; -- -- lockdep_assert_irqs_disabled(); -- -- if (unlikely(!p)) { -- scx_ops_error("called with NULL task"); -- return false; -- } -- -- if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { -- scx_ops_error("invalid enq_flags 0x%llx", enq_flags); -- return false; -- } -- -- return true; --} -- --static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) --{ -- struct task_struct *ddsp_task; -- int idx; -- -- ddsp_task = __this_cpu_read(direct_dispatch_task); -- if (ddsp_task) { -- direct_dispatch(ddsp_task, p, dsq_id, enq_flags); -- return; -- } -- -- idx = __this_cpu_read(scx_dsp_ctx.buf_cursor); -- if (unlikely(idx >= scx_dsp_max_batch)) { -- scx_ops_error("dispatch buffer overflow"); -- return; -- } -- -- this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){ -- .task = p, -- .qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK, -- .dsq_id = dsq_id, -- .enq_flags = enq_flags, -- }; -- __this_cpu_inc(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe -- * to call this function spuriously. Can be called from ops.enqueue() and -- * ops.dispatch(). -- * -- * When called from ops.enqueue(), it's for direct dispatch and @p must match -- * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the -- * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be -- * on the target CPU in the first place. -- * -- * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id -- * and this function can be called upto ops.dispatch_max_batch times to dispatch -- * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the -- * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. -- * -- * This function doesn't have any locking restrictions and may be called under -- * BPF locks (in the future when BPF introduces more flexible locking). -- * -- * @p is allowed to run for @slice. The scheduling path is triggered on slice -- * exhaustion. If zero, the current residual slice is maintained. If -- * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with -- * scx_bpf_kick_cpu() to trigger scheduling. -- */ --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- scx_dispatch_commit(p, dsq_id, enq_flags); --} -- --/** -- * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. -- * Tasks queued into the priority queue are ordered by @vtime and always -- * consumed after the tasks in the FIFO queue. All other aspects are identical -- * to scx_bpf_dispatch(). -- * -- * @vtime ordering is according to time_before64() which considers wrapping. A -- * numerically larger vtime may indicate an earlier position in the ordering and -- * vice-versa. -- */ --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 vtime, u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- p->scx->dsq_vtime = vtime; -- -- scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); --} -- --BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime) --BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_enqueue_dispatch, --}; -- --/** -- * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots -- * -- * Can only be called from ops.dispatch(). -- */ --u32 scx_bpf_dispatch_nr_slots(void) --{ -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return 0; -- -- return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ -- * @dsq_id: DSQ to consume -- * -- * Consume a task from the non-local DSQ identified by @dsq_id and transfer it -- * to the current CPU's local DSQ for execution. Can only be called from -- * ops.dispatch(). -- * -- * This function flushes the in-flight dispatches from scx_bpf_dispatch() before -- * trying to consume the specified DSQ. It may also grab rq locks and thus can't -- * be called under any BPF locks. -- * -- * Returns %true if a task has been consumed, %false if there isn't any task to -- * consume. -- */ --bool scx_bpf_consume(u64 dsq_id) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- struct scx_dispatch_q *dsq; -- -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return false; -- -- flush_dispatch_buf(dspc->rq, dspc->rf); -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); -- return false; -- } -- -- if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) { -- /* -- * A successfully consumed task can be dequeued before it starts -- * running while the CPU is trying to migrate other dispatched -- * tasks. Bump nr_tasks to tell balance_scx() to retry on empty -- * local DSQ. -- */ -- dspc->nr_tasks++; -- return true; -- } else { -- return false; -- } --} -- --BTF_SET8_START(scx_kfunc_ids_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) --BTF_ID_FLAGS(func, scx_bpf_consume) --BTF_SET8_END(scx_kfunc_ids_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_dispatch, --}; -- --/** -- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ -- * -- * Iterate over all of the tasks currently enqueued on the local DSQ of the -- * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of -- * processed tasks. Can only be called from ops.cpu_release(). -- */ --u32 scx_bpf_reenqueue_local(void) --{ -- u32 nr_enqueued, i; -- struct rq *rq; -- struct scx_rq *scx_rq; -- -- if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) -- return 0; -- -- rq = cpu_rq(smp_processor_id()); -- lockdep_assert_rq_held(rq); -- scx_rq = rq->scx; -- -- /* -- * Get the number of tasks on the local DSQ before iterating over it to -- * pull off tasks. The enqueue callback below can signal that it wants -- * the task to stay on the local DSQ, and we want to prevent the BPF -- * scheduler from causing us to loop indefinitely. -- */ -- nr_enqueued = scx_rq->local_dsq.nr; -- for (i = 0; i < nr_enqueued; i++) { -- struct task_struct *p; -- -- p = first_local_task(rq); -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- WARN_ON_ONCE(p->scx->holding_cpu != -1); -- dispatch_dequeue(scx_rq, p); -- do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); -- } -- -- return nr_enqueued; --} -- --BTF_SET8_START(scx_kfunc_ids_cpu_release) --BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) --BTF_SET8_END(scx_kfunc_ids_cpu_release) -- --static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_cpu_release, --}; -- --/** -- * scx_bpf_kick_cpu - Trigger reschedule on a CPU -- * @cpu: cpu to kick -- * @flags: SCX_KICK_* flags -- * -- * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or -- * trigger rescheduling on a busy CPU. This can be called from any online -- * scx_ops operation and the actual kicking is performed asynchronously through -- * an irq work. -- */ --void scx_bpf_kick_cpu(s32 cpu, u64 flags) --{ -- struct rq *rq; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d", cpu); -- return; -- } -- -- preempt_disable(); -- rq = this_rq(); -- -- /* -- * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting -- * rq locks. We can probably be smarter and avoid bouncing if called -- * from ops which don't hold a rq lock. -- */ -- cpumask_set_cpu(cpu, rq->scx->cpus_to_kick); -- if (flags & SCX_KICK_PREEMPT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt); -- if (flags & SCX_KICK_WAIT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_wait); -- -- irq_work_queue(&rq->scx->kick_cpus_irq_work); -- preempt_enable(); --} -- --/** -- * scx_bpf_dsq_nr_queued - Return the number of queued tasks -- * @dsq_id: id of the DSQ -- * -- * Return the number of tasks in the DSQ matching @dsq_id. If not found, -- * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops -- * operations. -- */ --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) --{ -- struct scx_dispatch_q *dsq; -- -- lockdep_assert(rcu_read_lock_any_held()); -- -- if (dsq_id == SCX_DSQ_LOCAL) { -- return this_rq()->scx->local_dsq.nr; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (ops_cpu_valid(cpu)) -- return cpu_rq(cpu)->scx->local_dsq.nr; -- } else { -- dsq = find_non_local_dsq(dsq_id); -- if (dsq) -- return dsq->nr; -- } -- return -ENOENT; --} -- --/** -- * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state -- * @cpu: cpu to test and clear idle for -- * -- * Returns %true if @cpu was idle and its idle state was successfully cleared. -- * %false otherwise. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return false; -- } -- -- if (ops_cpu_valid(cpu)) -- return test_and_clear_cpu_idle(cpu); -- else -- return false; --} -- --/** -- * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu -- * @cpus_allowed: Allowed cpumask -- * -- * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked -- * idle cpu number on success. -%EBUSY if no matching cpu was found. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return -EBUSY; -- } -- -- return scx_pick_idle_cpu(cpus_allowed); --} -- --/** -- * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking -- * per-CPU cpumask. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_cpumask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.cpu; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, -- * per-physical-core cpumask. Can be used to determine if an entire physical -- * core is free. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_smtmask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.smt; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to -- * either the percpu, or SMT idle-tracking cpumask. -- */ --void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) --{ -- /* -- * Empty function body because we aren't actually acquiring or -- * releasing a reference to a global idle cpumask, which is read-only -- * in the caller and is never released. The acquire / release semantics -- * here are just used to make the cpumask is a trusted pointer in the -- * caller. -- */ --} -- --struct scx_bpf_error_bstr_bufs { -- u64 data[MAX_BPRINTF_VARARGS]; -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs); -- --/** -- * scx_bpf_error_bstr - Indicate fatal error -- * @fmt: error message format string -- * @data: format string parameters packaged using ___bpf_fill() macro -- * @data__sz: @data len, must end in '__sz' for the verifier -- * -- * Indicate that the BPF scheduler encountered a fatal error and initiate ops -- * disabling. -- */ --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) --{ -- struct scx_bpf_error_bstr_bufs *bufs; -- unsigned long flags; -- struct bpf_bprintf_data args = {}; -- int ret; -- -- local_irq_save(flags); -- bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs); -- -- if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || -- (data__sz && !data)) { -- scx_ops_error("invalid data=%p and data__sz=%u", -- (void *)data, data__sz); -- goto out_restore; -- } -- -- ret = copy_from_kernel_nofault(bufs->data, data, data__sz); -- if (ret) { -- scx_ops_error("failed to read data fields (%d)", ret); -- goto out_restore; -- } -- -- ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args); -- if (ret < 0) { -- scx_ops_error("failed to format prepration (%d)", ret); -- goto out_restore; -- } -- -- ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt, -- args.bin_args); -- bpf_bprintf_cleanup(&args); -- if (ret < 0) { -- scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format", -- fmt, data, data__sz); -- goto out_restore; -- } -- -- scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg); --out_restore: -- local_irq_restore(flags); --} -- --/** -- * scx_bpf_destroy_dsq - Destroy a custom DSQ -- * @dsq_id: DSQ to destroy -- * -- * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with -- * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is -- * empty and no further tasks are dispatched to it. Ignored if called on a DSQ -- * which doesn't exist. Can be called from any online scx_ops operations. -- */ --void scx_bpf_destroy_dsq(u64 dsq_id) --{ -- destroy_dsq(dsq_id); --} -- --/** -- * scx_bpf_task_running - Is task currently running? -- * @p: task of interest -- */ --bool scx_bpf_task_running(const struct task_struct *p) --{ -- return task_rq(p)->curr == p; --} -- --/** -- * scx_bpf_task_cpu - CPU a task is currently associated with -- * @p: task of interest -- */ --s32 scx_bpf_task_cpu(const struct task_struct *p) --{ -- return task_cpu(p); --} -- --/** -- * scx_bpf_task_cgroup - Return the sched cgroup of a task -- * @p: task of interest -- * -- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with -- * from the scheduler's POV. SCX operations should use this function to -- * determine @p's current cgroup as, unlike following @p->cgroups, -- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all -- * rq-locked operations. Can be called on the parameter tasks of rq-locked -- * operations. The restriction guarantees that @p's rq is locked by the caller. -- */ --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) --{ -- struct task_group *tg = p->sched_task_group; -- struct cgroup *cgrp = &cgrp_dfl_root.cgrp; -- -- if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) -- goto out; -- -- /* -- * A task_group may either be a cgroup or an autogroup. In the latter -- * case, @tg->css.cgroup is %NULL. A task_group can't become the other -- * kind once created. -- */ -- if (tg && tg->css.cgroup) -- cgrp = tg->css.cgroup; -- else -- cgrp = &cgrp_dfl_root.cgrp; --out: -- cgroup_get(cgrp); -- return cgrp; --} -- --BTF_SET8_START(scx_kfunc_ids_any) --BTF_ID_FLAGS(func, scx_bpf_kick_cpu) --BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) --BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) --BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu) --BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) --BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) --BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) --BTF_ID_FLAGS(func, scx_bpf_task_running) --BTF_ID_FLAGS(func, scx_bpf_task_cpu) --BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE) --BTF_SET8_END(scx_kfunc_ids_any) -- --static const struct btf_kfunc_id_set scx_kfunc_set_any = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_any, --}; -- --__diag_pop(); -- --/* -- * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set() -- * needs most of the system to be up. -- */ --static int __init register_ext_kfuncs(void) --{ -- int ret; -- -- /* -- * Some kfuncs are context-sensitive and can only be called from -- * specific SCX ops. They are grouped into BTF sets accordingly. -- * Unfortunately, BPF currently doesn't have a way of enforcing such -- * restrictions. Eventually, the verifier should be able to enforce -- * them. For now, register them the same and make each kfunc explicitly -- * check using scx_kf_allowed(). -- */ -- if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_init)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_sleepable)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_enqueue_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_cpu_release)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_any))) { -- pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret); -- return ret; -- } -- -- return 0; --} --__initcall(register_ext_kfuncs); -diff --git a/kernel/sched/ext.h b/kernel/sched/ext.h -deleted file mode 100755 -index fba8ed845f99..000000000000 ---- a/kernel/sched/ext.h -+++ /dev/null -@@ -1,257 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ -- -- --/* -- * Tag marking a kernel function as a kfunc. This is meant to minimize the -- * amount of copy-paste that kfunc authors have to include for correctness so -- * as to avoid issues such as the compiler inlining or eliding either a static -- * kfunc, or a global kfunc in an LTO build. -- */ --#define __bpf_kfunc __used noinline -- --enum scx_wake_flags { -- /* expose select WF_* flags as enums */ -- SCX_WAKE_EXEC = WF_EXEC, -- SCX_WAKE_FORK = WF_FORK, -- SCX_WAKE_TTWU = WF_TTWU, -- SCX_WAKE_SYNC = WF_SYNC, --}; -- --enum scx_enq_flags { -- /* expose select ENQUEUE_* flags as enums */ -- SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, -- SCX_ENQ_HEAD = ENQUEUE_HEAD, -- -- /* high 32bits are SCX specific */ -- -- /* -- * Set the following to trigger preemption when calling -- * scx_bpf_dispatch() with a local dsq as the target. The slice of the -- * current task is cleared to zero and the CPU is kicked into the -- * scheduling path. Implies %SCX_ENQ_HEAD. -- */ -- SCX_ENQ_PREEMPT = 1LLU << 32, -- -- /* -- * The task being enqueued was previously enqueued on the current CPU's -- * %SCX_DSQ_LOCAL, but was removed from it in a call to the -- * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was -- * invoked in a ->cpu_release() callback, and the task is again -- * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the -- * task will not be scheduled on the CPU until at least the next invocation -- * of the ->cpu_acquire() callback. -- */ -- SCX_ENQ_REENQ = 1LLU << 40, -- -- /* -- * The task being enqueued is the only task available for the cpu. By -- * default, ext core keeps executing such tasks but when -- * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with -- * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set. -- * -- * If the BPF scheduler wants to continue executing the task, -- * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately. -- * If the task gets queued on a different dsq or the BPF side, the BPF -- * scheduler is responsible for triggering a follow-up scheduling event. -- * Otherwise, Execution may stall. -- */ -- SCX_ENQ_LAST = 1LLU << 41, -- -- /* -- * A hint indicating that it's advisable to enqueue the task on the -- * local dsq of the currently selected CPU. Currently used by -- * select_cpu_dfl() and together with %SCX_ENQ_LAST. -- */ -- SCX_ENQ_LOCAL = 1LLU << 42, -- -- /* high 8 bits are internal */ -- __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, -- -- SCX_ENQ_CLEAR_OPSS = 1LLU << 56, -- SCX_ENQ_DSQ_PRIQ = 1LLU << 57, --}; -- --enum scx_deq_flags { -- /* expose select DEQUEUE_* flags as enums */ -- SCX_DEQ_SLEEP = DEQUEUE_SLEEP, -- -- /* high 32bits are SCX specific */ -- -- /* -- * The generic core-sched layer decided to execute the task even though -- * it hasn't been dispatched yet. Dequeue from the BPF side. -- */ -- SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, --}; -- --enum scx_tg_flags { -- SCX_TG_ONLINE = 1U << 0, -- SCX_TG_INITED = 1U << 1, --}; -- --enum scx_kick_flags { -- SCX_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -- SCX_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ --}; -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --extern const struct sched_class ext_sched_class; --extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops; --extern const struct file_operations sched_ext_fops; --extern unsigned long scx_watchdog_timeout; --extern unsigned long scx_watchdog_timestamp; -- --DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); --DECLARE_STATIC_KEY_FALSE(__scx_switched_all); --#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) --#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) -- --DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); -- --bool task_on_scx(struct task_struct *p); --void scx_pre_fork(struct task_struct *p); --int scx_fork(struct task_struct *p); --void scx_post_fork(struct task_struct *p); --void scx_cancel_fork(struct task_struct *p); --int scx_check_setscheduler(struct task_struct *p, int policy); --bool scx_can_stop_tick(struct rq *rq); --void init_sched_ext_class(void); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...); --#define scx_ops_error(fmt, args...) \ -- scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args) -- --void __scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active); -- --static inline void scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active) --{ -- if (!scx_enabled()) -- return; --#ifdef CONFIG_SMP -- /* -- * Pairs with the smp_load_acquire() issued by a CPU in -- * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -- * resched. -- */ -- smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1); --#endif -- if (!static_branch_unlikely(&scx_ops_cpu_preempt)) -- return; -- __scx_notify_pick_next_task(rq, p, active); --} -- --//extern void scx_scheduler_tick(void); --static inline void scx_notify_sched_tick(void) --{ -- unsigned long last_check; -- -- if (!scx_enabled()) -- return; -- //scx_scheduler_tick(); -- -- last_check = scx_watchdog_timestamp; -- if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "watchdog failed to check in for %u.%03us", -- dur_ms / 1000, dur_ms % 1000); -- } --} -- --static inline const struct sched_class *next_active_class(const struct sched_class *class) --{ -- class++; -- if (scx_switched_all() && class == &fair_sched_class) -- class++; -- if (!scx_enabled() && class == &ext_sched_class) -- class++; -- return class; --} -- --#define for_active_class_range(class, _from, _to) \ -- for (class = (_from); class != (_to); class = next_active_class(class)) -- --#define for_each_active_class(class) \ -- for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -- --/* -- * SCX requires a balance() call before every pick_next_task() call including -- * when waking up from idle. -- */ --#define for_balance_class_range(class, prev_class, end_class) \ -- for_active_class_range(class, (prev_class) > &ext_sched_class ? \ -- &ext_sched_class : (prev_class), (end_class)) -- --#ifdef CONFIG_SCHED_CORE --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi); --#endif -- --#else /* CONFIG_SCHED_CLASS_EXT */ -- --#define scx_enabled() false --#define scx_switched_all() false -- --static inline void scx_pre_fork(struct task_struct *p) {} --static inline int scx_fork(struct task_struct *p) { return 0; } --static inline void scx_post_fork(struct task_struct *p) {} --static inline void scx_cancel_fork(struct task_struct *p) {} --static inline int scx_check_setscheduler(struct task_struct *p, -- int policy) { return 0; } --static inline bool scx_can_stop_tick(struct rq *rq) { return true; } --static inline void init_sched_ext_class(void) {} --static inline void scx_notify_pick_next_task(struct rq *rq, -- const struct task_struct *p, -- const struct sched_class *active) {} --static inline void scx_notify_sched_tick(void) {} -- --#define for_each_active_class for_each_class --#define for_balance_class_range for_class_range -- --#endif /* CONFIG_SCHED_CLASS_EXT */ -- --#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) --void __scx_update_idle(struct rq *rq, bool idle); -- --static inline void scx_update_idle(struct rq *rq, bool idle) --{ -- if (scx_enabled()) -- __scx_update_idle(rq, idle); --} --#else --static inline void scx_update_idle(struct rq *rq, bool idle) {} --#endif -- --#ifdef CONFIG_CGROUP_SCHED --#ifdef CONFIG_EXT_GROUP_SCHED --int scx_tg_online(struct task_group *tg); --void scx_tg_offline(struct task_group *tg); --int scx_cgroup_can_attach(struct cgroup_taskset *tset); --void scx_move_task(struct task_struct *p); --void scx_cgroup_finish_attach(void); --void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); --void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); --#else /* CONFIG_EXT_GROUP_SCHED */ --static inline int scx_tg_online(struct task_group *tg) { return 0; } --static inline void scx_tg_offline(struct task_group *tg) {} --static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } --static inline void scx_move_task(struct task_struct *p) {} --static inline void scx_cgroup_finish_attach(void) {} --static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} --static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} --#endif /* CONFIG_EXT_GROUP_SCHED */ --#endif /* CONFIG_CGROUP_SCHED */ -diff --git a/kernel/sched/hmbird.h b/kernel/sched/hmbird.h -new file mode 100755 -index 000000000000..d0d84ddee13d ---- /dev/null -+++ b/kernel/sched/hmbird.h -@@ -0,0 +1,343 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#ifndef _HMBIRD_H_ -+#define _HMBIRD_H_ -+ -+/* -+ * Tag marking a kernel function as a kfunc. This is meant to minimize the -+ * amount of copy-paste that kfunc authors have to include for correctness so -+ * as to avoid issues such as the compiler inlining or eliding either a static -+ * kfunc, or a global kfunc in an LTO build. -+ */ -+ -+#define DEBUG_INTERNAL (1 << 0) -+#define DEBUG_INFO_TRACE (1 << 1) -+#define DEBUG_INFO_SYSTRACE (1 << 2) -+ -+#define NUMS_CGROUP_KINDS (512) -+#define SLIM_FOR_SGAME (1 << 0) -+#define SLIM_FOR_GENSHIN (1 << 1) -+ -+#ifdef MAX_TASK_NR -+#define MAX_KEY_THREAD_RECORD ((MAX_TASK_NR + 1) >> 1) -+#else -+#define MAX_KEY_THREAD_RECORD (1 << 3) -+#endif /* MAX_TASK_NR */ -+ -+#define TOP_TASK_SHIFT (8) -+#define TOP_TASK_MAX (1 << TOP_TASK_SHIFT) -+#ifndef TOP_TASK_BITS_MASK -+#define TOP_TASK_BITS_MASK (TOP_TASK_MAX - 1) -+#endif /* TOP_TASK_BITS_MASK */ -+ -+extern struct md_info_t *md_info; -+ -+#define debug_enabled() \ -+ (unlikely(hmbirdcore_debug)) -+ -+#define hmbird_debug(fmt, ...) \ -+ pr_info(":"fmt, ##__VA_ARGS__) -+ -+#define hmbird_err(id, fmt, ...) \ -+do { \ -+ pr_err(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_deferred_err(id, fmt, ...) \ -+do { \ -+ printk_deferred(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \ -+do { \ -+ if (unlikely(cond)) { \ -+ hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+ } \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_info_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_info_trace(fmt, ...) -+#endif -+ -+#define hmbird_info_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+ } \ -+} while (0) -+ -+#define hmbird_output_systrace(fmt, ...) \ -+do { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_internal_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_internal_trace(fmt, ...) -+#endif -+ -+#define hmbird_internal_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) { \ -+ hmbird_output_systrace(fmt, ##__VA_ARGS__); \ -+ } \ -+} while (0) -+ -+enum hmbird_wake_flags { -+ /* expose select WF_* flags as enums */ -+ HMBIRD_WAKE_EXEC = WF_EXEC, -+ HMBIRD_WAKE_FORK = WF_FORK, -+ HMBIRD_WAKE_TTWU = WF_TTWU, -+ HMBIRD_WAKE_SYNC = WF_SYNC, -+}; -+ -+enum hmbird_enq_flags { -+ /* expose select ENQUEUE_* flags as enums */ -+ HMBIRD_ENQ_WAKEUP = ENQUEUE_WAKEUP, -+ HMBIRD_ENQ_HEAD = ENQUEUE_HEAD, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * Set the following to trigger preemption when calling -+ * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the -+ * current task is cleared to zero and the CPU is kicked into the -+ * scheduling path. Implies %HMBIRD_ENQ_HEAD. -+ */ -+ HMBIRD_ENQ_PREEMPT = 1LLU << 32, -+ HMBIRD_ENQ_REENQ = 1LLU << 40, -+ HMBIRD_ENQ_LAST = 1LLU << 41, -+ -+ /* -+ * A hint indicating that it's advisable to enqueue the task on the -+ * local dsq of the currently selected CPU. Currently used by -+ * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST. -+ */ -+ HMBIRD_ENQ_LOCAL = 1LLU << 42, -+ -+ /* high 8 bits are internal */ -+ __HMBIRD_ENQ_INTERNAL_MASK = 0xffLLU << 56, -+ -+ HMBIRD_ENQ_CLEAR_OPSS = 1LLU << 56, -+ HMBIRD_ENQ_DSQ_PRIQ = 1LLU << 57, -+}; -+ -+enum hmbird_deq_flags { -+ /* expose select DEQUEUE_* flags as enums */ -+ HMBIRD_DEQ_SLEEP = DEQUEUE_SLEEP, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * The generic core-sched layer decided to execute the task even though -+ * it hasn't been dispatched yet. Dequeue from the BPF side. -+ */ -+ HMBIRD_DEQ_CORE_SCHED_EXEC = 1LLU << 32, -+}; -+ -+enum hmbird_kick_flags { -+ HMBIRD_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -+ HMBIRD_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ -+}; -+ -+enum hmbird_task_prop_type { -+ HMBIRD_TASK_PROP_COMMON, -+ HMBIRD_TASK_PROP_PIPELINE, -+ HMBIRD_TASK_PROP_DEBUG_OR_LOG, -+ HMBIRD_TASK_PROP_HIGH_LOAD, -+ HMBIRD_TASK_PROP_IO, -+ HMBIRD_TASK_PROP_NETWORK, -+ HMBIRD_TASK_PROP_PERIODIC, -+ HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL, -+ HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL, -+ HMBIRD_TASK_PROP_ISOLATE, -+ HMBIRD_TASK_PROP_MAX, -+}; -+ -+enum hmbird_preempt_policy_id { -+ HMBIRD_PREEMPT_POLICY_NONE, -+ HMBIRD_PREEMPT_POLICY_PRIO_BASED, -+}; -+ -+extern const struct sched_class hmbird_sched_class; -+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops; -+extern const struct file_operations sched_hmbird_fops; -+extern unsigned long hmbird_watchdog_timeout; -+extern unsigned long hmbird_watchdog_timestamp; -+ -+enum scx_rq_flags { -+ HMBIRD_RQ_CAN_STOP_TICK = 1 << 0, -+}; -+ -+struct hmbird_rq { -+ struct hmbird_dispatch_q local_dsq; -+ struct list_head watchdog_list; -+ u64 ops_qseq; -+ u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -+ u32 nr_running; -+ u32 flags; -+ bool cpu_released; -+ cpumask_var_t cpus_to_kick; -+ cpumask_var_t cpus_to_preempt; -+ cpumask_var_t cpus_to_wait; -+ u64 pnt_seq; -+ u64 *prev_runnable_sum_fixed; -+ u32 *prev_window_size; -+ struct irq_work kick_cpus_irq_work; -+ bool pipeline; -+ bool exclusive; -+ bool period_disallow; -+ bool nonperiod_disallow; -+ struct rq *rq; -+ struct hmbird_sched_rq_stats *srq; -+}; -+ -+struct hmbird_sched_change_guard { -+ struct task_struct *p; -+ struct rq *rq; -+ bool queued; -+ bool running; -+ bool done; -+}; -+ -+extern struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -+ -+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags); -+ -+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -+ for (struct hmbird_sched_change_guard __cg = \ -+ hmbird_sched_change_guard_init(__rq, __p, __flags); \ -+ !__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags)) -+ -+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+bool task_on_hmbird(struct task_struct *p); -+int hmbird_pre_fork(struct task_struct *p); -+int hmbird_fork(struct task_struct *p); -+void hmbird_post_fork(struct task_struct *p); -+void hmbird_cancel_fork(struct task_struct *p); -+int hmbird_check_setscheduler(struct task_struct *p, int policy); -+bool hmbird_can_stop_tick(struct rq *rq); -+void init_sched_hmbird_class(void); -+ -+void hmbird_ops_exit(void); -+#define hmbird_ops_error(fmt, ...) \ -+do { \ -+ hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__); \ -+ hmbird_ops_exit(); \ -+} while (0) -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active); -+ -+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight) -+{ -+ return clamp_t(unsigned long, -+ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -+ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); -+} -+ -+static inline void hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active) -+{ -+ if (!hmbird_enabled()) -+ return; -+#ifdef CONFIG_SMP -+ /* -+ * Pairs with the smp_load_acquire() issued by a CPU in -+ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -+ * resched. -+ */ -+ smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1); -+#endif -+ if (!static_branch_unlikely(&hmbird_ops_cpu_preempt)) -+ return; -+ __hmbird_notify_pick_next_task(rq, p, active); -+} -+ -+extern void hmbird_scheduler_tick(void); -+void scan_timeout(struct rq *rq); -+void hmbird_notify_sched_tick(void); -+ -+static inline const struct sched_class *next_active_class(const struct sched_class *class) -+{ -+ class++; -+ -+ if (hmbird_enabled() && class == &fair_sched_class) -+ class++; -+ if (!hmbird_enabled() && class == &hmbird_sched_class) -+ class++; -+ return class; -+} -+ -+#define for_active_class_range(class, _from, _to) \ -+ for (class = (_from); class != (_to); class = next_active_class(class)) -+ -+#define for_each_active_class(class) \ -+ for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -+ -+/* -+ * HMBIRD requires a balance() call before every pick_next_task() call including -+ * when waking up from idle. -+ */ -+#define for_balance_class_range(class, prev_class, end_class) \ -+ for_active_class_range(class, (prev_class) > &hmbird_sched_class ? \ -+ &hmbird_sched_class : (prev_class), (end_class)) -+ -+#define MIN_CGROUP_DL_IDX (5) /* 8ms */ -+#define DEFAULT_CGROUP_DL_IDX (8) /* 64ms */ -+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS]; -+ -+ -+void __hmbird_update_idle(struct rq *rq, bool idle); -+ -+static inline void hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ if (hmbird_enabled()) -+ __hmbird_update_idle(rq, idle); -+} -+ -+int hmbird_ctrl(bool enable); -+ -+int hmbird_tg_online(struct task_group *tg); -+ -+ -+static inline u16 hmbird_task_util(struct task_struct *p) -+{ -+ return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0; -+} -+u16 hmbird_cpu_util(int cpu); -+ -+bool get_hmbird_ops_enabled(void); -+bool get_non_hmbird_task(void); -+void set_cpu_cluster(u64 cpu_cluster); -+#endif /*_EXT_H_*/ -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -new file mode 100755 -index 000000000000..ce05928392bf ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird.c -@@ -0,0 +1,4313 @@ -+// SPDX-License-Identifier: GPL-2.0 -+ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#include -+#include -+ -+#include "slim.h" -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+#include -+ -+#define CLUSTER_SEPARATE -+ -+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0); -+atomic_t non_hmbird_task; -+atomic_t hmbird_module_loaded = ATOMIC_INIT(0); -+int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0}; -+ -+enum hmbird_internal_consts { -+ HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ, -+}; -+ -+enum hmbird_ops_enable_state { -+ HMBIRD_OPS_PREPPING, -+ HMBIRD_OPS_ENABLING, -+ HMBIRD_OPS_ENABLED, -+ HMBIRD_OPS_DISABLING, -+ HMBIRD_OPS_DISABLED, -+}; -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+/* -+ * hmbird_entity->ops_state -+ * -+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler. -+ * State transitions look as follows: -+ * -+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING -+ * ^ | | -+ * | v v -+ * \-------------------------------/ -+ * -+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -+ * sites for explanations on the conditions being waited upon and why they are -+ * safe. Transitions out of them into NONE or QUEUED must store_release and the -+ * waiters should load_acquire. -+ * -+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether -+ * any given task can be dispatched by the BPF scheduler at all times and thus -+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core -+ * can safely reject invalid dispatches. -+ */ -+enum hmbird_ops_state { -+ HMBIRD_OPSS_NONE, /* owned by the HMBIRD core */ -+ HMBIRD_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -+ HMBIRD_OPSS_QUEUED, /* owned by the BPF scheduler */ -+ HMBIRD_OPSS_DISPATCHING, /* in transit back to the HMBIRD core */ -+ -+ /* -+ * QSEQ brands each QUEUED instance so that, when dispatch races -+ * dequeue/requeue, the dispatcher can tell whether it still has a claim -+ * on the task being dispatched. -+ */ -+ HMBIRD_OPSS_QSEQ_SHIFT = 2, -+ HMBIRD_OPSS_STATE_MASK = (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1, -+ HMBIRD_OPSS_QSEQ_MASK = ~HMBIRD_OPSS_STATE_MASK, -+}; -+ -+enum switch_stat { -+ HMBIRD_DISABLED, -+ HMBIRD_SWITCH_PREP, -+ HMBIRD_RQ_SWITCH_BEGIN, -+ HMBIRD_RQ_SWITCH_DONE, -+ HMBIRD_ENABLED, -+}; -+enum switch_stat curr_ss; -+ -+/* -+ * During exit, a task may schedule after losing its PIDs. When disabling the -+ * BPF scheduler, we need to be able to iterate tasks in every state to -+ * guarantee system safety. Maintain a dedicated task list which contains every -+ * task between its fork and eventual free. -+ */ -+DEFINE_SPINLOCK(hmbird_tasks_lock); -+static LIST_HEAD(hmbird_tasks); -+ -+/* ops enable/disable */ -+static struct kthread_worker *hmbird_ops_helper; -+static DEFINE_MUTEX(hmbird_ops_enable_mutex); -+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem); -+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED); -+ -+static bool hmbird_warned_zero_slice; -+enum hmbird_switch_type sw_type; -+static int skip_num[MAX_GLOBAL_DSQS]; -+static int big_distribute_mask_prev; -+static int little_distribute_mask_prev; -+ -+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0); -+ -+/* -+ * The maximum amount of time in jiffies that a task may be runnable without -+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger -+ * hmbird_ops_error(). -+ */ -+unsigned long hmbird_watchdog_timeout; -+ -+/* -+ * The last time the delayed work was run. This delayed work relies on -+ * ksoftirqd being able to run to service timer interrupts, so it's possible -+ * that this work itself could get wedged. To account for this, we check that -+ * it's not stalled in the timer tick, and trigger an error if it is. -+ */ -+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES; -+ -+static struct delayed_work hmbird_watchdog_work; -+static struct work_struct hmbird_err_exit_work; -+ -+/* idle tracking */ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_CPUMASK_OFFSTACK -+#define CL_ALIGNED_IF_ONSTACK -+#else -+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp -+#endif -+ -+static struct { -+ cpumask_var_t cpu; -+ cpumask_var_t smt; -+} idle_masks CL_ALIGNED_IF_ONSTACK; -+ -+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus; -+#endif /* CONFIG_SMP */ -+ -+/* dispatch queues */ -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global; -+ -+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128}; -+u32 pcp_dsq_deadline = 20; -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS]; -+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq); -+ -+static u64 max_hmbird_dsq_internal_id; -+ -+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/ -+#define CLUSTER_SEPARATE_IDX (8) -+#define GDSQS_ID_BASE (3) -+#define UX_COMPATIBLE_IDX (4) -+#define NON_PERIOD_START (5) -+#define NON_PERIOD_END (MAX_GLOBAL_DSQS) -+#define CREATE_DSQ_LEVEL_WITHIN (1) -+ -+struct hmbird_sched_info { -+ spinlock_t lock; -+ int curr_idx[2]; -+ int rtime[MAX_GLOBAL_DSQS]; -+}; -+ -+struct pcp_sched_info { -+ s64 pcp_seq; -+ int rtime; -+ bool pcp_round; -+}; -+ -+/* -+ * pcp_info may rw by another cpu. -+ * protected by rq->lock. -+ */ -+atomic64_t pcp_dsq_round; -+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info); -+ -+struct md_info_t *md_info; -+ -+static int b_rescue_l, l_rescue_b; -+static struct hmbird_sched_info sinfo; -+ -+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC; -+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = { -+ 0, 0, 0, 0, 0, -+ 32 * NSEC_PER_MSEC, -+ 20 * NSEC_PER_MSEC, -+ 14 * NSEC_PER_MSEC, -+ 8 * NSEC_PER_MSEC, -+ 6 * NSEC_PER_MSEC -+}; -+ -+struct cluster_ctx { -+ /* cpu-dsq map must within [lower, upper) */ -+ int upper; -+ int lower; -+ int tidx; -+}; -+ -+enum stat_items { -+ GLOBAL_STAT, -+ CPU_ALLOW_FAIL, -+ RT_CNT, -+ KEY_TASK_CNT, -+ SWITCH_IDX, -+ TIMEOUT_CNT, -+ -+ TOTAL_DSP_CNT, -+ MOVE_RQ_CNT, -+ SELECT_CPU, -+ -+ DWORD_STAT_END = SELECT_CPU, -+ -+ GDSQ_CNT, -+ ERR_IDX, -+ PCP_TIMEOUT_CNT, -+ PCP_LDSQ_CNT, -+ PCP_ENQL_CNT, -+ -+ MAX_ITEMS, -+}; -+static DEFINE_SPINLOCK(stats_lock); -+static char *stats_str[MAX_ITEMS] = { -+ "global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt", -+ "switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt", -+ "select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt", -+ "pcp_ldsq_cnt", "pcp_enql_cnt" -+}; -+ -+ -+struct stats_struct { -+ u64 global_stat[2]; -+ u64 cpu_allow_fail[2]; -+ u64 rt_cnt[2]; -+ u64 key_task_cnt[2]; -+ u64 switch_idx[2]; -+ u64 timeout_cnt[2]; -+ -+ u64 total_dsp_cnt[2]; -+ u64 move_rq_cnt[2]; -+ u64 select_cpu[2]; -+ -+ u64 gdsq_count[MAX_GLOBAL_DSQS][2]; -+ u64 err_idx[5]; -+ u64 pcp_timeout_cnt[NR_CPUS]; -+ u64 pcp_ldsq_count[NR_CPUS][2]; -+ u64 pcp_enql_cnt[NR_CPUS]; -+} stats_data; -+ -+static struct { -+ cpumask_var_t ex_free; -+ cpumask_var_t exclusive; -+ cpumask_var_t partial; -+ cpumask_var_t big; -+ cpumask_var_t little; -+} iso_masks __read_mostly; -+ -+/* -+ * Need more synchronization for these two variables? -+ * I choose not to. -+ */ -+static int l_need_rescue, b_need_rescue; -+ -+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...) \ -+{ \ -+ char buf[MAX_FATAL_INFO]; \ -+ \ -+ scnprintf(buf, MAX_FATAL_INFO, fmt, ##args); \ -+ hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf); \ -+ trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable), \ -+ READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf); \ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); \ -+} \ -+ -+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2) -+{ -+ int c1, c2; -+ struct cpumask mask = {.bits = {0}}; -+ struct cpumask tmp = {.bits = {0}}; -+ -+ if (!slim_stats) -+ return false; -+ -+ if (!rq1 || !rq2) -+ return false; -+ -+ c1 = cpu_of(rq1); -+ c2 = cpu_of(rq2); -+ cpumask_set_cpu(c1, &mask); -+ cpumask_set_cpu(c2, &mask); -+ -+ if (cpumask_and(&tmp, iso_masks.little, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.big, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.partial, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ return false; -+} -+ -+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu) -+{ -+ unsigned long flags; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ if (!slim_stats) -+ return; -+ -+ switch (item) { -+ case GLOBAL_STAT: -+ fallthrough; -+ case CPU_ALLOW_FAIL: -+ fallthrough; -+ case RT_CNT: -+ fallthrough; -+ case KEY_TASK_CNT: -+ fallthrough; -+ case SWITCH_IDX: -+ fallthrough; -+ case TIMEOUT_CNT: -+ fallthrough; -+ case TOTAL_DSP_CNT: -+ fallthrough; -+ case MOVE_RQ_CNT: -+ fallthrough; -+ case SELECT_CPU: -+ pval = pbase + item * 2 + idx; -+ break; -+ case GDSQ_CNT: -+ pval = &stats_data.gdsq_count[dsq_id][idx]; -+ break; -+ case ERR_IDX: -+ pval = &stats_data.err_idx[idx]; -+ break; -+ case PCP_TIMEOUT_CNT: -+ pval = &stats_data.pcp_timeout_cnt[cpu]; -+ break; -+ case PCP_LDSQ_CNT: -+ pval = &stats_data.pcp_ldsq_count[cpu][idx]; -+ break; -+ case PCP_ENQL_CNT: -+ pval = &stats_data.pcp_enql_cnt[cpu]; -+ break; -+ default: -+ return; -+ } -+ -+ spin_lock_irqsave(&stats_lock, flags); -+ *pval += 1; -+ spin_unlock_irqrestore(&stats_lock, flags); -+} -+ -+static inline bool handle_ret(int ret, int *idx, int len) -+{ -+ if (ret < 0 || ret >= len - *idx) -+ return true; -+ *idx += ret; -+ return false; -+} -+ -+#define PRINT_INTV (5 * HZ) -+void stats_print(char *buf, int len) -+{ -+ int idx = 0, i, j, ret; -+ int item = 0; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ for (item = 0; item < MAX_ITEMS; item++) { -+ if (item <= DWORD_STAT_END) { -+ pval = pbase + item * 2; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n", -+ stats_str[item], pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == GDSQ_CNT) { -+ for (j = 0; j < MAX_GLOBAL_DSQS; j++) { -+ pval = (u64 *)&stats_data.gdsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == ERR_IDX) { -+ pval = (u64 *)&stats_data.err_idx; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n", -+ stats_str[item], pval[0], -+ pval[1], pval[2], pval[3], pval[4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == PCP_TIMEOUT_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_timeout_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_LDSQ_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_ldsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_ENQL_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_enql_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } -+ } -+ -+ if (!md_info) { -+ buf[idx] = '\0'; -+ return; -+ } -+ -+ ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_SWITCHS; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "sw_rec[%d] = %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.sw_rec[i].switch_at, -+ md_info->kern_dump.sw_rec[i].is_success, -+ md_info->kern_dump.sw_rec[i].end_state, -+ md_info->kern_dump.sw_rec[i].switch_reason); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_EXCEP_ID; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "excep[%d] = %llu %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.excep_rec[i][0], -+ md_info->kern_dump.excep_rec[i][1], -+ md_info->kern_dump.excep_rec[i][2], -+ md_info->kern_dump.excep_rec[i][3], -+ md_info->kern_dump.excep_rec[i][4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n", -+ i, md_info->kern_dump.excep_idx[i]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ -+ buf[idx] = '\0'; -+} -+ -+enum cpu_type { -+ LITTLE, -+ BIG, -+ PARTIAL, -+ EXCLUSIVE, -+ EX_FREE, -+ INVALID -+}; -+ -+enum dsq_type { -+ GLOBAL_DSQ, -+ PCP_DSQ, -+ OTHER, -+ MAX_DSQ_TYPE, -+}; -+ -+static enum cpu_type cpu_cluster(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (get_hmbird_rq(rq)->exclusive) { -+ return EXCLUSIVE; -+ } else { -+ if (cpumask_test_cpu(cpu, iso_masks.little)) -+ return LITTLE; -+ else if (cpumask_test_cpu(cpu, iso_masks.big)) -+ return BIG; -+ else if (cpumask_test_cpu(cpu, iso_masks.partial)) -+ return PARTIAL; -+ else if (cpumask_test_cpu(cpu, iso_masks.exclusive)) -+ return EXCLUSIVE; -+ else if (cpumask_test_cpu(cpu, iso_masks.ex_free)) -+ return EX_FREE; -+ } -+ return INVALID; -+} -+ -+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq) -+{ -+ if (!dsq) -+ return OTHER; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= GDSQS_ID_BASE) && -+ ((dsq->id & 0xff) < MAX_GLOBAL_DSQS)) -+ return GLOBAL_DSQ; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) && -+ ((dsq->id & 0xff) < max_hmbird_dsq_internal_id)) -+ return PCP_DSQ; -+ -+ return OTHER; -+} -+ -+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq) -+{ -+ enum dsq_type type; -+ -+ type = get_dsq_type(dsq); -+ switch (type) { -+ case GLOBAL_DSQ: -+ case PCP_DSQ: -+ return (dsq->id & 0xff) - GDSQS_ID_BASE; -+ default: -+ return -1; -+ } -+ return -1; -+} -+ -+static void update_cpus_idle(bool set, struct cpumask *mask) -+{ -+ int cpu; -+ struct rq *rq; -+ -+ if (set) { -+ for_each_cpu(cpu, mask) { -+ rq = cpu_rq(cpu); -+ if (is_idle_task(rq->curr)) -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ } -+ } else -+ cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask); -+} -+ -+static void set_partial_status(bool enable, bool little, bool big) -+{ -+ WRITE_ONCE(partial_enable, enable); -+ WRITE_ONCE(l_need_rescue, little); -+ WRITE_ONCE(b_need_rescue, big); -+} -+ -+static bool is_little_need_rescue(void) -+{ -+ return READ_ONCE(l_need_rescue); -+} -+ -+static bool is_big_need_rescue(void) -+{ -+ return READ_ONCE(b_need_rescue); -+} -+ -+static bool is_partial_enabled(void) -+{ -+ return READ_ONCE(partial_enable); -+} -+ -+static bool is_partial_cpu(int cpu) -+{ -+ return cpumask_test_cpu(cpu, iso_masks.partial); -+} -+ -+static void set_iso_par_free(bool enable) -+{ -+ WRITE_ONCE(isolate_ctrl, enable); -+} -+ -+static bool is_iso_par_free(void) -+{ -+ return READ_ONCE(isolate_ctrl); -+} -+ -+static bool skip_update_idle(void) -+{ -+ int cpu = smp_processor_id(); -+ enum cpu_type type = cpu_cluster(cpu); -+ -+ if ((type == EXCLUSIVE && !is_iso_par_free()) || -+ /* partial enable may changed during idle, it doesn't matter. */ -+ (!is_partial_enabled() && type == PARTIAL)) -+ return true; -+ -+ return false; -+} -+ -+static void init_isolate_cpus(void) -+{ -+ WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -+ cpumask_set_cpu(0, iso_masks.big); -+ cpumask_set_cpu(1, iso_masks.big); -+ cpumask_set_cpu(2, iso_masks.big); -+ cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(4, iso_masks.big); -+ cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(6, iso_masks.partial); -+ cpumask_set_cpu(7, iso_masks.ex_free); -+} -+ -+extern spinlock_t css_set_lock; -+ -+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp) -+{ -+ int i; -+ struct cgroup *anc; -+ -+ spin_lock_irq(&css_set_lock); -+ for (i = 0; i < cgrp->level; i++) { -+ anc = cgrp->ancestors[i]; -+ if (anc->level != CREATE_DSQ_LEVEL_WITHIN) -+ continue; -+ cgroup_get(anc); -+ spin_unlock_irq(&css_set_lock); -+ return anc; -+ } -+ spin_unlock_irq(&css_set_lock); -+ hmbird_err(NO_CGROUP_L1, ":error cgroup = %s\n", cgrp->kn->name); -+ return NULL; -+} -+ -+#define PCP_IDX_BIT (1 << 31) -+ -+static bool is_pcp_rt(struct task_struct *p) -+{ -+ return rt_prio(p->prio) && (p->nr_cpus_allowed == 1); -+} -+ -+static bool is_pcp_idx(int idx) -+{ -+ return idx >= MAX_GLOBAL_DSQS; -+} -+ -+static bool is_critical_system_task(struct task_struct *p) -+{ -+ int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK; -+ -+ return (p->prio < (MAX_RT_PRIO >> 1) && -+ (sp_dl < SCHED_PROP_DEADLINE_LEVEL3)); -+} -+ -+#define ISOLATE_TASK_TOP_BIT (1 << 17) -+#define PIPELINE_TASK_TOP_BIT (1 << 9) -+static bool is_pipeline_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT); -+} -+ -+static bool is_isolate_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT); -+} -+ -+static bool is_critical_app_task_without_isolate(struct task_struct *p) -+{ -+ return task_is_top_task(p) && !is_isolate_task(p); -+} -+ -+static int find_idx_from_task(struct task_struct *p) -+{ -+ int idx, cpu; -+ int sp_dl; -+ struct task_group *tg = p->sched_task_group; -+ -+ if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) { -+ cpu = cpumask_any(p->cpus_ptr); -+ idx = cpu + MAX_GLOBAL_DSQS; -+ return idx; -+ } -+ -+ if (is_critical_system_task(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL0; -+ goto done; -+ } -+ -+ if (is_critical_app_task_without_isolate(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL1; -+ goto done; -+ } -+ -+ sp_dl = hmbird_get_dsq_id(p); -+ if (sp_dl) { -+ idx = sp_dl; -+ goto done; -+ } -+ -+ if (rt_prio(p->prio)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL3; -+ goto done; -+ } -+ -+ if (tg && tg->css.cgroup && tg->css.cgroup->kn) { -+ if (likely(tg->css.cgroup->kn->id >= 0 && -+ tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS)) -+ idx = cgroup_ids_table[tg->css.cgroup->kn->id]; -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ -+done: -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) { -+ hmbird_err(DSQ_ID_ERR, " : idx error, idx = %d-----\n", idx); -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } -+ return idx; -+} -+ -+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p) -+{ -+ int idx; -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq; -+ -+ if (!p) -+ return NULL; -+ -+ idx = find_idx_from_task(p); -+ if (is_pcp_idx(idx)) { -+ idx -= MAX_GLOBAL_DSQS; -+ dsq = &per_cpu(pcp_ldsq, idx); -+ get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq); -+ slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx); -+ } else { -+ dsq = &gdsqs[idx]; -+ get_hmbird_ts(p)->gdsq_idx = idx; -+ slim_stats_record(GDSQ_CNT, 0, idx, 0); -+ } -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ dsq->last_consume_at = jiffies; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ return dsq; -+} -+ -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq); -+ -+static void set_partial_rescue(bool p_state, bool l_over, bool b_over) -+{ -+ set_partial_status(p_state, l_over, b_over); -+ update_cpus_idle(p_state, iso_masks.partial); -+ hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled()); -+ hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue()); -+ hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue()); -+} -+ -+static void free_isocpu(bool enable) -+{ -+ set_iso_par_free(enable); -+ update_cpus_idle(enable, iso_masks.ex_free); -+ hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free()); -+} -+ -+inline u64 get_hmbird_cpu_util(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed) -+ return 0; -+ u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed); -+ u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size); -+ -+ do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT); -+ -+ return prev_runnable_sum_fixed; -+} -+ -+static inline unsigned int get_scaling_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->max; -+} -+ -+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static u64 get_cpus_max_util(struct cpumask *mask) -+{ -+ int cpu; -+ u64 max = 0; -+ u64 util, ratio; -+ unsigned long effective_cap = 0; -+ for_each_cpu(cpu, mask) { -+ if (slim_walt_ctrl) -+ slim_get_cpu_util(cpu, &util); -+ else -+ util = get_hmbird_cpu_util(cpu); -+ -+ /* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/ -+ if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu))) -+ effective_cap = arch_scale_cpu_capacity(cpu); -+ else -+ effective_cap = arch_scale_cpu_capacity(cpu) * -+ get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu); -+ -+ ratio = util * 100 / effective_cap; -+ hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util); -+ hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n", -+ cpu, (u64)arch_scale_cpu_capacity(cpu)); -+ hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n", -+ cpu, (u64)effective_cap); -+ -+ if (ratio > 100) -+ ratio = 100; -+ -+ if (ratio > max) -+ max = ratio; -+ } -+ return max; -+} -+ -+static bool cluster_need_rescue(struct cpumask *mask, int hres) -+{ -+ return get_cpus_max_util(mask) > hres; -+} -+ -+void partial_dynamic_ctrl(void) -+{ -+ u64 lmax = 0, bmax = 0; -+ bool l_over, l_under, b_over, b_under; -+ static bool last_l_over, last_b_over; -+ static unsigned long last_check; -+ -+ /* Check partial every jiffies. need lock? */ -+ if (time_before_eq(jiffies, READ_ONCE(last_check))) -+ return; -+ -+ WRITE_ONCE(last_check, jiffies); -+ -+ lmax = get_cpus_max_util(iso_masks.little); -+ l_over = lmax > parctrl_high_ratio_l; -+ l_under = lmax < parctrl_low_ratio_l; -+ -+ bmax = get_cpus_max_util(iso_masks.big); -+ b_over = bmax > parctrl_high_ratio; -+ b_under = bmax < parctrl_low_ratio; -+ -+ if (is_partial_enabled() && (l_over || b_over)) { -+ if (last_l_over != l_over || last_b_over != b_over) -+ set_partial_rescue(true, l_over, b_over); -+ } else if (!is_partial_enabled() && (l_over || b_over)) { -+ set_partial_rescue(true, l_over, b_over); -+ } else if (is_partial_enabled() && l_under && b_under) { -+ set_partial_rescue(false, false, false); -+ } -+ last_l_over = l_over; -+ last_b_over = b_over; -+ -+ if (is_partial_enabled()) { -+ bmax = get_cpus_max_util(iso_masks.partial); -+ if (!is_iso_par_free() && bmax > isoctrl_high_ratio) { -+ hmbird_info_trace("partial max = %llu\n", bmax); -+ free_isocpu(true); -+ } else if (is_iso_par_free() && bmax < isoctrl_low_ratio) -+ free_isocpu(false); -+ } else if (is_iso_par_free()) { -+ free_isocpu(false); -+ } -+} -+ -+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx) -+{ -+ hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx); -+} -+ -+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx) -+{ -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[idx])) { -+ slim_stats_record(GDSQ_CNT, 1, idx, 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ int i; -+ -+ for (i = 0; i < UX_COMPATIBLE_IDX; i++) { -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ slim_stats_record(GDSQ_CNT, 1, i, 0); -+ return 1; -+ } -+ } -+ return 0; -+} -+ -+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) { -+ slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ goto clear_timeout; -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) -+ goto clear_timeout; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ hmbird_info_trace( -+ "dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n", -+ dsq_id_to_internal(dsq), entity->task->comm, -+ jiffies, msecs_to_jiffies(deadline), entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1); -+ return; -+ -+clear_timeout: -+ hmbird_info_trace("dsq[%d] clear timeout\n", -+ dsq_id_to_internal(dsq)); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0); -+ dsq->is_timeout = false; -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq)); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime) -+{ -+ hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime); -+} -+ -+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any) -+{ -+ bool is_timeout; -+ int cpu = cpu_of(rq); -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu); -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ is_timeout = dsq->is_timeout; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ /* -+ * dsq->is_timeout may change here, let it be. -+ * it won't cause serious problems. -+ * the same for consume_dispatch_q later. -+ */ -+ if (!is_timeout && !any) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, dsq)) { -+ if (is_timeout) { -+ hmbird_info_trace("dsq[%d] consume a pcp timeout task\n", -+ dsq_id_to_internal(dsq)); -+ update_timeout_stats(rq, dsq, pcp_dsq_deadline); -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu); -+ } -+ slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu); -+ return 1; -+ } -+ /* -+ * No pcp task, clear quota. -+ */ -+ if (any) { -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+ } -+ return 0; -+} -+ -+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf) -+{ -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ } -+ return 0; -+} -+ -+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf, -+ int tmp, int cidx, int tidx) -+{ -+ unsigned long flags; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) { -+ if (tmp != cidx) { -+ spin_lock(&sinfo.lock); -+ sinfo.curr_idx[tidx] = tmp; -+ spin_unlock(&sinfo.lock); -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]); -+ slim_stats_record(SWITCH_IDX, 0, 0, 0); -+ } -+ slim_stats_record(GDSQ_CNT, 1, tmp, 0); -+ -+ raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags); -+ gdsqs[tmp].last_consume_at = jiffies; -+ raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags); -+ return 1; -+ } -+ return 0; -+ -+} -+ -+static int get_cidx(struct cluster_ctx *ctx) -+{ -+ int cidx; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ if (cidx < ctx->lower || cidx >= ctx->upper) { -+ sinfo.curr_idx[ctx->tidx] = ctx->lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]); -+ slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ } -+ spin_unlock(&sinfo.lock); -+ -+ return cidx; -+} -+ -+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = CLUSTER_SEPARATE_IDX; -+ if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) { -+ pr_debug(" %s iso_masks.little first is %d\n", -+ __func__, cpumask_first(iso_masks.little)); -+ ctx->upper = NON_PERIOD_END; -+ } -+ ctx->tidx = 0; -+ break; -+ case LITTLE: -+ ctx->lower = CLUSTER_SEPARATE_IDX; -+ ctx->upper = NON_PERIOD_END; -+ if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) { -+ pr_debug(" %s iso_masks.big first is %d", -+ __func__, cpumask_first(iso_masks.big)); -+ ctx->lower = NON_PERIOD_START; -+ } -+ ctx->tidx = 1; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ case LITTLE: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = NON_PERIOD_END; -+ ctx->tidx = 0; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ if (cluster_separate) { -+ return gen_cluster_ctx_separate(ctx, type); -+ } else { -+ return gen_cluster_ctx_common(ctx, type); -+ } -+} -+ -+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ int i; -+ bool is_timeout; -+ unsigned long flags; -+ struct cluster_ctx ctx; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ /* Third param means only consume timeout pcp dsq. */ -+ if (consume_pcp_dsq(rq, rf, false)) -+ return 1; -+ -+ for (i = ctx.lower; i < ctx.upper; i++) { -+ raw_spin_lock_irqsave(&gdsqs[i].lock, flags); -+ is_timeout = gdsqs[i].is_timeout; -+ raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags); -+ /* gdsqs[i].is_timeout may change here, let it be... */ -+ if (is_timeout) { -+ /* -+ * consume_dispatch_q will acquire dsq-lock, -+ * So cannot keep lock here, annoy enough. -+ * may rewrite a consume_dispatch_q_locked, TODO. -+ */ -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ hmbird_info_trace("dsq[%d] consume a timeout task\n", i); -+ slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0); -+ update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]); -+ return 1; -+ } -+ } -+ } -+ return 0; -+} -+ -+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ struct cluster_ctx ctx; -+ int cidx; -+ int tmp; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ cidx = get_cidx(&ctx); -+ tmp = cidx; -+ do { -+ if (check_pcp_dsq_round(rq, rf)) -+ return 1; -+ if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx)) -+ return 1; -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[tmp] = 0; -+ systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]); -+ tmp++; -+ if (tmp >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ tmp = ctx.lower; -+ } -+ spin_unlock(&sinfo.lock); -+ } while (tmp != cidx); -+ -+ return consume_pcp_dsq(rq, rf, true); -+} -+ -+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ enum cpu_type type = cpu_cluster(cpu_of(rq)); -+ bool period_allowed = !get_hmbird_rq(rq)->period_disallow; -+ bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow; -+ -+ switch (type) { -+ case EX_FREE: -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ break; -+ case EXCLUSIVE: -+ if (!is_iso_par_free()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ } -+ /* Only non-period task can run on isolate cpus */ -+ if (!READ_ONCE(iso_free_rescue)) { -+ if (is_big_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ } else { -+ /* Free run, can run on any task. */ -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ case PARTIAL: -+ if (!is_partial_enabled()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ return 0; -+ } -+ if (is_big_need_rescue()) { -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ -+ case BIG: -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ if (is_iso_par_free() || (is_little_need_rescue() && -+ !cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) { -+ hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++); -+ return 1; -+ } -+ } -+ break; -+ -+ case LITTLE: -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ -+ if (is_iso_par_free() || (is_big_need_rescue() && -+ !cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) { -+ hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++); -+ return 1; -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return 0; -+} -+ -+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf) -+{ -+ return consume_hmbird_global_dsq(rq, rf); -+} -+ -+ -+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time) -+{ -+ int idx; -+ -+ /* which dsq belongs to while task enqueue, task will consume its running time. */ -+ idx = get_hmbird_ts(p)->gdsq_idx; -+ /* Only non-period dsq share running time between each other. */ -+ if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id) -+ return; -+ -+ if (idx >= MAX_GLOBAL_DSQS) { -+ per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } else { -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[idx] += exec_time; -+ spin_unlock(&sinfo.lock); -+ systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]); -+ } -+} -+ -+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type) -+{ -+ int cidx; -+ struct cluster_ctx ctx; -+ int cpu = cpu_of(rq); -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ if (cidx < ctx.lower || cidx >= ctx.upper) { -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ slim_stats_record(ERR_IDX, ctx.tidx, 0, 0); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ } -+ -+ while (1) { -+ if (per_cpu(pcp_info, cpu).pcp_round) { -+ if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) { -+ hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n", -+ cpu, per_cpu(pcp_info, cpu).rtime); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } -+ } -+ if (sinfo.rtime[cidx] < dsq_quota[cidx]) -+ break; -+ -+ /* clear current dsq rtime */ -+ sinfo.rtime[cidx] = 0; -+ systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]); -+ -+ sinfo.curr_idx[ctx.tidx]++; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", -+ ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ } -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ slim_stats_record(SWITCH_IDX, 1, 0, 0); -+ } -+ spin_unlock(&sinfo.lock); -+} -+ -+ -+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p) -+{ -+ enum cpu_type type; -+ -+ if (!rq || !p) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ switch (type) { -+ case PARTIAL: -+ return; -+ case EXCLUSIVE: -+ return; -+ case EX_FREE: -+ return; -+ default: -+ break; -+ } -+ update_dsq_idx(rq, p, type); -+} -+ -+ -+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ int dsq_id; -+ -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo) || dsq->is_timeout) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ hmbird_deferred_err(SCAN_ENTITY_NULL, -+ " : entity is NULL, dsq->id = %llu\n", dsq->id); -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ dsq->is_timeout = true; -+ dsq_id = dsq_id_to_internal(dsq); -+ hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n", -+ dsq_id, entity->task->comm, jiffies, entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1); -+ raw_spin_unlock(&dsq->lock); -+ -+ return true; -+} -+ -+void scan_timeout(struct rq *rq) -+{ -+ int i; -+ int cpu = cpu_of(rq); -+ struct hmbird_dispatch_q *dsq; -+ static u64 last_scan_at; -+ static DEFINE_PER_CPU(u64, pcp_last_scan_at); -+ -+ if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu))) -+ return; -+ per_cpu(pcp_last_scan_at, cpu) = jiffies; -+ -+ dsq = &per_cpu(pcp_ldsq, cpu); -+ scan_dsq_timeout(rq, dsq, pcp_dsq_deadline); -+ -+ if (time_before_eq(jiffies, (unsigned long)last_scan_at)) -+ return; -+ last_scan_at = jiffies; -+ -+ for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) { -+ dsq = &gdsqs[i]; -+ scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]); -+ } -+} -+ -+/*******************************Initialize***********************************/ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id); -+static void init_dsq_at_boot(void) -+{ -+ int i, cpu; -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i)); -+ } -+ for_each_possible_cpu(cpu) -+ init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i + cpu)); -+ -+ max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu; -+ spin_lock_init(&sinfo.lock); -+} -+ -+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgrp_name_to_idx(struct cgroup *cgrp) -+{ -+ int idx; -+ -+ if (!cgrp) -+ return -1; -+ -+ if (!strcmp(cgrp->kn->name, "display") -+ || !strcmp(cgrp->kn->name, "multimedia")) -+ idx = 5; /* 8ms */ -+ else if (!strcmp(cgrp->kn->name, "top-app") -+ || !strcmp(cgrp->kn->name, "ss-top")) -+ idx = 6; /* 16ms */ -+ else if (!strcmp(cgrp->kn->name, "ssfg") -+ || !strcmp(cgrp->kn->name, "foreground")) -+ idx = 7; /* 32ms */ -+ else if (!strcmp(cgrp->kn->name, "bg") -+ || !strcmp(cgrp->kn->name, "log") -+ || !strcmp(cgrp->kn->name, "dex2oat") -+ || !strcmp(cgrp->kn->name, "background")) -+ idx = 9; /* 128ms */ -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */ -+ -+ return idx; -+} -+ -+static void init_root_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX); -+} -+ -+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) { -+ pr_err("%s idx err!\n", __func__); -+ return; -+ } -+ -+ if (cgroup_ids_table[cgrp->kn->id] == -1) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp)); -+} -+ -+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ struct cgroup *l1cgrp; -+ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ l1cgrp = cgroup_ancestor_l1(cgrp); -+ if (l1cgrp) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp)); -+ cgroup_put(l1cgrp); -+} -+ -+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg) -+{ -+ switch (cgrp->level) { -+ case 0: -+ init_root_tg(cgrp, tg); -+ break; -+ case 1: -+ init_level1_tg(cgrp, tg); -+ break; -+ default: -+ init_child_tg(cgrp, tg); -+ break; -+ } -+} -+ -+/**************************************************************************/ -+ -+struct hmbird_task_iter { -+ struct hmbird_entity cursor; -+ struct task_struct *locked; -+ struct rq *rq; -+ struct rq_flags rf; -+}; -+ -+/** -+ * hmbird_task_iter_init - Initialize a task iterator -+ * @iter: iterator to init -+ * -+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized, -+ * @iter must eventually be exited with hmbird_task_iter_exit(). -+ * -+ * hmbird_tasks_lock may be released between this and the first next() call or -+ * between any two next() calls. If hmbird_tasks_lock is released between two -+ * next() calls, the caller is responsible for ensuring that the task being -+ * iterated remains accessible either through RCU read lock or obtaining a -+ * reference count. -+ * -+ * All tasks which existed when the iteration started are guaranteed to be -+ * visited as long as they still exist. -+ */ -+static void hmbird_task_iter_init(struct hmbird_task_iter *iter) -+{ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR }; -+ list_add(&iter->cursor.tasks_node, &hmbird_tasks); -+ iter->locked = NULL; -+} -+ -+/** -+ * hmbird_task_iter_exit - Exit a task iterator -+ * @iter: iterator to exit -+ * -+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held. -+ * If the iterator holds a task's rq lock, that rq lock is released. See -+ * hmbird_task_iter_init() for details. -+ */ -+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ if (list_empty(cursor)) -+ return; -+ -+ list_del_init(cursor); -+} -+ -+/** -+ * hmbird_task_iter_next - Next task -+ * @iter: iterator to walk -+ * -+ * Visit the next task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ struct hmbird_entity *pos; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ list_for_each_entry(pos, cursor, tasks_node) { -+ if (&pos->tasks_node == &hmbird_tasks) -+ return NULL; -+ if (!(pos->flags & HMBIRD_TASK_CURSOR)) { -+ list_move(cursor, &pos->tasks_node); -+ return pos->task; -+ } -+ } -+ -+ /* can't happen, should always terminate at hmbird_tasks above */ -+ hmbird_deferred_err(ITER_RET_NULL, " : unreachable path in scx_task_iter_next\n"); -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered - Next non-idle task -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ while ((p = hmbird_task_iter_next(iter))) { -+ if (!is_idle_task(p)) -+ return p; -+ } -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init() -+ * for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ p = hmbird_task_iter_next_filtered(iter); -+ if (!p) -+ return NULL; -+ -+ iter->rq = task_rq_lock(p, &iter->rf); -+ iter->locked = p; -+ return p; -+} -+ -+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void) -+{ -+ return atomic_read(&hmbird_ops_enable_state_var); -+} -+ -+static enum hmbird_ops_enable_state -+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to) -+{ -+ return atomic_xchg(&hmbird_ops_enable_state_var, to); -+} -+ -+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to, -+ enum hmbird_ops_enable_state from) -+{ -+ int from_v = from; -+ -+ return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to); -+} -+ -+static bool hmbird_ops_disabling(void) -+{ -+ return false; -+} -+ -+/** -+ * wait_ops_state - Busy-wait the specified ops state to end -+ * @p: target task -+ * @opss: state to wait the end of -+ * -+ * Busy-wait for @p to transition out of @opss. This can only be used when the -+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also -+ * has load_acquire semantics to ensure that the caller can see the updates made -+ * in the enqueueing and dispatching paths. -+ */ -+static void wait_ops_state(struct task_struct *p, u64 opss) -+{ -+ do { -+ cpu_relax(); -+ } while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss); -+} -+ -+ -+static void update_curr_hmbird(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ u64 now = rq_clock_task(rq); -+ u64 delta_exec; -+ -+ if (time_before_eq64(now, curr->se.exec_start)) -+ return; -+ -+ delta_exec = now - curr->se.exec_start; -+ curr->se.exec_start = now; -+ update_runningtime(rq, curr, delta_exec); -+ curr->se.sum_exec_runtime += delta_exec; -+ account_group_exec_runtime(curr, delta_exec); -+ cgroup_account_cputime(curr, delta_exec); -+ -+ if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF) -+ get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec); -+ -+ trace_sched_stat_runtime(curr, delta_exec, 0); -+} -+ -+static bool hmbird_dsq_priq_less(struct rb_node *node_a, -+ const struct rb_node *node_b) -+{ -+ const struct hmbird_entity *a = -+ container_of(node_a, struct hmbird_entity, dsq_node.priq); -+ const struct hmbird_entity *b = -+ container_of(node_b, struct hmbird_entity, dsq_node.priq); -+ -+ return time_before64(a->dsq_vtime, b->dsq_vtime); -+} -+ -+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ bool is_local = dsq->id == HMBIRD_DSQ_LOCAL; -+ unsigned long flags; -+ -+ hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq || -+ !list_empty(&get_hmbird_ts(p)->dsq_node.fifo), -+ "task = %s, dsq->id = %llu\n", p->comm, dsq->id); -+ hmbird_cond_deferred_err(ENQ_EXIST2, -+ (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq), -+ "task = %s\n", p->comm); -+ -+ if (!is_local) { -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ); -+ hmbird_ops_error(": %s\n", -+ "attempting to dispatch to a destroyed dsq"); -+ /* fall back to the global dsq */ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ dsq = &hmbird_dsq_global; -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ } -+ } -+ -+ if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) { -+ get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ; -+ rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq, -+ hmbird_dsq_priq_less); -+ } else { -+ if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT)) -+ list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ else -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ } -+ dsq->nr++; -+ get_hmbird_ts(p)->dsq = dsq; -+ -+ /* -+ * We're transitioning out of QUEUEING or DISPATCHING. store_release to -+ * match waiters' load_acquire. -+ */ -+ if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS) -+ atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE); -+ -+ if (is_local) { -+ struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq); -+ struct rq *rq = hmbird->rq; -+ bool preempt = false; -+ -+ if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr && -+ rq->curr->sched_class == &hmbird_sched_class) { -+ get_hmbird_ts(rq->curr)->slice = 0; -+ preempt = true; -+ } -+ -+ if (preempt || sched_class_above(&hmbird_sched_class, -+ rq->curr->sched_class)) -+ resched_curr(rq); -+ } else { -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ } -+} -+ -+static void task_unlink_from_dsq(struct task_struct *p, -+ struct hmbird_dispatch_q *dsq) -+{ -+ if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) { -+ rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ; -+ } else { -+ list_del_init(&get_hmbird_ts(p)->dsq_node.fifo); -+ } -+} -+ -+static bool task_linked_on_dsq(struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+} -+ -+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq; -+ bool is_local = dsq == &hmbird_rq->local_dsq; -+ -+ if (!dsq) { -+ hmbird_cond_deferred_err(TASK_LINKED1, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ /* -+ * When dispatching directly from the BPF scheduler to a local -+ * DSQ, the task isn't associated with any DSQ but -+ * @get_hmbird_ts(p)->holding_cpu may be set under the protection of -+ * %HMBIRD_OPSS_DISPATCHING. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu >= 0) -+ get_hmbird_ts(p)->holding_cpu = -1; -+ return; -+ } -+ -+ if (!is_local) -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ /* -+ * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node -+ * can't change underneath us. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu < 0) { -+ /* @p must still be on @dsq, dequeue */ -+ hmbird_cond_deferred_err(TASK_UNLINKED, -+ !task_linked_on_dsq(p), "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ } else { -+ /* -+ * We're racing against dispatch_to_local_dsq() which already -+ * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the -+ * holding_cpu which tells dispatch_to_local_dsq() that it lost -+ * the race. -+ */ -+ hmbird_cond_deferred_err(TASK_LINKED2, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ get_hmbird_ts(p)->holding_cpu = -1; -+ } -+ get_hmbird_ts(p)->dsq = NULL; -+ -+ if (!is_local) -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+ -+static bool test_rq_online(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ return rq->online; -+#else -+ return true; -+#endif -+} -+ -+static void refill_task_slice(struct task_struct *p) -+{ -+ if (is_isolate_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO; -+ else if (is_pipeline_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2; -+ else -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+} -+ -+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -+ int sticky_cpu) -+{ -+ struct hmbird_dispatch_q *d; -+ s32 cpu = -1; -+ -+ hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), -+ "task = %s\n", p->comm); -+ -+ if (is_pcp_rt(p)) { -+ /* Enqueue percpu rt task to local directly. */ -+ /* Or cause a bug when disable dispatch. */ -+ if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ } -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (cpu >= 0) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ -+ if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ /* rq migration */ -+ if (sticky_cpu == cpu_of(rq)) -+ goto local_norefill; -+ /* -+ * If !rq->online, we already told the BPF scheduler that the CPU is -+ * offline. We're just trying to on/offline the CPU. Don't bother the -+ * BPF scheduler. -+ */ -+ if (unlikely(!test_rq_online(rq))) -+ goto local; -+ -+ /* see %HMBIRD_OPS_ENQ_LAST */ -+ if (enq_flags & HMBIRD_ENQ_LAST) -+ goto local; -+ -+ if (enq_flags & HMBIRD_ENQ_LOCAL) -+ goto local; -+ else -+ goto global; -+local: -+ /* -+ * For task-ordering, slice refill must be treated as implying the end -+ * of the current slice. Otherwise, the longer @p stays on the CPU, the -+ * higher priority it becomes from hmbird_prio_less()'s POV. -+ */ -+ refill_task_slice(p); -+local_norefill: -+ dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags); -+ slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq)); -+ return; -+ -+global: -+ d = find_dsq_from_task(p); -+ if (d) { -+ refill_task_slice(p); -+ dispatch_enqueue(d, p, enq_flags); -+ return; -+ } -+ slim_stats_record(GLOBAL_STAT, 0, 0, 0); -+ refill_task_slice(p); -+ dispatch_enqueue(&hmbird_dsq_global, p, enq_flags); -+} -+ -+static bool watchdog_task_watched(const struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->watchdog_node); -+} -+ -+static void watchdog_watch_task(struct rq *rq, struct task_struct *p) -+{ -+ lockdep_assert_rq_held(rq); -+ if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ get_hmbird_ts(p)->runnable_at = jiffies; -+ clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list); -+} -+ -+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) -+{ -+ list_del_init(&get_hmbird_ts(p)->watchdog_node); -+ if (reset_timeout) -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags) -+{ -+ int sticky_cpu = get_hmbird_ts(p)->sticky_cpu; -+ -+ enq_flags |= get_hmbird_rq(rq)->extra_enq_flags; -+ -+ if (sticky_cpu >= 0) -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ -+ /* -+ * Restoring a running task will be immediately followed by -+ * set_next_task_hmbird() which expects the task to not be on the BPF -+ * scheduler as tasks can only start running through local DSQs. Force -+ * direct-dispatch into the local DSQ by setting the sticky_cpu. -+ */ -+ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -+ sticky_cpu = cpu_of(rq); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_UNWATCHED, -+ !watchdog_task_watched(p), "task = %s\n", p->comm); -+ return; -+ } -+ -+ watchdog_watch_task(rq, p); -+ set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ get_hmbird_rq(rq)->nr_running++; -+ add_nr_running(rq, 1); -+ -+ do_enqueue_task(rq, p, enq_flags, sticky_cpu); -+} -+ -+static void ops_dequeue(struct task_struct *p, u64 deq_flags) -+{ -+ u64 opss; -+ -+ watchdog_unwatch_task(p, false); -+ -+ /* acquire ensures that we see the preceding updates on QUEUED */ -+ opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state); -+ -+ switch (opss & HMBIRD_OPSS_STATE_MASK) { -+ case HMBIRD_OPSS_NONE: -+ break; -+ case HMBIRD_OPSS_QUEUEING: -+ /* -+ * QUEUEING is started and finished while holding @p's rq lock. -+ * As we're holding the rq lock now, we shouldn't see QUEUEING. -+ */ -+ hmbird_deferred_err(DEQ_DEQING, " : unreachable path in %s\n", __func__); -+ break; -+ case HMBIRD_OPSS_QUEUED: -+ if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss, -+ HMBIRD_OPSS_NONE)) -+ break; -+ fallthrough; -+ case HMBIRD_OPSS_DISPATCHING: -+ /* -+ * If @p is being dispatched from the BPF scheduler to a DSQ, -+ * wait for the transfer to complete so that @p doesn't get -+ * added to its DSQ after dequeueing is complete. -+ * -+ * As we're waiting on DISPATCHING with the rq locked, the -+ * dispatching side shouldn't try to lock the rq while -+ * DISPATCHING is set. See dispatch_to_local_dsq(). -+ * -+ * DISPATCHING shouldn't have qseq set and control can reach -+ * here with NONE @opss from the above QUEUED case block. -+ * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss. -+ */ -+ wait_ops_state(p, HMBIRD_OPSS_DISPATCHING); -+ hmbird_cond_deferred_err(HMBIRD_OPN, -+ atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE, -+ "task = %s\n", p->comm); -+ break; -+ } -+} -+ -+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ -+ if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p), -+ "task = %s\n", p->comm); -+ return; -+ } -+ -+ ops_dequeue(p, deq_flags); -+ -+ if (slim_walt_ctrl) { -+ if (task_current(rq, p)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ if (deq_flags & HMBIRD_DEQ_SLEEP) -+ set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else -+ clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), -+ (unsigned long *)&get_hmbird_ts(p)->flags); -+ -+ clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm); -+ hmbird_rq->nr_running--; -+ sub_nr_running(rq, 1); -+ dispatch_dequeue(hmbird_rq, p); -+} -+ -+static void yield_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ get_hmbird_ts(p)->slice = 0; -+} -+ -+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to) -+{ -+ return false; -+} -+ -+#ifdef CONFIG_SMP -+/** -+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -+ * @rq: rq to move the task into, currently locked -+ * @p: task to move -+ * @enq_flags: %HMBIRD_ENQ_* -+ * -+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller -+ * must: -+ * -+ * 1. Start with exclusive access to @p either through its DSQ lock or -+ * %HMBIRD_OPSS_DISPATCHING flag. -+ * -+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id(). -+ * -+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't -+ * deadlock with dequeue. -+ * -+ * 4. Lock @rq and the task_rq from #3. -+ * -+ * 5. Call this function. -+ * -+ * Returns %true if @p was successfully moved. %false after racing dequeue and -+ * losing. -+ */ -+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ struct rq *task_rq; -+ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * If dequeue got to @p while we were trying to lock both rq's, it'd -+ * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have -+ * updated it to different values afterwards, as this operation can't be -+ * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become -+ * raw_smp_processor_id() again before we're done. Thus, we can tell -+ * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is -+ * still raw_smp_processor_id(). -+ * -+ * See dispatch_dequeue() for the counterpart. -+ */ -+ if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id())) -+ return false; -+ -+ /* @p->rq couldn't have changed if we're still the holding cpu */ -+ task_rq = task_rq(p); -+ lockdep_assert_rq_held(task_rq); -+ deactivate_task(task_rq, p, 0); -+ set_task_cpu(p, cpu_of(rq)); -+ get_hmbird_ts(p)->sticky_cpu = cpu_of(rq); -+ -+ /* -+ * We want to pass hmbird-specific enq_flags but activate_task() will -+ * truncate the upper 32 bit. As we own @rq, we can pass them through -+ * @get_hmbird_rq(rq)->extra_enq_flags instead. -+ */ -+ hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags, -+ "task = %s\n", p->comm); -+ get_hmbird_rq(rq)->extra_enq_flags = enq_flags; -+ activate_task(rq, p, 0); -+ get_hmbird_rq(rq)->extra_enq_flags = 0; -+ -+ return true; -+} -+ -+#endif /* CONFIG_SMP */ -+ -+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu) -+{ -+ int fitable = 1; -+ -+ return fitable; -+} -+ -+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq, -+ struct hmbird_dispatch_q *dsq) -+{ -+ bool dsq_misfit; -+ int cpu = cpu_of(rq); -+ u64 task_util = 0; -+ struct cluster_ctx ctx; -+ int dsq_int = dsq_id_to_internal(dsq); -+ -+ if (!cpumask_test_cpu(cpu, iso_masks.little)) -+ return false; -+ -+ gen_cluster_ctx(&ctx, BIG); -+ dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 && -+ dsq_int <= SCHED_PROP_DEADLINE_LEVEL4); -+#ifdef CLUSTER_SEPARATE -+ /* In rescue mode, little will consume big cluster's dsq.*/ -+ dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper); -+#endif -+ if (!dsq_misfit) -+ return false; -+ -+ if (p) { -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &task_util); -+ else -+ task_util = get_hmbird_ts(p)->demand_scaled; -+ } -+ -+ if (task_util <= misfit_ds) -+ return false; -+ -+ hmbird_info_trace(":task %s can't run on cpu%d, util = %llu\n", -+ p->comm, cpu, task_util); -+ return true; -+} -+ -+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq) -+{ -+ if (!task_fits_cpu_hmbird(p, cpu_of(rq))) -+ return false; -+ -+ if (check_misfit_task_on_little(p, rq, dsq)) -+ return false; -+ -+ return likely(test_rq_online(rq)); -+} -+ -+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return; -+ -+ if (add) -+ skipn[idx]++; -+ else -+ skipn[idx] = 0; -+} -+ -+static bool skip_too_much(struct hmbird_dispatch_q *dsq) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return false; -+ -+ if (skip_num[idx] > 3) { -+ skip_num[idx] = 0; -+ return true; -+ } -+ -+ return false; -+} -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rb_node *rb_node; -+ struct rq *task_rq; -+ unsigned long flags; -+ bool moved = false; -+ struct task_struct *may_fit = NULL; -+ int skip = 0; -+ -+retry: -+ if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -+ return false; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) { -+ set_skip_num(dsq, skip_num, (bool)may_fit); -+ goto this_rq; -+ } -+ if (skip_too_much(dsq)) -+ goto remote_rq; -+ if (!may_fit) -+ may_fit = p; -+ if (++skip <= 3) -+ continue; -+ /* -+ * If the recent 3 tasks not fit, use the first one. -+ * and clear the skip, because the first one is consumed. -+ */ -+ set_skip_num(dsq, skip_num, false); -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ /* No more task, use the first may fit task.*/ -+ if (may_fit) { -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ -+ for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) -+ goto this_rq; -+ goto remote_rq; -+ } -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ return false; -+ -+this_rq: -+ /* @dsq is locked and @p is on this rq */ -+ hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo); -+ dsq->nr--; -+ hmbird_rq->local_dsq.nr++; -+ get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ -+remote_rq: -+#ifdef CONFIG_SMP -+ if (cpu_same_cluster_stat(p, rq, task_rq)) -+ slim_stats_record(MOVE_RQ_CNT, 0, 0, 0); -+ else -+ slim_stats_record(MOVE_RQ_CNT, 1, 0, 0); -+ /* -+ * @dsq is locked and @p is on a remote rq. @p is currently protected by -+ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -+ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -+ * rq lock or fail, do a little dancing from our side. See -+ * move_task_to_local_dsq(). -+ */ -+ hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id(); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ rq_unpin_lock(rq, rf); -+ double_lock_balance(rq, task_rq); -+ rq_repin_lock(rq, rf); -+ -+ moved = move_task_to_local_dsq(rq, p, 0); -+ -+ double_unlock_balance(rq, task_rq); -+#endif /* CONFIG_SMP */ -+ if (likely(moved)) { -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ } -+ may_fit = NULL; -+ goto retry; -+} -+ -+ -+static int balance_one(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf, bool local) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class; -+ -+ if (!hmbird_rq) -+ return 1; -+ -+ lockdep_assert_rq_held(rq); -+ -+ if (static_branch_unlikely(&hmbird_ops_cpu_preempt) && -+ unlikely(get_hmbird_rq(rq)->cpu_released)) { -+ /* -+ * If the previous sched_class for the current CPU was not HMBIRD, -+ * notify the BPF scheduler that it again has control of the -+ * core. This callback complements ->cpu_release(), which is -+ * emitted in hmbird_notify_pick_next_task(). -+ */ -+ get_hmbird_rq(rq)->cpu_released = false; -+ } -+ -+ if (prev_on_hmbird) -+ update_curr_hmbird(rq); -+ /* if there already are tasks to run, nothing to do */ -+ if (hmbird_rq->local_dsq.nr) -+ return 1; -+ -+ if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) { -+ slim_stats_record(GLOBAL_STAT, 1, 0, 0); -+ return 1; -+ } -+ -+ if (consume_dispatch_global(rq, rf)) -+ return 1; -+ -+ return 0; -+} -+ -+static int balance_hmbird(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf) -+{ -+ return balance_one(rq, prev, rf, true); -+} -+ -+/* -+ * output task util to systrace, only for debug mode. -+ * we can not output too many logs to systrace buffer even in debug mode -+ * only output debug-info while it exceed misfit_ds. -+ */ -+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p) -+{ -+ static DEFINE_PER_CPU(int, is_last_exceed); -+ int cpu = cpu_of(rq); -+ u64 util = 0; -+ -+ if (likely(!debug_enabled())) -+ return; -+ -+ if (!p) -+ return; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ util = uclamp_rq_util_with(rq, util, p); -+ -+ if (util >= misfit_ds) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util); -+ per_cpu(is_last_exceed, cpu) = true; -+ } else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0); -+ per_cpu(is_last_exceed, cpu) = false; -+ } else { -+ } -+} -+ -+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first) -+{ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ /* -+ * Core-sched might decide to execute @p before it is -+ * dispatched. Call ops_dequeue() to notify the BPF scheduler. -+ */ -+ ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC); -+ dispatch_dequeue(get_hmbird_rq(rq), p); -+ } -+ -+ p->se.exec_start = rq_clock_task(rq); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK); -+ } -+ -+ watchdog_unwatch_task(p, true); -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx); -+ systrace_output_cpu_ds(rq, p); -+ /* -+ * @p is getting newly scheduled or got kicked after someone updated its -+ * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird(). -+ */ -+ if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) != -+ (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) { -+ if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) -+ get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK; -+ else -+ get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK; -+ -+ sched_update_tick_dependency(rq); -+ } -+ -+ p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime; -+} -+ -+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p) -+{ -+ update_curr_hmbird(rq); -+ -+ update_dispatch_dsq_info(rq, p); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ watchdog_watch_task(rq, p); -+ -+ if (is_pipeline_task(p)) { -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1); -+ return; -+ } -+ -+ /* -+ * If we're in the pick_next_task path, balance_hmbird() should -+ * have already populated the local DSQ if there are any other -+ * available tasks. If empty, tell ops.enqueue() that @p is the -+ * only one available for this cpu. ops.enqueue() should put it -+ * on the local DSQ so that the subsequent pick_next_task_hmbird() -+ * can find the task unless it wants to trigger a separate -+ * follow-up scheduling event. -+ */ -+ if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1); -+ else -+ do_enqueue_task(rq, p, 0, -1); -+ } -+} -+ -+static struct task_struct *first_local_task(struct rq *rq) -+{ -+ struct rb_node *rb_node; -+ struct hmbird_entity *entity; -+ -+ if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) { -+ entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo, -+ struct hmbird_entity, dsq_node.fifo); -+ return entity->task; -+ } -+ -+ rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq); -+ if (rb_node) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ return entity->task; -+ } -+ return NULL; -+} -+ -+static struct task_struct *pick_next_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p; -+ -+ p = first_local_task(rq); -+ if (!p) -+ return NULL; -+ -+ if (unlikely(!get_hmbird_ts(p)->slice)) { -+ if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice) -+ hmbird_warned_zero_slice = true; -+ -+ refill_task_slice(p); -+ } -+ -+ set_next_task_hmbird(rq, p, true); -+ -+ return p; -+} -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task, -+ const struct sched_class *active) -+{ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * The callback is conceptually meant to convey that the CPU is no -+ * longer under the control of HMBIRD. Therefore, don't invoke the -+ * callback if the CPU is staying on HMBIRD, or going idle (in which -+ * case the HMBIRD scheduler has actively decided not to schedule any -+ * tasks on the CPU). -+ */ -+ if (likely(active >= &hmbird_sched_class)) -+ return; -+ -+ /* -+ * At this point we know that HMBIRD was preempted by a higher priority -+ * sched_class, so invoke the ->cpu_release() callback if we have not -+ * done so already. We only send the callback once between HMBIRD being -+ * preempted, and it regaining control of the CPU. -+ * -+ * ->cpu_release() complements ->cpu_acquire(), which is emitted the -+ * next time that balance_hmbird() is invoked. -+ */ -+ if (!get_hmbird_rq(rq)->cpu_released) -+ get_hmbird_rq(rq)->cpu_released = true; -+} -+ -+#ifdef CONFIG_SMP -+ -+static bool test_and_clear_cpu_idle(int cpu) -+{ -+ if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -+ if (cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ return true; -+ } else { -+ return false; -+ } -+} -+ -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) -+{ -+ int cpu; -+ -+ do { -+ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -+ if (cpu >= nr_cpu_ids) -+ return -EBUSY; -+ } while (!test_and_clear_cpu_idle(cpu)); -+ -+ return cpu; -+} -+ -+ -+static bool prev_cpu_misfit(int prev) -+{ -+ if (!is_partial_enabled() && is_partial_cpu(prev)) -+ return true; -+ -+ return false; -+} -+ -+static int heavy_rt_placement(struct task_struct *p, int prev) -+{ -+ struct cpumask tmp = {.bits = {0}}; -+ int cpu; -+ u64 util = 0; -+ -+ if (!rt_prio(p->prio)) -+ return -EFAULT; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ -+ if (util < misfit_ds) -+ return -EFAULT; -+ -+ if (is_partial_enabled()) -+ cpumask_or(&tmp, iso_masks.big, iso_masks.partial); -+ else -+ cpumask_copy(&tmp, iso_masks.big); -+ -+ cpu = hmbird_pick_idle_cpu(&tmp); -+ if (cpu >= 0) -+ return cpu; -+ -+ if (cpumask_test_cpu(prev, iso_masks.big) || -+ (is_partial_enabled() && is_partial_cpu(prev))) -+ return prev; -+ -+ return cpumask_first(&tmp); -+} -+ -+static int spec_task_before_pick_idle(struct task_struct *p, int prev) -+{ -+ int cpu; -+ -+ cpu = heavy_rt_placement(p, prev); -+ if (cpu >= 0) -+ return cpu; -+ return -EFAULT; -+} -+ -+static int cpumask_distribute_next(struct cpumask *mask, int *prev) -+{ -+ int p = *prev, n; -+ -+ n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1); -+ if (n < nr_cpu_ids) -+ WRITE_ONCE(*prev, n); -+ return n; -+} -+ -+static int repick_fallback_cpu(void) -+{ -+ /* -+ * partial cpu follow big cluster's Scheduling policy, -+ * simply return first bit cpu. -+ */ -+ return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev); -+} -+ -+/* -+ * Must return a valid cpu num, as this task's cpu. -+ */ -+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu, -+ struct cpumask *mask, int *prev_mask) -+{ -+ int cpu; -+ -+ cpu = hmbird_pick_idle_cpu(mask); -+ if (cpu >= 0) -+ return cpu; -+ return cpumask_distribute_next(mask, prev_mask); -+} -+ -+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type) -+{ -+ int idx; -+ struct cluster_ctx ctx; -+ -+ idx = find_idx_from_task(p); -+ if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS) -+ return false; -+ -+ gen_cluster_ctx(&ctx, type); -+ if (idx >= ctx.lower && idx < ctx.upper) -+ return true; -+ return false; -+} -+ -+static bool is_valid_cpu(int cpu) -+{ -+ return (cpu >= 0) && (cpu < nr_cpu_ids); -+} -+ -+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) -+{ -+ s32 cpu = -1; -+ struct cpumask mask = {.bits = {0}}; -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (is_valid_cpu(cpu)) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ partial_dynamic_ctrl(); -+ -+ if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ iso_masks.big, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ if (p->nr_cpus_allowed == 1) { -+ cpu = cpumask_any(p->cpus_ptr); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* For non-period global dsq, not contain pcp task. */ -+ if (task_only_blongs_to_cluster(p, LITTLE)) { -+ cpumask_copy(&mask, iso_masks.little); -+ if (unlikely(l_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &little_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ if (task_only_blongs_to_cluster(p, BIG)) { -+ cpumask_copy(&mask, iso_masks.big); -+ if (unlikely(b_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ cpu = spec_task_before_pick_idle(p, prev_cpu); -+ if (is_valid_cpu(cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* if the previous CPU is idle, dispatch directly to it */ -+ if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+ } -+ -+ cpu = hmbird_pick_idle_cpu(cpu_possible_mask); -+ if (is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ if (prev_cpu_misfit(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ cpu = repick_fallback_cpu(); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+} -+ -+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags) -+{ -+ return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags); -+} -+ -+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx) -+{ -+ set_cpus_allowed_common(p, ctx); -+} -+ -+static void reset_idle_masks(void) -+{ -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little); -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big); -+ if (is_partial_enabled()) -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial); -+ hmbird_has_idle_cpus = true; -+} -+ -+void __hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ int cpu = cpu_of(rq); -+ struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -+ -+ if (skip_update_idle()) -+ return; -+ -+ if (idle) { -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ if (!hmbird_has_idle_cpus) -+ hmbird_has_idle_cpus = true; -+ -+ /* -+ * idle_masks.smt handling is racy but that's fine as it's only -+ * for optimization and self-correcting. -+ */ -+ for_each_cpu(cpu, sib_mask) { -+ if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -+ return; -+ } -+ cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -+ } else { -+ cpumask_clear_cpu(cpu, idle_masks.cpu); -+ if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ -+ cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -+ } -+} -+ -+#else /* !CONFIG_SMP */ -+ -+static bool test_and_clear_cpu_idle(int cpu) { return false; } -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } -+static void reset_idle_masks(void) {} -+ -+#endif /* CONFIG_SMP */ -+ -+static bool check_rq_for_timeouts(struct rq *rq) -+{ -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rq_flags rf; -+ -+ rq_lock_irqsave(rq, &rf); -+ list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) { -+ unsigned long last_runnable; -+ -+ p = entity->task; -+ last_runnable = get_hmbird_ts(p)->runnable_at; -+ -+ if (unlikely(time_after(jiffies, -+ last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -+ -+ rq_unlock_irqrestore(rq, &rf); -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb", -+ p->comm, p->pid, -+ dur_ms / 1000, dur_ms % 1000, -+ get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0, -+ cpumask_pr_args(&p->cpus_mask)); -+ return 1; -+ } -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ return 0; -+} -+ -+static void hmbird_watchdog_workfn(struct work_struct *work) -+{ -+ int cpu; -+ bool timeout; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ -+ for_each_online_cpu(cpu) { -+ timeout = check_rq_for_timeouts(cpu_rq(cpu)); -+ if (unlikely(timeout)) -+ break; -+ -+ cond_resched(); -+ } -+ if (!timeout) -+ queue_delayed_work(system_unbound_wq, to_delayed_work(work), -+ hmbird_watchdog_timeout / 2); -+} -+ -+static void set_pcp_round(struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) { -+ per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round); -+ per_cpu(pcp_info, cpu).pcp_round = true; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+} -+ -+ -+/* -+ * Just for debug: output hmbird on/off state per 10s. -+ */ -+#define OUTPUT_INTVAL (msecs_to_jiffies(10 * 1000)) -+static void inform_hmbird_onoff_from_systrace(void) -+{ -+ static unsigned long __read_mostly next_print; -+ -+ if (time_before(jiffies, READ_ONCE(next_print))) -+ return; -+ -+ WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL); -+ hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio); -+} -+ -+void hmbird_notify_sched_tick(void) -+{ -+ unsigned long last_check; -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ hmbird_scheduler_tick(); -+ -+ if (!hmbird_enabled()) -+ return; -+ -+ last_check = hmbird_watchdog_timestamp; -+ if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -+ -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "watchdog failed to check in for %u.%03us", -+ dur_ms / 1000, dur_ms % 1000); -+ } -+ scan_timeout(rq); -+} -+ -+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued) -+{ -+ update_curr_hmbird(rq); -+ -+ set_pcp_round(rq); -+ -+ if (slim_walt_ctrl) -+ hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE); -+ /* -+ * While disabling, always resched and refresh core-sched timestamp as -+ * we can't trust the slice management or ops.core_sched_before(). -+ */ -+ if (hmbird_ops_disabling()) -+ get_hmbird_ts(curr)->slice = 0; -+ -+ if (!get_hmbird_ts(curr)->slice) -+ resched_curr(rq); -+ -+ inform_hmbird_onoff_from_systrace(); -+} -+ -+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg) -+{ -+ hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ get_hmbird_ts(p)->disallow = false; -+ -+ hmbird_sched_init_task(p); -+ -+ set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ return 0; -+} -+ -+static void hmbird_ops_enable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void hmbird_ops_disable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void set_task_hmbird_weight(struct task_struct *p) -+{ -+ u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -+ -+ get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight); -+} -+ -+/** -+ * refresh_hmbird_weight - Refresh a task's hmbird weight -+ * @p: task to refresh hmbird weight for -+ * -+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to -+ * enable easy access from the BPF scheduler. To keep it synchronized with the -+ * current task priority, this function should be called when a new task is -+ * created, priority is changed for a task on hmbird, and a task is switched -+ * to hmbird from other classes. -+ */ -+static void refresh_hmbird_weight(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ set_task_hmbird_weight(p); -+} -+ -+int hmbird_pre_fork(struct task_struct *p) -+{ -+ int ret = 0; -+ -+ p->android_oem_data1[HMBIRD_TS_IDX] = -+ (u64)(kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL)); -+ if (!get_hmbird_ts(p)) { -+ ret = -1; -+ goto lock; -+ } -+ -+ get_hmbird_ts(p)->dsq = NULL; -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node); -+ get_hmbird_ts(p)->flags = 0; -+ get_hmbird_ts(p)->weight = 0; -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ get_hmbird_ts(p)->holding_cpu = -1; -+ get_hmbird_ts(p)->kf_mask = 0; -+ atomic64_set(&get_hmbird_ts(p)->ops_state, 0); -+ get_hmbird_ts(p)->runnable_at = INITIAL_JIFFIES; -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+ get_hmbird_ts(p)->task = p; -+ hmbird_set_sched_prop(p, 0); -+ -+ get_hmbird_ts(p)->critical_affinity_cpu = -1; -+ get_hmbird_ts(p)->sched_class = &hmbird_sched_class; -+ -+ /* -+ * BPF scheduler enable/disable paths want to be able to iterate and -+ * update all tasks which can become complex when racing forks. As -+ * enable/disable are very cold paths, let's use a percpu_rwsem to -+ * exclude forks. -+ */ -+lock: -+ percpu_down_read(&hmbird_fork_rwsem); -+ -+ return ret; -+} -+ -+int hmbird_fork(struct task_struct *p) -+{ -+ percpu_rwsem_assert_held(&hmbird_fork_rwsem); -+ -+ if (hmbird_enabled()) -+ return hmbird_ops_prepare_task(p, task_group(p)); -+ else -+ return 0; -+} -+ -+void hmbird_post_fork(struct task_struct *p) -+{ -+ if (hmbird_enabled()) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ /* -+ * Set the weight manually before calling ops.enable() so that -+ * the scheduler doesn't see a stale value if they inspect the -+ * task struct. We'll invoke ops.set_weight() afterwards, as it -+ * would be odd to receive a callback on the task before we -+ * tell the scheduler that it's been fully enabled. -+ */ -+ set_task_hmbird_weight(p); -+ hmbird_ops_enable_task(p); -+ refresh_hmbird_weight(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ -+ spin_lock_irq(&hmbird_tasks_lock); -+ list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_cancel_fork(struct task_struct *p) -+{ -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ if (hmbird_enabled()) -+ hmbird_ops_disable_task(p); -+ -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_free(struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+ list_del_init(&get_hmbird_ts(p)->tasks_node); -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+ -+ /* -+ * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED -> -+ * ENABLED transitions can't race us. Disable ops for @p. -+ */ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) || -+ test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ hmbird_ops_disable_task(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+} -+ -+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio) -+{ -+} -+ -+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type) -+{ -+ return (prop >> TOP_TASK_SHIFT) & (1 << type); -+} -+ -+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p) -+{ -+ uint32_t prop = get_top_task_prop(p); -+ -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE)) -+ return HMBIRD_TASK_PROP_PIPELINE; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) || -+ !task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) { -+ return HMBIRD_TASK_PROP_COMMON; -+ } -+ return HMBIRD_TASK_PROP_DEBUG_OR_LOG; -+} -+ -+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b) -+{ -+ int type_a = hmbird_get_task_type(a); -+ int type_b = hmbird_get_task_type(b); -+ -+ return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b]; -+} -+ -+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ enum cpu_type type; -+ int sp_dl; -+ struct task_struct *curr = NULL; -+ -+ switch (hmbird_preempt_policy) { -+ case HMBIRD_PREEMPT_POLICY_PRIO_BASED: -+ curr = rq->curr; -+ if (curr && hmbird_prio_higher(p, curr)) -+ goto preempt; -+ break; -+ default: -+ break; -+ } -+ if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) || -+ (is_critical_system_task(p) && !is_critical_system_task(rq->curr))) -+ goto preempt; -+ -+ sp_dl = find_idx_from_task(p); -+ if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled())) -+ return; -+ -+ if (rq->curr->prio > p->prio) -+ goto preempt; -+ -+ return; -+ -+preempt: -+ resched_curr(rq); -+} -+ -+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {} -+ -+int hmbird_check_setscheduler(struct task_struct *p, int policy) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ /* if disallow, reject transitioning into HMBIRD */ -+ if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) && -+ p->policy != policy && policy == SCHED_HMBIRD) -+ return -EACCES; -+ -+ return 0; -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool hmbird_can_stop_tick(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ if (hmbird_ops_disabling()) -+ return false; -+ -+ if (p->sched_class != &hmbird_sched_class) -+ return true; -+ -+ return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK; -+} -+#endif -+ -+int hmbird_tg_online(struct task_group *tg) -+{ -+ struct cgroup *cgrp; -+ -+ if (!tg) -+ return 0; -+ cgrp = tg->css.cgroup; -+ if (!cgrp || !(cgrp->kn)) -+ return 0; -+ update_cgroup_ids_table(cgrp->kn->id, -1); -+ return 0; -+} -+ -+/* -+ * Omitted operations: -+ * -+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -+ * task isn't tied to the CPU at that point. Preemption is implemented by -+ * resetting the victim task's slice to 0 and triggering reschedule on the -+ * target CPU. -+ * -+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -+ * -+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -+ * their current sched_class. Call them directly from sched core instead. -+ * -+ * - task_woken, switched_from: Unnecessary. -+ */ -+DEFINE_SCHED_CLASS(hmbird) = { -+ .enqueue_task = enqueue_task_hmbird, -+ .dequeue_task = dequeue_task_hmbird, -+ .yield_task = yield_task_hmbird, -+ .yield_to_task = yield_to_task_hmbird, -+ -+ .check_preempt_curr = check_preempt_curr_hmbird, -+ -+ .pick_next_task = pick_next_task_hmbird, -+ -+ .put_prev_task = put_prev_task_hmbird, -+ .set_next_task = set_next_task_hmbird, -+ -+#ifdef CONFIG_SMP -+ .balance = balance_hmbird, -+ .select_task_rq = select_task_rq_hmbird, -+ .set_cpus_allowed = set_cpus_allowed_hmbird, -+#endif -+ .task_tick = task_tick_hmbird, -+ -+ .switched_to = switched_to_hmbird, -+ .prio_changed = prio_changed_hmbird, -+ -+ .update_curr = update_curr_hmbird, -+ -+#ifdef CONFIG_UCLAMP_TASK -+ .uclamp_enabled = 0, -+#endif -+}; -+ -+/* -+ * Must with rq lock held. -+ */ -+bool task_is_hmbird(struct task_struct *p) -+{ -+ return p->sched_class == &hmbird_sched_class; -+} -+ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id) -+{ -+ memset(dsq, 0, sizeof(*dsq)); -+ -+ raw_spin_lock_init(&dsq->lock); -+ INIT_LIST_HEAD(&dsq->fifo); -+ dsq->id = dsq_id; -+} -+ -+static int hmbird_cgroup_init(void) -+{ -+ struct cgroup_subsys_state *css; -+ -+ css_for_each_descendant_pre(css, &root_task_group.css) { -+ struct task_group *tg = css_tg(css); -+ -+ cgrp_dsq_idx_init(css->cgroup, tg); -+ } -+ return 0; -+} -+ -+/* -+ * Used by sched_fork() and __setscheduler_prio() to pick the matching -+ * sched_class. dl/rt are already handled. -+ */ -+bool task_on_hmbird(struct task_struct *p) -+{ -+ return hmbird_enabled(); -+} -+ -+static void __setscheduler_prio(struct task_struct *p, int prio) -+{ -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) { -+ p->prio = prio; -+ return; -+ } else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (on_hmbird && rt_prio(prio)) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ -+ p->prio = prio; -+} -+ -+/* -+ * Heartbeat, avoid humbird keep running while APP already exit. -+ * Check whether APP send alive-signal periodly. -+ */ -+#define HEARTBEAT_TIMEOUT (msecs_to_jiffies(2500)) -+#define HEARTBEAT_CHECK_INTERVAL (msecs_to_jiffies(1000)) -+static struct timer_list hb_timer; -+static unsigned long next_hb; -+ -+void hb_timer_handler(struct timer_list *timer) -+{ -+ if (!heartbeat_enable) -+ goto refill; -+ -+ pr_err(": enter timer.\n"); -+ if (heartbeat) { -+ heartbeat = 0; -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ } -+ -+ /* can't detect heartbeat, disable ext. */ -+ if (time_after(jiffies, READ_ONCE(next_hb))) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT, -+ "can't detect heartbeat, disable ext\n"); -+ } -+refill: -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_start(void) -+{ -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_init(void) -+{ -+ timer_setup(&hb_timer, hb_timer_handler, 0); -+} -+ -+static void hb_timer_exit(void) -+{ -+ del_timer(&hb_timer); -+} -+ -+static bool check_and_disable_cpuhp(void) -+{ -+ struct rq *rq; -+ struct rq_flags rf; -+ int cpu; -+ -+ cpu_hotplug_disable(); -+ cpus_read_lock(); -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ rq_lock_irqsave(rq, &rf); -+ if (!rq->online) { -+ rq_unlock_irqrestore(rq, &rf); -+ goto err_offline; -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ cpus_read_unlock(); -+ return true; -+ -+err_offline: -+ cpus_read_unlock(); -+ cpu_hotplug_enable(); -+ return false; -+} -+ -+static void reenable_cpuhp(void) -+{ -+ cpu_hotplug_enable(); -+} -+ -+static void scheduler_switch_done(bool final_state) -+{ -+ /* set scx_enable again, switch may fail. */ -+ scx_enable = final_state; -+ /* reset heartbeat*/ -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ -+ if (final_state) -+ hb_timer_start(); -+ else -+ hb_timer_exit(); -+} -+ -+/** -+ * ss : curr switch state -+ * finish : success or fail -+ * enable : curr operation is diabling or enabling? -+ * fail_reason : string output when failed, empty string when success. -+ */ -+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason) -+{ -+ char *s1 = finish ? "finished" : "failed"; -+ char *s2 = enable ? "enabled" : "disabled"; -+ -+ hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss); -+ if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) { -+ sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type)); -+ hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n", -+ s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason); -+ } -+ curr_ss = ss; -+} -+ -+bool get_hmbird_ops_enabled(void) -+{ -+ return atomic_read(&__hmbird_ops_enabled); -+} -+ -+bool get_non_hmbird_task(void) -+{ -+ return atomic_read(&non_hmbird_task); -+} -+ -+static void hmbird_ops_disable_workfn(struct kthread_work *work) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int cpu; -+ -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, ""); -+ cancel_delayed_work_sync(&hmbird_watchdog_work); -+ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) { -+ case HMBIRD_OPS_DISABLED: -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled"); -+ scheduler_switch_done(false); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return; -+ case HMBIRD_OPS_PREPPING: -+ fallthrough; -+ case HMBIRD_OPS_DISABLING: -+ /* shouldn't happen but handle it like ENABLING if it does */ -+ WARN_ONCE(true, "hmbird: duplicate disabling instance?"); -+ fallthrough; -+ case HMBIRD_OPS_ENABLING: -+ case HMBIRD_OPS_ENABLED: -+ break; -+ } -+ -+ /* kick all CPUs to restore ticks */ -+ for_each_possible_cpu(cpu) -+ resched_cpu(cpu); -+ -+ /* avoid racing against fork and cgroup changes */ -+ cpus_read_lock(); -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, ""); -+ spin_lock_irq(&hmbird_tasks_lock); -+ atomic_set(&__hmbird_ops_enabled, false); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ bool alive = READ_ONCE(p->__state) != TASK_DEAD; -+ -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ get_hmbird_ts(p)->slice = min_t(u64, -+ get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL); -+ -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ if (alive) -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ -+ hmbird_ops_disable_task(p); -+ } -+ hmbird_task_iter_exit(&sti); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, ""); -+ -+ atomic_set(&non_hmbird_task, true); -+ /* no task is on hmbird, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt); -+ synchronize_rcu(); -+ -+ percpu_up_write(&hmbird_fork_rwsem); -+ cpus_read_unlock(); -+ -+ if (slim_walt_ctrl) -+ slim_walt_enable(false); -+ -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 1, 0, ""); -+ scheduler_switch_done(false); -+ reenable_cpuhp(); -+} -+ -+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn); -+ -+static void schedule_hmbird_ops_disable_work(void) -+{ -+ struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper); -+ -+ /* -+ * We may be called spuriously before the first bpf_hmbird_reg(). If -+ * hmbird_ops_helper isn't set up yet, there's nothing to do. -+ */ -+ if (helper) -+ kthread_queue_work(helper, &hmbird_ops_disable_work); -+} -+ -+static void hmbird_ops_disable(void) -+{ -+ schedule_hmbird_ops_disable_work(); -+} -+ -+static void hmbird_err_exit_workfn(struct work_struct *work) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ hmbird_ctrl(false); -+ -+ for_each_present_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy) -+ continue; -+ if (cpu != policy->cpu) -+ goto put; -+ down_write(&policy->rwsem); -+ WARN_ON(store_scaling_governor(policy, -+ saved_gov[cpu], strlen(saved_gov[cpu])) <= 0); -+ up_write(&policy->rwsem); -+ hmbird_info_trace(":restore origin gov : %s\n", saved_gov[cpu]); -+put: -+ cpufreq_cpu_put(policy); -+ } -+ memset((char *)saved_gov, 0, sizeof(saved_gov)); -+} -+ -+void hmbird_ops_exit(void) -+{ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); -+} -+ -+static struct kthread_worker *hmbird_create_rt_helper(const char *name) -+{ -+ struct kthread_worker *helper; -+ -+ helper = kthread_create_worker(0, name); -+ if (helper) -+ sched_set_fifo(helper->task); -+ return helper; -+} -+ -+static inline void set_audio_thread_sched_prop(struct task_struct *p) -+{ -+ struct cgroup_subsys_state *css; -+ -+ if (likely(p->prio >= MAX_RT_PRIO)) -+ return; -+ -+ rcu_read_lock(); -+ css = task_css(p, cpuset_cgrp_id); -+ if (!css) { -+ rcu_read_unlock(); -+ return; -+ } -+ rcu_read_unlock(); -+ -+ if (!strcmp(css->cgroup->kn->name, "audio-app")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+} -+ -+static int hmbird_ops_enable(void *unused) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int ret; -+ int tcnt = 0; -+ unsigned long long start = 0; -+ -+ if (!check_and_disable_cpuhp()) { -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline"); -+ return -EBUSY; -+ } -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, ""); -+ mutex_lock(&hmbird_ops_enable_mutex); -+ -+ if (!hmbird_ops_helper) { -+ WRITE_ONCE(hmbird_ops_helper, -+ hmbird_create_rt_helper("hmbird_ops_helper")); -+ if (!hmbird_ops_helper) { -+ ret = -ENOMEM; -+ goto err_unlock; -+ } -+ } -+ -+ if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) { -+ ret = -EBUSY; -+ goto err_unlock; -+ } -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) != -+ HMBIRD_OPS_DISABLED); -+ -+ hmbird_warned_zero_slice = false; -+ -+ atomic64_set(&hmbird_nr_rejected, 0); -+ -+ /* -+ * Keep CPUs stable during enable so that the BPF scheduler can track -+ * online CPUs by watching ->on/offline_cpu() after ->init(). -+ */ -+ cpus_read_lock(); -+ -+ hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work, -+ hmbird_watchdog_timeout / 2); -+ -+ /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ */ -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ reset_idle_masks(); -+ -+ /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. -+ */ -+ ret = hmbird_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock; -+ -+ /* -+ * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem -+ * preventing new tasks from being added. No need to exclude tasks -+ * leaving as hmbird_free() can handle both prepped and enabled -+ * tasks. Prep all tasks first and then enable them with preemption -+ * disabled. -+ */ -+ spin_lock_irq(&hmbird_tasks_lock); -+ -+ atomic_set(&non_hmbird_task, false); -+ atomic_set(&__hmbird_ops_enabled, true); -+ -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered(&sti))) -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ hmbird_task_iter_exit(&sti); -+ -+ /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. -+ */ -+ preempt_disable(); -+ -+ /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ */ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) { -+ atomic_set(&non_hmbird_task, true); -+ atomic_set(&__hmbird_ops_enabled, false); -+ preempt_enable(); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ ret = -EBUSY; -+ goto err_disable_unlock; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED -+ * transitions here are synchronized against hmbird_free() through -+ * hmbird_tasks_lock. -+ */ -+ start = sched_clock(); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, ""); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ tcnt++; -+ if (READ_ONCE(p->__state) != TASK_DEAD) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ -+ set_audio_thread_sched_prop(p); -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ hmbird_ops_enable_task(p); -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ } else { -+ hmbird_ops_disable_task(p); -+ } -+ } -+ hmbird_task_iter_exit(&sti); -+ -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, ""); -+ preempt_enable(); -+ percpu_up_write(&hmbird_fork_rwsem); -+ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) { -+ ret = -EBUSY; -+ goto err_disable; -+ } -+ -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_ENABLED, 1, 1, ""); -+ scheduler_switch_done(true); -+ -+ return 0; -+ -+err_unlock: -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock"); -+ scheduler_switch_done(false); -+ return ret; -+ -+err_disable_unlock: -+ percpu_up_write(&hmbird_fork_rwsem); -+err_disable: -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ /* must be fully disabled before returning */ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable"); -+ scheduler_switch_done(false); -+ return ret; -+} -+ -+#ifdef CONFIG_SCHED_DEBUG -+static const char *hmbird_ops_enable_state_str[] = { -+ [HMBIRD_OPS_PREPPING] = "prepping", -+ [HMBIRD_OPS_ENABLING] = "enabling", -+ [HMBIRD_OPS_ENABLED] = "enabled", -+ [HMBIRD_OPS_DISABLING] = "disabling", -+ [HMBIRD_OPS_DISABLED] = "disabled", -+}; -+ -+static int hmbird_debug_show(struct seq_file *m, void *v) -+{ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled()); -+ seq_printf(m, "%-30s: %s\n", "enable_state", -+ hmbird_ops_enable_state_str[hmbird_ops_enable_state()]); -+ seq_printf(m, "%-30s: %llu\n", "nr_rejected", -+ atomic64_read(&hmbird_nr_rejected)); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return 0; -+} -+ -+static int hmbird_debug_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_debug_show, NULL); -+} -+ -+const struct file_operations sched_hmbird_fops = { -+ .open = hmbird_debug_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif -+ -+ -+static int bpf_hmbird_reg(void *kdata) -+{ -+ return hmbird_ops_enable(kdata); -+} -+ -+static int bpf_hmbird_unreg(void *kdata) -+{ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ return 0; -+} -+ -+void set_hmbird_module_loaded(int is_loaded) -+{ -+ atomic_set(&hmbird_module_loaded, is_loaded); -+} -+ -+/* -+ * MUST load hmbird module before enable hmbird scheduler -+ * load track & hmbird gover implemented in hmbird module -+ */ -+int hmbird_ctrl(bool enable) -+{ -+ if (!atomic_read(&hmbird_module_loaded)) { -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "ext module unloaded\n"); -+ return -EINVAL; -+ } -+ -+ if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED -+ || hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING -+ || hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already enabled(ing)\n"); -+ hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING || -+ hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already disabled(ing)\n"); -+ hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (enable) -+ return bpf_hmbird_reg(NULL); -+ else -+ return bpf_hmbird_unreg(NULL); -+} -+ -+void set_cpu_isomask(int cpu, cpumask_var_t *mask) -+{ -+ cpumask_clear_cpu(cpu, iso_masks.ex_free); -+ cpumask_clear_cpu(cpu, iso_masks.exclusive); -+ cpumask_clear_cpu(cpu, iso_masks.partial); -+ cpumask_clear_cpu(cpu, iso_masks.big); -+ cpumask_clear_cpu(cpu, iso_masks.little); -+ cpumask_set_cpu(cpu, *mask); -+} -+ -+void set_cpu_cluster(u64 cpu_cluster) -+{ -+ int cpu; -+ -+ for_each_present_cpu(cpu) { -+ u64 pos = 1 << cpu; -+ -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.ex_free)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.exclusive)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.partial)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.big)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) -+ set_cpu_isomask(cpu, &(iso_masks.little)); -+ } -+} -+ -+static void get_hmbird_snapshot(struct panic_snapshot_t *p) -+{ -+ struct hmbird_dispatch_q *dsq; -+ struct rq *rq; -+ int cpu, i; -+ -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ p->rq_nr[cpu] = rq->nr_running; -+ p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running; -+ } -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ struct hmbird_entity *entity; -+ -+ dsq = &gdsqs[i]; -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo)) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ p->runnable_at[i] = entity->runnable_at; -+ raw_spin_unlock(&dsq->lock); -+ -+ } -+ -+ p->snap_misc.hmbird_enabled = hmbird_enabled(); -+ p->snap_misc.curr_ss = curr_ss; -+ p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state(); -+ p->snap_misc.parctrl_high_ratio = parctrl_high_ratio; -+ p->snap_misc.parctrl_low_ratio = parctrl_low_ratio; -+ p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l; -+ p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l; -+ p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio; -+ p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio; -+ p->snap_misc.misfit_ds = misfit_ds; -+ p->snap_misc.partial_enable = partial_enable; -+ p->snap_misc.iso_free_rescue = iso_free_rescue; -+ p->snap_misc.isolate_ctrl = isolate_ctrl; -+ p->snap_misc.snap_jiffies = jiffies; -+ p->snap_misc.snap_time = local_clock(); -+} -+ -+// MTK minidump begin -+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3) -+{ -+ strscpy(m->desc_str, str, DESC_STR_LEN); -+ m->len = d1 * d2 * d3; -+ m->parse[0] = d1; -+ m->parse[1] = d2; -+ m->parse[2] = d3; -+} -+ -+static void init_desc_metas(struct md_info_t *m) -+{ -+ init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :", -+ 1, MAX_SWITCHS, SWITCH_ITEMS); -+ -+ init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1); -+ -+ init_desc_meta(&m->kern_dump.excep_rec_meta, -+ "excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS); -+ -+ init_desc_meta(&m->kern_dump.excep_idx_meta, -+ "excep idx :", 1, 1, MAX_EXCEP_ID); -+ -+ init_desc_meta(&m->kern_dump.snap.runnable_at_meta, -+ "each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS); -+ -+ init_desc_meta(&m->kern_dump.snap.rq_nr_meta, -+ "rq runnable task nr:", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta, -+ "scxrq runnable task nr :", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.snap_misc_meta, -+ "misc snap params :", 1, 1, SNAP_ITEMS); -+} -+ -+static void init_md_meta(struct md_info_t *m) -+{ -+ m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64); -+ m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64); -+ m->meta.unit_size = sizeof(u64); -+ m->meta.switches = MAX_SWITCHS; -+ m->meta.exceps = MAX_EXCEPS; -+ m->meta.global_dsqs = MAX_GLOBAL_DSQS; -+ m->meta.parse_dimens = PARSE_DIMENS; -+ m->meta.nr_cpus = num_possible_cpus(); -+ m->meta.real_cpus = nr_cpu_ids; -+ m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64); -+ m->meta.nr_meta_desc = 8; -+ m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64); -+ -+ init_desc_metas(m); -+} -+ -+#define MINIDUMP_DFL_SIZE (4 * 1024) -+ -+struct notifier_block hmbird_panic_blk; -+static int hmbird_panic_handler(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ if (!md_info) -+ return NOTIFY_DONE; -+ -+ get_hmbird_snapshot(&md_info->kern_dump.snap); -+ -+ return NOTIFY_DONE; -+} -+ -+static void panic_blk_init(void) -+{ -+ int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE); -+ -+ md_info = kzalloc(dump_size, GFP_KERNEL); -+ if (!md_info) -+ return; -+ init_md_meta(md_info); -+ -+ hmbird_panic_blk.notifier_call = hmbird_panic_handler; -+ /* make sure to execute before minidump. */ -+ hmbird_panic_blk.priority = INT_MAX; -+ atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk); -+ -+ hmbird_debug("register minidump.\n"); -+} -+ -+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size) -+{ -+ *vaddr = (unsigned long)md_info; -+ *size = sizeof(struct md_info_t); -+} -+// MTK minidump end -+ -+static inline void init_sched_prop_to_preempt_prio(void) -+{ -+ for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) { -+ switch (i) { -+ case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 5; -+ break; -+ -+ case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 4; -+ break; -+ -+ case HMBIRD_TASK_PROP_PIPELINE: -+ case HMBIRD_TASK_PROP_ISOLATE: -+ sched_prop_to_preempt_prio[i] = 3; -+ break; -+ -+ case HMBIRD_TASK_PROP_COMMON: -+ sched_prop_to_preempt_prio[i] = 1; -+ break; -+ -+ case HMBIRD_TASK_PROP_DEBUG_OR_LOG: -+ sched_prop_to_preempt_prio[i] = 0; -+ break; -+ -+ default: -+ sched_prop_to_preempt_prio[i] = 2; -+ break; -+ } -+ } -+} -+ -+void __init init_sched_hmbird_class(void) -+{ -+ int cpu; -+ u32 v; -+ struct hmbird_entity *init_hmbird; -+ -+ /* -+ * The following is to prevent the compiler from optimizing out the enum -+ * definitions so that BPF scheduler implementations can use them -+ * through the generated vmlinux.h. -+ */ -+ WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP | -+ HMBIRD_KICK_PREEMPT); -+ -+ init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL); -+ init_dsq_at_boot(); -+ init_isolate_cpus(); -+ hb_timer_init(); -+ init_sched_prop_to_preempt_prio(); -+#ifdef CONFIG_SMP -+ WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); -+#endif -+ -+ /* -+ * we can't static init init_task's hmbird struct, init here. -+ * init_task->hmbird would not use during boot. -+ */ -+ init_hmbird = kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL); -+ init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird; -+ if (init_hmbird) { -+ INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo); -+ INIT_LIST_HEAD(&init_hmbird->watchdog_node); -+ init_hmbird->sticky_cpu = -1; -+ init_hmbird->holding_cpu = -1; -+ atomic64_set(&init_hmbird->ops_state, 0); -+ init_hmbird->runnable_at = jiffies; -+ init_hmbird->slice = HMBIRD_SLICE_DFL; -+ init_hmbird->task = &init_task; -+ hmbird_set_sched_prop(&init_task, 0); -+ } else { -+ hmbird_err(INIT_TASK_FAIL, ":alloc init_task.scx failed!!!\n"); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ /* -+ * exec during boot phase, no need to care about alloc failed. -+ * lifecycle same to rq, no need to free. -+ */ -+ rq->android_oem_data1[HMBIRD_RQ_IDX] = -+ (u64)kmalloc(sizeof(struct hmbird_rq), GFP_KERNEL); -+ if (get_hmbird_rq(rq)) { -+ get_hmbird_rq(rq)->rq = rq; -+ get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window; -+ } else { -+ hmbird_err(ALLOC_RQSCX_FAIL, ":alloc rq->scx failed!!!\n"); -+ } -+ -+ rq->android_oem_data1[HMBIRD_OPS_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL); -+ if (!get_hmbird_ops(rq)) -+ pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n"); -+ init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL); -+ INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list); -+ -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL)); -+ -+ hmbird_ops_init(get_hmbird_ops(rq)); -+ } -+ -+ INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn); -+ INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn); -+ hmbird_misc_init(); -+ -+ panic_blk_init(); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_misc.c b/kernel/sched/hmbird/hmbird_misc.c -new file mode 100755 -index 000000000000..52582c22052c ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_misc.c -@@ -0,0 +1,124 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * @Description: -+ * @Version: -+ * @Author: wangrui8 -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include "hmbird_sched.h" -+#include -+#include -+#include "slim.h" -+ -+noinline int tracing_mark_write(const char *buf) -+{ -+ trace_printk(buf); -+ return 0; -+} -+ -+struct yield_opt_params yield_opt_params = { -+ .enable = 0, -+ .frame_per_sec = 120, -+ .frame_time_ns = NSEC_PER_SEC / 120, -+ .yield_headroom = 10, -+}; -+ -+DEFINE_PER_CPU(struct sched_yield_state, ystate); -+ -+static inline void hmbird_yield_state_update(struct sched_yield_state *ys) -+{ -+ if (!raw_spin_is_locked(&ys->lock)) -+ return; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ -+ if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1 -+ || ys->yield_cnt_after_sleep > yield_headroom) { -+ ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP); -+ } else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) { -+ ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP); -+ } -+ ys->yield_cnt = 0; -+ ys->sleep_times = 0; -+ ys->yield_cnt_after_sleep = 0; -+} -+ -+void hmbird_skip_yield(long *skip) -+{ -+ if (!get_hmbird_ops_enabled() || !yield_opt_params.enable) -+ return; -+ unsigned long flags, sleep_now = 0; -+ struct sched_yield_state *ys; -+ int cpu = raw_smp_processor_id(), cont_yield, new_frame; -+ int frame_time_ns = yield_opt_params.frame_time_ns; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ u64 wc; -+ -+ if (!(*skip)) { -+ wc = sched_clock(); -+ ys = &per_cpu(ystate, cpu); -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ -+ cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP; -+ new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1); -+ -+ if (!cont_yield && new_frame) { -+ hmbird_yield_state_update(ys); -+ ys->last_update_time = wc; -+ ys->sleep_end = ys->last_yield_time + frame_time_ns -+ - yield_headroom * YIELD_DURATION; -+ } -+ -+ if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) { -+ *skip = true; -+ -+ sleep_now = ys->sleep_times ? -+ max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep; -+ if (wc + sleep_now > ys->sleep_end) { -+ u64 delta = ys->sleep_end - wc; -+ -+ if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION) -+ sleep_now = delta; -+ else -+ sleep_now = 0; -+ } -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ if (sleep_now) { -+ sleep_now = div64_u64(sleep_now, 1000); -+ usleep_range_state(sleep_now, sleep_now, TASK_IDLE); -+ } -+ ys->sleep_times++; -+ ys->last_yield_time = sched_clock(); -+ return; -+ } -+ if (ys->sleep_times) -+ ys->yield_cnt_after_sleep++; -+ else -+ (ys->yield_cnt)++; -+ ys->last_yield_time = wc; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+} -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops) -+{ -+ hmbird_ops->scx_enable = get_hmbird_ops_enabled; -+ hmbird_ops->check_non_task = get_non_hmbird_task; -+ hmbird_ops->hmbird_get_md_info = hmbird_get_md_info; -+} -+ -+void hmbird_misc_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_init(&ys->lock); -+ } -+ -+ set_hmbird_module_loaded(1); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_sched.h b/kernel/sched/hmbird/hmbird_sched.h -new file mode 100755 -index 000000000000..6b5ef43cb827 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef __HMBIRD_SCHED__ -+#define __HMBIRD_SCHED__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include "../../time/tick-sched.h" -+#include "../../sched/sched.h" -+#include -+#include -+#include -+ -+#define REGISTER_TRACE_VH(vender_hook, handler) \ -+{ \ -+ ret = register_trace_##vender_hook(handler, NULL); \ -+ if (ret) { \ -+ pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \ -+ } \ -+} -+#define REGISTER_TRACE(vendor_hook, handler, data, err) \ -+do { \ -+ ret = register_trace_##vendor_hook(handler, data); \ -+ if (ret) { \ -+ pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret); \ -+ goto err; \ -+ } \ -+} while (0) -+ -+#define UNREGISTER_TRACE(vendor_hook, handler, data) \ -+ unregister_trace_##vendor_hook(handler, data) \ -+ -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+ -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int sched_ravg_window_frame_per_sec; -+extern int slim_gov_debug; -+extern int cpu7_tl; -+extern int scx_gov_ctrl; -+extern spinlock_t new_sched_ravg_window_lock; -+extern int cluster_separate; -+ -+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER BIT(31) -+#define MAX_YIELD_SLEEP (2000000ULL) -+#define MIN_YIELD_SLEEP (200000ULL) -+#define YIELD_DURATION (5000ULL) -+#define DEFAULT_YIELD_SLEEP_TH (10) -+ -+struct sched_yield_state { -+ raw_spinlock_t lock; -+ u64 last_yield_time; -+ u64 last_update_time; -+ u64 sleep_end; -+ unsigned long yield_cnt; -+ unsigned long yield_cnt_after_sleep; -+ unsigned long sleep; -+ int sleep_times; -+}; -+ -+DECLARE_PER_CPU(struct sched_yield_state, ystate); -+ -+void hmbird_window_rollover_run_once(struct rq *rq); -+void hmbird_yield_state_update_per_frame(void); -+void hmbird_misc_init(void); -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops); -+#endif /*__HMBIRD_SCHED__*/ -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.c b/kernel/sched/hmbird/hmbird_sched_proc.c -new file mode 100755 -index 000000000000..234768fc767a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.c -@@ -0,0 +1,527 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "hmbird_sched_proc.h" -+#include -+ -+#include "hmbird_util_track.h" -+#include "slim.h" -+ -+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched" -+#define SLIM_FREQ_GOV_DIR "slim_freq_gov" -+#define LOAD_TRACK_DIR "slim_walt" -+#define HMBIRD_PROC_PERMISSION 0666 -+ -+int scx_enable; -+int partial_enable; -+int cpuctrl_high_ratio = 55; -+int cpuctrl_low_ratio = 40; -+int slim_stats; -+int hmbirdcore_debug; -+int slim_for_app; -+int misfit_ds = 90; -+unsigned int highres_tick_ctrl; -+unsigned int highres_tick_ctrl_dbg; -+int cpu7_tl = 70; -+int slim_walt_ctrl; -+int slim_walt_dump; -+int slim_walt_policy; -+int slim_gov_debug; -+int scx_gov_ctrl = 1; -+int sched_ravg_window_frame_per_sec = 125; -+int parctrl_high_ratio = 55; -+int parctrl_low_ratio = 40; -+int parctrl_high_ratio_l = 65; -+int parctrl_low_ratio_l = 50; -+int isoctrl_high_ratio = 75; -+int isoctrl_low_ratio = 60; -+int isolate_ctrl; -+int iso_free_rescue; -+int heartbeat; -+int heartbeat_enable = 1; -+int watchdog_enable; -+int save_gov; -+u64 cpu_cluster_masks; -+int hmbird_preempt_policy; -+int cluster_separate; -+ -+char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= 32) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtoint(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoint\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+static int set_proc_buf_val_u64(struct file *file, const char __user *buf, -+ size_t count, u64 *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= sizeof(kbuf)) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtou64(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoul\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/* common ops begin */ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ return count; -+} -+ -+static int hmbird_common_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%d\n", *(int *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_show, pde_data(inode)); -+} -+ -+static int hmbird_common_ul_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%lu\n", *(unsigned long *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_ul_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_ul_show, pde_data(inode)); -+} -+ -+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write); -+/* common ops end */ -+ -+/* scx_enable ops begin */ -+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC); -+ if (hmbird_ctrl(*pval)) -+ return -EFAULT; -+ -+ return count; -+} -+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write); -+/* scx_enable ops end */ -+ -+/* hmbird_stats ops begin */ -+#define MAX_STATS_BUF (4096) -+static int hmbird_stats_proc_show(struct seq_file *m, void *v) -+{ -+ char *buf; -+ -+ buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ stats_print(buf, MAX_STATS_BUF); -+ -+ seq_printf(m, "%s\n", buf); -+ -+ kfree(buf); -+ return 0; -+} -+ -+static int hmbird_stats_proc_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_stats_proc_show, inode); -+} -+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL); -+/* hmbird_stats ops end */ -+ -+/* sched_ravg_window_frame_per_sec ops begin */ -+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file, -+ const char __user *buf, size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ sched_ravg_window_change(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open, -+ sched_ravg_window_frame_per_sec_proc_write); -+/* sched_ravg_window_frame_per_sec ops end */ -+ -+static ssize_t save_gov_str(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ for_each_possible_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy || (cpu != policy->cpu)) -+ continue; -+ WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0); -+ hmbird_info_systrace(":save origin gov : %s\n", saved_gov[cpu]); -+ } -+ return count; -+} -+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str); -+ -+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ u64 *pval = (u64 *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val_u64(file, buf, count, pval)) -+ return -EFAULT; -+ -+ if (scx_enable == 0) -+ set_cpu_cluster(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write); -+ -+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ int tmp_val; -+ -+ if (set_proc_buf_val(file, buf, count, &tmp_val)) -+ return -EFAULT; -+ -+ slim_walt_enable(tmp_val); -+ *pval = tmp_val; -+ -+ return count; -+} -+ -+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write); -+ -+/* yield_opt ops begin */ -+static int yield_opt_show(struct seq_file *m, void *v) -+{ -+ struct yield_opt_params *data = m->private; -+ -+ seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n", -+ data->enable, data->frame_per_sec, data->yield_headroom); -+ return 0; -+} -+ -+static int yield_opt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, yield_opt_show, pde_data(inode)); -+} -+ -+static ssize_t yield_opt_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu; -+ unsigned long flags; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if (copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp -+ != 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) || -+ (yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp; -+ yield_opt_params.frame_per_sec = frame_per_sec_tmp; -+ yield_opt_params.yield_headroom = yield_headroom_tmp; -+ yield_opt_params.enable = enable_tmp; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ ys->last_yield_time = 0; -+ ys->last_update_time = 0; -+ ys->sleep_end = 0; -+ ys->yield_cnt = 0; -+ ys->yield_cnt_after_sleep = 0; -+ ys->sleep = 0; -+ ys->sleep_times = 0; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write); -+ -+ -+static int __init hmbird_proc_init(void) -+{ -+ struct proc_dir_entry *hmbird_dir; -+ struct proc_dir_entry *load_track_dir; -+ struct proc_dir_entry *freq_gov_dir; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ -+ /* mkdir /proc/hmbird_sched */ -+ hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL); -+ if (!hmbird_dir) { -+ pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &scx_enable_proc_ops, -+ &scx_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &partial_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_stats); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbirdcore_debug); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_for_app); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &misfit_ds); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl_dbg); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpu7_tl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &cpu_cluster_masks_proc_ops, -+ &cpu_cluster_masks); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &save_gov_proc_ops, -+ &save_gov); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &watchdog_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isolate_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &iso_free_rescue); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_stats_proc_ops); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &yield_opt_proc_ops, -+ &yield_opt_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbird_preempt_policy); -+ /* /proc/hmbird_sched--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_walt */ -+ load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir); -+ if (!load_track_dir) { -+ pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_walt--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &slim_walt_ctrl_proc_ops, -+ &slim_walt_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_dump); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_policy); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &sched_ravg_window_frame_per_sec_proc_ops, -+ &sched_ravg_window_frame_per_sec); -+ /* /proc/hmbird_sched/slim_walt--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_freq_gov */ -+ freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir); -+ if (!freq_gov_dir) { -+ pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_freq_gov--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &slim_gov_debug); -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &scx_gov_ctrl); -+ /* /proc/hmbird_sched/slim_freq_gov--end */ -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cluster_separate); -+ -+ return 0; -+} -+ -+device_initcall(hmbird_proc_init); -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.h b/kernel/sched/hmbird/hmbird_sched_proc.h -new file mode 100755 -index 000000000000..e22bc75f1dd7 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SCHED_PROC_H__ -+#define __HMBIRD_SCHED_PROC_H__ -+ -+#include -+#include -+ -+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \ -+ do { \ -+ if (!proc_create(name, mode, parent, proc_ops)) { \ -+ pr_err("Error creating proc entry %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \ -+ do { \ -+ if (!proc_create_data(name, mode, parent, proc_ops, data)) { \ -+ pr_err("Error creating proc entry with data %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_PROC_OPS(name, open_func, write_func) \ -+ static const struct proc_ops name##_proc_ops = { \ -+ .proc_open = open_func, \ -+ .proc_write = write_func, \ -+ .proc_read = seq_read, \ -+ .proc_lseek = seq_lseek, \ -+ .proc_release = single_release, \ -+ } -+ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos); -+static int hmbird_common_show(struct seq_file *m, void *v); -+static int hmbird_common_open(struct inode *inode, struct file *file); -+ -+#endif -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.c b/kernel/sched/hmbird/hmbird_shadow_tick.c -new file mode 100755 -index 000000000000..21de551c5132 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.c -@@ -0,0 +1,159 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include -+#include "../../time/tick-sched.h" -+#include -+ -+#include "hmbird_shadow_tick.h" -+ -+#define HIGHRES_WATCH_CPU 0 -+ -+#include -+static bool shadow_tick_enable(void) {return highres_tick_ctrl; } -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; } -+#endif -+static bool shadow_tick_timer_init_flag; -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define shadow_tick_printk(fmt, args...) \ -+do { \ -+ int cpu = smp_processor_id(); \ -+ if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU) \ -+ trace_printk("hmbird shadow tick :"fmt, args); \ -+} while (0) -+#else -+#define shadow_tick_printk(fmt, args...) -+#endif -+ -+DEFINE_PER_CPU(struct hrtimer, stt); -+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu))) -+#define STOP_IDLE_TRIGGER (1) -+#define PERIODIC_TICK_TRIGGER (2) -+#define TICK_INTVAL (1000000ULL) -+/* -+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms, -+ * so we can't restart hrtimer in scheduler tick. -+ */ -+static DEFINE_PER_CPU(u8, trigger_event); -+ -+/* -+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick. -+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler. -+ */ -+static DEFINE_PER_CPU(u8, tick_phase); -+ -+static inline void highres_timer_ctrl(bool enable, int cpu) -+{ -+ if (enable && hmbird_enabled()) { -+ if (!hrtimer_active(shadow_tick_timer(cpu))) -+ hrtimer_start(shadow_tick_timer(cpu), -+ ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED); -+ } else { -+ if (!enable) -+ hrtimer_cancel(shadow_tick_timer(cpu)); -+ } -+} -+ -+static inline void high_res_clear_phase(int cpu) -+{ -+ per_cpu(tick_phase, cpu) = 0; -+} -+ -+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer) -+{ -+ per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3; -+ if (per_cpu(tick_phase, cpu)) { -+ hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) { -+ per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ struct rq_flags rf; -+ -+ rq_lock(rq, &rf); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ rq_unlock(rq, &rf); -+ -+ return highres_next_phase(cpu, timer); -+} -+ -+void shadow_tick_timer_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance; -+ } -+} -+ -+void start_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable()) { -+ if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER) -+ highres_timer_ctrl(false, cpu); -+ per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static void stop_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ per_cpu(trigger_event, cpu) = 0; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(false, cpu); -+} -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ stop_shadow_tick_timer(); -+} -+ -+void scheduler_tick_handler(void *unused, struct rq *rq) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ start_shadow_tick_timer(); -+} -+ -+static int __init hmbird_shadow_tick_init(void) -+{ -+ int ret = 0; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ shadow_tick_timer_init(); -+ shadow_tick_timer_init_flag = true; -+ return ret; -+} -+ -+device_initcall(hmbird_shadow_tick_init); -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.h b/kernel/sched/hmbird/hmbird_shadow_tick.h -new file mode 100755 -index 000000000000..0c26fd984f0a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.h -@@ -0,0 +1,11 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SHADOW_TICK_H__ -+#define __HMBIRD_SHADOW_TICK_H__ -+ -+#include -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+void scheduler_tick_handler(void *unused, struct rq *rq); -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state); -+#endif -diff --git a/kernel/sched/hmbird/hmbird_trace.h b/kernel/sched/hmbird/hmbird_trace.h -new file mode 100755 -index 000000000000..8468b406f347 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_trace.h -@@ -0,0 +1,92 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hmbird -+ -+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HMBIRD_H -+ -+#include -+#include -+#include -+ -+#define MAX_FATAL_INFO (64) -+ -+TRACE_EVENT(hmbird_fatal_info, -+ -+ TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info), -+ -+ TP_ARGS(type, pe, lnr, bnr, info), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, type) -+ __field(int, pe) -+ __field(int, lnr) -+ __field(int, bnr) -+ __array(char, info, MAX_FATAL_INFO)), -+ -+ TP_fast_assign( -+ __entry->type = type; -+ __entry->pe = pe; -+ __entry->lnr = lnr; -+ __entry->bnr = bnr; -+ memcpy(__entry->info, info, MAX_FATAL_INFO);), -+ -+ TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s", -+ __entry->type, __entry->pe, __entry->lnr, __entry->bnr, -+ __entry->info) -+); -+ -+TRACE_EVENT(hmbird_update_history, -+ -+ TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq, -+ struct task_struct *p, u32 runtime, int samples, int event), -+ -+ TP_ARGS(hmbird, rq, p, runtime, samples, event), -+ -+ TP_STRUCT__entry( -+ __array(char, comm, TASK_COMM_LEN) -+ __field(pid_t, pid) -+ __field(unsigned int, runtime) -+ __field(int, samples) -+ __field(int, event) -+ __field(unsigned int, demand) -+ __array(u32, hist, RAVG_HIST_SIZE) -+ __field(u16, task_util) -+ __field(int, cpu)), -+ -+ TP_fast_assign( -+ memcpy(__entry->comm, p->comm, TASK_COMM_LEN); -+ __entry->pid = p->pid; -+ __entry->runtime = runtime; -+ __entry->samples = samples; -+ __entry->event = event; -+ __entry->demand = hmbird->sts.demand; -+ memcpy(__entry->hist, hmbird->sts.sum_history, -+ RAVG_HIST_SIZE * sizeof(u32)); -+ __entry->task_util = hmbird->sts.demand_scaled; -+ __entry->cpu = rq->cpu;), -+ -+ TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d", -+ __entry->comm, __entry->pid, -+ __entry->runtime, __entry->samples, -+ __entry->event, -+ __entry->demand, -+ __entry->hist[0], __entry->hist[1], -+ __entry->hist[2], __entry->hist[3], -+ __entry->hist[4], -+ __entry->task_util, -+ __entry->cpu) -+); -+ -+#endif /*_TRACE_HMBIRD_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird -+ -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE hmbird_trace -+/* This part must be outside protection */ -+#include -diff --git a/kernel/sched/hmbird/hmbird_util_track.c b/kernel/sched/hmbird/hmbird_util_track.c -new file mode 100755 -index 000000000000..d520a8403401 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.c -@@ -0,0 +1,626 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+ -+#define CREATE_TRACE_POINTS -+#include "hmbird_trace.h" -+#undef CREATE_TRACE_POINTS -+ -+#define HMBIRD_DEBUG_PANIC (1 << 3) -+static bool init_irq_work_inited; -+ -+extern noinline int tracing_mark_write(const char *buf); -+ -+int hmbird_sched_ravg_window = 8000000; -+int new_hmbird_sched_ravg_window = 8000000; -+DEFINE_SPINLOCK(new_sched_ravg_window_lock); -+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+ -+static struct irq_work hmbird_slim_walt_irq_work; -+ -+/*Sysctl related interface*/ -+#define WINDOW_STATS_RECENT 0 -+#define WINDOW_STATS_MAX 1 -+#define WINDOW_STATS_MAX_RECENT_AVG 2 -+#define WINDOW_STATS_AVG 3 -+#define WINDOW_STATS_INVALID_POLICY 4 -+ -+ -+#define SCX_SCHED_CAPACITY_SHIFT 10 -+#define SCHED_ACCOUNT_WAIT_TIME 0 -+ -+atomic64_t hmbird_irq_work_lastq_ws; -+static u64 tick_sched_clock; -+ -+static inline u64 scale_exec_time(u64 delta, struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT; -+} -+ -+static inline u64 scale_time_to_util(u64 d) -+{ -+ do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT); -+ return d; -+} -+ -+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ delta = scale_exec_time(delta, rq); -+ sts->sum += delta; -+ if (unlikely(sts->sum > hmbird_sched_ravg_window)) -+ sts->sum = hmbird_sched_ravg_window; -+ -+ return delta; -+} -+ -+ -+static int -+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) -+{ -+ /* -+ * No need to bother updating task demand for the idle task. -+ */ -+ if (is_idle_task(p)) -+ return 0; -+ -+ /* -+ * When a task is waking up it is completing a segment of non-busy -+ * time. Likewise, if wait time is not treated as busy time, then -+ * when a task begins to run or is migrated, it is not running and -+ * is completing a segment of non-busy time. -+ */ -+ if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && -+ (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) -+ return 0; -+ -+ /* -+ * The idle exit time is not accounted for the first task _picked_ up to -+ * run on the idle CPU. -+ */ -+ if (event == PICK_NEXT_TASK && rq->curr == rq->idle) -+ return 0; -+ -+ /* -+ * TASK_UPDATE can be called on sleeping task, when its moved between -+ * related groups -+ */ -+ if (event == TASK_UPDATE) { -+ if (rq->curr == p) -+ return 1; -+ -+ return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; -+ } -+ -+ return 1; -+} -+ -+static void rollover_cpu_window(struct rq *rq, bool full_window) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 curr_sum = srq->curr_runnable_sum; -+ -+ if (unlikely(full_window)) -+ curr_sum = 0; -+ -+ srq->prev_runnable_sum = curr_sum; -+ srq->curr_runnable_sum = 0; -+} -+ -+static u64 -+update_window_start(struct rq *rq, u64 wallclock, int event) -+{ -+ s64 delta; -+ int nr_windows; -+ bool full_window; -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start = srq->window_start; -+ -+ if (wallclock < srq->latest_clock) -+ wallclock = srq->latest_clock; -+ delta = wallclock - srq->window_start; -+ if (delta < 0) { -+ delta = 0; -+ wallclock = srq->window_start; -+ } -+ srq->latest_clock = wallclock; -+ if (delta < hmbird_sched_ravg_window) -+ return old_window_start; -+ -+ nr_windows = div64_u64(delta, hmbird_sched_ravg_window); -+ srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window; -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ full_window = nr_windows > 1; -+ rollover_cpu_window(rq, full_window); -+ -+ return old_window_start; -+} -+ -+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -+ -+static inline unsigned int get_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static inline unsigned int cpu_cur_freq(int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cur; -+} -+ -+static void -+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, -+ u64 wallclock) -+{ -+ int cpu = cpu_of(rq); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * -+ arch_scale_cpu_capacity(cpu), get_max_freq(cpu)); -+} -+ -+/* -+ * Called when new window is starting for a task, to record cpu usage over -+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 -+ * when, say, a real-time task runs without preemption for several windows at a -+ * stretch. -+ */ -+static void update_history(struct rq *rq, struct task_struct *p, -+ u32 runtime, int samples, int event) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u32 *hist = &sts->sum_history[0]; -+ int i; -+ u32 max = 0, avg, demand; -+ u64 sum = 0; -+ u16 demand_scaled; -+ -+ /* Ignore windows where task had no activity */ -+ if (!runtime || is_idle_task(p) || !samples) -+ goto done; -+ -+ /* Push new 'runtime' value onto stack */ -+ for (; samples > 0; samples--) { -+ hist[sts->cidx] = runtime; -+ sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE; -+ } -+ -+ for (i = 0; i < RAVG_HIST_SIZE; i++) { -+ sum += hist[i]; -+ if (hist[i] > max) -+ max = hist[i]; -+ } -+ -+ sts->sum = 0; -+ avg = div64_u64(sum, RAVG_HIST_SIZE); -+ -+ switch (slim_walt_policy) { -+ case WINDOW_STATS_RECENT: -+ demand = runtime; -+ break; -+ case WINDOW_STATS_MAX: -+ demand = max; -+ break; -+ case WINDOW_STATS_AVG: -+ demand = avg; -+ break; -+ default: -+ demand = max(avg, runtime); -+ } -+ -+ demand_scaled = scale_time_to_util(demand); -+ -+ sts->demand = demand; -+ sts->demand_scaled = demand_scaled; -+ -+done: -+ return; -+} -+ -+ -+static u64 update_task_demand(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ -+ u64 delta, window_start = srq->window_start; -+ int new_window, nr_full_windows; -+ u32 window_size = hmbird_sched_ravg_window; -+ u64 runtime; -+ -+ new_window = mark_start < window_start; -+ if (!account_busy_for_task_demand(rq, p, event)) { -+ if (new_window) -+ /* -+ * If the time accounted isn't being accounted as -+ * busy time, and a new window started, only the -+ * previous window need be closed out with the -+ * pre-existing demand. Multiple windows may have -+ * elapsed, but since empty windows are dropped, -+ * it is not necessary to account those. -+ */ -+ update_history(rq, p, sts->sum, 1, event); -+ return 0; -+ } -+ -+ if (!new_window) { -+ /* -+ * The simple case - busy time contained within the existing -+ * window. -+ */ -+ return add_to_task_demand(rq, p, wallclock - mark_start); -+ } -+ -+ /* -+ * Busy time spans at least two windows. Temporarily rewind -+ * window_start to first window boundary after mark_start. -+ */ -+ delta = window_start - mark_start; -+ nr_full_windows = div64_u64(delta, window_size); -+ window_start -= (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (window_start - mark_start) first */ -+ runtime = add_to_task_demand(rq, p, window_start - mark_start); -+ -+ /* Push new sample(s) into task's demand history */ -+ update_history(rq, p, sts->sum, 1, event); -+ if (nr_full_windows) { -+ u64 scaled_window = scale_exec_time(window_size, rq); -+ -+ update_history(rq, p, scaled_window, nr_full_windows, event); -+ runtime += nr_full_windows * scaled_window; -+ } -+ -+ /* -+ * Roll window_start back to current to process any remainder -+ * in current window. -+ */ -+ window_start += (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (wallclock - window_start) next */ -+ mark_start = window_start; -+ runtime += add_to_task_demand(rq, p, wallclock - mark_start); -+ -+ return runtime; -+} -+ -+u16 slim_walt_cpu_util(int cpu) -+{ -+ u64 prev_runnable_sum; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ prev_runnable_sum = srq->prev_runnable_sum; -+ do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT); -+ -+ return (u16)prev_runnable_sum; -+} -+ -+static DEFINE_PER_CPU(u16, prev_cpu_util); -+static inline void cpu_util_update_systrace_c(int cpu) -+{ -+ char buf[256]; -+ u16 cpu_util = slim_walt_cpu_util(cpu); -+ -+ if (cpu_util != per_cpu(prev_cpu_util, cpu)) { -+ snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n", -+ cpu, cpu_util); -+ tracing_mark_write(buf); -+ per_cpu(prev_cpu_util, cpu) = cpu_util; -+ } -+} -+ -+ -+static inline int account_busy_for_cpu_time(struct rq *rq, -+ struct task_struct *p, -+ int event) -+{ -+ return !is_idle_task(p) && (event == PUT_PREV_TASK -+ || event == TASK_UPDATE); -+} -+ -+ -+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ int new_window, full_window = 0; -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 window_start = srq->window_start; -+ u32 window_size = srq->prev_window_size; -+ u64 delta; -+ u64 *curr_runnable_sum = &srq->curr_runnable_sum; -+ u64 *prev_runnable_sum = &srq->prev_runnable_sum; -+ -+ new_window = mark_start < window_start; -+ if (new_window) -+ full_window = (window_start - mark_start) >= window_size; -+ -+ -+ if (!account_busy_for_cpu_time(rq, p, event)) -+ goto done; -+ -+ -+ if (!new_window) { -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. No rollover -+ * since we didn't start a new window. An example of this is -+ * when a task starts execution and then sleeps within the -+ * same window. -+ */ -+ delta = wallclock - mark_start; -+ -+ delta = scale_exec_time(delta, rq); -+ *curr_runnable_sum += delta; -+ -+ goto done; -+ } -+ -+ /* -+ * situations below this need window rollover, -+ * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done -+ * in update_window_start() -+ * -+ * For task counters curr/prev_window[_cpu] are rolled over in the early part of -+ * this function. If full_window(s) have expired and time since last update needs -+ * to be accounted as busy time, set the prev to a complete window size time, else -+ * add the prev window portion. -+ * -+ * For task curr counters a new window has begun, always assign -+ */ -+ -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. A new window -+ * must have been started in udpate_window_start() -+ * If any of these three above conditions are true -+ * then this busy time can't be accounted as irqtime. -+ * -+ * Busy time for the idle task need not be accounted. -+ * -+ * An example of this would be a task that starts execution -+ * and then sleeps once a new window has begun. -+ */ -+ -+ /* -+ * A full window hasn't elapsed, account partial -+ * contribution to previous completed window. -+ */ -+ -+ -+ delta = full_window ? scale_exec_time(window_size, rq) : -+ scale_exec_time(window_start - mark_start, rq); -+ -+ *prev_runnable_sum += delta; -+ -+ /* Account piece of busy time in the current window. */ -+ delta = scale_exec_time(wallclock - window_start, rq); -+ *curr_runnable_sum += delta; -+ -+done: -+ if (slim_walt_dump && new_window) -+ cpu_util_update_systrace_c(rq->cpu); -+} -+ -+ -+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq) -+{ -+ u64 result; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 new_window_start = srq->window_start; -+ -+ if (old_window_start == new_window_start) -+ return; -+ -+ result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws, -+ old_window_start, new_window_start); -+ if (result != old_window_start) -+ return; -+ -+ if (likely(cpu_online(raw_smp_processor_id()))) -+ irq_work_queue(&hmbird_slim_walt_irq_work); -+ else -+ irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask)); -+} -+ -+/* -+ * In the core scheduler, most of the load update points update the rq_clock after -+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of -+ * obtaining the time, but to prevent the subsequent migration of the load update -+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock -+ * update here. -+ */ -+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p, -+ struct rq *rq, int event) -+{ -+ if (!(rq->clock_update_flags & RQCF_UPDATED)) -+ update_rq_clock(rq); -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock)); -+} -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start; -+ -+ if (!slim_walt_ctrl) -+ return; -+ -+ if (!srq->window_start || sts->mark_start == wallclock) -+ return; -+ -+ old_window_start = update_window_start(rq, wallclock, event); -+ -+ if (!sts->window_start) -+ sts->window_start = srq->window_start; -+ -+ if (!sts->mark_start) -+ goto done; -+ -+ update_task_rq_cpu_cycles(p, rq, event, wallclock); -+ update_task_demand(p, rq, event, wallclock); -+ update_cpu_busy_time(p, rq, event, wallclock); -+ -+ sts->window_start = srq->window_start; -+ -+done: -+ sts->mark_start = wallclock; -+ slim_walt_window_rollover_run_once(old_window_start, rq); -+} -+ -+static void slim_walt_irq_work(struct irq_work *irq_work) -+{ -+ cpumask_t lock_cpus; -+ struct hmbird_sched_rq_stats *srq; -+ struct rq *rq; -+ int cpu; -+ int level = 0; -+ u64 wc; -+ unsigned long flags; -+ -+ cpumask_copy(&lock_cpus, cpu_possible_mask); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ if (level == 0) -+ raw_spin_lock(&cpu_rq(cpu)->__lock); -+ else -+ raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); -+ level++; -+ } -+ -+ wc = sched_clock(); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ rq = cpu_rq(cpu); -+ hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc); -+ } -+ -+ cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER); -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) { -+ srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id()); -+ if (wc < srq->window_start + new_hmbird_sched_ravg_window) -+ hmbird_sched_ravg_window = new_hmbird_sched_ravg_window; -+ } -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ raw_spin_unlock(&cpu_rq(cpu)->__lock); -+ } -+} -+static void hmbird_sched_init_rq(struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ srq->task_exec_scale = 1024; -+ srq->window_start = 0; -+} -+ -+void hmbird_sched_init_task(struct task_struct *p) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ memset(sts, 0, sizeof(struct hmbird_sched_task_stats)); -+} -+ -+static void hmbird_sched_stats_init(void) -+{ -+ unsigned long flags; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ raw_spin_lock_irqsave(&rq->__lock, flags); -+ hmbird_sched_init_rq(rq); -+ raw_spin_unlock_irqrestore(&rq->__lock, flags); -+ } -+ slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG; -+ -+ if (false == init_irq_work_inited) { -+ init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work); -+ init_irq_work_inited = true; -+ } -+} -+ -+void hmbird_scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (unlikely(!tick_sched_clock)) { -+ /* -+ * Let the window begin 20us prior to the tick, -+ * that way we are guaranteed a rollover when the tick occurs. -+ * Use rq->clock directly instead of rq_clock() since -+ * we do not have the rq lock and -+ * rq->clock was updated in the tick callpath. -+ */ -+ if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000)) -+ return; -+ for_each_possible_cpu(cpu) { -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ srq->window_start = tick_sched_clock; -+ } -+ atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock); -+ } -+} -+ -+void slim_walt_enable(int enable) -+{ -+ if (1 == !!enable) { -+ hmbird_sched_stats_init(); -+ WRITE_ONCE(tick_sched_clock, 0); -+ } else -+ slim_walt_ctrl = 0; -+} -+ -+void slim_get_cpu_util(int cpu, u64 *util) -+{ -+ if (cpu < 0) -+ return; -+ -+ *util = slim_walt_cpu_util(cpu); -+} -+ -+void slim_get_task_util(struct task_struct *p, u64 *util) -+{ -+ if (p == NULL) -+ return; -+ -+ *util = get_hmbird_ts(p)->sts.demand_scaled; -+} -+ -+void sched_ravg_window_change(int frame_per_sec) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec; -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+} -diff --git a/kernel/sched/hmbird/hmbird_util_track.h b/kernel/sched/hmbird/hmbird_util_track.h -new file mode 100755 -index 000000000000..17b85df7f83b ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.h -@@ -0,0 +1,33 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef __HMBIRD_UTIL_TRACK_H__ -+#define __HMBIRD_UTIL_TRACK_H__ -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock); -+void hmbird_sched_init_task(struct task_struct *p); -+void slim_walt_enable(int enable); -+void slim_get_cpu_util(int cpu, u64 *util); -+void slim_get_task_util(struct task_struct *p, u64 *util); -+ -+extern atomic64_t hmbird_irq_work_lastq_ws; -+ -+enum task_event { -+ PUT_PREV_TASK = 0, -+ PICK_NEXT_TASK = 1, -+ TASK_WAKE = 2, -+ TASK_MIGRATE = 3, -+ TASK_UPDATE = 4, -+ IRQ_UPDATE = 5, -+}; -+ -+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+#endif /* __HMBIRD_UTIL_TRACK_H__ */ -diff --git a/kernel/sched/hmbird/slim.h b/kernel/sched/hmbird/slim.h -new file mode 100755 -index 000000000000..eb386260e950 ---- /dev/null -+++ b/kernel/sched/hmbird/slim.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+ -+#ifndef __SLIM_H -+#define __SLIM_H -+ -+extern atomic_t __hmbird_ops_enabled; -+extern atomic_t non_hmbird_task; -+extern int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+extern int heartbeat; -+extern int heartbeat_enable; -+extern int watchdog_enable; -+extern int isolate_ctrl; -+extern int parctrl_high_ratio; -+extern int parctrl_low_ratio; -+extern int isoctrl_high_ratio; -+extern int isoctrl_low_ratio; -+extern int iso_free_rescue; -+extern int yield_opt; -+ -+extern enum hmbird_switch_type sw_type; -+extern noinline int tracing_mark_write(const char *buf); -+int task_top_id(struct task_struct *p); -+void stats_print(char *buf, int len); -+void hmbird_skip_yield(long *skip); -+extern spinlock_t hmbird_tasks_lock; -+ -+struct yield_opt_params { -+ int enable; -+ int frame_per_sec; -+ u64 frame_time_ns; -+ int yield_headroom; -+}; -+ -+extern struct yield_opt_params yield_opt_params; -+ -+#define MAX_GOV_LEN (16) -+extern char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+#endif -diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c -index b33cefeb4188..487255ac6832 100644 ---- a/kernel/sched/idle.c -+++ b/kernel/sched/idle.c -@@ -408,13 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl - - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) - { -- scx_update_idle(rq, false); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, false); -+#endif - } - - static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) - { - update_idle_core(rq); -- scx_update_idle(rq, true); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, true); -+#endif - schedstat_inc(rq->sched_goidle); - } - -diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h -index cbc478992cc4..e35473a1f0ea 100644 ---- a/kernel/sched/sched.h -+++ b/kernel/sched/sched.h -@@ -187,18 +187,13 @@ static inline int idle_policy(int policy) - return policy == SCHED_IDLE; - } - --static inline int normal_policy(int policy) --{ --#ifdef CONFIG_SCHED_CLASS_EXT -- if (policy == SCHED_EXT) -- return true; --#endif -- return policy == SCHED_NORMAL; --} -- - static inline int fair_policy(int policy) - { -- return normal_policy(policy) || policy == SCHED_BATCH; -+#ifdef CONFIG_HMBIRD_SCHED -+ return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#else -+ return policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#endif - } - - static inline int rt_policy(int policy) -@@ -246,24 +241,6 @@ static inline void update_avg(u64 *avg, u64 sample) - #define shr_bound(val, shift) \ - (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) - --/* -- * cgroup weight knobs should use the common MIN, DFL and MAX values which are -- * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it -- * maps pretty well onto the shares value used by scheduler and the round-trip -- * conversions preserve the original value over the entire range. -- */ --static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) --{ -- return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); --} -- --static inline unsigned long sched_weight_to_cgroup(unsigned long weight) --{ -- return clamp_t(unsigned long, -- DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -- CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); --} -- - /* - * !! For sched_setattr_nocheck() (kernel) only !! - * -@@ -531,10 +508,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) { } - #endif /* CONFIG_SMP */ - #else /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_HMBIRD_SCHED - static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) - { - return 0; - } -+#endif - #endif /* CONFIG_FAIR_GROUP_SCHED */ - - #else /* CONFIG_CGROUP_SCHED */ -@@ -699,29 +678,6 @@ struct cfs_rq { - #endif /* CONFIG_FAIR_GROUP_SCHED */ - }; - --#ifdef CONFIG_SCHED_CLASS_EXT --/* scx_rq->flags, protected by the rq lock */ --enum scx_rq_flags { -- SCX_RQ_CAN_STOP_TICK = 1 << 0, --}; -- --struct scx_rq { -- struct scx_dispatch_q local_dsq; -- struct list_head watchdog_list; -- u64 ops_qseq; -- u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -- u32 nr_running; -- u32 flags; -- bool cpu_released; -- cpumask_var_t cpus_to_kick; -- cpumask_var_t cpus_to_preempt; -- cpumask_var_t cpus_to_wait; -- u64 pnt_seq; -- struct irq_work kick_cpus_irq_work; -- struct rq *rq; --}; --#endif /* CONFIG_SCHED_CLASS_EXT */ -- - static inline int rt_bandwidth_enabled(void) - { - return sysctl_sched_rt_runtime >= 0; -@@ -1257,11 +1213,7 @@ struct rq { - - ANDROID_OEM_DATA_ARRAY(1, 16); - --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, struct scx_rq *scx); --#else - ANDROID_KABI_RESERVE(1); --#endif - ANDROID_KABI_RESERVE(2); - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); -@@ -2339,11 +2291,6 @@ struct affinity_context { - unsigned int flags; - }; - --enum rq_onoff_reason { -- RQ_ONOFF_HOTPLUG, /* CPU is going on/offline */ -- RQ_ONOFF_TOPOLOGY, /* sched domain topology update */ --}; -- - struct sched_class { - - #ifdef CONFIG_UCLAMP_TASK -@@ -2550,7 +2497,6 @@ extern void init_sched_rt_class(void); - extern void init_sched_fair_class(void); - - extern void reweight_task(struct task_struct *p, int prio); --extern void __setscheduler_prio(struct task_struct *p, int prio); - - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); -@@ -2631,53 +2577,6 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) - extern void activate_task(struct rq *rq, struct task_struct *p, int flags); - extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); - --struct sched_change_guard { -- struct task_struct *p; -- struct rq *rq; -- bool queued; -- bool running; -- bool done; --}; -- --extern struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -- --extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags); -- --/** -- * SCHED_CHANGE_BLOCK - Nested block for task attribute updates -- * @__rq: Runqueue the target task belongs to -- * @__p: Target task -- * @__flags: DEQUEUE/ENQUEUE_* flags -- * -- * A task may need to be dequeued and put_prev_task'd for attribute updates and -- * set_next_task'd and re-enqueued afterwards. This helper defines a nested -- * block which automatically handles these preparation and cleanup operations. -- * -- * SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- * update_attribute(p); -- * ... -- * } -- * -- * If @__flags is a variable, the variable may be updated in the block body and -- * the updated value will be used when re-enqueueing @p. -- * -- * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling -- * update_rq_clock() beforehand. Otherwise, the rq clock is automatically -- * updated iff the task needs to be dequeued and re-enqueued. Only the former -- * case guarantees that the rq clock is up-to-date inside and after the block. -- */ --#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -- for (struct sched_change_guard __cg = \ -- sched_change_guard_init(__rq, __p, __flags); \ -- !__cg.done; sched_change_guard_fini(&__cg, __flags)) -- --extern void check_class_changing(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class); --extern void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio); -- - extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); - - #ifdef CONFIG_PREEMPT_RT -@@ -3709,6 +3608,8 @@ static inline bool cpu_busy_with_softirqs(int cpu) - } - #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ - --#include "ext.h" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird.h" -+#endif - - #endif /* _KERNEL_SCHED_SCHED_H */ -diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c -index c197c61d6ea9..919fed4ff08b 100644 ---- a/kernel/time/tick-sched.c -+++ b/kernel/time/tick-sched.c -@@ -34,6 +34,10 @@ - - #include - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "../sched/hmbird/hmbird_shadow_tick.h" -+#endif -+ - /* - * Per-CPU nohz control structure - */ -@@ -1112,6 +1116,9 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) - return true; - } - -+#ifdef CONFIG_HMBIRD_SCHED -+extern void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+#endif - /** - * tick_nohz_idle_stop_tick - stop the idle tick from the idle task - * -@@ -1124,7 +1131,9 @@ void tick_nohz_idle_stop_tick(void) - ktime_t expires; - - trace_android_vh_tick_nohz_idle_stop_tick(NULL); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL); -+#endif - /* - * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the - * tick timer expiration time is known already. -diff --git a/tools/sched_ext/.gitignore b/tools/sched_ext/.gitignore -deleted file mode 100644 -index a3240f9f7eba..000000000000 ---- a/tools/sched_ext/.gitignore -+++ /dev/null -@@ -1,9 +0,0 @@ --scx_example_simple --scx_example_qmap --scx_example_central --scx_example_pair --scx_example_flatcg --scx_example_userland --*.skel.h --*.subskel.h --/tools/ -diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile -deleted file mode 100644 -index 73c43782837d..000000000000 ---- a/tools/sched_ext/Makefile -+++ /dev/null -@@ -1,216 +0,0 @@ --# SPDX-License-Identifier: GPL-2.0 --# Copyright (c) 2022 Meta Platforms, Inc. and affiliates. --include ../build/Build.include --include ../scripts/Makefile.arch --include ../scripts/Makefile.include -- --ifneq ($(LLVM),) --ifneq ($(filter %/,$(LLVM)),) --LLVM_PREFIX := $(LLVM) --else ifneq ($(filter -%,$(LLVM)),) --LLVM_SUFFIX := $(LLVM) --endif -- --CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi --CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu --CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl --CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu --CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu --CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu --CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu --CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu --CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu --CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) -- --ifeq ($(CROSS_COMPILE),) --ifeq ($(CLANG_TARGET_FLAGS),) --$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk --else --CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) --endif # CLANG_TARGET_FLAGS --else --CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) --endif # CROSS_COMPILE -- --CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as --else --CC := $(CROSS_COMPILE)gcc --endif # LLVM -- --CURDIR := $(abspath .) --TOOLSDIR := $(abspath ..) --LIBDIR := $(TOOLSDIR)/lib --BPFDIR := $(LIBDIR)/bpf --TOOLSINCDIR := $(TOOLSDIR)/include --BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool --APIDIR := $(TOOLSINCDIR)/uapi --GENDIR := $(abspath ../../include/generated) --GENHDR := $(GENDIR)/autoconf.h -- --SCRATCH_DIR := $(CURDIR)/tools --BUILD_DIR := $(SCRATCH_DIR)/build --INCLUDE_DIR := $(SCRATCH_DIR)/include --BPFOBJ_DIR := $(BUILD_DIR)/libbpf --BPFOBJ := $(BPFOBJ_DIR)/libbpf.a --ifneq ($(CROSS_COMPILE),) --HOST_BUILD_DIR := $(BUILD_DIR)/host --HOST_SCRATCH_DIR := host-tools --HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include --else --HOST_BUILD_DIR := $(BUILD_DIR) --HOST_SCRATCH_DIR := $(SCRATCH_DIR) --HOST_INCLUDE_DIR := $(INCLUDE_DIR) --endif --HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a --RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids --DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -- --VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ -- $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ -- ../../vmlinux \ -- /sys/kernel/btf/vmlinux \ -- /boot/vmlinux-$(shell uname -r) --VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) --ifeq ($(VMLINUX_BTF),) --$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") --endif -- --BPFTOOL ?= $(DEFAULT_BPFTOOL) -- --ifneq ($(wildcard $(GENHDR)),) -- GENFLAGS := -DHAVE_GENHDR --endif -- --CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ -- -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -- -I$(TOOLSINCDIR) -I$(APIDIR) -- --CARGOFLAGS := --release -- --# Silence some warnings when compiled with clang --ifneq ($(LLVM),) --CFLAGS += -Wno-unused-command-line-argument --endif -- --LDFLAGS = -lelf -lz -lpthread -- --IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - &1 \ -- | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ --$(shell $(1) -dM -E - $@ --else -- $(call msg,CP,,$@) -- $(Q)cp "$(VMLINUX_H)" $@ --endif -- --%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \ -- | $(BPFOBJ) -- $(call msg,CLNG-BPF,,$@) -- $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ -- --%.skel.h: %.bpf.o $(BPFTOOL) -- $(call msg,GEN-SKEL,,$@) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) -- $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) -- $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@ -- $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h) -- --scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_userland: scx_example_userland.c scx_example_userland.skel.h \ -- scx_example_userland_common.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR) --atropos: export ATROPOS_CLANG = $(CLANG) --atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS) --atropos: $(INCLUDE_DIR)/vmlinux.h -- cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS) -- --clean: -- cargo clean --manifest-path=atropos/Cargo.toml -- rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) -- rm -f *.o *.bpf.o *.skel.h *.subskel.h -- rm -f scx_example_simple scx_example_qmap scx_example_central \ -- scx_example_pair scx_example_flatcg scx_example_userland -- --.PHONY: all atropos clean -- --# delete failed targets --.DELETE_ON_ERROR: -- --# keep intermediate (.skel.h, .bpf.o, etc) targets --.SECONDARY: -diff --git a/tools/sched_ext/atropos/.gitignore b/tools/sched_ext/atropos/.gitignore -deleted file mode 100644 -index 186dba259ec2..000000000000 ---- a/tools/sched_ext/atropos/.gitignore -+++ /dev/null -@@ -1,3 +0,0 @@ --src/bpf/.output --Cargo.lock --target -diff --git a/tools/sched_ext/atropos/Cargo.toml b/tools/sched_ext/atropos/Cargo.toml -deleted file mode 100644 -index 7462a836d53d..000000000000 ---- a/tools/sched_ext/atropos/Cargo.toml -+++ /dev/null -@@ -1,28 +0,0 @@ --[package] --name = "scx_atropos" --version = "0.5.0" --authors = ["Dan Schatzberg ", "Meta"] --edition = "2021" --description = "Userspace scheduling with BPF" --license = "GPL-2.0-only" -- --[dependencies] --anyhow = "1.0.65" --bitvec = { version = "1.0", features = ["serde"] } --clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } --ctrlc = { version = "3.1", features = ["termination"] } --fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"} --hex = "0.4.3" --libbpf-rs = "0.19.1" --libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] } --libc = "0.2.137" --log = "0.4.17" --ordered-float = "3.4.0" --simplelog = "0.12.0" -- --[build-dependencies] --bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false } --libbpf-cargo = "0.13.0" -- --[features] --enable_backtrace = [] -diff --git a/tools/sched_ext/atropos/build.rs b/tools/sched_ext/atropos/build.rs -deleted file mode 100644 -index 26e792c5e17e..000000000000 ---- a/tools/sched_ext/atropos/build.rs -+++ /dev/null -@@ -1,70 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --extern crate bindgen; -- --use std::env; --use std::fs::create_dir_all; --use std::path::Path; --use std::path::PathBuf; -- --use libbpf_cargo::SkeletonBuilder; -- --const HEADER_PATH: &str = "src/bpf/atropos.h"; -- --fn bindgen_atropos() { -- // Tell cargo to invalidate the built crate whenever the wrapper changes -- println!("cargo:rerun-if-changed={}", HEADER_PATH); -- -- // The bindgen::Builder is the main entry point -- // to bindgen, and lets you build up options for -- // the resulting bindings. -- let bindings = bindgen::Builder::default() -- // The input header we would like to generate -- // bindings for. -- .header(HEADER_PATH) -- // Tell cargo to invalidate the built crate whenever any of the -- // included header files changed. -- .parse_callbacks(Box::new(bindgen::CargoCallbacks)) -- // Finish the builder and generate the bindings. -- .generate() -- // Unwrap the Result and panic on failure. -- .expect("Unable to generate bindings"); -- -- // Write the bindings to the $OUT_DIR/bindings.rs file. -- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); -- bindings -- .write_to_file(out_path.join("atropos-sys.rs")) -- .expect("Couldn't write bindings!"); --} -- --fn gen_bpf_sched(name: &str) { -- let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap(); -- let clang = env::var("ATROPOS_CLANG").unwrap(); -- eprintln!("{}", clang); -- let outpath = format!("./src/bpf/.output/{}.skel.rs", name); -- let skel = Path::new(&outpath); -- let src = format!("./src/bpf/{}.bpf.c", name); -- SkeletonBuilder::new() -- .source(src.clone()) -- .clang(clang) -- .clang_args(bpf_cflags) -- .build_and_generate(&skel) -- .unwrap(); -- println!("cargo:rerun-if-changed={}", src); --} -- --fn main() { -- bindgen_atropos(); -- // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton. -- // Reasons are because the generated skeleton contains compiler attributes -- // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]` -- // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside -- // the path attribute either (see https://github.com/rust-lang/rust/pull/83366). -- // -- // However, there is hope! When the above feature stabilizes we can clean this -- // all up. -- create_dir_all("./src/bpf/.output").unwrap(); -- gen_bpf_sched("atropos"); --} -diff --git a/tools/sched_ext/atropos/rustfmt.toml b/tools/sched_ext/atropos/rustfmt.toml -deleted file mode 100644 -index b7258ed0a8d8..000000000000 ---- a/tools/sched_ext/atropos/rustfmt.toml -+++ /dev/null -@@ -1,8 +0,0 @@ --# Get help on options with `rustfmt --help=config` --# Please keep these in alphabetical order. --edition = "2021" --group_imports = "StdExternalCrate" --imports_granularity = "Item" --merge_derives = false --use_field_init_shorthand = true --version = "Two" -diff --git a/tools/sched_ext/atropos/src/atropos_sys.rs b/tools/sched_ext/atropos/src/atropos_sys.rs -deleted file mode 100644 -index bbeaf856d40e..000000000000 ---- a/tools/sched_ext/atropos/src/atropos_sys.rs -+++ /dev/null -@@ -1,10 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#![allow(non_upper_case_globals)] --#![allow(non_camel_case_types)] --#![allow(non_snake_case)] --#![allow(dead_code)] -- --include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs")); -diff --git a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -deleted file mode 100644 -index 3905a403e940..000000000000 ---- a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -+++ /dev/null -@@ -1,743 +0,0 @@ --/* Copyright (c) Meta Platforms, Inc. and affiliates. */ --/* -- * This software may be used and distributed according to the terms of the -- * GNU General Public License version 2. -- * -- * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF -- * part does simple round robin in each domain and the userspace part -- * calculates the load factor of each domain and tells the BPF part how to load -- * balance the domains. -- * -- * Every task has an entry in the task_data map which lists which domain the -- * task belongs to. When a task first enters the system (atropos_prep_enable), -- * they are round-robined to a domain. -- * -- * atropos_select_cpu is the primary scheduling logic, invoked when a task -- * becomes runnable. The lb_data map is populated by userspace to inform the BPF -- * scheduler that a task should be migrated to a new domain. Otherwise, the task -- * is scheduled in priority order as follows: -- * * The current core if the task was woken up synchronously and there are idle -- * cpus in the system -- * * The previous core, if idle -- * * The pinned-to core if the task is pinned to a specific core -- * * Any idle cpu in the domain -- * -- * If none of the above conditions are met, then the task is enqueued to a -- * dispatch queue corresponding to the domain (atropos_enqueue). -- * -- * atropos_dispatch will attempt to consume a task from its domain's -- * corresponding dispatch queue (this occurs after scheduling any tasks directly -- * assigned to it due to the logic in atropos_select_cpu). If no task is found, -- * then greedy load stealing will attempt to find a task on another dispatch -- * queue to run. -- * -- * Load balancing is almost entirely handled by userspace. BPF populates the -- * task weight, dom mask and current dom in the task_data map and executes the -- * load balance based on userspace populating the lb_data map. -- */ --#include "../../../scx_common.bpf.h" --#include "atropos.h" -- --#include --#include --#include --#include --#include --#include -- --char _license[] SEC("license") = "GPL"; -- --/* -- * const volatiles are set during initialization and treated as consts by the -- * jit compiler. -- */ -- --/* -- * Domains and cpus -- */ --const volatile __u32 nr_doms = 32; /* !0 for veristat, set during init */ --const volatile __u32 nr_cpus = 64; /* !0 for veristat, set during init */ --const volatile __u32 cpu_dom_id_map[MAX_CPUS]; --const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; -- --const volatile bool kthreads_local; --const volatile bool fifo_sched; --const volatile bool switch_partial; --const volatile __u32 greedy_threshold; -- --/* base slice duration */ --const volatile __u64 slice_us = 20000; -- --/* -- * Exit info -- */ --int exit_type = SCX_EXIT_NONE; --char exit_msg[SCX_EXIT_MSG_LEN]; -- --struct pcpu_ctx { -- __u32 dom_rr_cur; /* used when scanning other doms */ -- -- /* libbpf-rs does not respect the alignment, so pad out the struct explicitly */ -- __u8 _padding[CACHELINE_SIZE - sizeof(u64)]; --} __attribute__((aligned(CACHELINE_SIZE))); -- --struct pcpu_ctx pcpu_ctx[MAX_CPUS]; -- --/* -- * Domain context -- */ --struct dom_ctx { -- struct bpf_cpumask __kptr *cpumask; -- u64 vtime_now; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __type(key, u32); -- __type(value, struct dom_ctx); -- __uint(max_entries, MAX_DOMS); -- __uint(map_flags, 0); --} dom_ctx SEC(".maps"); -- --/* -- * Statistics -- */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, ATROPOS_NR_STATS); --} stats SEC(".maps"); -- --static inline void stat_add(enum stat_idx idx, u64 addend) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p) += addend; --} -- --/* Map pid -> task_ctx */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, struct task_ctx); -- __uint(max_entries, 1000000); -- __uint(map_flags, 0); --} task_data SEC(".maps"); -- --/* -- * This is populated from userspace to indicate which pids should be reassigned -- * to new doms. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, u32); -- __uint(max_entries, 1000); -- __uint(map_flags, 0); --} lb_data SEC(".maps"); -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p, -- u32 new_dom_id) --{ -- struct dom_ctx *old_domc, *new_domc; -- struct bpf_cpumask *d_cpumask, *t_cpumask; -- u32 old_dom_id = task_ctx->dom_id; -- s64 vtime_delta; -- -- old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id); -- if (!old_domc) { -- scx_bpf_error("No dom%u", old_dom_id); -- return false; -- } -- -- vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now; -- -- new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id); -- if (!new_domc) { -- scx_bpf_error("No dom%u", new_dom_id); -- return false; -- } -- -- d_cpumask = new_domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to get domain %u cpumask kptr", -- new_dom_id); -- return false; -- } -- -- t_cpumask = task_ctx->cpumask; -- if (!t_cpumask) { -- scx_bpf_error("Failed to look up task cpumask"); -- return false; -- } -- -- /* -- * set_cpumask might have happened between userspace requesting LB and -- * here and @p might not be able to run in @dom_id anymore. Verify. -- */ -- if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- p->cpus_ptr)) { -- p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta; -- task_ctx->dom_id = new_dom_id; -- bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask, -- p->cpus_ptr); -- } -- -- return task_ctx->dom_id == new_dom_id; --} -- --s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu, -- u32 wake_flags) --{ -- s32 cpu; -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- struct bpf_cpumask *p_cpumask; -- -- if (!task_ctx) -- return -ENOENT; -- -- if (kthreads_local && -- (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- cpu = prev_cpu; -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local dsq of the waker. -- */ -- if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) { -- struct task_struct *current = (void *)bpf_get_current_task(); -- -- if (!(BPF_CORE_READ(current, flags) & PF_EXITING) && -- task_ctx->dom_id < MAX_DOMS) { -- struct dom_ctx *domc; -- struct bpf_cpumask *d_cpumask; -- const struct cpumask *idle_cpumask; -- bool has_idle; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id); -- if (!domc) { -- scx_bpf_error("Failed to find dom%u", -- task_ctx->dom_id); -- return prev_cpu; -- } -- d_cpumask = domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to acquire domain %u cpumask kptr", -- task_ctx->dom_id); -- return prev_cpu; -- } -- -- idle_cpumask = scx_bpf_get_idle_cpumask(); -- -- has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- idle_cpumask); -- -- scx_bpf_put_idle_cpumask(idle_cpumask); -- -- if (has_idle) { -- cpu = bpf_get_smp_processor_id(); -- if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- stat_add(ATROPOS_STAT_WAKE_SYNC, 1); -- goto local; -- } -- } -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- stat_add(ATROPOS_STAT_PREV_IDLE, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- /* If only one core is allowed, dispatch */ -- if (p->nr_cpus_allowed == 1) { -- stat_add(ATROPOS_STAT_PINNED, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) -- return -ENOENT; -- -- /* If there is an eligible idle CPU, dispatch directly */ -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- if (cpu >= 0) { -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * @prev_cpu may be in a different domain. Returning an out-of-domain -- * CPU can lead to stalls as all in-domain CPUs may be idle by the time -- * @p gets enqueued. -- */ -- if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask)) -- cpu = prev_cpu; -- else -- cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask); -- -- return cpu; -- --local: -- task_ctx->dispatch_local = true; -- return cpu; --} -- --void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- u32 *new_dom; -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- new_dom = bpf_map_lookup_elem(&lb_data, &pid); -- if (new_dom && *new_dom != task_ctx->dom_id && -- task_set_dsq(task_ctx, p, *new_dom)) { -- struct bpf_cpumask *p_cpumask; -- s32 cpu; -- -- stat_add(ATROPOS_STAT_LOAD_BALANCE, 1); -- -- /* -- * If dispatch_local is set, We own @p's idle state but we are -- * not gonna put the task in the associated local dsq which can -- * cause the CPU to stall. Kick it. -- */ -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0); -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) { -- scx_bpf_error("Failed to get task_ctx->cpumask"); -- return; -- } -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- } -- -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags); -- return; -- } -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000, -- enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- u32 dom_id = task_ctx->dom_id; -- struct dom_ctx *domc; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, domc->vtime_now - slice_us * 1000)) -- vtime = domc->vtime_now - slice_us * 1000; -- -- scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --static u32 cpu_to_dom_id(s32 cpu) --{ -- const volatile u32 *dom_idp; -- -- if (nr_doms <= 1) -- return 0; -- -- dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]); -- if (!dom_idp) -- return MAX_DOMS; -- -- return *dom_idp; --} -- --static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) --{ -- s32 cpu; -- -- if (dom_id >= MAX_DOMS) -- return false; -- -- bpf_for(cpu, 0, nr_cpus) { -- if (bpf_cpumask_test_cpu(cpu, cpumask) && -- (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64)))) -- return true; -- } -- return false; --} -- --static u32 dom_rr_next(s32 cpu) --{ -- struct pcpu_ctx *pcpuc; -- u32 dom_id; -- -- pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]); -- if (!pcpuc) -- return 0; -- -- dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms; -- -- if (dom_id == cpu_to_dom_id(cpu)) -- dom_id = (dom_id + 1) % nr_doms; -- -- pcpuc->dom_rr_cur = dom_id; -- return dom_id; --} -- --void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 dom = cpu_to_dom_id(cpu); -- -- if (scx_bpf_consume(dom)) { -- stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1); -- return; -- } -- -- if (!greedy_threshold) -- return; -- -- bpf_repeat(nr_doms - 1) { -- u32 dom_id = dom_rr_next(cpu); -- -- if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold && -- scx_bpf_consume(dom_id)) { -- stat_add(ATROPOS_STAT_GREEDY, 1); -- break; -- } -- } --} -- --void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_at = bpf_ktime_get_ns(); --} -- --void BPF_STRUCT_OPS(atropos_running, struct task_struct *p) --{ -- struct task_ctx *taskc; -- struct dom_ctx *domc; -- pid_t pid = p->pid; -- u32 dom_id; -- -- if (fifo_sched) -- return; -- -- taskc = bpf_map_lookup_elem(&task_data, &pid); -- if (!taskc) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- dom_id = taskc->dom_id; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(domc->vtime_now, p->scx.dsq_vtime)) -- domc->vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at; -- task_ctx->runnable_at = 0; --} -- --void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->weight = weight; --} -- --struct pick_task_domain_loop_ctx { -- struct task_struct *p; -- const struct cpumask *cpumask; -- u64 dom_mask; -- u32 dom_rr_base; -- u32 dom_id; --}; -- --static int pick_task_domain_loopfn(u32 idx, void *data) --{ -- struct pick_task_domain_loop_ctx *lctx = data; -- u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms; -- -- if (dom_id >= MAX_DOMS) -- return 1; -- -- if (cpumask_intersects_domain(lctx->cpumask, dom_id)) { -- lctx->dom_mask |= 1LLU << dom_id; -- if (lctx->dom_id == MAX_DOMS) -- lctx->dom_id = dom_id; -- } -- return 0; --} -- --static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- struct pick_task_domain_loop_ctx lctx = { -- .p = p, -- .cpumask = cpumask, -- .dom_id = MAX_DOMS, -- }; -- s32 cpu = bpf_get_smp_processor_id(); -- -- if (cpu < 0 || cpu >= MAX_CPUS) -- return MAX_DOMS; -- -- lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur); -- -- bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0); -- task_ctx->dom_mask = lctx.dom_mask; -- -- return lctx.dom_id; --} -- --static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- u32 dom_id = 0; -- -- if (nr_doms > 1) -- dom_id = pick_task_domain(task_ctx, p, cpumask); -- -- if (!task_set_dsq(task_ctx, p, dom_id)) -- scx_bpf_error("Failed to set domain %d for %s[%d]", -- dom_id, p->comm, p->pid); --} -- --void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_set_domain(task_ctx, p, cpumask); --} -- --s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct bpf_cpumask *cpumask; -- struct task_ctx task_ctx, *map_value; -- long ret; -- pid_t pid; -- -- memset(&task_ctx, 0, sizeof(task_ctx)); -- -- pid = p->pid; -- ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return ret; -- } -- -- /* -- * Read the entry from the map immediately so we can add the cpumask -- * with bpf_kptr_xchg(). -- */ -- map_value = bpf_map_lookup_elem(&task_data, &pid); -- if (!map_value) -- /* Should never happen -- it was just inserted above. */ -- return -EINVAL; -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- bpf_map_delete_elem(&task_data, &pid); -- return -ENOMEM; -- } -- -- cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask); -- if (cpumask) { -- /* Should never happen as we just inserted it above. */ -- bpf_cpumask_release(cpumask); -- bpf_map_delete_elem(&task_data, &pid); -- return -EINVAL; -- } -- -- task_set_domain(map_value, p, p->cpus_ptr); -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p) --{ -- pid_t pid = p->pid; -- long ret = bpf_map_delete_elem(&task_data, &pid); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return; -- } --} -- --static int create_dom_dsq(u32 idx, void *data) --{ -- struct dom_ctx domc_init = {}, *domc; -- struct bpf_cpumask *cpumask; -- u32 cpu, dom_id = idx; -- s32 ret; -- -- ret = scx_bpf_create_dsq(dom_id, -1); -- if (ret < 0) { -- scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret); -- return 1; -- } -- -- ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0); -- if (ret) { -- scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret); -- return 1; -- } -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- /* Should never happen, we just inserted it above. */ -- scx_bpf_error("No dom%u", dom_id); -- return 1; -- } -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id); -- return 1; -- } -- -- for (cpu = 0; cpu < MAX_CPUS; cpu++) { -- const volatile __u64 *dmask; -- -- dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]); -- if (!dmask) { -- scx_bpf_error("array index error"); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- if (*dmask & (1LLU << (cpu % 64))) -- bpf_cpumask_set_cpu(cpu, cpumask); -- } -- -- cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask); -- if (cpumask) { -- scx_bpf_error("Domain %u was already present", dom_id); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(atropos_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- bpf_loop(nr_doms, create_dom_dsq, NULL, 0); -- -- for (u32 i = 0; i < nr_cpus; i++) -- pcpu_ctx[i].dom_rr_cur = i; -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg); -- exit_type = ei->type; --} -- --SEC(".struct_ops") --struct sched_ext_ops atropos = { -- .select_cpu = (void *)atropos_select_cpu, -- .enqueue = (void *)atropos_enqueue, -- .dispatch = (void *)atropos_dispatch, -- .runnable = (void *)atropos_runnable, -- .running = (void *)atropos_running, -- .stopping = (void *)atropos_stopping, -- .quiescent = (void *)atropos_quiescent, -- .set_weight = (void *)atropos_set_weight, -- .set_cpumask = (void *)atropos_set_cpumask, -- .prep_enable = (void *)atropos_prep_enable, -- .disable = (void *)atropos_disable, -- .init = (void *)atropos_init, -- .exit = (void *)atropos_exit, -- .flags = 0, -- .name = "atropos", --}; -diff --git a/tools/sched_ext/atropos/src/bpf/atropos.h b/tools/sched_ext/atropos/src/bpf/atropos.h -deleted file mode 100644 -index addf29ca104a..000000000000 ---- a/tools/sched_ext/atropos/src/bpf/atropos.h -+++ /dev/null -@@ -1,44 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#ifndef __ATROPOS_H --#define __ATROPOS_H -- --#include --#ifndef __kptr --#ifdef __KERNEL__ --#error "__kptr_ref not defined in the kernel" --#endif --#define __kptr --#endif -- --#define MAX_CPUS 512 --#define MAX_DOMS 64 /* limited to avoid complex bitmask ops */ --#define CACHELINE_SIZE 64 -- --/* Statistics */ --enum stat_idx { -- ATROPOS_STAT_TASK_GET_ERR, -- ATROPOS_STAT_WAKE_SYNC, -- ATROPOS_STAT_PREV_IDLE, -- ATROPOS_STAT_PINNED, -- ATROPOS_STAT_DIRECT_DISPATCH, -- ATROPOS_STAT_DSQ_DISPATCH, -- ATROPOS_STAT_GREEDY, -- ATROPOS_STAT_LOAD_BALANCE, -- ATROPOS_STAT_LAST_TASK, -- ATROPOS_NR_STATS, --}; -- --struct task_ctx { -- unsigned long long dom_mask; /* the domains this task can run on */ -- struct bpf_cpumask __kptr *cpumask; -- unsigned int dom_id; -- unsigned int weight; -- unsigned long long runnable_at; -- unsigned long long runnable_for; -- bool dispatch_local; --}; -- --#endif /* __ATROPOS_H */ -diff --git a/tools/sched_ext/atropos/src/main.rs b/tools/sched_ext/atropos/src/main.rs -deleted file mode 100644 -index 0d313662f713..000000000000 ---- a/tools/sched_ext/atropos/src/main.rs -+++ /dev/null -@@ -1,942 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#[path = "bpf/.output/atropos.skel.rs"] --mod atropos; --pub use atropos::*; --pub mod atropos_sys; -- --use std::cell::Cell; --use std::collections::{BTreeMap, BTreeSet}; --use std::ffi::CStr; --use std::ops::Bound::{Included, Unbounded}; --use std::sync::atomic::{AtomicBool, Ordering}; --use std::sync::Arc; --use std::time::{Duration, SystemTime}; -- --use ::fb_procfs as procfs; --use anyhow::{anyhow, bail, Context, Result}; --use bitvec::prelude::*; --use clap::Parser; --use log::{info, trace, warn}; --use ordered_float::OrderedFloat; -- --/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF --/// part does simple round robin in each domain and the userspace part --/// calculates the load factor of each domain and tells the BPF part how to load --/// balance the domains. -- --/// This scheduler demonstrates dividing scheduling logic between BPF and --/// userspace and using rust to build the userspace part. An earlier variant of --/// this scheduler was used to balance across six domains, each representing a --/// chiplet in a six-chiplet AMD processor, and could match the performance of --/// production setup using CFS. --#[derive(Debug, Parser)] --struct Opts { -- /// Scheduling slice duration in microseconds. -- #[clap(short, long, default_value = "20000")] -- slice_us: u64, -- -- /// Monitoring and load balance interval in seconds. -- #[clap(short, long, default_value = "2.0")] -- interval: f64, -- -- /// Build domains according to how CPUs are grouped at this cache level -- /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id. -- #[clap(short = 'c', long, default_value = "3")] -- cache_level: u32, -- -- /// Instead of using cache locality, set the cpumask for each domain -- /// manually, provide multiple --cpumasks, one for each domain. E.g. -- /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with -- /// the corresponding CPUs belonging to each domain. Each CPU must -- /// belong to precisely one domain. -- #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")] -- cpumasks: Vec, -- -- /// When non-zero, enable greedy task stealing. When a domain is idle, a -- /// cpu will attempt to steal tasks from a domain with at least -- /// greedy_threshold tasks enqueued. These tasks aren't permanently -- /// stolen from the domain. -- #[clap(short, long, default_value = "4")] -- greedy_threshold: u32, -- -- /// The load decay factor. Every interval, the existing load is decayed -- /// by this factor and new load is added. Must be in the range [0.0, -- /// 0.99]. The smaller the value, the more sensitive load calculation -- /// is to recent changes. When 0.0, history is ignored and the load -- /// value from the latest period is used directly. -- #[clap(short, long, default_value = "0.5")] -- load_decay_factor: f64, -- -- /// Disable load balancing. Unless disabled, periodically userspace will -- /// calculate the load factor of each domain and instruct BPF which -- /// processes to move. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- no_load_balance: bool, -- -- /// Put per-cpu kthreads directly into local dsq's. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- kthreads_local: bool, -- -- /// Use FIFO scheduling instead of weighted vtime scheduling. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- fifo_sched: bool, -- -- /// If specified, only tasks which have their scheduling policy set to -- /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all -- /// tasks are switched. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- partial: bool, -- -- /// Enable verbose output including libbpf details. Specify multiple -- /// times to increase verbosity. -- #[clap(short, long, action = clap::ArgAction::Count)] -- verbose: u8, --} -- --fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result { -- Ok(reader -- .read_stat() -- .context("Failed to read procfs")? -- .total_cpu -- .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?) --} -- --fn now_monotonic() -> u64 { -- let mut time = libc::timespec { -- tv_sec: 0, -- tv_nsec: 0, -- }; -- let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) }; -- assert!(ret == 0); -- time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 --} -- --fn clear_map(map: &mut libbpf_rs::Map) { -- // XXX: libbpf_rs has some design flaw that make it impossible to -- // delete while iterating despite it being safe so we alias it here -- let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) }; -- for key in map.keys() { -- let _ = deleter.delete(&key); -- } --} -- --#[derive(Debug)] --struct TaskLoad { -- runnable_for: u64, -- load: f64, --} -- --#[derive(Debug)] --struct TaskInfo { -- pid: i32, -- dom_mask: u64, -- migrated: Cell, --} -- --struct LoadBalancer<'a, 'b, 'c> { -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- -- tasks_by_load: Vec, TaskInfo>>, -- load_avg: f64, -- dom_loads: Vec, -- -- imbal: Vec, -- doms_to_push: BTreeMap, u32>, -- doms_to_pull: BTreeMap, u32>, -- -- nr_lb_data_errors: &'c mut u64, --} -- --impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> { -- const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10; -- const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1; -- const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50; -- -- fn new( -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- nr_lb_data_errors: &'c mut u64, -- ) -> Self { -- Self { -- maps, -- task_loads, -- nr_doms, -- load_decay_factor, -- -- tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(), -- load_avg: 0f64, -- dom_loads: vec![0.0; nr_doms], -- -- imbal: vec![0.0; nr_doms], -- doms_to_pull: BTreeMap::new(), -- doms_to_push: BTreeMap::new(), -- -- nr_lb_data_errors, -- } -- } -- -- fn read_task_loads(&mut self, period: Duration) -> Result<()> { -- let now_mono = now_monotonic(); -- let task_data = self.maps.task_data(); -- let mut this_task_loads = BTreeMap::::new(); -- let mut load_sum = 0.0f64; -- self.dom_loads = vec![0f64; self.nr_doms]; -- -- for key in task_data.keys() { -- if let Some(task_ctx_vec) = task_data -- .lookup(&key, libbpf_rs::MapFlags::ANY) -- .context("Failed to lookup task_data")? -- { -- let task_ctx = -- unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) }; -- let pid = i32::from_ne_bytes( -- key.as_slice() -- .try_into() -- .context("Invalid key length in task_data map")?, -- ); -- -- let (this_at, this_for, weight) = unsafe { -- ( -- std::ptr::read_volatile(&task_ctx.runnable_at as *const u64), -- std::ptr::read_volatile(&task_ctx.runnable_for as *const u64), -- std::ptr::read_volatile(&task_ctx.weight as *const u32), -- ) -- }; -- -- let (mut delta, prev_load) = match self.task_loads.get(&pid) { -- Some(prev) => (this_for - prev.runnable_for, Some(prev.load)), -- None => (this_for, None), -- }; -- -- // Non-zero this_at indicates that the task is currently -- // runnable. Note that we read runnable_at and runnable_for -- // without any synchronization and there is a small window -- // where we end up misaccounting. While this can cause -- // temporary error, it's unlikely to cause any noticeable -- // misbehavior especially given the load value clamping. -- if this_at > 0 && this_at < now_mono { -- delta += now_mono - this_at; -- } -- -- delta = delta.min(period.as_nanos() as u64); -- let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64) -- .clamp(0.0, weight as f64); -- -- let this_load = match prev_load { -- Some(prev_load) => { -- prev_load * self.load_decay_factor -- + this_load * (1.0 - self.load_decay_factor) -- } -- None => this_load, -- }; -- -- this_task_loads.insert( -- pid, -- TaskLoad { -- runnable_for: this_for, -- load: this_load, -- }, -- ); -- -- load_sum += this_load; -- self.dom_loads[task_ctx.dom_id as usize] += this_load; -- // Only record pids that are eligible for load balancing -- if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) { -- continue; -- } -- self.tasks_by_load[task_ctx.dom_id as usize].insert( -- OrderedFloat(this_load), -- TaskInfo { -- pid, -- dom_mask: task_ctx.dom_mask, -- migrated: Cell::new(false), -- }, -- ); -- } -- } -- -- self.load_avg = load_sum / self.nr_doms as f64; -- *self.task_loads = this_task_loads; -- Ok(()) -- } -- -- // To balance dom loads we identify doms with lower and higher load than average -- fn calculate_dom_load_balance(&mut self) -> Result<()> { -- for (dom, dom_load) in self.dom_loads.iter().enumerate() { -- let imbal = dom_load - self.load_avg; -- if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO { -- if imbal > 0f64 { -- self.doms_to_push.insert(OrderedFloat(imbal), dom as u32); -- } else { -- self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32); -- } -- self.imbal[dom] = imbal; -- } -- } -- Ok(()) -- } -- -- // Find the first candidate pid which hasn't already been migrated and -- // can run in @pull_dom. -- fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)> -- where -- I: IntoIterator, &'d TaskInfo)>, -- { -- match tasks_by_load -- .into_iter() -- .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0) -- .next() -- { -- Some((OrderedFloat(load), task)) => Some((*load, task)), -- None => None, -- } -- } -- -- fn pick_victim( -- &self, -- (push_dom, to_push): (u32, f64), -- (pull_dom, to_pull): (u32, f64), -- ) -> Option<(&TaskInfo, f64)> { -- let to_xfer = to_pull.min(to_push); -- -- trace!( -- "considering dom {}@{:.2} -> {}@{:.2}", -- push_dom, -- to_push, -- pull_dom, -- to_pull -- ); -- -- let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs(); -- -- trace!( -- "to_xfer={:.2} tasks_by_load={:?}", -- to_xfer, -- &self.tasks_by_load[push_dom as usize] -- ); -- -- // We want to pick a task to transfer from push_dom to pull_dom to -- // maximize the reduction of load imbalance between the two. IOW, -- // pick a task which has the closest load value to $to_xfer that can -- // be migrated. Find such task by locating the first migratable task -- // while scanning left from $to_xfer and the counterpart while -- // scanning right and picking the better of the two. -- let (load, task, new_imbal) = match ( -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Unbounded, Included(&OrderedFloat(to_xfer)))) -- .rev(), -- pull_dom, -- ), -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Included(&OrderedFloat(to_xfer)), Unbounded)), -- pull_dom, -- ), -- ) { -- (None, None) => return None, -- (Some((load, task)), None) | (None, Some((load, task))) => { -- (load, task, calc_new_imbal(load)) -- } -- (Some((load0, task0)), Some((load1, task1))) => { -- let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1)); -- if new_imbal0 <= new_imbal1 { -- (load0, task0, new_imbal0) -- } else { -- (load1, task1, new_imbal1) -- } -- } -- }; -- -- // If the best candidate can't reduce the imbalance, there's nothing -- // to do for this pair. -- let old_imbal = to_push + to_pull; -- if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal { -- trace!( -- "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal -- ); -- return None; -- } -- -- trace!( -- "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal, -- ); -- -- Some((task, load)) -- } -- -- // Actually execute the load balancing. Concretely this writes pid -> dom -- // entries into the lb_data map for bpf side to consume. -- fn load_balance(&mut self) -> Result<()> { -- clear_map(self.maps.lb_data()); -- -- trace!("imbal={:?}", &self.imbal); -- trace!("doms_to_push={:?}", &self.doms_to_push); -- trace!("doms_to_pull={:?}", &self.doms_to_pull); -- -- // Push from the most imbalanced to least. -- while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() { -- let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO; -- let mut pushed = 0f64; -- -- // Transfer tasks from push_dom to reduce imbalance. -- loop { -- let last_pushed = pushed; -- -- // Pull from the most imbalaned to least. -- let mut doms_to_pull = BTreeMap::<_, _>::new(); -- std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull); -- let mut pull_doms = doms_to_pull.into_iter().rev().collect::>(); -- -- for (to_pull, pull_dom) in pull_doms.iter_mut() { -- if let Some((task, load)) = -- self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull))) -- { -- // Execute migration. -- task.migrated.set(true); -- to_push -= load; -- *to_pull -= load; -- pushed += load; -- -- // Ask BPF code to execute the migration. -- let pid = task.pid; -- let cpid = (pid as libc::pid_t).to_ne_bytes(); -- if let Err(e) = self.maps.lb_data().update( -- &cpid, -- &pull_dom.to_ne_bytes(), -- libbpf_rs::MapFlags::NO_EXIST, -- ) { -- warn!( -- "Failed to update lb_data map for pid={} error={:?}", -- pid, &e -- ); -- *self.nr_lb_data_errors += 1; -- } -- -- // Always break after a successful migration so that -- // the pulling domains are always considered in the -- // descending imbalance order. -- break; -- } -- } -- -- pull_doms -- .into_iter() -- .map(|(k, v)| self.doms_to_pull.insert(k, v)) -- .count(); -- -- // Stop repeating if nothing got transferred or pushed enough. -- if pushed == last_pushed || pushed >= push_max { -- break; -- } -- } -- } -- Ok(()) -- } --} -- --struct Scheduler<'a> { -- skel: AtroposSkel<'a>, -- struct_ops: Option, -- -- nr_cpus: usize, -- nr_doms: usize, -- load_decay_factor: f64, -- balance_load: bool, -- -- proc_reader: procfs::ProcReader, -- -- prev_at: SystemTime, -- prev_total_cpu: procfs::CpuStat, -- task_loads: BTreeMap, -- -- nr_lb_data_errors: u64, --} -- --impl<'a> Scheduler<'a> { -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn parse_cpusets( -- cpumasks: &[String], -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- if cpumasks.len() > atropos_sys::MAX_DOMS as usize { -- bail!( -- "Number of requested DSQs ({}) is greater than MAX_DOMS ({})", -- cpumasks.len(), -- atropos_sys::MAX_DOMS -- ); -- } -- let mut cpus = vec![-1i32; nr_cpus]; -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()]; -- for (dq, cpumask) in cpumasks.iter().enumerate() { -- let hex_str = { -- let mut tmp_str = cpumask -- .strip_prefix("0x") -- .unwrap_or(cpumask) -- .replace('_', ""); -- if tmp_str.len() % 2 != 0 { -- tmp_str = "0".to_string() + &tmp_str; -- } -- tmp_str -- }; -- let byte_vec = hex::decode(&hex_str) -- .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?; -- -- for (index, &val) in byte_vec.iter().rev().enumerate() { -- let mut v = val; -- while v != 0 { -- let lsb = v.trailing_zeros() as usize; -- v &= !(1 << lsb); -- let cpu = index * 8 + lsb; -- if cpu > nr_cpus { -- bail!( -- concat!( -- "Found cpu ({}) in cpumask ({}) which is larger", -- " than the number of cpus on the machine ({})" -- ), -- cpu, -- cpumask, -- nr_cpus -- ); -- } -- if cpus[cpu] != -1 { -- bail!( -- "Found cpu ({}) with dq ({}) but also in cpumask ({})", -- cpu, -- cpus[cpu], -- cpumask -- ); -- } -- cpus[cpu] = dq as i32; -- cpusets[dq].set(cpu, true); -- } -- } -- cpusets[dq].set_uninitialized(false); -- } -- -- for (cpu, &dq) in cpus.iter().enumerate() { -- if dq < 0 { -- bail!( -- "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.", -- cpu -- ); -- } -- } -- -- Ok((cpusets, cpus)) -- } -- -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn cpusets_from_cache( -- level: u32, -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- let mut cpu_to_cache = vec![]; // (cpu_id, cache_id) -- let mut cache_ids = BTreeSet::::new(); -- let mut nr_not_found = 0; -- -- // Build cpu -> cache ID mapping. -- for cpu in 0..nr_cpus { -- let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level); -- let id = match std::fs::read_to_string(&path) { -- Ok(val) => val -- .trim() -- .parse::() -- .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?, -- Err(e) if e.kind() == std::io::ErrorKind::NotFound => { -- nr_not_found += 1; -- 0 -- } -- Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)), -- }; -- -- cpu_to_cache.push(id); -- cache_ids.insert(id); -- } -- -- if nr_not_found > 1 { -- warn!( -- "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0", -- level, nr_not_found, nr_cpus -- ); -- } -- -- // Cache IDs may have holes. Assign consecutive domain IDs to -- // existing cache IDs. -- let mut cache_to_dom = BTreeMap::::new(); -- let mut nr_doms = 0; -- for cache_id in cache_ids.iter() { -- cache_to_dom.insert(*cache_id, nr_doms); -- nr_doms += 1; -- } -- -- if nr_doms > atropos_sys::MAX_DOMS { -- bail!( -- "Total number of doms {} is greater than MAX_DOMS ({})", -- nr_doms, -- atropos_sys::MAX_DOMS -- ); -- } -- -- // Build and return dom -> cpumask and cpu -> dom mappings. -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize]; -- let mut cpu_to_dom = vec![]; -- -- for cpu in 0..nr_cpus { -- let dom_id = cache_to_dom[&cpu_to_cache[cpu]]; -- cpusets[dom_id as usize].set(cpu, true); -- cpu_to_dom.push(dom_id as i32); -- } -- -- Ok((cpusets, cpu_to_dom)) -- } -- -- fn init(opts: &Opts) -> Result { -- // Open the BPF prog first for verification. -- let mut skel_builder = AtroposSkelBuilder::default(); -- skel_builder.obj_builder.debug(opts.verbose > 0); -- let mut skel = skel_builder.open().context("Failed to open BPF program")?; -- -- let nr_cpus = libbpf_rs::num_possible_cpus().unwrap(); -- if nr_cpus > atropos_sys::MAX_CPUS as usize { -- bail!( -- "nr_cpus ({}) is greater than MAX_CPUS ({})", -- nr_cpus, -- atropos_sys::MAX_CPUS -- ); -- } -- -- // Initialize skel according to @opts. -- let (cpusets, cpus) = if opts.cpumasks.len() > 0 { -- Self::parse_cpusets(&opts.cpumasks, nr_cpus)? -- } else { -- Self::cpusets_from_cache(opts.cache_level, nr_cpus)? -- }; -- let nr_doms = cpusets.len(); -- skel.rodata().nr_doms = nr_doms as u32; -- skel.rodata().nr_cpus = nr_cpus as u32; -- -- for (cpu, dom) in cpus.iter().enumerate() { -- skel.rodata().cpu_dom_id_map[cpu] = *dom as u32; -- } -- -- for (dom, cpuset) in cpusets.iter().enumerate() { -- let raw_cpuset_slice = cpuset.as_raw_slice(); -- let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom]; -- let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len()); -- left.clone_from_slice(cpuset.as_raw_slice()); -- let cpumask_str = dom_cpumask_slice -- .iter() -- .take((nr_cpus + 63) / 64) -- .rev() -- .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x)); -- info!( -- "DOM[{:02}] cpumask{} ({} cpus)", -- dom, -- &cpumask_str, -- cpuset.count_ones() -- ); -- } -- -- skel.rodata().slice_us = opts.slice_us; -- skel.rodata().kthreads_local = opts.kthreads_local; -- skel.rodata().fifo_sched = opts.fifo_sched; -- skel.rodata().switch_partial = opts.partial; -- skel.rodata().greedy_threshold = opts.greedy_threshold; -- -- // Attach. -- let mut skel = skel.load().context("Failed to load BPF program")?; -- skel.attach().context("Failed to attach BPF program")?; -- let struct_ops = Some( -- skel.maps_mut() -- .atropos() -- .attach_struct_ops() -- .context("Failed to attach atropos struct ops")?, -- ); -- info!("Atropos Scheduler Attached"); -- -- // Other stuff. -- let mut proc_reader = procfs::ProcReader::new(); -- let prev_total_cpu = read_total_cpu(&mut proc_reader)?; -- -- Ok(Self { -- skel, -- struct_ops, // should be held to keep it attached -- -- nr_cpus, -- nr_doms, -- load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99), -- balance_load: !opts.no_load_balance, -- -- proc_reader, -- -- prev_at: SystemTime::now(), -- prev_total_cpu, -- task_loads: BTreeMap::new(), -- -- nr_lb_data_errors: 0, -- }) -- } -- -- fn get_cpu_busy(&mut self) -> Result { -- let total_cpu = read_total_cpu(&mut self.proc_reader)?; -- let busy = match (&self.prev_total_cpu, &total_cpu) { -- ( -- procfs::CpuStat { -- user_usec: Some(prev_user), -- nice_usec: Some(prev_nice), -- system_usec: Some(prev_system), -- idle_usec: Some(prev_idle), -- iowait_usec: Some(prev_iowait), -- irq_usec: Some(prev_irq), -- softirq_usec: Some(prev_softirq), -- stolen_usec: Some(prev_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- procfs::CpuStat { -- user_usec: Some(curr_user), -- nice_usec: Some(curr_nice), -- system_usec: Some(curr_system), -- idle_usec: Some(curr_idle), -- iowait_usec: Some(curr_iowait), -- irq_usec: Some(curr_irq), -- softirq_usec: Some(curr_softirq), -- stolen_usec: Some(curr_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- ) => { -- let idle_usec = curr_idle - prev_idle; -- let iowait_usec = curr_iowait - prev_iowait; -- let user_usec = curr_user - prev_user; -- let system_usec = curr_system - prev_system; -- let nice_usec = curr_nice - prev_nice; -- let irq_usec = curr_irq - prev_irq; -- let softirq_usec = curr_softirq - prev_softirq; -- let stolen_usec = curr_stolen - prev_stolen; -- -- let busy_usec = -- user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec; -- let total_usec = idle_usec + busy_usec + iowait_usec; -- busy_usec as f64 / total_usec as f64 -- } -- _ => { -- bail!("Some procfs stats are not populated!"); -- } -- }; -- -- self.prev_total_cpu = total_cpu; -- Ok(busy) -- } -- -- fn read_bpf_stats(&mut self) -> Result> { -- let mut maps = self.skel.maps_mut(); -- let stats_map = maps.stats(); -- let mut stats: Vec = Vec::new(); -- let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus]; -- -- for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS { -- let cpu_stat_vec = stats_map -- .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY) -- .with_context(|| format!("Failed to lookup stat {}", stat))? -- .expect("per-cpu stat should exist"); -- let sum = cpu_stat_vec -- .iter() -- .map(|val| { -- u64::from_ne_bytes( -- val.as_slice() -- .try_into() -- .expect("Invalid value length in stat map"), -- ) -- }) -- .sum(); -- stats_map -- .update_percpu( -- &(stat as u32).to_ne_bytes(), -- &zero_vec, -- libbpf_rs::MapFlags::ANY, -- ) -- .context("Failed to zero stat")?; -- stats.push(sum); -- } -- Ok(stats) -- } -- -- fn report( -- &self, -- stats: &Vec, -- cpu_busy: f64, -- processing_dur: Duration, -- load_avg: f64, -- dom_loads: &Vec, -- imbal: &Vec, -- ) { -- let stat = |idx| stats[idx as usize]; -- let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK); -- -- info!( -- "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms", -- cpu_busy * 100.0, -- load_avg, -- stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize], -- stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize], -- self.nr_lb_data_errors, -- processing_dur.as_millis(), -- ); -- -- let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0; -- -- info!( -- "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}", -- total, -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY), -- ); -- -- for i in 0..self.nr_doms { -- info!( -- "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}", -- i, -- dom_loads[i], -- if imbal[i] < 0.0 { -imbal[i] } else { 0.0 }, -- if imbal[i] > 0.0 { imbal[i] } else { 0.0 }, -- ); -- } -- } -- -- fn step(&mut self) -> Result<()> { -- let started_at = std::time::SystemTime::now(); -- let bpf_stats = self.read_bpf_stats()?; -- let cpu_busy = self.get_cpu_busy()?; -- -- let mut lb = LoadBalancer::new( -- self.skel.maps_mut(), -- &mut self.task_loads, -- self.nr_doms, -- self.load_decay_factor, -- &mut self.nr_lb_data_errors, -- ); -- -- lb.read_task_loads(started_at.duration_since(self.prev_at)?)?; -- lb.calculate_dom_load_balance()?; -- -- if self.balance_load { -- lb.load_balance()?; -- } -- -- // Extract fields needed for reporting and drop lb to release -- // mutable borrows. -- let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal); -- -- self.report( -- &bpf_stats, -- cpu_busy, -- std::time::SystemTime::now().duration_since(started_at)?, -- load_avg, -- &dom_loads, -- &imbal, -- ); -- -- self.prev_at = started_at; -- Ok(()) -- } -- -- fn read_bpf_exit_type(&mut self) -> i32 { -- unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) } -- } -- -- fn report_bpf_exit_type(&mut self) -> Result<()> { -- // Report msg if EXT_OPS_EXIT_ERROR. -- match self.read_bpf_exit_type() { -- 0 => Ok(()), -- etype if etype == 2 => { -- let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) }; -- let msg = cstr -- .to_str() -- .context("Failed to convert exit msg to string") -- .unwrap(); -- bail!("BPF exit_type={} msg={}", etype, msg); -- } -- etype => { -- info!("BPF exit_type={}", etype); -- Ok(()) -- } -- } -- } --} -- --impl<'a> Drop for Scheduler<'a> { -- fn drop(&mut self) { -- if let Some(struct_ops) = self.struct_ops.take() { -- drop(struct_ops); -- } -- } --} -- --fn main() -> Result<()> { -- let opts = Opts::parse(); -- -- let llv = match opts.verbose { -- 0 => simplelog::LevelFilter::Info, -- 1 => simplelog::LevelFilter::Debug, -- _ => simplelog::LevelFilter::Trace, -- }; -- let mut lcfg = simplelog::ConfigBuilder::new(); -- lcfg.set_time_level(simplelog::LevelFilter::Error) -- .set_location_level(simplelog::LevelFilter::Off) -- .set_target_level(simplelog::LevelFilter::Off) -- .set_thread_level(simplelog::LevelFilter::Off); -- simplelog::TermLogger::init( -- llv, -- lcfg.build(), -- simplelog::TerminalMode::Stderr, -- simplelog::ColorChoice::Auto, -- )?; -- -- let shutdown = Arc::new(AtomicBool::new(false)); -- let shutdown_clone = shutdown.clone(); -- ctrlc::set_handler(move || { -- shutdown_clone.store(true, Ordering::Relaxed); -- }) -- .context("Error setting Ctrl-C handler")?; -- -- let mut sched = Scheduler::init(&opts)?; -- -- while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 { -- std::thread::sleep(Duration::from_secs_f64(opts.interval)); -- sched.step()?; -- } -- -- sched.report_bpf_exit_type() --} -diff --git a/tools/sched_ext/gnu/stubs.h b/tools/sched_ext/gnu/stubs.h -deleted file mode 100644 -index 719225b16626..000000000000 ---- a/tools/sched_ext/gnu/stubs.h -+++ /dev/null -@@ -1 +0,0 @@ --/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ -diff --git a/tools/sched_ext/scx_common.bpf.h b/tools/sched_ext/scx_common.bpf.h -deleted file mode 100644 -index e56de9dc86f2..000000000000 ---- a/tools/sched_ext/scx_common.bpf.h -+++ /dev/null -@@ -1,288 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __SCHED_EXT_COMMON_BPF_H --#define __SCHED_EXT_COMMON_BPF_H -- --#include "vmlinux.h" --#include --#include --#include --#include "user_exit_info.h" -- --#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ --#define PF_EXITING 0x00000004 --#define CLOCK_MONOTONIC 1 -- --/* -- * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can -- * lead to really confusing misbehaviors. Let's trigger a build failure. -- */ --static inline void ___vmlinux_h_sanity_check___(void) --{ -- _Static_assert(SCX_DSQ_FLAG_BUILTIN, -- "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); --} -- --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; -- --static inline __attribute__((format(printf, 1, 2))) --void ___scx_bpf_error_format_checker(const char *fmt, ...) {} -- --/* -- * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments -- * instead of an array of u64. Note that __param[] must have at least one -- * element to keep the verifier happy. -- */ --#define scx_bpf_error(fmt, args...) \ --({ \ -- static char ___fmt[] = fmt; \ -- unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ -- \ -- _Pragma("GCC diagnostic push") \ -- _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ -- ___bpf_fill(___param, args); \ -- _Pragma("GCC diagnostic pop") \ -- \ -- scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ -- \ -- ___scx_bpf_error_format_checker(fmt, ##args); \ --}) -- --void scx_bpf_switch_all(void) __ksym; --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; --bool scx_bpf_consume(u64 dsq_id) __ksym; --u32 scx_bpf_dispatch_nr_slots(void) __ksym; --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym; --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym; --void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; --s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym; --const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; --const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; --void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; --void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; --bool scx_bpf_task_running(const struct task_struct *p) __ksym; --s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym; --u32 scx_bpf_reenqueue_local(void) __ksym; -- --#define BPF_STRUCT_OPS(name, args...) \ --SEC("struct_ops/"#name) \ --BPF_PROG(name, ##args) -- --#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ --SEC("struct_ops.s/"#name) \ --BPF_PROG(name, ##args) -- --/** -- * MEMBER_VPTR - Obtain the verified pointer to a struct or array member -- * @base: struct or array to index -- * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...) -- * -- * The verifier often gets confused by the instruction sequence the compiler -- * generates for indexing struct fields or arrays. This macro forces the -- * compiler to generate a code sequence which first calculates the byte offset, -- * checks it against the struct or array size and add that byte offset to -- * generate the pointer to the member to help the verifier. -- * -- * Ideally, we want to abort if the calculated offset is out-of-bounds. However, -- * BPF currently doesn't support abort, so evaluate to NULL instead. The caller -- * must check for NULL and take appropriate action to appease the verifier. To -- * avoid confusing the verifier, it's best to check for NULL and dereference -- * immediately. -- * -- * vptr = MEMBER_VPTR(my_array, [i][j]); -- * if (!vptr) -- * return error; -- * *vptr = new_value; -- */ --#define MEMBER_VPTR(base, member) (typeof(base member) *)({ \ -- u64 __base = (u64)base; \ -- u64 __addr = (u64)&(base member) - __base; \ -- asm volatile ( \ -- "if %0 <= %[max] goto +2\n" \ -- "%0 = 0\n" \ -- "goto +1\n" \ -- "%0 += %1\n" \ -- : "+r"(__addr) \ -- : "r"(__base), \ -- [max]"i"(sizeof(base) - sizeof(base member))); \ -- __addr; \ --}) -- --/* -- * BPF core and other generic helpers -- */ -- --/* list and rbtree */ --#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) --#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) -- --void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; --void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; -- --#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) --#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) -- --void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; --struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; --struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, -- struct bpf_rb_node *node) __ksym; --void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, -- bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym; --struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; -- --/* task */ --struct task_struct *bpf_task_from_pid(s32 pid) __ksym; --struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; --void bpf_task_release(struct task_struct *p) __ksym; -- --/* cgroup */ --struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; --void bpf_cgroup_release(struct cgroup *cgrp) __ksym; --struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; -- --/* cpumask */ --struct bpf_cpumask *bpf_cpumask_create(void) __ksym; --struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; --u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; --u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym; -- --/* rcu */ --void bpf_rcu_read_lock(void) __ksym; --void bpf_rcu_read_unlock(void) __ksym; -- --/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */ --struct bpf_iter_num; -- --extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym; --extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym; --extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym; -- --#ifndef bpf_for_each --/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for -- * using BPF open-coded iterators without having to write mundane explicit -- * low-level loop logic. Instead, it provides for()-like generic construct -- * that can be used pretty naturally. E.g., for some hypothetical cgroup -- * iterator, you'd write: -- * -- * struct cgroup *cg, *parent_cg = <...>; -- * -- * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { -- * bpf_printk("Child cgroup id = %d", cg->cgroup_id); -- * if (cg->cgroup_id == 123) -- * break; -- * } -- * -- * I.e., it looks almost like high-level for each loop in other languages, -- * supports continue/break, and is verifiable by BPF verifier. -- * -- * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) -- * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to -- * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int -- * *`, not just `int`. So for integers bpf_for() is more convenient. -- * -- * Note: this macro relies on C99 feature of allowing to declare variables -- * inside for() loop, bound to for() loop lifetime. It also utilizes GCC -- * extension: __attribute__((cleanup())), supported by both GCC and -- * Clang. -- */ --#define bpf_for_each(type, cur, args...) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ -- cleanup(bpf_iter_##type##_destroy))), \ -- /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_##type##_new(&___it, ##args), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_##type##_destroy, (void *)0); \ -- /* iteration and termination check */ \ -- (((cur) = bpf_iter_##type##_next(&___it))); \ --) --#endif /* bpf_for_each */ -- --#ifndef bpf_for --/* bpf_for(i, start, end) implements a for()-like looping construct that sets -- * provided integer variable *i* to values starting from *start* through, -- * but not including, *end*. It also proves to BPF verifier that *i* belongs -- * to range [start, end), so this can be used for accessing arrays without -- * extra checks. -- * -- * Note: *start* and *end* are assumed to be expressions with no side effects -- * and whose values do not change throughout bpf_for() loop execution. They do -- * not have to be statically known or constant, though. -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_for(i, start, end) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, (start), (end)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- ({ \ -- /* iteration step */ \ -- int *___t = bpf_iter_num_next(&___it); \ -- /* termination and bounds check */ \ -- (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ -- }); \ --) --#endif /* bpf_for */ -- --#ifndef bpf_repeat --/* bpf_repeat(N) performs N iterations without exposing iteration number -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_repeat(N) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, 0, (N)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- bpf_iter_num_next(&___it); \ -- /* nothing here */ \ --) --#endif /* bpf_repeat */ -- --#endif /* __SCHED_EXT_COMMON_BPF_H */ -diff --git a/tools/sched_ext/scx_example_central.bpf.c b/tools/sched_ext/scx_example_central.bpf.c -deleted file mode 100644 -index 4cec04b4c2ed..000000000000 ---- a/tools/sched_ext/scx_example_central.bpf.c -+++ /dev/null -@@ -1,334 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A central FIFO sched_ext scheduler which demonstrates the followings: -- * -- * a. Making all scheduling decisions from one CPU: -- * -- * The central CPU is the only one making scheduling decisions. All other -- * CPUs kick the central CPU when they run out of tasks to run. -- * -- * There is one global BPF queue and the central CPU schedules all CPUs by -- * dispatching from the global queue to each CPU's local dsq from dispatch(). -- * This isn't the most straightforward. e.g. It'd be easier to bounce -- * through per-CPU BPF queues. The current design is chosen to maximally -- * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching. -- * -- * b. Tickless operation -- * -- * All tasks are dispatched with the infinite slice which allows stopping the -- * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full -- * parameter. The tickless operation can be observed through -- * /proc/interrupts. -- * -- * Periodic switching is enforced by a periodic timer checking all CPUs and -- * preempting them as necessary. Unfortunately, BPF timer currently doesn't -- * have a way to pin to a specific CPU, so the periodic timer isn't pinned to -- * the central CPU. -- * -- * c. Preemption -- * -- * Kthreads are unconditionally queued to the head of a matching local dsq -- * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always -- * prioritized over user threads, which is required for ensuring forward -- * progress as e.g. the periodic timer may run on a ksoftirqd and if the -- * ksoftirqd gets starved by a user thread, there may not be anything else to -- * vacate that user thread. -- * -- * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the -- * next tasks. -- * -- * This scheduler is designed to maximize usage of various SCX mechanisms. A -- * more practical implementation would likely put the scheduling loop outside -- * the central CPU's dispatch() path and add some form of priority mechanism. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --enum { -- FALLBACK_DSQ_ID = 0, -- MAX_CPUS = 4096, -- MS_TO_NS = 1000LLU * 1000, -- TIMER_INTERVAL_NS = 1 * MS_TO_NS, --}; -- --const volatile bool switch_partial; --const volatile s32 central_cpu; --const volatile u32 nr_cpu_ids = 64; /* !0 for veristat, set during init */ -- --u64 nr_total, nr_locals, nr_queued, nr_lost_pids; --u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries; --u64 nr_overflows; -- --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, s32); --} central_q SEC(".maps"); -- --/* can't use percpu map due to bad lookups */ --static bool cpu_gimme_task[MAX_CPUS]; --static u64 cpu_started_at[MAX_CPUS]; -- --struct central_timer { -- struct bpf_timer timer; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, 1); -- __type(key, u32); -- __type(value, struct central_timer); --} central_timer SEC(".maps"); -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- /* -- * Steer wakeups to the central CPU as much as possible to avoid -- * disturbing other CPUs. It's safe to blindly return the central cpu as -- * select_cpu() is a hint and if @p can't be on it, the kernel will -- * automatically pick a fallback CPU. -- */ -- return central_cpu; --} -- --void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags) --{ -- s32 pid = p->pid; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- /* -- * Push per-cpu kthreads at the head of local dsq's and preempt the -- * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked -- * behind other threads which is necessary for forward progress -- * guarantee as we depend on the BPF timer which may run from ksoftirqd. -- */ -- if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- __sync_fetch_and_add(&nr_locals, 1); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF, -- enq_flags | SCX_ENQ_PREEMPT); -- return; -- } -- -- if (bpf_map_push_elem(¢ral_q, &pid, 0)) { -- __sync_fetch_and_add(&nr_overflows, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_queued, 1); -- -- if (!scx_bpf_task_running(p)) -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); --} -- --static bool dispatch_to_cpu(s32 cpu) --{ -- struct task_struct *p; -- s32 pid; -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (bpf_map_pop_elem(¢ral_q, &pid)) -- break; -- -- __sync_fetch_and_sub(&nr_queued, 1); -- -- p = bpf_task_from_pid(pid); -- if (!p) { -- __sync_fetch_and_add(&nr_lost_pids, 1); -- continue; -- } -- -- /* -- * If we can't run the task at the top, do the dumb thing and -- * bounce it to the fallback dsq. -- */ -- if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- __sync_fetch_and_add(&nr_mismatches, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0); -- bpf_task_release(p); -- continue; -- } -- -- /* dispatch to local and mark that @cpu doesn't need more */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0); -- -- if (cpu != central_cpu) -- scx_bpf_kick_cpu(cpu, 0); -- -- bpf_task_release(p); -- return true; -- } -- -- return false; --} -- --void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (cpu == central_cpu) { -- /* dispatch for all other CPUs first */ -- __sync_fetch_and_add(&nr_dispatches, 1); -- -- bpf_for(cpu, 0, nr_cpu_ids) { -- bool *gimme; -- -- if (!scx_bpf_dispatch_nr_slots()) -- break; -- -- /* central's gimme is never set */ -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme && !*gimme) -- continue; -- -- if (dispatch_to_cpu(cpu)) -- *gimme = false; -- } -- -- /* -- * Retry if we ran out of dispatch buffer slots as we might have -- * skipped some CPUs and also need to dispatch for self. The ext -- * core automatically retries if the local dsq is empty but we -- * can't rely on that as we're dispatching for other CPUs too. -- * Kick self explicitly to retry. -- */ -- if (!scx_bpf_dispatch_nr_slots()) { -- __sync_fetch_and_add(&nr_retries, 1); -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- return; -- } -- -- /* look for a task to run on the central CPU */ -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- dispatch_to_cpu(central_cpu); -- } else { -- bool *gimme; -- -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme) -- *gimme = true; -- -- /* -- * Force dispatch on the scheduling CPU so that it finds a task -- * to run for us. -- */ -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- } --} -- --void BPF_STRUCT_OPS(central_running, struct task_struct *p) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ --} -- --void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = 0; --} -- --static int central_timerfn(void *map, int *key, struct bpf_timer *timer) --{ -- u64 now = bpf_ktime_get_ns(); -- u64 nr_to_kick = nr_queued; -- s32 i; -- -- bpf_for(i, 0, nr_cpu_ids) { -- s32 cpu = (nr_timers + i) % nr_cpu_ids; -- u64 *started_at; -- -- if (cpu == central_cpu) -- continue; -- -- /* kick iff the current one exhausted its slice */ -- started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at && *started_at && -- vtime_before(now, *started_at + SCX_SLICE_DFL)) -- continue; -- -- /* and there's something pending */ -- if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) || -- scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu)) -- ; -- else if (nr_to_kick) -- nr_to_kick--; -- else -- continue; -- -- scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); -- } -- -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- -- bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- __sync_fetch_and_add(&nr_timers, 1); -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(central_init) --{ -- u32 key = 0; -- struct bpf_timer *timer; -- int ret; -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1); -- if (ret) -- return ret; -- -- timer = bpf_map_lookup_elem(¢ral_timer, &key); -- if (!timer) -- return -ESRCH; -- -- bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); -- bpf_timer_set_callback(timer, central_timerfn); -- ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- return ret; --} -- --void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops central_ops = { -- /* -- * We are offloading all scheduling decisions to the central CPU and -- * thus being the last task on a given CPU doesn't mean anything -- * special. Enqueue the last tasks like any other tasks. -- */ -- .flags = SCX_OPS_ENQ_LAST, -- -- .select_cpu = (void *)central_select_cpu, -- .enqueue = (void *)central_enqueue, -- .dispatch = (void *)central_dispatch, -- .running = (void *)central_running, -- .stopping = (void *)central_stopping, -- .init = (void *)central_init, -- .exit = (void *)central_exit, -- .name = "central", --}; -diff --git a/tools/sched_ext/scx_example_central.c b/tools/sched_ext/scx_example_central.c -deleted file mode 100644 -index 7ad591cbdc65..000000000000 ---- a/tools/sched_ext/scx_example_central.c -+++ /dev/null -@@ -1,94 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_central.skel.h" -- --const char help_fmt[] = --"A central FIFO sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-c CPU] [-p]\n" --"\n" --" -c CPU Override the central CPU (default: 0)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_central *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_central__open(); -- assert(skel); -- -- skel->rodata->central_cpu = 0; -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "c:ph")) != -1) { -- switch (opt) { -- case 'c': -- skel->rodata->central_cpu = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_central__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.central_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf("total :%10lu local:%10lu queued:%10lu lost:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_locals, -- skel->bss->nr_queued, -- skel->bss->nr_lost_pids); -- printf("timer :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n", -- skel->bss->nr_timers, -- skel->bss->nr_dispatches, -- skel->bss->nr_mismatches, -- skel->bss->nr_retries); -- printf("overflow:%10lu\n", -- skel->bss->nr_overflows); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_central__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_flatcg.bpf.c b/tools/sched_ext/scx_example_flatcg.bpf.c -deleted file mode 100644 -index f6078b9a681f..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.bpf.c -+++ /dev/null -@@ -1,872 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext flattened cgroup hierarchy scheduler. It implements -- * hierarchical weight-based cgroup CPU control by flattening the cgroup -- * hierarchy into a single layer by compounding the active weight share at each -- * level. Consider the following hierarchy with weights in parentheses: -- * -- * R + A (100) + B (100) -- * | \ C (100) -- * \ D (200) -- * -- * Ignoring the root and threaded cgroups, only B, C and D can contain tasks. -- * Let's say all three have runnable tasks. The total share that each of these -- * three cgroups is entitled to can be calculated by compounding its share at -- * each level. -- * -- * For example, B is competing against C and in that competition its share is -- * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's -- * share in that competition is 200/(200+100) == 1/3. B's eventual share in the -- * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's -- * eventual shaer is the same at 1/6. D is only competing at the top level and -- * its share is 200/(100+200) == 2/3. -- * -- * So, instead of hierarchically scheduling level-by-level, we can consider it -- * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3 -- * and keep updating the eventual shares as the cgroups' runnable states change. -- * -- * This flattening of hierarchy can bring a substantial performance gain when -- * the cgroup hierarchy is nested multiple levels. in a simple benchmark using -- * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it -- * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two -- * apache instances competing with 2:1 weight ratio nested four level deep. -- * -- * However, the gain comes at the cost of not being able to properly handle -- * thundering herd of cgroups. For example, if many cgroups which are nested -- * behind a low priority parent cgroup wake up around the same time, they may be -- * able to consume more CPU cycles than they are entitled to. In many use cases, -- * this isn't a real concern especially given the performance gain. Also, there -- * are ways to mitigate the problem further by e.g. introducing an extra -- * scheduling layer on cgroup delegation boundaries. -- * -- * The scheduler first picks the cgroup to run and then schedule the tasks -- * within by using nested weighted vtime scheduling by default. The -- * cgroup-internal scheduling can be switched to FIFO with the -f option. -- */ --#include "scx_common.bpf.h" --#include "user_exit_info.h" --#include "scx_example_flatcg.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */ --const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL; --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --u64 cvtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, u64); -- __uint(max_entries, FCG_NR_STATS); --} stats SEC(".maps"); -- --static void stat_inc(enum fcg_stat_idx idx) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p)++; --} -- --struct fcg_cpu_ctx { -- u64 cur_cgid; -- u64 cur_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, struct fcg_cpu_ctx); -- __uint(max_entries, 1); --} cpu_ctx SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_cgrp_ctx); --} cgrp_ctx SEC(".maps"); -- --struct cgv_node { -- struct bpf_rb_node rb_node; -- __u64 cvtime; -- __u64 cgid; --}; -- --private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock; --private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node); -- --struct cgv_node_stash { -- struct cgv_node __kptr *node; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, 16384); -- __type(key, __u64); -- __type(value, struct cgv_node_stash); --} cgv_node_stash SEC(".maps"); -- --struct fcg_task_ctx { -- u64 bypassed_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_task_ctx); --} task_ctx SEC(".maps"); -- --/* gets inc'd on weight tree changes to expire the cached hweights */ --unsigned long hweight_gen = 1; -- --static u64 div_round_up(u64 dividend, u64 divisor) --{ -- return (dividend + divisor - 1) / divisor; --} -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) --{ -- struct cgv_node *cgc_a, *cgc_b; -- -- cgc_a = container_of(a, struct cgv_node, rb_node); -- cgc_b = container_of(b, struct cgv_node, rb_node); -- -- return cgc_a->cvtime < cgc_b->cvtime; --} -- --static struct fcg_cpu_ctx *find_cpu_ctx(void) --{ -- struct fcg_cpu_ctx *cpuc; -- u32 idx = 0; -- -- cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx); -- if (!cpuc) { -- scx_bpf_error("cpu_ctx lookup failed"); -- return NULL; -- } -- return cpuc; --} -- --static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id); -- return NULL; -- } -- return cgc; --} -- --static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgrp = bpf_cgroup_ancestor(cgrp, level); -- if (!cgrp) { -- scx_bpf_error("ancestor cgroup lookup failed"); -- return NULL; -- } -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- scx_bpf_error("ancestor cgrp_ctx lookup failed"); -- bpf_cgroup_release(cgrp); -- return cgc; --} -- --static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- int level; -- -- if (!cgc->nr_active) { -- stat_inc(FCG_STAT_HWT_SKIP); -- return; -- } -- -- if (cgc->hweight_gen == hweight_gen) { -- stat_inc(FCG_STAT_HWT_CACHE); -- return; -- } -- -- stat_inc(FCG_STAT_HWT_UPDATES); -- bpf_for(level, 0, cgrp->level + 1) { -- struct fcg_cgrp_ctx *cgc; -- bool is_active; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- -- if (!level) { -- cgc->hweight = FCG_HWEIGHT_ONE; -- cgc->hweight_gen = hweight_gen; -- } else { -- struct fcg_cgrp_ctx *pcgc; -- -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- -- /* -- * We can be oppotunistic here and not grab the -- * cgv_tree_lock and deal with the occasional races. -- * However, hweight updates are already cached and -- * relatively low-frequency. Let's just do the -- * straightforward thing. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- is_active = cgc->nr_active; -- if (is_active) { -- cgc->hweight_gen = pcgc->hweight_gen; -- cgc->hweight = -- div_round_up(pcgc->hweight * cgc->weight, -- pcgc->child_weight_sum); -- } -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!is_active) { -- stat_inc(FCG_STAT_HWT_RACE); -- break; -- } -- } -- } --} -- --static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) --{ -- u64 delta, cvtime, max_budget; -- -- /* -- * A node which is on the rbtree can't be pointed to from elsewhere yet -- * and thus can't be updated and repositioned. Instead, we collect the -- * vtime deltas separately and apply it asynchronously here. -- */ -- delta = cgc->cvtime_delta; -- __sync_fetch_and_sub(&cgc->cvtime_delta, delta); -- cvtime = cgv_node->cvtime + delta; -- -- /* -- * Allow a cgroup to carry the maximum budget proportional to its -- * hweight such that a full-hweight cgroup can immediately take up half -- * of the CPUs at the most while staying at the front of the rbtree. -- */ -- max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / -- (2 * FCG_HWEIGHT_ONE); -- if (vtime_before(cvtime, cvtime_now - max_budget)) -- cvtime = cvtime_now - max_budget; -- -- cgv_node->cvtime = cvtime; --} -- --static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- u64 cgid = cgrp->kn->id; -- -- /* paired with cmpxchg in try_pick_next_cgroup() */ -- if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { -- stat_inc(FCG_STAT_ENQ_SKIP); -- return; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid); -- return; -- } -- -- /* NULL if the node is already on the rbtree */ -- cgv_node = bpf_kptr_xchg(&stash->node, NULL); -- if (!cgv_node) { -- stat_inc(FCG_STAT_ENQ_RACE); -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); --} -- --void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * If select_cpu_dfl() is recommending local enqueue, the target CPU is -- * idle. Follow it and charge the cgroup later in fcg_stopping() after -- * the fact. Use the same mechanism to deal with tasks with custom -- * affinities so that we don't have to worry about per-cgroup dq's -- * containing tasks that can't be executed from some CPUs. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) { -- /* -- * Tell fcg_stopping() that this bypassed the regular scheduling -- * path and should be force charged to the cgroup. 0 is used to -- * indicate that the task isn't bypassing, so if the current -- * runtime is 0, go back by one nanosecond. -- */ -- taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1; -- -- /* -- * The global dq is deprioritized as we don't want to let tasks -- * to boost themselves by constraining its cpumask. The -- * deprioritization is rather severe, so let's not apply that to -- * per-cpu kernel threads. This is ham-fisted. We probably wanna -- * implement per-cgroup fallback dq's instead so that we have -- * more control over when tasks with custom cpumask get issued. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || -- (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) { -- stat_inc(FCG_STAT_LOCAL); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- } else { -- stat_inc(FCG_STAT_GLOBAL); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } -- return; -- } -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- goto out_release; -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 tvtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) -- tvtime = cgc->tvtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, -- tvtime, enq_flags); -- } -- -- cgrp_enqueued(cgrp, cgc); --out_release: -- bpf_cgroup_release(cgrp); --} -- --/* -- * Walk the cgroup tree to update the active weight sums as tasks wake up and -- * sleep. The weight sums are used as the base when calculating the proportion a -- * given cgroup or task is entitled to at each level. -- */ --static void update_active_weight_sums(struct cgroup *cgrp, bool runnable) --{ -- struct fcg_cgrp_ctx *cgc; -- bool updated = false; -- int idx; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- /* -- * In most cases, a hot cgroup would have multiple threads going to -- * sleep and waking up while the whole cgroup stays active. In leaf -- * cgroups, ->nr_runnable which is updated with __sync operations gates -- * ->nr_active updates, so that we don't have to grab the cgv_tree_lock -- * repeatedly for a busy cgroup which is staying active. -- */ -- if (runnable) { -- if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_ACT); -- } else { -- if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_DEACT); -- } -- -- /* -- * If @cgrp is becoming runnable, its hweight should be refreshed after -- * it's added to the weight tree so that enqueue has the up-to-date -- * value. If @cgrp is becoming quiescent, the hweight should be -- * refreshed before it's removed from the weight tree so that the usage -- * charging which happens afterwards has access to the latest value. -- */ -- if (!runnable) -- cgrp_refresh_hweight(cgrp, cgc); -- -- /* propagate upwards */ -- bpf_for(idx, 0, cgrp->level) { -- int level = cgrp->level - idx; -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- bool propagate = false; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- if (level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- } -- -- /* -- * We need the propagation protected by a lock to synchronize -- * against weight changes. There's no reason to drop the lock at -- * each level but bpf_spin_lock() doesn't want any function -- * calls while locked. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- -- if (runnable) { -- if (!cgc->nr_active++) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum += cgc->weight; -- } -- } -- } else { -- if (!--cgc->nr_active) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum -= cgc->weight; -- } -- } -- } -- -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!propagate) -- break; -- } -- -- if (updated) -- __sync_fetch_and_add(&hweight_gen, 1); -- -- if (runnable) -- cgrp_refresh_hweight(cgrp, cgc); --} -- --void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, true); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) --{ -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- if (fifo_sched) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- /* -- * @cgc->tvtime_now always progresses forward as tasks start -- * executing. The test and update can be performed concurrently -- * from multiple CPUs and thus racy. Any error should be -- * contained and temporary. Let's just live with it. -- */ -- if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) -- cgc->tvtime_now = p->scx.dsq_vtime; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- /* scale the execution time by the inverse of the weight and charge */ -- if (!fifo_sched) -- p->scx.dsq_vtime += -- (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- if (!taskc->bypassed_at) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- __sync_fetch_and_add(&cgc->cvtime_delta, -- p->se.sum_exec_runtime - taskc->bypassed_at); -- taskc->bypassed_at = 0; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, false); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) --{ -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- if (cgrp->level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1); -- if (!pcgc) -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- if (pcgc && cgc->nr_active) -- pcgc->child_weight_sum += (s64)weight - cgc->weight; -- cgc->weight = weight; -- bpf_spin_unlock(&cgv_tree_lock); --} -- --static bool try_pick_next_cgroup(u64 *cgidp) --{ -- struct bpf_rb_node *rb_node; -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 cgid; -- -- /* pop the front cgroup and wind cvtime_now accordingly */ -- bpf_spin_lock(&cgv_tree_lock); -- -- rb_node = bpf_rbtree_first(&cgv_tree); -- if (!rb_node) { -- bpf_spin_unlock(&cgv_tree_lock); -- stat_inc(FCG_STAT_PNC_NO_CGRP); -- *cgidp = 0; -- return true; -- } -- -- rb_node = bpf_rbtree_remove(&cgv_tree, rb_node); -- bpf_spin_unlock(&cgv_tree_lock); -- -- cgv_node = container_of(rb_node, struct cgv_node, rb_node); -- cgid = cgv_node->cgid; -- -- if (vtime_before(cvtime_now, cgv_node->cvtime)) -- cvtime_now = cgv_node->cvtime; -- -- /* -- * If lookup fails, the cgroup's gone. Free and move on. See -- * fcg_cgroup_exit(). -- */ -- cgrp = bpf_cgroup_from_id(cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- if (!scx_bpf_consume(cgid)) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_EMPTY); -- goto out_stash; -- } -- -- /* -- * Successfully consumed from the cgroup. This will be our current -- * cgroup for the new slice. Refresh its hweight. -- */ -- cgrp_refresh_hweight(cgrp, cgc); -- -- bpf_cgroup_release(cgrp); -- -- /* -- * As the cgroup may have more tasks, add it back to the rbtree. Note -- * that here we charge the full slice upfront and then exact later -- * according to the actual consumption. This prevents lowpri thundering -- * herd from saturating the machine. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- -- *cgidp = cgid; -- stat_inc(FCG_STAT_PNC_NEXT); -- return true; -- --out_stash: -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- /* -- * Paired with cmpxchg in cgrp_enqueued(). If they see the following -- * transition, they'll enqueue the cgroup. If they are earlier, we'll -- * see their task in the dq below and requeue the cgroup. -- */ -- __sync_val_compare_and_swap(&cgc->queued, 1, 0); -- -- if (scx_bpf_dsq_nr_queued(cgid)) { -- bpf_spin_lock(&cgv_tree_lock); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- goto out_free; -- } -- } -- -- return false; -- --out_free: -- bpf_obj_drop(cgv_node); -- return false; --} -- --void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) --{ -- struct fcg_cpu_ctx *cpuc; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 now = bpf_ktime_get_ns(); -- -- cpuc = find_cpu_ctx(); -- if (!cpuc) -- return; -- -- if (!cpuc->cur_cgid) -- goto pick_next_cgroup; -- -- if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { -- if (scx_bpf_consume(cpuc->cur_cgid)) { -- stat_inc(FCG_STAT_CNS_KEEP); -- return; -- } -- stat_inc(FCG_STAT_CNS_EMPTY); -- } else { -- stat_inc(FCG_STAT_CNS_EXPIRE); -- } -- -- /* -- * The current cgroup is expiring. It was already charged a full slice. -- * Calculate the actual usage and accumulate the delta. -- */ -- cgrp = bpf_cgroup_from_id(cpuc->cur_cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_CNS_GONE); -- goto pick_next_cgroup; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (cgc) { -- /* -- * We want to update the vtime delta and then look for the next -- * cgroup to execute but the latter needs to be done in a loop -- * and we can't keep the lock held. Oh well... -- */ -- bpf_spin_lock(&cgv_tree_lock); -- __sync_fetch_and_add(&cgc->cvtime_delta, -- (cpuc->cur_at + cgrp_slice_ns - now) * -- FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- stat_inc(FCG_STAT_CNS_GONE); -- } -- -- bpf_cgroup_release(cgrp); -- --pick_next_cgroup: -- cpuc->cur_at = now; -- -- if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { -- cpuc->cur_cgid = 0; -- return; -- } -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_pick_next_cgroup(&cpuc->cur_cgid)) -- break; -- } --} -- --s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct fcg_task_ctx *taskc; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- taskc = bpf_task_storage_get(&task_ctx, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!taskc) -- return -ENOMEM; -- -- taskc->bypassed_at = 0; -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp, -- struct scx_cgroup_init_args *args) --{ -- struct fcg_cgrp_ctx *cgc; -- struct cgv_node *cgv_node; -- struct cgv_node_stash empty_stash = {}, *stash; -- u64 cgid = cgrp->kn->id; -- int ret; -- -- /* -- * Technically incorrect as cgroup ID is full 64bit while dq ID is -- * 63bit. Should not be a problem in practice and easy to spot in the -- * unlikely case that it breaks. -- */ -- ret = scx_bpf_create_dsq(cgid, -1); -- if (ret) -- return ret; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!cgc) { -- ret = -ENOMEM; -- goto err_destroy_dsq; -- } -- -- cgc->weight = args->weight; -- cgc->hweight = FCG_HWEIGHT_ONE; -- -- ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash, -- BPF_NOEXIST); -- if (ret) { -- if (ret != -ENOMEM) -- scx_bpf_error("unexpected stash creation error (%d)", -- ret); -- goto err_destroy_dsq; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("unexpected cgv_node stash lookup failure"); -- ret = -ENOENT; -- goto err_destroy_dsq; -- } -- -- cgv_node = bpf_obj_new(struct cgv_node); -- if (!cgv_node) { -- ret = -ENOMEM; -- goto err_del_cgv_node; -- } -- -- cgv_node->cgid = cgid; -- cgv_node->cvtime = cvtime_now; -- -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- ret = -EBUSY; -- goto err_drop; -- } -- -- return 0; -- --err_drop: -- bpf_obj_drop(cgv_node); --err_del_cgv_node: -- bpf_map_delete_elem(&cgv_node_stash, &cgid); --err_destroy_dsq: -- scx_bpf_destroy_dsq(cgid); -- return ret; --} -- --void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- -- /* -- * For now, there's no way find and remove the cgv_node if it's on the -- * cgv_tree. Let's drain them in the dispatch path as they get popped -- * off the front of the tree. -- */ -- bpf_map_delete_elem(&cgv_node_stash, &cgid); -- scx_bpf_destroy_dsq(cgid); --} -- --s32 BPF_STRUCT_OPS(fcg_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops flatcg_ops = { -- .enqueue = (void *)fcg_enqueue, -- .dispatch = (void *)fcg_dispatch, -- .runnable = (void *)fcg_runnable, -- .running = (void *)fcg_running, -- .stopping = (void *)fcg_stopping, -- .quiescent = (void *)fcg_quiescent, -- .prep_enable = (void *)fcg_prep_enable, -- .cgroup_set_weight = (void *)fcg_cgroup_set_weight, -- .cgroup_init = (void *)fcg_cgroup_init, -- .cgroup_exit = (void *)fcg_cgroup_exit, -- .init = (void *)fcg_init, -- .exit = (void *)fcg_exit, -- .flags = SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING, -- .name = "flatcg", --}; -diff --git a/tools/sched_ext/scx_example_flatcg.c b/tools/sched_ext/scx_example_flatcg.c -deleted file mode 100644 -index f9c8a5b84a70..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.c -+++ /dev/null -@@ -1,232 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2023 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2023 Tejun Heo -- * Copyright (c) 2023 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_flatcg.h" --#include "scx_example_flatcg.skel.h" -- --#ifndef FILEID_KERNFS --#define FILEID_KERNFS 0xfe --#endif -- --const char help_fmt[] = --"A flattened cgroup hierarchy sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -i INTERVAL Report interval\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --static float read_cpu_util(__u64 *last_sum, __u64 *last_idle) --{ -- FILE *fp; -- char buf[4096]; -- char *line, *cur = NULL, *tok; -- __u64 sum = 0, idle = 0; -- __u64 delta_sum, delta_idle; -- int idx; -- -- fp = fopen("/proc/stat", "r"); -- if (!fp) { -- perror("fopen(\"/proc/stat\")"); -- return 0.0; -- } -- -- if (!fgets(buf, sizeof(buf), fp)) { -- perror("fgets(\"/proc/stat\")"); -- fclose(fp); -- return 0.0; -- } -- fclose(fp); -- -- line = buf; -- for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) { -- char *endp = NULL; -- __u64 v; -- -- if (idx == 0) { -- line = NULL; -- continue; -- } -- v = strtoull(tok, &endp, 0); -- if (!endp || *endp != '\0') { -- fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n", -- idx, tok); -- continue; -- } -- sum += v; -- if (idx == 4) -- idle = v; -- } -- -- delta_sum = sum - *last_sum; -- delta_idle = idle - *last_idle; -- *last_sum = sum; -- *last_idle = idle; -- -- return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0; --} -- --static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats) --{ -- __u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus]; -- __u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); -- -- for (idx = 0; idx < FCG_NR_STATS; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_flatcg *skel; -- struct bpf_link *link; -- struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 }; -- bool dump_cgrps = false; -- __u64 last_cpu_sum = 0, last_cpu_idle = 0; -- __u64 last_stats[FCG_NR_STATS] = {}; -- unsigned long seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_flatcg__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open: %s\n", strerror(errno)); -- return 1; -- } -- -- skel->rodata->nr_cpus = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) { -- double v; -- -- switch (opt) { -- case 's': -- v = strtod(optarg, NULL); -- skel->rodata->cgrp_slice_ns = v * 1000; -- break; -- case 'i': -- v = strtod(optarg, NULL); -- intv_ts.tv_sec = v; -- intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000; -- break; -- case 'd': -- dump_cgrps = true; -- break; -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- case 'h': -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d", -- (double)skel->rodata->cgrp_slice_ns / 1000000.0, -- (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0, -- dump_cgrps); -- -- if (scx_example_flatcg__load(skel)) { -- fprintf(stderr, "Failed to load: %s\n", strerror(errno)); -- return 1; -- } -- -- link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops); -- if (!link) { -- fprintf(stderr, "Failed to attach_struct_ops: %s\n", -- strerror(errno)); -- return 1; -- } -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- __u64 acc_stats[FCG_NR_STATS]; -- __u64 stats[FCG_NR_STATS]; -- float cpu_util; -- int i; -- -- cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle); -- -- fcg_read_stats(skel, acc_stats); -- for (i = 0; i < FCG_NR_STATS; i++) -- stats[i] = acc_stats[i] - last_stats[i]; -- -- memcpy(last_stats, acc_stats, sizeof(acc_stats)); -- -- printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n", -- seq++, cpu_util * 100.0, skel->data->hweight_gen); -- printf(" act:%6llu deact:%6llu local:%6llu global:%6llu\n", -- stats[FCG_STAT_ACT], -- stats[FCG_STAT_DEACT], -- stats[FCG_STAT_LOCAL], -- stats[FCG_STAT_GLOBAL]); -- printf("HWT skip:%6llu race:%6llu cache:%6llu update:%6llu\n", -- stats[FCG_STAT_HWT_SKIP], -- stats[FCG_STAT_HWT_RACE], -- stats[FCG_STAT_HWT_CACHE], -- stats[FCG_STAT_HWT_UPDATES]); -- printf("ENQ skip:%6llu race:%6llu\n", -- stats[FCG_STAT_ENQ_SKIP], -- stats[FCG_STAT_ENQ_RACE]); -- printf("CNS keep:%6llu expire:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_CNS_KEEP], -- stats[FCG_STAT_CNS_EXPIRE], -- stats[FCG_STAT_CNS_EMPTY], -- stats[FCG_STAT_CNS_GONE]); -- printf("PNC nocgrp:%6llu next:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_PNC_NO_CGRP], -- stats[FCG_STAT_PNC_NEXT], -- stats[FCG_STAT_PNC_EMPTY], -- stats[FCG_STAT_PNC_GONE]); -- printf("BAD remove:%6llu\n", -- acc_stats[FCG_STAT_BAD_REMOVAL]); -- -- nanosleep(&intv_ts, NULL); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_flatcg__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_flatcg.h b/tools/sched_ext/scx_example_flatcg.h -deleted file mode 100644 -index 490758ed41f0..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.h -+++ /dev/null -@@ -1,49 +0,0 @@ --#ifndef __SCX_EXAMPLE_FLATCG_H --#define __SCX_EXAMPLE_FLATCG_H -- --enum { -- FCG_HWEIGHT_ONE = 1LLU << 16, --}; -- --enum fcg_stat_idx { -- FCG_STAT_ACT, -- FCG_STAT_DEACT, -- FCG_STAT_LOCAL, -- FCG_STAT_GLOBAL, -- -- FCG_STAT_HWT_UPDATES, -- FCG_STAT_HWT_CACHE, -- FCG_STAT_HWT_SKIP, -- FCG_STAT_HWT_RACE, -- -- FCG_STAT_ENQ_SKIP, -- FCG_STAT_ENQ_RACE, -- -- FCG_STAT_CNS_KEEP, -- FCG_STAT_CNS_EXPIRE, -- FCG_STAT_CNS_EMPTY, -- FCG_STAT_CNS_GONE, -- -- FCG_STAT_PNC_NO_CGRP, -- FCG_STAT_PNC_NEXT, -- FCG_STAT_PNC_EMPTY, -- FCG_STAT_PNC_GONE, -- -- FCG_STAT_BAD_REMOVAL, -- -- FCG_NR_STATS, --}; -- --struct fcg_cgrp_ctx { -- u32 nr_active; -- u32 nr_runnable; -- u32 queued; -- u32 weight; -- u32 hweight; -- u64 child_weight_sum; -- u64 hweight_gen; -- s64 cvtime_delta; -- u64 tvtime_now; --}; -- --#endif /* __SCX_EXAMPLE_FLATCG_H */ -diff --git a/tools/sched_ext/scx_example_pair.bpf.c b/tools/sched_ext/scx_example_pair.bpf.c -deleted file mode 100644 -index 279efe58b777..000000000000 ---- a/tools/sched_ext/scx_example_pair.bpf.c -+++ /dev/null -@@ -1,627 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext core-scheduler which always makes every sibling CPU pair -- * execute from the same CPU cgroup. -- * -- * This scheduler is a minimal implementation and would need some form of -- * priority handling both inside each cgroup and across the cgroups to be -- * practically useful. -- * -- * Each CPU in the system is paired with exactly one other CPU, according to a -- * "stride" value that can be specified when the BPF scheduler program is first -- * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee -- * that they will only ever schedule tasks that belong to the same CPU cgroup. -- * -- * Scheduler Initialization -- * ------------------------ -- * -- * The scheduler BPF program is first initialized from user space, before it is -- * enabled. During this initialization process, each CPU on the system is -- * assigned several values that are constant throughout its runtime: -- * -- * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling -- * decisions. Paired CPUs always schedule tasks from the same -- * CPU cgroup, and synchronize with each other to guarantee -- * that this constraint is not violated. -- * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access -- * a struct pair_ctx object that is shared between the pair. -- * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the -- * pair. Each struct pair_ctx has an active_mask field, -- * which is a bitmap used to indicate whether each core -- * in the pair currently has an actively running task. -- * This index specifies which entry in the bitmap corresponds -- * to each CPU in the pair. -- * -- * During this initialization, the CPUs are paired according to a "stride" that -- * may be specified when invoking the user space program that initializes and -- * loads the scheduler. By default, the stride is 1/2 the total number of CPUs. -- * -- * Tasks and cgroups -- * ----------------- -- * -- * Every cgroup in the system is registered with the scheduler using the -- * pair_cgroup_init() callback, and every task in the system is associated with -- * exactly one cgroup. At a high level, the idea with the pair scheduler is to -- * always schedule tasks from the same cgroup within a given CPU pair. When a -- * task is enqueued (i.e. passed to the pair_enqueue() callback function), its -- * cgroup ID is read from its task struct, and then a corresponding queue map -- * is used to FIFO-enqueue the task for that cgroup. -- * -- * If you look through the implementation of the scheduler, you'll notice that -- * there is quite a bit of complexity involved with looking up the per-cgroup -- * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash -- * BPF hash map that is used to map a cgroup ID to a globally unique ID that's -- * allocated in the BPF program. This is done because we use separate maps to -- * store the FIFO queue of tasks, and the length of that map, per cgroup. This -- * complexity is only present because of current deficiencies in BPF that will -- * soon be addressed. The main point to keep in mind is that newly enqueued -- * tasks are added to their cgroup's FIFO queue. -- * -- * Dispatching tasks -- * ----------------- -- * -- * This section will describe how enqueued tasks are dispatched and scheduled. -- * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is -- * as follows: -- * -- * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is -- * the structure that's used to synchronize amongst the two pair CPUs in their -- * scheduling decisions. After any of the following events have occurred: -- * -- * - The cgroup's slice run has expired, or -- * - The cgroup becomes empty, or -- * - Either CPU in the pair is preempted by a higher priority scheduling class -- * -- * The cgroup transitions to the draining state and stops executing new tasks -- * from the cgroup. -- * -- * 2. If the pair is still executing a task, mark the pair_ctx as draining, and -- * wait for the pair CPU to be preempted. -- * -- * 3. Otherwise, if the pair CPU is not running a task, we can move onto -- * scheduling new tasks. Pop the next cgroup id from the top_q queue. -- * -- * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it. -- * -- * Note again that this scheduling behavior is simple, but the implementation -- * is complex mostly because this it hits several BPF shortcomings and has to -- * work around in often awkward ways. Most of the shortcomings are expected to -- * be resolved in the near future which should allow greatly simplifying this -- * scheduler. -- * -- * Dealing with preemption -- * ----------------------- -- * -- * SCX is the lowest priority sched_class, and could be preempted by them at -- * any time. To address this, the scheduler implements pair_cpu_release() and -- * pair_cpu_acquire() callbacks which are invoked by the core scheduler when -- * the scheduler loses and gains control of the CPU respectively. -- * -- * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and -- * then invoke: -- * -- * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- * -- * This preempts the pair CPU, and waits until it has re-entered the scheduler -- * before returning. This is necessary to ensure that the higher priority -- * sched_class that preempted our scheduler does not schedule a task -- * concurrently with our pair CPU. -- * -- * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption -- * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable -- * pair scheduling. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include "scx_example_pair.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; -- --/* !0 for veristat, set during init */ --const volatile u32 nr_cpu_ids = 64; -- --/* a pair of CPUs stay on a cgroup for this duration */ --const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL; -- --/* cpu ID -> pair cpu ID */ --const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 }; -- --/* cpu ID -> pair_id */ --const volatile u32 pair_id[MAX_CPUS]; -- --/* CPU ID -> CPU # in the pair (0 or 1) */ --const volatile u32 in_pair_idx[MAX_CPUS]; -- --struct pair_ctx { -- struct bpf_spin_lock lock; -- -- /* the cgroup the pair is currently executing */ -- u64 cgid; -- -- /* the pair started executing the current cgroup at */ -- u64 started_at; -- -- /* whether the current cgroup is draining */ -- bool draining; -- -- /* the CPUs that are currently active on the cgroup */ -- u32 active_mask; -- -- /* -- * the CPUs that are currently preempted and running tasks in a -- * different scheduler. -- */ -- u32 preempted_mask; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, MAX_CPUS / 2); -- __type(key, u32); -- __type(value, struct pair_ctx); --} pair_ctx SEC(".maps"); -- --/* queue of cgrp_q's possibly with tasks on them */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- /* -- * Because it's difficult to build strong synchronization encompassing -- * multiple non-trivial operations in BPF, this queue is managed in an -- * opportunistic way so that we guarantee that a cgroup w/ active tasks -- * is always on it but possibly multiple times. Once we have more robust -- * synchronization constructs and e.g. linked list, we should be able to -- * do this in a prettier way but for now just size it big enough. -- */ -- __uint(max_entries, 4 * MAX_CGRPS); -- __type(value, u64); --} top_q SEC(".maps"); -- --/* per-cgroup q which FIFOs the tasks from the cgroup */ --struct cgrp_q { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, MAX_QUEUED); -- __type(value, u32); --}; -- --/* -- * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local -- * storage; however, a cgroup local storage can only be accessed from the BPF -- * progs attached to the cgroup. For now, work around by allocating array of -- * cgrp_q's and then allocating per-cgroup indices. -- * -- * Another caveat: It's difficult to populate a large array of maps statically -- * or from BPF. Initialize it from userland. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, MAX_CGRPS); -- __type(key, s32); -- __array(values, struct cgrp_q); --} cgrp_q_arr SEC(".maps"); -- --static u64 cgrp_q_len[MAX_CGRPS]; -- --/* -- * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be -- * useful to have as a map type. -- */ --static u32 cgrp_q_idx_cursor; --static u64 cgrp_q_idx_busy[MAX_CGRPS]; -- --/* -- * All added up, the following is what we do: -- * -- * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking -- * for a free ID. If not found, fail cgroup creation with -EBUSY. -- * -- * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following -- * cgrp_q_idx_hash. -- * -- * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from -- * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr. -- * -- * This is sadly complicated for something pretty simple. Hopefully, we should -- * be able to simplify in the future. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, MAX_CGRPS); -- __uint(key_size, sizeof(u64)); /* cgrp ID */ -- __uint(value_size, sizeof(s32)); /* cgrp_q idx */ --} cgrp_q_idx_hash SEC(".maps"); -- --/* statistics */ --u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions; --u64 nr_exps, nr_exp_waits, nr_exp_empty; --u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty; -- --struct user_exit_info uei; -- --static bool time_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- struct cgrp_q *cgq; -- s32 pid = p->pid; -- u64 cgid; -- u32 *q_idx; -- u64 *cgq_len; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- cgrp = scx_bpf_task_cgroup(p); -- cgid = cgrp->kn->id; -- bpf_cgroup_release(cgrp); -- -- /* find the cgroup's q and push @p into it */ -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!q_idx) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return; -- } -- -- cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx); -- if (!cgq) { -- scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]", -- cgid, *q_idx); -- return; -- } -- -- if (bpf_map_push_elem(cgq, &pid, 0)) { -- scx_bpf_error("cgroup[%llu] queue overflow", cgid); -- return; -- } -- -- /* bump q len, if going 0 -> 1, queue cgroup into the top_q */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len) { -- scx_bpf_error("MEMBER_VTPR malfunction"); -- return; -- } -- -- if (!__sync_fetch_and_add(cgq_len, 1) && -- bpf_map_push_elem(&top_q, &cgid, 0)) { -- scx_bpf_error("top_q overflow"); -- return; -- } --} -- --static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) --{ -- u32 *vptr; -- -- vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *pairc = bpf_map_lookup_elem(&pair_ctx, vptr); -- if (!(*pairc)) -- return -EINVAL; -- -- vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *mask = 1U << *vptr; -- -- return 0; --} -- --static int try_dispatch(s32 cpu) --{ -- struct pair_ctx *pairc; -- struct bpf_map *cgq_map; -- struct task_struct *p; -- u64 now = bpf_ktime_get_ns(); -- bool kick_pair = false; -- bool expired, pair_preempted; -- u32 *vptr, in_pair_mask; -- s32 pid, q_idx; -- u64 cgid; -- int ret; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) { -- scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]", -- cpu); -- return -ENOENT; -- } -- -- bpf_spin_lock(&pairc->lock); -- pairc->active_mask &= ~in_pair_mask; -- -- expired = time_before(pairc->started_at + pair_batch_dur_ns, now); -- if (expired || pairc->draining) { -- u64 new_cgid = 0; -- -- __sync_fetch_and_add(&nr_exps, 1); -- -- /* -- * We're done with the current cgid. An obvious optimization -- * would be not draining if the next cgroup is the current one. -- * For now, be dumb and always expire. -- */ -- pairc->draining = true; -- -- pair_preempted = pairc->preempted_mask; -- if (pairc->active_mask || pair_preempted) { -- /* -- * The other CPU is still active, or is no longer under -- * our control due to e.g. being preempted by a higher -- * priority sched_class. We want to wait until this -- * cgroup expires, or until control of our pair CPU has -- * been returned to us. -- * -- * If the pair controls its CPU, and the time already -- * expired, kick. When the other CPU arrives at -- * dispatch and clears its active mask, it'll push the -- * pair to the next cgroup and kick this CPU. -- */ -- __sync_fetch_and_add(&nr_exp_waits, 1); -- bpf_spin_unlock(&pairc->lock); -- if (expired && !pair_preempted) -- kick_pair = true; -- goto out_maybe_kick; -- } -- -- bpf_spin_unlock(&pairc->lock); -- -- /* -- * Pick the next cgroup. It'd be easier / cleaner to not drop -- * pairc->lock and use stronger synchronization here especially -- * given that we'll be switching cgroups significantly less -- * frequently than tasks. Unfortunately, bpf_spin_lock can't -- * really protect anything non-trivial. Let's do opportunistic -- * operations instead. -- */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u32 *q_idx; -- u64 *cgq_len; -- -- if (bpf_map_pop_elem(&top_q, &new_cgid)) { -- /* no active cgroup, go idle */ -- __sync_fetch_and_add(&nr_exp_empty, 1); -- return 0; -- } -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid); -- if (!q_idx) -- continue; -- -- /* -- * This is the only place where empty cgroups are taken -- * off the top_q. -- */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len || !*cgq_len) -- continue; -- -- /* -- * If it has any tasks, requeue as we may race and not -- * execute it. -- */ -- bpf_map_push_elem(&top_q, &new_cgid, 0); -- break; -- } -- -- bpf_spin_lock(&pairc->lock); -- -- /* -- * The other CPU may already have started on a new cgroup while -- * we dropped the lock. Make sure that we're still draining and -- * start on the new cgroup. -- */ -- if (pairc->draining && !pairc->active_mask) { -- __sync_fetch_and_add(&nr_cgrp_next, 1); -- pairc->cgid = new_cgid; -- pairc->started_at = now; -- pairc->draining = false; -- kick_pair = true; -- } else { -- __sync_fetch_and_add(&nr_cgrp_coll, 1); -- } -- } -- -- cgid = pairc->cgid; -- pairc->active_mask |= in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- -- /* again, it'd be better to do all these with the lock held, oh well */ -- vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!vptr) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return -ENOENT; -- } -- q_idx = *vptr; -- -- /* claim one task from cgrp_q w/ q_idx */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u64 *cgq_len, len; -- -- cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]); -- if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) { -- /* the cgroup must be empty, expire and repeat */ -- __sync_fetch_and_add(&nr_cgrp_empty, 1); -- bpf_spin_lock(&pairc->lock); -- pairc->draining = true; -- pairc->active_mask &= ~in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- return -EAGAIN; -- } -- -- if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len) -- continue; -- -- break; -- } -- -- cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx); -- if (!cgq_map) { -- scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- if (bpf_map_pop_elem(cgq_map, &pid)) { -- scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- p = bpf_task_from_pid(pid); -- if (p) { -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } else { -- /* we don't handle dequeues, retry on lost tasks */ -- __sync_fetch_and_add(&nr_missing, 1); -- return -EAGAIN; -- } -- --out_maybe_kick: -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } -- return 0; --} -- --void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) --{ -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_dispatch(cpu) != -EAGAIN) -- break; -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask &= ~in_pair_mask; -- /* Kick the pair CPU, unless it was also preempted. */ -- kick_pair = !pairc->preempted_mask; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask |= in_pair_mask; -- pairc->active_mask &= ~in_pair_mask; -- /* Kick the pair CPU if it's still running. */ -- kick_pair = pairc->active_mask; -- pairc->draining = true; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- } -- } -- __sync_fetch_and_add(&nr_preemptions, 1); --} -- --s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 i, q_idx; -- -- bpf_for(i, 0, MAX_CGRPS) { -- q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS; -- if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1)) -- break; -- } -- if (i == MAX_CGRPS) -- return -EBUSY; -- -- if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]); -- if (busy) -- *busy = 0; -- return -EBUSY; -- } -- -- return 0; --} -- --void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 *q_idx; -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (q_idx) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]); -- if (busy) -- *busy = 0; -- bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid); -- } --} -- --s32 BPF_STRUCT_OPS(pair_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops pair_ops = { -- .enqueue = (void *)pair_enqueue, -- .dispatch = (void *)pair_dispatch, -- .cpu_acquire = (void *)pair_cpu_acquire, -- .cpu_release = (void *)pair_cpu_release, -- .cgroup_init = (void *)pair_cgroup_init, -- .cgroup_exit = (void *)pair_cgroup_exit, -- .init = (void *)pair_init, -- .exit = (void *)pair_exit, -- .name = "pair", --}; -diff --git a/tools/sched_ext/scx_example_pair.c b/tools/sched_ext/scx_example_pair.c -deleted file mode 100644 -index 18e032bbc173..000000000000 ---- a/tools/sched_ext/scx_example_pair.c -+++ /dev/null -@@ -1,143 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_pair.h" --#include "scx_example_pair.skel.h" -- --const char help_fmt[] = --"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n" --"execute from the same CPU cgroup.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-S STRIDE] [-p]\n" --"\n" --" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_pair *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 stride, i, opt, outer_fd; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_pair__open(); -- assert(skel); -- -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- /* pair up the earlier half to the latter by default, override with -s */ -- stride = skel->rodata->nr_cpu_ids / 2; -- -- while ((opt = getopt(argc, argv, "S:ph")) != -1) { -- switch (opt) { -- case 'S': -- stride = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- for (i = 0; i < skel->rodata->nr_cpu_ids; i++) { -- if (skel->rodata->pair_cpu[i] < 0) { -- skel->rodata->pair_cpu[i] = i + stride; -- skel->rodata->pair_cpu[i + stride] = i; -- skel->rodata->pair_id[i] = i; -- skel->rodata->pair_id[i + stride] = i; -- skel->rodata->in_pair_idx[i] = 0; -- skel->rodata->in_pair_idx[i + stride] = 1; -- } -- } -- -- assert(!scx_example_pair__load(skel)); -- -- /* -- * Populate the cgrp_q_arr map which is an array containing per-cgroup -- * queues. It'd probably be better to do this from BPF but there are too -- * many to initialize statically and there's no way to dynamically -- * populate from BPF. -- */ -- outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr); -- assert(outer_fd >= 0); -- -- printf("Initializing"); -- for (i = 0; i < MAX_CGRPS; i++) { -- s32 inner_fd; -- -- if (exit_req) -- break; -- -- inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, -- sizeof(u32), MAX_QUEUED, NULL); -- assert(inner_fd >= 0); -- assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY)); -- close(inner_fd); -- -- if (!(i % 10)) -- printf("."); -- fflush(stdout); -- } -- printf("\n"); -- -- /* -- * Fully initialized, attach and run. -- */ -- link = bpf_map__attach_struct_ops(skel->maps.pair_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf(" total:%10lu dispatch:%10lu missing:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_dispatched, -- skel->bss->nr_missing); -- printf(" kicks:%10lu preemptions:%7lu\n", -- skel->bss->nr_kicks, -- skel->bss->nr_preemptions); -- printf(" exp:%10lu exp_wait:%10lu exp_empty:%10lu\n", -- skel->bss->nr_exps, -- skel->bss->nr_exp_waits, -- skel->bss->nr_exp_empty); -- printf("cgnext:%10lu cgcoll:%10lu cgempty:%10lu\n", -- skel->bss->nr_cgrp_next, -- skel->bss->nr_cgrp_coll, -- skel->bss->nr_cgrp_empty); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_pair__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_pair.h b/tools/sched_ext/scx_example_pair.h -deleted file mode 100644 -index f60b824272f7..000000000000 ---- a/tools/sched_ext/scx_example_pair.h -+++ /dev/null -@@ -1,10 +0,0 @@ --#ifndef __SCX_EXAMPLE_PAIR_H --#define __SCX_EXAMPLE_PAIR_H -- --enum { -- MAX_CPUS = 4096, -- MAX_QUEUED = 4096, -- MAX_CGRPS = 4096, --}; -- --#endif /* __SCX_EXAMPLE_PAIR_H */ -diff --git a/tools/sched_ext/scx_example_qmap.bpf.c b/tools/sched_ext/scx_example_qmap.bpf.c -deleted file mode 100644 -index 579ab21ae403..000000000000 ---- a/tools/sched_ext/scx_example_qmap.bpf.c -+++ /dev/null -@@ -1,401 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple five-level FIFO queue scheduler. -- * -- * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets -- * assigned to one depending on its compound weight. Each CPU round robins -- * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from -- * queue0, 2 from queue1, 4 from queue2 and so on. -- * -- * This scheduler demonstrates: -- * -- * - BPF-side queueing using PIDs. -- * - Sleepable per-task storage allocation using ops.prep_enable(). -- * - Using ops.cpu_release() to handle a higher priority scheduling class taking -- * the CPU away. -- * - Core-sched support. -- * -- * This scheduler is primarily for demonstration and testing of sched_ext -- * features and unlikely to be useful for actual workloads. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include -- --char _license[] SEC("license") = "GPL"; -- --const volatile u64 slice_ns = SCX_SLICE_DFL; --const volatile bool switch_partial; --const volatile u32 stall_user_nth; --const volatile u32 stall_kernel_nth; --const volatile u32 dsp_inf_loop_after; --const volatile s32 disallow_tgid; -- --u32 test_error_cnt; -- --struct user_exit_info uei; -- --struct qmap { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, u32); --} queue0 SEC(".maps"), -- queue1 SEC(".maps"), -- queue2 SEC(".maps"), -- queue3 SEC(".maps"), -- queue4 SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, 5); -- __type(key, int); -- __array(values, struct qmap); --} queue_arr SEC(".maps") = { -- .values = { -- [0] = &queue0, -- [1] = &queue1, -- [2] = &queue2, -- [3] = &queue3, -- [4] = &queue4, -- }, --}; -- --/* -- * Per-queue sequence numbers to implement core-sched ordering. -- * -- * Tail seq is assigned to each queued task and incremented. Head seq tracks the -- * sequence number of the latest dispatched task. The distance between the a -- * task's seq and the associated queue's head seq is called the queue distance -- * and used when comparing two tasks for ordering. See qmap_core_sched_before(). -- */ --static u64 core_sched_head_seqs[5]; --static u64 core_sched_tail_seqs[5]; -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local_dsq */ -- u64 core_sched_seq; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --/* Per-cpu dispatch index and remaining count */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(max_entries, 2); -- __type(key, u32); -- __type(value, u64); --} dispatch_idx_cnt SEC(".maps"); -- --/* Statistics */ --unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued; --unsigned long nr_core_sched_execed; -- --s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- struct task_ctx *tctx; -- s32 cpu; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- return cpu; -- -- return prev_cpu; --} -- --static int weight_to_idx(u32 weight) --{ -- /* Coarsely map the compound weight to a FIFO. */ -- if (weight <= 25) -- return 0; -- else if (weight <= 50) -- return 1; -- else if (weight < 200) -- return 2; -- else if (weight < 400) -- return 3; -- else -- return 4; --} -- --void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) --{ -- static u32 user_cnt, kernel_cnt; -- struct task_ctx *tctx; -- u32 pid = p->pid; -- int idx = weight_to_idx(p->scx.weight); -- void *ring; -- -- if (p->flags & PF_KTHREAD) { -- if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth)) -- return; -- } else { -- if (stall_user_nth && !(++user_cnt % stall_user_nth)) -- return; -- } -- -- if (test_error_cnt && !--test_error_cnt) -- scx_bpf_error("test triggering error"); -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * All enqueued tasks must have their core_sched_seq updated for correct -- * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in -- * qmap_ops.flags. -- */ -- tctx->core_sched_seq = core_sched_tail_seqs[idx]++; -- -- /* -- * If qmap_select_cpu() is telling us to or this is the last runnable -- * task on the CPU, enqueue locally. -- */ -- if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) { -- tctx->force_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); -- return; -- } -- -- /* -- * If the task was re-enqueued due to the CPU being preempted by a -- * higher priority scheduling class, just re-enqueue the task directly -- * on the global DSQ. As we want another CPU to pick it up, find and -- * kick an idle CPU. -- */ -- if (enq_flags & SCX_ENQ_REENQ) { -- s32 cpu; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags); -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- return; -- } -- -- ring = bpf_map_lookup_elem(&queue_arr, &idx); -- if (!ring) { -- scx_bpf_error("failed to find ring %d", idx); -- return; -- } -- -- /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */ -- if (bpf_map_push_elem(ring, &pid, 0)) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_enqueued, 1); --} -- --/* -- * The BPF queue map doesn't support removal and sched_ext can handle spurious -- * dispatches. qmap_dequeue() is only used to collect statistics. -- */ --void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags) --{ -- __sync_fetch_and_add(&nr_dequeued, 1); -- if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) -- __sync_fetch_and_add(&nr_core_sched_execed, 1); --} -- --static void update_core_sched_head_seq(struct task_struct *p) --{ -- struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- int idx = weight_to_idx(p->scx.weight); -- -- if (tctx) -- core_sched_head_seqs[idx] = tctx->core_sched_seq; -- else -- scx_bpf_error("task_ctx lookup failed"); --} -- --void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 zero = 0, one = 1; -- u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero); -- u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one); -- void *fifo; -- s32 pid; -- int i; -- -- if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) { -- struct task_struct *p; -- -- /* -- * PID 2 should be kthreadd which should mostly be idle and off -- * the scheduler. Let's keep dispatching it to force the kernel -- * to call this function over and over again. -- */ -- p = bpf_task_from_pid(2); -- if (p) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- if (!idx || !cnt) { -- scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt); -- return; -- } -- -- for (i = 0; i < 5; i++) { -- /* Advance the dispatch cursor and pick the fifo. */ -- if (!*cnt) { -- *idx = (*idx + 1) % 5; -- *cnt = 1 << *idx; -- } -- (*cnt)--; -- -- fifo = bpf_map_lookup_elem(&queue_arr, idx); -- if (!fifo) { -- scx_bpf_error("failed to find ring %llu", *idx); -- return; -- } -- -- /* Dispatch or advance. */ -- if (!bpf_map_pop_elem(fifo, &pid)) { -- struct task_struct *p; -- -- p = bpf_task_from_pid(pid); -- if (p) { -- update_core_sched_head_seq(p); -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- *cnt = 0; -- } --} -- --/* -- * The distance from the head of the queue scaled by the weight of the queue. -- * The lower the number, the older the task and the higher the priority. -- */ --static s64 task_qdist(struct task_struct *p) --{ -- int idx = weight_to_idx(p->scx.weight); -- struct task_ctx *tctx; -- s64 qdist; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return 0; -- } -- -- qdist = tctx->core_sched_seq - core_sched_head_seqs[idx]; -- -- /* -- * As queue index increments, the priority doubles. The queue w/ index 3 -- * is dispatched twice more frequently than 2. Reflect the difference by -- * scaling qdists accordingly. Note that the shift amount needs to be -- * flipped depending on the sign to avoid flipping priority direction. -- */ -- if (qdist >= 0) -- return qdist << (4 - idx); -- else -- return qdist << idx; --} -- --/* -- * This is called to determine the task ordering when core-sched is picking -- * tasks to execute on SMT siblings and should encode about the same ordering as -- * the regular scheduling path. Use the priority-scaled distances from the head -- * of the queues to compare the two tasks which should be consistent with the -- * dispatch path behavior. -- */ --bool BPF_STRUCT_OPS(qmap_core_sched_before, -- struct task_struct *a, struct task_struct *b) --{ -- return task_qdist(a) > task_qdist(b); --} -- --void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- u32 cnt; -- -- /* -- * Called when @cpu is taken by a higher priority scheduling class. This -- * makes @cpu no longer available for executing sched_ext tasks. As we -- * don't want the tasks in @cpu's local dsq to sit there until @cpu -- * becomes available again, re-enqueue them into the global dsq. See -- * %SCX_ENQ_REENQ handling in qmap_enqueue(). -- */ -- cnt = scx_bpf_reenqueue_local(); -- if (cnt) -- __sync_fetch_and_add(&nr_reenqueued, cnt); --} -- --s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (p->tgid == disallow_tgid) -- p->scx.disallow = true; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(qmap_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops qmap_ops = { -- .select_cpu = (void *)qmap_select_cpu, -- .enqueue = (void *)qmap_enqueue, -- .dequeue = (void *)qmap_dequeue, -- .dispatch = (void *)qmap_dispatch, -- .core_sched_before = (void *)qmap_core_sched_before, -- .cpu_release = (void *)qmap_cpu_release, -- .prep_enable = (void *)qmap_prep_enable, -- .init = (void *)qmap_init, -- .exit = (void *)qmap_exit, -- .flags = SCX_OPS_ENQ_LAST, -- .timeout_ms = 5000U, -- .name = "qmap", --}; -diff --git a/tools/sched_ext/scx_example_qmap.c b/tools/sched_ext/scx_example_qmap.c -deleted file mode 100644 -index ccb4814ee61b..000000000000 ---- a/tools/sched_ext/scx_example_qmap.c -+++ /dev/null -@@ -1,107 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_qmap.skel.h" -- --const char help_fmt[] = --"A simple five-level FIFO queue sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" --" -t COUNT Stall every COUNT'th user thread\n" --" -T COUNT Stall every COUNT'th kernel thread\n" --" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n" --" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_qmap *skel; -- struct bpf_link *link; -- int opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_qmap__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) { -- switch (opt) { -- case 's': -- skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; -- break; -- case 'e': -- skel->bss->test_error_cnt = strtoul(optarg, NULL, 0); -- break; -- case 't': -- skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0); -- break; -- case 'T': -- skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0); -- break; -- case 'l': -- skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0); -- break; -- case 'd': -- skel->rodata->disallow_tgid = strtol(optarg, NULL, 0); -- if (skel->rodata->disallow_tgid < 0) -- skel->rodata->disallow_tgid = getpid(); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_qmap__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.qmap_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- long nr_enqueued = skel->bss->nr_enqueued; -- long nr_dispatched = skel->bss->nr_dispatched; -- -- printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n", -- nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, -- skel->bss->nr_reenqueued, skel->bss->nr_dequeued, -- skel->bss->nr_core_sched_execed); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_qmap__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_simple.bpf.c b/tools/sched_ext/scx_example_simple.bpf.c -deleted file mode 100644 -index 4bccca3e2047..000000000000 ---- a/tools/sched_ext/scx_example_simple.bpf.c -+++ /dev/null -@@ -1,128 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple scheduler. -- * -- * By default, it operates as a simple global weighted vtime scheduler and can -- * be switched to FIFO scheduling. It also demonstrates the following niceties. -- * -- * - Statistics tracking how many tasks are queued to local and global dsq's. -- * - Termination notification for userspace. -- * -- * While very simple, this scheduler should work reasonably well on CPUs with a -- * uniform L3 cache topology. While preemption is not implemented, the fact that -- * the scheduling queue is shared across all CPUs means that whatever is at the -- * front of the queue is likely to be executed fairly quickly given enough -- * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads -- * but comes with the usual problems with FIFO scheduling where saturating -- * threads can easily drown out interactive ones. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --static u64 vtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, 2); /* [local, global] */ --} stats SEC(".maps"); -- --static void stat_inc(u32 idx) --{ -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); -- if (cnt_p) -- (*cnt_p)++; --} -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) --{ -- /* -- * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that -- * running @p on its CPU directly shouldn't affect fairness. Just queue -- * it on the local FIFO. -- */ -- if (enq_flags & SCX_ENQ_LOCAL) { -- stat_inc(0); /* count local queueing */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- return; -- } -- -- stat_inc(1); /* count global queueing */ -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) -- vtime = vtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --void BPF_STRUCT_OPS(simple_running, struct task_struct *p) --{ -- if (fifo_sched) -- return; -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(vtime_now, p->scx.dsq_vtime)) -- vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --s32 BPF_STRUCT_OPS(simple_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .running = (void *)simple_running, -- .stopping = (void *)simple_stopping, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", --}; -diff --git a/tools/sched_ext/scx_example_simple.c b/tools/sched_ext/scx_example_simple.c -deleted file mode 100644 -index 486b401f7c95..000000000000 ---- a/tools/sched_ext/scx_example_simple.c -+++ /dev/null -@@ -1,101 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_simple.skel.h" -- --const char help_fmt[] = --"A simple sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-f] [-p]\n" --"\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int simple) --{ -- exit_req = 1; --} -- --static void read_stats(struct scx_example_simple *skel, u64 *stats) --{ -- int nr_cpus = libbpf_num_possible_cpus(); -- u64 cnts[2][nr_cpus]; -- u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * 2); -- -- for (idx = 0; idx < 2; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_simple *skel; -- struct bpf_link *link; -- u32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_simple__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "fph")) != -1) { -- switch (opt) { -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_simple__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.simple_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- u64 stats[2]; -- -- read_stats(skel, stats); -- printf("local=%lu global=%lu\n", stats[0], stats[1]); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_simple__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_userland.bpf.c b/tools/sched_ext/scx_example_userland.bpf.c -deleted file mode 100644 -index a089bc6bbe86..000000000000 ---- a/tools/sched_ext/scx_example_userland.bpf.c -+++ /dev/null -@@ -1,269 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A minimal userland scheduler. -- * -- * In terms of scheduling, this provides two different types of behaviors: -- * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity. -- * All such tasks are direct-dispatched from the kernel, and are never -- * enqueued in user space. -- * 2. A primitive vruntime scheduler that is implemented in user space, for all -- * other tasks. -- * -- * Some parts of this example user space scheduler could be implemented more -- * efficiently using more complex and sophisticated data structures. For -- * example, rather than using BPF_MAP_TYPE_QUEUE's, -- * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between -- * user space and kernel space. Similarly, we use a simple vruntime-sorted list -- * in user space, but an rbtree could be used instead. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include --#include "scx_common.bpf.h" --#include "scx_example_userland_common.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; --const volatile s32 usersched_pid; -- --/* !0 for veristat, set during init */ --const volatile u32 num_possible_cpus = 64; -- --/* Stats that are printed by user space. */ --u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues; -- --struct user_exit_info uei; -- --/* -- * Whether the user space scheduler needs to be scheduled due to a task being -- * enqueued in user space. -- */ --static bool usersched_needed; -- --/* -- * The map containing tasks that are enqueued in user space from the kernel. -- * -- * This map is drained by the user space scheduler. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, struct scx_userland_enqueued_task); --} enqueued SEC(".maps"); -- --/* -- * The map containing tasks that are dispatched to the kernel from user space. -- * -- * Drained by the kernel in userland_dispatch(). -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, s32); --} dispatched SEC(".maps"); -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local DSQ */ --}; -- --/* Map that contains task-local storage. */ --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --static bool is_usersched_task(const struct task_struct *p) --{ -- return p->pid == usersched_pid; --} -- --static bool keep_in_kernel(const struct task_struct *p) --{ -- return p->nr_cpus_allowed < num_possible_cpus; --} -- --static struct task_struct *usersched_task(void) --{ -- struct task_struct *p; -- -- p = bpf_task_from_pid(usersched_pid); -- /* -- * Should never happen -- the usersched task should always be managed -- * by sched_ext. -- */ -- if (!p) { -- scx_bpf_error("Failed to find usersched task %d", usersched_pid); -- /* -- * We should never hit this path, and we error out of the -- * scheduler above just in case, so the scheduler will soon be -- * be evicted regardless. So as to simplify the logic in the -- * caller to not have to check for NULL, return an acquired -- * reference to the current task here rather than NULL. -- */ -- return bpf_task_acquire(bpf_get_current_task_btf()); -- } -- -- return p; --} -- --s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- if (keep_in_kernel(p)) { -- s32 cpu; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to look up task-local storage for %s", p->comm); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- tctx->force_local = true; -- return cpu; -- } -- } -- -- return prev_cpu; --} -- --static void dispatch_user_scheduler(void) --{ -- struct task_struct *p; -- -- usersched_needed = false; -- p = usersched_task(); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); --} -- --static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags) --{ -- struct scx_userland_enqueued_task task; -- -- memset(&task, 0, sizeof(task)); -- task.pid = p->pid; -- task.sum_exec_runtime = p->se.sum_exec_runtime; -- task.weight = p->scx.weight; -- -- if (bpf_map_push_elem(&enqueued, &task, 0)) { -- /* -- * If we fail to enqueue the task in user space, put it -- * directly on the global DSQ. -- */ -- __sync_fetch_and_add(&nr_failed_enqueues, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- __sync_fetch_and_add(&nr_user_enqueues, 1); -- usersched_needed = true; -- } --} -- --void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags) --{ -- if (keep_in_kernel(p)) { -- u64 dsq_id = SCX_DSQ_GLOBAL; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to lookup task ctx for %s", p->comm); -- return; -- } -- -- if (tctx->force_local) -- dsq_id = SCX_DSQ_LOCAL; -- tctx->force_local = false; -- scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags); -- __sync_fetch_and_add(&nr_kernel_enqueues, 1); -- return; -- } else if (!is_usersched_task(p)) { -- enqueue_task_in_user_space(p, enq_flags); -- } --} -- --void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (usersched_needed) -- dispatch_user_scheduler(); -- -- bpf_repeat(4096) { -- s32 pid; -- struct task_struct *p; -- -- if (bpf_map_pop_elem(&dispatched, &pid)) -- break; -- -- /* -- * The task could have exited by the time we get around to -- * dispatching it. Treat this as a normal occurrence, and simply -- * move onto the next iteration. -- */ -- p = bpf_task_from_pid(pid); -- if (!p) -- continue; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } --} -- --s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(userland_init) --{ -- if (num_possible_cpus == 0) { -- scx_bpf_error("User scheduler # CPUs uninitialized (%d)", -- num_possible_cpus); -- return -EINVAL; -- } -- -- if (usersched_pid <= 0) { -- scx_bpf_error("User scheduler pid uninitialized (%d)", -- usersched_pid); -- return -EINVAL; -- } -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops userland_ops = { -- .select_cpu = (void *)userland_select_cpu, -- .enqueue = (void *)userland_enqueue, -- .dispatch = (void *)userland_dispatch, -- .prep_enable = (void *)userland_prep_enable, -- .init = (void *)userland_init, -- .exit = (void *)userland_exit, -- .timeout_ms = 3000, -- .name = "userland", --}; -diff --git a/tools/sched_ext/scx_example_userland.c b/tools/sched_ext/scx_example_userland.c -deleted file mode 100644 -index 4152b1e65fe1..000000000000 ---- a/tools/sched_ext/scx_example_userland.c -+++ /dev/null -@@ -1,402 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext user space scheduler which provides vruntime semantics -- * using a simple ordered-list implementation. -- * -- * Each CPU in the system resides in a single, global domain. This precludes -- * the need to do any load balancing between domains. The scheduler could -- * easily be extended to support multiple domains, with load balancing -- * happening in user space. -- * -- * Any task which has any CPU affinity is scheduled entirely in BPF. This -- * program only schedules tasks which may run on any CPU. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include "user_exit_info.h" --#include "scx_example_userland_common.h" --#include "scx_example_userland.skel.h" -- --const char help_fmt[] = --"A minimal userland sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-b BATCH] [-p]\n" --"\n" --" -b BATCH The number of tasks to batch when dispatching (default: 8)\n" --" -p Don't switch all, switch only tasks on SCHED_EXT policy\n" --" -h Display this help and exit\n"; -- --/* Defined in UAPI */ --#define SCHED_EXT 7 -- --/* Number of tasks to batch when dispatching to user space. */ --static __u32 batch_size = 8; -- --static volatile int exit_req; --static int enqueued_fd, dispatched_fd; -- --static struct scx_example_userland *skel; --static struct bpf_link *ops_link; -- --/* Stats collected in user space. */ --static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches; -- --/* The data structure containing tasks that are enqueued in user space. */ --struct enqueued_task { -- LIST_ENTRY(enqueued_task) entries; -- __u64 sum_exec_runtime; -- double vruntime; --}; -- --/* -- * Use a vruntime-sorted list to store tasks. This could easily be extended to -- * a more optimal data structure, such as an rbtree as is done in CFS. We -- * currently elect to use a sorted list to simplify the example for -- * illustrative purposes. -- */ --LIST_HEAD(listhead, enqueued_task); -- --/* -- * A vruntime-sorted list of tasks. The head of the list contains the task with -- * the lowest vruntime. That is, the task that has the "highest" claim to be -- * scheduled. -- */ --static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head); -- --/* -- * The statically allocated array of tasks. We use a statically allocated list -- * here to avoid having to allocate on the enqueue path, which could cause a -- * deadlock. A more substantive user space scheduler could e.g. provide a hook -- * for newly enabled tasks that are passed to the scheduler from the -- * .prep_enable() callback to allows the scheduler to allocate on safe paths. -- */ --struct enqueued_task tasks[USERLAND_MAX_TASKS]; -- --static double min_vruntime; -- --static void sigint_handler(int userland) --{ -- exit_req = 1; --} -- --static __u32 task_pid(const struct enqueued_task *task) --{ -- return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task); --} -- --static int dispatch_task(s32 pid) --{ -- int err; -- -- err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d\n", pid); -- exit_req = 1; -- } else { -- nr_vruntime_dispatches++; -- } -- -- return err; --} -- --static struct enqueued_task *get_enqueued_task(__s32 pid) --{ -- if (pid >= USERLAND_MAX_TASKS) -- return NULL; -- -- return &tasks[pid]; --} -- --static double calc_vruntime_delta(__u64 weight, __u64 delta) --{ -- double weight_f = (double)weight / 100.0; -- double delta_f = (double)delta; -- -- return delta_f / weight_f; --} -- --static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task) --{ -- __u64 delta; -- -- delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime; -- -- enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta); -- if (min_vruntime > enqueued->vruntime) -- enqueued->vruntime = min_vruntime; -- enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime; --} -- --static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task) --{ -- struct enqueued_task *curr, *enqueued, *prev; -- -- curr = get_enqueued_task(bpf_task->pid); -- if (!curr) -- return ENOENT; -- -- update_enqueued(curr, bpf_task); -- nr_vruntime_enqueues++; -- -- /* -- * Enqueue the task in a vruntime-sorted list. A more optimal data -- * structure such as an rbtree could easily be used as well. We elect -- * to use a list here simply because it's less code, and thus the -- * example is less convoluted and better serves to illustrate what a -- * user space scheduler could look like. -- */ -- -- if (LIST_EMPTY(&vruntime_head)) { -- LIST_INSERT_HEAD(&vruntime_head, curr, entries); -- return 0; -- } -- -- LIST_FOREACH(enqueued, &vruntime_head, entries) { -- if (curr->vruntime <= enqueued->vruntime) { -- LIST_INSERT_BEFORE(enqueued, curr, entries); -- return 0; -- } -- prev = enqueued; -- } -- -- LIST_INSERT_AFTER(prev, curr, entries); -- -- return 0; --} -- --static void drain_enqueued_map(void) --{ -- while (1) { -- struct scx_userland_enqueued_task task; -- int err; -- -- if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) -- return; -- -- err = vruntime_enqueue(&task); -- if (err) { -- fprintf(stderr, "Failed to enqueue task %d: %s\n", -- task.pid, strerror(err)); -- exit_req = 1; -- return; -- } -- } --} -- --static void dispatch_batch(void) --{ -- __u32 i; -- -- for (i = 0; i < batch_size; i++) { -- struct enqueued_task *task; -- int err; -- __s32 pid; -- -- task = LIST_FIRST(&vruntime_head); -- if (!task) -- return; -- -- min_vruntime = task->vruntime; -- pid = task_pid(task); -- LIST_REMOVE(task, entries); -- err = dispatch_task(pid); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d in %u\n", -- pid, i); -- return; -- } -- } --} -- --static void *run_stats_printer(void *arg) --{ -- while (!exit_req) { -- __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; -- -- nr_failed_enqueues = skel->bss->nr_failed_enqueues; -- nr_kernel_enqueues = skel->bss->nr_kernel_enqueues; -- nr_user_enqueues = skel->bss->nr_user_enqueues; -- total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues; -- -- printf("o-----------------------o\n"); -- printf("| BPF ENQUEUES |\n"); -- printf("|-----------------------|\n"); -- printf("| kern: %10llu |\n", nr_kernel_enqueues); -- printf("| user: %10llu |\n", nr_user_enqueues); -- printf("| failed: %10llu |\n", nr_failed_enqueues); -- printf("| -------------------- |\n"); -- printf("| total: %10llu |\n", total); -- printf("| |\n"); -- printf("|-----------------------|\n"); -- printf("| VRUNTIME / USER |\n"); -- printf("|-----------------------|\n"); -- printf("| enq: %10llu |\n", nr_vruntime_enqueues); -- printf("| disp: %10llu |\n", nr_vruntime_dispatches); -- printf("o-----------------------o\n"); -- printf("\n\n"); -- sleep(1); -- } -- -- return NULL; --} -- --static int spawn_stats_thread(void) --{ -- pthread_t stats_printer; -- -- return pthread_create(&stats_printer, NULL, run_stats_printer, NULL); --} -- --static int bootstrap(int argc, char **argv) --{ -- int err; -- __u32 opt; -- struct sched_param sched_param = { -- .sched_priority = sched_get_priority_max(SCHED_EXT), -- }; -- bool switch_partial = false; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- /* -- * Enforce that the user scheduler task is managed by sched_ext. The -- * task eagerly drains the list of enqueued tasks in its main work -- * loop, and then yields the CPU. The BPF scheduler only schedules the -- * user space scheduler task when at least one other task in the system -- * needs to be scheduled. -- */ -- err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param); -- if (err) { -- fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err)); -- return err; -- } -- -- while ((opt = getopt(argc, argv, "b:ph")) != -1) { -- switch (opt) { -- case 'b': -- batch_size = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- exit(opt != 'h'); -- } -- } -- -- /* -- * It's not always safe to allocate in a user space scheduler, as an -- * enqueued task could hold a lock that we require in order to be able -- * to allocate. -- */ -- err = mlockall(MCL_CURRENT | MCL_FUTURE); -- if (err) { -- fprintf(stderr, "Failed to prefault and lock address space: %s\n", -- strerror(err)); -- return err; -- } -- -- skel = scx_example_userland__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno)); -- return errno; -- } -- skel->rodata->num_possible_cpus = libbpf_num_possible_cpus(); -- assert(skel->rodata->num_possible_cpus > 0); -- skel->rodata->usersched_pid = getpid(); -- assert(skel->rodata->usersched_pid > 0); -- skel->rodata->switch_partial = switch_partial; -- -- err = scx_example_userland__load(skel); -- if (err) { -- fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- enqueued_fd = bpf_map__fd(skel->maps.enqueued); -- dispatched_fd = bpf_map__fd(skel->maps.dispatched); -- assert(enqueued_fd > 0); -- assert(dispatched_fd > 0); -- -- err = spawn_stats_thread(); -- if (err) { -- fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops); -- if (!ops_link) { -- fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno)); -- err = errno; -- goto destroy_skel; -- } -- -- return 0; -- --destroy_skel: -- scx_example_userland__destroy(skel); -- exit_req = 1; -- return err; --} -- --static void sched_main_loop(void) --{ -- while (!exit_req) { -- /* -- * Perform the following work in the main user space scheduler -- * loop: -- * -- * 1. Drain all tasks from the enqueued map, and enqueue them -- * to the vruntime sorted list. -- * -- * 2. Dispatch a batch of tasks from the vruntime sorted list -- * down to the kernel. -- * -- * 3. Yield the CPU back to the system. The BPF scheduler will -- * reschedule the user space scheduler once another task has -- * been enqueued to user space. -- */ -- drain_enqueued_map(); -- dispatch_batch(); -- sched_yield(); -- } --} -- --int main(int argc, char **argv) --{ -- int err; -- -- err = bootstrap(argc, argv); -- if (err) { -- fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err)); -- return err; -- } -- -- sched_main_loop(); -- -- exit_req = 1; -- bpf_link__destroy(ops_link); -- uei_print(&skel->bss->uei); -- scx_example_userland__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_userland_common.h b/tools/sched_ext/scx_example_userland_common.h -deleted file mode 100644 -index 639c6809c5ff..000000000000 ---- a/tools/sched_ext/scx_example_userland_common.h -+++ /dev/null -@@ -1,19 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* Copyright (c) 2022 Meta, Inc */ -- --#ifndef __SCX_USERLAND_COMMON_H --#define __SCX_USERLAND_COMMON_H -- --#define USERLAND_MAX_TASKS 8192 -- --/* -- * An instance of a task that has been enqueued by the kernel for consumption -- * by a user space global scheduler thread. -- */ --struct scx_userland_enqueued_task { -- __s32 pid; -- u64 sum_exec_runtime; -- u64 weight; --}; -- --#endif // __SCX_USERLAND_COMMON_H -diff --git a/tools/sched_ext/user_exit_info.h b/tools/sched_ext/user_exit_info.h -deleted file mode 100644 -index e701ef0e0b86..000000000000 ---- a/tools/sched_ext/user_exit_info.h -+++ /dev/null -@@ -1,50 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Define struct user_exit_info which is shared between BPF and userspace parts -- * to communicate exit status and other information. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __USER_EXIT_INFO_H --#define __USER_EXIT_INFO_H -- --struct user_exit_info { -- int type; -- char reason[128]; -- char msg[1024]; --}; -- --#ifdef __bpf__ -- --#include "vmlinux.h" --#include -- --static inline void uei_record(struct user_exit_info *uei, -- const struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason); -- bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg); -- /* use __sync to force memory barrier */ -- __sync_val_compare_and_swap(&uei->type, uei->type, ei->type); --} -- --#else /* !__bpf__ */ -- --static inline bool uei_exited(struct user_exit_info *uei) --{ -- /* use __sync to force memory barrier */ -- return __sync_val_compare_and_swap(&uei->type, -1, -1); --} -- --static inline void uei_print(const struct user_exit_info *uei) --{ -- fprintf(stderr, "EXIT: %s", uei->reason); -- if (uei->msg[0] != '\0') -- fprintf(stderr, " (%s)", uei->msg); -- fputs("\n", stderr); --} -- --#endif /* __bpf__ */ --#endif /* __USER_EXIT_INFO_H */ -diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c -index 3a7bd62ef6b7..e8c98ab86084 100644 ---- a/drivers/cpufreq/cpufreq.c -+++ b/drivers/cpufreq/cpufreq.c -@@ -810,7 +810,7 @@ static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy, - /* - * show_scaling_governor - show the current policy for the specified CPU - */ --static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) -+ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) - { - if (policy->policy == CPUFREQ_POLICY_POWERSAVE) - return sprintf(buf, "powersave\n"); -@@ -825,7 +825,7 @@ static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) - /* - * store_scaling_governor - store policy for the specified CPU - */ --static ssize_t store_scaling_governor(struct cpufreq_policy *policy, -+ssize_t store_scaling_governor(struct cpufreq_policy *policy, - const char *buf, size_t count) - { - char str_governor[16]; -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -index ce05928392bf..467061a1ffa4 100755 ---- a/kernel/sched/hmbird/hmbird.c -+++ b/kernel/sched/hmbird/hmbird.c -@@ -633,14 +633,14 @@ static void init_isolate_cpus(void) - WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -- cpumask_set_cpu(0, iso_masks.big); -- cpumask_set_cpu(1, iso_masks.big); -- cpumask_set_cpu(2, iso_masks.big); -- cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(0, iso_masks.little); -+ cpumask_set_cpu(1, iso_masks.little); -+ cpumask_set_cpu(2, iso_masks.little); -+ cpumask_set_cpu(3, iso_masks.little); - cpumask_set_cpu(4, iso_masks.big); -- cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(5, iso_masks.big); - cpumask_set_cpu(6, iso_masks.partial); -- cpumask_set_cpu(7, iso_masks.ex_free); -+ cpumask_set_cpu(7, iso_masks.exclusive); - } - - extern spinlock_t css_set_lock; -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index 692c9aa0ffa6..c536364f4736 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -10942,7 +10942,6 @@ void sched_move_task(struct task_struct *tsk) - { - int queued, running, queue_flags = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; -- struct task_group *group; - struct rq_flags rf; - struct rq *rq; - -@@ -10958,7 +10957,7 @@ void sched_move_task(struct task_struct *tsk) - if (running) - put_prev_task(rq, tsk); - -- sched_change_group(tsk, group); -+ sched_change_group(tsk); - - if (queued) - enqueue_task(rq, tsk, queue_flags); diff --git a/oneplus/hmbird/fengchi_OP-ACE-6_A16.patch b/oneplus/hmbird/fengchi_OP-ACE-6_A16.patch deleted file mode 100644 index e6008b42..00000000 --- a/oneplus/hmbird/fengchi_OP-ACE-6_A16.patch +++ /dev/null @@ -1,19909 +0,0 @@ -diff --git b/Documentation/scheduler/index.rst a/Documentation/scheduler/index.rst -index 0b650bb550e6..3170747226f6 100644 ---- b/Documentation/scheduler/index.rst -+++ a/Documentation/scheduler/index.rst -@@ -19,7 +19,6 @@ Scheduler - sched-nice-design - sched-rt-group - sched-stats -- sched-ext - sched-debug - - text_files -diff --git b/Documentation/scheduler/sched-ext.rst a/Documentation/scheduler/sched-ext.rst -deleted file mode 100644 -index 84c30b44f104..000000000000 ---- b/Documentation/scheduler/sched-ext.rst -+++ /dev/null -@@ -1,230 +0,0 @@ --========================== --Extensible Scheduler Class --========================== -- --sched_ext is a scheduler class whose behavior can be defined by a set of BPF --programs - the BPF scheduler. -- --* sched_ext exports a full scheduling interface so that any scheduling -- algorithm can be implemented on top. -- --* The BPF scheduler can group CPUs however it sees fit and schedule them -- together, as tasks aren't tied to specific CPUs at the time of wakeup. -- --* The BPF scheduler can be turned on and off dynamically anytime. -- --* The system integrity is maintained no matter what the BPF scheduler does. -- The default scheduling behavior is restored anytime an error is detected, -- a runnable task stalls, or on invoking the SysRq key sequence -- :kbd:`SysRq-S`. -- --Switching to and from sched_ext --=============================== -- --``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and --``tools/sched_ext`` contains the example schedulers. -- --sched_ext is used only when the BPF scheduler is loaded and running. -- --If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be --treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is --loaded. On load, such tasks will be switched to and scheduled by sched_ext. -- --The BPF scheduler can choose to schedule all normal and lower class tasks by --calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this --case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and --``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers, --this mode can be selected with the ``-a`` option. -- --Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or --detection of any internal error including stalled runnable tasks aborts the --BPF scheduler and reverts all tasks back to CFS. -- --.. code-block:: none -- -- # make -j16 -C tools/sched_ext -- # tools/sched_ext/scx_example_simple -- local=0 global=3 -- local=5 global=24 -- local=9 global=44 -- local=13 global=56 -- local=17 global=72 -- ^CEXIT: BPF scheduler unregistered -- --If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler --and whether a given task is on sched_ext can be determined as follows: -- --.. code-block:: none -- -- # cat /sys/kernel/debug/sched/ext -- ops : simple -- enabled : 1 -- switching_all : 1 -- switched_all : 1 -- enable_state : enabled -- -- # grep ext /proc/self/sched -- ext.enabled : 1 -- --The Basics --========== -- --Userspace can implement an arbitrary BPF scheduler by loading a set of BPF --programs that implement ``struct sched_ext_ops``. The only mandatory field --is ``ops.name`` which must be a valid BPF object name. All operations are --optional. The following modified excerpt is from --``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO --scheduler. -- --.. code-block:: c -- -- s32 BPF_STRUCT_OPS(simple_init) -- { -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; -- } -- -- void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) -- { -- if (enq_flags & SCX_ENQ_LOCAL) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags); -- } -- -- void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) -- { -- exit_type = ei->type; -- } -- -- SEC(".struct_ops") -- struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", -- }; -- --Dispatch Queues ----------------- -- --To match the impedance between the scheduler core and the BPF scheduler, --sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a --priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``), --and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage --an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and --``scx_bpf_destroy_dsq()``. -- --A CPU always executes a task from its local DSQ. A task is "dispatched" to a --DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's --local DSQ. -- --When a CPU is looking for the next task to run, if the local DSQ is not --empty, the first task is picked. Otherwise, the CPU tries to consume the --global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()`` --is invoked. -- --Scheduling Cycle ------------------ -- --The following briefly shows how a waking task is scheduled and executed. -- --1. When a task is waking up, ``ops.select_cpu()`` is the first operation -- invoked. This serves two purposes. First, CPU selection optimization -- hint. Second, waking up the selected CPU if idle. -- -- The CPU selected by ``ops.select_cpu()`` is an optimization hint and not -- binding. The actual decision is made at the last step of scheduling. -- However, there is a small performance gain if the CPU -- ``ops.select_cpu()`` returns matches the CPU the task eventually runs on. -- -- A side-effect of selecting a CPU is waking it up from idle. While a BPF -- scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper, -- using ``ops.select_cpu()`` judiciously can be simpler and more efficient. -- -- Note that the scheduler core will ignore an invalid CPU selection, for -- example, if it's outside the allowed cpumask of the task. -- --2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can -- make one of the following decisions: -- -- * Immediately dispatch the task to either the global or local DSQ by -- calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or -- ``SCX_DSQ_LOCAL``, respectively. -- -- * Immediately dispatch the task to a custom DSQ by calling -- ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63. -- -- * Queue the task on the BPF side. -- --3. When a CPU is ready to schedule, it first looks at its local DSQ. If -- empty, it then looks at the global DSQ. If there still isn't a task to -- run, ``ops.dispatch()`` is invoked which can use the following two -- functions to populate the local DSQ. -- -- * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can -- be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``, -- ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()`` -- currently can't be called with BPF locks held, this is being worked on -- and will be supported. ``scx_bpf_dispatch()`` schedules dispatching -- rather than performing them immediately. There can be up to -- ``ops.dispatch_max_batch`` pending tasks. -- -- * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ -- to the dispatching DSQ. This function cannot be called with any BPF -- locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks -- before trying to consume the specified DSQ. -- --4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ, -- the CPU runs the first one. If empty, the following steps are taken: -- -- * Try to consume the global DSQ. If successful, run the task. -- -- * If ``ops.dispatch()`` has dispatched any tasks, retry #3. -- -- * If the previous task is an SCX task and still runnable, keep executing -- it (see ``SCX_OPS_ENQ_LAST``). -- -- * Go idle. -- --Note that the BPF scheduler can always choose to dispatch tasks immediately --in ``ops.enqueue()`` as illustrated in the above simple example. If only the --built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as --a task is never queued on the BPF scheduler and both the local and global --DSQs are consumed automatically. -- --``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use --``scx_bpf_dispatch_vtime()`` for the priority queue. See the function --documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for --more information. -- --Where to Look --============= -- --* ``include/linux/sched/ext.h`` defines the core data structures, ops table -- and constants. -- --* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. -- The functions prefixed with ``scx_bpf_`` can be called from the BPF -- scheduler. -- --* ``tools/sched_ext/`` hosts example BPF scheduler implementations. -- -- * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example -- using a custom DSQ. -- -- * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting -- five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. -- --ABI Instability --=============== -- --The APIs provided by sched_ext to BPF schedulers programs have no stability --guarantees. This includes the ops table callbacks and constants defined in --``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in --``kernel/sched/ext.c``. -- --While we will attempt to provide a relatively stable API surface when --possible, they are subject to change without warning between kernel --versions. -diff --git b/MAINTAINERS a/MAINTAINERS -index 9fc6108503c3..2384b55682ee 100644 ---- b/MAINTAINERS -+++ a/MAINTAINERS -@@ -19132,8 +19132,6 @@ R: Ben Segall (CONFIG_CFS_BANDWIDTH) - R: Mel Gorman (CONFIG_NUMA_BALANCING) - R: Daniel Bristot de Oliveira (SCHED_DEADLINE) - R: Valentin Schneider (TOPOLOGY) --R: Tejun Heo (SCHED_EXT) --R: David Vernet (SCHED_EXT) - L: linux-kernel@vger.kernel.org - S: Maintained - T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core -@@ -19142,7 +19140,6 @@ F: include/linux/sched.h - F: include/linux/wait.h - F: include/uapi/linux/sched.h - F: kernel/sched/ --F: tools/sched_ext/ - - SCSI LIBSAS SUBSYSTEM - R: John Garry -diff --git b/arch/arm64/configs/defconfig a/arch/arm64/configs/defconfig -index f93980c09d7f..c2d530535980 100644 ---- b/arch/arm64/configs/defconfig -+++ a/arch/arm64/configs/defconfig -@@ -6,8 +6,6 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_BPF_SYSCALL=y - CONFIG_BPF_JIT=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_BSD_PROCESS_ACCT=y - CONFIG_BSD_PROCESS_ACCT_V3=y -@@ -1587,3 +1585,4 @@ CONFIG_CORESIGHT_STM=m - CONFIG_CORESIGHT_CPU_DEBUG=m - CONFIG_CORESIGHT_CTI=m - CONFIG_MEMTEST=y -+CONFIG_HMBIRD_SCHED=y -diff --git b/arch/arm64/configs/gki_defconfig a/arch/arm64/configs/gki_defconfig -index 4f756dca5e05..6c1bb94f875d 100644 ---- b/arch/arm64/configs/gki_defconfig -+++ a/arch/arm64/configs/gki_defconfig -@@ -10,8 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y - # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set - CONFIG_BPF_LSM=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_TASKSTATS=y - CONFIG_TASK_DELAY_ACCT=y -diff --git b/drivers/gpu/drm/msm/msm_drv.c a/drivers/gpu/drm/msm/msm_drv.c -index 4bd028fa7500..5825ce9d6d7b 100755 ---- b/drivers/gpu/drm/msm/msm_drv.c -+++ a/drivers/gpu/drm/msm/msm_drv.c -@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) - } - - sched_set_fifo(ev_thread->worker->task); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3); -+#endif - } - - ret = drm_vblank_init(ddev, priv->num_crtcs); -diff --git b/drivers/tty/sysrq.c a/drivers/tty/sysrq.c -index bd001445815e..dcf2e1ebf9c5 100644 ---- b/drivers/tty/sysrq.c -+++ a/drivers/tty/sysrq.c -@@ -524,7 +524,6 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { - NULL, /* P */ - NULL, /* Q */ - NULL, /* R */ -- /* S: May be registered by sched_ext for resetting */ - NULL, /* S */ - NULL, /* T */ - NULL, /* U */ -diff --git b/include/asm-generic/vmlinux.lds.h a/include/asm-generic/vmlinux.lds.h -index c45078a445c8..6c15659f874e 100755 ---- b/include/asm-generic/vmlinux.lds.h -+++ a/include/asm-generic/vmlinux.lds.h -@@ -131,7 +131,7 @@ - *(__dl_sched_class) \ - *(__rt_sched_class) \ - *(__fair_sched_class) \ -- *(__ext_sched_class) \ -+ *(__hmbird_sched_class) \ - *(__idle_sched_class) \ - __sched_class_lowest = .; - -diff --git b/include/linux/sched.h a/include/linux/sched.h -index ba49d7bd2b67..dd6b1d58af01 100755 ---- b/include/linux/sched.h -+++ a/include/linux/sched.h -@@ -73,7 +73,9 @@ struct task_delay_info; - struct task_group; - struct user_event_mm; - --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - - /* - * Task state bitmask. NOTE! These bits are also -@@ -298,11 +300,6 @@ enum { - - extern void scheduler_tick(void); - --#ifdef CONFIG_SLIM_SCHED --extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer); --extern void stop_shadow_tick_timer(void); --#endif -- - #define MAX_SCHEDULE_TIMEOUT LONG_MAX - - extern long schedule_timeout(long timeout); -@@ -1523,13 +1520,8 @@ struct task_struct { - */ - struct callback_head l1d_flush_kill; - #endif --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, unsigned long sched_prop); -- ANDROID_KABI_USE(2, struct sched_ext_entity *scx); --#else - ANDROID_KABI_RESERVE(1); - ANDROID_KABI_RESERVE(2); --#endif - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); - ANDROID_KABI_RESERVE(5); -@@ -1933,6 +1925,91 @@ static inline int task_nice(const struct task_struct *p) - return PRIO_TO_NICE((p)->static_prio); - } - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline bool task_is_top_task(struct task_struct *p) -+{ -+ return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ ->top_task_prop & TOP_TASK_BITS_MASK); -+} -+ -+static inline int get_top_task_prop(struct task_struct *p) -+{ -+ return get_hmbird_ts(p)->top_task_prop; -+} -+ -+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear) -+{ -+ if (set) -+ get_hmbird_ts(p)->top_task_prop |= set; -+ if (clear) -+ get_hmbird_ts(p)->top_task_prop &= ~clear; -+ return 0; -+} -+ -+static inline void reset_top_task_prop(struct task_struct *p) -+{ -+ get_hmbird_ts(p)->top_task_prop = 0; -+} -+ -+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ entity->sched_prop = sp; -+ } -+ return 0; -+} -+ -+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->sched_prop; -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq) -+{ -+ unsigned long new_dsq; -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq; -+ entity->sched_prop = new_dsq; -+ } -+} -+ -+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK); -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ entity->dsq_sync_ux = val; -+} -+ -+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->dsq_sync_ux; -+ else -+ return 0; -+} -+#endif - - extern int can_nice(const struct task_struct *p, const int nice); - extern int task_curr(const struct task_struct *p); -diff --git b/include/linux/sched/ext.h a/include/linux/sched/ext.h -deleted file mode 100755 -index b737a00c1373..000000000000 ---- b/include/linux/sched/ext.h -+++ /dev/null -@@ -1,647 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef _LINUX_SCHED_EXT_H --#define _LINUX_SCHED_EXT_H -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --#include -- --enum scx_consts { -- SCX_OPS_NAME_LEN = 128, -- SCX_EXIT_REASON_LEN = 128, -- SCX_EXIT_BT_LEN = 64, -- SCX_EXIT_MSG_LEN = 1024, -- -- SCX_SLICE_DFL = 20 * NSEC_PER_MSEC, -- SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */ --}; -- --/* -- * DSQ (dispatch queue) IDs are 64bit of the format: -- * -- * Bits: [63] [62 .. 0] -- * [ B] [ ID ] -- * -- * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -- * ID: 63 bit ID -- * -- * Built-in IDs: -- * -- * Bits: [63] [62] [61..32] [31 .. 0] -- * [ 1] [ L] [ R ] [ V ] -- * -- * 1: 1 for built-in DSQs. -- * L: 1 for LOCAL_ON DSQ IDs, 0 for others -- * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -- */ --enum scx_dsq_id_flags { -- SCX_DSQ_FLAG_BUILTIN = 1LLU << 63, -- SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -- -- SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0, -- SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, -- SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, -- SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, -- SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, --}; -- --enum scx_exit_type { -- SCX_EXIT_NONE, -- SCX_EXIT_DONE, -- -- SCX_EXIT_UNREG = 64, /* BPF unregistration */ -- SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -- SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ -- SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ --}; -- --/* -- * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is -- * being disabled. -- */ --struct scx_exit_info { -- /* %SCX_EXIT_* - broad category of the exit reason */ -- enum scx_exit_type type; -- /* textual representation of the above */ -- char reason[SCX_EXIT_REASON_LEN]; -- /* number of entries in the backtrace */ -- u32 bt_len; -- /* backtrace if exiting due to an error */ -- unsigned long bt[SCX_EXIT_BT_LEN]; -- /* extra message */ -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --/* sched_ext_ops.flags */ --enum scx_ops_flags { -- /* -- * Keep built-in idle tracking even if ops.update_idle() is implemented. -- */ -- SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, -- -- /* -- * By default, if there are no other task to run on the CPU, ext core -- * keeps running the current task even after its slice expires. If this -- * flag is specified, such tasks are passed to ops.enqueue() with -- * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. -- */ -- SCX_OPS_ENQ_LAST = 1LLU << 1, -- -- /* -- * An exiting task may schedule after PF_EXITING is set. In such cases, -- * bpf_task_from_pid() may not be able to find the task and if the BPF -- * scheduler depends on pid lookup for dispatching, the task will be -- * lost leading to various issues including RCU grace period stalls. -- * -- * To mask this problem, by default, unhashed tasks are automatically -- * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't -- * depend on pid lookups and wants to handle these tasks directly, the -- * following flag can be used. -- */ -- SCX_OPS_ENQ_EXITING = 1LLU << 2, -- -- /* -- * CPU cgroup knob enable flags -- */ -- SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16, /* cpu.weight */ -- -- SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | -- SCX_OPS_ENQ_LAST | -- SCX_OPS_ENQ_EXITING | -- SCX_OPS_CGROUP_KNOB_WEIGHT, --}; -- --/* argument container for ops.enable() and friends */ --struct scx_enable_args { --}; -- --/* argument container for ops->cgroup_init() */ --struct scx_cgroup_init_args { -- /* the weight of the cgroup [1..10000] */ -- u32 weight; --}; -- --enum scx_cpu_preempt_reason { -- /* next task is being scheduled by &sched_class_rt */ -- SCX_CPU_PREEMPT_RT, -- /* next task is being scheduled by &sched_class_dl */ -- SCX_CPU_PREEMPT_DL, -- /* next task is being scheduled by &sched_class_stop */ -- SCX_CPU_PREEMPT_STOP, -- /* unknown reason for SCX being preempted */ -- SCX_CPU_PREEMPT_UNKNOWN, --}; -- --/* -- * Argument container for ops->cpu_acquire(). Currently empty, but may be -- * expanded in the future. -- */ --struct scx_cpu_acquire_args {}; -- --/* argument container for ops->cpu_release() */ --struct scx_cpu_release_args { -- /* the reason the CPU was preempted */ -- enum scx_cpu_preempt_reason reason; -- -- /* the task that's going to be scheduled on the CPU */ -- struct task_struct *task; --}; -- --/** -- * struct sched_ext_ops - Operation table for BPF scheduler implementation -- * -- * Userland can implement an arbitrary scheduling policy by implementing and -- * loading operations in this table. -- */ --struct sched_ext_ops { -- /** -- * select_cpu - Pick the target CPU for a task which is being woken up -- * @p: task being woken up -- * @prev_cpu: the cpu @p was on before sleeping -- * @wake_flags: SCX_WAKE_* -- * -- * Decision made here isn't final. @p may be moved to any CPU while it -- * is getting dispatched for execution later. However, as @p is not on -- * the rq at this point, getting the eventual execution CPU right here -- * saves a small bit of overhead down the line. -- * -- * If an idle CPU is returned, the CPU is kicked and will try to -- * dispatch. While an explicit custom mechanism can be added, -- * select_cpu() serves as the default way to wake up idle CPUs. -- */ -- s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); -- -- /** -- * enqueue - Enqueue a task on the BPF scheduler -- * @p: task being enqueued -- * @enq_flags: %SCX_ENQ_* -- * -- * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() -- * or enqueue on the BPF scheduler. If not directly dispatched, the bpf -- * scheduler owns @p and if it fails to dispatch @p, the task will -- * stall. -- */ -- void (*enqueue)(struct task_struct *p, u64 enq_flags); -- -- /** -- * dequeue - Remove a task from the BPF scheduler -- * @p: task being dequeued -- * @deq_flags: %SCX_DEQ_* -- * -- * Remove @p from the BPF scheduler. This is usually called to isolate -- * the task while updating its scheduling properties (e.g. priority). -- * -- * The ext core keeps track of whether the BPF side owns a given task or -- * not and can gracefully ignore spurious dispatches from BPF side, -- * which makes it safe to not implement this method. However, depending -- * on the scheduling logic, this can lead to confusing behaviors - e.g. -- * scheduling position not being updated across a priority change. -- */ -- void (*dequeue)(struct task_struct *p, u64 deq_flags); -- -- /** -- * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs -- * @cpu: CPU to dispatch tasks for -- * @prev: previous task being switched out -- * -- * Called when a CPU's local dsq is empty. The operation should dispatch -- * one or more tasks from the BPF scheduler into the DSQs using -- * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using -- * scx_bpf_consume(). -- * -- * The maximum number of times scx_bpf_dispatch() can be called without -- * an intervening scx_bpf_consume() is specified by -- * ops.dispatch_max_batch. See the comments on top of the two functions -- * for more details. -- * -- * When not %NULL, @prev is an SCX task with its slice depleted. If -- * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in -- * @prev->scx.flags, it is not enqueued yet and will be enqueued after -- * ops.dispatch() returns. To keep executing @prev, return without -- * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. -- */ -- void (*dispatch)(s32 cpu, struct task_struct *prev); -- -- /** -- * runnable - A task is becoming runnable on its associated CPU -- * @p: task becoming runnable -- * @enq_flags: %SCX_ENQ_* -- * -- * This and the following three functions can be used to track a task's -- * execution state transitions. A task becomes ->runnable() on a CPU, -- * and then goes through one or more ->running() and ->stopping() pairs -- * as it runs on the CPU, and eventually becomes ->quiescent() when it's -- * done running on the CPU. -- * -- * @p is becoming runnable on the CPU because it's -- * -- * - waking up (%SCX_ENQ_WAKEUP) -- * - being moved from another CPU -- * - being restored after temporarily taken off the queue for an -- * attribute change. -- * -- * This and ->enqueue() are related but not coupled. This operation -- * notifies @p's state transition and may not be followed by ->enqueue() -- * e.g. when @p is being dispatched to a remote CPU. Likewise, a task -- * may be ->enqueue()'d without being preceded by this operation e.g. -- * after exhausting its slice. -- */ -- void (*runnable)(struct task_struct *p, u64 enq_flags); -- -- /** -- * running - A task is starting to run on its associated CPU -- * @p: task starting to run -- * -- * See ->runnable() for explanation on the task state notifiers. -- */ -- void (*running)(struct task_struct *p); -- -- /** -- * stopping - A task is stopping execution -- * @p: task stopping to run -- * @runnable: is task @p still runnable? -- * -- * See ->runnable() for explanation on the task state notifiers. If -- * !@runnable, ->quiescent() will be invoked after this operation -- * returns. -- */ -- void (*stopping)(struct task_struct *p, bool runnable); -- -- /** -- * quiescent - A task is becoming not runnable on its associated CPU -- * @p: task becoming not runnable -- * @deq_flags: %SCX_DEQ_* -- * -- * See ->runnable() for explanation on the task state notifiers. -- * -- * @p is becoming quiescent on the CPU because it's -- * -- * - sleeping (%SCX_DEQ_SLEEP) -- * - being moved to another CPU -- * - being temporarily taken off the queue for an attribute change -- * (%SCX_DEQ_SAVE) -- * -- * This and ->dequeue() are related but not coupled. This operation -- * notifies @p's state transition and may not be preceded by ->dequeue() -- * e.g. when @p is being dispatched to a remote CPU. -- */ -- void (*quiescent)(struct task_struct *p, u64 deq_flags); -- -- /** -- * yield - Yield CPU -- * @from: yielding task -- * @to: optional yield target task -- * -- * If @to is NULL, @from is yielding the CPU to other runnable tasks. -- * The BPF scheduler should ensure that other available tasks are -- * dispatched before the yielding task. Return value is ignored in this -- * case. -- * -- * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf -- * scheduler can implement the request, return %true; otherwise, %false. -- */ -- bool (*yield)(struct task_struct *from, struct task_struct *to); -- -- /** -- * core_sched_before - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Used by core-sched to determine the ordering between two tasks. See -- * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on -- * core-sched. -- * -- * Both @a and @b are runnable and may or may not currently be queued on -- * the BPF scheduler. Should return %true if @a should run before @b. -- * %false if there's no required ordering or @b should run before @a. -- * -- * If not specified, the default is ordering them according to when they -- * became runnable. -- */ -- bool (*core_sched_before)(struct task_struct *a,struct task_struct *b); -- -- /** -- * set_weight - Set task weight -- * @p: task to set weight for -- * @weight: new eight [1..10000] -- * -- * Update @p's weight to @weight. -- */ -- void (*set_weight)(struct task_struct *p, u32 weight); -- -- /** -- * set_cpumask - Set CPU affinity -- * @p: task to set CPU affinity for -- * @cpumask: cpumask of cpus that @p can run on -- * -- * Update @p's CPU affinity to @cpumask. -- */ -- void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask); -- -- /** -- * update_idle - Update the idle state of a CPU -- * @cpu: CPU to udpate the idle state for -- * @idle: whether entering or exiting the idle state -- * -- * This operation is called when @rq's CPU goes or leaves the idle -- * state. By default, implementing this operation disables the built-in -- * idle CPU tracking and the following helpers become unavailable: -- * -- * - scx_bpf_select_cpu_dfl() -- * - scx_bpf_test_and_clear_cpu_idle() -- * - scx_bpf_pick_idle_cpu() -- * - scx_bpf_any_idle_cpu() -- * -- * The user also must implement ops.select_cpu() as the default -- * implementation relies on scx_bpf_select_cpu_dfl(). -- * -- * If you keep the built-in idle tracking, specify the -- * %SCX_OPS_KEEP_BUILTIN_IDLE flag. -- */ -- void (*update_idle)(s32 cpu, bool idle); -- -- /** -- * cpu_acquire - A CPU is becoming available to the BPF scheduler -- * @cpu: The CPU being acquired by the BPF scheduler. -- * @args: Acquire arguments, see the struct definition. -- * -- * A CPU that was previously released from the BPF scheduler is now once -- * again under its control. -- */ -- void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); -- -- /** -- * cpu_release - A CPU is taken away from the BPF scheduler -- * @cpu: The CPU being released by the BPF scheduler. -- * @args: Release arguments, see the struct definition. -- * -- * The specified CPU is no longer under the control of the BPF -- * scheduler. This could be because it was preempted by a higher -- * priority sched_class, though there may be other reasons as well. The -- * caller should consult @args->reason to determine the cause. -- */ -- void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -- -- /** -- * cpu_online - A CPU became online -- * @cpu: CPU which just came up -- * -- * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs beforehand. -- */ -- void (*cpu_online)(s32 cpu); -- -- /** -- * cpu_offline - A CPU is going offline -- * @cpu: CPU which is going offline -- * -- * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs afterwards. -- */ -- void (*cpu_offline)(s32 cpu); -- -- /** -- * prep_enable - Prepare to enable BPF scheduling for a task -- * @p: task to prepare BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Either we're loading a BPF scheduler or a new task is being forked. -- * Prepare BPF scheduling for @p. This operation may block and can be -- * used for allocations. -- * -- * Return 0 for success, -errno for failure. An error return while -- * loading will abort loading of the BPF scheduler. During a fork, will -- * abort the specific fork. -- */ -- s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * enable - Enable BPF scheduling for a task -- * @p: task to enable BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Enable @p for BPF scheduling. @p is now in the cgroup specified for -- * the preceding prep_enable() and will start running soon. -- */ -- void (*enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * cancel_enable - Cancel prep_enable() -- * @p: task being canceled -- * @args: enable arguments, see the struct definition -- * -- * @p was prep_enable()'d but failed before reaching enable(). Undo the -- * preparation. -- */ -- void (*cancel_enable)(struct task_struct *p, -- struct scx_enable_args *args); -- -- /** -- * disable - Disable BPF scheduling for a task -- * @p: task to disable BPF scheduling for -- * -- * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. -- * Disable BPF scheduling for @p. -- */ -- void (*disable)(struct task_struct *p); -- -- /* -- * All online ops must come before ops.init(). -- */ -- -- /** -- * init - Initialize the BPF scheduler -- */ -- s32 (*init)(void); -- -- /** -- * exit - Clean up after the BPF scheduler -- * @info: Exit info -- */ -- void (*exit)(struct scx_exit_info *info); -- -- /** -- * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch -- */ -- u32 dispatch_max_batch; -- -- /** -- * flags - %SCX_OPS_* flags -- */ -- u64 flags; -- -- /** -- * timeout_ms - The maximum amount of time, in milliseconds, that a -- * runnable task should be able to wait before being scheduled. The -- * maximum timeout may not exceed the default timeout of 30 seconds. -- * -- * Defaults to the maximum allowed timeout value of 30 seconds. -- */ -- u32 timeout_ms; -- -- /** -- * name - BPF scheduler's name -- * -- * Must be a non-zero valid BPF object name including only isalnum(), -- * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the -- * BPF scheduler is enabled. -- */ -- char name[SCX_OPS_NAME_LEN]; --}; -- --/* -- * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -- * scheduler core and the BPF scheduler. See the documentation for more details. -- */ --struct scx_dispatch_q { -- raw_spinlock_t lock; -- struct list_head fifo; /* processed in dispatching order */ -- struct rb_root_cached priq; /* processed in p->scx.dsq_vtime order */ -- u32 nr; -- u64 id; -- struct llist_node free_node; -- struct rcu_head rcu; --}; -- --/* scx_entity.flags */ --enum scx_ent_flags { -- SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ -- SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -- SCX_TASK_ENQ_LOCAL = 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */ -- -- SCX_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -- SCX_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -- -- SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -- SCX_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -- -- SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ --}; -- --/* scx_entity.dsq_flags */ --enum scx_ent_dsq_flags { -- SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ --}; -- --/* -- * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from -- * everywhere and the following bits track which kfunc sets are currently -- * allowed for %current. This simple per-task tracking works because SCX ops -- * nest in a limited way. BPF will likely implement a way to allow and disallow -- * kfuncs depending on the calling context which will replace this manual -- * mechanism. See scx_kf_allow(). -- */ --enum scx_kf_mask { -- SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ -- /* all non-sleepables may be nested inside INIT and SLEEPABLE */ -- SCX_KF_INIT = 1 << 0, /* running ops.init() */ -- SCX_KF_SLEEPABLE = 1 << 1, /* other sleepable init operations */ -- /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ -- SCX_KF_CPU_RELEASE = 1 << 2, /* ops.cpu_release() */ -- /* ops.dequeue (in REST) may be nested inside DISPATCH */ -- SCX_KF_DISPATCH = 1 << 3, /* ops.dispatch() */ -- SCX_KF_ENQUEUE = 1 << 4, /* ops.enqueue() */ -- SCX_KF_REST = 1 << 5, /* other rq-locked operations */ -- -- __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | -- SCX_KF_ENQUEUE | SCX_KF_REST, -- __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_REST, --}; -- --#define RAVG_HIST_SIZE 5 --struct scx_sched_task_stats { -- u64 mark_start; -- u64 window_start; -- u32 sum; -- u32 sum_history[RAVG_HIST_SIZE]; -- int cidx; -- -- u32 demand; -- u16 demand_scaled; -- void *sdsq; --}; -- -- -- --/* -- * The following is embedded in task_struct and contains all fields necessary -- * for a task to be scheduled by SCX. -- */ --struct sched_ext_entity { -- struct scx_dispatch_q *dsq; -- struct { -- struct list_head fifo; /* dispatch order */ -- struct rb_node priq; /* p->scx.dsq_vtime order */ -- } dsq_node; -- struct list_head watchdog_node; -- u32 flags; /* protected by rq lock */ -- u32 dsq_flags; /* protected by dsq lock */ -- u32 weight; -- s32 sticky_cpu; -- s32 holding_cpu; -- u32 kf_mask; /* see scx_kf_mask above */ -- struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ -- atomic64_t ops_state; -- unsigned long runnable_at; --#ifdef CONFIG_SCHED_CORE -- u64 core_sched_at; /* see scx_prio_less() */ --#endif -- -- /* BPF scheduler modifiable fields */ -- -- /* -- * Runtime budget in nsecs. This is usually set through -- * scx_bpf_dispatch() but can also be modified directly by the BPF -- * scheduler. Automatically decreased by SCX as the task executes. On -- * depletion, a scheduling event is triggered. -- * -- * This value is cleared to zero if the task is preempted by -- * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the -- * task ran. Use p->se.sum_exec_runtime instead. -- */ -- u64 slice; -- -- /* -- * Used to order tasks when dispatching to the vtime-ordered priority -- * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime() -- * but can also be modified directly by the BPF scheduler. Modifying it -- * while a task is queued on a dsq may mangle the ordering and is not -- * recommended. -- */ -- u64 dsq_vtime; -- -- /* -- * If set, reject future sched_setscheduler(2) calls updating the policy -- * to %SCHED_EXT with -%EACCES. -- * -- * If set from ops.prep_enable() and the task's policy is already -- * %SCHED_EXT, which can happen while the BPF scheduler is being loaded -- * or by inhering the parent's policy during fork, the task's policy is -- * rejected and forcefully reverted to %SCHED_NORMAL. The number of such -- * events are reported through /sys/kernel/debug/sched_ext::nr_rejected. -- */ -- bool disallow; /* reject switching into SCX */ -- -- /* cold fields */ -- struct list_head tasks_node; -- struct task_struct *task; -- struct scx_sched_task_stats sts; --}; -- --void sched_ext_free(struct task_struct *p); -- --#else /* !CONFIG_SCHED_CLASS_EXT */ -- --static inline void sched_ext_free(struct task_struct *p) {} -- --#endif /* CONFIG_SCHED_CLASS_EXT */ --#endif /* _LINUX_SCHED_EXT_H */ -diff --git b/include/linux/sched/hmbird_proc_val.h a/include/linux/sched/hmbird_proc_val.h -new file mode 100755 -index 000000000000..e59708b16ea3 ---- /dev/null -+++ a/include/linux/sched/hmbird_proc_val.h -@@ -0,0 +1,22 @@ -+#ifndef __HMBORD_PROC_VAL_H__ -+#define __HMBORD_PROC_VAL_H__ -+ -+extern int scx_enable; -+extern int partial_enable; -+extern int cpuctrl_high_ratio; -+extern int cpuctrl_low_ratio; -+extern int slim_stats; -+extern int hmbirdcore_debug; -+extern int slim_for_app; -+extern int misfit_ds; -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+extern int cpu7_tl; -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int slim_gov_debug; -+extern int scx_gov_ctrl; -+extern int sched_ravg_window_frame_per_sec; -+ -+#endif -\ No newline at end of file -diff --git b/include/linux/sched/hmbird_version.h a/include/linux/sched/hmbird_version.h -new file mode 100755 -index 000000000000..b2843881c26f ---- /dev/null -+++ a/include/linux/sched/hmbird_version.h -@@ -0,0 +1,52 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#ifndef _OPLUS_HMBIRD_VERSION_H_ -+#define _OPLUS_HMBIRD_VERSION_H_ -+#include -+#include -+#include -+ -+enum hmbird_version { -+ HMBIRD_UNINIT, -+ HMBIRD_GKI_VERSION, -+ HMBIRD_OGKI_VERSION, -+ HMBIRD_UNKNOW_VERSION, -+}; -+ -+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT; -+ -+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type" -+ -+static inline enum hmbird_version get_hmbird_version_type(void) -+{ -+ struct device_node *np = NULL; -+ const char *hmbird_version_str = NULL; -+ if (HMBIRD_UNINIT != hmbird_version_type) -+ return hmbird_version_type; -+ np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH); -+ if (np) { -+ of_property_read_string(np, "type", &hmbird_version_str); -+ if (NULL != hmbird_version_str) { -+ if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) { -+ hmbird_version_type = HMBIRD_OGKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi"); -+ } else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) { -+ hmbird_version_type = HMBIRD_GKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi"); -+ } else { -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi"); -+ } -+ return hmbird_version_type; -+ } -+ } -+ -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION"); -+ return hmbird_version_type; -+} -+ -+#endif /*_OPLUS_HMBIRD_VERSION_H_ */ -diff --git b/include/linux/sched/task.h a/include/linux/sched/task.h -index 03d35e3eda3c..6a5f0c0d74bd 100644 ---- b/include/linux/sched/task.h -+++ a/include/linux/sched/task.h -@@ -61,8 +61,10 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); - extern void init_idle(struct task_struct *idle, int cpu); - - extern int sched_fork(unsigned long clone_flags, struct task_struct *p); --extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); --extern void sched_cancel_fork(struct task_struct *p); -+extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); -+#ifdef CONFIG_HMBIRD_SCHED -+extern void hmbird_cancel_fork(struct task_struct *p); -+#endif - extern void sched_post_fork(struct task_struct *p); - extern void sched_dead(struct task_struct *p); - -diff --git b/include/uapi/linux/sched.h a/include/uapi/linux/sched.h -index 359a14cc76a4..969716ab4320 100755 ---- b/include/uapi/linux/sched.h -+++ a/include/uapi/linux/sched.h -@@ -118,7 +118,7 @@ struct clone_args { - /* SCHED_ISO: reserved but not implemented yet */ - #define SCHED_IDLE 5 - #define SCHED_DEADLINE 6 --#define SCHED_EXT 7 -+#define SCHED_HMBIRD 7 - - /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ - #define SCHED_RESET_ON_FORK 0x40000000 -diff --git b/init/Kconfig a/init/Kconfig -index 337276cbc7f8..bf052ca532b4 100644 ---- b/init/Kconfig -+++ a/init/Kconfig -@@ -1030,10 +1030,6 @@ config RT_GROUP_SCHED - realtime bandwidth for them. - See Documentation/scheduler/sched-rt-group.rst for more information. - --config EXT_GROUP_SCHED -- bool -- depends on SCHED_CLASS_EXT && CGROUP_SCHED -- default y - endif #CGROUP_SCHED - - config SCHED_MM_CID -diff --git b/init/init_task.c a/init/init_task.c -index 69a046388995..31ceb0e469f7 100755 ---- b/init/init_task.c -+++ a/init/init_task.c -@@ -6,7 +6,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -102,9 +101,6 @@ struct task_struct init_task - #endif - #ifdef CONFIG_CGROUP_SCHED - .sched_task_group = &root_task_group, --#endif --#ifdef CONFIG_SLIM_SCHED -- .scx = NULL, - #endif - .ptraced = LIST_HEAD_INIT(init_task.ptraced), - .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), -diff --git b/kernel/Kconfig.preempt a/kernel/Kconfig.preempt -index cf22ef6107b8..ca8de6eb1e16 100755 ---- b/kernel/Kconfig.preempt -+++ a/kernel/Kconfig.preempt -@@ -133,12 +133,8 @@ config SCHED_CORE - which is the likely usage by Linux distributions, there should - be no measurable impact on performance. - --config SLIM_SCHED -- bool "slim sched" -- default n --config SCHED_CLASS_EXT -- bool "Extensible Scheduling Class" -- depends on BPF_SYSCALL && BPF_JIT -+config HMBIRD_SCHED -+ bool "hmbird Scheduling Class(base on ext scheduler)" - help - This option enables a new scheduler class sched_ext (SCX), which - allows scheduling policies to be implemented as BPF programs to -diff --git b/kernel/bpf/bpf_struct_ops_types.h a/kernel/bpf/bpf_struct_ops_types.h -index 3618769d853d..5678a9ddf817 100644 ---- b/kernel/bpf/bpf_struct_ops_types.h -+++ a/kernel/bpf/bpf_struct_ops_types.h -@@ -9,8 +9,4 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) - #include - BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) - #endif --#ifdef CONFIG_SCHED_CLASS_EXT --#include --BPF_STRUCT_OPS_TYPE(sched_ext_ops) --#endif - #endif -diff --git b/kernel/fork.c a/kernel/fork.c -index a43f97302332..7a91505b9378 100755 ---- b/kernel/fork.c -+++ a/kernel/fork.c -@@ -23,7 +23,9 @@ - #include - #include - #include --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - #include - #include - #include -@@ -999,8 +1001,9 @@ void __put_task_struct(struct task_struct *tsk) - WARN_ON(!tsk->exit_state); - WARN_ON(refcount_read(&tsk->usage)); - WARN_ON(tsk == current); -- -- sched_ext_free(tsk); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_free(tsk); -+#endif - io_uring_free(tsk); - cgroup_free(tsk); - task_numa_free(tsk, true); -@@ -2517,7 +2520,11 @@ __latent_entropy struct task_struct *copy_process( - - retval = perf_event_init_task(p, clone_flags); - if (retval) -- goto bad_fork_sched_cancel_fork; -+#ifdef CONFIG_HMBIRD_SCHED -+ goto bad_fork_hmbird_cancel_fork; -+#else -+ goto bad_fork_cleanup_policy; -+#endif - retval = audit_alloc(p); - if (retval) - goto bad_fork_cleanup_perf; -@@ -2649,9 +2656,7 @@ __latent_entropy struct task_struct *copy_process( - * cgroup specific, it unconditionally needs to place the task on a - * runqueue. - */ -- retval = sched_cgroup_fork(p, args); -- if (retval) -- goto bad_fork_cancel_cgroup; -+ sched_cgroup_fork(p, args); - - /* - * From this point on we must avoid any synchronous user-space -@@ -2697,13 +2702,13 @@ __latent_entropy struct task_struct *copy_process( - /* Don't start children in a dying pid namespace */ - if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { - retval = -ENOMEM; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* Let kill terminate clone/fork in the middle */ - if (fatal_signal_pending(current)) { - retval = -EINTR; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* No more failure paths after this point. */ -@@ -2780,11 +2785,10 @@ __latent_entropy struct task_struct *copy_process( - - return p; - --bad_fork_core_free: -+bad_fork_cancel_cgroup: - sched_core_free(p); - spin_unlock(¤t->sighand->siglock); - write_unlock_irq(&tasklist_lock); --bad_fork_cancel_cgroup: - cgroup_cancel_fork(p, args); - bad_fork_put_pidfd: - if (clone_flags & CLONE_PIDFD) { -@@ -2823,8 +2827,10 @@ __latent_entropy struct task_struct *copy_process( - audit_free(p); - bad_fork_cleanup_perf: - perf_event_free_task(p); --bad_fork_sched_cancel_fork: -- sched_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+bad_fork_hmbird_cancel_fork: -+ hmbird_cancel_fork(p); -+#endif - bad_fork_cleanup_policy: - lockdep_free_task(p); - #ifdef CONFIG_NUMA -diff --git b/kernel/sched/build_policy.c a/kernel/sched/build_policy.c -index 005025f55bea..c091539a0f60 100755 ---- b/kernel/sched/build_policy.c -+++ a/kernel/sched/build_policy.c -@@ -28,8 +28,10 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED - #include - #include -+#endif - - #include - -@@ -54,6 +56,10 @@ - #include "cputime.c" - #include "deadline.c" - --#ifdef CONFIG_SCHED_CLASS_EXT --# include "ext.c" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/hmbird_util_track.c" -+#include "hmbird/hmbird_sched_proc.c" -+#include "hmbird/hmbird_shadow_tick.c" -+#include "hmbird/hmbird.c" -+#include "hmbird/hmbird_misc.c" - #endif -diff --git b/kernel/sched/core.c a/kernel/sched/core.c -index 25d5703df992..b894417ce7df 100755 ---- b/kernel/sched/core.c -+++ a/kernel/sched/core.c -@@ -96,6 +96,12 @@ - #include "../../io_uring/io-wq.h" - #include "../smpboot.h" - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/slim.h" -+#include "hmbird/hmbird_shadow_tick.h" -+#include "hmbird.h" -+#endif -+ - #include - #include - #include -@@ -170,7 +176,6 @@ __read_mostly int scheduler_running; - - DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); - -- - /* kernel prio, less is more */ - static inline int __task_prio(const struct task_struct *p) - { -@@ -183,11 +188,10 @@ static inline int __task_prio(const struct task_struct *p) - if (p->sched_class == &idle_sched_class) - return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - --#ifdef CONFIG_SCHED_CLASS_EXT -- if (p->sched_class == &ext_sched_class) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (p->sched_class == &hmbird_sched_class) - return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ - #endif -- - return MAX_RT_PRIO + MAX_NICE; /* 119, squash fair */ - } - -@@ -217,9 +221,9 @@ static inline bool prio_less(const struct task_struct *a, - if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ - return cfs_prio_less(a, b, in_fi); - --#ifdef CONFIG_SCHED_CLASS_EXT -+#ifdef CONFIG_HMBIRD_SCHED - if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ -- return scx_prio_less(a, b, in_fi); -+ return hmbird_prio_less(a, b, in_fi); - #endif - - return false; -@@ -1279,17 +1283,26 @@ bool sched_can_stop_tick(struct rq *rq) - if (fifo_nr_running) - return true; - -+#ifdef CONFIG_HMBIRD_SCHED - /* -- * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks -+ * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks - * left. For CFS, if there's more than one we need the tick for -- * involuntary preemption. For SCX, ask. -+ * involuntary preemption. For HMBIRD, ask. - */ -- if (!scx_switched_all() && rq->nr_running > 1) -+ if (!hmbird_enabled() && rq->nr_running > 1) - return false; - -- if (scx_enabled() && !scx_can_stop_tick(rq)) -+ if (hmbird_enabled() && !hmbird_can_stop_tick(rq)) - return false; -- -+#else -+ /* -+ * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; -+ * if there's more than one we need the tick for involuntary -+ * preemption. -+ */ -+ if (rq->nr_running > 1) -+ return false; -+#endif - /* - * If there is one task and it has CFS runtime bandwidth constraints - * and it's on the cpu now we don't want to stop the tick. -@@ -2222,10 +2235,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) - } - EXPORT_SYMBOL_GPL(deactivate_task); - --struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) -+#ifdef CONFIG_HMBIRD_SCHED -+struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - { -- struct sched_change_guard cg = { -+ struct hmbird_sched_change_guard cg = { - .rq = rq, - .p = p, - .queued = task_on_rq_queued(p), -@@ -2247,7 +2261,7 @@ sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - return cg; - } - --void sched_change_guard_fini(struct sched_change_guard *cg, int flags) -+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags) - { - if (cg->queued) - enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); -@@ -2255,6 +2269,7 @@ void sched_change_guard_fini(struct sched_change_guard *cg, int flags) - set_next_task(cg->rq, cg->p); - cg->done = true; - } -+#endif - - static inline int __normal_prio(int policy, int rt_prio, int nice) - { -@@ -2320,9 +2335,9 @@ inline int task_curr(const struct task_struct *p) - * this means any call to check_class_changed() must be followed by a call to - * balance_callback(). - */ --void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio) -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) - { - if (prev_class != p->sched_class) { - if (prev_class->switched_from) -@@ -2885,6 +2900,7 @@ static void - __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - { - struct rq *rq = task_rq(p); -+ bool queued, running; - - /* - * This here violates the locking rules for affinity, since we're only -@@ -2903,9 +2919,26 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - else - lockdep_assert_held(&p->pi_lock); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->sched_class->set_cpus_allowed(p, ctx); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ -+ if (queued) { -+ /* -+ * Because __kthread_bind() calls this on blocked tasks without -+ * holding rq->lock. -+ */ -+ lockdep_assert_rq_held(rq); -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); - } -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->sched_class->set_cpus_allowed(p, ctx); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - } - - /* -@@ -3772,7 +3805,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class)) -+#else - if (unlikely(!is_cpu_allowed(p, cpu))) -+#endif - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -@@ -4891,8 +4928,9 @@ late_initcall(sched_core_sysctl_init); - */ - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { -- -+#ifdef CONFIG_HMBIRD_SCHED - int ret; -+#endif - - trace_android_rvh_sched_fork(p); - -@@ -4933,21 +4971,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_reset_on_fork = 0; - } - -- scx_pre_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ ret = hmbird_pre_fork(p); -+ if (ret) -+ goto out_cancel; - - if (dl_prio(p->prio)) { - ret = -EAGAIN; - goto out_cancel; --#ifdef CONFIG_SCHED_CLASS_EXT -- } else if (task_on_scx(p)) { -- p->sched_class = &ext_sched_class; --#endif -+ } else if (task_on_hmbird(p)) { -+ p->sched_class = &hmbird_sched_class; - } else if (rt_prio(p->prio)) { - p->sched_class = &rt_sched_class; - } else { - p->sched_class = &fair_sched_class; - } -- -+#else -+ if (dl_prio(p->prio)) -+ return -EAGAIN; -+ else if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - init_entity_runnable_average(&p->se); - trace_android_rvh_finish_prio_fork(p); - -@@ -4966,12 +5012,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - #endif - return 0; - -+#ifdef CONFIG_HMBIRD_SCHED - out_cancel: -- scx_cancel_fork(p); -+ hmbird_cancel_fork(p); - return ret; -+#endif - } - --int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) -+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - { - unsigned long flags; - -@@ -4998,20 +5046,14 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -- -- return scx_fork(p); --} -- --void sched_cancel_fork(struct task_struct *p) --{ -- scx_cancel_fork(p); - } - - void sched_post_fork(struct task_struct *p) - { - uclamp_post_fork(p); -- -- scx_post_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_post_fork(p); -+#endif - } - - unsigned long to_ratio(u64 period, u64 runtime) -@@ -5875,21 +5917,28 @@ void scheduler_tick(void) - - if (sched_feat(LATENCY_WARN) && resched_latency) - resched_latency_warn(cpu, resched_latency); -- -- scx_notify_sched_tick(); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_notify_sched_tick(); -+#endif - perf_event_task_tick(); - - if (curr->flags & PF_WQ_WORKER) - wq_worker_tick(curr); - - #ifdef CONFIG_SMP -- if (!scx_switched_all()) { -+#ifdef CONFIG_HMBIRD_SCHED -+ if (!hmbird_enabled()) { -+#endif - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); -+#ifdef CONFIG_HMBIRD_SCHED - } -+#endif - #endif - trace_android_vh_scheduler_tick(rq); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ scheduler_tick_handler(NULL, NULL); -+#endif - } - - #ifdef CONFIG_NO_HZ_FULL -@@ -6191,7 +6240,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ -+#ifdef CONFIG_HMBIRD_SCHED - for_balance_class_range(class, prev->sched_class, &idle_sched_class) { -+#else -+ for_class_range(class, prev->sched_class, &idle_sched_class) { -+#endif - if (class->balance(rq, prev, rf)) - break; - } -@@ -6209,9 +6262,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - const struct sched_class *class; - struct task_struct *p; - -- if (scx_enabled()) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (hmbird_enabled()) - goto restart; -- -+#endif - /* - * Optimization: we know that if all tasks are in the fair class we can - * call that function directly, but only if the @prev task wasn't of a -@@ -6237,13 +6291,21 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - restart: - put_prev_task_balance(rq, prev, rf); - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { - p = class->pick_next_task(rq); - if (p) { -- scx_notify_pick_next_task(rq, p, class); -+ hmbird_notify_pick_next_task(rq, p, class); - return p; - } - } -+#else -+ for_each_class(class) { -+ p = class->pick_next_task(rq); -+ if (p) -+ return p; -+ } -+#endif - - BUG(); /* The idle class should always have a runnable task. */ - } -@@ -6272,7 +6334,11 @@ static inline struct task_struct *pick_task(struct rq *rq) - const struct sched_class *class; - struct task_struct *p; - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { -+#else -+ for_each_class(class) { -+#endif - p = class->pick_task(rq); - if (p) - return p; -@@ -6916,7 +6982,9 @@ static void __sched notrace __schedule(unsigned int sched_mode) - psi_sched_switch(prev, next, !task_on_rq_queued(prev)); - - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#endif - /* Also unlocks the rq: */ - rq = context_switch(rq, prev, next, &rf); - } else { -@@ -7244,24 +7312,31 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag - } - EXPORT_SYMBOL(default_wake_function); - --void __setscheduler_prio(struct task_struct *p, int prio) -+static void __setscheduler_prio(struct task_struct *p, int prio) - { -- /* -- * After switching all rt and fair class to ext, -- * stop class is switched to ext class too. Just -- * skip it. */ -+#ifdef CONFIG_HMBIRD_SCHED -+ bool on_hmbird = task_on_hmbird(p); -+ - if (p->sched_class == &stop_sched_class) -- ;/* do nothing */ -+ ; - else if (dl_prio(prio)) - p->sched_class = &dl_sched_class; --#ifdef CONFIG_SCHED_CLASS_EXT -- else if (task_on_scx(p)) -- p->sched_class = &ext_sched_class; --#endif -+ else if (rt_prio(prio) && on_hmbird) -+ p->sched_class = &hmbird_sched_class; - else if (rt_prio(prio)) - p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; - else - p->sched_class = &fair_sched_class; -+#else -+ if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - - p->prio = prio; - trace_android_rvh_setscheduler_prio(p); -@@ -7297,7 +7372,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - */ - void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - { -- int prio, oldprio, queue_flag = -+ int prio, oldprio, queued, running, queue_flag = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *prev_class; - struct rq_flags rf; -@@ -7360,42 +7435,52 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - queue_flag &= ~DEQUEUE_MOVE; - - prev_class = p->sched_class; -- SCHED_CHANGE_BLOCK(rq, p, queue_flag) { -- /* -- * Boosting condition are: -- * 1. -rt task is running and holds mutex A -- * --> -dl task blocks on mutex A -- * -- * 2. -dl task is running and holds mutex A -- * --> -dl task blocks on mutex A and could preempt the -- * running task -- */ -- if (dl_prio(prio)) { -- if (!dl_prio(p->normal_prio) || -- (pi_task && dl_prio(pi_task->prio) && -- dl_entity_preempt(&pi_task->dl, &p->dl))) { -- p->dl.pi_se = pi_task->dl.pi_se; -- queue_flag |= ENQUEUE_REPLENISH; -- } else { -- p->dl.pi_se = &p->dl; -- } -- } else if (rt_prio(prio)) { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- if (oldprio < prio) -- queue_flag |= ENQUEUE_HEAD; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flag); -+ if (running) -+ put_prev_task(rq, p); -+ -+ /* -+ * Boosting condition are: -+ * 1. -rt task is running and holds mutex A -+ * --> -dl task blocks on mutex A -+ * -+ * 2. -dl task is running and holds mutex A -+ * --> -dl task blocks on mutex A and could preempt the -+ * running task -+ */ -+ if (dl_prio(prio)) { -+ if (!dl_prio(p->normal_prio) || -+ (pi_task && dl_prio(pi_task->prio) && -+ dl_entity_preempt(&pi_task->dl, &p->dl))) { -+ p->dl.pi_se = pi_task->dl.pi_se; -+ queue_flag |= ENQUEUE_REPLENISH; - } else { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- else if (rt_prio(oldprio)) -- p->rt.timeout = 0; -- else if (!task_has_idle_policy(p)) -- reweight_task(p, prio - MAX_RT_PRIO); -+ p->dl.pi_se = &p->dl; - } -- -- __setscheduler_prio(p, prio); -+ } else if (rt_prio(prio)) { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ if (oldprio < prio) -+ queue_flag |= ENQUEUE_HEAD; -+ } else { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ else if (rt_prio(oldprio)) -+ p->rt.timeout = 0; -+ else if (!task_has_idle_policy(p)) -+ reweight_task(p, prio - MAX_RT_PRIO); - } - -+ __setscheduler_prio(p, prio); -+ -+ if (queued) -+ enqueue_task(rq, p, queue_flag); -+ if (running) -+ set_next_task(rq, p); -+ - check_class_changed(rq, p, prev_class, oldprio); - out_unlock: - /* Avoid rq from going away on us: */ -@@ -7416,7 +7501,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - - void set_user_nice(struct task_struct *p, long nice) - { -- bool allowed = false; -+ bool queued, running, allowed = false; - int old_prio; - struct rq_flags rf; - struct rq *rq; -@@ -7445,13 +7530,22 @@ void set_user_nice(struct task_struct *p, long nice) - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); -+ if (running) -+ put_prev_task(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->static_prio = NICE_TO_PRIO(nice); -- set_load_weight(p, true); -- old_prio = p->prio; -- p->prio = effective_prio(p); -- } -+ p->static_prio = NICE_TO_PRIO(nice); -+ set_load_weight(p, true); -+ old_prio = p->prio; -+ p->prio = effective_prio(p); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - - /* - * If the task increased its priority or is running and -@@ -7868,7 +7962,7 @@ static int __sched_setscheduler(struct task_struct *p, - bool user, bool pi) - { - int oldpolicy = -1, policy = attr->sched_policy; -- int retval, oldprio, newprio; -+ int retval, oldprio, newprio, queued, running; - const struct sched_class *prev_class; - struct balance_callback *head; - struct rq_flags rf; -@@ -7876,7 +7970,9 @@ static int __sched_setscheduler(struct task_struct *p, - int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct rq *rq; - bool cpuset_locked = false; -- -+#ifdef CONFIG_HMBIRD_SCHED -+ unsigned long flags; -+#endif - - /* The pi code expects interrupts enabled */ - BUG_ON(pi && in_interrupt()); -@@ -7942,7 +8038,9 @@ static int __sched_setscheduler(struct task_struct *p, - * To be able to change p->policy safely, the appropriate - * runqueue lock must be held. - */ -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+#endif - rq = task_rq_lock(p, &rf); - update_rq_clock(rq); - -@@ -7954,10 +8052,11 @@ static int __sched_setscheduler(struct task_struct *p, - goto unlock; - } - -- retval = scx_check_setscheduler(p, policy); -+#ifdef CONFIG_HMBIRD_SCHED -+ retval = hmbird_check_setscheduler(p, policy); - if (retval) - goto unlock; -- -+#endif - /* - * If not changing anything there's no need to proceed further, - * but store a possible modification of reset_on_fork. -@@ -8014,7 +8113,9 @@ static int __sched_setscheduler(struct task_struct *p, - if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { - policy = oldpolicy = -1; - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - goto recheck; -@@ -8047,16 +8148,23 @@ static int __sched_setscheduler(struct task_struct *p, - queue_flags &= ~DEQUEUE_MOVE; - } - -- SCHED_CHANGE_BLOCK(rq, p, queue_flags) { -- prev_class = p->sched_class; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flags); -+ if (running) -+ put_prev_task(rq, p); -+ -+ prev_class = p->sched_class; - -- if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -- __setscheduler_params(p, attr); -- __setscheduler_prio(p, newprio); -- trace_android_rvh_setscheduler(p); -- } -- __setscheduler_uclamp(p, attr); -+ if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -+ __setscheduler_params(p, attr); -+ __setscheduler_prio(p, newprio); -+ trace_android_rvh_setscheduler(p); -+ } -+ __setscheduler_uclamp(p, attr); - -+ if (queued) { - /* - * We enqueue to tail when the priority of a task is - * increased (user space view). -@@ -8064,7 +8172,10 @@ static int __sched_setscheduler(struct task_struct *p, - if (oldprio < p->prio) - queue_flags |= ENQUEUE_HEAD; - -+ enqueue_task(rq, p, queue_flags); - } -+ if (running) -+ set_next_task(rq, p); - - check_class_changed(rq, p, prev_class, oldprio); - -@@ -8072,7 +8183,9 @@ static int __sched_setscheduler(struct task_struct *p, - preempt_disable(); - head = splice_balance_callbacks(rq); - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (pi) { - if (cpuset_locked) - cpuset_unlock(); -@@ -8087,7 +8200,9 @@ static int __sched_setscheduler(struct task_struct *p, - - unlock: - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - return retval; -@@ -8785,6 +8900,9 @@ static void do_sched_yield(void) - long skip = 0; - - trace_android_rvh_before_do_sched_yield(&skip); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_skip_yield(&skip); -+#endif - if (skip) - return; - -@@ -9309,7 +9427,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - break; - } -@@ -9337,7 +9457,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - } - return ret; -@@ -9638,15 +9760,25 @@ int migrate_task_to(struct task_struct *p, int target_cpu) - */ - void sched_setnuma(struct task_struct *p, int nid) - { -+ bool queued, running; - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) { -- p->numa_preferred_nid = nid; -- } -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE); -+ if (running) -+ put_prev_task(rq, p); - -+ p->numa_preferred_nid = nid; -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - task_rq_unlock(rq, p, &rf); - } - #endif /* CONFIG_NUMA_BALANCING */ -@@ -10212,17 +10344,23 @@ void __init sched_init(void) - int i; - - /* Make sure the linker didn't screw up */ -+#ifdef CONFIG_HMBIRD_SCHED - #ifdef CONFIG_SMP -- BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+#endif -+ WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class)); -+#else -+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 || -+ &fair_sched_class != &rt_sched_class + 1 || -+ &rt_sched_class != &dl_sched_class + 1); -+#ifdef CONFIG_SMP -+ BUG_ON(&dl_sched_class != &stop_sched_class + 1); - #endif -- BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); -- BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); -- BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); --#ifdef CONFIG_SCHED_CLASS_EXT -- BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); -- BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); - #endif -- - wait_bit_init(); - - #ifdef CONFIG_FAIR_GROUP_SCHED -@@ -10392,8 +10530,9 @@ void __init sched_init(void) - balance_push_set(smp_processor_id(), false); - #endif - init_sched_fair_class(); -- init_sched_ext_class(); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ init_sched_hmbird_class(); -+#endif - psi_init(); - - init_uclamp(); -@@ -10863,6 +11002,11 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) - struct task_group *tg = css_tg(css); - struct task_group *parent = css_tg(css->parent); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_tg_online(tg); -+#endif -+ -+ - if (parent) - sched_online_group(tg, parent); - -@@ -10912,13 +11056,77 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) - } - #endif - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 dl) -+{ -+ int i; -+ -+ for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) { -+ if (dl < HMBIRD_BPF_DSQS_DEADLINE[i]) -+ break; -+ } -+ -+ i = max_t(int, i-1, MIN_CGROUP_DL_IDX); -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return 0; -+ update_cgroup_ids_table(css->cgroup->kn->id, i); -+ -+ return 0; -+} -+ -+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ u8 i; -+ -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX]; -+ i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1); -+ if (i < 0) { -+ pr_err(" <%s> i is %d, less than 0, name is %s\n", -+ __func__, i, css->cgroup->kn->name); -+ i = DEFAULT_CGROUP_DL_IDX; -+ } -+ -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[i]; -+} -+ -+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css, -+ struct task_struct *p, int prio) -+{ -+ if (!css || !rt_prio(prio)) -+ return; -+ -+ if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) { -+ if (!strcmp(css->cgroup->kn->name, "display")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3); -+ else if (!strcmp(css->cgroup->kn->name, "touch")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2); -+ else if (!strcmp(css->cgroup->kn->name, "multimedia")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+ } -+} -+#endif -+ - static void cpu_cgroup_attach(struct cgroup_taskset *tset) - { - struct task_struct *task; - struct cgroup_subsys_state *css; - - cgroup_taskset_for_each(task, css, tset) { -- -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_change_rt_sched_prop(css, task, task->prio); -+#endif - sched_move_task(task); - } - -@@ -11600,7 +11808,13 @@ static struct cftype cpu_legacy_files[] = { - .write_u64 = cpu_uclamp_ls_write_u64, - }, - #endif -- -+#ifdef CONFIG_HMBIRD_SCHED -+ { -+ .name = "hmbird.deadline", -+ .read_u64 = cgroup_read_hmbird_deadline, -+ .write_u64 = cgroup_write_hmbird_deadline, -+ }, -+#endif - { } /* Terminate */ - }; - -@@ -11649,7 +11863,6 @@ static int cpu_local_stat_show(struct seq_file *sf, - } - - #ifdef CONFIG_FAIR_GROUP_SCHED -- - static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, - struct cftype *cft) - { -diff --git b/kernel/sched/debug.c a/kernel/sched/debug.c -index a6c99df9edf9..e09904f4d6e5 100755 ---- b/kernel/sched/debug.c -+++ a/kernel/sched/debug.c -@@ -375,9 +375,8 @@ static __init int sched_init_debug(void) - #endif - - debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); -- --#ifdef CONFIG_SCHED_CLASS_EXT -- debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops); -+#ifdef CONFIG_HMBIRD_SCHED -+ debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops); - #endif - return 0; - } -@@ -1006,9 +1005,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - SEQ_printf(m, - "---------------------------------------------------------" - "----------\n"); --#ifdef CONFIG_SLIM_SCHED -- SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop); --#endif - - #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) - #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) -@@ -1104,8 +1100,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - } else if (fair_policy(p->policy)) { - P(se.slice); - } --#ifdef CONFIG_SCHED_CLASS_EXT -- __PS("ext.enabled", p->sched_class == &ext_sched_class); -+#ifdef CONFIG_HMBIRD_SCHED -+ __PS("hmbird.enabled", p->sched_class == &hmbird_sched_class); -+ __PS("sched_prop", get_hmbird_ts(p)->sched_prop); -+ __PS("top_task_prop", get_hmbird_ts(p)->top_task_prop); - #endif - #undef PN_SCHEDSTAT - #undef P_SCHEDSTAT -diff --git b/kernel/sched/ext.c a/kernel/sched/ext.c -deleted file mode 100755 -index 4845d441df2b..000000000000 ---- b/kernel/sched/ext.c -+++ /dev/null -@@ -1,3978 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -- --enum scx_internal_consts { -- SCX_NR_ONLINE_OPS = SCX_OP_IDX(init), -- SCX_DSP_DFL_MAX_BATCH = 32, -- SCX_DSP_MAX_LOOPS = 32, -- SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, --}; -- --enum scx_ops_enable_state { -- SCX_OPS_PREPPING, -- SCX_OPS_ENABLING, -- SCX_OPS_ENABLED, -- SCX_OPS_DISABLING, -- SCX_OPS_DISABLED, --}; -- --static inline struct task_group *css_tg(struct cgroup_subsys_state *css) --{ -- return css ? container_of(css, struct task_group, css) : NULL; --} -- --/* -- * sched_ext_entity->ops_state -- * -- * Used to track the task ownership between the SCX core and the BPF scheduler. -- * State transitions look as follows: -- * -- * NONE -> QUEUEING -> QUEUED -> DISPATCHING -- * ^ | | -- * | v v -- * \-------------------------------/ -- * -- * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -- * sites for explanations on the conditions being waited upon and why they are -- * safe. Transitions out of them into NONE or QUEUED must store_release and the -- * waiters should load_acquire. -- * -- * Tracking scx_ops_state enables sched_ext core to reliably determine whether -- * any given task can be dispatched by the BPF scheduler at all times and thus -- * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -- * to try to dispatch any task anytime regardless of its state as the SCX core -- * can safely reject invalid dispatches. -- */ --enum scx_ops_state { -- SCX_OPSS_NONE, /* owned by the SCX core */ -- SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -- SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ -- SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ -- -- /* -- * QSEQ brands each QUEUED instance so that, when dispatch races -- * dequeue/requeue, the dispatcher can tell whether it still has a claim -- * on the task being dispatched. -- */ -- SCX_OPSS_QSEQ_SHIFT = 2, -- SCX_OPSS_STATE_MASK = (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1, -- SCX_OPSS_QSEQ_MASK = ~SCX_OPSS_STATE_MASK, --}; -- --/* -- * During exit, a task may schedule after losing its PIDs. When disabling the -- * BPF scheduler, we need to be able to iterate tasks in every state to -- * guarantee system safety. Maintain a dedicated task list which contains every -- * task between its fork and eventual free. -- */ --static DEFINE_SPINLOCK(scx_tasks_lock); --static LIST_HEAD(scx_tasks); -- --/* ops enable/disable */ --static struct kthread_worker *scx_ops_helper; --static DEFINE_MUTEX(scx_ops_enable_mutex); --DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); --DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); --static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); --static bool scx_switch_all_req; --static bool scx_switching_all; --DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -- --static struct sched_ext_ops scx_ops; --static bool scx_warned_zero_slice; -- --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); --DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); --static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); -- --struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] = -- { [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT }; -- --static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE); --static struct scx_exit_info scx_exit_info; -- --static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0); -- --/* -- * The maximum amount of time in jiffies that a task may be runnable without -- * being scheduled on a CPU. If this timeout is exceeded, it will trigger -- * scx_ops_error(). -- */ --unsigned long scx_watchdog_timeout; -- --/* -- * The last time the delayed work was run. This delayed work relies on -- * ksoftirqd being able to run to service timer interrupts, so it's possible -- * that this work itself could get wedged. To account for this, we check that -- * it's not stalled in the timer tick, and trigger an error if it is. -- */ --unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; -- --static struct delayed_work scx_watchdog_work; -- --/* idle tracking */ --#ifdef CONFIG_SMP --#ifdef CONFIG_CPUMASK_OFFSTACK --#define CL_ALIGNED_IF_ONSTACK --#else --#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp --#endif -- --static struct { -- cpumask_var_t cpu; -- cpumask_var_t smt; --} idle_masks CL_ALIGNED_IF_ONSTACK; -- --static bool __cacheline_aligned_in_smp scx_has_idle_cpus; --#endif /* CONFIG_SMP */ -- --/* for %SCX_KICK_WAIT */ --static u64 __percpu *scx_kick_cpus_pnt_seqs; -- --/* -- * Direct dispatch marker. -- * -- * Non-NULL values are used for direct dispatch from enqueue path. A valid -- * pointer points to the task currently being enqueued. An ERR_PTR value is used -- * to indicate that direct dispatch has already happened. -- */ --static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); -- --/* dispatch queues */ --static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; -- --static LLIST_HEAD(dsqs_to_free); -- --/* dispatch buf */ --struct scx_dsp_buf_ent { -- struct task_struct *task; -- u64 qseq; -- u64 dsq_id; -- u64 enq_flags; --}; -- --static u32 scx_dsp_max_batch; --static struct scx_dsp_buf_ent __percpu *scx_dsp_buf; -- --struct scx_dsp_ctx { -- struct rq *rq; -- struct rq_flags *rf; -- u32 buf_cursor; -- u32 nr_tasks; --}; -- --static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx); -- --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags); --void scx_bpf_kick_cpu(s32 cpu, u64 flags); -- --struct scx_task_iter { -- struct sched_ext_entity cursor; -- struct task_struct *locked; -- struct rq *rq; -- struct rq_flags rf; --}; -- --#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) -- --/* if the highest set bit is N, return a mask with bits [N+1, 31] set */ --static u32 higher_bits(u32 flags) --{ -- return ~((1 << fls(flags)) - 1); --} -- --/* return the mask with only the highest bit set */ --static u32 highest_bit(u32 flags) --{ -- int bit = fls(flags); -- return bit ? 1 << (bit - 1) : 0; --} -- --/* -- * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX -- * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate -- * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check -- * whether it's running from an allowed context. -- * -- * @mask is constant, always inline to cull the mask calculations. -- */ --static __always_inline void scx_kf_allow(u32 mask) --{ -- /* nesting is allowed only in increasing scx_kf_mask order */ -- WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask, -- "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n", -- current->scx->kf_mask, mask); -- current->scx->kf_mask |= mask; --} -- --static void scx_kf_disallow(u32 mask) --{ -- current->scx->kf_mask &= ~mask; --} -- --#define SCX_CALL_OP(mask, op, args...) \ --do { \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- scx_ops.op(args); \ -- } \ --} while (0) -- --#define SCX_CALL_OP_RET(mask, op, args...) \ --({ \ -- __typeof__(scx_ops.op(args)) __ret; \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- __ret = scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- __ret = scx_ops.op(args); \ -- } \ -- __ret; \ --}) -- --/* -- * Some kfuncs are allowed only on the tasks that are subjects of the -- * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such -- * restrictions, the following SCX_CALL_OP_*() variants should be used when -- * invoking scx_ops operations that take task arguments. These can only be used -- * for non-nesting operations due to the way the tasks are tracked. -- * -- * kfuncs which can only operate on such tasks can in turn use -- * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on -- * the specific task. -- */ --#define SCX_CALL_OP_TASK(mask, op, task, args...) \ --do { \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- SCX_CALL_OP(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ --} while (0) -- --#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ --({ \ -- __typeof__(scx_ops.op(task, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- __ret; \ --}) -- --#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ --({ \ -- __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task0; \ -- current->scx->kf_tasks[1] = task1; \ -- __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- current->scx->kf_tasks[1] = NULL; \ -- __ret; \ --}) -- --//#include "./slim_walt.c" -- --/* @mask is constant, always inline to cull unnecessary branches */ --static __always_inline bool scx_kf_allowed(u32 mask) --{ -- if (unlikely(!(current->scx->kf_mask & mask))) { -- scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", -- mask, current->scx->kf_mask); -- return false; -- } -- -- if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) && -- in_interrupt())) { -- scx_ops_error("sleepable kfunc called from non-sleepable context"); -- return false; -- } -- -- /* -- * Enforce nesting boundaries. e.g. A kfunc which can be called from -- * DISPATCH must not be called if we're running DEQUEUE which is nested -- * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE -- * boundary thanks to the above in_interrupt() check. -- */ -- if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && -- (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { -- scx_ops_error("cpu_release kfunc called from a nested operation"); -- return false; -- } -- -- if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && -- (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) { -- scx_ops_error("dispatch kfunc called from a nested operation"); -- return false; -- } -- -- return true; --} -- --/* see SCX_CALL_OP_TASK() */ --static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, -- struct task_struct *p) --{ -- if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED)) -- return false; -- -- if (unlikely((p != current->scx->kf_tasks[0] && -- p != current->scx->kf_tasks[1]))) { -- scx_ops_error("called on a task not being operated on"); -- return false; -- } -- -- return true; --} -- --/** -- * scx_task_iter_init - Initialize a task iterator -- * @iter: iterator to init -- * -- * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized, -- * @iter must eventually be exited with scx_task_iter_exit(). -- * -- * scx_tasks_lock may be released between this and the first next() call or -- * between any two next() calls. If scx_tasks_lock is released between two -- * next() calls, the caller is responsible for ensuring that the task being -- * iterated remains accessible either through RCU read lock or obtaining a -- * reference count. -- * -- * All tasks which existed when the iteration started are guaranteed to be -- * visited as long as they still exist. -- */ --static void scx_task_iter_init(struct scx_task_iter *iter) --{ -- lockdep_assert_held(&scx_tasks_lock); -- -- iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; -- list_add(&iter->cursor.tasks_node, &scx_tasks); -- iter->locked = NULL; --} -- --/** -- * scx_task_iter_exit - Exit a task iterator -- * @iter: iterator to exit -- * -- * Exit a previously initialized @iter. Must be called with scx_tasks_lock held. -- * If the iterator holds a task's rq lock, that rq lock is released. See -- * scx_task_iter_init() for details. -- */ --static void scx_task_iter_exit(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- if (list_empty(cursor)) -- return; -- -- list_del_init(cursor); --} -- --/** -- * scx_task_iter_next - Next task -- * @iter: iterator to walk -- * -- * Visit the next task. See scx_task_iter_init() for details. -- */ --static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- struct sched_ext_entity *pos; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- list_for_each_entry(pos, cursor, tasks_node) { -- if (&pos->tasks_node == &scx_tasks) -- return NULL; -- if (!(pos->flags & SCX_TASK_CURSOR)) { -- list_move(cursor, &pos->tasks_node); -- return pos->task; -- } -- } -- -- /* can't happen, should always terminate at scx_tasks above */ -- BUG(); --} -- --/** -- * scx_task_iter_next_filtered - Next non-idle task -- * @iter: iterator to walk -- * -- * Visit the next non-idle task. See scx_task_iter_init() for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- while ((p = scx_task_iter_next(iter))) { -- if (!is_idle_task(p)) -- return p; -- } -- return NULL; --} -- --/** -- * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked -- * @iter: iterator to walk -- * -- * Visit the next non-idle task with its rq lock held. See scx_task_iter_init() -- * for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered_locked(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- p = scx_task_iter_next_filtered(iter); -- if (!p) -- return NULL; -- -- iter->rq = task_rq_lock(p, &iter->rf); -- iter->locked = p; -- return p; --} -- --static enum scx_ops_enable_state scx_ops_enable_state(void) --{ -- return atomic_read(&scx_ops_enable_state_var); --} -- --static enum scx_ops_enable_state --scx_ops_set_enable_state(enum scx_ops_enable_state to) --{ -- return atomic_xchg(&scx_ops_enable_state_var, to); --} -- --static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, -- enum scx_ops_enable_state from) --{ -- int from_v = from; -- -- return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); --} -- --static bool scx_ops_disabling(void) --{ -- return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING); --} -- --/** -- * wait_ops_state - Busy-wait the specified ops state to end -- * @p: target task -- * @opss: state to wait the end of -- * -- * Busy-wait for @p to transition out of @opss. This can only be used when the -- * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also -- * has load_acquire semantics to ensure that the caller can see the updates made -- * in the enqueueing and dispatching paths. -- */ --static void wait_ops_state(struct task_struct *p, u64 opss) --{ -- do { -- cpu_relax(); -- } while (atomic64_read_acquire(&p->scx->ops_state) == opss); --} -- --/** -- * ops_cpu_valid - Verify a cpu number -- * @cpu: cpu number which came from a BPF ops -- * -- * @cpu is a cpu number which came from the BPF scheduler and can be any value. -- * Verify that it is in range and one of the possible cpus. -- */ --static bool ops_cpu_valid(s32 cpu) --{ -- return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); --} -- --/** -- * ops_sanitize_err - Sanitize a -errno value -- * @ops_name: operation to blame on failure -- * @err: -errno value to sanitize -- * -- * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return -- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can -- * cause misbehaviors. For an example, a large negative return from -- * ops.prep_enable() triggers an oops when passed up the call chain because the -- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is -- * handled as a pointer. -- */ --static int ops_sanitize_err(const char *ops_name, s32 err) --{ -- if (err < 0 && err >= -MAX_ERRNO) -- return err; -- -- scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); -- return -EPROTO; --} -- --/** -- * touch_core_sched - Update timestamp used for core-sched task ordering -- * @rq: rq to read clock from, must be locked -- * @p: task to update the timestamp for -- * -- * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to -- * implement global or local-DSQ FIFO ordering for core-sched. Should be called -- * when a task becomes runnable and its turn on the CPU ends (e.g. slice -- * exhaustion). -- */ --static void touch_core_sched(struct rq *rq, struct task_struct *p) --{ --#ifdef CONFIG_SCHED_CORE -- /* -- * It's okay to update the timestamp spuriously. Use -- * sched_core_disabled() which is cheaper than enabled(). -- */ -- if (!sched_core_disabled()) -- p->scx->core_sched_at = rq_clock_task(rq); --#endif --} -- --/** -- * touch_core_sched_dispatch - Update core-sched timestamp on dispatch -- * @rq: rq to read clock from, must be locked -- * @p: task being dispatched -- * -- * If the BPF scheduler implements custom core-sched ordering via -- * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO -- * ordering within each local DSQ. This function is called from dispatch paths -- * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect. -- */ --static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- assert_clock_updated(rq); -- --#ifdef CONFIG_SCHED_CORE -- if (SCX_HAS_OP(core_sched_before)) -- touch_core_sched(rq, p); --#endif --} -- --static void update_curr_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- u64 now = rq_clock_task(rq); -- u64 delta_exec; -- -- if (time_before_eq64(now, curr->se.exec_start)) -- return; -- -- delta_exec = now - curr->se.exec_start; -- curr->se.exec_start = now; -- curr->se.sum_exec_runtime += delta_exec; -- account_group_exec_runtime(curr, delta_exec); -- cgroup_account_cputime(curr, delta_exec); -- -- if (curr->scx->slice != SCX_SLICE_INF) { -- curr->scx->slice -= min(curr->scx->slice, delta_exec); -- if (!curr->scx->slice) -- touch_core_sched(rq, curr); -- } --} -- --static bool scx_dsq_priq_less(struct rb_node *node_a, -- const struct rb_node *node_b) --{ -- const struct sched_ext_entity *a = -- container_of(node_a, struct sched_ext_entity, dsq_node.priq); -- const struct sched_ext_entity *b = -- container_of(node_b, struct sched_ext_entity, dsq_node.priq); -- -- return time_before64(a->dsq_vtime, b->dsq_vtime); --} -- --static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, -- u64 enq_flags) --{ -- bool is_local = dsq->id == SCX_DSQ_LOCAL; -- -- WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo)); -- WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq)); -- -- if (!is_local) { -- raw_spin_lock(&dsq->lock); -- if (unlikely(dsq->id == SCX_DSQ_INVALID)) { -- scx_ops_error("attempting to dispatch to a destroyed dsq"); -- /* fall back to the global dsq */ -- raw_spin_unlock(&dsq->lock); -- dsq = &scx_dsq_global; -- raw_spin_lock(&dsq->lock); -- } -- } -- -- if (enq_flags & SCX_ENQ_DSQ_PRIQ) { -- p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; -- rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq, -- scx_dsq_priq_less); -- } else { -- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) -- list_add(&p->scx->dsq_node.fifo, &dsq->fifo); -- else -- list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo); -- } -- dsq->nr++; -- p->scx->dsq = dsq; -- -- /* -- * We're transitioning out of QUEUEING or DISPATCHING. store_release to -- * match waiters' load_acquire. -- */ -- if (enq_flags & SCX_ENQ_CLEAR_OPSS) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- if (is_local) { -- struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq); -- struct rq *rq = scx->rq; -- bool preempt = false; -- -- if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && -- rq->curr->sched_class == &ext_sched_class) { -- rq->curr->scx->slice = 0; -- preempt = true; -- } -- -- if (preempt || sched_class_above(&ext_sched_class, -- rq->curr->sched_class)) -- resched_curr(rq); -- } else { -- raw_spin_unlock(&dsq->lock); -- } --} -- --static void task_unlink_from_dsq(struct task_struct *p, -- struct scx_dispatch_q *dsq) --{ -- if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { -- rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; -- } else { -- list_del_init(&p->scx->dsq_node.fifo); -- } -- --} -- --static bool task_linked_on_dsq(struct task_struct *p) --{ -- return !list_empty(&p->scx->dsq_node.fifo) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq); --} -- --static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p) --{ -- struct scx_dispatch_q *dsq = p->scx->dsq; -- bool is_local = dsq == &scx_rq->local_dsq; -- -- if (!dsq) { -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- /* -- * When dispatching directly from the BPF scheduler to a local -- * DSQ, the task isn't associated with any DSQ but -- * @p->scx->holding_cpu may be set under the protection of -- * %SCX_OPSS_DISPATCHING. -- */ -- if (p->scx->holding_cpu >= 0) -- p->scx->holding_cpu = -1; -- return; -- } -- -- if (!is_local) -- raw_spin_lock(&dsq->lock); -- -- /* -- * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node -- * can't change underneath us. -- */ -- if (p->scx->holding_cpu < 0) { -- /* @p must still be on @dsq, dequeue */ -- WARN_ON_ONCE(!task_linked_on_dsq(p)); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- } else { -- /* -- * We're racing against dispatch_to_local_dsq() which already -- * removed @p from @dsq and set @p->scx->holding_cpu. Clear the -- * holding_cpu which tells dispatch_to_local_dsq() that it lost -- * the race. -- */ -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- p->scx->holding_cpu = -1; -- } -- p->scx->dsq = NULL; -- -- if (!is_local) -- raw_spin_unlock(&dsq->lock); --} -- --static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) --{ -- lockdep_assert(rcu_read_lock_any_held()); -- -- return &scx_dsq_global; --} -- --static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, -- struct task_struct *p) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id == SCX_DSQ_LOCAL) -- return &rq->scx->local_dsq; -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", -- dsq_id, p->comm, p->pid); -- return &scx_dsq_global; -- } -- -- return dsq; --} -- --static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- -- /* @p must match the task which is being enqueued */ -- if (unlikely(p != ddsp_task)) { -- if (IS_ERR(ddsp_task)) -- scx_ops_error("%s[%d] already direct-dispatched", -- p->comm, p->pid); -- else -- scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]", -- ddsp_task->comm, ddsp_task->pid, -- p->comm, p->pid); -- return; -- } -- -- /* -- * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because -- * dispatching to the local DSQ of a different CPU requires unlocking -- * the current rq which isn't allowed in the enqueue path. Use -- * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL. -- */ -- if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) { -- scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch"); -- return; -- } -- -- touch_core_sched_dispatch(task_rq(p), p); -- -- dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- -- /* -- * Mark that dispatch already happened by spoiling direct_dispatch_task -- * with a non-NULL value which can never match a valid task pointer. -- */ -- __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); --} -- --static bool test_rq_online(struct rq *rq) --{ --#ifdef CONFIG_SMP -- return rq->online; --#else -- return true; --#endif --} -- --static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- int sticky_cpu) --{ -- struct task_struct **ddsp_taskp; -- u64 qseq; -- -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- if (p->scx->flags & SCX_TASK_ENQ_LOCAL) { -- enq_flags |= SCX_ENQ_LOCAL; -- p->scx->flags &= ~SCX_TASK_ENQ_LOCAL; -- } -- -- /* rq migration */ -- if (sticky_cpu == cpu_of(rq)) -- goto local_norefill; -- -- /* -- * If !rq->online, we already told the BPF scheduler that the CPU is -- * offline. We're just trying to on/offline the CPU. Don't bother the -- * BPF scheduler. -- */ -- if (unlikely(!test_rq_online(rq))) -- goto local; -- -- /* see %SCX_OPS_ENQ_EXITING */ -- if (!static_branch_unlikely(&scx_ops_enq_exiting) && -- unlikely(p->flags & PF_EXITING)) -- goto local; -- -- /* see %SCX_OPS_ENQ_LAST */ -- if (!static_branch_unlikely(&scx_ops_enq_last) && -- (enq_flags & SCX_ENQ_LAST)) -- goto local; -- -- if (!SCX_HAS_OP(enqueue)) { -- if (enq_flags & SCX_ENQ_LOCAL) -- goto local; -- else -- goto global; -- } -- -- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ -- qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; -- -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq); -- -- ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); -- WARN_ON_ONCE(*ddsp_taskp); -- *ddsp_taskp = p; -- -- SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); -- -- /* -- * If not directly dispatched, QUEUEING isn't clear yet and dispatch or -- * dequeue may be waiting. The store_release matches their load_acquire. -- */ -- if (*ddsp_taskp == p) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq); -- *ddsp_taskp = NULL; -- return; -- --local: -- /* -- * For task-ordering, slice refill must be treated as implying the end -- * of the current slice. Otherwise, the longer @p stays on the CPU, the -- * higher priority it becomes from scx_prio_less()'s POV. -- */ -- touch_core_sched(rq, p); -- p->scx->slice = SCX_SLICE_DFL; --local_norefill: -- dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags); -- return; -- --global: -- touch_core_sched(rq, p); /* see the comment in local: */ -- p->scx->slice = SCX_SLICE_DFL; -- dispatch_enqueue(&scx_dsq_global, p, enq_flags); --} -- --static bool watchdog_task_watched(const struct task_struct *p) --{ -- return !list_empty(&p->scx->watchdog_node); --} -- --static void watchdog_watch_task(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- if (p->scx->flags & SCX_TASK_WATCHDOG_RESET) -- p->scx->runnable_at = jiffies; -- p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET; -- list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list); --} -- --static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) --{ -- list_del_init(&p->scx->watchdog_node); -- if (reset_timeout) -- p->scx->flags |= SCX_TASK_WATCHDOG_RESET; --} -- --static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) --{ -- int sticky_cpu = p->scx->sticky_cpu; -- -- enq_flags |= rq->scx->extra_enq_flags; -- -- if (sticky_cpu >= 0) -- p->scx->sticky_cpu = -1; -- -- /* -- * Restoring a running task will be immediately followed by -- * set_next_task_scx() which expects the task to not be on the BPF -- * scheduler as tasks can only start running through local DSQs. Force -- * direct-dispatch into the local DSQ by setting the sticky_cpu. -- */ -- if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -- sticky_cpu = cpu_of(rq); -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- WARN_ON_ONCE(!watchdog_task_watched(p)); -- return; -- } -- -- watchdog_watch_task(rq, p); -- p->scx->flags |= SCX_TASK_QUEUED; -- rq->scx->nr_running++; -- add_nr_running(rq, 1); -- -- if (SCX_HAS_OP(runnable)) -- SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); -- -- if (enq_flags & SCX_ENQ_WAKEUP) -- touch_core_sched(rq, p); -- -- do_enqueue_task(rq, p, enq_flags, sticky_cpu); --} -- --static void ops_dequeue(struct task_struct *p, u64 deq_flags) --{ -- u64 opss; -- -- watchdog_unwatch_task(p, false); -- -- /* acquire ensures that we see the preceding updates on QUEUED */ -- opss = atomic64_read_acquire(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_NONE: -- break; -- case SCX_OPSS_QUEUEING: -- /* -- * QUEUEING is started and finished while holding @p's rq lock. -- * As we're holding the rq lock now, we shouldn't see QUEUEING. -- */ -- BUG(); -- case SCX_OPSS_QUEUED: -- if (SCX_HAS_OP(dequeue)) -- SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); -- -- if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_NONE)) -- break; -- fallthrough; -- case SCX_OPSS_DISPATCHING: -- /* -- * If @p is being dispatched from the BPF scheduler to a DSQ, -- * wait for the transfer to complete so that @p doesn't get -- * added to its DSQ after dequeueing is complete. -- * -- * As we're waiting on DISPATCHING with the rq locked, the -- * dispatching side shouldn't try to lock the rq while -- * DISPATCHING is set. See dispatch_to_local_dsq(). -- * -- * DISPATCHING shouldn't have qseq set and control can reach -- * here with NONE @opss from the above QUEUED case block. -- * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. -- */ -- wait_ops_state(p, SCX_OPSS_DISPATCHING); -- BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- break; -- } --} -- --static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) --{ -- struct scx_rq *scx_rq = rq->scx; -- -- if (!(p->scx->flags & SCX_TASK_QUEUED)) { -- WARN_ON_ONCE(watchdog_task_watched(p)); -- return; -- } -- -- ops_dequeue(p, deq_flags); -- -- /* -- * A currently running task which is going off @rq first gets dequeued -- * and then stops running. As we want running <-> stopping transitions -- * to be contained within runnable <-> quiescent transitions, trigger -- * ->stopping() early here instead of in put_prev_task_scx(). -- * -- * @p may go through multiple stopping <-> running transitions between -- * here and put_prev_task_scx() if task attribute changes occur while -- * balance_scx() leaves @rq unlocked. However, they don't contain any -- * information meaningful to the BPF scheduler and can be suppressed by -- * skipping the callbacks if the task is !QUEUED. -- */ -- if (SCX_HAS_OP(stopping) && task_current(rq, p)) { -- update_curr_scx(rq); -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); -- } -- -- //if(task_current(rq, p)) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- if (SCX_HAS_OP(quiescent)) -- SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); -- -- if (deq_flags & SCX_DEQ_SLEEP) -- p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP; -- else -- p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP; -- -- p->scx->flags &= ~SCX_TASK_QUEUED; -- BUG_ON(!scx_rq->nr_running); -- scx_rq->nr_running--; -- sub_nr_running(rq, 1); -- -- dispatch_dequeue(scx_rq, p); --} -- --static void yield_task_scx(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); -- else -- p->scx->slice = 0; --} -- --static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) --{ -- struct task_struct *from = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); -- else -- return false; --} -- --#ifdef CONFIG_SMP --/** -- * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -- * @rq: rq to move the task into, currently locked -- * @p: task to move -- * @enq_flags: %SCX_ENQ_* -- * -- * Move @p which is currently on a different rq to @rq's local DSQ. The caller -- * must: -- * -- * 1. Start with exclusive access to @p either through its DSQ lock or -- * %SCX_OPSS_DISPATCHING flag. -- * -- * 2. Set @p->scx->holding_cpu to raw_smp_processor_id(). -- * -- * 3. Remember task_rq(@p). Release the exclusive access so that we don't -- * deadlock with dequeue. -- * -- * 4. Lock @rq and the task_rq from #3. -- * -- * 5. Call this function. -- * -- * Returns %true if @p was successfully moved. %false after racing dequeue and -- * losing. -- */ --static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -- u64 enq_flags) --{ -- struct rq *task_rq; -- -- lockdep_assert_rq_held(rq); -- -- /* -- * If dequeue got to @p while we were trying to lock both rq's, it'd -- * have cleared @p->scx->holding_cpu to -1. While other cpus may have -- * updated it to different values afterwards, as this operation can't be -- * preempted or recurse, @p->scx->holding_cpu can never become -- * raw_smp_processor_id() again before we're done. Thus, we can tell -- * whether we lost to dequeue by testing whether @p->scx->holding_cpu is -- * still raw_smp_processor_id(). -- * -- * See dispatch_dequeue() for the counterpart. -- */ -- if (unlikely(p->scx->holding_cpu != raw_smp_processor_id())) -- return false; -- -- /* @p->rq couldn't have changed if we're still the holding cpu */ -- task_rq = task_rq(p); -- lockdep_assert_rq_held(task_rq); -- -- WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)); -- deactivate_task(task_rq, p, 0); -- set_task_cpu(p, cpu_of(rq)); -- p->scx->sticky_cpu = cpu_of(rq); -- -- /* -- * We want to pass scx-specific enq_flags but activate_task() will -- * truncate the upper 32 bit. As we own @rq, we can pass them through -- * @rq->scx->extra_enq_flags instead. -- */ -- WARN_ON_ONCE(rq->scx->extra_enq_flags); -- rq->scx->extra_enq_flags = enq_flags; -- activate_task(rq, p, 0); -- rq->scx->extra_enq_flags = 0; -- -- return true; --} -- --/** -- * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * We're holding @rq lock and trying to dispatch a task from @src_rq to -- * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether -- * @rq stays locked isn't important as long as the state is restored after -- * dispatch_to_local_dsq_unlock(). -- */ --static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- rq_unpin_lock(rq, rf); -- -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(rq); -- raw_spin_rq_lock(dst_rq); -- } else if (rq == src_rq) { -- double_lock_balance(rq, dst_rq); -- rq_repin_lock(rq, rf); -- } else if (rq == dst_rq) { -- double_lock_balance(rq, src_rq); -- rq_repin_lock(rq, rf); -- } else { -- raw_spin_rq_unlock(rq); -- double_rq_lock(src_rq, dst_rq); -- } --} -- --/** -- * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock() -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return. -- */ --static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } else if (rq == src_rq) { -- double_unlock_balance(rq, dst_rq); -- } else if (rq == dst_rq) { -- double_unlock_balance(rq, src_rq); -- } else { -- double_rq_unlock(src_rq, dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } --} --#endif /* CONFIG_SMP */ -- -- --static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq) --{ -- return likely(test_rq_online(rq)) && !is_migration_disabled(p) && -- cpumask_test_cpu(cpu_of(rq), p->cpus_ptr); --} -- --static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -- struct scx_dispatch_q *dsq) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rb_node *rb_node; -- struct rq *task_rq; -- bool moved = false; --retry: -- if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -- return false; -- -- raw_spin_lock(&dsq->lock); -- -- list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- for (rb_node = rb_first_cached(&dsq->priq); rb_node; -- rb_node = rb_next(rb_node)) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- raw_spin_unlock(&dsq->lock); -- return false; -- --this_rq: -- /* @dsq is locked and @p is on this rq */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo); -- dsq->nr--; -- scx_rq->local_dsq.nr++; -- p->scx->dsq = &scx_rq->local_dsq; -- raw_spin_unlock(&dsq->lock); -- return true; -- --remote_rq: --#ifdef CONFIG_SMP -- /* -- * @dsq is locked and @p is on a remote rq. @p is currently protected by -- * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -- * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -- * rq lock or fail, do a little dancing from our side. See -- * move_task_to_local_dsq(). -- */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- p->scx->holding_cpu = raw_smp_processor_id(); -- raw_spin_unlock(&dsq->lock); -- -- rq_unpin_lock(rq, rf); -- double_lock_balance(rq, task_rq); -- rq_repin_lock(rq, rf); -- -- moved = move_task_to_local_dsq(rq, p, 0); -- -- double_unlock_balance(rq, task_rq); --#endif /* CONFIG_SMP */ -- if (likely(moved)) -- return true; -- goto retry; --} -- --enum dispatch_to_local_dsq_ret { -- DTL_DISPATCHED, /* successfully dispatched */ -- DTL_LOST, /* lost race to dequeue */ -- DTL_NOT_LOCAL, /* destination is not a local DSQ */ -- DTL_INVALID, /* invalid local dsq_id */ --}; -- --/** -- * dispatch_to_local_dsq - Dispatch a task to a local dsq -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @dsq_id: destination dsq ID -- * @p: task to dispatch -- * @enq_flags: %SCX_ENQ_* -- * -- * We're holding @rq lock and want to dispatch @p to the local DSQ identified by -- * @dsq_id. This function performs all the synchronization dancing needed -- * because local DSQs are protected with rq locks. -- * -- * The caller must have exclusive ownership of @p (e.g. through -- * %SCX_OPSS_DISPATCHING). -- */ --static enum dispatch_to_local_dsq_ret --dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id, -- struct task_struct *p, u64 enq_flags) --{ -- struct rq *src_rq = task_rq(p); -- struct rq *dst_rq; -- -- /* -- * We're synchronized against dequeue through DISPATCHING. As @p can't -- * be dequeued, its task_rq and cpus_allowed are stable too. -- */ -- if (dsq_id == SCX_DSQ_LOCAL) { -- dst_rq = rq; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]", -- cpu, p->comm, p->pid); -- return DTL_INVALID; -- } -- dst_rq = cpu_rq(cpu); -- } else { -- return DTL_NOT_LOCAL; -- } -- -- /* if dispatching to @rq that @p is already on, no lock dancing needed */ -- if (rq == src_rq && rq == dst_rq) { -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags | SCX_ENQ_CLEAR_OPSS); -- return DTL_DISPATCHED; -- } -- --#ifdef CONFIG_SMP -- if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { -- struct rq *locked_dst_rq = dst_rq; -- bool dsp; -- -- /* -- * @p is on a possibly remote @src_rq which we need to lock to -- * move the task. If dequeue is in progress, it'd be locking -- * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq -- * lock while holding DISPATCHING. -- * -- * As DISPATCHING guarantees that @p is wholly ours, we can -- * pretend that we're moving from a DSQ and use the same -- * mechanism - mark the task under transfer with holding_cpu, -- * release DISPATCHING and then follow the same protocol. -- */ -- p->scx->holding_cpu = raw_smp_processor_id(); -- -- /* store_release ensures that dequeue sees the above */ -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq); -- -- /* -- * We don't require the BPF scheduler to avoid dispatching to -- * offline CPUs mostly for convenience but also because CPUs can -- * go offline between scx_bpf_dispatch() calls and here. If @p -- * is destined to an offline CPU, queue it on its current CPU -- * instead, which should always be safe. As this is an allowed -- * behavior, don't trigger an ops error. -- */ -- if (unlikely(!test_rq_online(dst_rq))) -- dst_rq = src_rq; -- -- if (src_rq == dst_rq) { -- /* -- * As @p is staying on the same rq, there's no need to -- * go through the full deactivate/activate cycle. -- * Optimize by abbreviating the operations in -- * move_task_to_local_dsq(). -- */ -- dsp = p->scx->holding_cpu == raw_smp_processor_id(); -- if (likely(dsp)) { -- p->scx->holding_cpu = -1; -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags); -- } -- } else { -- dsp = move_task_to_local_dsq(dst_rq, p, enq_flags); -- } -- -- /* if the destination CPU is idle, wake it up */ -- if (dsp && p->sched_class > dst_rq->curr->sched_class) -- resched_curr(dst_rq); -- -- dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq); -- -- return dsp ? DTL_DISPATCHED : DTL_LOST; -- } --#endif /* CONFIG_SMP */ -- -- scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", -- cpu_of(dst_rq), p->comm, p->pid); -- return DTL_INVALID; --} -- --/** -- * finish_dispatch - Asynchronously finish dispatching a task -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @p: task to finish dispatching -- * @qseq_at_dispatch: qseq when @p started getting dispatched -- * @dsq_id: destination DSQ ID -- * @enq_flags: %SCX_ENQ_* -- * -- * Dispatching to local DSQs may need to wait for queueing to complete or -- * require rq lock dancing. As we don't wanna do either while inside -- * ops.dispatch() to avoid locking order inversion, we split dispatching into -- * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the -- * task and its qseq. Once ops.dispatch() returns, this function is called to -- * finish up. -- * -- * There is no guarantee that @p is still valid for dispatching or even that it -- * was valid in the first place. Make sure that the task is still owned by the -- * BPF scheduler and claim the ownership before dispatching. -- */ --static void finish_dispatch(struct rq *rq, struct rq_flags *rf, -- struct task_struct *p, u64 qseq_at_dispatch, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- u64 opss; -- -- touch_core_sched_dispatch(rq, p); --retry: -- /* -- * No need for _acquire here. @p is accessed only after a successful -- * try_cmpxchg to DISPATCHING. -- */ -- opss = atomic64_read(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_DISPATCHING: -- case SCX_OPSS_NONE: -- /* someone else already got to it */ -- return; -- case SCX_OPSS_QUEUED: -- /* -- * If qseq doesn't match, @p has gone through at least one -- * dispatch/dequeue and re-enqueue cycle between -- * scx_bpf_dispatch() and here and we have no claim on it. -- */ -- if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) -- return; -- -- /* -- * While we know @p is accessible, we don't yet have a claim on -- * it - the BPF scheduler is allowed to dispatch tasks -- * spuriously and there can be a racing dequeue attempt. Let's -- * claim @p by atomically transitioning it from QUEUED to -- * DISPATCHING. -- */ -- if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_DISPATCHING))) -- break; -- goto retry; -- case SCX_OPSS_QUEUEING: -- /* -- * do_enqueue_task() is in the process of transferring the task -- * to the BPF scheduler while holding @p's rq lock. As we aren't -- * holding any kernel or BPF resource that the enqueue path may -- * depend upon, it's safe to wait. -- */ -- wait_ops_state(p, opss); -- goto retry; -- } -- -- BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) { -- case DTL_DISPATCHED: -- break; -- case DTL_LOST: -- break; -- case DTL_INVALID: -- dsq_id = SCX_DSQ_GLOBAL; -- fallthrough; -- case DTL_NOT_LOCAL: -- dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()), -- dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- break; -- } --} -- --static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- u32 u; -- -- for (u = 0; u < dspc->buf_cursor; u++) { -- struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u]; -- -- finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id, -- ent->enq_flags); -- } -- -- dspc->nr_tasks += dspc->buf_cursor; -- dspc->buf_cursor = 0; --} -- --static int balance_one(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf, bool local) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- bool prev_on_scx = prev->sched_class == &ext_sched_class; -- int nr_loops = SCX_DSP_MAX_LOOPS; -- -- lockdep_assert_rq_held(rq); -- -- if (static_branch_unlikely(&scx_ops_cpu_preempt) && -- unlikely(rq->scx->cpu_released)) { -- /* -- * If the previous sched_class for the current CPU was not SCX, -- * notify the BPF scheduler that it again has control of the -- * core. This callback complements ->cpu_release(), which is -- * emitted in scx_notify_pick_next_task(). -- */ -- if (SCX_HAS_OP(cpu_acquire)) -- SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq), -- NULL); -- rq->scx->cpu_released = false; -- } -- -- if (prev_on_scx) { -- WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP)); -- update_curr_scx(rq); -- -- /* -- * If @prev is runnable & has slice left, it has priority and -- * fetching more just increases latency for the fetched tasks. -- * Tell put_prev_task_scx() to put @prev on local_dsq. If the -- * BPF scheduler wants to handle this explicitly, it should -- * implement ->cpu_released(). -- * -- * See scx_ops_disable_workfn() for the explanation on the -- * disabling() test. -- * -- * When balancing a remote CPU for core-sched, there won't be a -- * following put_prev_task_scx() call and we don't own -- * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the -- * same conditions later and pick @rq->curr accordingly. -- */ -- if ((prev->scx->flags & SCX_TASK_QUEUED) && -- prev->scx->slice && !scx_ops_disabling()) { -- if (local) -- prev->scx->flags |= SCX_TASK_BAL_KEEP; -- return 1; -- } -- } -- -- /* if there already are tasks to run, nothing to do */ -- if (scx_rq->local_dsq.nr) -- return 1; -- -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- if (!SCX_HAS_OP(dispatch)) -- return 0; -- -- dspc->rq = rq; -- dspc->rf = rf; -- -- /* -- * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, -- * the local DSQ might still end up empty after a successful -- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() -- * produced some tasks, retry. The BPF scheduler may depend on this -- * looping behavior to simplify its implementation. -- */ -- do { -- dspc->nr_tasks = 0; -- -- SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), -- prev_on_scx ? prev : NULL); -- -- flush_dispatch_buf(rq, rf); -- -- if (scx_rq->local_dsq.nr) -- return 1; -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- /* -- * ops.dispatch() can trap us in this loop by repeatedly -- * dispatching ineligible tasks. Break out once in a while to -- * allow the watchdog to run. As IRQ can't be enabled in -- * balance(), we want to complete this scheduling cycle and then -- * start a new one. IOW, we want to call resched_curr() on the -- * next, most likely idle, task, not the current one. Use -- * scx_bpf_kick_cpu() for deferred kicking. -- */ -- if (unlikely(!--nr_loops)) { -- scx_bpf_kick_cpu(cpu_of(rq), 0); -- break; -- } -- } while (dspc->nr_tasks); -- -- return 0; --} -- --static int balance_scx(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf) --{ -- int ret; -- -- ret = balance_one(rq, prev, rf, true); --#ifdef CONFIG_SCHED_SMT -- /* -- * When core-sched is enabled, this ops.balance() call will be followed -- * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx() -- * on the SMT siblings. Balance the siblings too. -- */ -- if (sched_core_enabled(rq)) { -- const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); -- int scpu; -- -- for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { -- struct rq *srq = cpu_rq(scpu); -- struct rq_flags srf; -- struct task_struct *sprev = srq->curr; -- -- /* -- * While core-scheduling, rq lock is shared among -- * siblings but the debug annotations and rq clock -- * aren't. Do pinning dance to transfer the ownership. -- */ -- WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); -- rq_unpin_lock(rq, rf); -- rq_pin_lock(srq, &srf); -- -- update_rq_clock(srq); -- balance_one(srq, sprev, &srf, false); -- -- rq_unpin_lock(srq, &srf); -- rq_repin_lock(rq, rf); -- } -- } --#endif -- return ret; --} -- --static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) --{ -- if (p->scx->flags & SCX_TASK_QUEUED) { -- /* -- * Core-sched might decide to execute @p before it is -- * dispatched. Call ops_dequeue() to notify the BPF scheduler. -- */ -- ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); -- dispatch_dequeue(rq->scx, p); -- } -- -- p->se.exec_start = rq_clock_task(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, running, p); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock); -- -- watchdog_unwatch_task(p, true); -- -- /* -- * @p is getting newly scheduled or got kicked after someone updated its -- * slice. Refresh whether tick can be stopped. See can_stop_tick_scx(). -- */ -- if ((p->scx->slice == SCX_SLICE_INF) != -- (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) { -- if (p->scx->slice == SCX_SLICE_INF) -- rq->scx->flags |= SCX_RQ_CAN_STOP_TICK; -- else -- rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK; -- -- sched_update_tick_dependency(rq); -- } --} -- --static void put_prev_task_scx(struct rq *rq, struct task_struct *p) --{ --#ifndef CONFIG_SMP -- /* -- * UP workaround. -- * -- * Because SCX may transfer tasks across CPUs during dispatch, dispatch -- * is performed from its balance operation which isn't called in UP. -- * Let's work around by calling it from the operations which come right -- * after. -- * -- * 1. If the prev task is on SCX, pick_next_task() calls -- * .put_prev_task() right after. As .put_prev_task() is also called -- * from other places, we need to distinguish the calls which can be -- * done by looking at the previous task's state - if still queued or -- * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task(). -- * This case is handled here. -- * -- * 2. If the prev task is not on SCX, the first following call into SCX -- * will be .pick_next_task(), which is covered by calling -- * balance_scx() from pick_next_task_scx(). -- * -- * Note that we can't merge the first case into the second as -- * balance_scx() must be called before the previous SCX task goes -- * through put_prev_task_scx(). -- * -- * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf. -- * Pass in %NULL. -- */ -- if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP)) -- balance_scx(rq, p, NULL); --#endif -- -- update_curr_scx(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- -- /* -- * If we're being called from put_prev_task_balance(), balance_scx() may -- * have decided that @p should keep running. -- */ -- if (p->scx->flags & SCX_TASK_BAL_KEEP) { -- p->scx->flags &= ~SCX_TASK_BAL_KEEP; -- watchdog_watch_task(rq, p); -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- watchdog_watch_task(rq, p); -- -- /* -- * If @p has slice left and balance_scx() didn't tag it for -- * keeping, @p is getting preempted by a higher priority -- * scheduler class or core-sched forcing a different task. Leave -- * it at the head of the local DSQ. -- */ -- if (p->scx->slice && !scx_ops_disabling()) { -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- /* -- * If we're in the pick_next_task path, balance_scx() should -- * have already populated the local DSQ if there are any other -- * available tasks. If empty, tell ops.enqueue() that @p is the -- * only one available for this cpu. ops.enqueue() should put it -- * on the local DSQ so that the subsequent pick_next_task_scx() -- * can find the task unless it wants to trigger a separate -- * follow-up scheduling event. -- */ -- if (list_empty(&rq->scx->local_dsq.fifo)) -- do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1); -- else -- do_enqueue_task(rq, p, 0, -1); -- } --} -- --static struct task_struct *first_local_task(struct rq *rq) --{ -- struct rb_node *rb_node; -- struct sched_ext_entity *entity; -- -- if (!list_empty(&rq->scx->local_dsq.fifo)) { -- entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo); -- return entity->task; -- } -- -- rb_node = rb_first_cached(&rq->scx->local_dsq.priq); -- if (rb_node) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- return entity->task; -- } -- -- return NULL; --} -- --static struct task_struct *pick_next_task_scx(struct rq *rq) --{ -- struct task_struct *p; -- --#ifndef CONFIG_SMP -- /* UP workaround - see the comment at the head of put_prev_task_scx() */ -- if (unlikely(rq->curr->sched_class != &ext_sched_class)) -- balance_scx(rq, rq->curr, NULL); --#endif -- -- p = first_local_task(rq); -- if (!p) -- return NULL; -- -- if (unlikely(!p->scx->slice)) { -- if (!scx_ops_disabling() && !scx_warned_zero_slice) { -- printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n", -- p->comm, p->pid); -- scx_warned_zero_slice = true; -- } -- p->scx->slice = SCX_SLICE_DFL; -- } -- -- set_next_task_scx(rq, p, true); -- -- return p; --} -- --#ifdef CONFIG_SCHED_CORE --/** -- * scx_prio_less - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Core-sched is implemented as an additional scheduling layer on top of the -- * usual sched_class'es and needs to find out the expected task ordering. For -- * SCX, core-sched calls this function to interrogate the task ordering. -- * -- * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used -- * to implement the default task ordering. The older the timestamp, the higher -- * prority the task - the global FIFO ordering matching the default scheduling -- * behavior. -- * -- * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to -- * implement FIFO ordering within each local DSQ. See pick_task_scx(). -- */ --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi) --{ -- /* -- * The const qualifiers are dropped from task_struct pointers when -- * calling ops.core_sched_before(). Accesses are controlled by the -- * verifier. -- */ -- if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling()) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, -- (struct task_struct *)a, -- (struct task_struct *)b); -- else -- return time_after64(a->scx->core_sched_at, b->scx->core_sched_at); --} -- --/** -- * pick_task_scx - Pick a candidate task for core-sched -- * @rq: rq to pick the candidate task from -- * -- * Core-sched calls this function on each SMT sibling to determine the next -- * tasks to run on the SMT siblings. balance_one() has been called on all -- * siblings and put_prev_task_scx() has been called only for the current CPU. -- * -- * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look -- * at the first task in the local dsq. @rq->curr has to be considered explicitly -- * to mimic %SCX_TASK_BAL_KEEP. -- */ --static struct task_struct *pick_task_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- struct task_struct *first = first_local_task(rq); -- -- if (curr->scx->flags & SCX_TASK_QUEUED) { -- /* is curr the only runnable task? */ -- if (!first) -- return curr; -- -- /* -- * Does curr trump first? We can always go by core_sched_at for -- * this comparison as it represents global FIFO ordering when -- * the default core-sched ordering is used and local-DSQ FIFO -- * ordering otherwise. -- * -- * We can have a task with an earlier timestamp on the DSQ. For -- * example, when a current task is preempted by a sibling -- * picking a different cookie, the task would be requeued at the -- * head of the local DSQ with an earlier timestamp than the -- * core-sched picked next task. Besides, the BPF scheduler may -- * dispatch any tasks to the local DSQ anytime. -- */ -- if (curr->scx->slice && time_before64(curr->scx->core_sched_at, -- first->scx->core_sched_at)) -- return curr; -- } -- -- return first; /* this may be %NULL */ --} --#endif /* CONFIG_SCHED_CORE */ -- --static enum scx_cpu_preempt_reason --preempt_reason_from_class(const struct sched_class *class) --{ --#ifdef CONFIG_SMP -- if (class == &stop_sched_class) -- return SCX_CPU_PREEMPT_STOP; --#endif -- if (class == &dl_sched_class) -- return SCX_CPU_PREEMPT_DL; -- if (class == &rt_sched_class) -- return SCX_CPU_PREEMPT_RT; -- return SCX_CPU_PREEMPT_UNKNOWN; --} -- --void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task, -- const struct sched_class *active) --{ -- lockdep_assert_rq_held(rq); -- -- /* -- * The callback is conceptually meant to convey that the CPU is no -- * longer under the control of SCX. Therefore, don't invoke the -- * callback if the CPU is is staying on SCX, or going idle (in which -- * case the SCX scheduler has actively decided not to schedule any -- * tasks on the CPU). -- */ -- if (likely(active >= &ext_sched_class)) -- return; -- -- /* -- * At this point we know that SCX was preempted by a higher priority -- * sched_class, so invoke the ->cpu_release() callback if we have not -- * done so already. We only send the callback once between SCX being -- * preempted, and it regaining control of the CPU. -- * -- * ->cpu_release() complements ->cpu_acquire(), which is emitted the -- * next time that balance_scx() is invoked. -- */ -- if (!rq->scx->cpu_released) { -- if (SCX_HAS_OP(cpu_release)) { -- struct scx_cpu_release_args args = { -- .reason = preempt_reason_from_class(active), -- .task = task, -- }; -- -- SCX_CALL_OP(SCX_KF_CPU_RELEASE, -- cpu_release, cpu_of(rq), &args); -- } -- rq->scx->cpu_released = true; -- } --} -- --#ifdef CONFIG_SMP -- --static bool test_and_clear_cpu_idle(int cpu) --{ -- if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -- if (cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- return true; -- } else { -- return false; -- } --} -- --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- int cpu; -- -- do { -- cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); -- if (cpu < nr_cpu_ids) { -- const struct cpumask *sbm = topology_sibling_cpumask(cpu); -- -- /* -- * If offline, @cpu is not its own sibling and we can -- * get caught in an infinite loop as @cpu is never -- * cleared from idle_masks.smt. Clear @cpu directly in -- * such cases. -- */ -- if (likely(cpumask_test_cpu(cpu, sbm))) -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm); -- else -- cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu)); -- } else { -- cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -- if (cpu >= nr_cpu_ids) -- return -EBUSY; -- } -- } while (!test_and_clear_cpu_idle(cpu)); -- -- return cpu; --} -- --static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) --{ -- s32 cpu; -- -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return prev_cpu; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local DSQ of the waker. -- */ -- if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 && -- scx_has_idle_cpus && !(current->flags & PF_EXITING)) { -- cpu = smp_processor_id(); -- if (cpumask_test_cpu(cpu, p->cpus_ptr)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (test_and_clear_cpu_idle(prev_cpu)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return prev_cpu; -- } -- -- if (p->nr_cpus_allowed == 1) -- return prev_cpu; -- -- cpu = scx_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- -- return prev_cpu; --} -- --static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) --{ -- if (SCX_HAS_OP(select_cpu)) { -- s32 cpu; -- -- cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu, -- wake_flags); -- if (ops_cpu_valid(cpu)) { -- return cpu; -- } else { -- scx_ops_error("select_cpu returned invalid cpu %d", cpu); -- return prev_cpu; -- } -- } else { -- return scx_select_cpu_dfl(p, prev_cpu, wake_flags); -- } --} -- --static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx) --{ -- set_cpus_allowed_common(p, ctx); -- -- /* -- * The effective cpumask is stored in @p->cpus_ptr which may temporarily -- * differ from the configured one in @p->cpus_mask. Always tell the bpf -- * scheduler the effective one. -- * -- * Fine-grained memory write control is enforced by BPF making the const -- * designation pointless. Cast it away when calling the operation. -- */ -- if (SCX_HAS_OP(set_cpumask)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, -- (struct cpumask *)p->cpus_ptr); --} -- --static void reset_idle_masks(void) --{ -- /* consider all cpus idle, should converge to the actual state quickly */ -- cpumask_setall(idle_masks.cpu); -- cpumask_setall(idle_masks.smt); -- scx_has_idle_cpus = true; --} -- --void __scx_update_idle(struct rq *rq, bool idle) --{ -- int cpu = cpu_of(rq); -- struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -- -- if (SCX_HAS_OP(update_idle)) { -- SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); -- if (!static_branch_unlikely(&scx_builtin_idle_enabled)) -- return; -- } -- -- if (idle) { -- cpumask_set_cpu(cpu, idle_masks.cpu); -- if (!scx_has_idle_cpus) -- scx_has_idle_cpus = true; -- -- /* -- * idle_masks.smt handling is racy but that's fine as it's only -- * for optimization and self-correcting. -- */ -- for_each_cpu(cpu, sib_mask) { -- if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -- return; -- } -- cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -- } else { -- cpumask_clear_cpu(cpu, idle_masks.cpu); -- if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -- } --} -- --#else /* !CONFIG_SMP */ -- --static bool test_and_clear_cpu_idle(int cpu) { return false; } --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } --static void reset_idle_masks(void) {} -- --#endif /* CONFIG_SMP */ -- --static bool check_rq_for_timeouts(struct rq *rq) --{ -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rq_flags rf; -- bool timed_out = false; -- -- rq_lock_irqsave(rq, &rf); -- list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) { -- unsigned long last_runnable; -- -- p = entity->task; -- last_runnable = p->scx->runnable_at; -- -- if (unlikely(time_after(jiffies, -- last_runnable + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "%s[%d] failed to run for %u.%03us", -- p->comm, p->pid, -- dur_ms / 1000, dur_ms % 1000); -- timed_out = true; -- break; -- } -- } -- rq_unlock_irqrestore(rq, &rf); -- -- return timed_out; --} -- --static void scx_watchdog_workfn(struct work_struct *work) --{ -- int cpu; -- -- scx_watchdog_timestamp = jiffies; -- -- for_each_online_cpu(cpu) { -- if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) -- break; -- -- cond_resched(); -- } -- queue_delayed_work(system_unbound_wq, to_delayed_work(work), -- scx_watchdog_timeout / 2); --} -- --static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) --{ -- update_curr_scx(rq); -- //scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock); -- /* -- * While disabling, always resched and refresh core-sched timestamp as -- * we can't trust the slice management or ops.core_sched_before(). -- */ -- if (scx_ops_disabling()) { -- curr->scx->slice = 0; -- touch_core_sched(rq, curr); -- } -- -- if (!curr->scx->slice) -- resched_curr(rq); --} -- --#define SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- --static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg) --{ -- int ret; -- -- WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED); -- -- p->scx->disallow = false; -- -- if (SCX_HAS_OP(prep_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- }; -- -- ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args); -- if (unlikely(ret)) { -- ret = ops_sanitize_err("prep_enable", ret); -- return ret; -- } -- } -- //scx_sched_init_task(p); -- -- if (p->scx->disallow) { -- struct rq *rq; -- struct rq_flags rf; -- -- rq = task_rq_lock(p, &rf); -- -- /* -- * We're either in fork or load path and @p->policy will be -- * applied right after. Reverting @p->policy here and rejecting -- * %SCHED_EXT transitions from scx_check_setscheduler() -- * guarantees that if ops.prep_enable() sets @p->disallow, @p -- * can never be in SCX. -- */ -- if (p->policy == SCHED_EXT) { -- p->policy = SCHED_NORMAL; -- atomic64_inc(&scx_nr_rejected); -- } -- -- task_rq_unlock(rq, p, &rf); -- } -- -- p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET); -- return 0; --} -- --static void scx_ops_enable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED)); -- -- if (SCX_HAS_OP(enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- p->scx->flags |= SCX_TASK_OPS_ENABLED; --} -- --static void scx_ops_disable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- if (p->scx->flags & SCX_TASK_OPS_PREPPED) { -- if (SCX_HAS_OP(cancel_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- } else if (p->scx->flags & SCX_TASK_OPS_ENABLED) { -- if (SCX_HAS_OP(disable)) -- SCX_CALL_OP(SCX_KF_REST, disable, p); -- p->scx->flags &= ~SCX_TASK_OPS_ENABLED; -- } --} -- --static void set_task_scx_weight(struct task_struct *p) --{ -- u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -- -- p->scx->weight = sched_weight_to_cgroup(weight); --} -- --/** -- * refresh_scx_weight - Refresh a task's ext weight -- * @p: task to refresh ext weight for -- * -- * @p->scx->weight carries the task's static priority in cgroup weight scale to -- * enable easy access from the BPF scheduler. To keep it synchronized with the -- * current task priority, this function should be called when a new task is -- * created, priority is changed for a task on sched_ext, and a task is switched -- * to sched_ext from other classes. -- */ --static void refresh_scx_weight(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- set_task_scx_weight(p); -- if (SCX_HAS_OP(set_weight)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight); --} -- --void scx_pre_fork(struct task_struct *p) --{ -- p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL); -- if (!p->scx) { -- goto lock; -- } -- -- p->scx->dsq = NULL; -- INIT_LIST_HEAD(&p->scx->dsq_node.fifo); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- INIT_LIST_HEAD(&p->scx->watchdog_node); -- p->scx->flags = 0; -- p->scx->weight = 0; -- p->scx->sticky_cpu = -1; -- p->scx->holding_cpu = -1; -- p->scx->kf_mask = 0; -- atomic64_set(&p->scx->ops_state, 0); -- p->scx->runnable_at = INITIAL_JIFFIES; -- p->scx->slice = SCX_SLICE_DFL; -- p->scx->task = p; -- p->sched_prop = 0; -- -- /* -- * BPF scheduler enable/disable paths want to be able to iterate and -- * update all tasks which can become complex when racing forks. As -- * enable/disable are very cold paths, let's use a percpu_rwsem to -- * exclude forks. -- */ --lock: -- percpu_down_read(&scx_fork_rwsem); --} -- --int scx_fork(struct task_struct *p) --{ -- percpu_rwsem_assert_held(&scx_fork_rwsem); -- -- if (scx_enabled()) -- return scx_ops_prepare_task(p, task_group(p)); -- else -- return 0; --} -- --void scx_post_fork(struct task_struct *p) --{ -- if (scx_enabled()) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- /* -- * Set the weight manually before calling ops.enable() so that -- * the scheduler doesn't see a stale value if they inspect the -- * task struct. We'll invoke ops.set_weight() afterwards, as it -- * would be odd to receive a callback on the task before we -- * tell the scheduler that it's been fully enabled. -- */ -- set_task_scx_weight(p); -- scx_ops_enable_task(p); -- refresh_scx_weight(p); -- task_rq_unlock(rq, p, &rf); -- } -- -- spin_lock_irq(&scx_tasks_lock); -- list_add_tail(&p->scx->tasks_node, &scx_tasks); -- spin_unlock_irq(&scx_tasks_lock); -- -- percpu_up_read(&scx_fork_rwsem); --} -- --void scx_cancel_fork(struct task_struct *p) --{ -- if (scx_enabled()) -- scx_ops_disable_task(p); -- percpu_up_read(&scx_fork_rwsem); --} -- --void sched_ext_free(struct task_struct *p) --{ -- unsigned long flags; -- -- spin_lock_irqsave(&scx_tasks_lock, flags); -- list_del_init(&p->scx->tasks_node); -- spin_unlock_irqrestore(&scx_tasks_lock, flags); -- -- /* -- * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED -> -- * ENABLED transitions can't race us. Disable ops for @p. -- */ -- if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- scx_ops_disable_task(p); -- task_rq_unlock(rq, p, &rf); -- } --} -- --static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) --{ --} -- --static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} --static void switched_to_scx(struct rq *rq, struct task_struct *p) {} -- --int scx_check_setscheduler(struct task_struct *p, int policy) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- /* if disallow, reject transitioning into SCX */ -- if (scx_enabled() && READ_ONCE(p->scx->disallow) && -- p->policy != policy && policy == SCHED_EXT) -- return -EACCES; -- -- return 0; --} -- --#ifdef CONFIG_NO_HZ_FULL --bool scx_can_stop_tick(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (scx_ops_disabling()) -- return false; -- -- if (p->sched_class != &ext_sched_class) -- return true; -- -- /* -- * @rq can dispatch from different DSQs, so we can't tell whether it -- * needs the tick or not by looking at nr_running. Allow stopping ticks -- * iff the BPF scheduler indicated so. See set_next_task_scx(). -- */ -- return rq->scx->flags & SCX_RQ_CAN_STOP_TICK; --} --#endif -- --static inline void scx_cgroup_lock(void) {} --static inline void scx_cgroup_unlock(void) {} -- --/* -- * Omitted operations: -- * -- * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -- * task isn't tied to the CPU at that point. Preemption is implemented by -- * resetting the victim task's slice to 0 and triggering reschedule on the -- * target CPU. -- * -- * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -- * -- * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -- * their current sched_class. Call them directly from sched core instead. -- * -- * - task_woken, switched_from: Unnecessary. -- */ --DEFINE_SCHED_CLASS(ext) = { -- .enqueue_task = enqueue_task_scx, -- .dequeue_task = dequeue_task_scx, -- .yield_task = yield_task_scx, -- .yield_to_task = yield_to_task_scx, -- -- .check_preempt_curr = check_preempt_curr_scx, -- -- .pick_next_task = pick_next_task_scx, -- -- .put_prev_task = put_prev_task_scx, -- .set_next_task = set_next_task_scx, -- --#ifdef CONFIG_SMP -- .balance = balance_scx, -- .select_task_rq = select_task_rq_scx, -- .set_cpus_allowed = set_cpus_allowed_scx, --#endif -- --#ifdef CONFIG_SCHED_CORE -- .pick_task = pick_task_scx, --#endif -- -- .task_tick = task_tick_scx, -- -- .switched_to = switched_to_scx, -- .prio_changed = prio_changed_scx, -- -- .update_curr = update_curr_scx, -- --#ifdef CONFIG_UCLAMP_TASK -- .uclamp_enabled = 0, --#endif --}; -- --static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) --{ -- memset(dsq, 0, sizeof(*dsq)); -- -- raw_spin_lock_init(&dsq->lock); -- INIT_LIST_HEAD(&dsq->fifo); -- dsq->id = dsq_id; --} -- --static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id & SCX_DSQ_FLAG_BUILTIN) -- return ERR_PTR(-EINVAL); -- -- dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node); -- if (!dsq) -- return ERR_PTR(-ENOMEM); -- -- init_dsq(dsq, dsq_id); -- -- return dsq; --} -- --static void free_dsq_irq_workfn(struct irq_work *irq_work) --{ -- struct llist_node *to_free = llist_del_all(&dsqs_to_free); -- struct scx_dispatch_q *dsq, *tmp_dsq; -- -- llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) -- kfree_rcu(dsq, rcu); --} -- --static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); -- --static void destroy_dsq(u64 dsq_id) --{ --} -- --static void scx_cgroup_exit(void) {} --static int scx_cgroup_init(void) { return 0; } --static void scx_cgroup_config_knobs(void) {} -- --/* -- * Used by sched_fork() and __setscheduler_prio() to pick the matching -- * sched_class. dl/rt are already handled. -- */ --bool task_on_scx(struct task_struct *p) --{ -- if (!scx_enabled() || scx_ops_disabling()) -- return false; -- if (READ_ONCE(scx_switching_all)) -- return true; -- return p->policy == SCHED_EXT; --} -- --static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags) --{ -- if (enq_flags & SCX_ENQ_LAST) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); --} -- --static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {} -- --static void scx_ops_disable_workfn(struct kthread_work *work) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- struct scx_task_iter sti; -- struct task_struct *p; -- const char *reason; -- int i, cpu, type; -- -- type = atomic_read(&scx_exit_type); -- while (true) { -- /* -- * NONE indicates that a new scx_ops has been registered since -- * disable was scheduled - don't kill the new ops. DONE -- * indicates that the ops has already been disabled. -- */ -- if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE) -- return; -- if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE)) -- break; -- } -- -- cancel_delayed_work_sync(&scx_watchdog_work); -- -- switch (type) { -- case SCX_EXIT_UNREG: -- reason = "BPF scheduler unregistered"; -- break; -- case SCX_EXIT_SYSRQ: -- reason = "disabled by sysrq-S"; -- break; -- case SCX_EXIT_ERROR: -- reason = "runtime error"; -- break; -- case SCX_EXIT_ERROR_BPF: -- reason = "scx_bpf_error"; -- break; -- case SCX_EXIT_ERROR_STALL: -- reason = "runnable task stall"; -- break; -- default: -- reason = ""; -- } -- -- ei->type = type; -- strlcpy(ei->reason, reason, sizeof(ei->reason)); -- -- switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { -- case SCX_OPS_DISABLED: -- pr_warn("sched_ext: ops error detected without ops (%s)\n", -- scx_exit_info.msg); -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- return; -- case SCX_OPS_PREPPING: -- goto forward_progress_guaranteed; -- case SCX_OPS_DISABLING: -- /* shouldn't happen but handle it like ENABLING if it does */ -- WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); -- fallthrough; -- case SCX_OPS_ENABLING: -- case SCX_OPS_ENABLED: -- break; -- } -- -- /* -- * DISABLING is set and ops was either ENABLING or ENABLED indicating -- * that the ops and static branches are set. -- * -- * We must guarantee that all runnable tasks make forward progress -- * without trusting the BPF scheduler. We can't grab any mutexes or -- * rwsems as they might be held by tasks that the BPF scheduler is -- * forgetting to run, which unfortunately also excludes toggling the -- * static branches. -- * -- * Let's work around by overriding a couple ops and modifying behaviors -- * based on the DISABLING state and then cycling the tasks through -- * dequeue/enqueue to force global FIFO scheduling. -- * -- * a. ops.enqueue() and .dispatch() are overridden for simple global -- * FIFO scheduling. -- * -- * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value -- * can't be trusted. Whenever a tick triggers, the running task is -- * rotated to the tail of the queue with core_sched_at touched. -- * -- * c. pick_next_task() suppresses zero slice warning. -- * -- * d. scx_prio_less() reverts to the default core_sched_at order. -- */ -- scx_ops.enqueue = scx_ops_fallback_enqueue; -- scx_ops.dispatch = scx_ops_fallback_dispatch; -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- SCHED_CHANGE_BLOCK(task_rq(p), p, -- DEQUEUE_SAVE | DEQUEUE_MOVE) { -- /* cycling deq/enq is enough, see above */ -- } -- } -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* kick all CPUs to restore ticks */ -- for_each_possible_cpu(cpu) -- resched_cpu(cpu); -- --forward_progress_guaranteed: -- /* -- * Here, every runnable task is guaranteed to make forward progress and -- * we can safely use blocking synchronization constructs. Actually -- * disable ops. -- */ -- mutex_lock(&scx_ops_enable_mutex); -- -- static_branch_disable(&__scx_switched_all); -- WRITE_ONCE(scx_switching_all, false); -- -- /* avoid racing against fork and cgroup changes */ -- cpus_read_lock(); -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- bool alive = READ_ONCE(p->__state) != TASK_DEAD; -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL); -- -- __setscheduler_prio(p, p->prio); -- } -- -- if (alive) -- check_class_changed(task_rq(p), p, old_class, p->prio); -- -- scx_ops_disable_task(p); -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* no task is on scx, turn off all the switches and flush in-progress calls */ -- static_branch_disable_cpuslocked(&__scx_ops_enabled); -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- static_branch_disable_cpuslocked(&scx_has_op[i]); -- static_branch_disable_cpuslocked(&scx_ops_enq_last); -- static_branch_disable_cpuslocked(&scx_ops_enq_exiting); -- static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- synchronize_rcu(); -- -- scx_cgroup_exit(); -- -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- cpus_read_unlock(); -- -- if (ei->type >= SCX_EXIT_ERROR) { -- printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name); -- -- if (ei->msg[0] == '\0') -- printk(KERN_ERR "sched_ext: %s\n", ei->reason); -- else -- printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg); -- -- stack_trace_print(ei->bt, ei->bt_len, 2); -- } -- -- if (scx_ops.exit) -- SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); -- -- memset(&scx_ops, 0, sizeof(scx_ops)); -- -- free_percpu(scx_dsp_buf); -- scx_dsp_buf = NULL; -- scx_dsp_max_batch = 0; -- -- mutex_unlock(&scx_ops_enable_mutex); -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- -- scx_cgroup_config_knobs(); --} -- --static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); -- --static void schedule_scx_ops_disable_work(void) --{ -- struct kthread_worker *helper = READ_ONCE(scx_ops_helper); -- -- /* -- * We may be called spuriously before the first bpf_sched_ext_reg(). If -- * scx_ops_helper isn't set up yet, there's nothing to do. -- */ -- if (helper) -- kthread_queue_work(helper, &scx_ops_disable_work); --} -- --static void scx_ops_disable(enum scx_exit_type type) --{ -- int none = SCX_EXIT_NONE; -- -- if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)) -- type = SCX_EXIT_ERROR; -- -- atomic_try_cmpxchg(&scx_exit_type, &none, type); -- -- schedule_scx_ops_disable_work(); --} -- --static void scx_ops_error_irq_workfn(struct irq_work *irq_work) --{ -- schedule_scx_ops_disable_work(); --} -- --static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- int none = SCX_EXIT_NONE; -- va_list args; -- -- if (!atomic_try_cmpxchg(&scx_exit_type, &none, type)) -- return; -- -- ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1); -- -- va_start(args, fmt); -- vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args); -- va_end(args); -- -- irq_work_queue(&scx_ops_error_irq_work); --} -- --static struct kthread_worker *scx_create_rt_helper(const char *name) --{ -- struct kthread_worker *helper; -- -- helper = kthread_create_worker(0, name); -- if (helper) -- sched_set_fifo(helper->task); -- return helper; --} -- --static int scx_ops_enable(struct sched_ext_ops *ops) --{ -- struct scx_task_iter sti; -- struct task_struct *p; -- int i, ret; -- int tcnt = 0; -- unsigned long long start = 0; -- unsigned long long total_start = sched_clock(); -- -- mutex_lock(&scx_ops_enable_mutex); -- -- if (!scx_ops_helper) { -- WRITE_ONCE(scx_ops_helper, -- scx_create_rt_helper("sched_ext_ops_helper")); -- if (!scx_ops_helper) { -- ret = -ENOMEM; -- goto err_unlock; -- } -- } -- -- if (scx_ops_enable_state() != SCX_OPS_DISABLED) { -- ret = -EBUSY; -- goto err_unlock; -- } -- -- //slim_walt_enable(true); -- -- /* -- * Set scx_ops, transition to PREPPING and clear exit info to arm the -- * disable path. Failure triggers full disabling from here on. -- */ -- scx_ops = *ops; -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != -- SCX_OPS_DISABLED); -- -- memset(&scx_exit_info, 0, sizeof(scx_exit_info)); -- atomic_set(&scx_exit_type, SCX_EXIT_NONE); -- scx_warned_zero_slice = false; -- -- atomic64_set(&scx_nr_rejected, 0); -- -- /* -- * Keep CPUs stable during enable so that the BPF scheduler can track -- * online CPUs by watching ->on/offline_cpu() after ->init(). -- */ -- cpus_read_lock(); -- -- scx_switch_all_req = true; -- if (scx_ops.init) { -- ret = SCX_CALL_OP_RET(SCX_KF_INIT, init); -- if (ret) { -- ret = ops_sanitize_err("init", ret); -- goto err_disable; -- } -- -- /* -- * Exit early if ops.init() triggered scx_bpf_error(). Not -- * strictly necessary as we'll fail transitioning into ENABLING -- * later but that'd be after calling ops.prep_enable() on all -- * tasks and with -EBUSY which isn't very intuitive. Let's exit -- * early with success so that the condition is notified through -- * ops.exit() like other scx_bpf_error() invocations. -- */ -- if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE) -- goto err_disable; -- } -- -- WARN_ON_ONCE(scx_dsp_buf); -- scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; -- scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch, -- __alignof__(scx_dsp_buf[0])); -- if (!scx_dsp_buf) { -- ret = -ENOMEM; -- goto err_disable; -- } -- -- scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; -- if (ops->timeout_ms) -- scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); -- -- scx_watchdog_timestamp = jiffies; -- queue_delayed_work(system_unbound_wq, &scx_watchdog_work, -- scx_watchdog_timeout / 2); -- -- /* -- * Lock out forks, cgroup on/offlining and moves before opening the -- * floodgate so that they don't wander into the operations prematurely. -- */ -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- if (((void (**)(void))ops)[i]) -- static_branch_enable_cpuslocked(&scx_has_op[i]); -- -- if (ops->flags & SCX_OPS_ENQ_LAST) -- static_branch_enable_cpuslocked(&scx_ops_enq_last); -- -- if (ops->flags & SCX_OPS_ENQ_EXITING) -- static_branch_enable_cpuslocked(&scx_ops_enq_exiting); -- if (scx_ops.cpu_acquire || scx_ops.cpu_release) -- static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); -- -- if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { -- reset_idle_masks(); -- static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); -- } else { -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- } -- -- /* -- * All cgroups should be initialized before letting in tasks. cgroup -- * on/offlining and task migrations are already locked out. -- */ -- ret = scx_cgroup_init(); -- if (ret) -- goto err_disable_unlock; -- -- static_branch_enable_cpuslocked(&__scx_ops_enabled); -- -- /* -- * Enable ops for every task. Fork is excluded by scx_fork_rwsem -- * preventing new tasks from being added. No need to exclude tasks -- * leaving as sched_ext_free() can handle both prepped and enabled -- * tasks. Prep all tasks first and then enable them with preemption -- * disabled. -- */ -- spin_lock_irq(&scx_tasks_lock); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered(&sti))) { -- get_task_struct(p); -- spin_unlock_irq(&scx_tasks_lock); -- -- ret = scx_ops_prepare_task(p, task_group(p)); -- if (ret) { -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n", -- ret, p->comm, p->pid); -- goto err_disable_unlock; -- } -- -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- } -- scx_task_iter_exit(&sti); -- -- /* -- * All tasks are prepped but are still ops-disabled. Ensure that -- * %current can't be scheduled out and switch everyone. -- * preempt_disable() is necessary because we can't guarantee that -- * %current won't be starved if scheduled out while switching. -- */ -- preempt_disable(); -- -- /* -- * From here on, the disable path must assume that tasks have ops -- * enabled and need to be recovered. -- */ -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { -- preempt_enable(); -- spin_unlock_irq(&scx_tasks_lock); -- ret = -EBUSY; -- goto err_disable_unlock; -- } -- -- /* -- * We're fully committed and can't fail. The PREPPED -> ENABLED -- * transitions here are synchronized against sched_ext_free() through -- * scx_tasks_lock. -- */ -- WRITE_ONCE(scx_switching_all, scx_switch_all_req); -- start = sched_clock(); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- tcnt++; -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- scx_ops_enable_task(p); -- __setscheduler_prio(p, p->prio); -- } -- -- check_class_changed(task_rq(p), p, old_class, p->prio); -- } else { -- scx_ops_disable_task(p); -- } -- } -- printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id()); -- scx_task_iter_exit(&sti); -- -- spin_unlock_irq(&scx_tasks_lock); -- preempt_enable(); -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { -- ret = -EBUSY; -- goto err_disable; -- } -- -- if (scx_switch_all_req) -- static_branch_enable_cpuslocked(&__scx_switched_all); -- -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- -- scx_cgroup_config_knobs(); -- printk("\n total duration = %llu , current cpu = %d\n", sched_clock() - total_start, smp_processor_id()); -- -- return 0; -- --err_unlock: -- mutex_unlock(&scx_ops_enable_mutex); -- return ret; -- --err_disable_unlock: -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); --err_disable: -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- /* must be fully disabled before returning */ -- scx_ops_disable(SCX_EXIT_ERROR); -- kthread_flush_work(&scx_ops_disable_work); -- return ret; --} -- --#ifdef CONFIG_SCHED_DEBUG --static const char *scx_ops_enable_state_str[] = { -- [SCX_OPS_PREPPING] = "prepping", -- [SCX_OPS_ENABLING] = "enabling", -- [SCX_OPS_ENABLED] = "enabled", -- [SCX_OPS_DISABLING] = "disabling", -- [SCX_OPS_DISABLED] = "disabled", --}; -- --static int scx_debug_show(struct seq_file *m, void *v) --{ -- mutex_lock(&scx_ops_enable_mutex); -- seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name); -- seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled()); -- seq_printf(m, "%-30s: %d\n", "switching_all", -- READ_ONCE(scx_switching_all)); -- seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all()); -- seq_printf(m, "%-30s: %s\n", "enable_state", -- scx_ops_enable_state_str[scx_ops_enable_state()]); -- seq_printf(m, "%-30s: %llu\n", "nr_rejected", -- atomic64_read(&scx_nr_rejected)); -- mutex_unlock(&scx_ops_enable_mutex); -- return 0; --} -- --static int scx_debug_open(struct inode *inode, struct file *file) --{ -- return single_open(file, scx_debug_show, NULL); --} -- --const struct file_operations sched_ext_fops = { -- .open = scx_debug_open, -- .read = seq_read, -- .llseek = seq_lseek, -- .release = single_release, --}; --#endif -- --/******************************************************************************** -- * bpf_struct_ops plumbing. -- */ --#include --#include --#include -- --extern struct btf *btf_vmlinux; --static const struct btf_type *task_struct_type; -- --static bool bpf_scx_is_valid_access(int off, int size, -- enum bpf_access_type type, -- const struct bpf_prog *prog, -- struct bpf_insn_access_aux *info) --{ -- if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) -- return false; -- if (type != BPF_READ) -- return false; -- if (off % size != 0) -- return false; -- -- return btf_ctx_access(off, size, type, prog, info); --} -- --static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, -- const struct bpf_reg_state *reg, -- int off, int size) --{ -- return 0; --} -- --static const struct bpf_func_proto * --bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) --{ -- switch (func_id) { -- case BPF_FUNC_task_storage_get: -- return &bpf_task_storage_get_proto; -- case BPF_FUNC_task_storage_delete: -- return &bpf_task_storage_delete_proto; -- default: -- return bpf_base_func_proto(func_id); -- } --} -- --const struct bpf_verifier_ops bpf_scx_verifier_ops = { -- .get_func_proto = bpf_scx_get_func_proto, -- .is_valid_access = bpf_scx_is_valid_access, -- .btf_struct_access = bpf_scx_btf_struct_access, --}; -- --static int bpf_scx_init_member(const struct btf_type *t, -- const struct btf_member *member, -- void *kdata, const void *udata) --{ -- const struct sched_ext_ops *uops = udata; -- struct sched_ext_ops *ops = kdata; -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- int ret; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, dispatch_max_batch): -- if (*(u32 *)(udata + moff) > INT_MAX) -- return -E2BIG; -- ops->dispatch_max_batch = *(u32 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, flags): -- if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) -- return -EINVAL; -- ops->flags = *(u64 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, name): -- ret = bpf_obj_name_cpy(ops->name, uops->name, -- sizeof(ops->name)); -- if (ret < 0) -- return ret; -- if (ret == 0) -- return -EINVAL; -- return 1; -- case offsetof(struct sched_ext_ops, timeout_ms): -- if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT) -- return -E2BIG; -- ops->timeout_ms = *(u32 *)(udata + moff); -- return 1; -- } -- -- return 0; --} -- --static int bpf_scx_check_member(const struct btf_type *t, -- const struct btf_member *member, -- const struct bpf_prog *prog) --{ -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, prep_enable): -- case offsetof(struct sched_ext_ops, init): -- case offsetof(struct sched_ext_ops, exit): -- break; -- default: -- /* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */ -- /* if (prog->aux->sleepable) -- return -EINVAL; */ -- break; -- } -- -- return 0; --} -- --static int bpf_scx_reg(void *kdata) --{ -- return scx_ops_enable(kdata); --} -- --static void bpf_scx_unreg(void *kdata) --{ -- scx_ops_disable(SCX_EXIT_UNREG); -- kthread_flush_work(&scx_ops_disable_work); --} -- --static int bpf_scx_init(struct btf *btf) --{ -- u32 type_id; -- -- type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); -- if (type_id < 0) -- return -EINVAL; -- task_struct_type = btf_type_by_id(btf, type_id); -- -- return 0; --} -- --/* "extern" to avoid sparse warning, only used in this file */ --extern struct bpf_struct_ops bpf_sched_ext_ops; -- --struct bpf_struct_ops bpf_sched_ext_ops = { -- .verifier_ops = &bpf_scx_verifier_ops, -- .reg = bpf_scx_reg, -- .unreg = bpf_scx_unreg, -- .check_member = bpf_scx_check_member, -- .init_member = bpf_scx_init_member, -- .init = bpf_scx_init, -- .name = "sched_ext_ops", --}; -- --static void sysrq_handle_sched_ext_reset(u8 key) --{ -- if (scx_ops_helper) -- scx_ops_disable(SCX_EXIT_SYSRQ); -- else -- pr_info("sched_ext: BPF scheduler not yet used\n"); --} -- --static const struct sysrq_key_op sysrq_sched_ext_reset_op = { -- .handler = sysrq_handle_sched_ext_reset, -- .help_msg = "reset-sched-ext(S)", -- .action_msg = "Disable sched_ext and revert all tasks to CFS", -- .enable_mask = SYSRQ_ENABLE_RTNICE, --}; -- --static void kick_cpus_irq_workfn(struct irq_work *irq_work) --{ -- struct rq *this_rq = this_rq(); -- u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); -- int this_cpu = cpu_of(this_rq); -- int cpu; -- -- for_each_cpu(cpu, this_rq->scx->cpus_to_kick) { -- struct rq *rq = cpu_rq(cpu); -- unsigned long flags; -- -- raw_spin_rq_lock_irqsave(rq, flags); -- -- if (cpu_online(cpu) || cpu == this_cpu) { -- if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) && -- rq->curr->sched_class == &ext_sched_class) -- rq->curr->scx->slice = 0; -- pseqs[cpu] = rq->scx->pnt_seq; -- resched_curr(rq); -- } else { -- cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait); -- } -- -- raw_spin_rq_unlock_irqrestore(rq, flags); -- } -- -- for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait, -- cpumask_of(this_cpu)) { -- /* -- * Pairs with smp_store_release() issued by this CPU in -- * scx_notify_pick_next_task() on the resched path. -- * -- * We busy-wait here to guarantee that no other task can be -- * scheduled on our core before the target CPU has entered the -- * resched path. -- */ -- while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu]) -- cpu_relax(); -- } -- -- cpumask_clear(this_rq->scx->cpus_to_kick); -- cpumask_clear(this_rq->scx->cpus_to_preempt); -- cpumask_clear(this_rq->scx->cpus_to_wait); --} -- --void __init init_sched_ext_class(void) --{ -- int cpu; -- u32 v; -- -- /* -- * The following is to prevent the compiler from optimizing out the enum -- * definitions so that BPF scheduler implementations can use them -- * through the generated vmlinux.h. -- */ -- WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | -- SCX_TG_ONLINE | SCX_KICK_PREEMPT); -- -- init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); --#ifdef CONFIG_SMP -- BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -- BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); --#endif -- scx_kick_cpus_pnt_seqs = -- __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * -- num_possible_cpus(), -- __alignof__(scx_kick_cpus_pnt_seqs[0])); -- BUG_ON(!scx_kick_cpus_pnt_seqs); -- -- for_each_possible_cpu(cpu) { -- struct rq *rq = cpu_rq(cpu); -- -- rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL); -- if (rq->scx) -- rq->scx->rq = rq; -- else -- pr_err("fatal error : alloc rq->scx failed!!!\n"); -- -- init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL); -- INIT_LIST_HEAD(&rq->scx->watchdog_list); -- -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL)); -- init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn); -- } -- -- register_sysrq_key('S', &sysrq_sched_ext_reset_op); -- INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); -- scx_cgroup_config_knobs(); --} -- -- --/******************************************************************************** -- * Helpers that can be called from the BPF scheduler. -- */ --#include -- --/* Disables missing prototype warnings for kfuncs */ --__diag_push(); --__diag_ignore_all("-Wmissing-prototypes", -- "Global functions as their definitions will be in vmlinux BTF"); -- --/** -- * scx_bpf_switch_all - Switch all tasks into SCX -- * @into_scx: switch direction -- * -- * If @into_scx is %true, all existing and future non-dl/rt tasks are switched -- * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on -- * SCX. The actual switching is asynchronous. Can be called from ops.init(). -- */ --void scx_bpf_switch_all(void) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return; -- -- scx_switch_all_req = true; --} -- --BTF_SET8_START(scx_kfunc_ids_init) --BTF_ID_FLAGS(func, scx_bpf_switch_all) --BTF_SET8_END(scx_kfunc_ids_init) -- --static const struct btf_kfunc_id_set scx_kfunc_set_init = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_init, --}; -- --/** -- * scx_bpf_create_dsq - Create a custom DSQ -- * @dsq_id: DSQ to create -- * @node: NUMA node to allocate from -- * -- * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(), -- * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move(). -- */ --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return -EINVAL; -- -- if (unlikely(node >= (int)nr_node_ids || -- (node < 0 && node != NUMA_NO_NODE))) -- return -EINVAL; -- return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); --} -- --BTF_SET8_START(scx_kfunc_ids_sleepable) --BTF_ID_FLAGS(func, scx_bpf_create_dsq) --BTF_SET8_END(scx_kfunc_ids_sleepable) -- --static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_sleepable, --}; -- --static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) --{ -- if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) -- return false; -- -- lockdep_assert_irqs_disabled(); -- -- if (unlikely(!p)) { -- scx_ops_error("called with NULL task"); -- return false; -- } -- -- if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { -- scx_ops_error("invalid enq_flags 0x%llx", enq_flags); -- return false; -- } -- -- return true; --} -- --static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) --{ -- struct task_struct *ddsp_task; -- int idx; -- -- ddsp_task = __this_cpu_read(direct_dispatch_task); -- if (ddsp_task) { -- direct_dispatch(ddsp_task, p, dsq_id, enq_flags); -- return; -- } -- -- idx = __this_cpu_read(scx_dsp_ctx.buf_cursor); -- if (unlikely(idx >= scx_dsp_max_batch)) { -- scx_ops_error("dispatch buffer overflow"); -- return; -- } -- -- this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){ -- .task = p, -- .qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK, -- .dsq_id = dsq_id, -- .enq_flags = enq_flags, -- }; -- __this_cpu_inc(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe -- * to call this function spuriously. Can be called from ops.enqueue() and -- * ops.dispatch(). -- * -- * When called from ops.enqueue(), it's for direct dispatch and @p must match -- * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the -- * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be -- * on the target CPU in the first place. -- * -- * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id -- * and this function can be called upto ops.dispatch_max_batch times to dispatch -- * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the -- * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. -- * -- * This function doesn't have any locking restrictions and may be called under -- * BPF locks (in the future when BPF introduces more flexible locking). -- * -- * @p is allowed to run for @slice. The scheduling path is triggered on slice -- * exhaustion. If zero, the current residual slice is maintained. If -- * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with -- * scx_bpf_kick_cpu() to trigger scheduling. -- */ --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- scx_dispatch_commit(p, dsq_id, enq_flags); --} -- --/** -- * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. -- * Tasks queued into the priority queue are ordered by @vtime and always -- * consumed after the tasks in the FIFO queue. All other aspects are identical -- * to scx_bpf_dispatch(). -- * -- * @vtime ordering is according to time_before64() which considers wrapping. A -- * numerically larger vtime may indicate an earlier position in the ordering and -- * vice-versa. -- */ --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 vtime, u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- p->scx->dsq_vtime = vtime; -- -- scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); --} -- --BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime) --BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_enqueue_dispatch, --}; -- --/** -- * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots -- * -- * Can only be called from ops.dispatch(). -- */ --u32 scx_bpf_dispatch_nr_slots(void) --{ -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return 0; -- -- return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ -- * @dsq_id: DSQ to consume -- * -- * Consume a task from the non-local DSQ identified by @dsq_id and transfer it -- * to the current CPU's local DSQ for execution. Can only be called from -- * ops.dispatch(). -- * -- * This function flushes the in-flight dispatches from scx_bpf_dispatch() before -- * trying to consume the specified DSQ. It may also grab rq locks and thus can't -- * be called under any BPF locks. -- * -- * Returns %true if a task has been consumed, %false if there isn't any task to -- * consume. -- */ --bool scx_bpf_consume(u64 dsq_id) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- struct scx_dispatch_q *dsq; -- -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return false; -- -- flush_dispatch_buf(dspc->rq, dspc->rf); -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); -- return false; -- } -- -- if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) { -- /* -- * A successfully consumed task can be dequeued before it starts -- * running while the CPU is trying to migrate other dispatched -- * tasks. Bump nr_tasks to tell balance_scx() to retry on empty -- * local DSQ. -- */ -- dspc->nr_tasks++; -- return true; -- } else { -- return false; -- } --} -- --BTF_SET8_START(scx_kfunc_ids_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) --BTF_ID_FLAGS(func, scx_bpf_consume) --BTF_SET8_END(scx_kfunc_ids_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_dispatch, --}; -- --/** -- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ -- * -- * Iterate over all of the tasks currently enqueued on the local DSQ of the -- * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of -- * processed tasks. Can only be called from ops.cpu_release(). -- */ --u32 scx_bpf_reenqueue_local(void) --{ -- u32 nr_enqueued, i; -- struct rq *rq; -- struct scx_rq *scx_rq; -- -- if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) -- return 0; -- -- rq = cpu_rq(smp_processor_id()); -- lockdep_assert_rq_held(rq); -- scx_rq = rq->scx; -- -- /* -- * Get the number of tasks on the local DSQ before iterating over it to -- * pull off tasks. The enqueue callback below can signal that it wants -- * the task to stay on the local DSQ, and we want to prevent the BPF -- * scheduler from causing us to loop indefinitely. -- */ -- nr_enqueued = scx_rq->local_dsq.nr; -- for (i = 0; i < nr_enqueued; i++) { -- struct task_struct *p; -- -- p = first_local_task(rq); -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- WARN_ON_ONCE(p->scx->holding_cpu != -1); -- dispatch_dequeue(scx_rq, p); -- do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); -- } -- -- return nr_enqueued; --} -- --BTF_SET8_START(scx_kfunc_ids_cpu_release) --BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) --BTF_SET8_END(scx_kfunc_ids_cpu_release) -- --static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_cpu_release, --}; -- --/** -- * scx_bpf_kick_cpu - Trigger reschedule on a CPU -- * @cpu: cpu to kick -- * @flags: SCX_KICK_* flags -- * -- * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or -- * trigger rescheduling on a busy CPU. This can be called from any online -- * scx_ops operation and the actual kicking is performed asynchronously through -- * an irq work. -- */ --void scx_bpf_kick_cpu(s32 cpu, u64 flags) --{ -- struct rq *rq; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d", cpu); -- return; -- } -- -- preempt_disable(); -- rq = this_rq(); -- -- /* -- * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting -- * rq locks. We can probably be smarter and avoid bouncing if called -- * from ops which don't hold a rq lock. -- */ -- cpumask_set_cpu(cpu, rq->scx->cpus_to_kick); -- if (flags & SCX_KICK_PREEMPT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt); -- if (flags & SCX_KICK_WAIT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_wait); -- -- irq_work_queue(&rq->scx->kick_cpus_irq_work); -- preempt_enable(); --} -- --/** -- * scx_bpf_dsq_nr_queued - Return the number of queued tasks -- * @dsq_id: id of the DSQ -- * -- * Return the number of tasks in the DSQ matching @dsq_id. If not found, -- * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops -- * operations. -- */ --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) --{ -- struct scx_dispatch_q *dsq; -- -- lockdep_assert(rcu_read_lock_any_held()); -- -- if (dsq_id == SCX_DSQ_LOCAL) { -- return this_rq()->scx->local_dsq.nr; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (ops_cpu_valid(cpu)) -- return cpu_rq(cpu)->scx->local_dsq.nr; -- } else { -- dsq = find_non_local_dsq(dsq_id); -- if (dsq) -- return dsq->nr; -- } -- return -ENOENT; --} -- --/** -- * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state -- * @cpu: cpu to test and clear idle for -- * -- * Returns %true if @cpu was idle and its idle state was successfully cleared. -- * %false otherwise. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return false; -- } -- -- if (ops_cpu_valid(cpu)) -- return test_and_clear_cpu_idle(cpu); -- else -- return false; --} -- --/** -- * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu -- * @cpus_allowed: Allowed cpumask -- * -- * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked -- * idle cpu number on success. -%EBUSY if no matching cpu was found. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return -EBUSY; -- } -- -- return scx_pick_idle_cpu(cpus_allowed); --} -- --/** -- * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking -- * per-CPU cpumask. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_cpumask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.cpu; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, -- * per-physical-core cpumask. Can be used to determine if an entire physical -- * core is free. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_smtmask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.smt; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to -- * either the percpu, or SMT idle-tracking cpumask. -- */ --void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) --{ -- /* -- * Empty function body because we aren't actually acquiring or -- * releasing a reference to a global idle cpumask, which is read-only -- * in the caller and is never released. The acquire / release semantics -- * here are just used to make the cpumask is a trusted pointer in the -- * caller. -- */ --} -- --struct scx_bpf_error_bstr_bufs { -- u64 data[MAX_BPRINTF_VARARGS]; -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs); -- --/** -- * scx_bpf_error_bstr - Indicate fatal error -- * @fmt: error message format string -- * @data: format string parameters packaged using ___bpf_fill() macro -- * @data__sz: @data len, must end in '__sz' for the verifier -- * -- * Indicate that the BPF scheduler encountered a fatal error and initiate ops -- * disabling. -- */ --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) --{ -- struct scx_bpf_error_bstr_bufs *bufs; -- unsigned long flags; -- struct bpf_bprintf_data args = {}; -- int ret; -- -- local_irq_save(flags); -- bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs); -- -- if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || -- (data__sz && !data)) { -- scx_ops_error("invalid data=%p and data__sz=%u", -- (void *)data, data__sz); -- goto out_restore; -- } -- -- ret = copy_from_kernel_nofault(bufs->data, data, data__sz); -- if (ret) { -- scx_ops_error("failed to read data fields (%d)", ret); -- goto out_restore; -- } -- -- ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args); -- if (ret < 0) { -- scx_ops_error("failed to format prepration (%d)", ret); -- goto out_restore; -- } -- -- ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt, -- args.bin_args); -- bpf_bprintf_cleanup(&args); -- if (ret < 0) { -- scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format", -- fmt, data, data__sz); -- goto out_restore; -- } -- -- scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg); --out_restore: -- local_irq_restore(flags); --} -- --/** -- * scx_bpf_destroy_dsq - Destroy a custom DSQ -- * @dsq_id: DSQ to destroy -- * -- * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with -- * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is -- * empty and no further tasks are dispatched to it. Ignored if called on a DSQ -- * which doesn't exist. Can be called from any online scx_ops operations. -- */ --void scx_bpf_destroy_dsq(u64 dsq_id) --{ -- destroy_dsq(dsq_id); --} -- --/** -- * scx_bpf_task_running - Is task currently running? -- * @p: task of interest -- */ --bool scx_bpf_task_running(const struct task_struct *p) --{ -- return task_rq(p)->curr == p; --} -- --/** -- * scx_bpf_task_cpu - CPU a task is currently associated with -- * @p: task of interest -- */ --s32 scx_bpf_task_cpu(const struct task_struct *p) --{ -- return task_cpu(p); --} -- --/** -- * scx_bpf_task_cgroup - Return the sched cgroup of a task -- * @p: task of interest -- * -- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with -- * from the scheduler's POV. SCX operations should use this function to -- * determine @p's current cgroup as, unlike following @p->cgroups, -- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all -- * rq-locked operations. Can be called on the parameter tasks of rq-locked -- * operations. The restriction guarantees that @p's rq is locked by the caller. -- */ --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) --{ -- struct task_group *tg = p->sched_task_group; -- struct cgroup *cgrp = &cgrp_dfl_root.cgrp; -- -- if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) -- goto out; -- -- /* -- * A task_group may either be a cgroup or an autogroup. In the latter -- * case, @tg->css.cgroup is %NULL. A task_group can't become the other -- * kind once created. -- */ -- if (tg && tg->css.cgroup) -- cgrp = tg->css.cgroup; -- else -- cgrp = &cgrp_dfl_root.cgrp; --out: -- cgroup_get(cgrp); -- return cgrp; --} -- --BTF_SET8_START(scx_kfunc_ids_any) --BTF_ID_FLAGS(func, scx_bpf_kick_cpu) --BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) --BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) --BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu) --BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) --BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) --BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) --BTF_ID_FLAGS(func, scx_bpf_task_running) --BTF_ID_FLAGS(func, scx_bpf_task_cpu) --BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE) --BTF_SET8_END(scx_kfunc_ids_any) -- --static const struct btf_kfunc_id_set scx_kfunc_set_any = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_any, --}; -- --__diag_pop(); -- --/* -- * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set() -- * needs most of the system to be up. -- */ --static int __init register_ext_kfuncs(void) --{ -- int ret; -- -- /* -- * Some kfuncs are context-sensitive and can only be called from -- * specific SCX ops. They are grouped into BTF sets accordingly. -- * Unfortunately, BPF currently doesn't have a way of enforcing such -- * restrictions. Eventually, the verifier should be able to enforce -- * them. For now, register them the same and make each kfunc explicitly -- * check using scx_kf_allowed(). -- */ -- if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_init)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_sleepable)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_enqueue_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_cpu_release)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_any))) { -- pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret); -- return ret; -- } -- -- return 0; --} --__initcall(register_ext_kfuncs); -diff --git b/kernel/sched/ext.h a/kernel/sched/ext.h -deleted file mode 100755 -index fba8ed845f99..000000000000 ---- b/kernel/sched/ext.h -+++ /dev/null -@@ -1,257 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ -- -- --/* -- * Tag marking a kernel function as a kfunc. This is meant to minimize the -- * amount of copy-paste that kfunc authors have to include for correctness so -- * as to avoid issues such as the compiler inlining or eliding either a static -- * kfunc, or a global kfunc in an LTO build. -- */ --#define __bpf_kfunc __used noinline -- --enum scx_wake_flags { -- /* expose select WF_* flags as enums */ -- SCX_WAKE_EXEC = WF_EXEC, -- SCX_WAKE_FORK = WF_FORK, -- SCX_WAKE_TTWU = WF_TTWU, -- SCX_WAKE_SYNC = WF_SYNC, --}; -- --enum scx_enq_flags { -- /* expose select ENQUEUE_* flags as enums */ -- SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, -- SCX_ENQ_HEAD = ENQUEUE_HEAD, -- -- /* high 32bits are SCX specific */ -- -- /* -- * Set the following to trigger preemption when calling -- * scx_bpf_dispatch() with a local dsq as the target. The slice of the -- * current task is cleared to zero and the CPU is kicked into the -- * scheduling path. Implies %SCX_ENQ_HEAD. -- */ -- SCX_ENQ_PREEMPT = 1LLU << 32, -- -- /* -- * The task being enqueued was previously enqueued on the current CPU's -- * %SCX_DSQ_LOCAL, but was removed from it in a call to the -- * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was -- * invoked in a ->cpu_release() callback, and the task is again -- * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the -- * task will not be scheduled on the CPU until at least the next invocation -- * of the ->cpu_acquire() callback. -- */ -- SCX_ENQ_REENQ = 1LLU << 40, -- -- /* -- * The task being enqueued is the only task available for the cpu. By -- * default, ext core keeps executing such tasks but when -- * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with -- * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set. -- * -- * If the BPF scheduler wants to continue executing the task, -- * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately. -- * If the task gets queued on a different dsq or the BPF side, the BPF -- * scheduler is responsible for triggering a follow-up scheduling event. -- * Otherwise, Execution may stall. -- */ -- SCX_ENQ_LAST = 1LLU << 41, -- -- /* -- * A hint indicating that it's advisable to enqueue the task on the -- * local dsq of the currently selected CPU. Currently used by -- * select_cpu_dfl() and together with %SCX_ENQ_LAST. -- */ -- SCX_ENQ_LOCAL = 1LLU << 42, -- -- /* high 8 bits are internal */ -- __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, -- -- SCX_ENQ_CLEAR_OPSS = 1LLU << 56, -- SCX_ENQ_DSQ_PRIQ = 1LLU << 57, --}; -- --enum scx_deq_flags { -- /* expose select DEQUEUE_* flags as enums */ -- SCX_DEQ_SLEEP = DEQUEUE_SLEEP, -- -- /* high 32bits are SCX specific */ -- -- /* -- * The generic core-sched layer decided to execute the task even though -- * it hasn't been dispatched yet. Dequeue from the BPF side. -- */ -- SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, --}; -- --enum scx_tg_flags { -- SCX_TG_ONLINE = 1U << 0, -- SCX_TG_INITED = 1U << 1, --}; -- --enum scx_kick_flags { -- SCX_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -- SCX_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ --}; -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --extern const struct sched_class ext_sched_class; --extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops; --extern const struct file_operations sched_ext_fops; --extern unsigned long scx_watchdog_timeout; --extern unsigned long scx_watchdog_timestamp; -- --DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); --DECLARE_STATIC_KEY_FALSE(__scx_switched_all); --#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) --#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) -- --DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); -- --bool task_on_scx(struct task_struct *p); --void scx_pre_fork(struct task_struct *p); --int scx_fork(struct task_struct *p); --void scx_post_fork(struct task_struct *p); --void scx_cancel_fork(struct task_struct *p); --int scx_check_setscheduler(struct task_struct *p, int policy); --bool scx_can_stop_tick(struct rq *rq); --void init_sched_ext_class(void); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...); --#define scx_ops_error(fmt, args...) \ -- scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args) -- --void __scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active); -- --static inline void scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active) --{ -- if (!scx_enabled()) -- return; --#ifdef CONFIG_SMP -- /* -- * Pairs with the smp_load_acquire() issued by a CPU in -- * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -- * resched. -- */ -- smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1); --#endif -- if (!static_branch_unlikely(&scx_ops_cpu_preempt)) -- return; -- __scx_notify_pick_next_task(rq, p, active); --} -- --//extern void scx_scheduler_tick(void); --static inline void scx_notify_sched_tick(void) --{ -- unsigned long last_check; -- -- if (!scx_enabled()) -- return; -- //scx_scheduler_tick(); -- -- last_check = scx_watchdog_timestamp; -- if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "watchdog failed to check in for %u.%03us", -- dur_ms / 1000, dur_ms % 1000); -- } --} -- --static inline const struct sched_class *next_active_class(const struct sched_class *class) --{ -- class++; -- if (scx_switched_all() && class == &fair_sched_class) -- class++; -- if (!scx_enabled() && class == &ext_sched_class) -- class++; -- return class; --} -- --#define for_active_class_range(class, _from, _to) \ -- for (class = (_from); class != (_to); class = next_active_class(class)) -- --#define for_each_active_class(class) \ -- for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -- --/* -- * SCX requires a balance() call before every pick_next_task() call including -- * when waking up from idle. -- */ --#define for_balance_class_range(class, prev_class, end_class) \ -- for_active_class_range(class, (prev_class) > &ext_sched_class ? \ -- &ext_sched_class : (prev_class), (end_class)) -- --#ifdef CONFIG_SCHED_CORE --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi); --#endif -- --#else /* CONFIG_SCHED_CLASS_EXT */ -- --#define scx_enabled() false --#define scx_switched_all() false -- --static inline void scx_pre_fork(struct task_struct *p) {} --static inline int scx_fork(struct task_struct *p) { return 0; } --static inline void scx_post_fork(struct task_struct *p) {} --static inline void scx_cancel_fork(struct task_struct *p) {} --static inline int scx_check_setscheduler(struct task_struct *p, -- int policy) { return 0; } --static inline bool scx_can_stop_tick(struct rq *rq) { return true; } --static inline void init_sched_ext_class(void) {} --static inline void scx_notify_pick_next_task(struct rq *rq, -- const struct task_struct *p, -- const struct sched_class *active) {} --static inline void scx_notify_sched_tick(void) {} -- --#define for_each_active_class for_each_class --#define for_balance_class_range for_class_range -- --#endif /* CONFIG_SCHED_CLASS_EXT */ -- --#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) --void __scx_update_idle(struct rq *rq, bool idle); -- --static inline void scx_update_idle(struct rq *rq, bool idle) --{ -- if (scx_enabled()) -- __scx_update_idle(rq, idle); --} --#else --static inline void scx_update_idle(struct rq *rq, bool idle) {} --#endif -- --#ifdef CONFIG_CGROUP_SCHED --#ifdef CONFIG_EXT_GROUP_SCHED --int scx_tg_online(struct task_group *tg); --void scx_tg_offline(struct task_group *tg); --int scx_cgroup_can_attach(struct cgroup_taskset *tset); --void scx_move_task(struct task_struct *p); --void scx_cgroup_finish_attach(void); --void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); --void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); --#else /* CONFIG_EXT_GROUP_SCHED */ --static inline int scx_tg_online(struct task_group *tg) { return 0; } --static inline void scx_tg_offline(struct task_group *tg) {} --static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } --static inline void scx_move_task(struct task_struct *p) {} --static inline void scx_cgroup_finish_attach(void) {} --static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} --static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} --#endif /* CONFIG_EXT_GROUP_SCHED */ --#endif /* CONFIG_CGROUP_SCHED */ -diff --git b/kernel/sched/hmbird.h a/kernel/sched/hmbird.h -new file mode 100755 -index 000000000000..fa76c7f09977 ---- /dev/null -+++ a/kernel/sched/hmbird.h -@@ -0,0 +1,342 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#ifndef _HMBIRD_H_ -+#define _HMBIRD_H_ -+ -+/* -+ * Tag marking a kernel function as a kfunc. This is meant to minimize the -+ * amount of copy-paste that kfunc authors have to include for correctness so -+ * as to avoid issues such as the compiler inlining or eliding either a static -+ * kfunc, or a global kfunc in an LTO build. -+ */ -+ -+#define DEBUG_INTERNAL (1 << 0) -+#define DEBUG_INFO_TRACE (1 << 1) -+#define DEBUG_INFO_SYSTRACE (1 << 2) -+ -+#define NUMS_CGROUP_KINDS (512) -+#define SLIM_FOR_SGAME (1 << 0) -+#define SLIM_FOR_GENSHIN (1 << 1) -+ -+#ifdef MAX_TASK_NR -+#define MAX_KEY_THREAD_RECORD ((MAX_TASK_NR + 1) >> 1) -+#else -+#define MAX_KEY_THREAD_RECORD (1 << 3) -+#endif /* MAX_TASK_NR */ -+ -+#define TOP_TASK_SHIFT (8) -+#define TOP_TASK_MAX (1 << TOP_TASK_SHIFT) -+#ifndef TOP_TASK_BITS_MASK -+#define TOP_TASK_BITS_MASK (TOP_TASK_MAX - 1) -+#endif /* TOP_TASK_BITS_MASK */ -+ -+extern struct md_info_t *md_info; -+ -+#define debug_enabled() \ -+ (unlikely(hmbirdcore_debug)) -+ -+#define hmbird_debug(fmt, ...) \ -+ pr_info(":"fmt, ##__VA_ARGS__) -+ -+#define hmbird_err(id, fmt, ...) \ -+do { \ -+ pr_err(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_deferred_err(id, fmt, ...) \ -+do { \ -+ printk_deferred(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \ -+do { \ -+ if (unlikely(cond)) { \ -+ hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+ } \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_info_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_info_trace(fmt, ...) -+#endif -+ -+#define hmbird_info_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+ } \ -+} while (0) -+ -+#define hmbird_output_systrace(fmt, ...) \ -+do { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_internal_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_internal_trace(fmt, ...) -+#endif -+ -+#define hmbird_internal_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) { \ -+ hmbird_output_systrace(fmt, ##__VA_ARGS__); \ -+ } \ -+} while (0) -+ -+enum hmbird_wake_flags { -+ /* expose select WF_* flags as enums */ -+ HMBIRD_WAKE_EXEC = WF_EXEC, -+ HMBIRD_WAKE_FORK = WF_FORK, -+ HMBIRD_WAKE_TTWU = WF_TTWU, -+ HMBIRD_WAKE_SYNC = WF_SYNC, -+}; -+ -+enum hmbird_enq_flags { -+ /* expose select ENQUEUE_* flags as enums */ -+ HMBIRD_ENQ_WAKEUP = ENQUEUE_WAKEUP, -+ HMBIRD_ENQ_HEAD = ENQUEUE_HEAD, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * Set the following to trigger preemption when calling -+ * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the -+ * current task is cleared to zero and the CPU is kicked into the -+ * scheduling path. Implies %HMBIRD_ENQ_HEAD. -+ */ -+ HMBIRD_ENQ_PREEMPT = 1LLU << 32, -+ HMBIRD_ENQ_REENQ = 1LLU << 40, -+ HMBIRD_ENQ_LAST = 1LLU << 41, -+ -+ /* -+ * A hint indicating that it's advisable to enqueue the task on the -+ * local dsq of the currently selected CPU. Currently used by -+ * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST. -+ */ -+ HMBIRD_ENQ_LOCAL = 1LLU << 42, -+ -+ /* high 8 bits are internal */ -+ __HMBIRD_ENQ_INTERNAL_MASK = 0xffLLU << 56, -+ -+ HMBIRD_ENQ_CLEAR_OPSS = 1LLU << 56, -+ HMBIRD_ENQ_DSQ_PRIQ = 1LLU << 57, -+}; -+ -+enum hmbird_deq_flags { -+ /* expose select DEQUEUE_* flags as enums */ -+ HMBIRD_DEQ_SLEEP = DEQUEUE_SLEEP, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * The generic core-sched layer decided to execute the task even though -+ * it hasn't been dispatched yet. Dequeue from the BPF side. -+ */ -+ HMBIRD_DEQ_CORE_SCHED_EXEC = 1LLU << 32, -+}; -+ -+enum hmbird_kick_flags { -+ HMBIRD_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -+ HMBIRD_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ -+}; -+ -+enum hmbird_task_prop_type { -+ HMBIRD_TASK_PROP_COMMON, -+ HMBIRD_TASK_PROP_PIPELINE, -+ HMBIRD_TASK_PROP_DEBUG_OR_LOG, -+ HMBIRD_TASK_PROP_HIGH_LOAD, -+ HMBIRD_TASK_PROP_IO, -+ HMBIRD_TASK_PROP_NETWORK, -+ HMBIRD_TASK_PROP_PERIODIC, -+ HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL, -+ HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL, -+ HMBIRD_TASK_PROP_ISOLATE, -+ HMBIRD_TASK_PROP_MAX, -+}; -+ -+enum hmbird_preempt_policy_id { -+ HMBIRD_PREEMPT_POLICY_NONE, -+ HMBIRD_PREEMPT_POLICY_PRIO_BASED, -+}; -+ -+extern const struct sched_class hmbird_sched_class; -+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops; -+extern const struct file_operations sched_hmbird_fops; -+extern unsigned long hmbird_watchdog_timeout; -+extern unsigned long hmbird_watchdog_timestamp; -+ -+enum scx_rq_flags { -+ HMBIRD_RQ_CAN_STOP_TICK = 1 << 0, -+}; -+ -+struct hmbird_rq { -+ struct hmbird_dispatch_q local_dsq; -+ struct list_head watchdog_list; -+ u64 ops_qseq; -+ u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -+ u32 nr_running; -+ u32 flags; -+ bool cpu_released; -+ cpumask_var_t cpus_to_kick; -+ cpumask_var_t cpus_to_preempt; -+ cpumask_var_t cpus_to_wait; -+ u64 pnt_seq; -+ u64 *prev_runnable_sum_fixed; -+ u32 *prev_window_size; -+ struct irq_work kick_cpus_irq_work; -+ bool pipeline; -+ bool exclusive; -+ bool period_disallow; -+ bool nonperiod_disallow; -+ struct rq *rq; -+ struct hmbird_sched_rq_stats *srq; -+}; -+ -+struct hmbird_sched_change_guard { -+ struct task_struct *p; -+ struct rq *rq; -+ bool queued; -+ bool running; -+ bool done; -+}; -+ -+extern struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -+ -+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags); -+ -+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -+ for (struct hmbird_sched_change_guard __cg = \ -+ hmbird_sched_change_guard_init(__rq, __p, __flags); \ -+ !__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags)) -+ -+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+bool task_on_hmbird(struct task_struct *p); -+int hmbird_pre_fork(struct task_struct *p); -+void hmbird_post_fork(struct task_struct *p); -+void hmbird_cancel_fork(struct task_struct *p); -+int hmbird_check_setscheduler(struct task_struct *p, int policy); -+bool hmbird_can_stop_tick(struct rq *rq); -+void init_sched_hmbird_class(void); -+ -+void hmbird_ops_exit(void); -+#define hmbird_ops_error(fmt, ...) \ -+do { \ -+ hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__); \ -+ hmbird_ops_exit(); \ -+} while (0) -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active); -+ -+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight) -+{ -+ return clamp_t(unsigned long, -+ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -+ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); -+} -+ -+static inline void hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active) -+{ -+ if (!hmbird_enabled()) -+ return; -+#ifdef CONFIG_SMP -+ /* -+ * Pairs with the smp_load_acquire() issued by a CPU in -+ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -+ * resched. -+ */ -+ smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1); -+#endif -+ if (!static_branch_unlikely(&hmbird_ops_cpu_preempt)) -+ return; -+ __hmbird_notify_pick_next_task(rq, p, active); -+} -+ -+extern void hmbird_scheduler_tick(void); -+void scan_timeout(struct rq *rq); -+void hmbird_notify_sched_tick(void); -+ -+static inline const struct sched_class *next_active_class(const struct sched_class *class) -+{ -+ class++; -+ -+ if (hmbird_enabled() && class == &fair_sched_class) -+ class++; -+ if (!hmbird_enabled() && class == &hmbird_sched_class) -+ class++; -+ return class; -+} -+ -+#define for_active_class_range(class, _from, _to) \ -+ for (class = (_from); class != (_to); class = next_active_class(class)) -+ -+#define for_each_active_class(class) \ -+ for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -+ -+/* -+ * HMBIRD requires a balance() call before every pick_next_task() call including -+ * when waking up from idle. -+ */ -+#define for_balance_class_range(class, prev_class, end_class) \ -+ for_active_class_range(class, (prev_class) > &hmbird_sched_class ? \ -+ &hmbird_sched_class : (prev_class), (end_class)) -+ -+#define MIN_CGROUP_DL_IDX (5) /* 8ms */ -+#define DEFAULT_CGROUP_DL_IDX (8) /* 64ms */ -+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS]; -+ -+ -+void __hmbird_update_idle(struct rq *rq, bool idle); -+ -+static inline void hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ if (hmbird_enabled()) -+ __hmbird_update_idle(rq, idle); -+} -+ -+int hmbird_ctrl(bool enable); -+ -+int hmbird_tg_online(struct task_group *tg); -+ -+ -+static inline u16 hmbird_task_util(struct task_struct *p) -+{ -+ return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0; -+} -+u16 hmbird_cpu_util(int cpu); -+ -+bool get_hmbird_ops_enabled(void); -+bool get_non_hmbird_task(void); -+void set_cpu_cluster(u64 cpu_cluster); -+#endif /*_EXT_H_*/ -diff --git b/kernel/sched/hmbird/hmbird.c a/kernel/sched/hmbird/hmbird.c -new file mode 100755 -index 000000000000..86f9fd092424 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird.c -@@ -0,0 +1,4354 @@ -+// SPDX-License-Identifier: GPL-2.0 -+ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#include -+#include -+ -+#include "slim.h" -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+#include -+ -+#define CLUSTER_SEPARATE -+ -+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0); -+atomic_t non_hmbird_task; -+atomic_t hmbird_module_loaded = ATOMIC_INIT(0); -+int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0}; -+ -+enum hmbird_internal_consts { -+ HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ, -+}; -+ -+enum hmbird_ops_enable_state { -+ HMBIRD_OPS_PREPPING, -+ HMBIRD_OPS_ENABLING, -+ HMBIRD_OPS_ENABLED, -+ HMBIRD_OPS_DISABLING, -+ HMBIRD_OPS_DISABLED, -+}; -+ -+static inline void put_hmbird_ts(struct task_struct *p) -+{ -+ kfree((void *)p->android_oem_data1[HMBIRD_TS_IDX]); -+ p->android_oem_data1[HMBIRD_TS_IDX] = 0; -+} -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+/* -+ * hmbird_entity->ops_state -+ * -+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler. -+ * State transitions look as follows: -+ * -+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING -+ * ^ | | -+ * | v v -+ * \-------------------------------/ -+ * -+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -+ * sites for explanations on the conditions being waited upon and why they are -+ * safe. Transitions out of them into NONE or QUEUED must store_release and the -+ * waiters should load_acquire. -+ * -+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether -+ * any given task can be dispatched by the BPF scheduler at all times and thus -+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core -+ * can safely reject invalid dispatches. -+ */ -+enum hmbird_ops_state { -+ HMBIRD_OPSS_NONE, /* owned by the HMBIRD core */ -+ HMBIRD_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -+ HMBIRD_OPSS_QUEUED, /* owned by the BPF scheduler */ -+ HMBIRD_OPSS_DISPATCHING, /* in transit back to the HMBIRD core */ -+ -+ /* -+ * QSEQ brands each QUEUED instance so that, when dispatch races -+ * dequeue/requeue, the dispatcher can tell whether it still has a claim -+ * on the task being dispatched. -+ */ -+ HMBIRD_OPSS_QSEQ_SHIFT = 2, -+ HMBIRD_OPSS_STATE_MASK = (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1, -+ HMBIRD_OPSS_QSEQ_MASK = ~HMBIRD_OPSS_STATE_MASK, -+}; -+ -+enum switch_stat { -+ HMBIRD_DISABLED, -+ HMBIRD_SWITCH_PREP, -+ HMBIRD_RQ_SWITCH_BEGIN, -+ HMBIRD_RQ_SWITCH_DONE, -+ HMBIRD_ENABLED, -+}; -+enum switch_stat curr_ss; -+ -+/* -+ * During exit, a task may schedule after losing its PIDs. When disabling the -+ * BPF scheduler, we need to be able to iterate tasks in every state to -+ * guarantee system safety. Maintain a dedicated task list which contains every -+ * task between its fork and eventual free. -+ */ -+DEFINE_SPINLOCK(hmbird_tasks_lock); -+static LIST_HEAD(hmbird_tasks); -+ -+/* ops enable/disable */ -+static struct kthread_worker *hmbird_ops_helper; -+static DEFINE_MUTEX(hmbird_ops_enable_mutex); -+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem); -+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED); -+ -+static bool hmbird_warned_zero_slice; -+enum hmbird_switch_type sw_type; -+static int skip_num[MAX_GLOBAL_DSQS]; -+static int big_distribute_mask_prev; -+static int little_distribute_mask_prev; -+ -+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0); -+ -+/* -+ * The maximum amount of time in jiffies that a task may be runnable without -+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger -+ * hmbird_ops_error(). -+ */ -+unsigned long hmbird_watchdog_timeout; -+ -+/* -+ * The last time the delayed work was run. This delayed work relies on -+ * ksoftirqd being able to run to service timer interrupts, so it's possible -+ * that this work itself could get wedged. To account for this, we check that -+ * it's not stalled in the timer tick, and trigger an error if it is. -+ */ -+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES; -+ -+static struct delayed_work hmbird_watchdog_work; -+static struct work_struct hmbird_err_exit_work; -+ -+/* idle tracking */ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_CPUMASK_OFFSTACK -+#define CL_ALIGNED_IF_ONSTACK -+#else -+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp -+#endif -+ -+static struct { -+ cpumask_var_t cpu; -+ cpumask_var_t smt; -+} idle_masks CL_ALIGNED_IF_ONSTACK; -+ -+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus; -+#endif /* CONFIG_SMP */ -+ -+/* dispatch queues */ -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global; -+ -+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128}; -+u32 pcp_dsq_deadline = 20; -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS]; -+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq); -+ -+static u64 max_hmbird_dsq_internal_id; -+ -+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/ -+#define CLUSTER_SEPARATE_IDX (8) -+#define GDSQS_ID_BASE (3) -+#define UX_COMPATIBLE_IDX (4) -+#define NON_PERIOD_START (5) -+#define NON_PERIOD_END (MAX_GLOBAL_DSQS) -+#define CREATE_DSQ_LEVEL_WITHIN (1) -+ -+struct hmbird_sched_info { -+ spinlock_t lock; -+ int curr_idx[2]; -+ int rtime[MAX_GLOBAL_DSQS]; -+}; -+ -+struct pcp_sched_info { -+ s64 pcp_seq; -+ int rtime; -+ bool pcp_round; -+}; -+ -+/* -+ * pcp_info may rw by another cpu. -+ * protected by rq->lock. -+ */ -+atomic64_t pcp_dsq_round; -+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info); -+ -+struct md_info_t *md_info; -+ -+static int b_rescue_l, l_rescue_b; -+static struct hmbird_sched_info sinfo; -+ -+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC; -+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = { -+ 0, 0, 0, 0, 0, -+ 32 * NSEC_PER_MSEC, -+ 20 * NSEC_PER_MSEC, -+ 14 * NSEC_PER_MSEC, -+ 8 * NSEC_PER_MSEC, -+ 6 * NSEC_PER_MSEC -+}; -+ -+struct cluster_ctx { -+ /* cpu-dsq map must within [lower, upper) */ -+ int upper; -+ int lower; -+ int tidx; -+}; -+ -+enum stat_items { -+ GLOBAL_STAT, -+ CPU_ALLOW_FAIL, -+ RT_CNT, -+ KEY_TASK_CNT, -+ SWITCH_IDX, -+ TIMEOUT_CNT, -+ -+ TOTAL_DSP_CNT, -+ MOVE_RQ_CNT, -+ SELECT_CPU, -+ -+ DWORD_STAT_END = SELECT_CPU, -+ -+ GDSQ_CNT, -+ ERR_IDX, -+ PCP_TIMEOUT_CNT, -+ PCP_LDSQ_CNT, -+ PCP_ENQL_CNT, -+ -+ MAX_ITEMS, -+}; -+static DEFINE_SPINLOCK(stats_lock); -+static char *stats_str[MAX_ITEMS] = { -+ "global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt", -+ "switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt", -+ "select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt", -+ "pcp_ldsq_cnt", "pcp_enql_cnt" -+}; -+ -+ -+struct stats_struct { -+ u64 global_stat[2]; -+ u64 cpu_allow_fail[2]; -+ u64 rt_cnt[2]; -+ u64 key_task_cnt[2]; -+ u64 switch_idx[2]; -+ u64 timeout_cnt[2]; -+ -+ u64 total_dsp_cnt[2]; -+ u64 move_rq_cnt[2]; -+ u64 select_cpu[2]; -+ -+ u64 gdsq_count[MAX_GLOBAL_DSQS][2]; -+ u64 err_idx[5]; -+ u64 pcp_timeout_cnt[NR_CPUS]; -+ u64 pcp_ldsq_count[NR_CPUS][2]; -+ u64 pcp_enql_cnt[NR_CPUS]; -+} stats_data; -+ -+static struct { -+ cpumask_var_t ex_free; -+ cpumask_var_t exclusive; -+ cpumask_var_t partial; -+ cpumask_var_t big; -+ cpumask_var_t little; -+} iso_masks __read_mostly; -+ -+/* -+ * Need more synchronization for these two variables? -+ * I choose not to. -+ */ -+static int l_need_rescue, b_need_rescue; -+ -+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...) \ -+{ \ -+ char buf[MAX_FATAL_INFO]; \ -+ \ -+ scnprintf(buf, MAX_FATAL_INFO, fmt, ##args); \ -+ hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf); \ -+ trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable), \ -+ READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf); \ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); \ -+} \ -+ -+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2) -+{ -+ int c1, c2; -+ struct cpumask mask = {.bits = {0}}; -+ struct cpumask tmp = {.bits = {0}}; -+ -+ if (!slim_stats) -+ return false; -+ -+ if (!rq1 || !rq2) -+ return false; -+ -+ c1 = cpu_of(rq1); -+ c2 = cpu_of(rq2); -+ cpumask_set_cpu(c1, &mask); -+ cpumask_set_cpu(c2, &mask); -+ -+ if (cpumask_and(&tmp, iso_masks.little, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.big, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.partial, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ return false; -+} -+ -+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu) -+{ -+ unsigned long flags; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ if (!slim_stats) -+ return; -+ -+ switch (item) { -+ case GLOBAL_STAT: -+ fallthrough; -+ case CPU_ALLOW_FAIL: -+ fallthrough; -+ case RT_CNT: -+ fallthrough; -+ case KEY_TASK_CNT: -+ fallthrough; -+ case SWITCH_IDX: -+ fallthrough; -+ case TIMEOUT_CNT: -+ fallthrough; -+ case TOTAL_DSP_CNT: -+ fallthrough; -+ case MOVE_RQ_CNT: -+ fallthrough; -+ case SELECT_CPU: -+ pval = pbase + item * 2 + idx; -+ break; -+ case GDSQ_CNT: -+ pval = &stats_data.gdsq_count[dsq_id][idx]; -+ break; -+ case ERR_IDX: -+ pval = &stats_data.err_idx[idx]; -+ break; -+ case PCP_TIMEOUT_CNT: -+ pval = &stats_data.pcp_timeout_cnt[cpu]; -+ break; -+ case PCP_LDSQ_CNT: -+ pval = &stats_data.pcp_ldsq_count[cpu][idx]; -+ break; -+ case PCP_ENQL_CNT: -+ pval = &stats_data.pcp_enql_cnt[cpu]; -+ break; -+ default: -+ return; -+ } -+ -+ spin_lock_irqsave(&stats_lock, flags); -+ *pval += 1; -+ spin_unlock_irqrestore(&stats_lock, flags); -+} -+ -+static inline bool handle_ret(int ret, int *idx, int len) -+{ -+ if (ret < 0 || ret >= len - *idx) -+ return true; -+ *idx += ret; -+ return false; -+} -+ -+#define PRINT_INTV (5 * HZ) -+void stats_print(char *buf, int len) -+{ -+ int idx = 0, i, j, ret; -+ int item = 0; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ for (item = 0; item < MAX_ITEMS; item++) { -+ if (item <= DWORD_STAT_END) { -+ pval = pbase + item * 2; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n", -+ stats_str[item], pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == GDSQ_CNT) { -+ for (j = 0; j < MAX_GLOBAL_DSQS; j++) { -+ pval = (u64 *)&stats_data.gdsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == ERR_IDX) { -+ pval = (u64 *)&stats_data.err_idx; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n", -+ stats_str[item], pval[0], -+ pval[1], pval[2], pval[3], pval[4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == PCP_TIMEOUT_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_timeout_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_LDSQ_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_ldsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_ENQL_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_enql_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } -+ } -+ -+ if (!md_info) { -+ buf[idx] = '\0'; -+ return; -+ } -+ -+ ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_SWITCHS; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "sw_rec[%d] = %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.sw_rec[i].switch_at, -+ md_info->kern_dump.sw_rec[i].is_success, -+ md_info->kern_dump.sw_rec[i].end_state, -+ md_info->kern_dump.sw_rec[i].switch_reason); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_EXCEP_ID; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "excep[%d] = %llu %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.excep_rec[i][0], -+ md_info->kern_dump.excep_rec[i][1], -+ md_info->kern_dump.excep_rec[i][2], -+ md_info->kern_dump.excep_rec[i][3], -+ md_info->kern_dump.excep_rec[i][4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n", -+ i, md_info->kern_dump.excep_idx[i]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ -+ buf[idx] = '\0'; -+} -+ -+enum cpu_type { -+ LITTLE, -+ BIG, -+ PARTIAL, -+ EXCLUSIVE, -+ EX_FREE, -+ INVALID -+}; -+ -+enum dsq_type { -+ GLOBAL_DSQ, -+ PCP_DSQ, -+ OTHER, -+ MAX_DSQ_TYPE, -+}; -+ -+static enum cpu_type cpu_cluster(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (get_hmbird_rq(rq)->exclusive) { -+ return EXCLUSIVE; -+ } else { -+ if (cpumask_test_cpu(cpu, iso_masks.little)) -+ return LITTLE; -+ else if (cpumask_test_cpu(cpu, iso_masks.big)) -+ return BIG; -+ else if (cpumask_test_cpu(cpu, iso_masks.partial)) -+ return PARTIAL; -+ else if (cpumask_test_cpu(cpu, iso_masks.exclusive)) -+ return EXCLUSIVE; -+ else if (cpumask_test_cpu(cpu, iso_masks.ex_free)) -+ return EX_FREE; -+ } -+ return INVALID; -+} -+ -+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq) -+{ -+ if (!dsq) -+ return OTHER; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= GDSQS_ID_BASE) && -+ ((dsq->id & 0xff) < MAX_GLOBAL_DSQS)) -+ return GLOBAL_DSQ; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) && -+ ((dsq->id & 0xff) < max_hmbird_dsq_internal_id)) -+ return PCP_DSQ; -+ -+ return OTHER; -+} -+ -+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq) -+{ -+ enum dsq_type type; -+ -+ type = get_dsq_type(dsq); -+ switch (type) { -+ case GLOBAL_DSQ: -+ case PCP_DSQ: -+ return (dsq->id & 0xff) - GDSQS_ID_BASE; -+ default: -+ return -1; -+ } -+ return -1; -+} -+ -+static void update_cpus_idle(bool set, struct cpumask *mask) -+{ -+ int cpu; -+ struct rq *rq; -+ -+ if (set) { -+ for_each_cpu(cpu, mask) { -+ rq = cpu_rq(cpu); -+ if (is_idle_task(rq->curr)) -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ } -+ } else -+ cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask); -+} -+ -+static void set_partial_status(bool enable, bool little, bool big) -+{ -+ WRITE_ONCE(partial_enable, enable); -+ WRITE_ONCE(l_need_rescue, little); -+ WRITE_ONCE(b_need_rescue, big); -+} -+ -+static bool is_little_need_rescue(void) -+{ -+ return READ_ONCE(l_need_rescue); -+} -+ -+static bool is_big_need_rescue(void) -+{ -+ return READ_ONCE(b_need_rescue); -+} -+ -+static bool is_partial_enabled(void) -+{ -+ return READ_ONCE(partial_enable); -+} -+ -+static bool is_partial_cpu(int cpu) -+{ -+ return cpumask_test_cpu(cpu, iso_masks.partial); -+} -+ -+static void set_iso_par_free(bool enable) -+{ -+ WRITE_ONCE(isolate_ctrl, enable); -+} -+ -+static bool is_iso_par_free(void) -+{ -+ return READ_ONCE(isolate_ctrl); -+} -+ -+static bool skip_update_idle(void) -+{ -+ int cpu = smp_processor_id(); -+ enum cpu_type type = cpu_cluster(cpu); -+ -+ if ((type == EXCLUSIVE && !is_iso_par_free()) || -+ /* partial enable may changed during idle, it doesn't matter. */ -+ (!is_partial_enabled() && type == PARTIAL)) -+ return true; -+ -+ return false; -+} -+ -+static void init_isolate_cpus(void) -+{ -+ WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -+ cpumask_set_cpu(0, iso_masks.big); -+ cpumask_set_cpu(1, iso_masks.big); -+ cpumask_set_cpu(2, iso_masks.big); -+ cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(4, iso_masks.big); -+ cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(6, iso_masks.partial); -+ cpumask_set_cpu(7, iso_masks.ex_free); -+} -+ -+extern spinlock_t css_set_lock; -+ -+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp) -+{ -+ int i; -+ struct cgroup *anc; -+ -+ spin_lock_irq(&css_set_lock); -+ for (i = 0; i < cgrp->level; i++) { -+ anc = cgrp->ancestors[i]; -+ if (anc->level != CREATE_DSQ_LEVEL_WITHIN) -+ continue; -+ cgroup_get(anc); -+ spin_unlock_irq(&css_set_lock); -+ return anc; -+ } -+ spin_unlock_irq(&css_set_lock); -+ hmbird_err(NO_CGROUP_L1, ":error cgroup = %s\n", cgrp->kn->name); -+ return NULL; -+} -+ -+#define PCP_IDX_BIT (1 << 31) -+ -+static bool is_pcp_rt(struct task_struct *p) -+{ -+ return rt_prio(p->prio) && (p->nr_cpus_allowed == 1); -+} -+ -+static bool is_pcp_idx(int idx) -+{ -+ return idx >= MAX_GLOBAL_DSQS; -+} -+ -+static bool is_critical_system_task(struct task_struct *p) -+{ -+ int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK; -+ -+ return (p->prio < (MAX_RT_PRIO >> 1) && -+ (sp_dl < SCHED_PROP_DEADLINE_LEVEL3)); -+} -+ -+#define ISOLATE_TASK_TOP_BIT (1 << 17) -+#define PIPELINE_TASK_TOP_BIT (1 << 9) -+static bool is_pipeline_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT); -+} -+ -+static bool is_isolate_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT); -+} -+ -+static bool is_critical_app_task_without_isolate(struct task_struct *p) -+{ -+ return task_is_top_task(p) && !is_isolate_task(p); -+} -+ -+static int find_idx_from_task(struct task_struct *p) -+{ -+ int idx, cpu; -+ int sp_dl; -+ struct task_group *tg = p->sched_task_group; -+ -+ if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) { -+ cpu = cpumask_any(p->cpus_ptr); -+ idx = cpu + MAX_GLOBAL_DSQS; -+ return idx; -+ } -+ -+ if (is_critical_system_task(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL0; -+ goto done; -+ } -+ -+ if (is_critical_app_task_without_isolate(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL1; -+ goto done; -+ } -+ -+ if (p->pid == scx_systemui_pid) { -+ idx = SCHED_PROP_DEADLINE_LEVEL4; -+ goto done; -+ } -+ -+ sp_dl = hmbird_get_dsq_id(p); -+ if (sp_dl) { -+ idx = sp_dl; -+ goto done; -+ } -+ -+ if (rt_prio(p->prio)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL3; -+ goto done; -+ } -+ -+ if (tg && tg->css.cgroup && tg->css.cgroup->kn) { -+ if (likely(tg->css.cgroup->kn->id >= 0 && -+ tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS)) -+ idx = cgroup_ids_table[tg->css.cgroup->kn->id]; -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ -+done: -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) { -+ hmbird_err(DSQ_ID_ERR, " : idx error, idx = %d-----\n", idx); -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } -+ return idx; -+} -+ -+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p) -+{ -+ int idx; -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq; -+ -+ if (!p) -+ return NULL; -+ -+ idx = find_idx_from_task(p); -+ if (is_pcp_idx(idx)) { -+ idx -= MAX_GLOBAL_DSQS; -+ dsq = &per_cpu(pcp_ldsq, idx); -+ get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq); -+ slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx); -+ } else { -+ dsq = &gdsqs[idx]; -+ get_hmbird_ts(p)->gdsq_idx = idx; -+ slim_stats_record(GDSQ_CNT, 0, idx, 0); -+ } -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ dsq->last_consume_at = jiffies; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ return dsq; -+} -+ -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq); -+ -+static void set_partial_rescue(bool p_state, bool l_over, bool b_over) -+{ -+ set_partial_status(p_state, l_over, b_over); -+ update_cpus_idle(p_state, iso_masks.partial); -+ hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled()); -+ hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue()); -+ hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue()); -+} -+ -+static void free_isocpu(bool enable) -+{ -+ set_iso_par_free(enable); -+ update_cpus_idle(enable, iso_masks.ex_free); -+ hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free()); -+} -+ -+inline u64 get_hmbird_cpu_util(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed) -+ return 0; -+ u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed); -+ u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size); -+ -+ do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT); -+ -+ return prev_runnable_sum_fixed; -+} -+ -+static inline unsigned int get_scaling_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->max; -+} -+ -+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static u64 get_cpus_max_util(struct cpumask *mask) -+{ -+ int cpu; -+ u64 max = 0; -+ u64 util, ratio; -+ unsigned long effective_cap = 0; -+ -+ for_each_cpu(cpu, mask) { -+ if (slim_walt_ctrl) -+ slim_get_cpu_util(cpu, &util); -+ else -+ util = get_hmbird_cpu_util(cpu); -+ -+ /* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/ -+ if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu))) -+ effective_cap = arch_scale_cpu_capacity(cpu); -+ else -+ effective_cap = arch_scale_cpu_capacity(cpu) * -+ get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu); -+ -+ ratio = util * 100 / effective_cap; -+ hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util); -+ hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n", -+ cpu, (u64)arch_scale_cpu_capacity(cpu)); -+ hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n", -+ cpu, (u64)effective_cap); -+ -+ if (ratio > 100) -+ ratio = 100; -+ -+ if (ratio > max) -+ max = ratio; -+ } -+ return max; -+} -+ -+static bool cluster_need_rescue(struct cpumask *mask, int hres) -+{ -+ return get_cpus_max_util(mask) > hres; -+} -+ -+void partial_dynamic_ctrl(void) -+{ -+ u64 lmax = 0, bmax = 0; -+ bool l_over, l_under, b_over, b_under; -+ static bool last_l_over, last_b_over; -+ static unsigned long last_check; -+ -+ /* Check partial every jiffies. need lock? */ -+ if (time_before_eq(jiffies, READ_ONCE(last_check))) -+ return; -+ -+ WRITE_ONCE(last_check, jiffies); -+ -+ lmax = get_cpus_max_util(iso_masks.little); -+ l_over = lmax > parctrl_high_ratio_l; -+ l_under = lmax < parctrl_low_ratio_l; -+ -+ bmax = get_cpus_max_util(iso_masks.big); -+ b_over = bmax > parctrl_high_ratio; -+ b_under = bmax < parctrl_low_ratio; -+ -+ if (is_partial_enabled() && (l_over || b_over)) { -+ if (last_l_over != l_over || last_b_over != b_over) -+ set_partial_rescue(true, l_over, b_over); -+ } else if (!is_partial_enabled() && (l_over || b_over)) { -+ set_partial_rescue(true, l_over, b_over); -+ } else if (is_partial_enabled() && l_under && b_under) { -+ set_partial_rescue(false, false, false); -+ } -+ last_l_over = l_over; -+ last_b_over = b_over; -+ -+ if (is_partial_enabled()) { -+ if (cpumask_empty(iso_masks.partial)) { -+ bmax = bmax > lmax ? bmax : lmax; -+ } else { -+ bmax = get_cpus_max_util(iso_masks.partial); -+ } -+ if (!is_iso_par_free() && bmax > isoctrl_high_ratio) { -+ hmbird_info_trace("partial max = %llu\n", bmax); -+ free_isocpu(true); -+ } else if (is_iso_par_free() && bmax < isoctrl_low_ratio) -+ free_isocpu(false); -+ } else if (is_iso_par_free()) { -+ free_isocpu(false); -+ } -+} -+ -+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx) -+{ -+ hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx); -+} -+ -+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx) -+{ -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[idx])) { -+ slim_stats_record(GDSQ_CNT, 1, idx, 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ int i; -+ -+ for (i = 0; i < UX_COMPATIBLE_IDX; i++) { -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ slim_stats_record(GDSQ_CNT, 1, i, 0); -+ return 1; -+ } -+ } -+ return 0; -+} -+ -+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) { -+ slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ goto clear_timeout; -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) -+ goto clear_timeout; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ hmbird_info_trace( -+ "dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n", -+ dsq_id_to_internal(dsq), entity->task->comm, -+ jiffies, msecs_to_jiffies(deadline), entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1); -+ return; -+ -+clear_timeout: -+ hmbird_info_trace("dsq[%d] clear timeout\n", -+ dsq_id_to_internal(dsq)); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0); -+ dsq->is_timeout = false; -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq)); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime) -+{ -+ hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime); -+} -+ -+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any) -+{ -+ bool is_timeout; -+ int cpu = cpu_of(rq); -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu); -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ is_timeout = dsq->is_timeout; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ /* -+ * dsq->is_timeout may change here, let it be. -+ * it won't cause serious problems. -+ * the same for consume_dispatch_q later. -+ */ -+ if (!is_timeout && !any) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, dsq)) { -+ if (is_timeout) { -+ hmbird_info_trace("dsq[%d] consume a pcp timeout task\n", -+ dsq_id_to_internal(dsq)); -+ update_timeout_stats(rq, dsq, pcp_dsq_deadline); -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu); -+ } -+ slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu); -+ return 1; -+ } -+ /* -+ * No pcp task, clear quota. -+ */ -+ if (any) { -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+ } -+ return 0; -+} -+ -+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf) -+{ -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ } -+ return 0; -+} -+ -+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf, -+ int tmp, int cidx, int tidx) -+{ -+ unsigned long flags; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) { -+ if (tmp != cidx) { -+ spin_lock(&sinfo.lock); -+ sinfo.curr_idx[tidx] = tmp; -+ spin_unlock(&sinfo.lock); -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]); -+ slim_stats_record(SWITCH_IDX, 0, 0, 0); -+ } -+ slim_stats_record(GDSQ_CNT, 1, tmp, 0); -+ -+ raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags); -+ gdsqs[tmp].last_consume_at = jiffies; -+ raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags); -+ return 1; -+ } -+ return 0; -+ -+} -+ -+static int get_cidx(struct cluster_ctx *ctx) -+{ -+ int cidx; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ if (cidx < ctx->lower || cidx >= ctx->upper) { -+ sinfo.curr_idx[ctx->tidx] = ctx->lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]); -+ slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ } -+ spin_unlock(&sinfo.lock); -+ -+ return cidx; -+} -+ -+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = CLUSTER_SEPARATE_IDX; -+ if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) { -+ pr_debug(" %s iso_masks.little first is %d\n", -+ __func__, cpumask_first(iso_masks.little)); -+ ctx->upper = NON_PERIOD_END; -+ } -+ ctx->tidx = 0; -+ break; -+ case LITTLE: -+ ctx->lower = CLUSTER_SEPARATE_IDX; -+ ctx->upper = NON_PERIOD_END; -+ if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) { -+ pr_debug(" %s iso_masks.big first is %d", -+ __func__, cpumask_first(iso_masks.big)); -+ ctx->lower = NON_PERIOD_START; -+ } -+ ctx->tidx = 1; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ case LITTLE: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = NON_PERIOD_END; -+ ctx->tidx = 0; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ if (cluster_separate) { -+ return gen_cluster_ctx_separate(ctx, type); -+ } else { -+ return gen_cluster_ctx_common(ctx, type); -+ } -+} -+ -+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ int i; -+ bool is_timeout; -+ unsigned long flags; -+ struct cluster_ctx ctx; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ /* Third param means only consume timeout pcp dsq. */ -+ if (consume_pcp_dsq(rq, rf, false)) -+ return 1; -+ -+ for (i = ctx.lower; i < ctx.upper; i++) { -+ raw_spin_lock_irqsave(&gdsqs[i].lock, flags); -+ is_timeout = gdsqs[i].is_timeout; -+ raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags); -+ /* gdsqs[i].is_timeout may change here, let it be... */ -+ if (is_timeout) { -+ /* -+ * consume_dispatch_q will acquire dsq-lock, -+ * So cannot keep lock here, annoy enough. -+ * may rewrite a consume_dispatch_q_locked, TODO. -+ */ -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ hmbird_info_trace("dsq[%d] consume a timeout task\n", i); -+ slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0); -+ update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]); -+ return 1; -+ } -+ } -+ } -+ return 0; -+} -+ -+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ struct cluster_ctx ctx; -+ int cidx; -+ int tmp; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ cidx = get_cidx(&ctx); -+ tmp = cidx; -+ do { -+ if (check_pcp_dsq_round(rq, rf)) -+ return 1; -+ if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx)) -+ return 1; -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[tmp] = 0; -+ systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]); -+ tmp++; -+ if (tmp >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ tmp = ctx.lower; -+ } -+ spin_unlock(&sinfo.lock); -+ } while (tmp != cidx); -+ -+ return consume_pcp_dsq(rq, rf, true); -+} -+ -+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ enum cpu_type type = cpu_cluster(cpu_of(rq)); -+ bool period_allowed = !get_hmbird_rq(rq)->period_disallow; -+ bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow; -+ -+ switch (type) { -+ case EX_FREE: -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ break; -+ case EXCLUSIVE: -+ if (!is_iso_par_free()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ } -+ /* Only non-period task can run on isolate cpus */ -+ if (!READ_ONCE(iso_free_rescue)) { -+ if (is_big_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ } else { -+ /* Free run, can run on any task. */ -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ case PARTIAL: -+ if (!is_partial_enabled()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ return 0; -+ } -+ if (is_big_need_rescue()) { -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ -+ case BIG: -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ if (is_iso_par_free() || (is_little_need_rescue() && -+ !cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) { -+ hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++); -+ return 1; -+ } -+ } -+ break; -+ -+ case LITTLE: -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ -+ if (is_iso_par_free() || (is_big_need_rescue() && -+ !cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) { -+ hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++); -+ return 1; -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return 0; -+} -+ -+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf) -+{ -+ return consume_hmbird_global_dsq(rq, rf); -+} -+ -+ -+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time) -+{ -+ int idx; -+ -+ /* which dsq belongs to while task enqueue, task will consume its running time. */ -+ idx = get_hmbird_ts(p)->gdsq_idx; -+ /* Only non-period dsq share running time between each other. */ -+ if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id) -+ return; -+ -+ if (idx >= MAX_GLOBAL_DSQS) { -+ per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } else { -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[idx] += exec_time; -+ spin_unlock(&sinfo.lock); -+ systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]); -+ } -+} -+ -+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type) -+{ -+ int cidx; -+ struct cluster_ctx ctx; -+ int cpu = cpu_of(rq); -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ if (cidx < ctx.lower || cidx >= ctx.upper) { -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ slim_stats_record(ERR_IDX, ctx.tidx, 0, 0); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ } -+ -+ while (1) { -+ if (per_cpu(pcp_info, cpu).pcp_round) { -+ if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) { -+ hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n", -+ cpu, per_cpu(pcp_info, cpu).rtime); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } -+ } -+ if (sinfo.rtime[cidx] < dsq_quota[cidx]) -+ break; -+ -+ /* clear current dsq rtime */ -+ sinfo.rtime[cidx] = 0; -+ systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]); -+ -+ sinfo.curr_idx[ctx.tidx]++; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", -+ ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ } -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ slim_stats_record(SWITCH_IDX, 1, 0, 0); -+ } -+ spin_unlock(&sinfo.lock); -+} -+ -+ -+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p) -+{ -+ enum cpu_type type; -+ -+ if (!rq || !p) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ switch (type) { -+ case PARTIAL: -+ return; -+ case EXCLUSIVE: -+ return; -+ case EX_FREE: -+ return; -+ default: -+ break; -+ } -+ update_dsq_idx(rq, p, type); -+} -+ -+ -+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ int dsq_id; -+ -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo) || dsq->is_timeout) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ hmbird_deferred_err(SCAN_ENTITY_NULL, -+ " : entity is NULL, dsq->id = %llu\n", dsq->id); -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ dsq->is_timeout = true; -+ dsq_id = dsq_id_to_internal(dsq); -+ hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n", -+ dsq_id, entity->task->comm, jiffies, entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1); -+ raw_spin_unlock(&dsq->lock); -+ -+ return true; -+} -+ -+void scan_timeout(struct rq *rq) -+{ -+ int i; -+ int cpu = cpu_of(rq); -+ struct hmbird_dispatch_q *dsq; -+ static u64 last_scan_at; -+ static DEFINE_PER_CPU(u64, pcp_last_scan_at); -+ -+ if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu))) -+ return; -+ per_cpu(pcp_last_scan_at, cpu) = jiffies; -+ -+ dsq = &per_cpu(pcp_ldsq, cpu); -+ scan_dsq_timeout(rq, dsq, pcp_dsq_deadline); -+ -+ if (time_before_eq(jiffies, (unsigned long)last_scan_at)) -+ return; -+ last_scan_at = jiffies; -+ -+ for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) { -+ dsq = &gdsqs[i]; -+ scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]); -+ } -+} -+ -+/*******************************Initialize***********************************/ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id); -+static void init_dsq_at_boot(void) -+{ -+ int i, cpu; -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i)); -+ } -+ for_each_possible_cpu(cpu) -+ init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i + cpu)); -+ -+ max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu; -+ spin_lock_init(&sinfo.lock); -+} -+ -+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgrp_name_to_idx(struct cgroup *cgrp) -+{ -+ int idx; -+ -+ if (!cgrp) -+ return -1; -+ -+ if (!strcmp(cgrp->kn->name, "display") -+ || !strcmp(cgrp->kn->name, "multimedia")) -+ idx = 5; /* 8ms */ -+ else if (!strcmp(cgrp->kn->name, "top-app") -+ || !strcmp(cgrp->kn->name, "ss-top")) -+ idx = 6; /* 16ms */ -+ else if (!strcmp(cgrp->kn->name, "ssfg") -+ || !strcmp(cgrp->kn->name, "foreground")) -+ idx = 7; /* 32ms */ -+ else if (!strcmp(cgrp->kn->name, "bg") -+ || !strcmp(cgrp->kn->name, "log") -+ || !strcmp(cgrp->kn->name, "dex2oat") -+ || !strcmp(cgrp->kn->name, "background")) -+ idx = 9; /* 128ms */ -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */ -+ -+ return idx; -+} -+ -+static void init_root_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX); -+} -+ -+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) { -+ pr_err("%s idx err!\n", __func__); -+ return; -+ } -+ -+ if (cgroup_ids_table[cgrp->kn->id] == -1) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp)); -+} -+ -+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ struct cgroup *l1cgrp; -+ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ l1cgrp = cgroup_ancestor_l1(cgrp); -+ if (l1cgrp) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp)); -+ cgroup_put(l1cgrp); -+} -+ -+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg) -+{ -+ switch (cgrp->level) { -+ case 0: -+ init_root_tg(cgrp, tg); -+ break; -+ case 1: -+ init_level1_tg(cgrp, tg); -+ break; -+ default: -+ init_child_tg(cgrp, tg); -+ break; -+ } -+} -+ -+/**************************************************************************/ -+ -+struct hmbird_task_iter { -+ struct hmbird_entity cursor; -+ struct task_struct *locked; -+ struct rq *rq; -+ struct rq_flags rf; -+}; -+ -+/** -+ * hmbird_task_iter_init - Initialize a task iterator -+ * @iter: iterator to init -+ * -+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized, -+ * @iter must eventually be exited with hmbird_task_iter_exit(). -+ * -+ * hmbird_tasks_lock may be released between this and the first next() call or -+ * between any two next() calls. If hmbird_tasks_lock is released between two -+ * next() calls, the caller is responsible for ensuring that the task being -+ * iterated remains accessible either through RCU read lock or obtaining a -+ * reference count. -+ * -+ * All tasks which existed when the iteration started are guaranteed to be -+ * visited as long as they still exist. -+ */ -+static void hmbird_task_iter_init(struct hmbird_task_iter *iter) -+{ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR }; -+ list_add(&iter->cursor.tasks_node, &hmbird_tasks); -+ iter->locked = NULL; -+} -+ -+/** -+ * hmbird_task_iter_exit - Exit a task iterator -+ * @iter: iterator to exit -+ * -+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held. -+ * If the iterator holds a task's rq lock, that rq lock is released. See -+ * hmbird_task_iter_init() for details. -+ */ -+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ if (list_empty(cursor)) -+ return; -+ -+ list_del_init(cursor); -+} -+ -+/** -+ * hmbird_task_iter_next - Next task -+ * @iter: iterator to walk -+ * -+ * Visit the next task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ struct hmbird_entity *pos; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ list_for_each_entry(pos, cursor, tasks_node) { -+ if (&pos->tasks_node == &hmbird_tasks) -+ return NULL; -+ if (!(pos->flags & HMBIRD_TASK_CURSOR)) { -+ list_move(cursor, &pos->tasks_node); -+ return pos->task; -+ } -+ } -+ -+ /* can't happen, should always terminate at hmbird_tasks above */ -+ hmbird_deferred_err(ITER_RET_NULL, " : unreachable path in scx_task_iter_next\n"); -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered - Next non-idle task -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ while ((p = hmbird_task_iter_next(iter))) { -+ if (!is_idle_task(p)) -+ return p; -+ } -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init() -+ * for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ p = hmbird_task_iter_next_filtered(iter); -+ if (!p) -+ return NULL; -+ -+ iter->rq = task_rq_lock(p, &iter->rf); -+ iter->locked = p; -+ return p; -+} -+ -+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void) -+{ -+ return atomic_read(&hmbird_ops_enable_state_var); -+} -+ -+static enum hmbird_ops_enable_state -+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to) -+{ -+ return atomic_xchg(&hmbird_ops_enable_state_var, to); -+} -+ -+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to, -+ enum hmbird_ops_enable_state from) -+{ -+ int from_v = from; -+ -+ return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to); -+} -+ -+static bool hmbird_ops_disabling(void) -+{ -+ return false; -+} -+ -+/** -+ * wait_ops_state - Busy-wait the specified ops state to end -+ * @p: target task -+ * @opss: state to wait the end of -+ * -+ * Busy-wait for @p to transition out of @opss. This can only be used when the -+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also -+ * has load_acquire semantics to ensure that the caller can see the updates made -+ * in the enqueueing and dispatching paths. -+ */ -+static void wait_ops_state(struct task_struct *p, u64 opss) -+{ -+ do { -+ cpu_relax(); -+ } while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss); -+} -+ -+ -+static void update_curr_hmbird(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ u64 now = rq_clock_task(rq); -+ u64 delta_exec; -+ -+ if (time_before_eq64(now, curr->se.exec_start)) -+ return; -+ -+ delta_exec = now - curr->se.exec_start; -+ curr->se.exec_start = now; -+ update_runningtime(rq, curr, delta_exec); -+ curr->se.sum_exec_runtime += delta_exec; -+ account_group_exec_runtime(curr, delta_exec); -+ cgroup_account_cputime(curr, delta_exec); -+ -+ if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF) -+ get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec); -+ -+ trace_sched_stat_runtime(curr, delta_exec, 0); -+} -+ -+static bool hmbird_dsq_priq_less(struct rb_node *node_a, -+ const struct rb_node *node_b) -+{ -+ const struct hmbird_entity *a = -+ container_of(node_a, struct hmbird_entity, dsq_node.priq); -+ const struct hmbird_entity *b = -+ container_of(node_b, struct hmbird_entity, dsq_node.priq); -+ -+ return time_before64(a->dsq_vtime, b->dsq_vtime); -+} -+ -+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ bool is_local = dsq->id == HMBIRD_DSQ_LOCAL; -+ unsigned long flags; -+ -+ hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq || -+ !list_empty(&get_hmbird_ts(p)->dsq_node.fifo), -+ "task = %s, dsq->id = %llu\n", p->comm, dsq->id); -+ hmbird_cond_deferred_err(ENQ_EXIST2, -+ (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq), -+ "task = %s\n", p->comm); -+ -+ if (!is_local) { -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ); -+ hmbird_ops_error(": %s\n", -+ "attempting to dispatch to a destroyed dsq"); -+ /* fall back to the global dsq */ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ dsq = &hmbird_dsq_global; -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ } -+ } -+ -+ if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) { -+ get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ; -+ rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq, -+ hmbird_dsq_priq_less); -+ } else { -+ if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT)) -+ list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ else -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ } -+ dsq->nr++; -+ get_hmbird_ts(p)->dsq = dsq; -+ -+ /* -+ * We're transitioning out of QUEUEING or DISPATCHING. store_release to -+ * match waiters' load_acquire. -+ */ -+ if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS) -+ atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE); -+ -+ if (is_local) { -+ struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq); -+ struct rq *rq = hmbird->rq; -+ bool preempt = false; -+ -+ if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr && -+ rq->curr->sched_class == &hmbird_sched_class) { -+ get_hmbird_ts(rq->curr)->slice = 0; -+ preempt = true; -+ } -+ -+ if (preempt || sched_class_above(&hmbird_sched_class, -+ rq->curr->sched_class)) -+ resched_curr(rq); -+ } else { -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ } -+} -+ -+static void task_unlink_from_dsq(struct task_struct *p, -+ struct hmbird_dispatch_q *dsq) -+{ -+ if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) { -+ rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ; -+ } else { -+ list_del_init(&get_hmbird_ts(p)->dsq_node.fifo); -+ } -+} -+ -+static bool task_linked_on_dsq(struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+} -+ -+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq; -+ bool is_local = dsq == &hmbird_rq->local_dsq; -+ -+ if (!dsq) { -+ hmbird_cond_deferred_err(TASK_LINKED1, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ /* -+ * When dispatching directly from the BPF scheduler to a local -+ * DSQ, the task isn't associated with any DSQ but -+ * @get_hmbird_ts(p)->holding_cpu may be set under the protection of -+ * %HMBIRD_OPSS_DISPATCHING. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu >= 0) -+ get_hmbird_ts(p)->holding_cpu = -1; -+ return; -+ } -+ -+ if (!is_local) -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ /* -+ * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node -+ * can't change underneath us. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu < 0) { -+ /* @p must still be on @dsq, dequeue */ -+ hmbird_cond_deferred_err(TASK_UNLINKED, -+ !task_linked_on_dsq(p), "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ } else { -+ /* -+ * We're racing against dispatch_to_local_dsq() which already -+ * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the -+ * holding_cpu which tells dispatch_to_local_dsq() that it lost -+ * the race. -+ */ -+ hmbird_cond_deferred_err(TASK_LINKED2, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ get_hmbird_ts(p)->holding_cpu = -1; -+ } -+ get_hmbird_ts(p)->dsq = NULL; -+ -+ if (!is_local) -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+ -+static bool test_rq_online(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ return rq->online; -+#else -+ return true; -+#endif -+} -+ -+static void refill_task_slice(struct task_struct *p) -+{ -+ if (is_isolate_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO; -+ else if (is_pipeline_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2; -+ else -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+} -+ -+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -+ int sticky_cpu) -+{ -+ struct hmbird_dispatch_q *d; -+ s32 cpu = -1; -+ -+ hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), -+ "task = %s\n", p->comm); -+ -+ if (is_pcp_rt(p)) { -+ /* Enqueue percpu rt task to local directly. */ -+ /* Or cause a bug when disable dispatch. */ -+ if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ } -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (cpu >= 0) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ -+ if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ /* rq migration */ -+ if (sticky_cpu == cpu_of(rq)) -+ goto local_norefill; -+ /* -+ * If !rq->online, we already told the BPF scheduler that the CPU is -+ * offline. We're just trying to on/offline the CPU. Don't bother the -+ * BPF scheduler. -+ */ -+ if (unlikely(!test_rq_online(rq))) -+ goto local; -+ -+ /* see %HMBIRD_OPS_ENQ_LAST */ -+ if (enq_flags & HMBIRD_ENQ_LAST) -+ goto local; -+ -+ if (enq_flags & HMBIRD_ENQ_LOCAL) -+ goto local; -+ else -+ goto global; -+local: -+ /* -+ * For task-ordering, slice refill must be treated as implying the end -+ * of the current slice. Otherwise, the longer @p stays on the CPU, the -+ * higher priority it becomes from hmbird_prio_less()'s POV. -+ */ -+ refill_task_slice(p); -+local_norefill: -+ dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags); -+ slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq)); -+ return; -+ -+global: -+ d = find_dsq_from_task(p); -+ if (d) { -+ refill_task_slice(p); -+ dispatch_enqueue(d, p, enq_flags); -+ return; -+ } -+ slim_stats_record(GLOBAL_STAT, 0, 0, 0); -+ refill_task_slice(p); -+ dispatch_enqueue(&hmbird_dsq_global, p, enq_flags); -+} -+ -+static bool watchdog_task_watched(const struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->watchdog_node); -+} -+ -+static void watchdog_watch_task(struct rq *rq, struct task_struct *p) -+{ -+ lockdep_assert_rq_held(rq); -+ if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ get_hmbird_ts(p)->runnable_at = jiffies; -+ clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list); -+} -+ -+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) -+{ -+ list_del_init(&get_hmbird_ts(p)->watchdog_node); -+ if (reset_timeout) -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags) -+{ -+ int sticky_cpu = get_hmbird_ts(p)->sticky_cpu; -+ -+ enq_flags |= get_hmbird_rq(rq)->extra_enq_flags; -+ -+ if (sticky_cpu >= 0) -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ -+ /* -+ * Restoring a running task will be immediately followed by -+ * set_next_task_hmbird() which expects the task to not be on the BPF -+ * scheduler as tasks can only start running through local DSQs. Force -+ * direct-dispatch into the local DSQ by setting the sticky_cpu. -+ */ -+ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -+ sticky_cpu = cpu_of(rq); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_UNWATCHED, -+ !watchdog_task_watched(p), "task = %s\n", p->comm); -+ return; -+ } -+ -+ watchdog_watch_task(rq, p); -+ set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ get_hmbird_rq(rq)->nr_running++; -+ add_nr_running(rq, 1); -+ -+ do_enqueue_task(rq, p, enq_flags, sticky_cpu); -+} -+ -+static void ops_dequeue(struct task_struct *p, u64 deq_flags) -+{ -+ u64 opss; -+ -+ watchdog_unwatch_task(p, false); -+ -+ /* acquire ensures that we see the preceding updates on QUEUED */ -+ opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state); -+ -+ switch (opss & HMBIRD_OPSS_STATE_MASK) { -+ case HMBIRD_OPSS_NONE: -+ break; -+ case HMBIRD_OPSS_QUEUEING: -+ /* -+ * QUEUEING is started and finished while holding @p's rq lock. -+ * As we're holding the rq lock now, we shouldn't see QUEUEING. -+ */ -+ hmbird_deferred_err(DEQ_DEQING, " : unreachable path in %s\n", __func__); -+ break; -+ case HMBIRD_OPSS_QUEUED: -+ if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss, -+ HMBIRD_OPSS_NONE)) -+ break; -+ fallthrough; -+ case HMBIRD_OPSS_DISPATCHING: -+ /* -+ * If @p is being dispatched from the BPF scheduler to a DSQ, -+ * wait for the transfer to complete so that @p doesn't get -+ * added to its DSQ after dequeueing is complete. -+ * -+ * As we're waiting on DISPATCHING with the rq locked, the -+ * dispatching side shouldn't try to lock the rq while -+ * DISPATCHING is set. See dispatch_to_local_dsq(). -+ * -+ * DISPATCHING shouldn't have qseq set and control can reach -+ * here with NONE @opss from the above QUEUED case block. -+ * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss. -+ */ -+ wait_ops_state(p, HMBIRD_OPSS_DISPATCHING); -+ hmbird_cond_deferred_err(HMBIRD_OPN, -+ atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE, -+ "task = %s\n", p->comm); -+ break; -+ } -+} -+ -+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ -+ if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p), -+ "task = %s\n", p->comm); -+ return; -+ } -+ -+ ops_dequeue(p, deq_flags); -+ -+ if (slim_walt_ctrl) { -+ if (task_current(rq, p)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ if (deq_flags & HMBIRD_DEQ_SLEEP) -+ set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else -+ clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), -+ (unsigned long *)&get_hmbird_ts(p)->flags); -+ -+ clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm); -+ hmbird_rq->nr_running--; -+ sub_nr_running(rq, 1); -+ dispatch_dequeue(hmbird_rq, p); -+} -+ -+static void yield_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ get_hmbird_ts(p)->slice = 0; -+} -+ -+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to) -+{ -+ return false; -+} -+ -+#ifdef CONFIG_SMP -+/** -+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -+ * @rq: rq to move the task into, currently locked -+ * @p: task to move -+ * @enq_flags: %HMBIRD_ENQ_* -+ * -+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller -+ * must: -+ * -+ * 1. Start with exclusive access to @p either through its DSQ lock or -+ * %HMBIRD_OPSS_DISPATCHING flag. -+ * -+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id(). -+ * -+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't -+ * deadlock with dequeue. -+ * -+ * 4. Lock @rq and the task_rq from #3. -+ * -+ * 5. Call this function. -+ * -+ * Returns %true if @p was successfully moved. %false after racing dequeue and -+ * losing. -+ */ -+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ struct rq *task_rq; -+ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * If dequeue got to @p while we were trying to lock both rq's, it'd -+ * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have -+ * updated it to different values afterwards, as this operation can't be -+ * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become -+ * raw_smp_processor_id() again before we're done. Thus, we can tell -+ * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is -+ * still raw_smp_processor_id(). -+ * -+ * See dispatch_dequeue() for the counterpart. -+ */ -+ if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id())) -+ return false; -+ -+ /* @p->rq couldn't have changed if we're still the holding cpu */ -+ task_rq = task_rq(p); -+ lockdep_assert_rq_held(task_rq); -+ deactivate_task(task_rq, p, 0); -+ set_task_cpu(p, cpu_of(rq)); -+ get_hmbird_ts(p)->sticky_cpu = cpu_of(rq); -+ -+ /* -+ * We want to pass hmbird-specific enq_flags but activate_task() will -+ * truncate the upper 32 bit. As we own @rq, we can pass them through -+ * @get_hmbird_rq(rq)->extra_enq_flags instead. -+ */ -+ hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags, -+ "task = %s\n", p->comm); -+ get_hmbird_rq(rq)->extra_enq_flags = enq_flags; -+ activate_task(rq, p, 0); -+ get_hmbird_rq(rq)->extra_enq_flags = 0; -+ -+ return true; -+} -+ -+#endif /* CONFIG_SMP */ -+ -+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu) -+{ -+ int fitable = 1; -+ -+ return fitable; -+} -+ -+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq, -+ struct hmbird_dispatch_q *dsq) -+{ -+ bool dsq_misfit; -+ int cpu = cpu_of(rq); -+ u64 task_util = 0; -+ struct cluster_ctx ctx; -+ int dsq_int = dsq_id_to_internal(dsq); -+ -+ if (!cpumask_test_cpu(cpu, iso_masks.little)) -+ return false; -+ if (p->pid == scx_systemui_pid) -+ return true; -+ -+ gen_cluster_ctx(&ctx, BIG); -+ dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 && -+ dsq_int <= SCHED_PROP_DEADLINE_LEVEL4); -+#ifdef CLUSTER_SEPARATE -+ /* In rescue mode, little will consume big cluster's dsq.*/ -+ dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper); -+#endif -+ if (!dsq_misfit) -+ return false; -+ -+ if (p) { -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &task_util); -+ else -+ task_util = get_hmbird_ts(p)->demand_scaled; -+ } -+ -+ if (task_util <= misfit_ds) -+ return false; -+ -+ hmbird_info_trace(":task %s can't run on cpu%d, util = %llu\n", -+ p->comm, cpu, task_util); -+ return true; -+} -+ -+static int check_misfit_task_on_fake_big(struct task_struct *p, struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (likely(p->pid != scx_systemui_pid)) -+ return false; -+ -+ if (topology_cluster_id(num_possible_cpus() - 1) > 1 && -+ arch_scale_cpu_capacity(cpu) == arch_scale_cpu_capacity(0) && -+ (parctrl_high_ratio <= 0 || parctrl_high_ratio_l <= 0)) -+ return true; -+ -+ return false; -+} -+ -+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq) -+{ -+ if (!cpumask_test_cpu(cpu_of(rq), task_cpu_possible_mask(p))) -+ return false; -+ -+ if (!task_fits_cpu_hmbird(p, cpu_of(rq))) -+ return false; -+ -+ if (check_misfit_task_on_little(p, rq, dsq)) -+ return false; -+ -+ if (check_misfit_task_on_fake_big(p, rq)) -+ return false; -+ -+ return likely(test_rq_online(rq)); -+} -+ -+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return; -+ -+ if (add) -+ skipn[idx]++; -+ else -+ skipn[idx] = 0; -+} -+ -+static bool skip_too_much(struct hmbird_dispatch_q *dsq) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return false; -+ -+ if (skip_num[idx] > 3) { -+ skip_num[idx] = 0; -+ return true; -+ } -+ -+ return false; -+} -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rb_node *rb_node; -+ struct rq *task_rq; -+ unsigned long flags; -+ bool moved = false; -+ struct task_struct *may_fit = NULL; -+ int skip = 0; -+ -+retry: -+ if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -+ return false; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) { -+ set_skip_num(dsq, skip_num, (bool)may_fit); -+ goto this_rq; -+ } -+ if (skip_too_much(dsq)) -+ goto remote_rq; -+ if (!may_fit) -+ may_fit = p; -+ if (++skip <= 3) -+ continue; -+ /* -+ * If the recent 3 tasks not fit, use the first one. -+ * and clear the skip, because the first one is consumed. -+ */ -+ set_skip_num(dsq, skip_num, false); -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ /* No more task, use the first may fit task.*/ -+ if (may_fit) { -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ -+ for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) -+ goto this_rq; -+ goto remote_rq; -+ } -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ return false; -+ -+this_rq: -+ /* @dsq is locked and @p is on this rq */ -+ hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo); -+ dsq->nr--; -+ hmbird_rq->local_dsq.nr++; -+ get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ -+remote_rq: -+#ifdef CONFIG_SMP -+ if (cpu_same_cluster_stat(p, rq, task_rq)) -+ slim_stats_record(MOVE_RQ_CNT, 0, 0, 0); -+ else -+ slim_stats_record(MOVE_RQ_CNT, 1, 0, 0); -+ /* -+ * @dsq is locked and @p is on a remote rq. @p is currently protected by -+ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -+ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -+ * rq lock or fail, do a little dancing from our side. See -+ * move_task_to_local_dsq(). -+ */ -+ hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id(); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ rq_unpin_lock(rq, rf); -+ double_lock_balance(rq, task_rq); -+ rq_repin_lock(rq, rf); -+ -+ moved = move_task_to_local_dsq(rq, p, 0); -+ -+ double_unlock_balance(rq, task_rq); -+#endif /* CONFIG_SMP */ -+ if (likely(moved)) { -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ } -+ may_fit = NULL; -+ goto retry; -+} -+ -+ -+static int balance_one(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf, bool local) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class; -+ -+ if (!hmbird_rq) -+ return 1; -+ -+ lockdep_assert_rq_held(rq); -+ -+ if (static_branch_unlikely(&hmbird_ops_cpu_preempt) && -+ unlikely(get_hmbird_rq(rq)->cpu_released)) { -+ /* -+ * If the previous sched_class for the current CPU was not HMBIRD, -+ * notify the BPF scheduler that it again has control of the -+ * core. This callback complements ->cpu_release(), which is -+ * emitted in hmbird_notify_pick_next_task(). -+ */ -+ get_hmbird_rq(rq)->cpu_released = false; -+ } -+ -+ if (prev_on_hmbird) -+ update_curr_hmbird(rq); -+ /* if there already are tasks to run, nothing to do */ -+ if (hmbird_rq->local_dsq.nr) -+ return 1; -+ -+ if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) { -+ slim_stats_record(GLOBAL_STAT, 1, 0, 0); -+ return 1; -+ } -+ -+ if (consume_dispatch_global(rq, rf)) -+ return 1; -+ -+ return 0; -+} -+ -+static int balance_hmbird(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf) -+{ -+ return balance_one(rq, prev, rf, true); -+} -+ -+/* -+ * output task util to systrace, only for debug mode. -+ * we can not output too many logs to systrace buffer even in debug mode -+ * only output debug-info while it exceed misfit_ds. -+ */ -+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p) -+{ -+ static DEFINE_PER_CPU(int, is_last_exceed); -+ int cpu = cpu_of(rq); -+ u64 util = 0; -+ -+ if (likely(!debug_enabled())) -+ return; -+ -+ if (!p) -+ return; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ util = uclamp_rq_util_with(rq, util, p); -+ -+ if (util >= misfit_ds) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util); -+ per_cpu(is_last_exceed, cpu) = true; -+ } else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0); -+ per_cpu(is_last_exceed, cpu) = false; -+ } else { -+ } -+} -+ -+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first) -+{ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ /* -+ * Core-sched might decide to execute @p before it is -+ * dispatched. Call ops_dequeue() to notify the BPF scheduler. -+ */ -+ ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC); -+ dispatch_dequeue(get_hmbird_rq(rq), p); -+ } -+ -+ p->se.exec_start = rq_clock_task(rq); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK); -+ } -+ -+ watchdog_unwatch_task(p, true); -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx); -+ systrace_output_cpu_ds(rq, p); -+ /* -+ * @p is getting newly scheduled or got kicked after someone updated its -+ * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird(). -+ */ -+ if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) != -+ (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) { -+ if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) -+ get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK; -+ else -+ get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK; -+ -+ sched_update_tick_dependency(rq); -+ } -+ -+ p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime; -+} -+ -+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p) -+{ -+ update_curr_hmbird(rq); -+ -+ update_dispatch_dsq_info(rq, p); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ watchdog_watch_task(rq, p); -+ -+ if (is_pipeline_task(p)) { -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1); -+ return; -+ } -+ -+ /* -+ * If we're in the pick_next_task path, balance_hmbird() should -+ * have already populated the local DSQ if there are any other -+ * available tasks. If empty, tell ops.enqueue() that @p is the -+ * only one available for this cpu. ops.enqueue() should put it -+ * on the local DSQ so that the subsequent pick_next_task_hmbird() -+ * can find the task unless it wants to trigger a separate -+ * follow-up scheduling event. -+ */ -+ if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1); -+ else -+ do_enqueue_task(rq, p, 0, -1); -+ } -+} -+ -+static struct task_struct *first_local_task(struct rq *rq) -+{ -+ struct rb_node *rb_node; -+ struct hmbird_entity *entity; -+ -+ if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) { -+ entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo, -+ struct hmbird_entity, dsq_node.fifo); -+ return entity->task; -+ } -+ -+ rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq); -+ if (rb_node) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ return entity->task; -+ } -+ return NULL; -+} -+ -+static struct task_struct *pick_next_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p; -+ -+ p = first_local_task(rq); -+ if (!p) -+ return NULL; -+ -+ if (unlikely(!get_hmbird_ts(p)->slice)) { -+ if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice) -+ hmbird_warned_zero_slice = true; -+ -+ refill_task_slice(p); -+ } -+ -+ set_next_task_hmbird(rq, p, true); -+ -+ return p; -+} -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task, -+ const struct sched_class *active) -+{ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * The callback is conceptually meant to convey that the CPU is no -+ * longer under the control of HMBIRD. Therefore, don't invoke the -+ * callback if the CPU is staying on HMBIRD, or going idle (in which -+ * case the HMBIRD scheduler has actively decided not to schedule any -+ * tasks on the CPU). -+ */ -+ if (likely(active >= &hmbird_sched_class)) -+ return; -+ -+ /* -+ * At this point we know that HMBIRD was preempted by a higher priority -+ * sched_class, so invoke the ->cpu_release() callback if we have not -+ * done so already. We only send the callback once between HMBIRD being -+ * preempted, and it regaining control of the CPU. -+ * -+ * ->cpu_release() complements ->cpu_acquire(), which is emitted the -+ * next time that balance_hmbird() is invoked. -+ */ -+ if (!get_hmbird_rq(rq)->cpu_released) -+ get_hmbird_rq(rq)->cpu_released = true; -+} -+ -+#ifdef CONFIG_SMP -+ -+static bool test_and_clear_cpu_idle(int cpu) -+{ -+ if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -+ if (cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ return true; -+ } else { -+ return false; -+ } -+} -+ -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) -+{ -+ int cpu; -+ -+ do { -+ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -+ if (cpu >= nr_cpu_ids) -+ return -EBUSY; -+ } while (!test_and_clear_cpu_idle(cpu)); -+ -+ return cpu; -+} -+ -+ -+static bool prev_cpu_misfit(int prev) -+{ -+ if (!is_partial_enabled() && is_partial_cpu(prev)) -+ return true; -+ -+ return false; -+} -+ -+static int heavy_rt_placement(struct task_struct *p, int prev) -+{ -+ struct cpumask tmp = {.bits = {0}}; -+ int cpu; -+ u64 util = 0; -+ -+ if (!rt_prio(p->prio)) -+ return -EFAULT; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ -+ if (util < misfit_ds) -+ return -EFAULT; -+ -+ if (is_partial_enabled()) -+ cpumask_or(&tmp, iso_masks.big, iso_masks.partial); -+ else -+ cpumask_copy(&tmp, iso_masks.big); -+ -+ cpu = hmbird_pick_idle_cpu(&tmp); -+ if (cpu >= 0) -+ return cpu; -+ -+ if (cpumask_test_cpu(prev, iso_masks.big) || -+ (is_partial_enabled() && is_partial_cpu(prev))) -+ return prev; -+ -+ return cpumask_first(&tmp); -+} -+ -+static int spec_task_before_pick_idle(struct task_struct *p, int prev) -+{ -+ int cpu; -+ -+ cpu = heavy_rt_placement(p, prev); -+ if (cpu >= 0) -+ return cpu; -+ return -EFAULT; -+} -+ -+static int cpumask_distribute_next(struct cpumask *mask, int *prev) -+{ -+ int p = *prev, n; -+ -+ n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1); -+ if (n < nr_cpu_ids) -+ WRITE_ONCE(*prev, n); -+ return n; -+} -+ -+static int repick_fallback_cpu(void) -+{ -+ /* -+ * partial cpu follow big cluster's Scheduling policy, -+ * simply return first bit cpu. -+ */ -+ return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev); -+} -+ -+/* -+ * Must return a valid cpu num, as this task's cpu. -+ */ -+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu, -+ struct cpumask *mask, int *prev_mask) -+{ -+ int cpu; -+ -+ cpu = hmbird_pick_idle_cpu(mask); -+ if (cpu >= 0) -+ return cpu; -+ return cpumask_distribute_next(mask, prev_mask); -+} -+ -+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type) -+{ -+ int idx; -+ struct cluster_ctx ctx; -+ -+ idx = find_idx_from_task(p); -+ if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS) -+ return false; -+ -+ gen_cluster_ctx(&ctx, type); -+ if (idx >= ctx.lower && idx < ctx.upper) -+ return true; -+ return false; -+} -+ -+static bool is_valid_cpu(int cpu) -+{ -+ return (cpu >= 0) && (cpu < nr_cpu_ids); -+} -+ -+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) -+{ -+ s32 cpu = -1; -+ struct cpumask mask = {.bits = {0}}; -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (is_valid_cpu(cpu)) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ partial_dynamic_ctrl(); -+ -+ if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ iso_masks.big, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ if (p->nr_cpus_allowed == 1) { -+ cpu = cpumask_any(p->cpus_ptr); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* For non-period global dsq, not contain pcp task. */ -+ if (task_only_blongs_to_cluster(p, LITTLE)) { -+ cpumask_copy(&mask, iso_masks.little); -+ if (unlikely(l_need_rescue)) { -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ cpumask_or(&mask, iso_masks.ex_free, &mask); -+ } -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &little_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ if (task_only_blongs_to_cluster(p, BIG)) { -+ cpumask_copy(&mask, iso_masks.big); -+ if (unlikely(b_need_rescue)) { -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ cpumask_or(&mask, iso_masks.ex_free, &mask); -+ } -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ cpu = spec_task_before_pick_idle(p, prev_cpu); -+ if (is_valid_cpu(cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* if the previous CPU is idle, dispatch directly to it */ -+ if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+ } -+ -+ cpu = hmbird_pick_idle_cpu(cpu_possible_mask); -+ if (is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ if (prev_cpu_misfit(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ cpu = repick_fallback_cpu(); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+} -+ -+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags) -+{ -+ return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags); -+} -+ -+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx) -+{ -+ set_cpus_allowed_common(p, ctx); -+} -+ -+static void reset_idle_masks(void) -+{ -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little); -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big); -+ if (is_partial_enabled()) -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial); -+ hmbird_has_idle_cpus = true; -+} -+ -+void __hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ int cpu = cpu_of(rq); -+ struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -+ -+ if (skip_update_idle()) -+ return; -+ -+ if (idle) { -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ if (!hmbird_has_idle_cpus) -+ hmbird_has_idle_cpus = true; -+ -+ /* -+ * idle_masks.smt handling is racy but that's fine as it's only -+ * for optimization and self-correcting. -+ */ -+ for_each_cpu(cpu, sib_mask) { -+ if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -+ return; -+ } -+ cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -+ } else { -+ cpumask_clear_cpu(cpu, idle_masks.cpu); -+ if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ -+ cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -+ } -+} -+ -+#else /* !CONFIG_SMP */ -+ -+static bool test_and_clear_cpu_idle(int cpu) { return false; } -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } -+static void reset_idle_masks(void) {} -+ -+#endif /* CONFIG_SMP */ -+ -+static bool check_rq_for_timeouts(struct rq *rq) -+{ -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rq_flags rf; -+ -+ rq_lock_irqsave(rq, &rf); -+ list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) { -+ unsigned long last_runnable; -+ -+ p = entity->task; -+ last_runnable = get_hmbird_ts(p)->runnable_at; -+ -+ if (unlikely(time_after(jiffies, -+ last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -+ -+ rq_unlock_irqrestore(rq, &rf); -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb", -+ p->comm, p->pid, -+ dur_ms / 1000, dur_ms % 1000, -+ get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0, -+ cpumask_pr_args(&p->cpus_mask)); -+ return 1; -+ } -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ return 0; -+} -+ -+static void hmbird_watchdog_workfn(struct work_struct *work) -+{ -+ int cpu; -+ bool timeout; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ -+ for_each_online_cpu(cpu) { -+ timeout = check_rq_for_timeouts(cpu_rq(cpu)); -+ if (unlikely(timeout)) -+ break; -+ -+ cond_resched(); -+ } -+ if (!timeout) -+ queue_delayed_work(system_unbound_wq, to_delayed_work(work), -+ hmbird_watchdog_timeout / 2); -+} -+ -+static void set_pcp_round(struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) { -+ per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round); -+ per_cpu(pcp_info, cpu).pcp_round = true; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+} -+ -+ -+/* -+ * Just for debug: output hmbird on/off state per 10s. -+ */ -+#define OUTPUT_INTVAL (msecs_to_jiffies(10 * 1000)) -+static void inform_hmbird_onoff_from_systrace(void) -+{ -+ static unsigned long __read_mostly next_print; -+ -+ if (time_before(jiffies, READ_ONCE(next_print))) -+ return; -+ -+ WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL); -+ hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio); -+} -+ -+void hmbird_notify_sched_tick(void) -+{ -+ unsigned long last_check; -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ hmbird_scheduler_tick(); -+ -+ if (!hmbird_enabled()) -+ return; -+ -+ last_check = hmbird_watchdog_timestamp; -+ if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -+ -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "watchdog failed to check in for %u.%03us", -+ dur_ms / 1000, dur_ms % 1000); -+ } -+ scan_timeout(rq); -+} -+ -+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued) -+{ -+ update_curr_hmbird(rq); -+ -+ set_pcp_round(rq); -+ -+ if (slim_walt_ctrl) -+ hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE); -+ /* -+ * While disabling, always resched and refresh core-sched timestamp as -+ * we can't trust the slice management or ops.core_sched_before(). -+ */ -+ if (hmbird_ops_disabling()) -+ get_hmbird_ts(curr)->slice = 0; -+ -+ if (!get_hmbird_ts(curr)->slice) -+ resched_curr(rq); -+ -+ inform_hmbird_onoff_from_systrace(); -+} -+ -+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg) -+{ -+ hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ get_hmbird_ts(p)->disallow = false; -+ -+ hmbird_sched_init_task(p); -+ -+ set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ return 0; -+} -+ -+static void hmbird_ops_enable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void hmbird_ops_disable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void set_task_hmbird_weight(struct task_struct *p) -+{ -+ u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -+ -+ get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight); -+} -+ -+/** -+ * refresh_hmbird_weight - Refresh a task's hmbird weight -+ * @p: task to refresh hmbird weight for -+ * -+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to -+ * enable easy access from the BPF scheduler. To keep it synchronized with the -+ * current task priority, this function should be called when a new task is -+ * created, priority is changed for a task on hmbird, and a task is switched -+ * to hmbird from other classes. -+ */ -+static void refresh_hmbird_weight(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ set_task_hmbird_weight(p); -+} -+ -+int hmbird_pre_fork(struct task_struct *p) -+{ -+ int ret = 0; -+ -+ p->android_oem_data1[HMBIRD_TS_IDX] = -+ (u64)(kzalloc(sizeof(struct hmbird_entity), GFP_KERNEL)); -+ if (!get_hmbird_ts(p)) -+ return -1; -+ -+ get_hmbird_ts(p)->dsq = NULL; -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node); -+ get_hmbird_ts(p)->flags = 0; -+ get_hmbird_ts(p)->weight = 0; -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ get_hmbird_ts(p)->holding_cpu = -1; -+ get_hmbird_ts(p)->kf_mask = 0; -+ atomic64_set(&get_hmbird_ts(p)->ops_state, 0); -+ get_hmbird_ts(p)->runnable_at = INITIAL_JIFFIES; -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+ get_hmbird_ts(p)->task = p; -+ hmbird_set_sched_prop(p, 0); -+ -+ get_hmbird_ts(p)->critical_affinity_cpu = -1; -+ get_hmbird_ts(p)->sched_class = &hmbird_sched_class; -+ get_hmbird_ts(p)->tick_hit_count = 0; -+ get_hmbird_ts(p)->start_jiffies = 0; -+ -+ /* -+ * BPF scheduler enable/disable paths want to be able to iterate and -+ * update all tasks which can become complex when racing forks. As -+ * enable/disable are very cold paths, let's use a percpu_rwsem to -+ * exclude forks. -+ */ -+ -+ return ret; -+} -+ -+void hmbird_post_fork(struct task_struct *p) -+{ -+ percpu_down_read(&hmbird_fork_rwsem); -+ -+ if (hmbird_enabled()) { -+ struct rq_flags rf; -+ struct rq *rq; -+ p->sched_class = &hmbird_sched_class; -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ rq = task_rq_lock(p, &rf); -+ /* -+ * Set the weight manually before calling ops.enable() so that -+ * the scheduler doesn't see a stale value if they inspect the -+ * task struct. We'll invoke ops.set_weight() afterwards, as it -+ * would be odd to receive a callback on the task before we -+ * tell the scheduler that it's been fully enabled. -+ */ -+ set_task_hmbird_weight(p); -+ hmbird_ops_enable_task(p); -+ refresh_hmbird_weight(p); -+ task_rq_unlock(rq, p, &rf); -+ } else { -+ if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ } -+ -+ spin_lock_irq(&hmbird_tasks_lock); -+ list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_cancel_fork(struct task_struct *p) -+{ -+ if (hmbird_enabled()) -+ hmbird_ops_disable_task(p); -+ put_hmbird_ts(p); -+} -+ -+void hmbird_free(struct task_struct *p) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+ list_del_init(&get_hmbird_ts(p)->tasks_node); -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+ -+ /* -+ * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED -> -+ * ENABLED transitions can't race us. Disable ops for @p. -+ */ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) || -+ test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ hmbird_ops_disable_task(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ put_hmbird_ts(p); -+} -+ -+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio) -+{ -+} -+ -+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type) -+{ -+ return (prop >> TOP_TASK_SHIFT) & (1 << type); -+} -+ -+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p) -+{ -+ uint32_t prop = get_top_task_prop(p); -+ -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE)) -+ return HMBIRD_TASK_PROP_PIPELINE; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) || -+ !task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) { -+ return HMBIRD_TASK_PROP_COMMON; -+ } -+ return HMBIRD_TASK_PROP_DEBUG_OR_LOG; -+} -+ -+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b) -+{ -+ int type_a = hmbird_get_task_type(a); -+ int type_b = hmbird_get_task_type(b); -+ -+ return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b]; -+} -+ -+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ enum cpu_type type; -+ int sp_dl; -+ struct task_struct *curr = NULL; -+ -+ switch (hmbird_preempt_policy) { -+ case HMBIRD_PREEMPT_POLICY_PRIO_BASED: -+ curr = rq->curr; -+ if (curr && hmbird_prio_higher(p, curr)) -+ goto preempt; -+ break; -+ default: -+ break; -+ } -+ if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) || -+ (is_critical_system_task(p) && !is_critical_system_task(rq->curr))) -+ goto preempt; -+ -+ sp_dl = find_idx_from_task(p); -+ if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled())) -+ return; -+ -+ if (rq->curr->prio > p->prio) -+ goto preempt; -+ -+ return; -+ -+preempt: -+ resched_curr(rq); -+} -+ -+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {} -+ -+int hmbird_check_setscheduler(struct task_struct *p, int policy) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ /* if disallow, reject transitioning into HMBIRD */ -+ if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) && -+ p->policy != policy && policy == SCHED_HMBIRD) -+ return -EACCES; -+ -+ return 0; -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool hmbird_can_stop_tick(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ if (hmbird_ops_disabling()) -+ return false; -+ -+ if (p->sched_class != &hmbird_sched_class) -+ return true; -+ -+ return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK; -+} -+#endif -+ -+int hmbird_tg_online(struct task_group *tg) -+{ -+ struct cgroup *cgrp; -+ -+ if (!tg) -+ return 0; -+ cgrp = tg->css.cgroup; -+ if (!cgrp || !(cgrp->kn)) -+ return 0; -+ update_cgroup_ids_table(cgrp->kn->id, -1); -+ return 0; -+} -+ -+/* -+ * Omitted operations: -+ * -+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -+ * task isn't tied to the CPU at that point. Preemption is implemented by -+ * resetting the victim task's slice to 0 and triggering reschedule on the -+ * target CPU. -+ * -+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -+ * -+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -+ * their current sched_class. Call them directly from sched core instead. -+ * -+ * - task_woken, switched_from: Unnecessary. -+ */ -+DEFINE_SCHED_CLASS(hmbird) = { -+ .enqueue_task = enqueue_task_hmbird, -+ .dequeue_task = dequeue_task_hmbird, -+ .yield_task = yield_task_hmbird, -+ .yield_to_task = yield_to_task_hmbird, -+ -+ .check_preempt_curr = check_preempt_curr_hmbird, -+ -+ .pick_next_task = pick_next_task_hmbird, -+ -+ .put_prev_task = put_prev_task_hmbird, -+ .set_next_task = set_next_task_hmbird, -+ -+#ifdef CONFIG_SMP -+ .balance = balance_hmbird, -+ .select_task_rq = select_task_rq_hmbird, -+ .set_cpus_allowed = set_cpus_allowed_hmbird, -+#endif -+ .task_tick = task_tick_hmbird, -+ -+ .switched_to = switched_to_hmbird, -+ .prio_changed = prio_changed_hmbird, -+ -+ .update_curr = update_curr_hmbird, -+ -+#ifdef CONFIG_UCLAMP_TASK -+ .uclamp_enabled = 0, -+#endif -+}; -+ -+/* -+ * Must with rq lock held. -+ */ -+bool task_is_hmbird(struct task_struct *p) -+{ -+ return p->sched_class == &hmbird_sched_class; -+} -+ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id) -+{ -+ memset(dsq, 0, sizeof(*dsq)); -+ -+ raw_spin_lock_init(&dsq->lock); -+ INIT_LIST_HEAD(&dsq->fifo); -+ dsq->id = dsq_id; -+} -+ -+static int hmbird_cgroup_init(void) -+{ -+ struct cgroup_subsys_state *css; -+ -+ css_for_each_descendant_pre(css, &root_task_group.css) { -+ struct task_group *tg = css_tg(css); -+ -+ cgrp_dsq_idx_init(css->cgroup, tg); -+ } -+ return 0; -+} -+ -+/* -+ * Used by sched_fork() and __setscheduler_prio() to pick the matching -+ * sched_class. dl/rt are already handled. -+ */ -+bool task_on_hmbird(struct task_struct *p) -+{ -+ return hmbird_enabled(); -+} -+ -+static void __setscheduler_prio(struct task_struct *p, int prio) -+{ -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) { -+ p->prio = prio; -+ return; -+ } else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (on_hmbird && rt_prio(prio)) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ -+ p->prio = prio; -+} -+ -+/* -+ * Heartbeat, avoid humbird keep running while APP already exit. -+ * Check whether APP send alive-signal periodly. -+ */ -+#define HEARTBEAT_TIMEOUT (msecs_to_jiffies(2500)) -+#define HEARTBEAT_CHECK_INTERVAL (msecs_to_jiffies(1000)) -+static struct timer_list hb_timer; -+static unsigned long next_hb; -+ -+void hb_timer_handler(struct timer_list *timer) -+{ -+ if (!heartbeat_enable) -+ goto refill; -+ -+ pr_info(": enter timer.\n"); -+ if (heartbeat) { -+ heartbeat = 0; -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ } -+ -+ /* can't detect heartbeat, disable ext. */ -+ if (time_after(jiffies, READ_ONCE(next_hb))) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT, -+ "can't detect heartbeat, disable ext\n"); -+ } -+refill: -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_start(void) -+{ -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_init(void) -+{ -+ timer_setup(&hb_timer, hb_timer_handler, 0); -+} -+ -+static void hb_timer_exit(void) -+{ -+ del_timer(&hb_timer); -+} -+ -+static bool check_and_disable_cpuhp(void) -+{ -+ struct rq *rq; -+ struct rq_flags rf; -+ int cpu; -+ -+ cpu_hotplug_disable(); -+ cpus_read_lock(); -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ rq_lock_irqsave(rq, &rf); -+ if (!rq->online) { -+ rq_unlock_irqrestore(rq, &rf); -+ goto err_offline; -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ cpus_read_unlock(); -+ return true; -+ -+err_offline: -+ cpus_read_unlock(); -+ cpu_hotplug_enable(); -+ return false; -+} -+ -+static void reenable_cpuhp(void) -+{ -+ cpu_hotplug_enable(); -+} -+ -+static void scheduler_switch_done(bool final_state) -+{ -+ /* set scx_enable again, switch may fail. */ -+ scx_enable = final_state; -+ /* reset heartbeat*/ -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ -+ if (final_state) -+ hb_timer_start(); -+ else -+ hb_timer_exit(); -+} -+ -+/** -+ * ss : curr switch state -+ * finish : success or fail -+ * enable : curr operation is diabling or enabling? -+ * fail_reason : string output when failed, empty string when success. -+ */ -+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason) -+{ -+ char *s1 = finish ? "finished" : "failed"; -+ char *s2 = enable ? "enabled" : "disabled"; -+ -+ hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss); -+ if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) { -+ sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type)); -+ hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n", -+ s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason); -+ } -+ curr_ss = ss; -+} -+ -+bool get_hmbird_ops_enabled(void) -+{ -+ return atomic_read(&__hmbird_ops_enabled); -+} -+ -+bool get_non_hmbird_task(void) -+{ -+ return atomic_read(&non_hmbird_task); -+} -+ -+static void hmbird_ops_disable_workfn(struct kthread_work *work) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int cpu; -+ -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, ""); -+ cancel_delayed_work_sync(&hmbird_watchdog_work); -+ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) { -+ case HMBIRD_OPS_DISABLED: -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled"); -+ scheduler_switch_done(false); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return; -+ case HMBIRD_OPS_PREPPING: -+ fallthrough; -+ case HMBIRD_OPS_DISABLING: -+ /* shouldn't happen but handle it like ENABLING if it does */ -+ WARN_ONCE(true, "hmbird: duplicate disabling instance?"); -+ fallthrough; -+ case HMBIRD_OPS_ENABLING: -+ case HMBIRD_OPS_ENABLED: -+ break; -+ } -+ -+ /* kick all CPUs to restore ticks */ -+ for_each_possible_cpu(cpu) -+ resched_cpu(cpu); -+ -+ /* avoid racing against fork and cgroup changes */ -+ cpus_read_lock(); -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, ""); -+ spin_lock_irq(&hmbird_tasks_lock); -+ atomic_set(&__hmbird_ops_enabled, false); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ bool alive = READ_ONCE(p->__state) != TASK_DEAD; -+ -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ get_hmbird_ts(p)->slice = min_t(u64, -+ get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL); -+ -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ if (alive) -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ -+ hmbird_ops_disable_task(p); -+ } -+ hmbird_task_iter_exit(&sti); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, ""); -+ -+ atomic_set(&non_hmbird_task, true); -+ /* no task is on hmbird, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt); -+ synchronize_rcu(); -+ -+ percpu_up_write(&hmbird_fork_rwsem); -+ cpus_read_unlock(); -+ -+ if (slim_walt_ctrl) -+ slim_walt_enable(false); -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ -+ hmbird_switch_log(HMBIRD_DISABLED, 1, 0, ""); -+ scheduler_switch_done(false); -+ reenable_cpuhp(); -+} -+ -+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn); -+ -+static void schedule_hmbird_ops_disable_work(void) -+{ -+ struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper); -+ -+ /* -+ * We may be called spuriously before the first bpf_hmbird_reg(). If -+ * hmbird_ops_helper isn't set up yet, there's nothing to do. -+ */ -+ if (helper) -+ kthread_queue_work(helper, &hmbird_ops_disable_work); -+} -+ -+static void hmbird_ops_disable(void) -+{ -+ schedule_hmbird_ops_disable_work(); -+} -+ -+static void hmbird_err_exit_workfn(struct work_struct *work) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ hmbird_ctrl(false); -+ -+ for_each_present_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy) -+ continue; -+ if (cpu != policy->cpu) -+ goto put; -+ down_write(&policy->rwsem); -+ WARN_ON(store_scaling_governor(policy, -+ saved_gov[cpu], strlen(saved_gov[cpu])) <= 0); -+ up_write(&policy->rwsem); -+ hmbird_info_trace(":restore origin gov : %s\n", saved_gov[cpu]); -+put: -+ cpufreq_cpu_put(policy); -+ } -+ memset((char *)saved_gov, 0, sizeof(saved_gov)); -+} -+ -+void hmbird_ops_exit(void) -+{ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); -+} -+ -+static struct kthread_worker *hmbird_create_rt_helper(const char *name) -+{ -+ struct kthread_worker *helper; -+ -+ helper = kthread_create_worker(KTW_FREEZABLE, name); -+ if (helper) -+ sched_set_fifo(helper->task); -+ return helper; -+} -+ -+static inline void set_audio_thread_sched_prop(struct task_struct *p) -+{ -+ struct cgroup_subsys_state *css; -+ -+ if (likely(p->prio >= MAX_RT_PRIO)) -+ return; -+ -+ rcu_read_lock(); -+ css = task_css(p, cpuset_cgrp_id); -+ if (!css) { -+ rcu_read_unlock(); -+ return; -+ } -+ rcu_read_unlock(); -+ -+ if (!strcmp(css->cgroup->kn->name, "audio-app")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+} -+ -+int scx_systemui_pid = -1; -+void set_systemui_thread_pid(struct task_struct *p) -+{ -+ if ((strcmp(p->comm, "ndroid.systemui") == 0) && (p->pid == p->tgid)) -+ scx_systemui_pid = p->pid; -+} -+ -+static int hmbird_ops_enable(void *unused) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int ret; -+ int tcnt = 0; -+ unsigned long long start = 0; -+ -+ if (!check_and_disable_cpuhp()) { -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline"); -+ return -EBUSY; -+ } -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, ""); -+ mutex_lock(&hmbird_ops_enable_mutex); -+ -+ if (!hmbird_ops_helper) { -+ WRITE_ONCE(hmbird_ops_helper, -+ hmbird_create_rt_helper("hmbird_ops_helper")); -+ if (!hmbird_ops_helper) { -+ ret = -ENOMEM; -+ goto err_unlock; -+ } -+ } -+ -+ if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) { -+ ret = -EBUSY; -+ goto err_unlock; -+ } -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) != -+ HMBIRD_OPS_DISABLED); -+ -+ hmbird_warned_zero_slice = false; -+ -+ atomic64_set(&hmbird_nr_rejected, 0); -+ -+ /* -+ * Keep CPUs stable during enable so that the BPF scheduler can track -+ * online CPUs by watching ->on/offline_cpu() after ->init(). -+ */ -+ cpus_read_lock(); -+ -+ hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work, -+ hmbird_watchdog_timeout / 2); -+ -+ /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ */ -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ reset_idle_masks(); -+ -+ /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. -+ */ -+ ret = hmbird_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock; -+ -+ /* -+ * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem -+ * preventing new tasks from being added. No need to exclude tasks -+ * leaving as hmbird_free() can handle both prepped and enabled -+ * tasks. Prep all tasks first and then enable them with preemption -+ * disabled. -+ */ -+ spin_lock_irq(&hmbird_tasks_lock); -+ -+ atomic_set(&non_hmbird_task, false); -+ atomic_set(&__hmbird_ops_enabled, true); -+ -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered(&sti))) -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ hmbird_task_iter_exit(&sti); -+ -+ /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. -+ */ -+ preempt_disable(); -+ -+ /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ */ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) { -+ atomic_set(&non_hmbird_task, true); -+ atomic_set(&__hmbird_ops_enabled, false); -+ preempt_enable(); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ ret = -EBUSY; -+ goto err_disable_unlock; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED -+ * transitions here are synchronized against hmbird_free() through -+ * hmbird_tasks_lock. -+ */ -+ start = sched_clock(); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, ""); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ tcnt++; -+ if (READ_ONCE(p->__state) != TASK_DEAD) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ -+ set_audio_thread_sched_prop(p); -+ set_systemui_thread_pid(p); -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ hmbird_ops_enable_task(p); -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ } else { -+ hmbird_ops_disable_task(p); -+ } -+ } -+ hmbird_task_iter_exit(&sti); -+ -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, ""); -+ preempt_enable(); -+ percpu_up_write(&hmbird_fork_rwsem); -+ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) { -+ ret = -EBUSY; -+ goto err_disable; -+ } -+ -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_ENABLED, 1, 1, ""); -+ scheduler_switch_done(true); -+ -+ return 0; -+ -+err_unlock: -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock"); -+ scheduler_switch_done(false); -+ return ret; -+ -+err_disable_unlock: -+ percpu_up_write(&hmbird_fork_rwsem); -+err_disable: -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ /* must be fully disabled before returning */ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable"); -+ scheduler_switch_done(false); -+ return ret; -+} -+ -+#ifdef CONFIG_SCHED_DEBUG -+static const char *hmbird_ops_enable_state_str[] = { -+ [HMBIRD_OPS_PREPPING] = "prepping", -+ [HMBIRD_OPS_ENABLING] = "enabling", -+ [HMBIRD_OPS_ENABLED] = "enabled", -+ [HMBIRD_OPS_DISABLING] = "disabling", -+ [HMBIRD_OPS_DISABLED] = "disabled", -+}; -+ -+static int hmbird_debug_show(struct seq_file *m, void *v) -+{ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled()); -+ seq_printf(m, "%-30s: %s\n", "enable_state", -+ hmbird_ops_enable_state_str[hmbird_ops_enable_state()]); -+ seq_printf(m, "%-30s: %llu\n", "nr_rejected", -+ atomic64_read(&hmbird_nr_rejected)); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return 0; -+} -+ -+static int hmbird_debug_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_debug_show, NULL); -+} -+ -+const struct file_operations sched_hmbird_fops = { -+ .open = hmbird_debug_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif -+ -+ -+static int bpf_hmbird_reg(void *kdata) -+{ -+ return hmbird_ops_enable(kdata); -+} -+ -+static int bpf_hmbird_unreg(void *kdata) -+{ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ return 0; -+} -+ -+void set_hmbird_module_loaded(int is_loaded) -+{ -+ atomic_set(&hmbird_module_loaded, is_loaded); -+} -+ -+/* -+ * MUST load hmbird module before enable hmbird scheduler -+ * load track & hmbird gover implemented in hmbird module -+ */ -+int hmbird_ctrl(bool enable) -+{ -+ if (!atomic_read(&hmbird_module_loaded)) { -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "ext module unloaded\n"); -+ return -EINVAL; -+ } -+ -+ if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED -+ || hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING -+ || hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already enabled(ing)\n"); -+ hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING || -+ hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already disabled(ing)\n"); -+ hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (enable) -+ return bpf_hmbird_reg(NULL); -+ else -+ return bpf_hmbird_unreg(NULL); -+} -+ -+void set_cpu_isomask(int cpu, cpumask_var_t *mask) -+{ -+ cpumask_clear_cpu(cpu, iso_masks.ex_free); -+ cpumask_clear_cpu(cpu, iso_masks.exclusive); -+ cpumask_clear_cpu(cpu, iso_masks.partial); -+ cpumask_clear_cpu(cpu, iso_masks.big); -+ cpumask_clear_cpu(cpu, iso_masks.little); -+ cpumask_set_cpu(cpu, *mask); -+} -+ -+void set_cpu_cluster(u64 cpu_cluster) -+{ -+ int cpu; -+ -+ for_each_present_cpu(cpu) { -+ u64 pos = 1 << cpu; -+ -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.ex_free)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.exclusive)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.partial)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.big)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) -+ set_cpu_isomask(cpu, &(iso_masks.little)); -+ } -+} -+ -+static void get_hmbird_snapshot(struct panic_snapshot_t *p) -+{ -+ struct hmbird_dispatch_q *dsq; -+ struct rq *rq; -+ int cpu, i; -+ -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ p->rq_nr[cpu] = rq->nr_running; -+ p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running; -+ } -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ struct hmbird_entity *entity; -+ -+ dsq = &gdsqs[i]; -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo)) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ p->runnable_at[i] = entity->runnable_at; -+ raw_spin_unlock(&dsq->lock); -+ -+ } -+ -+ p->snap_misc.hmbird_enabled = hmbird_enabled(); -+ p->snap_misc.curr_ss = curr_ss; -+ p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state(); -+ p->snap_misc.parctrl_high_ratio = parctrl_high_ratio; -+ p->snap_misc.parctrl_low_ratio = parctrl_low_ratio; -+ p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l; -+ p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l; -+ p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio; -+ p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio; -+ p->snap_misc.misfit_ds = misfit_ds; -+ p->snap_misc.partial_enable = partial_enable; -+ p->snap_misc.iso_free_rescue = iso_free_rescue; -+ p->snap_misc.isolate_ctrl = isolate_ctrl; -+ p->snap_misc.snap_jiffies = jiffies; -+ p->snap_misc.snap_time = local_clock(); -+} -+ -+// MTK minidump begin -+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3) -+{ -+ strscpy(m->desc_str, str, DESC_STR_LEN); -+ m->len = d1 * d2 * d3; -+ m->parse[0] = d1; -+ m->parse[1] = d2; -+ m->parse[2] = d3; -+} -+ -+static void init_desc_metas(struct md_info_t *m) -+{ -+ init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :", -+ 1, MAX_SWITCHS, SWITCH_ITEMS); -+ -+ init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1); -+ -+ init_desc_meta(&m->kern_dump.excep_rec_meta, -+ "excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS); -+ -+ init_desc_meta(&m->kern_dump.excep_idx_meta, -+ "excep idx :", 1, 1, MAX_EXCEP_ID); -+ -+ init_desc_meta(&m->kern_dump.snap.runnable_at_meta, -+ "each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS); -+ -+ init_desc_meta(&m->kern_dump.snap.rq_nr_meta, -+ "rq runnable task nr:", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta, -+ "scxrq runnable task nr :", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.snap_misc_meta, -+ "misc snap params :", 1, 1, SNAP_ITEMS); -+} -+ -+static void init_md_meta(struct md_info_t *m) -+{ -+ m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64); -+ m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64); -+ m->meta.unit_size = sizeof(u64); -+ m->meta.switches = MAX_SWITCHS; -+ m->meta.exceps = MAX_EXCEPS; -+ m->meta.global_dsqs = MAX_GLOBAL_DSQS; -+ m->meta.parse_dimens = PARSE_DIMENS; -+ m->meta.nr_cpus = num_possible_cpus(); -+ m->meta.real_cpus = nr_cpu_ids; -+ m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64); -+ m->meta.nr_meta_desc = 8; -+ m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64); -+ -+ init_desc_metas(m); -+} -+ -+#define MINIDUMP_DFL_SIZE (4 * 1024) -+ -+struct notifier_block hmbird_panic_blk; -+static int hmbird_panic_handler(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ if (!md_info) -+ return NOTIFY_DONE; -+ -+ get_hmbird_snapshot(&md_info->kern_dump.snap); -+ -+ return NOTIFY_DONE; -+} -+ -+static void panic_blk_init(void) -+{ -+ int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE); -+ -+ md_info = kzalloc(dump_size, GFP_KERNEL); -+ if (!md_info) -+ return; -+ init_md_meta(md_info); -+ -+ hmbird_panic_blk.notifier_call = hmbird_panic_handler; -+ /* make sure to execute before minidump. */ -+ hmbird_panic_blk.priority = INT_MAX; -+ atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk); -+ -+ hmbird_debug("register minidump.\n"); -+} -+ -+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size) -+{ -+ *vaddr = (unsigned long)md_info; -+ *size = sizeof(struct md_info_t); -+} -+// MTK minidump end -+ -+static inline void init_sched_prop_to_preempt_prio(void) -+{ -+ for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) { -+ switch (i) { -+ case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 5; -+ break; -+ -+ case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 4; -+ break; -+ -+ case HMBIRD_TASK_PROP_PIPELINE: -+ case HMBIRD_TASK_PROP_ISOLATE: -+ sched_prop_to_preempt_prio[i] = 3; -+ break; -+ -+ case HMBIRD_TASK_PROP_COMMON: -+ sched_prop_to_preempt_prio[i] = 1; -+ break; -+ -+ case HMBIRD_TASK_PROP_DEBUG_OR_LOG: -+ sched_prop_to_preempt_prio[i] = 0; -+ break; -+ -+ default: -+ sched_prop_to_preempt_prio[i] = 2; -+ break; -+ } -+ } -+} -+ -+void __init init_sched_hmbird_class(void) -+{ -+ int cpu; -+ u32 v; -+ struct hmbird_entity *init_hmbird; -+ -+ /* -+ * The following is to prevent the compiler from optimizing out the enum -+ * definitions so that BPF scheduler implementations can use them -+ * through the generated vmlinux.h. -+ */ -+ WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP | -+ HMBIRD_KICK_PREEMPT); -+ -+ init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL); -+ init_dsq_at_boot(); -+ init_isolate_cpus(); -+ hb_timer_init(); -+ init_sched_prop_to_preempt_prio(); -+#ifdef CONFIG_SMP -+ WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); -+#endif -+ -+ /* -+ * we can't static init init_task's hmbird struct, init here. -+ * init_task->hmbird would not use during boot. -+ */ -+ init_hmbird = kzalloc(sizeof(struct hmbird_entity), GFP_KERNEL); -+ init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird; -+ if (init_hmbird) { -+ INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo); -+ INIT_LIST_HEAD(&init_hmbird->watchdog_node); -+ init_hmbird->sticky_cpu = -1; -+ init_hmbird->holding_cpu = -1; -+ atomic64_set(&init_hmbird->ops_state, 0); -+ init_hmbird->runnable_at = jiffies; -+ init_hmbird->slice = HMBIRD_SLICE_DFL; -+ init_hmbird->task = &init_task; -+ hmbird_set_sched_prop(&init_task, 0); -+ } else { -+ hmbird_err(INIT_TASK_FAIL, ":alloc init_task.scx failed!!!\n"); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ /* -+ * exec during boot phase, no need to care about alloc failed. -+ * lifecycle same to rq, no need to free. -+ */ -+ rq->android_oem_data1[HMBIRD_RQ_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_rq), GFP_KERNEL); -+ if (get_hmbird_rq(rq)) { -+ get_hmbird_rq(rq)->rq = rq; -+ get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window; -+ } else { -+ hmbird_err(ALLOC_RQSCX_FAIL, ":alloc rq->scx failed!!!\n"); -+ } -+ -+ rq->android_oem_data1[HMBIRD_OPS_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL); -+ if (!get_hmbird_ops(rq)) -+ pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n"); -+ init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL); -+ INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list); -+ -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL)); -+ -+ hmbird_ops_init(get_hmbird_ops(rq)); -+ } -+ -+ INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn); -+ INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn); -+ hmbird_misc_init(); -+ -+ panic_blk_init(); -+} -+ -diff --git b/kernel/sched/hmbird/hmbird_misc.c a/kernel/sched/hmbird/hmbird_misc.c -new file mode 100755 -index 000000000000..895e8a29df33 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_misc.c -@@ -0,0 +1,148 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * @Description: -+ * @Version: -+ * @Author: wangrui8 -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include "hmbird_sched.h" -+#include -+#include -+#include "slim.h" -+ -+noinline int tracing_mark_write(const char *buf) -+{ -+ trace_printk(buf); -+ return 0; -+} -+ -+struct yield_opt_params yield_opt_params = { -+ .enable = 0, -+ .frame_per_sec = 120, -+ .frame_time_ns = NSEC_PER_SEC / 120, -+ .yield_headroom = 10, -+}; -+ -+DEFINE_PER_CPU(struct sched_yield_state, ystate); -+ -+static inline void hmbird_yield_state_update(struct sched_yield_state *ys) -+{ -+ if (!raw_spin_is_locked(&ys->lock)) -+ return; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ -+ if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1 -+ || ys->yield_cnt_after_sleep > yield_headroom) { -+ ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP); -+ } else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) { -+ ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP); -+ } -+ ys->yield_cnt = 0; -+ ys->sleep_times = 0; -+ ys->yield_cnt_after_sleep = 0; -+} -+ -+void hmbird_skip_yield(long *skip) -+{ -+ if (!get_hmbird_ops_enabled() || !yield_opt_params.enable) -+ return; -+ unsigned long flags, sleep_now = 0; -+ struct sched_yield_state *ys; -+ int cpu = raw_smp_processor_id(), cont_yield, new_frame; -+ int frame_time_ns = yield_opt_params.frame_time_ns; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ u64 wc; -+ -+ if (!(*skip)) { -+ wc = sched_clock(); -+ ys = &per_cpu(ystate, cpu); -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ -+ cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP; -+ new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1); -+ -+ if (!cont_yield && new_frame) { -+ hmbird_yield_state_update(ys); -+ ys->last_update_time = wc; -+ ys->sleep_end = ys->last_yield_time + frame_time_ns -+ - yield_headroom * YIELD_DURATION; -+ } -+ -+ if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) { -+ *skip = true; -+ -+ sleep_now = ys->sleep_times ? -+ max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep; -+ if (wc + sleep_now > ys->sleep_end) { -+ u64 delta = ys->sleep_end - wc; -+ -+ if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION) -+ sleep_now = delta; -+ else -+ sleep_now = 0; -+ } -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ if (sleep_now) { -+ sleep_now = div64_u64(sleep_now, 1000); -+ usleep_range_state(sleep_now, sleep_now, TASK_IDLE); -+ } -+ ys->sleep_times++; -+ ys->last_yield_time = sched_clock(); -+ return; -+ } -+ if (ys->sleep_times) -+ ys->yield_cnt_after_sleep++; -+ else -+ (ys->yield_cnt)++; -+ ys->last_yield_time = wc; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+} -+ -+struct boost_policy_params boost_policy_params = { -+ .enable = 0, -+ .bottom_freq = 1200000, -+ .boost_weight = 120, -+}; -+ -+int hmbird_get_boost_enable(void) -+{ -+ return boost_policy_params.enable; -+} -+ -+unsigned int hmbird_get_boost_bottom_freq(void) -+{ -+ return boost_policy_params.bottom_freq; -+} -+ -+int hmbird_get_boost_weight(void) -+{ -+ return boost_policy_params.boost_weight; -+} -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops) -+{ -+ hmbird_ops->scx_enable = get_hmbird_ops_enabled; -+ hmbird_ops->check_non_task = get_non_hmbird_task; -+ hmbird_ops->hmbird_get_md_info = hmbird_get_md_info; -+ hmbird_ops->hmbird_get_boost_enable = hmbird_get_boost_enable; -+ hmbird_ops->hmbird_get_boost_bottom_freq = hmbird_get_boost_bottom_freq; -+ hmbird_ops->hmbird_get_boost_weight = hmbird_get_boost_weight; -+} -+ -+void hmbird_misc_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_init(&ys->lock); -+ } -+ -+ set_hmbird_module_loaded(1); -+} -+ -diff --git b/kernel/sched/hmbird/hmbird_sched.h a/kernel/sched/hmbird/hmbird_sched.h -new file mode 100755 -index 000000000000..76acbe9685e4 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef __HMBIRD_SCHED__ -+#define __HMBIRD_SCHED__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include <../../kernel/time/tick-sched.h> -+#include <../../kernel/sched/sched.h> -+#include -+#include -+#include -+ -+#define REGISTER_TRACE_VH(vender_hook, handler) \ -+{ \ -+ ret = register_trace_##vender_hook(handler, NULL); \ -+ if (ret) { \ -+ pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \ -+ } \ -+} -+#define REGISTER_TRACE(vendor_hook, handler, data, err) \ -+do { \ -+ ret = register_trace_##vendor_hook(handler, data); \ -+ if (ret) { \ -+ pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret); \ -+ goto err; \ -+ } \ -+} while (0) -+ -+#define UNREGISTER_TRACE(vendor_hook, handler, data) \ -+ unregister_trace_##vendor_hook(handler, data) \ -+ -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+ -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int sched_ravg_window_frame_per_sec; -+extern int slim_gov_debug; -+extern int cpu7_tl; -+extern int scx_gov_ctrl; -+extern spinlock_t new_sched_ravg_window_lock; -+extern int cluster_separate; -+ -+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER BIT(31) -+#define MAX_YIELD_SLEEP (2000000ULL) -+#define MIN_YIELD_SLEEP (200000ULL) -+#define YIELD_DURATION (5000ULL) -+#define DEFAULT_YIELD_SLEEP_TH (10) -+ -+struct sched_yield_state { -+ raw_spinlock_t lock; -+ u64 last_yield_time; -+ u64 last_update_time; -+ u64 sleep_end; -+ unsigned long yield_cnt; -+ unsigned long yield_cnt_after_sleep; -+ unsigned long sleep; -+ int sleep_times; -+}; -+ -+DECLARE_PER_CPU(struct sched_yield_state, ystate); -+ -+void hmbird_window_rollover_run_once(struct rq *rq); -+void hmbird_yield_state_update_per_frame(void); -+void hmbird_misc_init(void); -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops); -+#endif /*__HMBIRD_SCHED__*/ -diff --git b/kernel/sched/hmbird/hmbird_sched_proc.c a/kernel/sched/hmbird/hmbird_sched_proc.c -new file mode 100755 -index 000000000000..295d45404608 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched_proc.c -@@ -0,0 +1,640 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "hmbird_sched_proc.h" -+#include -+ -+#include "hmbird_util_track.h" -+#include "slim.h" -+ -+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched" -+#define SLIM_FREQ_GOV_DIR "slim_freq_gov" -+#define LOAD_TRACK_DIR "slim_walt" -+#define HMBIRD_PROC_PERMISSION 0666 -+ -+int scx_enable; -+int partial_enable; -+int cpuctrl_high_ratio = 55; -+int cpuctrl_low_ratio = 40; -+int slim_stats; -+int hmbirdcore_debug; -+int slim_for_app; -+int misfit_ds = 90; -+unsigned int highres_tick_ctrl; -+unsigned int highres_tick_ctrl_dbg; -+int cpu7_tl = 70; -+int slim_walt_ctrl; -+int slim_walt_dump; -+int slim_walt_policy; -+int slim_gov_debug; -+int scx_gov_ctrl = 1; -+int sched_ravg_window_frame_per_sec = 125; -+int parctrl_high_ratio = 55; -+int parctrl_low_ratio = 40; -+int parctrl_high_ratio_l = 65; -+int parctrl_low_ratio_l = 50; -+int isoctrl_high_ratio = 75; -+int isoctrl_low_ratio = 60; -+int isolate_ctrl; -+int iso_free_rescue; -+int heartbeat; -+int heartbeat_enable = 1; -+int watchdog_enable; -+int save_gov; -+u64 cpu_cluster_masks; -+int hmbird_preempt_policy; -+int cluster_separate; -+ -+char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= 32) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtoint(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoint\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+static int set_proc_buf_val_u64(struct file *file, const char __user *buf, -+ size_t count, u64 *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= sizeof(kbuf)) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtou64(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoul\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/* common ops begin */ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ return count; -+} -+ -+static int hmbird_common_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%d\n", *(int *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_show, pde_data(inode)); -+} -+ -+static int hmbird_common_ul_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%lu\n", *(unsigned long *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_ul_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_ul_show, pde_data(inode)); -+} -+ -+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write); -+/* common ops end */ -+ -+/* scx_enable ops begin */ -+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC); -+ if (hmbird_ctrl(*pval)) -+ return -EFAULT; -+ -+ return count; -+} -+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write); -+/* scx_enable ops end */ -+ -+/* hmbird_stats ops begin */ -+#define MAX_STATS_BUF (4096) -+static int hmbird_stats_proc_show(struct seq_file *m, void *v) -+{ -+ char *buf; -+ -+ buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ stats_print(buf, MAX_STATS_BUF); -+ -+ seq_printf(m, "%s\n", buf); -+ -+ kfree(buf); -+ return 0; -+} -+ -+static int hmbird_stats_proc_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_stats_proc_show, inode); -+} -+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL); -+/* hmbird_stats ops end */ -+ -+/* sched_ravg_window_frame_per_sec ops begin */ -+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file, -+ const char __user *buf, size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ sched_ravg_window_change(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open, -+ sched_ravg_window_frame_per_sec_proc_write); -+/* sched_ravg_window_frame_per_sec ops end */ -+ -+static ssize_t save_gov_str(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ for_each_possible_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy || (cpu != policy->cpu)) -+ continue; -+ WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0); -+ hmbird_info_systrace(":save origin gov : %s\n", saved_gov[cpu]); -+ } -+ return count; -+} -+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str); -+ -+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ u64 *pval = (u64 *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val_u64(file, buf, count, pval)) -+ return -EFAULT; -+ -+ if (scx_enable == 0) -+ set_cpu_cluster(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write); -+ -+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ int tmp_val; -+ -+ if (set_proc_buf_val(file, buf, count, &tmp_val)) -+ return -EFAULT; -+ -+ slim_walt_enable(tmp_val); -+ *pval = tmp_val; -+ -+ return count; -+} -+ -+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write); -+ -+/* yield_opt ops begin */ -+static int yield_opt_show(struct seq_file *m, void *v) -+{ -+ struct yield_opt_params *data = m->private; -+ -+ seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n", -+ data->enable, data->frame_per_sec, data->yield_headroom); -+ return 0; -+} -+ -+static int yield_opt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, yield_opt_show, pde_data(inode)); -+} -+ -+static ssize_t yield_opt_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu; -+ unsigned long flags; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if (copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp -+ != 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) || -+ (yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp; -+ yield_opt_params.frame_per_sec = frame_per_sec_tmp; -+ yield_opt_params.yield_headroom = yield_headroom_tmp; -+ yield_opt_params.enable = enable_tmp; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ ys->last_yield_time = 0; -+ ys->last_update_time = 0; -+ ys->sleep_end = 0; -+ ys->yield_cnt = 0; -+ ys->yield_cnt_after_sleep = 0; -+ ys->sleep = 0; -+ ys->sleep_times = 0; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write); -+ -+/* boost_policy ops begin */ -+static int boost_policy_show(struct seq_file *m, void *v) -+{ -+ struct boost_policy_params *data = m->private; -+ -+ seq_printf(m, "boost_policy:{\"enable\":%d; \"bottom_freq\":%u; \"boost_weight\":%d}\n", -+ data->enable, data->bottom_freq, data->boost_weight); -+ return 0; -+} -+ -+static int boost_policy_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, boost_policy_show, pde_data(inode)); -+} -+ -+static ssize_t boost_policy_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, bottom_freq_tmp, boost_weight_tmp; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if(copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &bottom_freq_tmp, &boost_weight_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || -+ (boost_weight_tmp < 50 || boost_weight_tmp > 300) || -+ (bottom_freq_tmp < 400000 || bottom_freq_tmp > 2200000)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ boost_policy_params.bottom_freq = bottom_freq_tmp; -+ boost_policy_params.boost_weight = boost_weight_tmp; -+ boost_policy_params.enable = enable_tmp; -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(boost_policy, boost_policy_open, boost_policy_write); -+ -+/* tick_hit ops begin */ -+static int tick_hit_show(struct seq_file *m, void *v) -+{ -+ struct tick_hit_params *data = m->private; -+ -+ seq_printf(m, "tick_hit:{\"enable\":%d; \"hit_count_thres\":%d; \"jiffies_num\":%lu}\n", -+ data->enable, data->hit_count_thres, data->jiffies_num); -+ return 0; -+} -+ -+static int tick_hit_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, tick_hit_show, pde_data(inode)); -+} -+ -+static ssize_t tick_hit_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, hit_count_thres_tmp, jiffies_num_tmp; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if(copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &hit_count_thres_tmp, &jiffies_num_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ tick_hit_params.hit_count_thres = hit_count_thres_tmp; -+ tick_hit_params.jiffies_num = jiffies_num_tmp; -+ tick_hit_params.enable = enable_tmp; -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(tick_hit, tick_hit_open, tick_hit_write); -+ -+static int __init hmbird_proc_init(void) -+{ -+ struct proc_dir_entry *hmbird_dir; -+ struct proc_dir_entry *load_track_dir; -+ struct proc_dir_entry *freq_gov_dir; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ -+ /* mkdir /proc/hmbird_sched */ -+ hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL); -+ if (!hmbird_dir) { -+ pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &scx_enable_proc_ops, -+ &scx_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &partial_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_stats); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbirdcore_debug); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_for_app); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &misfit_ds); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl_dbg); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpu7_tl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &cpu_cluster_masks_proc_ops, -+ &cpu_cluster_masks); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &save_gov_proc_ops, -+ &save_gov); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &watchdog_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isolate_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &iso_free_rescue); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_stats_proc_ops); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &yield_opt_proc_ops, -+ &yield_opt_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("boost_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &boost_policy_proc_ops, -+ &boost_policy_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("tick_hit", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &tick_hit_proc_ops, -+ &tick_hit_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbird_preempt_policy); -+ /* /proc/hmbird_sched--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_walt */ -+ load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir); -+ if (!load_track_dir) { -+ pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_walt--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &slim_walt_ctrl_proc_ops, -+ &slim_walt_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_dump); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_policy); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &sched_ravg_window_frame_per_sec_proc_ops, -+ &sched_ravg_window_frame_per_sec); -+ /* /proc/hmbird_sched/slim_walt--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_freq_gov */ -+ freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir); -+ if (!freq_gov_dir) { -+ pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_freq_gov--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &slim_gov_debug); -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &scx_gov_ctrl); -+ /* /proc/hmbird_sched/slim_freq_gov--end */ -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cluster_separate); -+ -+ return 0; -+} -+ -+device_initcall(hmbird_proc_init); -diff --git b/kernel/sched/hmbird/hmbird_sched_proc.h a/kernel/sched/hmbird/hmbird_sched_proc.h -new file mode 100755 -index 000000000000..e22bc75f1dd7 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched_proc.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SCHED_PROC_H__ -+#define __HMBIRD_SCHED_PROC_H__ -+ -+#include -+#include -+ -+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \ -+ do { \ -+ if (!proc_create(name, mode, parent, proc_ops)) { \ -+ pr_err("Error creating proc entry %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \ -+ do { \ -+ if (!proc_create_data(name, mode, parent, proc_ops, data)) { \ -+ pr_err("Error creating proc entry with data %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_PROC_OPS(name, open_func, write_func) \ -+ static const struct proc_ops name##_proc_ops = { \ -+ .proc_open = open_func, \ -+ .proc_write = write_func, \ -+ .proc_read = seq_read, \ -+ .proc_lseek = seq_lseek, \ -+ .proc_release = single_release, \ -+ } -+ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos); -+static int hmbird_common_show(struct seq_file *m, void *v); -+static int hmbird_common_open(struct inode *inode, struct file *file); -+ -+#endif -diff --git b/kernel/sched/hmbird/hmbird_shadow_tick.c a/kernel/sched/hmbird/hmbird_shadow_tick.c -new file mode 100755 -index 000000000000..2744cd42bbfd ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_shadow_tick.c -@@ -0,0 +1,198 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include -+#include <../../kernel/time/tick-sched.h> -+#include -+ -+#include "hmbird_shadow_tick.h" -+#include "slim.h" -+ -+#define HIGHRES_WATCH_CPU 0 -+ -+#include -+static bool shadow_tick_enable(void) {return highres_tick_ctrl; } -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; } -+#endif -+static bool shadow_tick_timer_init_flag; -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define shadow_tick_printk(fmt, args...) \ -+do { \ -+ int cpu = smp_processor_id(); \ -+ if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU) \ -+ trace_printk("hmbird shadow tick :"fmt, args); \ -+} while (0) -+#else -+#define shadow_tick_printk(fmt, args...) -+#endif -+ -+DEFINE_PER_CPU(struct hrtimer, stt); -+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu))) -+#define STOP_IDLE_TRIGGER (1) -+#define PERIODIC_TICK_TRIGGER (2) -+#define TICK_INTVAL (1000000ULL) -+#define HMBIRD_TICK_HIT_BOOST BIT(29) -+/* -+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms, -+ * so we can't restart hrtimer in scheduler tick. -+ */ -+static DEFINE_PER_CPU(u8, trigger_event); -+ -+/* -+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick. -+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler. -+ */ -+static DEFINE_PER_CPU(u8, tick_phase); -+ -+static inline void highres_timer_ctrl(bool enable, int cpu) -+{ -+ if (enable && hmbird_enabled()) { -+ if (!hrtimer_active(shadow_tick_timer(cpu))) -+ hrtimer_start(shadow_tick_timer(cpu), -+ ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED); -+ } else { -+ if (!enable) -+ hrtimer_cancel(shadow_tick_timer(cpu)); -+ } -+} -+ -+static inline void high_res_clear_phase(int cpu) -+{ -+ per_cpu(tick_phase, cpu) = 0; -+} -+ -+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer) -+{ -+ per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3; -+ if (per_cpu(tick_phase, cpu)) { -+ hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) { -+ per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+struct tick_hit_params tick_hit_params = { -+ .enable = 0, -+ .jiffies_num = 2, -+ .hit_count_thres = 6, -+}; -+ -+static void tick_hit_critical_task(struct task_struct *curr, struct rq *rq) -+{ -+ struct hmbird_entity *see = get_hmbird_ts(curr); -+ if (!tick_hit_params.enable || !see) -+ return; -+ -+ if ((!task_is_top_task(curr) || curr->pid != curr->tgid) && (curr->pid != scx_systemui_pid)) -+ return; -+ -+ if (see->tick_hit_count == 0) { -+ if (see->start_jiffies == 0) { -+ see->start_jiffies = jiffies; -+ see->tick_hit_count = 1; -+ } else { -+ see->start_jiffies = 0; /* status reset */ -+ see->tick_hit_count = 0; -+ } -+ } else { -+ if (time_before_eq(jiffies, see->start_jiffies + tick_hit_params.jiffies_num)) { -+ see->tick_hit_count++; -+ if (see->tick_hit_count >= tick_hit_params.hit_count_thres) { -+ cpufreq_update_util(rq, HMBIRD_TICK_HIT_BOOST); -+ } -+ } else { -+ see->tick_hit_count = 1; -+ see->start_jiffies = jiffies; -+ } -+ } -+} -+ -+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ struct rq_flags rf; -+ -+ rq_lock(rq, &rf); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ tick_hit_critical_task(curr, rq); -+ rq_unlock(rq, &rf); -+ -+ return highres_next_phase(cpu, timer); -+} -+ -+void shadow_tick_timer_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance; -+ } -+} -+ -+void start_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable()) { -+ if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER) -+ highres_timer_ctrl(false, cpu); -+ per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static void stop_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ per_cpu(trigger_event, cpu) = 0; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(false, cpu); -+} -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ stop_shadow_tick_timer(); -+} -+ -+void scheduler_tick_handler(void *unused, struct rq *rq) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ start_shadow_tick_timer(); -+} -+ -+static int __init hmbird_shadow_tick_init(void) -+{ -+ int ret = 0; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ shadow_tick_timer_init(); -+ shadow_tick_timer_init_flag = true; -+ return ret; -+} -+ -+device_initcall(hmbird_shadow_tick_init); -diff --git b/kernel/sched/hmbird/hmbird_shadow_tick.h a/kernel/sched/hmbird/hmbird_shadow_tick.h -new file mode 100755 -index 000000000000..0c26fd984f0a ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_shadow_tick.h -@@ -0,0 +1,11 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SHADOW_TICK_H__ -+#define __HMBIRD_SHADOW_TICK_H__ -+ -+#include -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+void scheduler_tick_handler(void *unused, struct rq *rq); -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state); -+#endif -diff --git b/kernel/sched/hmbird/hmbird_trace.h a/kernel/sched/hmbird/hmbird_trace.h -new file mode 100755 -index 000000000000..8468b406f347 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_trace.h -@@ -0,0 +1,92 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hmbird -+ -+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HMBIRD_H -+ -+#include -+#include -+#include -+ -+#define MAX_FATAL_INFO (64) -+ -+TRACE_EVENT(hmbird_fatal_info, -+ -+ TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info), -+ -+ TP_ARGS(type, pe, lnr, bnr, info), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, type) -+ __field(int, pe) -+ __field(int, lnr) -+ __field(int, bnr) -+ __array(char, info, MAX_FATAL_INFO)), -+ -+ TP_fast_assign( -+ __entry->type = type; -+ __entry->pe = pe; -+ __entry->lnr = lnr; -+ __entry->bnr = bnr; -+ memcpy(__entry->info, info, MAX_FATAL_INFO);), -+ -+ TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s", -+ __entry->type, __entry->pe, __entry->lnr, __entry->bnr, -+ __entry->info) -+); -+ -+TRACE_EVENT(hmbird_update_history, -+ -+ TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq, -+ struct task_struct *p, u32 runtime, int samples, int event), -+ -+ TP_ARGS(hmbird, rq, p, runtime, samples, event), -+ -+ TP_STRUCT__entry( -+ __array(char, comm, TASK_COMM_LEN) -+ __field(pid_t, pid) -+ __field(unsigned int, runtime) -+ __field(int, samples) -+ __field(int, event) -+ __field(unsigned int, demand) -+ __array(u32, hist, RAVG_HIST_SIZE) -+ __field(u16, task_util) -+ __field(int, cpu)), -+ -+ TP_fast_assign( -+ memcpy(__entry->comm, p->comm, TASK_COMM_LEN); -+ __entry->pid = p->pid; -+ __entry->runtime = runtime; -+ __entry->samples = samples; -+ __entry->event = event; -+ __entry->demand = hmbird->sts.demand; -+ memcpy(__entry->hist, hmbird->sts.sum_history, -+ RAVG_HIST_SIZE * sizeof(u32)); -+ __entry->task_util = hmbird->sts.demand_scaled; -+ __entry->cpu = rq->cpu;), -+ -+ TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d", -+ __entry->comm, __entry->pid, -+ __entry->runtime, __entry->samples, -+ __entry->event, -+ __entry->demand, -+ __entry->hist[0], __entry->hist[1], -+ __entry->hist[2], __entry->hist[3], -+ __entry->hist[4], -+ __entry->task_util, -+ __entry->cpu) -+); -+ -+#endif /*_TRACE_HMBIRD_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird -+ -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE hmbird_trace -+/* This part must be outside protection */ -+#include -diff --git b/kernel/sched/hmbird/hmbird_util_track.c a/kernel/sched/hmbird/hmbird_util_track.c -new file mode 100755 -index 000000000000..d520a8403401 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_util_track.c -@@ -0,0 +1,626 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+ -+#define CREATE_TRACE_POINTS -+#include "hmbird_trace.h" -+#undef CREATE_TRACE_POINTS -+ -+#define HMBIRD_DEBUG_PANIC (1 << 3) -+static bool init_irq_work_inited; -+ -+extern noinline int tracing_mark_write(const char *buf); -+ -+int hmbird_sched_ravg_window = 8000000; -+int new_hmbird_sched_ravg_window = 8000000; -+DEFINE_SPINLOCK(new_sched_ravg_window_lock); -+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+ -+static struct irq_work hmbird_slim_walt_irq_work; -+ -+/*Sysctl related interface*/ -+#define WINDOW_STATS_RECENT 0 -+#define WINDOW_STATS_MAX 1 -+#define WINDOW_STATS_MAX_RECENT_AVG 2 -+#define WINDOW_STATS_AVG 3 -+#define WINDOW_STATS_INVALID_POLICY 4 -+ -+ -+#define SCX_SCHED_CAPACITY_SHIFT 10 -+#define SCHED_ACCOUNT_WAIT_TIME 0 -+ -+atomic64_t hmbird_irq_work_lastq_ws; -+static u64 tick_sched_clock; -+ -+static inline u64 scale_exec_time(u64 delta, struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT; -+} -+ -+static inline u64 scale_time_to_util(u64 d) -+{ -+ do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT); -+ return d; -+} -+ -+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ delta = scale_exec_time(delta, rq); -+ sts->sum += delta; -+ if (unlikely(sts->sum > hmbird_sched_ravg_window)) -+ sts->sum = hmbird_sched_ravg_window; -+ -+ return delta; -+} -+ -+ -+static int -+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) -+{ -+ /* -+ * No need to bother updating task demand for the idle task. -+ */ -+ if (is_idle_task(p)) -+ return 0; -+ -+ /* -+ * When a task is waking up it is completing a segment of non-busy -+ * time. Likewise, if wait time is not treated as busy time, then -+ * when a task begins to run or is migrated, it is not running and -+ * is completing a segment of non-busy time. -+ */ -+ if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && -+ (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) -+ return 0; -+ -+ /* -+ * The idle exit time is not accounted for the first task _picked_ up to -+ * run on the idle CPU. -+ */ -+ if (event == PICK_NEXT_TASK && rq->curr == rq->idle) -+ return 0; -+ -+ /* -+ * TASK_UPDATE can be called on sleeping task, when its moved between -+ * related groups -+ */ -+ if (event == TASK_UPDATE) { -+ if (rq->curr == p) -+ return 1; -+ -+ return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; -+ } -+ -+ return 1; -+} -+ -+static void rollover_cpu_window(struct rq *rq, bool full_window) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 curr_sum = srq->curr_runnable_sum; -+ -+ if (unlikely(full_window)) -+ curr_sum = 0; -+ -+ srq->prev_runnable_sum = curr_sum; -+ srq->curr_runnable_sum = 0; -+} -+ -+static u64 -+update_window_start(struct rq *rq, u64 wallclock, int event) -+{ -+ s64 delta; -+ int nr_windows; -+ bool full_window; -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start = srq->window_start; -+ -+ if (wallclock < srq->latest_clock) -+ wallclock = srq->latest_clock; -+ delta = wallclock - srq->window_start; -+ if (delta < 0) { -+ delta = 0; -+ wallclock = srq->window_start; -+ } -+ srq->latest_clock = wallclock; -+ if (delta < hmbird_sched_ravg_window) -+ return old_window_start; -+ -+ nr_windows = div64_u64(delta, hmbird_sched_ravg_window); -+ srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window; -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ full_window = nr_windows > 1; -+ rollover_cpu_window(rq, full_window); -+ -+ return old_window_start; -+} -+ -+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -+ -+static inline unsigned int get_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static inline unsigned int cpu_cur_freq(int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cur; -+} -+ -+static void -+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, -+ u64 wallclock) -+{ -+ int cpu = cpu_of(rq); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * -+ arch_scale_cpu_capacity(cpu), get_max_freq(cpu)); -+} -+ -+/* -+ * Called when new window is starting for a task, to record cpu usage over -+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 -+ * when, say, a real-time task runs without preemption for several windows at a -+ * stretch. -+ */ -+static void update_history(struct rq *rq, struct task_struct *p, -+ u32 runtime, int samples, int event) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u32 *hist = &sts->sum_history[0]; -+ int i; -+ u32 max = 0, avg, demand; -+ u64 sum = 0; -+ u16 demand_scaled; -+ -+ /* Ignore windows where task had no activity */ -+ if (!runtime || is_idle_task(p) || !samples) -+ goto done; -+ -+ /* Push new 'runtime' value onto stack */ -+ for (; samples > 0; samples--) { -+ hist[sts->cidx] = runtime; -+ sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE; -+ } -+ -+ for (i = 0; i < RAVG_HIST_SIZE; i++) { -+ sum += hist[i]; -+ if (hist[i] > max) -+ max = hist[i]; -+ } -+ -+ sts->sum = 0; -+ avg = div64_u64(sum, RAVG_HIST_SIZE); -+ -+ switch (slim_walt_policy) { -+ case WINDOW_STATS_RECENT: -+ demand = runtime; -+ break; -+ case WINDOW_STATS_MAX: -+ demand = max; -+ break; -+ case WINDOW_STATS_AVG: -+ demand = avg; -+ break; -+ default: -+ demand = max(avg, runtime); -+ } -+ -+ demand_scaled = scale_time_to_util(demand); -+ -+ sts->demand = demand; -+ sts->demand_scaled = demand_scaled; -+ -+done: -+ return; -+} -+ -+ -+static u64 update_task_demand(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ -+ u64 delta, window_start = srq->window_start; -+ int new_window, nr_full_windows; -+ u32 window_size = hmbird_sched_ravg_window; -+ u64 runtime; -+ -+ new_window = mark_start < window_start; -+ if (!account_busy_for_task_demand(rq, p, event)) { -+ if (new_window) -+ /* -+ * If the time accounted isn't being accounted as -+ * busy time, and a new window started, only the -+ * previous window need be closed out with the -+ * pre-existing demand. Multiple windows may have -+ * elapsed, but since empty windows are dropped, -+ * it is not necessary to account those. -+ */ -+ update_history(rq, p, sts->sum, 1, event); -+ return 0; -+ } -+ -+ if (!new_window) { -+ /* -+ * The simple case - busy time contained within the existing -+ * window. -+ */ -+ return add_to_task_demand(rq, p, wallclock - mark_start); -+ } -+ -+ /* -+ * Busy time spans at least two windows. Temporarily rewind -+ * window_start to first window boundary after mark_start. -+ */ -+ delta = window_start - mark_start; -+ nr_full_windows = div64_u64(delta, window_size); -+ window_start -= (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (window_start - mark_start) first */ -+ runtime = add_to_task_demand(rq, p, window_start - mark_start); -+ -+ /* Push new sample(s) into task's demand history */ -+ update_history(rq, p, sts->sum, 1, event); -+ if (nr_full_windows) { -+ u64 scaled_window = scale_exec_time(window_size, rq); -+ -+ update_history(rq, p, scaled_window, nr_full_windows, event); -+ runtime += nr_full_windows * scaled_window; -+ } -+ -+ /* -+ * Roll window_start back to current to process any remainder -+ * in current window. -+ */ -+ window_start += (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (wallclock - window_start) next */ -+ mark_start = window_start; -+ runtime += add_to_task_demand(rq, p, wallclock - mark_start); -+ -+ return runtime; -+} -+ -+u16 slim_walt_cpu_util(int cpu) -+{ -+ u64 prev_runnable_sum; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ prev_runnable_sum = srq->prev_runnable_sum; -+ do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT); -+ -+ return (u16)prev_runnable_sum; -+} -+ -+static DEFINE_PER_CPU(u16, prev_cpu_util); -+static inline void cpu_util_update_systrace_c(int cpu) -+{ -+ char buf[256]; -+ u16 cpu_util = slim_walt_cpu_util(cpu); -+ -+ if (cpu_util != per_cpu(prev_cpu_util, cpu)) { -+ snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n", -+ cpu, cpu_util); -+ tracing_mark_write(buf); -+ per_cpu(prev_cpu_util, cpu) = cpu_util; -+ } -+} -+ -+ -+static inline int account_busy_for_cpu_time(struct rq *rq, -+ struct task_struct *p, -+ int event) -+{ -+ return !is_idle_task(p) && (event == PUT_PREV_TASK -+ || event == TASK_UPDATE); -+} -+ -+ -+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ int new_window, full_window = 0; -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 window_start = srq->window_start; -+ u32 window_size = srq->prev_window_size; -+ u64 delta; -+ u64 *curr_runnable_sum = &srq->curr_runnable_sum; -+ u64 *prev_runnable_sum = &srq->prev_runnable_sum; -+ -+ new_window = mark_start < window_start; -+ if (new_window) -+ full_window = (window_start - mark_start) >= window_size; -+ -+ -+ if (!account_busy_for_cpu_time(rq, p, event)) -+ goto done; -+ -+ -+ if (!new_window) { -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. No rollover -+ * since we didn't start a new window. An example of this is -+ * when a task starts execution and then sleeps within the -+ * same window. -+ */ -+ delta = wallclock - mark_start; -+ -+ delta = scale_exec_time(delta, rq); -+ *curr_runnable_sum += delta; -+ -+ goto done; -+ } -+ -+ /* -+ * situations below this need window rollover, -+ * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done -+ * in update_window_start() -+ * -+ * For task counters curr/prev_window[_cpu] are rolled over in the early part of -+ * this function. If full_window(s) have expired and time since last update needs -+ * to be accounted as busy time, set the prev to a complete window size time, else -+ * add the prev window portion. -+ * -+ * For task curr counters a new window has begun, always assign -+ */ -+ -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. A new window -+ * must have been started in udpate_window_start() -+ * If any of these three above conditions are true -+ * then this busy time can't be accounted as irqtime. -+ * -+ * Busy time for the idle task need not be accounted. -+ * -+ * An example of this would be a task that starts execution -+ * and then sleeps once a new window has begun. -+ */ -+ -+ /* -+ * A full window hasn't elapsed, account partial -+ * contribution to previous completed window. -+ */ -+ -+ -+ delta = full_window ? scale_exec_time(window_size, rq) : -+ scale_exec_time(window_start - mark_start, rq); -+ -+ *prev_runnable_sum += delta; -+ -+ /* Account piece of busy time in the current window. */ -+ delta = scale_exec_time(wallclock - window_start, rq); -+ *curr_runnable_sum += delta; -+ -+done: -+ if (slim_walt_dump && new_window) -+ cpu_util_update_systrace_c(rq->cpu); -+} -+ -+ -+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq) -+{ -+ u64 result; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 new_window_start = srq->window_start; -+ -+ if (old_window_start == new_window_start) -+ return; -+ -+ result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws, -+ old_window_start, new_window_start); -+ if (result != old_window_start) -+ return; -+ -+ if (likely(cpu_online(raw_smp_processor_id()))) -+ irq_work_queue(&hmbird_slim_walt_irq_work); -+ else -+ irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask)); -+} -+ -+/* -+ * In the core scheduler, most of the load update points update the rq_clock after -+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of -+ * obtaining the time, but to prevent the subsequent migration of the load update -+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock -+ * update here. -+ */ -+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p, -+ struct rq *rq, int event) -+{ -+ if (!(rq->clock_update_flags & RQCF_UPDATED)) -+ update_rq_clock(rq); -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock)); -+} -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start; -+ -+ if (!slim_walt_ctrl) -+ return; -+ -+ if (!srq->window_start || sts->mark_start == wallclock) -+ return; -+ -+ old_window_start = update_window_start(rq, wallclock, event); -+ -+ if (!sts->window_start) -+ sts->window_start = srq->window_start; -+ -+ if (!sts->mark_start) -+ goto done; -+ -+ update_task_rq_cpu_cycles(p, rq, event, wallclock); -+ update_task_demand(p, rq, event, wallclock); -+ update_cpu_busy_time(p, rq, event, wallclock); -+ -+ sts->window_start = srq->window_start; -+ -+done: -+ sts->mark_start = wallclock; -+ slim_walt_window_rollover_run_once(old_window_start, rq); -+} -+ -+static void slim_walt_irq_work(struct irq_work *irq_work) -+{ -+ cpumask_t lock_cpus; -+ struct hmbird_sched_rq_stats *srq; -+ struct rq *rq; -+ int cpu; -+ int level = 0; -+ u64 wc; -+ unsigned long flags; -+ -+ cpumask_copy(&lock_cpus, cpu_possible_mask); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ if (level == 0) -+ raw_spin_lock(&cpu_rq(cpu)->__lock); -+ else -+ raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); -+ level++; -+ } -+ -+ wc = sched_clock(); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ rq = cpu_rq(cpu); -+ hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc); -+ } -+ -+ cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER); -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) { -+ srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id()); -+ if (wc < srq->window_start + new_hmbird_sched_ravg_window) -+ hmbird_sched_ravg_window = new_hmbird_sched_ravg_window; -+ } -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ raw_spin_unlock(&cpu_rq(cpu)->__lock); -+ } -+} -+static void hmbird_sched_init_rq(struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ srq->task_exec_scale = 1024; -+ srq->window_start = 0; -+} -+ -+void hmbird_sched_init_task(struct task_struct *p) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ memset(sts, 0, sizeof(struct hmbird_sched_task_stats)); -+} -+ -+static void hmbird_sched_stats_init(void) -+{ -+ unsigned long flags; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ raw_spin_lock_irqsave(&rq->__lock, flags); -+ hmbird_sched_init_rq(rq); -+ raw_spin_unlock_irqrestore(&rq->__lock, flags); -+ } -+ slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG; -+ -+ if (false == init_irq_work_inited) { -+ init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work); -+ init_irq_work_inited = true; -+ } -+} -+ -+void hmbird_scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (unlikely(!tick_sched_clock)) { -+ /* -+ * Let the window begin 20us prior to the tick, -+ * that way we are guaranteed a rollover when the tick occurs. -+ * Use rq->clock directly instead of rq_clock() since -+ * we do not have the rq lock and -+ * rq->clock was updated in the tick callpath. -+ */ -+ if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000)) -+ return; -+ for_each_possible_cpu(cpu) { -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ srq->window_start = tick_sched_clock; -+ } -+ atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock); -+ } -+} -+ -+void slim_walt_enable(int enable) -+{ -+ if (1 == !!enable) { -+ hmbird_sched_stats_init(); -+ WRITE_ONCE(tick_sched_clock, 0); -+ } else -+ slim_walt_ctrl = 0; -+} -+ -+void slim_get_cpu_util(int cpu, u64 *util) -+{ -+ if (cpu < 0) -+ return; -+ -+ *util = slim_walt_cpu_util(cpu); -+} -+ -+void slim_get_task_util(struct task_struct *p, u64 *util) -+{ -+ if (p == NULL) -+ return; -+ -+ *util = get_hmbird_ts(p)->sts.demand_scaled; -+} -+ -+void sched_ravg_window_change(int frame_per_sec) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec; -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+} -diff --git b/kernel/sched/hmbird/hmbird_util_track.h a/kernel/sched/hmbird/hmbird_util_track.h -new file mode 100644 -index 000000000000..17b85df7f83b ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_util_track.h -@@ -0,0 +1,33 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef __HMBIRD_UTIL_TRACK_H__ -+#define __HMBIRD_UTIL_TRACK_H__ -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock); -+void hmbird_sched_init_task(struct task_struct *p); -+void slim_walt_enable(int enable); -+void slim_get_cpu_util(int cpu, u64 *util); -+void slim_get_task_util(struct task_struct *p, u64 *util); -+ -+extern atomic64_t hmbird_irq_work_lastq_ws; -+ -+enum task_event { -+ PUT_PREV_TASK = 0, -+ PICK_NEXT_TASK = 1, -+ TASK_WAKE = 2, -+ TASK_MIGRATE = 3, -+ TASK_UPDATE = 4, -+ IRQ_UPDATE = 5, -+}; -+ -+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+#endif /* __HMBIRD_UTIL_TRACK_H__ */ -diff --git b/kernel/sched/hmbird/slim.h a/kernel/sched/hmbird/slim.h -new file mode 100755 -index 000000000000..dffe6506658e ---- /dev/null -+++ a/kernel/sched/hmbird/slim.h -@@ -0,0 +1,56 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+ -+#ifndef __SLIM_H -+#define __SLIM_H -+ -+extern atomic_t __hmbird_ops_enabled; -+extern atomic_t non_hmbird_task; -+extern int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+extern int heartbeat; -+extern int heartbeat_enable; -+extern int watchdog_enable; -+extern int isolate_ctrl; -+extern int parctrl_high_ratio; -+extern int parctrl_low_ratio; -+extern int isoctrl_high_ratio; -+extern int isoctrl_low_ratio; -+extern int iso_free_rescue; -+extern int yield_opt; -+ -+extern enum hmbird_switch_type sw_type; -+extern noinline int tracing_mark_write(const char *buf); -+int task_top_id(struct task_struct *p); -+void stats_print(char *buf, int len); -+void hmbird_skip_yield(long *skip); -+extern spinlock_t hmbird_tasks_lock; -+extern int scx_systemui_pid; -+ -+struct yield_opt_params { -+ int enable; -+ int frame_per_sec; -+ u64 frame_time_ns; -+ int yield_headroom; -+}; -+ -+extern struct yield_opt_params yield_opt_params; -+ -+struct tick_hit_params { -+ int enable; -+ unsigned long jiffies_num; -+ int hit_count_thres; -+}; -+ -+extern struct tick_hit_params tick_hit_params; -+ -+struct boost_policy_params { -+ int enable; -+ unsigned int bottom_freq; -+ int boost_weight; -+}; -+ -+extern struct boost_policy_params boost_policy_params; -+ -+#define MAX_GOV_LEN (16) -+extern char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+#endif -diff --git b/kernel/sched/idle.c a/kernel/sched/idle.c -index b33cefeb4188..487255ac6832 100755 ---- b/kernel/sched/idle.c -+++ a/kernel/sched/idle.c -@@ -408,13 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl - - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) - { -- scx_update_idle(rq, false); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, false); -+#endif - } - - static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) - { - update_idle_core(rq); -- scx_update_idle(rq, true); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, true); -+#endif - schedstat_inc(rq->sched_goidle); - } - -diff --git b/kernel/sched/sched.h a/kernel/sched/sched.h -index 509e8dc616ab..a7e9caea1efe 100755 ---- b/kernel/sched/sched.h -+++ a/kernel/sched/sched.h -@@ -188,18 +188,13 @@ static inline int idle_policy(int policy) - return policy == SCHED_IDLE; - } - --static inline int normal_policy(int policy) --{ --#ifdef CONFIG_SCHED_CLASS_EXT -- if (policy == SCHED_EXT) -- return true; --#endif -- return policy == SCHED_NORMAL; --} -- - static inline int fair_policy(int policy) - { -- return normal_policy(policy) || policy == SCHED_BATCH; -+#ifdef CONFIG_HMBIRD_SCHED -+ return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#else -+ return policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#endif - } - - static inline int rt_policy(int policy) -@@ -247,25 +242,7 @@ static inline void update_avg(u64 *avg, u64 sample) - #define shr_bound(val, shift) \ - (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) - --/* -- * cgroup weight knobs should use the common MIN, DFL and MAX values which are -- * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it -- * maps pretty well onto the shares value used by scheduler and the round-trip -- * conversions preserve the original value over the entire range. -- */ --static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) --{ -- return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); --} -- --static inline unsigned long sched_weight_to_cgroup(unsigned long weight) --{ -- return clamp_t(unsigned long, -- DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -- CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); --} -- --/* -+/* - * !! For sched_setattr_nocheck() (kernel) only !! - * - * This is actually gross. :( -@@ -532,10 +509,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) { } - #endif /* CONFIG_SMP */ - #else /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_HMBIRD_SCHED - static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) - { - return 0; - } -+#endif - #endif /* CONFIG_FAIR_GROUP_SCHED */ - - #else /* CONFIG_CGROUP_SCHED */ -@@ -700,29 +679,6 @@ struct cfs_rq { - #endif /* CONFIG_FAIR_GROUP_SCHED */ - }; - --#ifdef CONFIG_SCHED_CLASS_EXT --/* scx_rq->flags, protected by the rq lock */ --enum scx_rq_flags { -- SCX_RQ_CAN_STOP_TICK = 1 << 0, --}; -- --struct scx_rq { -- struct scx_dispatch_q local_dsq; -- struct list_head watchdog_list; -- u64 ops_qseq; -- u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -- u32 nr_running; -- u32 flags; -- bool cpu_released; -- cpumask_var_t cpus_to_kick; -- cpumask_var_t cpus_to_preempt; -- cpumask_var_t cpus_to_wait; -- u64 pnt_seq; -- struct irq_work kick_cpus_irq_work; -- struct rq *rq; --}; --#endif /* CONFIG_SCHED_CLASS_EXT */ -- - static inline int rt_bandwidth_enabled(void) - { - return sysctl_sched_rt_runtime >= 0; -@@ -1258,11 +1214,7 @@ struct rq { - - ANDROID_OEM_DATA_ARRAY(1, 16); - --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, struct scx_rq *scx); --#else - ANDROID_KABI_RESERVE(1); --#endif - ANDROID_KABI_RESERVE(2); - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); -@@ -2340,11 +2292,6 @@ struct affinity_context { - unsigned int flags; - }; - --enum rq_onoff_reason { -- RQ_ONOFF_HOTPLUG, /* CPU is going on/offline */ -- RQ_ONOFF_TOPOLOGY, /* sched domain topology update */ --}; -- - struct sched_class { - - #ifdef CONFIG_UCLAMP_TASK -@@ -2551,7 +2498,6 @@ extern void init_sched_rt_class(void); - extern void init_sched_fair_class(void); - - extern void reweight_task(struct task_struct *p, int prio); --extern void __setscheduler_prio(struct task_struct *p, int prio); - - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); -@@ -2632,53 +2578,6 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) - extern void activate_task(struct rq *rq, struct task_struct *p, int flags); - extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); - --struct sched_change_guard { -- struct task_struct *p; -- struct rq *rq; -- bool queued; -- bool running; -- bool done; --}; -- --extern struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -- --extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags); -- --/** -- * SCHED_CHANGE_BLOCK - Nested block for task attribute updates -- * @__rq: Runqueue the target task belongs to -- * @__p: Target task -- * @__flags: DEQUEUE/ENQUEUE_* flags -- * -- * A task may need to be dequeued and put_prev_task'd for attribute updates and -- * set_next_task'd and re-enqueued afterwards. This helper defines a nested -- * block which automatically handles these preparation and cleanup operations. -- * -- * SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- * update_attribute(p); -- * ... -- * } -- * -- * If @__flags is a variable, the variable may be updated in the block body and -- * the updated value will be used when re-enqueueing @p. -- * -- * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling -- * update_rq_clock() beforehand. Otherwise, the rq clock is automatically -- * updated iff the task needs to be dequeued and re-enqueued. Only the former -- * case guarantees that the rq clock is up-to-date inside and after the block. -- */ --#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -- for (struct sched_change_guard __cg = \ -- sched_change_guard_init(__rq, __p, __flags); \ -- !__cg.done; sched_change_guard_fini(&__cg, __flags)) -- --extern void check_class_changing(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class); --extern void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio); -- - extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); - - #ifdef CONFIG_PREEMPT_RT -@@ -3710,6 +3609,8 @@ static inline bool cpu_busy_with_softirqs(int cpu) - } - #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ - --#include "ext.h" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird.h" -+#endif - - #endif /* _KERNEL_SCHED_SCHED_H */ -diff --git b/kernel/time/tick-sched.c a/kernel/time/tick-sched.c -index 998c2eefe173..c13772ee823c 100755 ---- b/kernel/time/tick-sched.c -+++ a/kernel/time/tick-sched.c -@@ -34,6 +34,10 @@ - - #include - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "../sched/hmbird/hmbird_shadow_tick.h" -+#endif -+ - /* - * Per-CPU nohz control structure - */ -@@ -1124,6 +1128,9 @@ void tick_nohz_idle_stop_tick(void) - ktime_t expires; - - trace_android_vh_tick_nohz_idle_stop_tick(NULL); -+#ifdef CONFIG_HMBIRD_SCHED -+ android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL); -+#endif - /* - * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the - * tick timer expiration time is known already. -diff --git b/tools/sched_ext/.gitignore a/tools/sched_ext/.gitignore -deleted file mode 100644 -index a3240f9f7eba..000000000000 ---- b/tools/sched_ext/.gitignore -+++ /dev/null -@@ -1,9 +0,0 @@ --scx_example_simple --scx_example_qmap --scx_example_central --scx_example_pair --scx_example_flatcg --scx_example_userland --*.skel.h --*.subskel.h --/tools/ -diff --git b/tools/sched_ext/Makefile a/tools/sched_ext/Makefile -deleted file mode 100644 -index 73c43782837d..000000000000 ---- b/tools/sched_ext/Makefile -+++ /dev/null -@@ -1,216 +0,0 @@ --# SPDX-License-Identifier: GPL-2.0 --# Copyright (c) 2022 Meta Platforms, Inc. and affiliates. --include ../build/Build.include --include ../scripts/Makefile.arch --include ../scripts/Makefile.include -- --ifneq ($(LLVM),) --ifneq ($(filter %/,$(LLVM)),) --LLVM_PREFIX := $(LLVM) --else ifneq ($(filter -%,$(LLVM)),) --LLVM_SUFFIX := $(LLVM) --endif -- --CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi --CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu --CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl --CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu --CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu --CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu --CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu --CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu --CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu --CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) -- --ifeq ($(CROSS_COMPILE),) --ifeq ($(CLANG_TARGET_FLAGS),) --$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk --else --CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) --endif # CLANG_TARGET_FLAGS --else --CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) --endif # CROSS_COMPILE -- --CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as --else --CC := $(CROSS_COMPILE)gcc --endif # LLVM -- --CURDIR := $(abspath .) --TOOLSDIR := $(abspath ..) --LIBDIR := $(TOOLSDIR)/lib --BPFDIR := $(LIBDIR)/bpf --TOOLSINCDIR := $(TOOLSDIR)/include --BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool --APIDIR := $(TOOLSINCDIR)/uapi --GENDIR := $(abspath ../../include/generated) --GENHDR := $(GENDIR)/autoconf.h -- --SCRATCH_DIR := $(CURDIR)/tools --BUILD_DIR := $(SCRATCH_DIR)/build --INCLUDE_DIR := $(SCRATCH_DIR)/include --BPFOBJ_DIR := $(BUILD_DIR)/libbpf --BPFOBJ := $(BPFOBJ_DIR)/libbpf.a --ifneq ($(CROSS_COMPILE),) --HOST_BUILD_DIR := $(BUILD_DIR)/host --HOST_SCRATCH_DIR := host-tools --HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include --else --HOST_BUILD_DIR := $(BUILD_DIR) --HOST_SCRATCH_DIR := $(SCRATCH_DIR) --HOST_INCLUDE_DIR := $(INCLUDE_DIR) --endif --HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a --RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids --DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -- --VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ -- $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ -- ../../vmlinux \ -- /sys/kernel/btf/vmlinux \ -- /boot/vmlinux-$(shell uname -r) --VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) --ifeq ($(VMLINUX_BTF),) --$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") --endif -- --BPFTOOL ?= $(DEFAULT_BPFTOOL) -- --ifneq ($(wildcard $(GENHDR)),) -- GENFLAGS := -DHAVE_GENHDR --endif -- --CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ -- -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -- -I$(TOOLSINCDIR) -I$(APIDIR) -- --CARGOFLAGS := --release -- --# Silence some warnings when compiled with clang --ifneq ($(LLVM),) --CFLAGS += -Wno-unused-command-line-argument --endif -- --LDFLAGS = -lelf -lz -lpthread -- --IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - &1 \ -- | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ --$(shell $(1) -dM -E - $@ --else -- $(call msg,CP,,$@) -- $(Q)cp "$(VMLINUX_H)" $@ --endif -- --%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \ -- | $(BPFOBJ) -- $(call msg,CLNG-BPF,,$@) -- $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ -- --%.skel.h: %.bpf.o $(BPFTOOL) -- $(call msg,GEN-SKEL,,$@) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) -- $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) -- $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@ -- $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h) -- --scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_userland: scx_example_userland.c scx_example_userland.skel.h \ -- scx_example_userland_common.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR) --atropos: export ATROPOS_CLANG = $(CLANG) --atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS) --atropos: $(INCLUDE_DIR)/vmlinux.h -- cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS) -- --clean: -- cargo clean --manifest-path=atropos/Cargo.toml -- rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) -- rm -f *.o *.bpf.o *.skel.h *.subskel.h -- rm -f scx_example_simple scx_example_qmap scx_example_central \ -- scx_example_pair scx_example_flatcg scx_example_userland -- --.PHONY: all atropos clean -- --# delete failed targets --.DELETE_ON_ERROR: -- --# keep intermediate (.skel.h, .bpf.o, etc) targets --.SECONDARY: -diff --git b/tools/sched_ext/atropos/.gitignore a/tools/sched_ext/atropos/.gitignore -deleted file mode 100644 -index 186dba259ec2..000000000000 ---- b/tools/sched_ext/atropos/.gitignore -+++ /dev/null -@@ -1,3 +0,0 @@ --src/bpf/.output --Cargo.lock --target -diff --git b/tools/sched_ext/atropos/Cargo.toml a/tools/sched_ext/atropos/Cargo.toml -deleted file mode 100644 -index 7462a836d53d..000000000000 ---- b/tools/sched_ext/atropos/Cargo.toml -+++ /dev/null -@@ -1,28 +0,0 @@ --[package] --name = "scx_atropos" --version = "0.5.0" --authors = ["Dan Schatzberg ", "Meta"] --edition = "2021" --description = "Userspace scheduling with BPF" --license = "GPL-2.0-only" -- --[dependencies] --anyhow = "1.0.65" --bitvec = { version = "1.0", features = ["serde"] } --clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } --ctrlc = { version = "3.1", features = ["termination"] } --fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"} --hex = "0.4.3" --libbpf-rs = "0.19.1" --libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] } --libc = "0.2.137" --log = "0.4.17" --ordered-float = "3.4.0" --simplelog = "0.12.0" -- --[build-dependencies] --bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false } --libbpf-cargo = "0.13.0" -- --[features] --enable_backtrace = [] -diff --git b/tools/sched_ext/atropos/build.rs a/tools/sched_ext/atropos/build.rs -deleted file mode 100644 -index 26e792c5e17e..000000000000 ---- b/tools/sched_ext/atropos/build.rs -+++ /dev/null -@@ -1,70 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --extern crate bindgen; -- --use std::env; --use std::fs::create_dir_all; --use std::path::Path; --use std::path::PathBuf; -- --use libbpf_cargo::SkeletonBuilder; -- --const HEADER_PATH: &str = "src/bpf/atropos.h"; -- --fn bindgen_atropos() { -- // Tell cargo to invalidate the built crate whenever the wrapper changes -- println!("cargo:rerun-if-changed={}", HEADER_PATH); -- -- // The bindgen::Builder is the main entry point -- // to bindgen, and lets you build up options for -- // the resulting bindings. -- let bindings = bindgen::Builder::default() -- // The input header we would like to generate -- // bindings for. -- .header(HEADER_PATH) -- // Tell cargo to invalidate the built crate whenever any of the -- // included header files changed. -- .parse_callbacks(Box::new(bindgen::CargoCallbacks)) -- // Finish the builder and generate the bindings. -- .generate() -- // Unwrap the Result and panic on failure. -- .expect("Unable to generate bindings"); -- -- // Write the bindings to the $OUT_DIR/bindings.rs file. -- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); -- bindings -- .write_to_file(out_path.join("atropos-sys.rs")) -- .expect("Couldn't write bindings!"); --} -- --fn gen_bpf_sched(name: &str) { -- let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap(); -- let clang = env::var("ATROPOS_CLANG").unwrap(); -- eprintln!("{}", clang); -- let outpath = format!("./src/bpf/.output/{}.skel.rs", name); -- let skel = Path::new(&outpath); -- let src = format!("./src/bpf/{}.bpf.c", name); -- SkeletonBuilder::new() -- .source(src.clone()) -- .clang(clang) -- .clang_args(bpf_cflags) -- .build_and_generate(&skel) -- .unwrap(); -- println!("cargo:rerun-if-changed={}", src); --} -- --fn main() { -- bindgen_atropos(); -- // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton. -- // Reasons are because the generated skeleton contains compiler attributes -- // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]` -- // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside -- // the path attribute either (see https://github.com/rust-lang/rust/pull/83366). -- // -- // However, there is hope! When the above feature stabilizes we can clean this -- // all up. -- create_dir_all("./src/bpf/.output").unwrap(); -- gen_bpf_sched("atropos"); --} -diff --git b/tools/sched_ext/atropos/rustfmt.toml a/tools/sched_ext/atropos/rustfmt.toml -deleted file mode 100644 -index b7258ed0a8d8..000000000000 ---- b/tools/sched_ext/atropos/rustfmt.toml -+++ /dev/null -@@ -1,8 +0,0 @@ --# Get help on options with `rustfmt --help=config` --# Please keep these in alphabetical order. --edition = "2021" --group_imports = "StdExternalCrate" --imports_granularity = "Item" --merge_derives = false --use_field_init_shorthand = true --version = "Two" -diff --git b/tools/sched_ext/atropos/src/atropos_sys.rs a/tools/sched_ext/atropos/src/atropos_sys.rs -deleted file mode 100644 -index bbeaf856d40e..000000000000 ---- b/tools/sched_ext/atropos/src/atropos_sys.rs -+++ /dev/null -@@ -1,10 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#![allow(non_upper_case_globals)] --#![allow(non_camel_case_types)] --#![allow(non_snake_case)] --#![allow(dead_code)] -- --include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs")); -diff --git b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -deleted file mode 100644 -index 3905a403e940..000000000000 ---- b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -+++ /dev/null -@@ -1,743 +0,0 @@ --/* Copyright (c) Meta Platforms, Inc. and affiliates. */ --/* -- * This software may be used and distributed according to the terms of the -- * GNU General Public License version 2. -- * -- * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF -- * part does simple round robin in each domain and the userspace part -- * calculates the load factor of each domain and tells the BPF part how to load -- * balance the domains. -- * -- * Every task has an entry in the task_data map which lists which domain the -- * task belongs to. When a task first enters the system (atropos_prep_enable), -- * they are round-robined to a domain. -- * -- * atropos_select_cpu is the primary scheduling logic, invoked when a task -- * becomes runnable. The lb_data map is populated by userspace to inform the BPF -- * scheduler that a task should be migrated to a new domain. Otherwise, the task -- * is scheduled in priority order as follows: -- * * The current core if the task was woken up synchronously and there are idle -- * cpus in the system -- * * The previous core, if idle -- * * The pinned-to core if the task is pinned to a specific core -- * * Any idle cpu in the domain -- * -- * If none of the above conditions are met, then the task is enqueued to a -- * dispatch queue corresponding to the domain (atropos_enqueue). -- * -- * atropos_dispatch will attempt to consume a task from its domain's -- * corresponding dispatch queue (this occurs after scheduling any tasks directly -- * assigned to it due to the logic in atropos_select_cpu). If no task is found, -- * then greedy load stealing will attempt to find a task on another dispatch -- * queue to run. -- * -- * Load balancing is almost entirely handled by userspace. BPF populates the -- * task weight, dom mask and current dom in the task_data map and executes the -- * load balance based on userspace populating the lb_data map. -- */ --#include "../../../scx_common.bpf.h" --#include "atropos.h" -- --#include --#include --#include --#include --#include --#include -- --char _license[] SEC("license") = "GPL"; -- --/* -- * const volatiles are set during initialization and treated as consts by the -- * jit compiler. -- */ -- --/* -- * Domains and cpus -- */ --const volatile __u32 nr_doms = 32; /* !0 for veristat, set during init */ --const volatile __u32 nr_cpus = 64; /* !0 for veristat, set during init */ --const volatile __u32 cpu_dom_id_map[MAX_CPUS]; --const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; -- --const volatile bool kthreads_local; --const volatile bool fifo_sched; --const volatile bool switch_partial; --const volatile __u32 greedy_threshold; -- --/* base slice duration */ --const volatile __u64 slice_us = 20000; -- --/* -- * Exit info -- */ --int exit_type = SCX_EXIT_NONE; --char exit_msg[SCX_EXIT_MSG_LEN]; -- --struct pcpu_ctx { -- __u32 dom_rr_cur; /* used when scanning other doms */ -- -- /* libbpf-rs does not respect the alignment, so pad out the struct explicitly */ -- __u8 _padding[CACHELINE_SIZE - sizeof(u64)]; --} __attribute__((aligned(CACHELINE_SIZE))); -- --struct pcpu_ctx pcpu_ctx[MAX_CPUS]; -- --/* -- * Domain context -- */ --struct dom_ctx { -- struct bpf_cpumask __kptr *cpumask; -- u64 vtime_now; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __type(key, u32); -- __type(value, struct dom_ctx); -- __uint(max_entries, MAX_DOMS); -- __uint(map_flags, 0); --} dom_ctx SEC(".maps"); -- --/* -- * Statistics -- */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, ATROPOS_NR_STATS); --} stats SEC(".maps"); -- --static inline void stat_add(enum stat_idx idx, u64 addend) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p) += addend; --} -- --/* Map pid -> task_ctx */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, struct task_ctx); -- __uint(max_entries, 1000000); -- __uint(map_flags, 0); --} task_data SEC(".maps"); -- --/* -- * This is populated from userspace to indicate which pids should be reassigned -- * to new doms. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, u32); -- __uint(max_entries, 1000); -- __uint(map_flags, 0); --} lb_data SEC(".maps"); -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p, -- u32 new_dom_id) --{ -- struct dom_ctx *old_domc, *new_domc; -- struct bpf_cpumask *d_cpumask, *t_cpumask; -- u32 old_dom_id = task_ctx->dom_id; -- s64 vtime_delta; -- -- old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id); -- if (!old_domc) { -- scx_bpf_error("No dom%u", old_dom_id); -- return false; -- } -- -- vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now; -- -- new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id); -- if (!new_domc) { -- scx_bpf_error("No dom%u", new_dom_id); -- return false; -- } -- -- d_cpumask = new_domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to get domain %u cpumask kptr", -- new_dom_id); -- return false; -- } -- -- t_cpumask = task_ctx->cpumask; -- if (!t_cpumask) { -- scx_bpf_error("Failed to look up task cpumask"); -- return false; -- } -- -- /* -- * set_cpumask might have happened between userspace requesting LB and -- * here and @p might not be able to run in @dom_id anymore. Verify. -- */ -- if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- p->cpus_ptr)) { -- p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta; -- task_ctx->dom_id = new_dom_id; -- bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask, -- p->cpus_ptr); -- } -- -- return task_ctx->dom_id == new_dom_id; --} -- --s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu, -- u32 wake_flags) --{ -- s32 cpu; -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- struct bpf_cpumask *p_cpumask; -- -- if (!task_ctx) -- return -ENOENT; -- -- if (kthreads_local && -- (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- cpu = prev_cpu; -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local dsq of the waker. -- */ -- if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) { -- struct task_struct *current = (void *)bpf_get_current_task(); -- -- if (!(BPF_CORE_READ(current, flags) & PF_EXITING) && -- task_ctx->dom_id < MAX_DOMS) { -- struct dom_ctx *domc; -- struct bpf_cpumask *d_cpumask; -- const struct cpumask *idle_cpumask; -- bool has_idle; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id); -- if (!domc) { -- scx_bpf_error("Failed to find dom%u", -- task_ctx->dom_id); -- return prev_cpu; -- } -- d_cpumask = domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to acquire domain %u cpumask kptr", -- task_ctx->dom_id); -- return prev_cpu; -- } -- -- idle_cpumask = scx_bpf_get_idle_cpumask(); -- -- has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- idle_cpumask); -- -- scx_bpf_put_idle_cpumask(idle_cpumask); -- -- if (has_idle) { -- cpu = bpf_get_smp_processor_id(); -- if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- stat_add(ATROPOS_STAT_WAKE_SYNC, 1); -- goto local; -- } -- } -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- stat_add(ATROPOS_STAT_PREV_IDLE, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- /* If only one core is allowed, dispatch */ -- if (p->nr_cpus_allowed == 1) { -- stat_add(ATROPOS_STAT_PINNED, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) -- return -ENOENT; -- -- /* If there is an eligible idle CPU, dispatch directly */ -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- if (cpu >= 0) { -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * @prev_cpu may be in a different domain. Returning an out-of-domain -- * CPU can lead to stalls as all in-domain CPUs may be idle by the time -- * @p gets enqueued. -- */ -- if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask)) -- cpu = prev_cpu; -- else -- cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask); -- -- return cpu; -- --local: -- task_ctx->dispatch_local = true; -- return cpu; --} -- --void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- u32 *new_dom; -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- new_dom = bpf_map_lookup_elem(&lb_data, &pid); -- if (new_dom && *new_dom != task_ctx->dom_id && -- task_set_dsq(task_ctx, p, *new_dom)) { -- struct bpf_cpumask *p_cpumask; -- s32 cpu; -- -- stat_add(ATROPOS_STAT_LOAD_BALANCE, 1); -- -- /* -- * If dispatch_local is set, We own @p's idle state but we are -- * not gonna put the task in the associated local dsq which can -- * cause the CPU to stall. Kick it. -- */ -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0); -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) { -- scx_bpf_error("Failed to get task_ctx->cpumask"); -- return; -- } -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- } -- -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags); -- return; -- } -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000, -- enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- u32 dom_id = task_ctx->dom_id; -- struct dom_ctx *domc; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, domc->vtime_now - slice_us * 1000)) -- vtime = domc->vtime_now - slice_us * 1000; -- -- scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --static u32 cpu_to_dom_id(s32 cpu) --{ -- const volatile u32 *dom_idp; -- -- if (nr_doms <= 1) -- return 0; -- -- dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]); -- if (!dom_idp) -- return MAX_DOMS; -- -- return *dom_idp; --} -- --static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) --{ -- s32 cpu; -- -- if (dom_id >= MAX_DOMS) -- return false; -- -- bpf_for(cpu, 0, nr_cpus) { -- if (bpf_cpumask_test_cpu(cpu, cpumask) && -- (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64)))) -- return true; -- } -- return false; --} -- --static u32 dom_rr_next(s32 cpu) --{ -- struct pcpu_ctx *pcpuc; -- u32 dom_id; -- -- pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]); -- if (!pcpuc) -- return 0; -- -- dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms; -- -- if (dom_id == cpu_to_dom_id(cpu)) -- dom_id = (dom_id + 1) % nr_doms; -- -- pcpuc->dom_rr_cur = dom_id; -- return dom_id; --} -- --void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 dom = cpu_to_dom_id(cpu); -- -- if (scx_bpf_consume(dom)) { -- stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1); -- return; -- } -- -- if (!greedy_threshold) -- return; -- -- bpf_repeat(nr_doms - 1) { -- u32 dom_id = dom_rr_next(cpu); -- -- if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold && -- scx_bpf_consume(dom_id)) { -- stat_add(ATROPOS_STAT_GREEDY, 1); -- break; -- } -- } --} -- --void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_at = bpf_ktime_get_ns(); --} -- --void BPF_STRUCT_OPS(atropos_running, struct task_struct *p) --{ -- struct task_ctx *taskc; -- struct dom_ctx *domc; -- pid_t pid = p->pid; -- u32 dom_id; -- -- if (fifo_sched) -- return; -- -- taskc = bpf_map_lookup_elem(&task_data, &pid); -- if (!taskc) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- dom_id = taskc->dom_id; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(domc->vtime_now, p->scx.dsq_vtime)) -- domc->vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at; -- task_ctx->runnable_at = 0; --} -- --void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->weight = weight; --} -- --struct pick_task_domain_loop_ctx { -- struct task_struct *p; -- const struct cpumask *cpumask; -- u64 dom_mask; -- u32 dom_rr_base; -- u32 dom_id; --}; -- --static int pick_task_domain_loopfn(u32 idx, void *data) --{ -- struct pick_task_domain_loop_ctx *lctx = data; -- u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms; -- -- if (dom_id >= MAX_DOMS) -- return 1; -- -- if (cpumask_intersects_domain(lctx->cpumask, dom_id)) { -- lctx->dom_mask |= 1LLU << dom_id; -- if (lctx->dom_id == MAX_DOMS) -- lctx->dom_id = dom_id; -- } -- return 0; --} -- --static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- struct pick_task_domain_loop_ctx lctx = { -- .p = p, -- .cpumask = cpumask, -- .dom_id = MAX_DOMS, -- }; -- s32 cpu = bpf_get_smp_processor_id(); -- -- if (cpu < 0 || cpu >= MAX_CPUS) -- return MAX_DOMS; -- -- lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur); -- -- bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0); -- task_ctx->dom_mask = lctx.dom_mask; -- -- return lctx.dom_id; --} -- --static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- u32 dom_id = 0; -- -- if (nr_doms > 1) -- dom_id = pick_task_domain(task_ctx, p, cpumask); -- -- if (!task_set_dsq(task_ctx, p, dom_id)) -- scx_bpf_error("Failed to set domain %d for %s[%d]", -- dom_id, p->comm, p->pid); --} -- --void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_set_domain(task_ctx, p, cpumask); --} -- --s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct bpf_cpumask *cpumask; -- struct task_ctx task_ctx, *map_value; -- long ret; -- pid_t pid; -- -- memset(&task_ctx, 0, sizeof(task_ctx)); -- -- pid = p->pid; -- ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return ret; -- } -- -- /* -- * Read the entry from the map immediately so we can add the cpumask -- * with bpf_kptr_xchg(). -- */ -- map_value = bpf_map_lookup_elem(&task_data, &pid); -- if (!map_value) -- /* Should never happen -- it was just inserted above. */ -- return -EINVAL; -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- bpf_map_delete_elem(&task_data, &pid); -- return -ENOMEM; -- } -- -- cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask); -- if (cpumask) { -- /* Should never happen as we just inserted it above. */ -- bpf_cpumask_release(cpumask); -- bpf_map_delete_elem(&task_data, &pid); -- return -EINVAL; -- } -- -- task_set_domain(map_value, p, p->cpus_ptr); -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p) --{ -- pid_t pid = p->pid; -- long ret = bpf_map_delete_elem(&task_data, &pid); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return; -- } --} -- --static int create_dom_dsq(u32 idx, void *data) --{ -- struct dom_ctx domc_init = {}, *domc; -- struct bpf_cpumask *cpumask; -- u32 cpu, dom_id = idx; -- s32 ret; -- -- ret = scx_bpf_create_dsq(dom_id, -1); -- if (ret < 0) { -- scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret); -- return 1; -- } -- -- ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0); -- if (ret) { -- scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret); -- return 1; -- } -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- /* Should never happen, we just inserted it above. */ -- scx_bpf_error("No dom%u", dom_id); -- return 1; -- } -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id); -- return 1; -- } -- -- for (cpu = 0; cpu < MAX_CPUS; cpu++) { -- const volatile __u64 *dmask; -- -- dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]); -- if (!dmask) { -- scx_bpf_error("array index error"); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- if (*dmask & (1LLU << (cpu % 64))) -- bpf_cpumask_set_cpu(cpu, cpumask); -- } -- -- cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask); -- if (cpumask) { -- scx_bpf_error("Domain %u was already present", dom_id); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(atropos_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- bpf_loop(nr_doms, create_dom_dsq, NULL, 0); -- -- for (u32 i = 0; i < nr_cpus; i++) -- pcpu_ctx[i].dom_rr_cur = i; -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg); -- exit_type = ei->type; --} -- --SEC(".struct_ops") --struct sched_ext_ops atropos = { -- .select_cpu = (void *)atropos_select_cpu, -- .enqueue = (void *)atropos_enqueue, -- .dispatch = (void *)atropos_dispatch, -- .runnable = (void *)atropos_runnable, -- .running = (void *)atropos_running, -- .stopping = (void *)atropos_stopping, -- .quiescent = (void *)atropos_quiescent, -- .set_weight = (void *)atropos_set_weight, -- .set_cpumask = (void *)atropos_set_cpumask, -- .prep_enable = (void *)atropos_prep_enable, -- .disable = (void *)atropos_disable, -- .init = (void *)atropos_init, -- .exit = (void *)atropos_exit, -- .flags = 0, -- .name = "atropos", --}; -diff --git b/tools/sched_ext/atropos/src/bpf/atropos.h a/tools/sched_ext/atropos/src/bpf/atropos.h -deleted file mode 100644 -index addf29ca104a..000000000000 ---- b/tools/sched_ext/atropos/src/bpf/atropos.h -+++ /dev/null -@@ -1,44 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#ifndef __ATROPOS_H --#define __ATROPOS_H -- --#include --#ifndef __kptr --#ifdef __KERNEL__ --#error "__kptr_ref not defined in the kernel" --#endif --#define __kptr --#endif -- --#define MAX_CPUS 512 --#define MAX_DOMS 64 /* limited to avoid complex bitmask ops */ --#define CACHELINE_SIZE 64 -- --/* Statistics */ --enum stat_idx { -- ATROPOS_STAT_TASK_GET_ERR, -- ATROPOS_STAT_WAKE_SYNC, -- ATROPOS_STAT_PREV_IDLE, -- ATROPOS_STAT_PINNED, -- ATROPOS_STAT_DIRECT_DISPATCH, -- ATROPOS_STAT_DSQ_DISPATCH, -- ATROPOS_STAT_GREEDY, -- ATROPOS_STAT_LOAD_BALANCE, -- ATROPOS_STAT_LAST_TASK, -- ATROPOS_NR_STATS, --}; -- --struct task_ctx { -- unsigned long long dom_mask; /* the domains this task can run on */ -- struct bpf_cpumask __kptr *cpumask; -- unsigned int dom_id; -- unsigned int weight; -- unsigned long long runnable_at; -- unsigned long long runnable_for; -- bool dispatch_local; --}; -- --#endif /* __ATROPOS_H */ -diff --git b/tools/sched_ext/atropos/src/main.rs a/tools/sched_ext/atropos/src/main.rs -deleted file mode 100644 -index 0d313662f713..000000000000 ---- b/tools/sched_ext/atropos/src/main.rs -+++ /dev/null -@@ -1,942 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#[path = "bpf/.output/atropos.skel.rs"] --mod atropos; --pub use atropos::*; --pub mod atropos_sys; -- --use std::cell::Cell; --use std::collections::{BTreeMap, BTreeSet}; --use std::ffi::CStr; --use std::ops::Bound::{Included, Unbounded}; --use std::sync::atomic::{AtomicBool, Ordering}; --use std::sync::Arc; --use std::time::{Duration, SystemTime}; -- --use ::fb_procfs as procfs; --use anyhow::{anyhow, bail, Context, Result}; --use bitvec::prelude::*; --use clap::Parser; --use log::{info, trace, warn}; --use ordered_float::OrderedFloat; -- --/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF --/// part does simple round robin in each domain and the userspace part --/// calculates the load factor of each domain and tells the BPF part how to load --/// balance the domains. -- --/// This scheduler demonstrates dividing scheduling logic between BPF and --/// userspace and using rust to build the userspace part. An earlier variant of --/// this scheduler was used to balance across six domains, each representing a --/// chiplet in a six-chiplet AMD processor, and could match the performance of --/// production setup using CFS. --#[derive(Debug, Parser)] --struct Opts { -- /// Scheduling slice duration in microseconds. -- #[clap(short, long, default_value = "20000")] -- slice_us: u64, -- -- /// Monitoring and load balance interval in seconds. -- #[clap(short, long, default_value = "2.0")] -- interval: f64, -- -- /// Build domains according to how CPUs are grouped at this cache level -- /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id. -- #[clap(short = 'c', long, default_value = "3")] -- cache_level: u32, -- -- /// Instead of using cache locality, set the cpumask for each domain -- /// manually, provide multiple --cpumasks, one for each domain. E.g. -- /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with -- /// the corresponding CPUs belonging to each domain. Each CPU must -- /// belong to precisely one domain. -- #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")] -- cpumasks: Vec, -- -- /// When non-zero, enable greedy task stealing. When a domain is idle, a -- /// cpu will attempt to steal tasks from a domain with at least -- /// greedy_threshold tasks enqueued. These tasks aren't permanently -- /// stolen from the domain. -- #[clap(short, long, default_value = "4")] -- greedy_threshold: u32, -- -- /// The load decay factor. Every interval, the existing load is decayed -- /// by this factor and new load is added. Must be in the range [0.0, -- /// 0.99]. The smaller the value, the more sensitive load calculation -- /// is to recent changes. When 0.0, history is ignored and the load -- /// value from the latest period is used directly. -- #[clap(short, long, default_value = "0.5")] -- load_decay_factor: f64, -- -- /// Disable load balancing. Unless disabled, periodically userspace will -- /// calculate the load factor of each domain and instruct BPF which -- /// processes to move. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- no_load_balance: bool, -- -- /// Put per-cpu kthreads directly into local dsq's. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- kthreads_local: bool, -- -- /// Use FIFO scheduling instead of weighted vtime scheduling. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- fifo_sched: bool, -- -- /// If specified, only tasks which have their scheduling policy set to -- /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all -- /// tasks are switched. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- partial: bool, -- -- /// Enable verbose output including libbpf details. Specify multiple -- /// times to increase verbosity. -- #[clap(short, long, action = clap::ArgAction::Count)] -- verbose: u8, --} -- --fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result { -- Ok(reader -- .read_stat() -- .context("Failed to read procfs")? -- .total_cpu -- .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?) --} -- --fn now_monotonic() -> u64 { -- let mut time = libc::timespec { -- tv_sec: 0, -- tv_nsec: 0, -- }; -- let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) }; -- assert!(ret == 0); -- time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 --} -- --fn clear_map(map: &mut libbpf_rs::Map) { -- // XXX: libbpf_rs has some design flaw that make it impossible to -- // delete while iterating despite it being safe so we alias it here -- let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) }; -- for key in map.keys() { -- let _ = deleter.delete(&key); -- } --} -- --#[derive(Debug)] --struct TaskLoad { -- runnable_for: u64, -- load: f64, --} -- --#[derive(Debug)] --struct TaskInfo { -- pid: i32, -- dom_mask: u64, -- migrated: Cell, --} -- --struct LoadBalancer<'a, 'b, 'c> { -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- -- tasks_by_load: Vec, TaskInfo>>, -- load_avg: f64, -- dom_loads: Vec, -- -- imbal: Vec, -- doms_to_push: BTreeMap, u32>, -- doms_to_pull: BTreeMap, u32>, -- -- nr_lb_data_errors: &'c mut u64, --} -- --impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> { -- const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10; -- const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1; -- const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50; -- -- fn new( -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- nr_lb_data_errors: &'c mut u64, -- ) -> Self { -- Self { -- maps, -- task_loads, -- nr_doms, -- load_decay_factor, -- -- tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(), -- load_avg: 0f64, -- dom_loads: vec![0.0; nr_doms], -- -- imbal: vec![0.0; nr_doms], -- doms_to_pull: BTreeMap::new(), -- doms_to_push: BTreeMap::new(), -- -- nr_lb_data_errors, -- } -- } -- -- fn read_task_loads(&mut self, period: Duration) -> Result<()> { -- let now_mono = now_monotonic(); -- let task_data = self.maps.task_data(); -- let mut this_task_loads = BTreeMap::::new(); -- let mut load_sum = 0.0f64; -- self.dom_loads = vec![0f64; self.nr_doms]; -- -- for key in task_data.keys() { -- if let Some(task_ctx_vec) = task_data -- .lookup(&key, libbpf_rs::MapFlags::ANY) -- .context("Failed to lookup task_data")? -- { -- let task_ctx = -- unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) }; -- let pid = i32::from_ne_bytes( -- key.as_slice() -- .try_into() -- .context("Invalid key length in task_data map")?, -- ); -- -- let (this_at, this_for, weight) = unsafe { -- ( -- std::ptr::read_volatile(&task_ctx.runnable_at as *const u64), -- std::ptr::read_volatile(&task_ctx.runnable_for as *const u64), -- std::ptr::read_volatile(&task_ctx.weight as *const u32), -- ) -- }; -- -- let (mut delta, prev_load) = match self.task_loads.get(&pid) { -- Some(prev) => (this_for - prev.runnable_for, Some(prev.load)), -- None => (this_for, None), -- }; -- -- // Non-zero this_at indicates that the task is currently -- // runnable. Note that we read runnable_at and runnable_for -- // without any synchronization and there is a small window -- // where we end up misaccounting. While this can cause -- // temporary error, it's unlikely to cause any noticeable -- // misbehavior especially given the load value clamping. -- if this_at > 0 && this_at < now_mono { -- delta += now_mono - this_at; -- } -- -- delta = delta.min(period.as_nanos() as u64); -- let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64) -- .clamp(0.0, weight as f64); -- -- let this_load = match prev_load { -- Some(prev_load) => { -- prev_load * self.load_decay_factor -- + this_load * (1.0 - self.load_decay_factor) -- } -- None => this_load, -- }; -- -- this_task_loads.insert( -- pid, -- TaskLoad { -- runnable_for: this_for, -- load: this_load, -- }, -- ); -- -- load_sum += this_load; -- self.dom_loads[task_ctx.dom_id as usize] += this_load; -- // Only record pids that are eligible for load balancing -- if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) { -- continue; -- } -- self.tasks_by_load[task_ctx.dom_id as usize].insert( -- OrderedFloat(this_load), -- TaskInfo { -- pid, -- dom_mask: task_ctx.dom_mask, -- migrated: Cell::new(false), -- }, -- ); -- } -- } -- -- self.load_avg = load_sum / self.nr_doms as f64; -- *self.task_loads = this_task_loads; -- Ok(()) -- } -- -- // To balance dom loads we identify doms with lower and higher load than average -- fn calculate_dom_load_balance(&mut self) -> Result<()> { -- for (dom, dom_load) in self.dom_loads.iter().enumerate() { -- let imbal = dom_load - self.load_avg; -- if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO { -- if imbal > 0f64 { -- self.doms_to_push.insert(OrderedFloat(imbal), dom as u32); -- } else { -- self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32); -- } -- self.imbal[dom] = imbal; -- } -- } -- Ok(()) -- } -- -- // Find the first candidate pid which hasn't already been migrated and -- // can run in @pull_dom. -- fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)> -- where -- I: IntoIterator, &'d TaskInfo)>, -- { -- match tasks_by_load -- .into_iter() -- .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0) -- .next() -- { -- Some((OrderedFloat(load), task)) => Some((*load, task)), -- None => None, -- } -- } -- -- fn pick_victim( -- &self, -- (push_dom, to_push): (u32, f64), -- (pull_dom, to_pull): (u32, f64), -- ) -> Option<(&TaskInfo, f64)> { -- let to_xfer = to_pull.min(to_push); -- -- trace!( -- "considering dom {}@{:.2} -> {}@{:.2}", -- push_dom, -- to_push, -- pull_dom, -- to_pull -- ); -- -- let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs(); -- -- trace!( -- "to_xfer={:.2} tasks_by_load={:?}", -- to_xfer, -- &self.tasks_by_load[push_dom as usize] -- ); -- -- // We want to pick a task to transfer from push_dom to pull_dom to -- // maximize the reduction of load imbalance between the two. IOW, -- // pick a task which has the closest load value to $to_xfer that can -- // be migrated. Find such task by locating the first migratable task -- // while scanning left from $to_xfer and the counterpart while -- // scanning right and picking the better of the two. -- let (load, task, new_imbal) = match ( -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Unbounded, Included(&OrderedFloat(to_xfer)))) -- .rev(), -- pull_dom, -- ), -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Included(&OrderedFloat(to_xfer)), Unbounded)), -- pull_dom, -- ), -- ) { -- (None, None) => return None, -- (Some((load, task)), None) | (None, Some((load, task))) => { -- (load, task, calc_new_imbal(load)) -- } -- (Some((load0, task0)), Some((load1, task1))) => { -- let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1)); -- if new_imbal0 <= new_imbal1 { -- (load0, task0, new_imbal0) -- } else { -- (load1, task1, new_imbal1) -- } -- } -- }; -- -- // If the best candidate can't reduce the imbalance, there's nothing -- // to do for this pair. -- let old_imbal = to_push + to_pull; -- if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal { -- trace!( -- "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal -- ); -- return None; -- } -- -- trace!( -- "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal, -- ); -- -- Some((task, load)) -- } -- -- // Actually execute the load balancing. Concretely this writes pid -> dom -- // entries into the lb_data map for bpf side to consume. -- fn load_balance(&mut self) -> Result<()> { -- clear_map(self.maps.lb_data()); -- -- trace!("imbal={:?}", &self.imbal); -- trace!("doms_to_push={:?}", &self.doms_to_push); -- trace!("doms_to_pull={:?}", &self.doms_to_pull); -- -- // Push from the most imbalanced to least. -- while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() { -- let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO; -- let mut pushed = 0f64; -- -- // Transfer tasks from push_dom to reduce imbalance. -- loop { -- let last_pushed = pushed; -- -- // Pull from the most imbalaned to least. -- let mut doms_to_pull = BTreeMap::<_, _>::new(); -- std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull); -- let mut pull_doms = doms_to_pull.into_iter().rev().collect::>(); -- -- for (to_pull, pull_dom) in pull_doms.iter_mut() { -- if let Some((task, load)) = -- self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull))) -- { -- // Execute migration. -- task.migrated.set(true); -- to_push -= load; -- *to_pull -= load; -- pushed += load; -- -- // Ask BPF code to execute the migration. -- let pid = task.pid; -- let cpid = (pid as libc::pid_t).to_ne_bytes(); -- if let Err(e) = self.maps.lb_data().update( -- &cpid, -- &pull_dom.to_ne_bytes(), -- libbpf_rs::MapFlags::NO_EXIST, -- ) { -- warn!( -- "Failed to update lb_data map for pid={} error={:?}", -- pid, &e -- ); -- *self.nr_lb_data_errors += 1; -- } -- -- // Always break after a successful migration so that -- // the pulling domains are always considered in the -- // descending imbalance order. -- break; -- } -- } -- -- pull_doms -- .into_iter() -- .map(|(k, v)| self.doms_to_pull.insert(k, v)) -- .count(); -- -- // Stop repeating if nothing got transferred or pushed enough. -- if pushed == last_pushed || pushed >= push_max { -- break; -- } -- } -- } -- Ok(()) -- } --} -- --struct Scheduler<'a> { -- skel: AtroposSkel<'a>, -- struct_ops: Option, -- -- nr_cpus: usize, -- nr_doms: usize, -- load_decay_factor: f64, -- balance_load: bool, -- -- proc_reader: procfs::ProcReader, -- -- prev_at: SystemTime, -- prev_total_cpu: procfs::CpuStat, -- task_loads: BTreeMap, -- -- nr_lb_data_errors: u64, --} -- --impl<'a> Scheduler<'a> { -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn parse_cpusets( -- cpumasks: &[String], -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- if cpumasks.len() > atropos_sys::MAX_DOMS as usize { -- bail!( -- "Number of requested DSQs ({}) is greater than MAX_DOMS ({})", -- cpumasks.len(), -- atropos_sys::MAX_DOMS -- ); -- } -- let mut cpus = vec![-1i32; nr_cpus]; -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()]; -- for (dq, cpumask) in cpumasks.iter().enumerate() { -- let hex_str = { -- let mut tmp_str = cpumask -- .strip_prefix("0x") -- .unwrap_or(cpumask) -- .replace('_', ""); -- if tmp_str.len() % 2 != 0 { -- tmp_str = "0".to_string() + &tmp_str; -- } -- tmp_str -- }; -- let byte_vec = hex::decode(&hex_str) -- .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?; -- -- for (index, &val) in byte_vec.iter().rev().enumerate() { -- let mut v = val; -- while v != 0 { -- let lsb = v.trailing_zeros() as usize; -- v &= !(1 << lsb); -- let cpu = index * 8 + lsb; -- if cpu > nr_cpus { -- bail!( -- concat!( -- "Found cpu ({}) in cpumask ({}) which is larger", -- " than the number of cpus on the machine ({})" -- ), -- cpu, -- cpumask, -- nr_cpus -- ); -- } -- if cpus[cpu] != -1 { -- bail!( -- "Found cpu ({}) with dq ({}) but also in cpumask ({})", -- cpu, -- cpus[cpu], -- cpumask -- ); -- } -- cpus[cpu] = dq as i32; -- cpusets[dq].set(cpu, true); -- } -- } -- cpusets[dq].set_uninitialized(false); -- } -- -- for (cpu, &dq) in cpus.iter().enumerate() { -- if dq < 0 { -- bail!( -- "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.", -- cpu -- ); -- } -- } -- -- Ok((cpusets, cpus)) -- } -- -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn cpusets_from_cache( -- level: u32, -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- let mut cpu_to_cache = vec![]; // (cpu_id, cache_id) -- let mut cache_ids = BTreeSet::::new(); -- let mut nr_not_found = 0; -- -- // Build cpu -> cache ID mapping. -- for cpu in 0..nr_cpus { -- let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level); -- let id = match std::fs::read_to_string(&path) { -- Ok(val) => val -- .trim() -- .parse::() -- .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?, -- Err(e) if e.kind() == std::io::ErrorKind::NotFound => { -- nr_not_found += 1; -- 0 -- } -- Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)), -- }; -- -- cpu_to_cache.push(id); -- cache_ids.insert(id); -- } -- -- if nr_not_found > 1 { -- warn!( -- "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0", -- level, nr_not_found, nr_cpus -- ); -- } -- -- // Cache IDs may have holes. Assign consecutive domain IDs to -- // existing cache IDs. -- let mut cache_to_dom = BTreeMap::::new(); -- let mut nr_doms = 0; -- for cache_id in cache_ids.iter() { -- cache_to_dom.insert(*cache_id, nr_doms); -- nr_doms += 1; -- } -- -- if nr_doms > atropos_sys::MAX_DOMS { -- bail!( -- "Total number of doms {} is greater than MAX_DOMS ({})", -- nr_doms, -- atropos_sys::MAX_DOMS -- ); -- } -- -- // Build and return dom -> cpumask and cpu -> dom mappings. -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize]; -- let mut cpu_to_dom = vec![]; -- -- for cpu in 0..nr_cpus { -- let dom_id = cache_to_dom[&cpu_to_cache[cpu]]; -- cpusets[dom_id as usize].set(cpu, true); -- cpu_to_dom.push(dom_id as i32); -- } -- -- Ok((cpusets, cpu_to_dom)) -- } -- -- fn init(opts: &Opts) -> Result { -- // Open the BPF prog first for verification. -- let mut skel_builder = AtroposSkelBuilder::default(); -- skel_builder.obj_builder.debug(opts.verbose > 0); -- let mut skel = skel_builder.open().context("Failed to open BPF program")?; -- -- let nr_cpus = libbpf_rs::num_possible_cpus().unwrap(); -- if nr_cpus > atropos_sys::MAX_CPUS as usize { -- bail!( -- "nr_cpus ({}) is greater than MAX_CPUS ({})", -- nr_cpus, -- atropos_sys::MAX_CPUS -- ); -- } -- -- // Initialize skel according to @opts. -- let (cpusets, cpus) = if opts.cpumasks.len() > 0 { -- Self::parse_cpusets(&opts.cpumasks, nr_cpus)? -- } else { -- Self::cpusets_from_cache(opts.cache_level, nr_cpus)? -- }; -- let nr_doms = cpusets.len(); -- skel.rodata().nr_doms = nr_doms as u32; -- skel.rodata().nr_cpus = nr_cpus as u32; -- -- for (cpu, dom) in cpus.iter().enumerate() { -- skel.rodata().cpu_dom_id_map[cpu] = *dom as u32; -- } -- -- for (dom, cpuset) in cpusets.iter().enumerate() { -- let raw_cpuset_slice = cpuset.as_raw_slice(); -- let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom]; -- let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len()); -- left.clone_from_slice(cpuset.as_raw_slice()); -- let cpumask_str = dom_cpumask_slice -- .iter() -- .take((nr_cpus + 63) / 64) -- .rev() -- .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x)); -- info!( -- "DOM[{:02}] cpumask{} ({} cpus)", -- dom, -- &cpumask_str, -- cpuset.count_ones() -- ); -- } -- -- skel.rodata().slice_us = opts.slice_us; -- skel.rodata().kthreads_local = opts.kthreads_local; -- skel.rodata().fifo_sched = opts.fifo_sched; -- skel.rodata().switch_partial = opts.partial; -- skel.rodata().greedy_threshold = opts.greedy_threshold; -- -- // Attach. -- let mut skel = skel.load().context("Failed to load BPF program")?; -- skel.attach().context("Failed to attach BPF program")?; -- let struct_ops = Some( -- skel.maps_mut() -- .atropos() -- .attach_struct_ops() -- .context("Failed to attach atropos struct ops")?, -- ); -- info!("Atropos Scheduler Attached"); -- -- // Other stuff. -- let mut proc_reader = procfs::ProcReader::new(); -- let prev_total_cpu = read_total_cpu(&mut proc_reader)?; -- -- Ok(Self { -- skel, -- struct_ops, // should be held to keep it attached -- -- nr_cpus, -- nr_doms, -- load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99), -- balance_load: !opts.no_load_balance, -- -- proc_reader, -- -- prev_at: SystemTime::now(), -- prev_total_cpu, -- task_loads: BTreeMap::new(), -- -- nr_lb_data_errors: 0, -- }) -- } -- -- fn get_cpu_busy(&mut self) -> Result { -- let total_cpu = read_total_cpu(&mut self.proc_reader)?; -- let busy = match (&self.prev_total_cpu, &total_cpu) { -- ( -- procfs::CpuStat { -- user_usec: Some(prev_user), -- nice_usec: Some(prev_nice), -- system_usec: Some(prev_system), -- idle_usec: Some(prev_idle), -- iowait_usec: Some(prev_iowait), -- irq_usec: Some(prev_irq), -- softirq_usec: Some(prev_softirq), -- stolen_usec: Some(prev_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- procfs::CpuStat { -- user_usec: Some(curr_user), -- nice_usec: Some(curr_nice), -- system_usec: Some(curr_system), -- idle_usec: Some(curr_idle), -- iowait_usec: Some(curr_iowait), -- irq_usec: Some(curr_irq), -- softirq_usec: Some(curr_softirq), -- stolen_usec: Some(curr_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- ) => { -- let idle_usec = curr_idle - prev_idle; -- let iowait_usec = curr_iowait - prev_iowait; -- let user_usec = curr_user - prev_user; -- let system_usec = curr_system - prev_system; -- let nice_usec = curr_nice - prev_nice; -- let irq_usec = curr_irq - prev_irq; -- let softirq_usec = curr_softirq - prev_softirq; -- let stolen_usec = curr_stolen - prev_stolen; -- -- let busy_usec = -- user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec; -- let total_usec = idle_usec + busy_usec + iowait_usec; -- busy_usec as f64 / total_usec as f64 -- } -- _ => { -- bail!("Some procfs stats are not populated!"); -- } -- }; -- -- self.prev_total_cpu = total_cpu; -- Ok(busy) -- } -- -- fn read_bpf_stats(&mut self) -> Result> { -- let mut maps = self.skel.maps_mut(); -- let stats_map = maps.stats(); -- let mut stats: Vec = Vec::new(); -- let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus]; -- -- for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS { -- let cpu_stat_vec = stats_map -- .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY) -- .with_context(|| format!("Failed to lookup stat {}", stat))? -- .expect("per-cpu stat should exist"); -- let sum = cpu_stat_vec -- .iter() -- .map(|val| { -- u64::from_ne_bytes( -- val.as_slice() -- .try_into() -- .expect("Invalid value length in stat map"), -- ) -- }) -- .sum(); -- stats_map -- .update_percpu( -- &(stat as u32).to_ne_bytes(), -- &zero_vec, -- libbpf_rs::MapFlags::ANY, -- ) -- .context("Failed to zero stat")?; -- stats.push(sum); -- } -- Ok(stats) -- } -- -- fn report( -- &self, -- stats: &Vec, -- cpu_busy: f64, -- processing_dur: Duration, -- load_avg: f64, -- dom_loads: &Vec, -- imbal: &Vec, -- ) { -- let stat = |idx| stats[idx as usize]; -- let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK); -- -- info!( -- "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms", -- cpu_busy * 100.0, -- load_avg, -- stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize], -- stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize], -- self.nr_lb_data_errors, -- processing_dur.as_millis(), -- ); -- -- let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0; -- -- info!( -- "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}", -- total, -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY), -- ); -- -- for i in 0..self.nr_doms { -- info!( -- "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}", -- i, -- dom_loads[i], -- if imbal[i] < 0.0 { -imbal[i] } else { 0.0 }, -- if imbal[i] > 0.0 { imbal[i] } else { 0.0 }, -- ); -- } -- } -- -- fn step(&mut self) -> Result<()> { -- let started_at = std::time::SystemTime::now(); -- let bpf_stats = self.read_bpf_stats()?; -- let cpu_busy = self.get_cpu_busy()?; -- -- let mut lb = LoadBalancer::new( -- self.skel.maps_mut(), -- &mut self.task_loads, -- self.nr_doms, -- self.load_decay_factor, -- &mut self.nr_lb_data_errors, -- ); -- -- lb.read_task_loads(started_at.duration_since(self.prev_at)?)?; -- lb.calculate_dom_load_balance()?; -- -- if self.balance_load { -- lb.load_balance()?; -- } -- -- // Extract fields needed for reporting and drop lb to release -- // mutable borrows. -- let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal); -- -- self.report( -- &bpf_stats, -- cpu_busy, -- std::time::SystemTime::now().duration_since(started_at)?, -- load_avg, -- &dom_loads, -- &imbal, -- ); -- -- self.prev_at = started_at; -- Ok(()) -- } -- -- fn read_bpf_exit_type(&mut self) -> i32 { -- unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) } -- } -- -- fn report_bpf_exit_type(&mut self) -> Result<()> { -- // Report msg if EXT_OPS_EXIT_ERROR. -- match self.read_bpf_exit_type() { -- 0 => Ok(()), -- etype if etype == 2 => { -- let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) }; -- let msg = cstr -- .to_str() -- .context("Failed to convert exit msg to string") -- .unwrap(); -- bail!("BPF exit_type={} msg={}", etype, msg); -- } -- etype => { -- info!("BPF exit_type={}", etype); -- Ok(()) -- } -- } -- } --} -- --impl<'a> Drop for Scheduler<'a> { -- fn drop(&mut self) { -- if let Some(struct_ops) = self.struct_ops.take() { -- drop(struct_ops); -- } -- } --} -- --fn main() -> Result<()> { -- let opts = Opts::parse(); -- -- let llv = match opts.verbose { -- 0 => simplelog::LevelFilter::Info, -- 1 => simplelog::LevelFilter::Debug, -- _ => simplelog::LevelFilter::Trace, -- }; -- let mut lcfg = simplelog::ConfigBuilder::new(); -- lcfg.set_time_level(simplelog::LevelFilter::Error) -- .set_location_level(simplelog::LevelFilter::Off) -- .set_target_level(simplelog::LevelFilter::Off) -- .set_thread_level(simplelog::LevelFilter::Off); -- simplelog::TermLogger::init( -- llv, -- lcfg.build(), -- simplelog::TerminalMode::Stderr, -- simplelog::ColorChoice::Auto, -- )?; -- -- let shutdown = Arc::new(AtomicBool::new(false)); -- let shutdown_clone = shutdown.clone(); -- ctrlc::set_handler(move || { -- shutdown_clone.store(true, Ordering::Relaxed); -- }) -- .context("Error setting Ctrl-C handler")?; -- -- let mut sched = Scheduler::init(&opts)?; -- -- while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 { -- std::thread::sleep(Duration::from_secs_f64(opts.interval)); -- sched.step()?; -- } -- -- sched.report_bpf_exit_type() --} -diff --git b/tools/sched_ext/gnu/stubs.h a/tools/sched_ext/gnu/stubs.h -deleted file mode 100644 -index 719225b16626..000000000000 ---- b/tools/sched_ext/gnu/stubs.h -+++ /dev/null -@@ -1 +0,0 @@ --/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ -diff --git b/tools/sched_ext/scx_common.bpf.h a/tools/sched_ext/scx_common.bpf.h -deleted file mode 100644 -index e56de9dc86f2..000000000000 ---- b/tools/sched_ext/scx_common.bpf.h -+++ /dev/null -@@ -1,288 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __SCHED_EXT_COMMON_BPF_H --#define __SCHED_EXT_COMMON_BPF_H -- --#include "vmlinux.h" --#include --#include --#include --#include "user_exit_info.h" -- --#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ --#define PF_EXITING 0x00000004 --#define CLOCK_MONOTONIC 1 -- --/* -- * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can -- * lead to really confusing misbehaviors. Let's trigger a build failure. -- */ --static inline void ___vmlinux_h_sanity_check___(void) --{ -- _Static_assert(SCX_DSQ_FLAG_BUILTIN, -- "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); --} -- --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; -- --static inline __attribute__((format(printf, 1, 2))) --void ___scx_bpf_error_format_checker(const char *fmt, ...) {} -- --/* -- * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments -- * instead of an array of u64. Note that __param[] must have at least one -- * element to keep the verifier happy. -- */ --#define scx_bpf_error(fmt, args...) \ --({ \ -- static char ___fmt[] = fmt; \ -- unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ -- \ -- _Pragma("GCC diagnostic push") \ -- _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ -- ___bpf_fill(___param, args); \ -- _Pragma("GCC diagnostic pop") \ -- \ -- scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ -- \ -- ___scx_bpf_error_format_checker(fmt, ##args); \ --}) -- --void scx_bpf_switch_all(void) __ksym; --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; --bool scx_bpf_consume(u64 dsq_id) __ksym; --u32 scx_bpf_dispatch_nr_slots(void) __ksym; --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym; --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym; --void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; --s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym; --const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; --const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; --void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; --void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; --bool scx_bpf_task_running(const struct task_struct *p) __ksym; --s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym; --u32 scx_bpf_reenqueue_local(void) __ksym; -- --#define BPF_STRUCT_OPS(name, args...) \ --SEC("struct_ops/"#name) \ --BPF_PROG(name, ##args) -- --#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ --SEC("struct_ops.s/"#name) \ --BPF_PROG(name, ##args) -- --/** -- * MEMBER_VPTR - Obtain the verified pointer to a struct or array member -- * @base: struct or array to index -- * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...) -- * -- * The verifier often gets confused by the instruction sequence the compiler -- * generates for indexing struct fields or arrays. This macro forces the -- * compiler to generate a code sequence which first calculates the byte offset, -- * checks it against the struct or array size and add that byte offset to -- * generate the pointer to the member to help the verifier. -- * -- * Ideally, we want to abort if the calculated offset is out-of-bounds. However, -- * BPF currently doesn't support abort, so evaluate to NULL instead. The caller -- * must check for NULL and take appropriate action to appease the verifier. To -- * avoid confusing the verifier, it's best to check for NULL and dereference -- * immediately. -- * -- * vptr = MEMBER_VPTR(my_array, [i][j]); -- * if (!vptr) -- * return error; -- * *vptr = new_value; -- */ --#define MEMBER_VPTR(base, member) (typeof(base member) *)({ \ -- u64 __base = (u64)base; \ -- u64 __addr = (u64)&(base member) - __base; \ -- asm volatile ( \ -- "if %0 <= %[max] goto +2\n" \ -- "%0 = 0\n" \ -- "goto +1\n" \ -- "%0 += %1\n" \ -- : "+r"(__addr) \ -- : "r"(__base), \ -- [max]"i"(sizeof(base) - sizeof(base member))); \ -- __addr; \ --}) -- --/* -- * BPF core and other generic helpers -- */ -- --/* list and rbtree */ --#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) --#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) -- --void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; --void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; -- --#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) --#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) -- --void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; --struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; --struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, -- struct bpf_rb_node *node) __ksym; --void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, -- bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym; --struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; -- --/* task */ --struct task_struct *bpf_task_from_pid(s32 pid) __ksym; --struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; --void bpf_task_release(struct task_struct *p) __ksym; -- --/* cgroup */ --struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; --void bpf_cgroup_release(struct cgroup *cgrp) __ksym; --struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; -- --/* cpumask */ --struct bpf_cpumask *bpf_cpumask_create(void) __ksym; --struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; --u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; --u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym; -- --/* rcu */ --void bpf_rcu_read_lock(void) __ksym; --void bpf_rcu_read_unlock(void) __ksym; -- --/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */ --struct bpf_iter_num; -- --extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym; --extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym; --extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym; -- --#ifndef bpf_for_each --/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for -- * using BPF open-coded iterators without having to write mundane explicit -- * low-level loop logic. Instead, it provides for()-like generic construct -- * that can be used pretty naturally. E.g., for some hypothetical cgroup -- * iterator, you'd write: -- * -- * struct cgroup *cg, *parent_cg = <...>; -- * -- * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { -- * bpf_printk("Child cgroup id = %d", cg->cgroup_id); -- * if (cg->cgroup_id == 123) -- * break; -- * } -- * -- * I.e., it looks almost like high-level for each loop in other languages, -- * supports continue/break, and is verifiable by BPF verifier. -- * -- * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) -- * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to -- * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int -- * *`, not just `int`. So for integers bpf_for() is more convenient. -- * -- * Note: this macro relies on C99 feature of allowing to declare variables -- * inside for() loop, bound to for() loop lifetime. It also utilizes GCC -- * extension: __attribute__((cleanup())), supported by both GCC and -- * Clang. -- */ --#define bpf_for_each(type, cur, args...) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ -- cleanup(bpf_iter_##type##_destroy))), \ -- /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_##type##_new(&___it, ##args), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_##type##_destroy, (void *)0); \ -- /* iteration and termination check */ \ -- (((cur) = bpf_iter_##type##_next(&___it))); \ --) --#endif /* bpf_for_each */ -- --#ifndef bpf_for --/* bpf_for(i, start, end) implements a for()-like looping construct that sets -- * provided integer variable *i* to values starting from *start* through, -- * but not including, *end*. It also proves to BPF verifier that *i* belongs -- * to range [start, end), so this can be used for accessing arrays without -- * extra checks. -- * -- * Note: *start* and *end* are assumed to be expressions with no side effects -- * and whose values do not change throughout bpf_for() loop execution. They do -- * not have to be statically known or constant, though. -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_for(i, start, end) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, (start), (end)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- ({ \ -- /* iteration step */ \ -- int *___t = bpf_iter_num_next(&___it); \ -- /* termination and bounds check */ \ -- (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ -- }); \ --) --#endif /* bpf_for */ -- --#ifndef bpf_repeat --/* bpf_repeat(N) performs N iterations without exposing iteration number -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_repeat(N) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, 0, (N)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- bpf_iter_num_next(&___it); \ -- /* nothing here */ \ --) --#endif /* bpf_repeat */ -- --#endif /* __SCHED_EXT_COMMON_BPF_H */ -diff --git b/tools/sched_ext/scx_example_central.bpf.c a/tools/sched_ext/scx_example_central.bpf.c -deleted file mode 100644 -index 4cec04b4c2ed..000000000000 ---- b/tools/sched_ext/scx_example_central.bpf.c -+++ /dev/null -@@ -1,334 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A central FIFO sched_ext scheduler which demonstrates the followings: -- * -- * a. Making all scheduling decisions from one CPU: -- * -- * The central CPU is the only one making scheduling decisions. All other -- * CPUs kick the central CPU when they run out of tasks to run. -- * -- * There is one global BPF queue and the central CPU schedules all CPUs by -- * dispatching from the global queue to each CPU's local dsq from dispatch(). -- * This isn't the most straightforward. e.g. It'd be easier to bounce -- * through per-CPU BPF queues. The current design is chosen to maximally -- * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching. -- * -- * b. Tickless operation -- * -- * All tasks are dispatched with the infinite slice which allows stopping the -- * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full -- * parameter. The tickless operation can be observed through -- * /proc/interrupts. -- * -- * Periodic switching is enforced by a periodic timer checking all CPUs and -- * preempting them as necessary. Unfortunately, BPF timer currently doesn't -- * have a way to pin to a specific CPU, so the periodic timer isn't pinned to -- * the central CPU. -- * -- * c. Preemption -- * -- * Kthreads are unconditionally queued to the head of a matching local dsq -- * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always -- * prioritized over user threads, which is required for ensuring forward -- * progress as e.g. the periodic timer may run on a ksoftirqd and if the -- * ksoftirqd gets starved by a user thread, there may not be anything else to -- * vacate that user thread. -- * -- * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the -- * next tasks. -- * -- * This scheduler is designed to maximize usage of various SCX mechanisms. A -- * more practical implementation would likely put the scheduling loop outside -- * the central CPU's dispatch() path and add some form of priority mechanism. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --enum { -- FALLBACK_DSQ_ID = 0, -- MAX_CPUS = 4096, -- MS_TO_NS = 1000LLU * 1000, -- TIMER_INTERVAL_NS = 1 * MS_TO_NS, --}; -- --const volatile bool switch_partial; --const volatile s32 central_cpu; --const volatile u32 nr_cpu_ids = 64; /* !0 for veristat, set during init */ -- --u64 nr_total, nr_locals, nr_queued, nr_lost_pids; --u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries; --u64 nr_overflows; -- --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, s32); --} central_q SEC(".maps"); -- --/* can't use percpu map due to bad lookups */ --static bool cpu_gimme_task[MAX_CPUS]; --static u64 cpu_started_at[MAX_CPUS]; -- --struct central_timer { -- struct bpf_timer timer; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, 1); -- __type(key, u32); -- __type(value, struct central_timer); --} central_timer SEC(".maps"); -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- /* -- * Steer wakeups to the central CPU as much as possible to avoid -- * disturbing other CPUs. It's safe to blindly return the central cpu as -- * select_cpu() is a hint and if @p can't be on it, the kernel will -- * automatically pick a fallback CPU. -- */ -- return central_cpu; --} -- --void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags) --{ -- s32 pid = p->pid; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- /* -- * Push per-cpu kthreads at the head of local dsq's and preempt the -- * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked -- * behind other threads which is necessary for forward progress -- * guarantee as we depend on the BPF timer which may run from ksoftirqd. -- */ -- if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- __sync_fetch_and_add(&nr_locals, 1); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF, -- enq_flags | SCX_ENQ_PREEMPT); -- return; -- } -- -- if (bpf_map_push_elem(¢ral_q, &pid, 0)) { -- __sync_fetch_and_add(&nr_overflows, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_queued, 1); -- -- if (!scx_bpf_task_running(p)) -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); --} -- --static bool dispatch_to_cpu(s32 cpu) --{ -- struct task_struct *p; -- s32 pid; -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (bpf_map_pop_elem(¢ral_q, &pid)) -- break; -- -- __sync_fetch_and_sub(&nr_queued, 1); -- -- p = bpf_task_from_pid(pid); -- if (!p) { -- __sync_fetch_and_add(&nr_lost_pids, 1); -- continue; -- } -- -- /* -- * If we can't run the task at the top, do the dumb thing and -- * bounce it to the fallback dsq. -- */ -- if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- __sync_fetch_and_add(&nr_mismatches, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0); -- bpf_task_release(p); -- continue; -- } -- -- /* dispatch to local and mark that @cpu doesn't need more */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0); -- -- if (cpu != central_cpu) -- scx_bpf_kick_cpu(cpu, 0); -- -- bpf_task_release(p); -- return true; -- } -- -- return false; --} -- --void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (cpu == central_cpu) { -- /* dispatch for all other CPUs first */ -- __sync_fetch_and_add(&nr_dispatches, 1); -- -- bpf_for(cpu, 0, nr_cpu_ids) { -- bool *gimme; -- -- if (!scx_bpf_dispatch_nr_slots()) -- break; -- -- /* central's gimme is never set */ -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme && !*gimme) -- continue; -- -- if (dispatch_to_cpu(cpu)) -- *gimme = false; -- } -- -- /* -- * Retry if we ran out of dispatch buffer slots as we might have -- * skipped some CPUs and also need to dispatch for self. The ext -- * core automatically retries if the local dsq is empty but we -- * can't rely on that as we're dispatching for other CPUs too. -- * Kick self explicitly to retry. -- */ -- if (!scx_bpf_dispatch_nr_slots()) { -- __sync_fetch_and_add(&nr_retries, 1); -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- return; -- } -- -- /* look for a task to run on the central CPU */ -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- dispatch_to_cpu(central_cpu); -- } else { -- bool *gimme; -- -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme) -- *gimme = true; -- -- /* -- * Force dispatch on the scheduling CPU so that it finds a task -- * to run for us. -- */ -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- } --} -- --void BPF_STRUCT_OPS(central_running, struct task_struct *p) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ --} -- --void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = 0; --} -- --static int central_timerfn(void *map, int *key, struct bpf_timer *timer) --{ -- u64 now = bpf_ktime_get_ns(); -- u64 nr_to_kick = nr_queued; -- s32 i; -- -- bpf_for(i, 0, nr_cpu_ids) { -- s32 cpu = (nr_timers + i) % nr_cpu_ids; -- u64 *started_at; -- -- if (cpu == central_cpu) -- continue; -- -- /* kick iff the current one exhausted its slice */ -- started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at && *started_at && -- vtime_before(now, *started_at + SCX_SLICE_DFL)) -- continue; -- -- /* and there's something pending */ -- if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) || -- scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu)) -- ; -- else if (nr_to_kick) -- nr_to_kick--; -- else -- continue; -- -- scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); -- } -- -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- -- bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- __sync_fetch_and_add(&nr_timers, 1); -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(central_init) --{ -- u32 key = 0; -- struct bpf_timer *timer; -- int ret; -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1); -- if (ret) -- return ret; -- -- timer = bpf_map_lookup_elem(¢ral_timer, &key); -- if (!timer) -- return -ESRCH; -- -- bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); -- bpf_timer_set_callback(timer, central_timerfn); -- ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- return ret; --} -- --void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops central_ops = { -- /* -- * We are offloading all scheduling decisions to the central CPU and -- * thus being the last task on a given CPU doesn't mean anything -- * special. Enqueue the last tasks like any other tasks. -- */ -- .flags = SCX_OPS_ENQ_LAST, -- -- .select_cpu = (void *)central_select_cpu, -- .enqueue = (void *)central_enqueue, -- .dispatch = (void *)central_dispatch, -- .running = (void *)central_running, -- .stopping = (void *)central_stopping, -- .init = (void *)central_init, -- .exit = (void *)central_exit, -- .name = "central", --}; -diff --git b/tools/sched_ext/scx_example_central.c a/tools/sched_ext/scx_example_central.c -deleted file mode 100644 -index 7ad591cbdc65..000000000000 ---- b/tools/sched_ext/scx_example_central.c -+++ /dev/null -@@ -1,94 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_central.skel.h" -- --const char help_fmt[] = --"A central FIFO sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-c CPU] [-p]\n" --"\n" --" -c CPU Override the central CPU (default: 0)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_central *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_central__open(); -- assert(skel); -- -- skel->rodata->central_cpu = 0; -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "c:ph")) != -1) { -- switch (opt) { -- case 'c': -- skel->rodata->central_cpu = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_central__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.central_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf("total :%10lu local:%10lu queued:%10lu lost:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_locals, -- skel->bss->nr_queued, -- skel->bss->nr_lost_pids); -- printf("timer :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n", -- skel->bss->nr_timers, -- skel->bss->nr_dispatches, -- skel->bss->nr_mismatches, -- skel->bss->nr_retries); -- printf("overflow:%10lu\n", -- skel->bss->nr_overflows); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_central__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_flatcg.bpf.c a/tools/sched_ext/scx_example_flatcg.bpf.c -deleted file mode 100644 -index f6078b9a681f..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.bpf.c -+++ /dev/null -@@ -1,872 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext flattened cgroup hierarchy scheduler. It implements -- * hierarchical weight-based cgroup CPU control by flattening the cgroup -- * hierarchy into a single layer by compounding the active weight share at each -- * level. Consider the following hierarchy with weights in parentheses: -- * -- * R + A (100) + B (100) -- * | \ C (100) -- * \ D (200) -- * -- * Ignoring the root and threaded cgroups, only B, C and D can contain tasks. -- * Let's say all three have runnable tasks. The total share that each of these -- * three cgroups is entitled to can be calculated by compounding its share at -- * each level. -- * -- * For example, B is competing against C and in that competition its share is -- * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's -- * share in that competition is 200/(200+100) == 1/3. B's eventual share in the -- * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's -- * eventual shaer is the same at 1/6. D is only competing at the top level and -- * its share is 200/(100+200) == 2/3. -- * -- * So, instead of hierarchically scheduling level-by-level, we can consider it -- * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3 -- * and keep updating the eventual shares as the cgroups' runnable states change. -- * -- * This flattening of hierarchy can bring a substantial performance gain when -- * the cgroup hierarchy is nested multiple levels. in a simple benchmark using -- * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it -- * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two -- * apache instances competing with 2:1 weight ratio nested four level deep. -- * -- * However, the gain comes at the cost of not being able to properly handle -- * thundering herd of cgroups. For example, if many cgroups which are nested -- * behind a low priority parent cgroup wake up around the same time, they may be -- * able to consume more CPU cycles than they are entitled to. In many use cases, -- * this isn't a real concern especially given the performance gain. Also, there -- * are ways to mitigate the problem further by e.g. introducing an extra -- * scheduling layer on cgroup delegation boundaries. -- * -- * The scheduler first picks the cgroup to run and then schedule the tasks -- * within by using nested weighted vtime scheduling by default. The -- * cgroup-internal scheduling can be switched to FIFO with the -f option. -- */ --#include "scx_common.bpf.h" --#include "user_exit_info.h" --#include "scx_example_flatcg.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */ --const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL; --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --u64 cvtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, u64); -- __uint(max_entries, FCG_NR_STATS); --} stats SEC(".maps"); -- --static void stat_inc(enum fcg_stat_idx idx) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p)++; --} -- --struct fcg_cpu_ctx { -- u64 cur_cgid; -- u64 cur_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, struct fcg_cpu_ctx); -- __uint(max_entries, 1); --} cpu_ctx SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_cgrp_ctx); --} cgrp_ctx SEC(".maps"); -- --struct cgv_node { -- struct bpf_rb_node rb_node; -- __u64 cvtime; -- __u64 cgid; --}; -- --private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock; --private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node); -- --struct cgv_node_stash { -- struct cgv_node __kptr *node; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, 16384); -- __type(key, __u64); -- __type(value, struct cgv_node_stash); --} cgv_node_stash SEC(".maps"); -- --struct fcg_task_ctx { -- u64 bypassed_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_task_ctx); --} task_ctx SEC(".maps"); -- --/* gets inc'd on weight tree changes to expire the cached hweights */ --unsigned long hweight_gen = 1; -- --static u64 div_round_up(u64 dividend, u64 divisor) --{ -- return (dividend + divisor - 1) / divisor; --} -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) --{ -- struct cgv_node *cgc_a, *cgc_b; -- -- cgc_a = container_of(a, struct cgv_node, rb_node); -- cgc_b = container_of(b, struct cgv_node, rb_node); -- -- return cgc_a->cvtime < cgc_b->cvtime; --} -- --static struct fcg_cpu_ctx *find_cpu_ctx(void) --{ -- struct fcg_cpu_ctx *cpuc; -- u32 idx = 0; -- -- cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx); -- if (!cpuc) { -- scx_bpf_error("cpu_ctx lookup failed"); -- return NULL; -- } -- return cpuc; --} -- --static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id); -- return NULL; -- } -- return cgc; --} -- --static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgrp = bpf_cgroup_ancestor(cgrp, level); -- if (!cgrp) { -- scx_bpf_error("ancestor cgroup lookup failed"); -- return NULL; -- } -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- scx_bpf_error("ancestor cgrp_ctx lookup failed"); -- bpf_cgroup_release(cgrp); -- return cgc; --} -- --static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- int level; -- -- if (!cgc->nr_active) { -- stat_inc(FCG_STAT_HWT_SKIP); -- return; -- } -- -- if (cgc->hweight_gen == hweight_gen) { -- stat_inc(FCG_STAT_HWT_CACHE); -- return; -- } -- -- stat_inc(FCG_STAT_HWT_UPDATES); -- bpf_for(level, 0, cgrp->level + 1) { -- struct fcg_cgrp_ctx *cgc; -- bool is_active; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- -- if (!level) { -- cgc->hweight = FCG_HWEIGHT_ONE; -- cgc->hweight_gen = hweight_gen; -- } else { -- struct fcg_cgrp_ctx *pcgc; -- -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- -- /* -- * We can be oppotunistic here and not grab the -- * cgv_tree_lock and deal with the occasional races. -- * However, hweight updates are already cached and -- * relatively low-frequency. Let's just do the -- * straightforward thing. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- is_active = cgc->nr_active; -- if (is_active) { -- cgc->hweight_gen = pcgc->hweight_gen; -- cgc->hweight = -- div_round_up(pcgc->hweight * cgc->weight, -- pcgc->child_weight_sum); -- } -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!is_active) { -- stat_inc(FCG_STAT_HWT_RACE); -- break; -- } -- } -- } --} -- --static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) --{ -- u64 delta, cvtime, max_budget; -- -- /* -- * A node which is on the rbtree can't be pointed to from elsewhere yet -- * and thus can't be updated and repositioned. Instead, we collect the -- * vtime deltas separately and apply it asynchronously here. -- */ -- delta = cgc->cvtime_delta; -- __sync_fetch_and_sub(&cgc->cvtime_delta, delta); -- cvtime = cgv_node->cvtime + delta; -- -- /* -- * Allow a cgroup to carry the maximum budget proportional to its -- * hweight such that a full-hweight cgroup can immediately take up half -- * of the CPUs at the most while staying at the front of the rbtree. -- */ -- max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / -- (2 * FCG_HWEIGHT_ONE); -- if (vtime_before(cvtime, cvtime_now - max_budget)) -- cvtime = cvtime_now - max_budget; -- -- cgv_node->cvtime = cvtime; --} -- --static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- u64 cgid = cgrp->kn->id; -- -- /* paired with cmpxchg in try_pick_next_cgroup() */ -- if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { -- stat_inc(FCG_STAT_ENQ_SKIP); -- return; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid); -- return; -- } -- -- /* NULL if the node is already on the rbtree */ -- cgv_node = bpf_kptr_xchg(&stash->node, NULL); -- if (!cgv_node) { -- stat_inc(FCG_STAT_ENQ_RACE); -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); --} -- --void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * If select_cpu_dfl() is recommending local enqueue, the target CPU is -- * idle. Follow it and charge the cgroup later in fcg_stopping() after -- * the fact. Use the same mechanism to deal with tasks with custom -- * affinities so that we don't have to worry about per-cgroup dq's -- * containing tasks that can't be executed from some CPUs. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) { -- /* -- * Tell fcg_stopping() that this bypassed the regular scheduling -- * path and should be force charged to the cgroup. 0 is used to -- * indicate that the task isn't bypassing, so if the current -- * runtime is 0, go back by one nanosecond. -- */ -- taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1; -- -- /* -- * The global dq is deprioritized as we don't want to let tasks -- * to boost themselves by constraining its cpumask. The -- * deprioritization is rather severe, so let's not apply that to -- * per-cpu kernel threads. This is ham-fisted. We probably wanna -- * implement per-cgroup fallback dq's instead so that we have -- * more control over when tasks with custom cpumask get issued. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || -- (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) { -- stat_inc(FCG_STAT_LOCAL); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- } else { -- stat_inc(FCG_STAT_GLOBAL); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } -- return; -- } -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- goto out_release; -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 tvtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) -- tvtime = cgc->tvtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, -- tvtime, enq_flags); -- } -- -- cgrp_enqueued(cgrp, cgc); --out_release: -- bpf_cgroup_release(cgrp); --} -- --/* -- * Walk the cgroup tree to update the active weight sums as tasks wake up and -- * sleep. The weight sums are used as the base when calculating the proportion a -- * given cgroup or task is entitled to at each level. -- */ --static void update_active_weight_sums(struct cgroup *cgrp, bool runnable) --{ -- struct fcg_cgrp_ctx *cgc; -- bool updated = false; -- int idx; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- /* -- * In most cases, a hot cgroup would have multiple threads going to -- * sleep and waking up while the whole cgroup stays active. In leaf -- * cgroups, ->nr_runnable which is updated with __sync operations gates -- * ->nr_active updates, so that we don't have to grab the cgv_tree_lock -- * repeatedly for a busy cgroup which is staying active. -- */ -- if (runnable) { -- if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_ACT); -- } else { -- if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_DEACT); -- } -- -- /* -- * If @cgrp is becoming runnable, its hweight should be refreshed after -- * it's added to the weight tree so that enqueue has the up-to-date -- * value. If @cgrp is becoming quiescent, the hweight should be -- * refreshed before it's removed from the weight tree so that the usage -- * charging which happens afterwards has access to the latest value. -- */ -- if (!runnable) -- cgrp_refresh_hweight(cgrp, cgc); -- -- /* propagate upwards */ -- bpf_for(idx, 0, cgrp->level) { -- int level = cgrp->level - idx; -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- bool propagate = false; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- if (level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- } -- -- /* -- * We need the propagation protected by a lock to synchronize -- * against weight changes. There's no reason to drop the lock at -- * each level but bpf_spin_lock() doesn't want any function -- * calls while locked. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- -- if (runnable) { -- if (!cgc->nr_active++) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum += cgc->weight; -- } -- } -- } else { -- if (!--cgc->nr_active) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum -= cgc->weight; -- } -- } -- } -- -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!propagate) -- break; -- } -- -- if (updated) -- __sync_fetch_and_add(&hweight_gen, 1); -- -- if (runnable) -- cgrp_refresh_hweight(cgrp, cgc); --} -- --void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, true); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) --{ -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- if (fifo_sched) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- /* -- * @cgc->tvtime_now always progresses forward as tasks start -- * executing. The test and update can be performed concurrently -- * from multiple CPUs and thus racy. Any error should be -- * contained and temporary. Let's just live with it. -- */ -- if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) -- cgc->tvtime_now = p->scx.dsq_vtime; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- /* scale the execution time by the inverse of the weight and charge */ -- if (!fifo_sched) -- p->scx.dsq_vtime += -- (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- if (!taskc->bypassed_at) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- __sync_fetch_and_add(&cgc->cvtime_delta, -- p->se.sum_exec_runtime - taskc->bypassed_at); -- taskc->bypassed_at = 0; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, false); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) --{ -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- if (cgrp->level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1); -- if (!pcgc) -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- if (pcgc && cgc->nr_active) -- pcgc->child_weight_sum += (s64)weight - cgc->weight; -- cgc->weight = weight; -- bpf_spin_unlock(&cgv_tree_lock); --} -- --static bool try_pick_next_cgroup(u64 *cgidp) --{ -- struct bpf_rb_node *rb_node; -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 cgid; -- -- /* pop the front cgroup and wind cvtime_now accordingly */ -- bpf_spin_lock(&cgv_tree_lock); -- -- rb_node = bpf_rbtree_first(&cgv_tree); -- if (!rb_node) { -- bpf_spin_unlock(&cgv_tree_lock); -- stat_inc(FCG_STAT_PNC_NO_CGRP); -- *cgidp = 0; -- return true; -- } -- -- rb_node = bpf_rbtree_remove(&cgv_tree, rb_node); -- bpf_spin_unlock(&cgv_tree_lock); -- -- cgv_node = container_of(rb_node, struct cgv_node, rb_node); -- cgid = cgv_node->cgid; -- -- if (vtime_before(cvtime_now, cgv_node->cvtime)) -- cvtime_now = cgv_node->cvtime; -- -- /* -- * If lookup fails, the cgroup's gone. Free and move on. See -- * fcg_cgroup_exit(). -- */ -- cgrp = bpf_cgroup_from_id(cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- if (!scx_bpf_consume(cgid)) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_EMPTY); -- goto out_stash; -- } -- -- /* -- * Successfully consumed from the cgroup. This will be our current -- * cgroup for the new slice. Refresh its hweight. -- */ -- cgrp_refresh_hweight(cgrp, cgc); -- -- bpf_cgroup_release(cgrp); -- -- /* -- * As the cgroup may have more tasks, add it back to the rbtree. Note -- * that here we charge the full slice upfront and then exact later -- * according to the actual consumption. This prevents lowpri thundering -- * herd from saturating the machine. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- -- *cgidp = cgid; -- stat_inc(FCG_STAT_PNC_NEXT); -- return true; -- --out_stash: -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- /* -- * Paired with cmpxchg in cgrp_enqueued(). If they see the following -- * transition, they'll enqueue the cgroup. If they are earlier, we'll -- * see their task in the dq below and requeue the cgroup. -- */ -- __sync_val_compare_and_swap(&cgc->queued, 1, 0); -- -- if (scx_bpf_dsq_nr_queued(cgid)) { -- bpf_spin_lock(&cgv_tree_lock); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- goto out_free; -- } -- } -- -- return false; -- --out_free: -- bpf_obj_drop(cgv_node); -- return false; --} -- --void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) --{ -- struct fcg_cpu_ctx *cpuc; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 now = bpf_ktime_get_ns(); -- -- cpuc = find_cpu_ctx(); -- if (!cpuc) -- return; -- -- if (!cpuc->cur_cgid) -- goto pick_next_cgroup; -- -- if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { -- if (scx_bpf_consume(cpuc->cur_cgid)) { -- stat_inc(FCG_STAT_CNS_KEEP); -- return; -- } -- stat_inc(FCG_STAT_CNS_EMPTY); -- } else { -- stat_inc(FCG_STAT_CNS_EXPIRE); -- } -- -- /* -- * The current cgroup is expiring. It was already charged a full slice. -- * Calculate the actual usage and accumulate the delta. -- */ -- cgrp = bpf_cgroup_from_id(cpuc->cur_cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_CNS_GONE); -- goto pick_next_cgroup; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (cgc) { -- /* -- * We want to update the vtime delta and then look for the next -- * cgroup to execute but the latter needs to be done in a loop -- * and we can't keep the lock held. Oh well... -- */ -- bpf_spin_lock(&cgv_tree_lock); -- __sync_fetch_and_add(&cgc->cvtime_delta, -- (cpuc->cur_at + cgrp_slice_ns - now) * -- FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- stat_inc(FCG_STAT_CNS_GONE); -- } -- -- bpf_cgroup_release(cgrp); -- --pick_next_cgroup: -- cpuc->cur_at = now; -- -- if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { -- cpuc->cur_cgid = 0; -- return; -- } -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_pick_next_cgroup(&cpuc->cur_cgid)) -- break; -- } --} -- --s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct fcg_task_ctx *taskc; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- taskc = bpf_task_storage_get(&task_ctx, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!taskc) -- return -ENOMEM; -- -- taskc->bypassed_at = 0; -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp, -- struct scx_cgroup_init_args *args) --{ -- struct fcg_cgrp_ctx *cgc; -- struct cgv_node *cgv_node; -- struct cgv_node_stash empty_stash = {}, *stash; -- u64 cgid = cgrp->kn->id; -- int ret; -- -- /* -- * Technically incorrect as cgroup ID is full 64bit while dq ID is -- * 63bit. Should not be a problem in practice and easy to spot in the -- * unlikely case that it breaks. -- */ -- ret = scx_bpf_create_dsq(cgid, -1); -- if (ret) -- return ret; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!cgc) { -- ret = -ENOMEM; -- goto err_destroy_dsq; -- } -- -- cgc->weight = args->weight; -- cgc->hweight = FCG_HWEIGHT_ONE; -- -- ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash, -- BPF_NOEXIST); -- if (ret) { -- if (ret != -ENOMEM) -- scx_bpf_error("unexpected stash creation error (%d)", -- ret); -- goto err_destroy_dsq; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("unexpected cgv_node stash lookup failure"); -- ret = -ENOENT; -- goto err_destroy_dsq; -- } -- -- cgv_node = bpf_obj_new(struct cgv_node); -- if (!cgv_node) { -- ret = -ENOMEM; -- goto err_del_cgv_node; -- } -- -- cgv_node->cgid = cgid; -- cgv_node->cvtime = cvtime_now; -- -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- ret = -EBUSY; -- goto err_drop; -- } -- -- return 0; -- --err_drop: -- bpf_obj_drop(cgv_node); --err_del_cgv_node: -- bpf_map_delete_elem(&cgv_node_stash, &cgid); --err_destroy_dsq: -- scx_bpf_destroy_dsq(cgid); -- return ret; --} -- --void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- -- /* -- * For now, there's no way find and remove the cgv_node if it's on the -- * cgv_tree. Let's drain them in the dispatch path as they get popped -- * off the front of the tree. -- */ -- bpf_map_delete_elem(&cgv_node_stash, &cgid); -- scx_bpf_destroy_dsq(cgid); --} -- --s32 BPF_STRUCT_OPS(fcg_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops flatcg_ops = { -- .enqueue = (void *)fcg_enqueue, -- .dispatch = (void *)fcg_dispatch, -- .runnable = (void *)fcg_runnable, -- .running = (void *)fcg_running, -- .stopping = (void *)fcg_stopping, -- .quiescent = (void *)fcg_quiescent, -- .prep_enable = (void *)fcg_prep_enable, -- .cgroup_set_weight = (void *)fcg_cgroup_set_weight, -- .cgroup_init = (void *)fcg_cgroup_init, -- .cgroup_exit = (void *)fcg_cgroup_exit, -- .init = (void *)fcg_init, -- .exit = (void *)fcg_exit, -- .flags = SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING, -- .name = "flatcg", --}; -diff --git b/tools/sched_ext/scx_example_flatcg.c a/tools/sched_ext/scx_example_flatcg.c -deleted file mode 100644 -index f9c8a5b84a70..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.c -+++ /dev/null -@@ -1,232 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2023 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2023 Tejun Heo -- * Copyright (c) 2023 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_flatcg.h" --#include "scx_example_flatcg.skel.h" -- --#ifndef FILEID_KERNFS --#define FILEID_KERNFS 0xfe --#endif -- --const char help_fmt[] = --"A flattened cgroup hierarchy sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -i INTERVAL Report interval\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --static float read_cpu_util(__u64 *last_sum, __u64 *last_idle) --{ -- FILE *fp; -- char buf[4096]; -- char *line, *cur = NULL, *tok; -- __u64 sum = 0, idle = 0; -- __u64 delta_sum, delta_idle; -- int idx; -- -- fp = fopen("/proc/stat", "r"); -- if (!fp) { -- perror("fopen(\"/proc/stat\")"); -- return 0.0; -- } -- -- if (!fgets(buf, sizeof(buf), fp)) { -- perror("fgets(\"/proc/stat\")"); -- fclose(fp); -- return 0.0; -- } -- fclose(fp); -- -- line = buf; -- for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) { -- char *endp = NULL; -- __u64 v; -- -- if (idx == 0) { -- line = NULL; -- continue; -- } -- v = strtoull(tok, &endp, 0); -- if (!endp || *endp != '\0') { -- fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n", -- idx, tok); -- continue; -- } -- sum += v; -- if (idx == 4) -- idle = v; -- } -- -- delta_sum = sum - *last_sum; -- delta_idle = idle - *last_idle; -- *last_sum = sum; -- *last_idle = idle; -- -- return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0; --} -- --static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats) --{ -- __u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus]; -- __u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); -- -- for (idx = 0; idx < FCG_NR_STATS; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_flatcg *skel; -- struct bpf_link *link; -- struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 }; -- bool dump_cgrps = false; -- __u64 last_cpu_sum = 0, last_cpu_idle = 0; -- __u64 last_stats[FCG_NR_STATS] = {}; -- unsigned long seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_flatcg__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open: %s\n", strerror(errno)); -- return 1; -- } -- -- skel->rodata->nr_cpus = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) { -- double v; -- -- switch (opt) { -- case 's': -- v = strtod(optarg, NULL); -- skel->rodata->cgrp_slice_ns = v * 1000; -- break; -- case 'i': -- v = strtod(optarg, NULL); -- intv_ts.tv_sec = v; -- intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000; -- break; -- case 'd': -- dump_cgrps = true; -- break; -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- case 'h': -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d", -- (double)skel->rodata->cgrp_slice_ns / 1000000.0, -- (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0, -- dump_cgrps); -- -- if (scx_example_flatcg__load(skel)) { -- fprintf(stderr, "Failed to load: %s\n", strerror(errno)); -- return 1; -- } -- -- link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops); -- if (!link) { -- fprintf(stderr, "Failed to attach_struct_ops: %s\n", -- strerror(errno)); -- return 1; -- } -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- __u64 acc_stats[FCG_NR_STATS]; -- __u64 stats[FCG_NR_STATS]; -- float cpu_util; -- int i; -- -- cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle); -- -- fcg_read_stats(skel, acc_stats); -- for (i = 0; i < FCG_NR_STATS; i++) -- stats[i] = acc_stats[i] - last_stats[i]; -- -- memcpy(last_stats, acc_stats, sizeof(acc_stats)); -- -- printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n", -- seq++, cpu_util * 100.0, skel->data->hweight_gen); -- printf(" act:%6llu deact:%6llu local:%6llu global:%6llu\n", -- stats[FCG_STAT_ACT], -- stats[FCG_STAT_DEACT], -- stats[FCG_STAT_LOCAL], -- stats[FCG_STAT_GLOBAL]); -- printf("HWT skip:%6llu race:%6llu cache:%6llu update:%6llu\n", -- stats[FCG_STAT_HWT_SKIP], -- stats[FCG_STAT_HWT_RACE], -- stats[FCG_STAT_HWT_CACHE], -- stats[FCG_STAT_HWT_UPDATES]); -- printf("ENQ skip:%6llu race:%6llu\n", -- stats[FCG_STAT_ENQ_SKIP], -- stats[FCG_STAT_ENQ_RACE]); -- printf("CNS keep:%6llu expire:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_CNS_KEEP], -- stats[FCG_STAT_CNS_EXPIRE], -- stats[FCG_STAT_CNS_EMPTY], -- stats[FCG_STAT_CNS_GONE]); -- printf("PNC nocgrp:%6llu next:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_PNC_NO_CGRP], -- stats[FCG_STAT_PNC_NEXT], -- stats[FCG_STAT_PNC_EMPTY], -- stats[FCG_STAT_PNC_GONE]); -- printf("BAD remove:%6llu\n", -- acc_stats[FCG_STAT_BAD_REMOVAL]); -- -- nanosleep(&intv_ts, NULL); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_flatcg__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_flatcg.h a/tools/sched_ext/scx_example_flatcg.h -deleted file mode 100644 -index 490758ed41f0..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.h -+++ /dev/null -@@ -1,49 +0,0 @@ --#ifndef __SCX_EXAMPLE_FLATCG_H --#define __SCX_EXAMPLE_FLATCG_H -- --enum { -- FCG_HWEIGHT_ONE = 1LLU << 16, --}; -- --enum fcg_stat_idx { -- FCG_STAT_ACT, -- FCG_STAT_DEACT, -- FCG_STAT_LOCAL, -- FCG_STAT_GLOBAL, -- -- FCG_STAT_HWT_UPDATES, -- FCG_STAT_HWT_CACHE, -- FCG_STAT_HWT_SKIP, -- FCG_STAT_HWT_RACE, -- -- FCG_STAT_ENQ_SKIP, -- FCG_STAT_ENQ_RACE, -- -- FCG_STAT_CNS_KEEP, -- FCG_STAT_CNS_EXPIRE, -- FCG_STAT_CNS_EMPTY, -- FCG_STAT_CNS_GONE, -- -- FCG_STAT_PNC_NO_CGRP, -- FCG_STAT_PNC_NEXT, -- FCG_STAT_PNC_EMPTY, -- FCG_STAT_PNC_GONE, -- -- FCG_STAT_BAD_REMOVAL, -- -- FCG_NR_STATS, --}; -- --struct fcg_cgrp_ctx { -- u32 nr_active; -- u32 nr_runnable; -- u32 queued; -- u32 weight; -- u32 hweight; -- u64 child_weight_sum; -- u64 hweight_gen; -- s64 cvtime_delta; -- u64 tvtime_now; --}; -- --#endif /* __SCX_EXAMPLE_FLATCG_H */ -diff --git b/tools/sched_ext/scx_example_pair.bpf.c a/tools/sched_ext/scx_example_pair.bpf.c -deleted file mode 100644 -index 279efe58b777..000000000000 ---- b/tools/sched_ext/scx_example_pair.bpf.c -+++ /dev/null -@@ -1,627 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext core-scheduler which always makes every sibling CPU pair -- * execute from the same CPU cgroup. -- * -- * This scheduler is a minimal implementation and would need some form of -- * priority handling both inside each cgroup and across the cgroups to be -- * practically useful. -- * -- * Each CPU in the system is paired with exactly one other CPU, according to a -- * "stride" value that can be specified when the BPF scheduler program is first -- * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee -- * that they will only ever schedule tasks that belong to the same CPU cgroup. -- * -- * Scheduler Initialization -- * ------------------------ -- * -- * The scheduler BPF program is first initialized from user space, before it is -- * enabled. During this initialization process, each CPU on the system is -- * assigned several values that are constant throughout its runtime: -- * -- * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling -- * decisions. Paired CPUs always schedule tasks from the same -- * CPU cgroup, and synchronize with each other to guarantee -- * that this constraint is not violated. -- * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access -- * a struct pair_ctx object that is shared between the pair. -- * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the -- * pair. Each struct pair_ctx has an active_mask field, -- * which is a bitmap used to indicate whether each core -- * in the pair currently has an actively running task. -- * This index specifies which entry in the bitmap corresponds -- * to each CPU in the pair. -- * -- * During this initialization, the CPUs are paired according to a "stride" that -- * may be specified when invoking the user space program that initializes and -- * loads the scheduler. By default, the stride is 1/2 the total number of CPUs. -- * -- * Tasks and cgroups -- * ----------------- -- * -- * Every cgroup in the system is registered with the scheduler using the -- * pair_cgroup_init() callback, and every task in the system is associated with -- * exactly one cgroup. At a high level, the idea with the pair scheduler is to -- * always schedule tasks from the same cgroup within a given CPU pair. When a -- * task is enqueued (i.e. passed to the pair_enqueue() callback function), its -- * cgroup ID is read from its task struct, and then a corresponding queue map -- * is used to FIFO-enqueue the task for that cgroup. -- * -- * If you look through the implementation of the scheduler, you'll notice that -- * there is quite a bit of complexity involved with looking up the per-cgroup -- * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash -- * BPF hash map that is used to map a cgroup ID to a globally unique ID that's -- * allocated in the BPF program. This is done because we use separate maps to -- * store the FIFO queue of tasks, and the length of that map, per cgroup. This -- * complexity is only present because of current deficiencies in BPF that will -- * soon be addressed. The main point to keep in mind is that newly enqueued -- * tasks are added to their cgroup's FIFO queue. -- * -- * Dispatching tasks -- * ----------------- -- * -- * This section will describe how enqueued tasks are dispatched and scheduled. -- * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is -- * as follows: -- * -- * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is -- * the structure that's used to synchronize amongst the two pair CPUs in their -- * scheduling decisions. After any of the following events have occurred: -- * -- * - The cgroup's slice run has expired, or -- * - The cgroup becomes empty, or -- * - Either CPU in the pair is preempted by a higher priority scheduling class -- * -- * The cgroup transitions to the draining state and stops executing new tasks -- * from the cgroup. -- * -- * 2. If the pair is still executing a task, mark the pair_ctx as draining, and -- * wait for the pair CPU to be preempted. -- * -- * 3. Otherwise, if the pair CPU is not running a task, we can move onto -- * scheduling new tasks. Pop the next cgroup id from the top_q queue. -- * -- * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it. -- * -- * Note again that this scheduling behavior is simple, but the implementation -- * is complex mostly because this it hits several BPF shortcomings and has to -- * work around in often awkward ways. Most of the shortcomings are expected to -- * be resolved in the near future which should allow greatly simplifying this -- * scheduler. -- * -- * Dealing with preemption -- * ----------------------- -- * -- * SCX is the lowest priority sched_class, and could be preempted by them at -- * any time. To address this, the scheduler implements pair_cpu_release() and -- * pair_cpu_acquire() callbacks which are invoked by the core scheduler when -- * the scheduler loses and gains control of the CPU respectively. -- * -- * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and -- * then invoke: -- * -- * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- * -- * This preempts the pair CPU, and waits until it has re-entered the scheduler -- * before returning. This is necessary to ensure that the higher priority -- * sched_class that preempted our scheduler does not schedule a task -- * concurrently with our pair CPU. -- * -- * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption -- * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable -- * pair scheduling. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include "scx_example_pair.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; -- --/* !0 for veristat, set during init */ --const volatile u32 nr_cpu_ids = 64; -- --/* a pair of CPUs stay on a cgroup for this duration */ --const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL; -- --/* cpu ID -> pair cpu ID */ --const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 }; -- --/* cpu ID -> pair_id */ --const volatile u32 pair_id[MAX_CPUS]; -- --/* CPU ID -> CPU # in the pair (0 or 1) */ --const volatile u32 in_pair_idx[MAX_CPUS]; -- --struct pair_ctx { -- struct bpf_spin_lock lock; -- -- /* the cgroup the pair is currently executing */ -- u64 cgid; -- -- /* the pair started executing the current cgroup at */ -- u64 started_at; -- -- /* whether the current cgroup is draining */ -- bool draining; -- -- /* the CPUs that are currently active on the cgroup */ -- u32 active_mask; -- -- /* -- * the CPUs that are currently preempted and running tasks in a -- * different scheduler. -- */ -- u32 preempted_mask; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, MAX_CPUS / 2); -- __type(key, u32); -- __type(value, struct pair_ctx); --} pair_ctx SEC(".maps"); -- --/* queue of cgrp_q's possibly with tasks on them */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- /* -- * Because it's difficult to build strong synchronization encompassing -- * multiple non-trivial operations in BPF, this queue is managed in an -- * opportunistic way so that we guarantee that a cgroup w/ active tasks -- * is always on it but possibly multiple times. Once we have more robust -- * synchronization constructs and e.g. linked list, we should be able to -- * do this in a prettier way but for now just size it big enough. -- */ -- __uint(max_entries, 4 * MAX_CGRPS); -- __type(value, u64); --} top_q SEC(".maps"); -- --/* per-cgroup q which FIFOs the tasks from the cgroup */ --struct cgrp_q { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, MAX_QUEUED); -- __type(value, u32); --}; -- --/* -- * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local -- * storage; however, a cgroup local storage can only be accessed from the BPF -- * progs attached to the cgroup. For now, work around by allocating array of -- * cgrp_q's and then allocating per-cgroup indices. -- * -- * Another caveat: It's difficult to populate a large array of maps statically -- * or from BPF. Initialize it from userland. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, MAX_CGRPS); -- __type(key, s32); -- __array(values, struct cgrp_q); --} cgrp_q_arr SEC(".maps"); -- --static u64 cgrp_q_len[MAX_CGRPS]; -- --/* -- * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be -- * useful to have as a map type. -- */ --static u32 cgrp_q_idx_cursor; --static u64 cgrp_q_idx_busy[MAX_CGRPS]; -- --/* -- * All added up, the following is what we do: -- * -- * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking -- * for a free ID. If not found, fail cgroup creation with -EBUSY. -- * -- * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following -- * cgrp_q_idx_hash. -- * -- * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from -- * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr. -- * -- * This is sadly complicated for something pretty simple. Hopefully, we should -- * be able to simplify in the future. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, MAX_CGRPS); -- __uint(key_size, sizeof(u64)); /* cgrp ID */ -- __uint(value_size, sizeof(s32)); /* cgrp_q idx */ --} cgrp_q_idx_hash SEC(".maps"); -- --/* statistics */ --u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions; --u64 nr_exps, nr_exp_waits, nr_exp_empty; --u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty; -- --struct user_exit_info uei; -- --static bool time_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- struct cgrp_q *cgq; -- s32 pid = p->pid; -- u64 cgid; -- u32 *q_idx; -- u64 *cgq_len; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- cgrp = scx_bpf_task_cgroup(p); -- cgid = cgrp->kn->id; -- bpf_cgroup_release(cgrp); -- -- /* find the cgroup's q and push @p into it */ -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!q_idx) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return; -- } -- -- cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx); -- if (!cgq) { -- scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]", -- cgid, *q_idx); -- return; -- } -- -- if (bpf_map_push_elem(cgq, &pid, 0)) { -- scx_bpf_error("cgroup[%llu] queue overflow", cgid); -- return; -- } -- -- /* bump q len, if going 0 -> 1, queue cgroup into the top_q */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len) { -- scx_bpf_error("MEMBER_VTPR malfunction"); -- return; -- } -- -- if (!__sync_fetch_and_add(cgq_len, 1) && -- bpf_map_push_elem(&top_q, &cgid, 0)) { -- scx_bpf_error("top_q overflow"); -- return; -- } --} -- --static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) --{ -- u32 *vptr; -- -- vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *pairc = bpf_map_lookup_elem(&pair_ctx, vptr); -- if (!(*pairc)) -- return -EINVAL; -- -- vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *mask = 1U << *vptr; -- -- return 0; --} -- --static int try_dispatch(s32 cpu) --{ -- struct pair_ctx *pairc; -- struct bpf_map *cgq_map; -- struct task_struct *p; -- u64 now = bpf_ktime_get_ns(); -- bool kick_pair = false; -- bool expired, pair_preempted; -- u32 *vptr, in_pair_mask; -- s32 pid, q_idx; -- u64 cgid; -- int ret; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) { -- scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]", -- cpu); -- return -ENOENT; -- } -- -- bpf_spin_lock(&pairc->lock); -- pairc->active_mask &= ~in_pair_mask; -- -- expired = time_before(pairc->started_at + pair_batch_dur_ns, now); -- if (expired || pairc->draining) { -- u64 new_cgid = 0; -- -- __sync_fetch_and_add(&nr_exps, 1); -- -- /* -- * We're done with the current cgid. An obvious optimization -- * would be not draining if the next cgroup is the current one. -- * For now, be dumb and always expire. -- */ -- pairc->draining = true; -- -- pair_preempted = pairc->preempted_mask; -- if (pairc->active_mask || pair_preempted) { -- /* -- * The other CPU is still active, or is no longer under -- * our control due to e.g. being preempted by a higher -- * priority sched_class. We want to wait until this -- * cgroup expires, or until control of our pair CPU has -- * been returned to us. -- * -- * If the pair controls its CPU, and the time already -- * expired, kick. When the other CPU arrives at -- * dispatch and clears its active mask, it'll push the -- * pair to the next cgroup and kick this CPU. -- */ -- __sync_fetch_and_add(&nr_exp_waits, 1); -- bpf_spin_unlock(&pairc->lock); -- if (expired && !pair_preempted) -- kick_pair = true; -- goto out_maybe_kick; -- } -- -- bpf_spin_unlock(&pairc->lock); -- -- /* -- * Pick the next cgroup. It'd be easier / cleaner to not drop -- * pairc->lock and use stronger synchronization here especially -- * given that we'll be switching cgroups significantly less -- * frequently than tasks. Unfortunately, bpf_spin_lock can't -- * really protect anything non-trivial. Let's do opportunistic -- * operations instead. -- */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u32 *q_idx; -- u64 *cgq_len; -- -- if (bpf_map_pop_elem(&top_q, &new_cgid)) { -- /* no active cgroup, go idle */ -- __sync_fetch_and_add(&nr_exp_empty, 1); -- return 0; -- } -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid); -- if (!q_idx) -- continue; -- -- /* -- * This is the only place where empty cgroups are taken -- * off the top_q. -- */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len || !*cgq_len) -- continue; -- -- /* -- * If it has any tasks, requeue as we may race and not -- * execute it. -- */ -- bpf_map_push_elem(&top_q, &new_cgid, 0); -- break; -- } -- -- bpf_spin_lock(&pairc->lock); -- -- /* -- * The other CPU may already have started on a new cgroup while -- * we dropped the lock. Make sure that we're still draining and -- * start on the new cgroup. -- */ -- if (pairc->draining && !pairc->active_mask) { -- __sync_fetch_and_add(&nr_cgrp_next, 1); -- pairc->cgid = new_cgid; -- pairc->started_at = now; -- pairc->draining = false; -- kick_pair = true; -- } else { -- __sync_fetch_and_add(&nr_cgrp_coll, 1); -- } -- } -- -- cgid = pairc->cgid; -- pairc->active_mask |= in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- -- /* again, it'd be better to do all these with the lock held, oh well */ -- vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!vptr) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return -ENOENT; -- } -- q_idx = *vptr; -- -- /* claim one task from cgrp_q w/ q_idx */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u64 *cgq_len, len; -- -- cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]); -- if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) { -- /* the cgroup must be empty, expire and repeat */ -- __sync_fetch_and_add(&nr_cgrp_empty, 1); -- bpf_spin_lock(&pairc->lock); -- pairc->draining = true; -- pairc->active_mask &= ~in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- return -EAGAIN; -- } -- -- if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len) -- continue; -- -- break; -- } -- -- cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx); -- if (!cgq_map) { -- scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- if (bpf_map_pop_elem(cgq_map, &pid)) { -- scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- p = bpf_task_from_pid(pid); -- if (p) { -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } else { -- /* we don't handle dequeues, retry on lost tasks */ -- __sync_fetch_and_add(&nr_missing, 1); -- return -EAGAIN; -- } -- --out_maybe_kick: -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } -- return 0; --} -- --void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) --{ -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_dispatch(cpu) != -EAGAIN) -- break; -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask &= ~in_pair_mask; -- /* Kick the pair CPU, unless it was also preempted. */ -- kick_pair = !pairc->preempted_mask; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask |= in_pair_mask; -- pairc->active_mask &= ~in_pair_mask; -- /* Kick the pair CPU if it's still running. */ -- kick_pair = pairc->active_mask; -- pairc->draining = true; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- } -- } -- __sync_fetch_and_add(&nr_preemptions, 1); --} -- --s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 i, q_idx; -- -- bpf_for(i, 0, MAX_CGRPS) { -- q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS; -- if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1)) -- break; -- } -- if (i == MAX_CGRPS) -- return -EBUSY; -- -- if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]); -- if (busy) -- *busy = 0; -- return -EBUSY; -- } -- -- return 0; --} -- --void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 *q_idx; -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (q_idx) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]); -- if (busy) -- *busy = 0; -- bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid); -- } --} -- --s32 BPF_STRUCT_OPS(pair_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops pair_ops = { -- .enqueue = (void *)pair_enqueue, -- .dispatch = (void *)pair_dispatch, -- .cpu_acquire = (void *)pair_cpu_acquire, -- .cpu_release = (void *)pair_cpu_release, -- .cgroup_init = (void *)pair_cgroup_init, -- .cgroup_exit = (void *)pair_cgroup_exit, -- .init = (void *)pair_init, -- .exit = (void *)pair_exit, -- .name = "pair", --}; -diff --git b/tools/sched_ext/scx_example_pair.c a/tools/sched_ext/scx_example_pair.c -deleted file mode 100644 -index 18e032bbc173..000000000000 ---- b/tools/sched_ext/scx_example_pair.c -+++ /dev/null -@@ -1,143 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_pair.h" --#include "scx_example_pair.skel.h" -- --const char help_fmt[] = --"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n" --"execute from the same CPU cgroup.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-S STRIDE] [-p]\n" --"\n" --" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_pair *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 stride, i, opt, outer_fd; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_pair__open(); -- assert(skel); -- -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- /* pair up the earlier half to the latter by default, override with -s */ -- stride = skel->rodata->nr_cpu_ids / 2; -- -- while ((opt = getopt(argc, argv, "S:ph")) != -1) { -- switch (opt) { -- case 'S': -- stride = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- for (i = 0; i < skel->rodata->nr_cpu_ids; i++) { -- if (skel->rodata->pair_cpu[i] < 0) { -- skel->rodata->pair_cpu[i] = i + stride; -- skel->rodata->pair_cpu[i + stride] = i; -- skel->rodata->pair_id[i] = i; -- skel->rodata->pair_id[i + stride] = i; -- skel->rodata->in_pair_idx[i] = 0; -- skel->rodata->in_pair_idx[i + stride] = 1; -- } -- } -- -- assert(!scx_example_pair__load(skel)); -- -- /* -- * Populate the cgrp_q_arr map which is an array containing per-cgroup -- * queues. It'd probably be better to do this from BPF but there are too -- * many to initialize statically and there's no way to dynamically -- * populate from BPF. -- */ -- outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr); -- assert(outer_fd >= 0); -- -- printf("Initializing"); -- for (i = 0; i < MAX_CGRPS; i++) { -- s32 inner_fd; -- -- if (exit_req) -- break; -- -- inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, -- sizeof(u32), MAX_QUEUED, NULL); -- assert(inner_fd >= 0); -- assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY)); -- close(inner_fd); -- -- if (!(i % 10)) -- printf("."); -- fflush(stdout); -- } -- printf("\n"); -- -- /* -- * Fully initialized, attach and run. -- */ -- link = bpf_map__attach_struct_ops(skel->maps.pair_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf(" total:%10lu dispatch:%10lu missing:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_dispatched, -- skel->bss->nr_missing); -- printf(" kicks:%10lu preemptions:%7lu\n", -- skel->bss->nr_kicks, -- skel->bss->nr_preemptions); -- printf(" exp:%10lu exp_wait:%10lu exp_empty:%10lu\n", -- skel->bss->nr_exps, -- skel->bss->nr_exp_waits, -- skel->bss->nr_exp_empty); -- printf("cgnext:%10lu cgcoll:%10lu cgempty:%10lu\n", -- skel->bss->nr_cgrp_next, -- skel->bss->nr_cgrp_coll, -- skel->bss->nr_cgrp_empty); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_pair__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_pair.h a/tools/sched_ext/scx_example_pair.h -deleted file mode 100644 -index f60b824272f7..000000000000 ---- b/tools/sched_ext/scx_example_pair.h -+++ /dev/null -@@ -1,10 +0,0 @@ --#ifndef __SCX_EXAMPLE_PAIR_H --#define __SCX_EXAMPLE_PAIR_H -- --enum { -- MAX_CPUS = 4096, -- MAX_QUEUED = 4096, -- MAX_CGRPS = 4096, --}; -- --#endif /* __SCX_EXAMPLE_PAIR_H */ -diff --git b/tools/sched_ext/scx_example_qmap.bpf.c a/tools/sched_ext/scx_example_qmap.bpf.c -deleted file mode 100644 -index 579ab21ae403..000000000000 ---- b/tools/sched_ext/scx_example_qmap.bpf.c -+++ /dev/null -@@ -1,401 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple five-level FIFO queue scheduler. -- * -- * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets -- * assigned to one depending on its compound weight. Each CPU round robins -- * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from -- * queue0, 2 from queue1, 4 from queue2 and so on. -- * -- * This scheduler demonstrates: -- * -- * - BPF-side queueing using PIDs. -- * - Sleepable per-task storage allocation using ops.prep_enable(). -- * - Using ops.cpu_release() to handle a higher priority scheduling class taking -- * the CPU away. -- * - Core-sched support. -- * -- * This scheduler is primarily for demonstration and testing of sched_ext -- * features and unlikely to be useful for actual workloads. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include -- --char _license[] SEC("license") = "GPL"; -- --const volatile u64 slice_ns = SCX_SLICE_DFL; --const volatile bool switch_partial; --const volatile u32 stall_user_nth; --const volatile u32 stall_kernel_nth; --const volatile u32 dsp_inf_loop_after; --const volatile s32 disallow_tgid; -- --u32 test_error_cnt; -- --struct user_exit_info uei; -- --struct qmap { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, u32); --} queue0 SEC(".maps"), -- queue1 SEC(".maps"), -- queue2 SEC(".maps"), -- queue3 SEC(".maps"), -- queue4 SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, 5); -- __type(key, int); -- __array(values, struct qmap); --} queue_arr SEC(".maps") = { -- .values = { -- [0] = &queue0, -- [1] = &queue1, -- [2] = &queue2, -- [3] = &queue3, -- [4] = &queue4, -- }, --}; -- --/* -- * Per-queue sequence numbers to implement core-sched ordering. -- * -- * Tail seq is assigned to each queued task and incremented. Head seq tracks the -- * sequence number of the latest dispatched task. The distance between the a -- * task's seq and the associated queue's head seq is called the queue distance -- * and used when comparing two tasks for ordering. See qmap_core_sched_before(). -- */ --static u64 core_sched_head_seqs[5]; --static u64 core_sched_tail_seqs[5]; -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local_dsq */ -- u64 core_sched_seq; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --/* Per-cpu dispatch index and remaining count */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(max_entries, 2); -- __type(key, u32); -- __type(value, u64); --} dispatch_idx_cnt SEC(".maps"); -- --/* Statistics */ --unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued; --unsigned long nr_core_sched_execed; -- --s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- struct task_ctx *tctx; -- s32 cpu; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- return cpu; -- -- return prev_cpu; --} -- --static int weight_to_idx(u32 weight) --{ -- /* Coarsely map the compound weight to a FIFO. */ -- if (weight <= 25) -- return 0; -- else if (weight <= 50) -- return 1; -- else if (weight < 200) -- return 2; -- else if (weight < 400) -- return 3; -- else -- return 4; --} -- --void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) --{ -- static u32 user_cnt, kernel_cnt; -- struct task_ctx *tctx; -- u32 pid = p->pid; -- int idx = weight_to_idx(p->scx.weight); -- void *ring; -- -- if (p->flags & PF_KTHREAD) { -- if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth)) -- return; -- } else { -- if (stall_user_nth && !(++user_cnt % stall_user_nth)) -- return; -- } -- -- if (test_error_cnt && !--test_error_cnt) -- scx_bpf_error("test triggering error"); -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * All enqueued tasks must have their core_sched_seq updated for correct -- * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in -- * qmap_ops.flags. -- */ -- tctx->core_sched_seq = core_sched_tail_seqs[idx]++; -- -- /* -- * If qmap_select_cpu() is telling us to or this is the last runnable -- * task on the CPU, enqueue locally. -- */ -- if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) { -- tctx->force_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); -- return; -- } -- -- /* -- * If the task was re-enqueued due to the CPU being preempted by a -- * higher priority scheduling class, just re-enqueue the task directly -- * on the global DSQ. As we want another CPU to pick it up, find and -- * kick an idle CPU. -- */ -- if (enq_flags & SCX_ENQ_REENQ) { -- s32 cpu; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags); -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- return; -- } -- -- ring = bpf_map_lookup_elem(&queue_arr, &idx); -- if (!ring) { -- scx_bpf_error("failed to find ring %d", idx); -- return; -- } -- -- /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */ -- if (bpf_map_push_elem(ring, &pid, 0)) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_enqueued, 1); --} -- --/* -- * The BPF queue map doesn't support removal and sched_ext can handle spurious -- * dispatches. qmap_dequeue() is only used to collect statistics. -- */ --void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags) --{ -- __sync_fetch_and_add(&nr_dequeued, 1); -- if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) -- __sync_fetch_and_add(&nr_core_sched_execed, 1); --} -- --static void update_core_sched_head_seq(struct task_struct *p) --{ -- struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- int idx = weight_to_idx(p->scx.weight); -- -- if (tctx) -- core_sched_head_seqs[idx] = tctx->core_sched_seq; -- else -- scx_bpf_error("task_ctx lookup failed"); --} -- --void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 zero = 0, one = 1; -- u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero); -- u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one); -- void *fifo; -- s32 pid; -- int i; -- -- if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) { -- struct task_struct *p; -- -- /* -- * PID 2 should be kthreadd which should mostly be idle and off -- * the scheduler. Let's keep dispatching it to force the kernel -- * to call this function over and over again. -- */ -- p = bpf_task_from_pid(2); -- if (p) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- if (!idx || !cnt) { -- scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt); -- return; -- } -- -- for (i = 0; i < 5; i++) { -- /* Advance the dispatch cursor and pick the fifo. */ -- if (!*cnt) { -- *idx = (*idx + 1) % 5; -- *cnt = 1 << *idx; -- } -- (*cnt)--; -- -- fifo = bpf_map_lookup_elem(&queue_arr, idx); -- if (!fifo) { -- scx_bpf_error("failed to find ring %llu", *idx); -- return; -- } -- -- /* Dispatch or advance. */ -- if (!bpf_map_pop_elem(fifo, &pid)) { -- struct task_struct *p; -- -- p = bpf_task_from_pid(pid); -- if (p) { -- update_core_sched_head_seq(p); -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- *cnt = 0; -- } --} -- --/* -- * The distance from the head of the queue scaled by the weight of the queue. -- * The lower the number, the older the task and the higher the priority. -- */ --static s64 task_qdist(struct task_struct *p) --{ -- int idx = weight_to_idx(p->scx.weight); -- struct task_ctx *tctx; -- s64 qdist; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return 0; -- } -- -- qdist = tctx->core_sched_seq - core_sched_head_seqs[idx]; -- -- /* -- * As queue index increments, the priority doubles. The queue w/ index 3 -- * is dispatched twice more frequently than 2. Reflect the difference by -- * scaling qdists accordingly. Note that the shift amount needs to be -- * flipped depending on the sign to avoid flipping priority direction. -- */ -- if (qdist >= 0) -- return qdist << (4 - idx); -- else -- return qdist << idx; --} -- --/* -- * This is called to determine the task ordering when core-sched is picking -- * tasks to execute on SMT siblings and should encode about the same ordering as -- * the regular scheduling path. Use the priority-scaled distances from the head -- * of the queues to compare the two tasks which should be consistent with the -- * dispatch path behavior. -- */ --bool BPF_STRUCT_OPS(qmap_core_sched_before, -- struct task_struct *a, struct task_struct *b) --{ -- return task_qdist(a) > task_qdist(b); --} -- --void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- u32 cnt; -- -- /* -- * Called when @cpu is taken by a higher priority scheduling class. This -- * makes @cpu no longer available for executing sched_ext tasks. As we -- * don't want the tasks in @cpu's local dsq to sit there until @cpu -- * becomes available again, re-enqueue them into the global dsq. See -- * %SCX_ENQ_REENQ handling in qmap_enqueue(). -- */ -- cnt = scx_bpf_reenqueue_local(); -- if (cnt) -- __sync_fetch_and_add(&nr_reenqueued, cnt); --} -- --s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (p->tgid == disallow_tgid) -- p->scx.disallow = true; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(qmap_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops qmap_ops = { -- .select_cpu = (void *)qmap_select_cpu, -- .enqueue = (void *)qmap_enqueue, -- .dequeue = (void *)qmap_dequeue, -- .dispatch = (void *)qmap_dispatch, -- .core_sched_before = (void *)qmap_core_sched_before, -- .cpu_release = (void *)qmap_cpu_release, -- .prep_enable = (void *)qmap_prep_enable, -- .init = (void *)qmap_init, -- .exit = (void *)qmap_exit, -- .flags = SCX_OPS_ENQ_LAST, -- .timeout_ms = 5000U, -- .name = "qmap", --}; -diff --git b/tools/sched_ext/scx_example_qmap.c a/tools/sched_ext/scx_example_qmap.c -deleted file mode 100644 -index ccb4814ee61b..000000000000 ---- b/tools/sched_ext/scx_example_qmap.c -+++ /dev/null -@@ -1,107 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_qmap.skel.h" -- --const char help_fmt[] = --"A simple five-level FIFO queue sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" --" -t COUNT Stall every COUNT'th user thread\n" --" -T COUNT Stall every COUNT'th kernel thread\n" --" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n" --" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_qmap *skel; -- struct bpf_link *link; -- int opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_qmap__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) { -- switch (opt) { -- case 's': -- skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; -- break; -- case 'e': -- skel->bss->test_error_cnt = strtoul(optarg, NULL, 0); -- break; -- case 't': -- skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0); -- break; -- case 'T': -- skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0); -- break; -- case 'l': -- skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0); -- break; -- case 'd': -- skel->rodata->disallow_tgid = strtol(optarg, NULL, 0); -- if (skel->rodata->disallow_tgid < 0) -- skel->rodata->disallow_tgid = getpid(); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_qmap__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.qmap_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- long nr_enqueued = skel->bss->nr_enqueued; -- long nr_dispatched = skel->bss->nr_dispatched; -- -- printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n", -- nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, -- skel->bss->nr_reenqueued, skel->bss->nr_dequeued, -- skel->bss->nr_core_sched_execed); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_qmap__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_simple.bpf.c a/tools/sched_ext/scx_example_simple.bpf.c -deleted file mode 100644 -index 4bccca3e2047..000000000000 ---- b/tools/sched_ext/scx_example_simple.bpf.c -+++ /dev/null -@@ -1,128 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple scheduler. -- * -- * By default, it operates as a simple global weighted vtime scheduler and can -- * be switched to FIFO scheduling. It also demonstrates the following niceties. -- * -- * - Statistics tracking how many tasks are queued to local and global dsq's. -- * - Termination notification for userspace. -- * -- * While very simple, this scheduler should work reasonably well on CPUs with a -- * uniform L3 cache topology. While preemption is not implemented, the fact that -- * the scheduling queue is shared across all CPUs means that whatever is at the -- * front of the queue is likely to be executed fairly quickly given enough -- * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads -- * but comes with the usual problems with FIFO scheduling where saturating -- * threads can easily drown out interactive ones. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --static u64 vtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, 2); /* [local, global] */ --} stats SEC(".maps"); -- --static void stat_inc(u32 idx) --{ -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); -- if (cnt_p) -- (*cnt_p)++; --} -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) --{ -- /* -- * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that -- * running @p on its CPU directly shouldn't affect fairness. Just queue -- * it on the local FIFO. -- */ -- if (enq_flags & SCX_ENQ_LOCAL) { -- stat_inc(0); /* count local queueing */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- return; -- } -- -- stat_inc(1); /* count global queueing */ -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) -- vtime = vtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --void BPF_STRUCT_OPS(simple_running, struct task_struct *p) --{ -- if (fifo_sched) -- return; -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(vtime_now, p->scx.dsq_vtime)) -- vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --s32 BPF_STRUCT_OPS(simple_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .running = (void *)simple_running, -- .stopping = (void *)simple_stopping, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", --}; -diff --git b/tools/sched_ext/scx_example_simple.c a/tools/sched_ext/scx_example_simple.c -deleted file mode 100644 -index 486b401f7c95..000000000000 ---- b/tools/sched_ext/scx_example_simple.c -+++ /dev/null -@@ -1,101 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_simple.skel.h" -- --const char help_fmt[] = --"A simple sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-f] [-p]\n" --"\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int simple) --{ -- exit_req = 1; --} -- --static void read_stats(struct scx_example_simple *skel, u64 *stats) --{ -- int nr_cpus = libbpf_num_possible_cpus(); -- u64 cnts[2][nr_cpus]; -- u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * 2); -- -- for (idx = 0; idx < 2; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_simple *skel; -- struct bpf_link *link; -- u32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_simple__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "fph")) != -1) { -- switch (opt) { -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_simple__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.simple_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- u64 stats[2]; -- -- read_stats(skel, stats); -- printf("local=%lu global=%lu\n", stats[0], stats[1]); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_simple__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_userland.bpf.c a/tools/sched_ext/scx_example_userland.bpf.c -deleted file mode 100644 -index a089bc6bbe86..000000000000 ---- b/tools/sched_ext/scx_example_userland.bpf.c -+++ /dev/null -@@ -1,269 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A minimal userland scheduler. -- * -- * In terms of scheduling, this provides two different types of behaviors: -- * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity. -- * All such tasks are direct-dispatched from the kernel, and are never -- * enqueued in user space. -- * 2. A primitive vruntime scheduler that is implemented in user space, for all -- * other tasks. -- * -- * Some parts of this example user space scheduler could be implemented more -- * efficiently using more complex and sophisticated data structures. For -- * example, rather than using BPF_MAP_TYPE_QUEUE's, -- * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between -- * user space and kernel space. Similarly, we use a simple vruntime-sorted list -- * in user space, but an rbtree could be used instead. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include --#include "scx_common.bpf.h" --#include "scx_example_userland_common.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; --const volatile s32 usersched_pid; -- --/* !0 for veristat, set during init */ --const volatile u32 num_possible_cpus = 64; -- --/* Stats that are printed by user space. */ --u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues; -- --struct user_exit_info uei; -- --/* -- * Whether the user space scheduler needs to be scheduled due to a task being -- * enqueued in user space. -- */ --static bool usersched_needed; -- --/* -- * The map containing tasks that are enqueued in user space from the kernel. -- * -- * This map is drained by the user space scheduler. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, struct scx_userland_enqueued_task); --} enqueued SEC(".maps"); -- --/* -- * The map containing tasks that are dispatched to the kernel from user space. -- * -- * Drained by the kernel in userland_dispatch(). -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, s32); --} dispatched SEC(".maps"); -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local DSQ */ --}; -- --/* Map that contains task-local storage. */ --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --static bool is_usersched_task(const struct task_struct *p) --{ -- return p->pid == usersched_pid; --} -- --static bool keep_in_kernel(const struct task_struct *p) --{ -- return p->nr_cpus_allowed < num_possible_cpus; --} -- --static struct task_struct *usersched_task(void) --{ -- struct task_struct *p; -- -- p = bpf_task_from_pid(usersched_pid); -- /* -- * Should never happen -- the usersched task should always be managed -- * by sched_ext. -- */ -- if (!p) { -- scx_bpf_error("Failed to find usersched task %d", usersched_pid); -- /* -- * We should never hit this path, and we error out of the -- * scheduler above just in case, so the scheduler will soon be -- * be evicted regardless. So as to simplify the logic in the -- * caller to not have to check for NULL, return an acquired -- * reference to the current task here rather than NULL. -- */ -- return bpf_task_acquire(bpf_get_current_task_btf()); -- } -- -- return p; --} -- --s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- if (keep_in_kernel(p)) { -- s32 cpu; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to look up task-local storage for %s", p->comm); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- tctx->force_local = true; -- return cpu; -- } -- } -- -- return prev_cpu; --} -- --static void dispatch_user_scheduler(void) --{ -- struct task_struct *p; -- -- usersched_needed = false; -- p = usersched_task(); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); --} -- --static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags) --{ -- struct scx_userland_enqueued_task task; -- -- memset(&task, 0, sizeof(task)); -- task.pid = p->pid; -- task.sum_exec_runtime = p->se.sum_exec_runtime; -- task.weight = p->scx.weight; -- -- if (bpf_map_push_elem(&enqueued, &task, 0)) { -- /* -- * If we fail to enqueue the task in user space, put it -- * directly on the global DSQ. -- */ -- __sync_fetch_and_add(&nr_failed_enqueues, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- __sync_fetch_and_add(&nr_user_enqueues, 1); -- usersched_needed = true; -- } --} -- --void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags) --{ -- if (keep_in_kernel(p)) { -- u64 dsq_id = SCX_DSQ_GLOBAL; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to lookup task ctx for %s", p->comm); -- return; -- } -- -- if (tctx->force_local) -- dsq_id = SCX_DSQ_LOCAL; -- tctx->force_local = false; -- scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags); -- __sync_fetch_and_add(&nr_kernel_enqueues, 1); -- return; -- } else if (!is_usersched_task(p)) { -- enqueue_task_in_user_space(p, enq_flags); -- } --} -- --void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (usersched_needed) -- dispatch_user_scheduler(); -- -- bpf_repeat(4096) { -- s32 pid; -- struct task_struct *p; -- -- if (bpf_map_pop_elem(&dispatched, &pid)) -- break; -- -- /* -- * The task could have exited by the time we get around to -- * dispatching it. Treat this as a normal occurrence, and simply -- * move onto the next iteration. -- */ -- p = bpf_task_from_pid(pid); -- if (!p) -- continue; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } --} -- --s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(userland_init) --{ -- if (num_possible_cpus == 0) { -- scx_bpf_error("User scheduler # CPUs uninitialized (%d)", -- num_possible_cpus); -- return -EINVAL; -- } -- -- if (usersched_pid <= 0) { -- scx_bpf_error("User scheduler pid uninitialized (%d)", -- usersched_pid); -- return -EINVAL; -- } -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops userland_ops = { -- .select_cpu = (void *)userland_select_cpu, -- .enqueue = (void *)userland_enqueue, -- .dispatch = (void *)userland_dispatch, -- .prep_enable = (void *)userland_prep_enable, -- .init = (void *)userland_init, -- .exit = (void *)userland_exit, -- .timeout_ms = 3000, -- .name = "userland", --}; -diff --git b/tools/sched_ext/scx_example_userland.c a/tools/sched_ext/scx_example_userland.c -deleted file mode 100644 -index 4152b1e65fe1..000000000000 ---- b/tools/sched_ext/scx_example_userland.c -+++ /dev/null -@@ -1,402 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext user space scheduler which provides vruntime semantics -- * using a simple ordered-list implementation. -- * -- * Each CPU in the system resides in a single, global domain. This precludes -- * the need to do any load balancing between domains. The scheduler could -- * easily be extended to support multiple domains, with load balancing -- * happening in user space. -- * -- * Any task which has any CPU affinity is scheduled entirely in BPF. This -- * program only schedules tasks which may run on any CPU. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include "user_exit_info.h" --#include "scx_example_userland_common.h" --#include "scx_example_userland.skel.h" -- --const char help_fmt[] = --"A minimal userland sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-b BATCH] [-p]\n" --"\n" --" -b BATCH The number of tasks to batch when dispatching (default: 8)\n" --" -p Don't switch all, switch only tasks on SCHED_EXT policy\n" --" -h Display this help and exit\n"; -- --/* Defined in UAPI */ --#define SCHED_EXT 7 -- --/* Number of tasks to batch when dispatching to user space. */ --static __u32 batch_size = 8; -- --static volatile int exit_req; --static int enqueued_fd, dispatched_fd; -- --static struct scx_example_userland *skel; --static struct bpf_link *ops_link; -- --/* Stats collected in user space. */ --static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches; -- --/* The data structure containing tasks that are enqueued in user space. */ --struct enqueued_task { -- LIST_ENTRY(enqueued_task) entries; -- __u64 sum_exec_runtime; -- double vruntime; --}; -- --/* -- * Use a vruntime-sorted list to store tasks. This could easily be extended to -- * a more optimal data structure, such as an rbtree as is done in CFS. We -- * currently elect to use a sorted list to simplify the example for -- * illustrative purposes. -- */ --LIST_HEAD(listhead, enqueued_task); -- --/* -- * A vruntime-sorted list of tasks. The head of the list contains the task with -- * the lowest vruntime. That is, the task that has the "highest" claim to be -- * scheduled. -- */ --static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head); -- --/* -- * The statically allocated array of tasks. We use a statically allocated list -- * here to avoid having to allocate on the enqueue path, which could cause a -- * deadlock. A more substantive user space scheduler could e.g. provide a hook -- * for newly enabled tasks that are passed to the scheduler from the -- * .prep_enable() callback to allows the scheduler to allocate on safe paths. -- */ --struct enqueued_task tasks[USERLAND_MAX_TASKS]; -- --static double min_vruntime; -- --static void sigint_handler(int userland) --{ -- exit_req = 1; --} -- --static __u32 task_pid(const struct enqueued_task *task) --{ -- return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task); --} -- --static int dispatch_task(s32 pid) --{ -- int err; -- -- err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d\n", pid); -- exit_req = 1; -- } else { -- nr_vruntime_dispatches++; -- } -- -- return err; --} -- --static struct enqueued_task *get_enqueued_task(__s32 pid) --{ -- if (pid >= USERLAND_MAX_TASKS) -- return NULL; -- -- return &tasks[pid]; --} -- --static double calc_vruntime_delta(__u64 weight, __u64 delta) --{ -- double weight_f = (double)weight / 100.0; -- double delta_f = (double)delta; -- -- return delta_f / weight_f; --} -- --static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task) --{ -- __u64 delta; -- -- delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime; -- -- enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta); -- if (min_vruntime > enqueued->vruntime) -- enqueued->vruntime = min_vruntime; -- enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime; --} -- --static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task) --{ -- struct enqueued_task *curr, *enqueued, *prev; -- -- curr = get_enqueued_task(bpf_task->pid); -- if (!curr) -- return ENOENT; -- -- update_enqueued(curr, bpf_task); -- nr_vruntime_enqueues++; -- -- /* -- * Enqueue the task in a vruntime-sorted list. A more optimal data -- * structure such as an rbtree could easily be used as well. We elect -- * to use a list here simply because it's less code, and thus the -- * example is less convoluted and better serves to illustrate what a -- * user space scheduler could look like. -- */ -- -- if (LIST_EMPTY(&vruntime_head)) { -- LIST_INSERT_HEAD(&vruntime_head, curr, entries); -- return 0; -- } -- -- LIST_FOREACH(enqueued, &vruntime_head, entries) { -- if (curr->vruntime <= enqueued->vruntime) { -- LIST_INSERT_BEFORE(enqueued, curr, entries); -- return 0; -- } -- prev = enqueued; -- } -- -- LIST_INSERT_AFTER(prev, curr, entries); -- -- return 0; --} -- --static void drain_enqueued_map(void) --{ -- while (1) { -- struct scx_userland_enqueued_task task; -- int err; -- -- if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) -- return; -- -- err = vruntime_enqueue(&task); -- if (err) { -- fprintf(stderr, "Failed to enqueue task %d: %s\n", -- task.pid, strerror(err)); -- exit_req = 1; -- return; -- } -- } --} -- --static void dispatch_batch(void) --{ -- __u32 i; -- -- for (i = 0; i < batch_size; i++) { -- struct enqueued_task *task; -- int err; -- __s32 pid; -- -- task = LIST_FIRST(&vruntime_head); -- if (!task) -- return; -- -- min_vruntime = task->vruntime; -- pid = task_pid(task); -- LIST_REMOVE(task, entries); -- err = dispatch_task(pid); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d in %u\n", -- pid, i); -- return; -- } -- } --} -- --static void *run_stats_printer(void *arg) --{ -- while (!exit_req) { -- __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; -- -- nr_failed_enqueues = skel->bss->nr_failed_enqueues; -- nr_kernel_enqueues = skel->bss->nr_kernel_enqueues; -- nr_user_enqueues = skel->bss->nr_user_enqueues; -- total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues; -- -- printf("o-----------------------o\n"); -- printf("| BPF ENQUEUES |\n"); -- printf("|-----------------------|\n"); -- printf("| kern: %10llu |\n", nr_kernel_enqueues); -- printf("| user: %10llu |\n", nr_user_enqueues); -- printf("| failed: %10llu |\n", nr_failed_enqueues); -- printf("| -------------------- |\n"); -- printf("| total: %10llu |\n", total); -- printf("| |\n"); -- printf("|-----------------------|\n"); -- printf("| VRUNTIME / USER |\n"); -- printf("|-----------------------|\n"); -- printf("| enq: %10llu |\n", nr_vruntime_enqueues); -- printf("| disp: %10llu |\n", nr_vruntime_dispatches); -- printf("o-----------------------o\n"); -- printf("\n\n"); -- sleep(1); -- } -- -- return NULL; --} -- --static int spawn_stats_thread(void) --{ -- pthread_t stats_printer; -- -- return pthread_create(&stats_printer, NULL, run_stats_printer, NULL); --} -- --static int bootstrap(int argc, char **argv) --{ -- int err; -- __u32 opt; -- struct sched_param sched_param = { -- .sched_priority = sched_get_priority_max(SCHED_EXT), -- }; -- bool switch_partial = false; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- /* -- * Enforce that the user scheduler task is managed by sched_ext. The -- * task eagerly drains the list of enqueued tasks in its main work -- * loop, and then yields the CPU. The BPF scheduler only schedules the -- * user space scheduler task when at least one other task in the system -- * needs to be scheduled. -- */ -- err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param); -- if (err) { -- fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err)); -- return err; -- } -- -- while ((opt = getopt(argc, argv, "b:ph")) != -1) { -- switch (opt) { -- case 'b': -- batch_size = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- exit(opt != 'h'); -- } -- } -- -- /* -- * It's not always safe to allocate in a user space scheduler, as an -- * enqueued task could hold a lock that we require in order to be able -- * to allocate. -- */ -- err = mlockall(MCL_CURRENT | MCL_FUTURE); -- if (err) { -- fprintf(stderr, "Failed to prefault and lock address space: %s\n", -- strerror(err)); -- return err; -- } -- -- skel = scx_example_userland__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno)); -- return errno; -- } -- skel->rodata->num_possible_cpus = libbpf_num_possible_cpus(); -- assert(skel->rodata->num_possible_cpus > 0); -- skel->rodata->usersched_pid = getpid(); -- assert(skel->rodata->usersched_pid > 0); -- skel->rodata->switch_partial = switch_partial; -- -- err = scx_example_userland__load(skel); -- if (err) { -- fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- enqueued_fd = bpf_map__fd(skel->maps.enqueued); -- dispatched_fd = bpf_map__fd(skel->maps.dispatched); -- assert(enqueued_fd > 0); -- assert(dispatched_fd > 0); -- -- err = spawn_stats_thread(); -- if (err) { -- fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops); -- if (!ops_link) { -- fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno)); -- err = errno; -- goto destroy_skel; -- } -- -- return 0; -- --destroy_skel: -- scx_example_userland__destroy(skel); -- exit_req = 1; -- return err; --} -- --static void sched_main_loop(void) --{ -- while (!exit_req) { -- /* -- * Perform the following work in the main user space scheduler -- * loop: -- * -- * 1. Drain all tasks from the enqueued map, and enqueue them -- * to the vruntime sorted list. -- * -- * 2. Dispatch a batch of tasks from the vruntime sorted list -- * down to the kernel. -- * -- * 3. Yield the CPU back to the system. The BPF scheduler will -- * reschedule the user space scheduler once another task has -- * been enqueued to user space. -- */ -- drain_enqueued_map(); -- dispatch_batch(); -- sched_yield(); -- } --} -- --int main(int argc, char **argv) --{ -- int err; -- -- err = bootstrap(argc, argv); -- if (err) { -- fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err)); -- return err; -- } -- -- sched_main_loop(); -- -- exit_req = 1; -- bpf_link__destroy(ops_link); -- uei_print(&skel->bss->uei); -- scx_example_userland__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_userland_common.h a/tools/sched_ext/scx_example_userland_common.h -deleted file mode 100644 -index 639c6809c5ff..000000000000 ---- b/tools/sched_ext/scx_example_userland_common.h -+++ /dev/null -@@ -1,19 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* Copyright (c) 2022 Meta, Inc */ -- --#ifndef __SCX_USERLAND_COMMON_H --#define __SCX_USERLAND_COMMON_H -- --#define USERLAND_MAX_TASKS 8192 -- --/* -- * An instance of a task that has been enqueued by the kernel for consumption -- * by a user space global scheduler thread. -- */ --struct scx_userland_enqueued_task { -- __s32 pid; -- u64 sum_exec_runtime; -- u64 weight; --}; -- --#endif // __SCX_USERLAND_COMMON_H -diff --git b/tools/sched_ext/user_exit_info.h a/tools/sched_ext/user_exit_info.h -deleted file mode 100644 -index e701ef0e0b86..000000000000 ---- b/tools/sched_ext/user_exit_info.h -+++ /dev/null -@@ -1,50 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Define struct user_exit_info which is shared between BPF and userspace parts -- * to communicate exit status and other information. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __USER_EXIT_INFO_H --#define __USER_EXIT_INFO_H -- --struct user_exit_info { -- int type; -- char reason[128]; -- char msg[1024]; --}; -- --#ifdef __bpf__ -- --#include "vmlinux.h" --#include -- --static inline void uei_record(struct user_exit_info *uei, -- const struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason); -- bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg); -- /* use __sync to force memory barrier */ -- __sync_val_compare_and_swap(&uei->type, uei->type, ei->type); --} -- --#else /* !__bpf__ */ -- --static inline bool uei_exited(struct user_exit_info *uei) --{ -- /* use __sync to force memory barrier */ -- return __sync_val_compare_and_swap(&uei->type, -1, -1); --} -- --static inline void uei_print(const struct user_exit_info *uei) --{ -- fprintf(stderr, "EXIT: %s", uei->reason); -- if (uei->msg[0] != '\0') -- fprintf(stderr, " (%s)", uei->msg); -- fputs("\n", stderr); --} -- --#endif /* __bpf__ */ --#endif /* __USER_EXIT_INFO_H */ diff --git a/oneplus/hmbird/fengchi_OP-NORD-6_A16.patch b/oneplus/hmbird/fengchi_OP-NORD-6_A16.patch deleted file mode 100644 index e6008b42..00000000 --- a/oneplus/hmbird/fengchi_OP-NORD-6_A16.patch +++ /dev/null @@ -1,19909 +0,0 @@ -diff --git b/Documentation/scheduler/index.rst a/Documentation/scheduler/index.rst -index 0b650bb550e6..3170747226f6 100644 ---- b/Documentation/scheduler/index.rst -+++ a/Documentation/scheduler/index.rst -@@ -19,7 +19,6 @@ Scheduler - sched-nice-design - sched-rt-group - sched-stats -- sched-ext - sched-debug - - text_files -diff --git b/Documentation/scheduler/sched-ext.rst a/Documentation/scheduler/sched-ext.rst -deleted file mode 100644 -index 84c30b44f104..000000000000 ---- b/Documentation/scheduler/sched-ext.rst -+++ /dev/null -@@ -1,230 +0,0 @@ --========================== --Extensible Scheduler Class --========================== -- --sched_ext is a scheduler class whose behavior can be defined by a set of BPF --programs - the BPF scheduler. -- --* sched_ext exports a full scheduling interface so that any scheduling -- algorithm can be implemented on top. -- --* The BPF scheduler can group CPUs however it sees fit and schedule them -- together, as tasks aren't tied to specific CPUs at the time of wakeup. -- --* The BPF scheduler can be turned on and off dynamically anytime. -- --* The system integrity is maintained no matter what the BPF scheduler does. -- The default scheduling behavior is restored anytime an error is detected, -- a runnable task stalls, or on invoking the SysRq key sequence -- :kbd:`SysRq-S`. -- --Switching to and from sched_ext --=============================== -- --``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and --``tools/sched_ext`` contains the example schedulers. -- --sched_ext is used only when the BPF scheduler is loaded and running. -- --If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be --treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is --loaded. On load, such tasks will be switched to and scheduled by sched_ext. -- --The BPF scheduler can choose to schedule all normal and lower class tasks by --calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this --case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and --``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers, --this mode can be selected with the ``-a`` option. -- --Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or --detection of any internal error including stalled runnable tasks aborts the --BPF scheduler and reverts all tasks back to CFS. -- --.. code-block:: none -- -- # make -j16 -C tools/sched_ext -- # tools/sched_ext/scx_example_simple -- local=0 global=3 -- local=5 global=24 -- local=9 global=44 -- local=13 global=56 -- local=17 global=72 -- ^CEXIT: BPF scheduler unregistered -- --If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler --and whether a given task is on sched_ext can be determined as follows: -- --.. code-block:: none -- -- # cat /sys/kernel/debug/sched/ext -- ops : simple -- enabled : 1 -- switching_all : 1 -- switched_all : 1 -- enable_state : enabled -- -- # grep ext /proc/self/sched -- ext.enabled : 1 -- --The Basics --========== -- --Userspace can implement an arbitrary BPF scheduler by loading a set of BPF --programs that implement ``struct sched_ext_ops``. The only mandatory field --is ``ops.name`` which must be a valid BPF object name. All operations are --optional. The following modified excerpt is from --``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO --scheduler. -- --.. code-block:: c -- -- s32 BPF_STRUCT_OPS(simple_init) -- { -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; -- } -- -- void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) -- { -- if (enq_flags & SCX_ENQ_LOCAL) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags); -- } -- -- void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) -- { -- exit_type = ei->type; -- } -- -- SEC(".struct_ops") -- struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", -- }; -- --Dispatch Queues ----------------- -- --To match the impedance between the scheduler core and the BPF scheduler, --sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a --priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``), --and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage --an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and --``scx_bpf_destroy_dsq()``. -- --A CPU always executes a task from its local DSQ. A task is "dispatched" to a --DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's --local DSQ. -- --When a CPU is looking for the next task to run, if the local DSQ is not --empty, the first task is picked. Otherwise, the CPU tries to consume the --global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()`` --is invoked. -- --Scheduling Cycle ------------------ -- --The following briefly shows how a waking task is scheduled and executed. -- --1. When a task is waking up, ``ops.select_cpu()`` is the first operation -- invoked. This serves two purposes. First, CPU selection optimization -- hint. Second, waking up the selected CPU if idle. -- -- The CPU selected by ``ops.select_cpu()`` is an optimization hint and not -- binding. The actual decision is made at the last step of scheduling. -- However, there is a small performance gain if the CPU -- ``ops.select_cpu()`` returns matches the CPU the task eventually runs on. -- -- A side-effect of selecting a CPU is waking it up from idle. While a BPF -- scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper, -- using ``ops.select_cpu()`` judiciously can be simpler and more efficient. -- -- Note that the scheduler core will ignore an invalid CPU selection, for -- example, if it's outside the allowed cpumask of the task. -- --2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can -- make one of the following decisions: -- -- * Immediately dispatch the task to either the global or local DSQ by -- calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or -- ``SCX_DSQ_LOCAL``, respectively. -- -- * Immediately dispatch the task to a custom DSQ by calling -- ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63. -- -- * Queue the task on the BPF side. -- --3. When a CPU is ready to schedule, it first looks at its local DSQ. If -- empty, it then looks at the global DSQ. If there still isn't a task to -- run, ``ops.dispatch()`` is invoked which can use the following two -- functions to populate the local DSQ. -- -- * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can -- be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``, -- ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()`` -- currently can't be called with BPF locks held, this is being worked on -- and will be supported. ``scx_bpf_dispatch()`` schedules dispatching -- rather than performing them immediately. There can be up to -- ``ops.dispatch_max_batch`` pending tasks. -- -- * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ -- to the dispatching DSQ. This function cannot be called with any BPF -- locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks -- before trying to consume the specified DSQ. -- --4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ, -- the CPU runs the first one. If empty, the following steps are taken: -- -- * Try to consume the global DSQ. If successful, run the task. -- -- * If ``ops.dispatch()`` has dispatched any tasks, retry #3. -- -- * If the previous task is an SCX task and still runnable, keep executing -- it (see ``SCX_OPS_ENQ_LAST``). -- -- * Go idle. -- --Note that the BPF scheduler can always choose to dispatch tasks immediately --in ``ops.enqueue()`` as illustrated in the above simple example. If only the --built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as --a task is never queued on the BPF scheduler and both the local and global --DSQs are consumed automatically. -- --``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use --``scx_bpf_dispatch_vtime()`` for the priority queue. See the function --documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for --more information. -- --Where to Look --============= -- --* ``include/linux/sched/ext.h`` defines the core data structures, ops table -- and constants. -- --* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. -- The functions prefixed with ``scx_bpf_`` can be called from the BPF -- scheduler. -- --* ``tools/sched_ext/`` hosts example BPF scheduler implementations. -- -- * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example -- using a custom DSQ. -- -- * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting -- five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. -- --ABI Instability --=============== -- --The APIs provided by sched_ext to BPF schedulers programs have no stability --guarantees. This includes the ops table callbacks and constants defined in --``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in --``kernel/sched/ext.c``. -- --While we will attempt to provide a relatively stable API surface when --possible, they are subject to change without warning between kernel --versions. -diff --git b/MAINTAINERS a/MAINTAINERS -index 9fc6108503c3..2384b55682ee 100644 ---- b/MAINTAINERS -+++ a/MAINTAINERS -@@ -19132,8 +19132,6 @@ R: Ben Segall (CONFIG_CFS_BANDWIDTH) - R: Mel Gorman (CONFIG_NUMA_BALANCING) - R: Daniel Bristot de Oliveira (SCHED_DEADLINE) - R: Valentin Schneider (TOPOLOGY) --R: Tejun Heo (SCHED_EXT) --R: David Vernet (SCHED_EXT) - L: linux-kernel@vger.kernel.org - S: Maintained - T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core -@@ -19142,7 +19140,6 @@ F: include/linux/sched.h - F: include/linux/wait.h - F: include/uapi/linux/sched.h - F: kernel/sched/ --F: tools/sched_ext/ - - SCSI LIBSAS SUBSYSTEM - R: John Garry -diff --git b/arch/arm64/configs/defconfig a/arch/arm64/configs/defconfig -index f93980c09d7f..c2d530535980 100644 ---- b/arch/arm64/configs/defconfig -+++ a/arch/arm64/configs/defconfig -@@ -6,8 +6,6 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_BPF_SYSCALL=y - CONFIG_BPF_JIT=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_BSD_PROCESS_ACCT=y - CONFIG_BSD_PROCESS_ACCT_V3=y -@@ -1587,3 +1585,4 @@ CONFIG_CORESIGHT_STM=m - CONFIG_CORESIGHT_CPU_DEBUG=m - CONFIG_CORESIGHT_CTI=m - CONFIG_MEMTEST=y -+CONFIG_HMBIRD_SCHED=y -diff --git b/arch/arm64/configs/gki_defconfig a/arch/arm64/configs/gki_defconfig -index 4f756dca5e05..6c1bb94f875d 100644 ---- b/arch/arm64/configs/gki_defconfig -+++ a/arch/arm64/configs/gki_defconfig -@@ -10,8 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y - # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set - CONFIG_BPF_LSM=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_TASKSTATS=y - CONFIG_TASK_DELAY_ACCT=y -diff --git b/drivers/gpu/drm/msm/msm_drv.c a/drivers/gpu/drm/msm/msm_drv.c -index 4bd028fa7500..5825ce9d6d7b 100755 ---- b/drivers/gpu/drm/msm/msm_drv.c -+++ a/drivers/gpu/drm/msm/msm_drv.c -@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) - } - - sched_set_fifo(ev_thread->worker->task); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3); -+#endif - } - - ret = drm_vblank_init(ddev, priv->num_crtcs); -diff --git b/drivers/tty/sysrq.c a/drivers/tty/sysrq.c -index bd001445815e..dcf2e1ebf9c5 100644 ---- b/drivers/tty/sysrq.c -+++ a/drivers/tty/sysrq.c -@@ -524,7 +524,6 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { - NULL, /* P */ - NULL, /* Q */ - NULL, /* R */ -- /* S: May be registered by sched_ext for resetting */ - NULL, /* S */ - NULL, /* T */ - NULL, /* U */ -diff --git b/include/asm-generic/vmlinux.lds.h a/include/asm-generic/vmlinux.lds.h -index c45078a445c8..6c15659f874e 100755 ---- b/include/asm-generic/vmlinux.lds.h -+++ a/include/asm-generic/vmlinux.lds.h -@@ -131,7 +131,7 @@ - *(__dl_sched_class) \ - *(__rt_sched_class) \ - *(__fair_sched_class) \ -- *(__ext_sched_class) \ -+ *(__hmbird_sched_class) \ - *(__idle_sched_class) \ - __sched_class_lowest = .; - -diff --git b/include/linux/sched.h a/include/linux/sched.h -index ba49d7bd2b67..dd6b1d58af01 100755 ---- b/include/linux/sched.h -+++ a/include/linux/sched.h -@@ -73,7 +73,9 @@ struct task_delay_info; - struct task_group; - struct user_event_mm; - --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - - /* - * Task state bitmask. NOTE! These bits are also -@@ -298,11 +300,6 @@ enum { - - extern void scheduler_tick(void); - --#ifdef CONFIG_SLIM_SCHED --extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer); --extern void stop_shadow_tick_timer(void); --#endif -- - #define MAX_SCHEDULE_TIMEOUT LONG_MAX - - extern long schedule_timeout(long timeout); -@@ -1523,13 +1520,8 @@ struct task_struct { - */ - struct callback_head l1d_flush_kill; - #endif --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, unsigned long sched_prop); -- ANDROID_KABI_USE(2, struct sched_ext_entity *scx); --#else - ANDROID_KABI_RESERVE(1); - ANDROID_KABI_RESERVE(2); --#endif - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); - ANDROID_KABI_RESERVE(5); -@@ -1933,6 +1925,91 @@ static inline int task_nice(const struct task_struct *p) - return PRIO_TO_NICE((p)->static_prio); - } - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline bool task_is_top_task(struct task_struct *p) -+{ -+ return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ ->top_task_prop & TOP_TASK_BITS_MASK); -+} -+ -+static inline int get_top_task_prop(struct task_struct *p) -+{ -+ return get_hmbird_ts(p)->top_task_prop; -+} -+ -+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear) -+{ -+ if (set) -+ get_hmbird_ts(p)->top_task_prop |= set; -+ if (clear) -+ get_hmbird_ts(p)->top_task_prop &= ~clear; -+ return 0; -+} -+ -+static inline void reset_top_task_prop(struct task_struct *p) -+{ -+ get_hmbird_ts(p)->top_task_prop = 0; -+} -+ -+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ entity->sched_prop = sp; -+ } -+ return 0; -+} -+ -+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->sched_prop; -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq) -+{ -+ unsigned long new_dsq; -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq; -+ entity->sched_prop = new_dsq; -+ } -+} -+ -+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK); -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ entity->dsq_sync_ux = val; -+} -+ -+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->dsq_sync_ux; -+ else -+ return 0; -+} -+#endif - - extern int can_nice(const struct task_struct *p, const int nice); - extern int task_curr(const struct task_struct *p); -diff --git b/include/linux/sched/ext.h a/include/linux/sched/ext.h -deleted file mode 100755 -index b737a00c1373..000000000000 ---- b/include/linux/sched/ext.h -+++ /dev/null -@@ -1,647 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef _LINUX_SCHED_EXT_H --#define _LINUX_SCHED_EXT_H -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --#include -- --enum scx_consts { -- SCX_OPS_NAME_LEN = 128, -- SCX_EXIT_REASON_LEN = 128, -- SCX_EXIT_BT_LEN = 64, -- SCX_EXIT_MSG_LEN = 1024, -- -- SCX_SLICE_DFL = 20 * NSEC_PER_MSEC, -- SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */ --}; -- --/* -- * DSQ (dispatch queue) IDs are 64bit of the format: -- * -- * Bits: [63] [62 .. 0] -- * [ B] [ ID ] -- * -- * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -- * ID: 63 bit ID -- * -- * Built-in IDs: -- * -- * Bits: [63] [62] [61..32] [31 .. 0] -- * [ 1] [ L] [ R ] [ V ] -- * -- * 1: 1 for built-in DSQs. -- * L: 1 for LOCAL_ON DSQ IDs, 0 for others -- * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -- */ --enum scx_dsq_id_flags { -- SCX_DSQ_FLAG_BUILTIN = 1LLU << 63, -- SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -- -- SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0, -- SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, -- SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, -- SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, -- SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, --}; -- --enum scx_exit_type { -- SCX_EXIT_NONE, -- SCX_EXIT_DONE, -- -- SCX_EXIT_UNREG = 64, /* BPF unregistration */ -- SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -- SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ -- SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ --}; -- --/* -- * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is -- * being disabled. -- */ --struct scx_exit_info { -- /* %SCX_EXIT_* - broad category of the exit reason */ -- enum scx_exit_type type; -- /* textual representation of the above */ -- char reason[SCX_EXIT_REASON_LEN]; -- /* number of entries in the backtrace */ -- u32 bt_len; -- /* backtrace if exiting due to an error */ -- unsigned long bt[SCX_EXIT_BT_LEN]; -- /* extra message */ -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --/* sched_ext_ops.flags */ --enum scx_ops_flags { -- /* -- * Keep built-in idle tracking even if ops.update_idle() is implemented. -- */ -- SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, -- -- /* -- * By default, if there are no other task to run on the CPU, ext core -- * keeps running the current task even after its slice expires. If this -- * flag is specified, such tasks are passed to ops.enqueue() with -- * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. -- */ -- SCX_OPS_ENQ_LAST = 1LLU << 1, -- -- /* -- * An exiting task may schedule after PF_EXITING is set. In such cases, -- * bpf_task_from_pid() may not be able to find the task and if the BPF -- * scheduler depends on pid lookup for dispatching, the task will be -- * lost leading to various issues including RCU grace period stalls. -- * -- * To mask this problem, by default, unhashed tasks are automatically -- * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't -- * depend on pid lookups and wants to handle these tasks directly, the -- * following flag can be used. -- */ -- SCX_OPS_ENQ_EXITING = 1LLU << 2, -- -- /* -- * CPU cgroup knob enable flags -- */ -- SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16, /* cpu.weight */ -- -- SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | -- SCX_OPS_ENQ_LAST | -- SCX_OPS_ENQ_EXITING | -- SCX_OPS_CGROUP_KNOB_WEIGHT, --}; -- --/* argument container for ops.enable() and friends */ --struct scx_enable_args { --}; -- --/* argument container for ops->cgroup_init() */ --struct scx_cgroup_init_args { -- /* the weight of the cgroup [1..10000] */ -- u32 weight; --}; -- --enum scx_cpu_preempt_reason { -- /* next task is being scheduled by &sched_class_rt */ -- SCX_CPU_PREEMPT_RT, -- /* next task is being scheduled by &sched_class_dl */ -- SCX_CPU_PREEMPT_DL, -- /* next task is being scheduled by &sched_class_stop */ -- SCX_CPU_PREEMPT_STOP, -- /* unknown reason for SCX being preempted */ -- SCX_CPU_PREEMPT_UNKNOWN, --}; -- --/* -- * Argument container for ops->cpu_acquire(). Currently empty, but may be -- * expanded in the future. -- */ --struct scx_cpu_acquire_args {}; -- --/* argument container for ops->cpu_release() */ --struct scx_cpu_release_args { -- /* the reason the CPU was preempted */ -- enum scx_cpu_preempt_reason reason; -- -- /* the task that's going to be scheduled on the CPU */ -- struct task_struct *task; --}; -- --/** -- * struct sched_ext_ops - Operation table for BPF scheduler implementation -- * -- * Userland can implement an arbitrary scheduling policy by implementing and -- * loading operations in this table. -- */ --struct sched_ext_ops { -- /** -- * select_cpu - Pick the target CPU for a task which is being woken up -- * @p: task being woken up -- * @prev_cpu: the cpu @p was on before sleeping -- * @wake_flags: SCX_WAKE_* -- * -- * Decision made here isn't final. @p may be moved to any CPU while it -- * is getting dispatched for execution later. However, as @p is not on -- * the rq at this point, getting the eventual execution CPU right here -- * saves a small bit of overhead down the line. -- * -- * If an idle CPU is returned, the CPU is kicked and will try to -- * dispatch. While an explicit custom mechanism can be added, -- * select_cpu() serves as the default way to wake up idle CPUs. -- */ -- s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); -- -- /** -- * enqueue - Enqueue a task on the BPF scheduler -- * @p: task being enqueued -- * @enq_flags: %SCX_ENQ_* -- * -- * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() -- * or enqueue on the BPF scheduler. If not directly dispatched, the bpf -- * scheduler owns @p and if it fails to dispatch @p, the task will -- * stall. -- */ -- void (*enqueue)(struct task_struct *p, u64 enq_flags); -- -- /** -- * dequeue - Remove a task from the BPF scheduler -- * @p: task being dequeued -- * @deq_flags: %SCX_DEQ_* -- * -- * Remove @p from the BPF scheduler. This is usually called to isolate -- * the task while updating its scheduling properties (e.g. priority). -- * -- * The ext core keeps track of whether the BPF side owns a given task or -- * not and can gracefully ignore spurious dispatches from BPF side, -- * which makes it safe to not implement this method. However, depending -- * on the scheduling logic, this can lead to confusing behaviors - e.g. -- * scheduling position not being updated across a priority change. -- */ -- void (*dequeue)(struct task_struct *p, u64 deq_flags); -- -- /** -- * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs -- * @cpu: CPU to dispatch tasks for -- * @prev: previous task being switched out -- * -- * Called when a CPU's local dsq is empty. The operation should dispatch -- * one or more tasks from the BPF scheduler into the DSQs using -- * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using -- * scx_bpf_consume(). -- * -- * The maximum number of times scx_bpf_dispatch() can be called without -- * an intervening scx_bpf_consume() is specified by -- * ops.dispatch_max_batch. See the comments on top of the two functions -- * for more details. -- * -- * When not %NULL, @prev is an SCX task with its slice depleted. If -- * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in -- * @prev->scx.flags, it is not enqueued yet and will be enqueued after -- * ops.dispatch() returns. To keep executing @prev, return without -- * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. -- */ -- void (*dispatch)(s32 cpu, struct task_struct *prev); -- -- /** -- * runnable - A task is becoming runnable on its associated CPU -- * @p: task becoming runnable -- * @enq_flags: %SCX_ENQ_* -- * -- * This and the following three functions can be used to track a task's -- * execution state transitions. A task becomes ->runnable() on a CPU, -- * and then goes through one or more ->running() and ->stopping() pairs -- * as it runs on the CPU, and eventually becomes ->quiescent() when it's -- * done running on the CPU. -- * -- * @p is becoming runnable on the CPU because it's -- * -- * - waking up (%SCX_ENQ_WAKEUP) -- * - being moved from another CPU -- * - being restored after temporarily taken off the queue for an -- * attribute change. -- * -- * This and ->enqueue() are related but not coupled. This operation -- * notifies @p's state transition and may not be followed by ->enqueue() -- * e.g. when @p is being dispatched to a remote CPU. Likewise, a task -- * may be ->enqueue()'d without being preceded by this operation e.g. -- * after exhausting its slice. -- */ -- void (*runnable)(struct task_struct *p, u64 enq_flags); -- -- /** -- * running - A task is starting to run on its associated CPU -- * @p: task starting to run -- * -- * See ->runnable() for explanation on the task state notifiers. -- */ -- void (*running)(struct task_struct *p); -- -- /** -- * stopping - A task is stopping execution -- * @p: task stopping to run -- * @runnable: is task @p still runnable? -- * -- * See ->runnable() for explanation on the task state notifiers. If -- * !@runnable, ->quiescent() will be invoked after this operation -- * returns. -- */ -- void (*stopping)(struct task_struct *p, bool runnable); -- -- /** -- * quiescent - A task is becoming not runnable on its associated CPU -- * @p: task becoming not runnable -- * @deq_flags: %SCX_DEQ_* -- * -- * See ->runnable() for explanation on the task state notifiers. -- * -- * @p is becoming quiescent on the CPU because it's -- * -- * - sleeping (%SCX_DEQ_SLEEP) -- * - being moved to another CPU -- * - being temporarily taken off the queue for an attribute change -- * (%SCX_DEQ_SAVE) -- * -- * This and ->dequeue() are related but not coupled. This operation -- * notifies @p's state transition and may not be preceded by ->dequeue() -- * e.g. when @p is being dispatched to a remote CPU. -- */ -- void (*quiescent)(struct task_struct *p, u64 deq_flags); -- -- /** -- * yield - Yield CPU -- * @from: yielding task -- * @to: optional yield target task -- * -- * If @to is NULL, @from is yielding the CPU to other runnable tasks. -- * The BPF scheduler should ensure that other available tasks are -- * dispatched before the yielding task. Return value is ignored in this -- * case. -- * -- * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf -- * scheduler can implement the request, return %true; otherwise, %false. -- */ -- bool (*yield)(struct task_struct *from, struct task_struct *to); -- -- /** -- * core_sched_before - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Used by core-sched to determine the ordering between two tasks. See -- * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on -- * core-sched. -- * -- * Both @a and @b are runnable and may or may not currently be queued on -- * the BPF scheduler. Should return %true if @a should run before @b. -- * %false if there's no required ordering or @b should run before @a. -- * -- * If not specified, the default is ordering them according to when they -- * became runnable. -- */ -- bool (*core_sched_before)(struct task_struct *a,struct task_struct *b); -- -- /** -- * set_weight - Set task weight -- * @p: task to set weight for -- * @weight: new eight [1..10000] -- * -- * Update @p's weight to @weight. -- */ -- void (*set_weight)(struct task_struct *p, u32 weight); -- -- /** -- * set_cpumask - Set CPU affinity -- * @p: task to set CPU affinity for -- * @cpumask: cpumask of cpus that @p can run on -- * -- * Update @p's CPU affinity to @cpumask. -- */ -- void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask); -- -- /** -- * update_idle - Update the idle state of a CPU -- * @cpu: CPU to udpate the idle state for -- * @idle: whether entering or exiting the idle state -- * -- * This operation is called when @rq's CPU goes or leaves the idle -- * state. By default, implementing this operation disables the built-in -- * idle CPU tracking and the following helpers become unavailable: -- * -- * - scx_bpf_select_cpu_dfl() -- * - scx_bpf_test_and_clear_cpu_idle() -- * - scx_bpf_pick_idle_cpu() -- * - scx_bpf_any_idle_cpu() -- * -- * The user also must implement ops.select_cpu() as the default -- * implementation relies on scx_bpf_select_cpu_dfl(). -- * -- * If you keep the built-in idle tracking, specify the -- * %SCX_OPS_KEEP_BUILTIN_IDLE flag. -- */ -- void (*update_idle)(s32 cpu, bool idle); -- -- /** -- * cpu_acquire - A CPU is becoming available to the BPF scheduler -- * @cpu: The CPU being acquired by the BPF scheduler. -- * @args: Acquire arguments, see the struct definition. -- * -- * A CPU that was previously released from the BPF scheduler is now once -- * again under its control. -- */ -- void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); -- -- /** -- * cpu_release - A CPU is taken away from the BPF scheduler -- * @cpu: The CPU being released by the BPF scheduler. -- * @args: Release arguments, see the struct definition. -- * -- * The specified CPU is no longer under the control of the BPF -- * scheduler. This could be because it was preempted by a higher -- * priority sched_class, though there may be other reasons as well. The -- * caller should consult @args->reason to determine the cause. -- */ -- void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -- -- /** -- * cpu_online - A CPU became online -- * @cpu: CPU which just came up -- * -- * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs beforehand. -- */ -- void (*cpu_online)(s32 cpu); -- -- /** -- * cpu_offline - A CPU is going offline -- * @cpu: CPU which is going offline -- * -- * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs afterwards. -- */ -- void (*cpu_offline)(s32 cpu); -- -- /** -- * prep_enable - Prepare to enable BPF scheduling for a task -- * @p: task to prepare BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Either we're loading a BPF scheduler or a new task is being forked. -- * Prepare BPF scheduling for @p. This operation may block and can be -- * used for allocations. -- * -- * Return 0 for success, -errno for failure. An error return while -- * loading will abort loading of the BPF scheduler. During a fork, will -- * abort the specific fork. -- */ -- s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * enable - Enable BPF scheduling for a task -- * @p: task to enable BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Enable @p for BPF scheduling. @p is now in the cgroup specified for -- * the preceding prep_enable() and will start running soon. -- */ -- void (*enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * cancel_enable - Cancel prep_enable() -- * @p: task being canceled -- * @args: enable arguments, see the struct definition -- * -- * @p was prep_enable()'d but failed before reaching enable(). Undo the -- * preparation. -- */ -- void (*cancel_enable)(struct task_struct *p, -- struct scx_enable_args *args); -- -- /** -- * disable - Disable BPF scheduling for a task -- * @p: task to disable BPF scheduling for -- * -- * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. -- * Disable BPF scheduling for @p. -- */ -- void (*disable)(struct task_struct *p); -- -- /* -- * All online ops must come before ops.init(). -- */ -- -- /** -- * init - Initialize the BPF scheduler -- */ -- s32 (*init)(void); -- -- /** -- * exit - Clean up after the BPF scheduler -- * @info: Exit info -- */ -- void (*exit)(struct scx_exit_info *info); -- -- /** -- * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch -- */ -- u32 dispatch_max_batch; -- -- /** -- * flags - %SCX_OPS_* flags -- */ -- u64 flags; -- -- /** -- * timeout_ms - The maximum amount of time, in milliseconds, that a -- * runnable task should be able to wait before being scheduled. The -- * maximum timeout may not exceed the default timeout of 30 seconds. -- * -- * Defaults to the maximum allowed timeout value of 30 seconds. -- */ -- u32 timeout_ms; -- -- /** -- * name - BPF scheduler's name -- * -- * Must be a non-zero valid BPF object name including only isalnum(), -- * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the -- * BPF scheduler is enabled. -- */ -- char name[SCX_OPS_NAME_LEN]; --}; -- --/* -- * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -- * scheduler core and the BPF scheduler. See the documentation for more details. -- */ --struct scx_dispatch_q { -- raw_spinlock_t lock; -- struct list_head fifo; /* processed in dispatching order */ -- struct rb_root_cached priq; /* processed in p->scx.dsq_vtime order */ -- u32 nr; -- u64 id; -- struct llist_node free_node; -- struct rcu_head rcu; --}; -- --/* scx_entity.flags */ --enum scx_ent_flags { -- SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ -- SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -- SCX_TASK_ENQ_LOCAL = 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */ -- -- SCX_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -- SCX_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -- -- SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -- SCX_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -- -- SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ --}; -- --/* scx_entity.dsq_flags */ --enum scx_ent_dsq_flags { -- SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ --}; -- --/* -- * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from -- * everywhere and the following bits track which kfunc sets are currently -- * allowed for %current. This simple per-task tracking works because SCX ops -- * nest in a limited way. BPF will likely implement a way to allow and disallow -- * kfuncs depending on the calling context which will replace this manual -- * mechanism. See scx_kf_allow(). -- */ --enum scx_kf_mask { -- SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ -- /* all non-sleepables may be nested inside INIT and SLEEPABLE */ -- SCX_KF_INIT = 1 << 0, /* running ops.init() */ -- SCX_KF_SLEEPABLE = 1 << 1, /* other sleepable init operations */ -- /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ -- SCX_KF_CPU_RELEASE = 1 << 2, /* ops.cpu_release() */ -- /* ops.dequeue (in REST) may be nested inside DISPATCH */ -- SCX_KF_DISPATCH = 1 << 3, /* ops.dispatch() */ -- SCX_KF_ENQUEUE = 1 << 4, /* ops.enqueue() */ -- SCX_KF_REST = 1 << 5, /* other rq-locked operations */ -- -- __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | -- SCX_KF_ENQUEUE | SCX_KF_REST, -- __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_REST, --}; -- --#define RAVG_HIST_SIZE 5 --struct scx_sched_task_stats { -- u64 mark_start; -- u64 window_start; -- u32 sum; -- u32 sum_history[RAVG_HIST_SIZE]; -- int cidx; -- -- u32 demand; -- u16 demand_scaled; -- void *sdsq; --}; -- -- -- --/* -- * The following is embedded in task_struct and contains all fields necessary -- * for a task to be scheduled by SCX. -- */ --struct sched_ext_entity { -- struct scx_dispatch_q *dsq; -- struct { -- struct list_head fifo; /* dispatch order */ -- struct rb_node priq; /* p->scx.dsq_vtime order */ -- } dsq_node; -- struct list_head watchdog_node; -- u32 flags; /* protected by rq lock */ -- u32 dsq_flags; /* protected by dsq lock */ -- u32 weight; -- s32 sticky_cpu; -- s32 holding_cpu; -- u32 kf_mask; /* see scx_kf_mask above */ -- struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ -- atomic64_t ops_state; -- unsigned long runnable_at; --#ifdef CONFIG_SCHED_CORE -- u64 core_sched_at; /* see scx_prio_less() */ --#endif -- -- /* BPF scheduler modifiable fields */ -- -- /* -- * Runtime budget in nsecs. This is usually set through -- * scx_bpf_dispatch() but can also be modified directly by the BPF -- * scheduler. Automatically decreased by SCX as the task executes. On -- * depletion, a scheduling event is triggered. -- * -- * This value is cleared to zero if the task is preempted by -- * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the -- * task ran. Use p->se.sum_exec_runtime instead. -- */ -- u64 slice; -- -- /* -- * Used to order tasks when dispatching to the vtime-ordered priority -- * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime() -- * but can also be modified directly by the BPF scheduler. Modifying it -- * while a task is queued on a dsq may mangle the ordering and is not -- * recommended. -- */ -- u64 dsq_vtime; -- -- /* -- * If set, reject future sched_setscheduler(2) calls updating the policy -- * to %SCHED_EXT with -%EACCES. -- * -- * If set from ops.prep_enable() and the task's policy is already -- * %SCHED_EXT, which can happen while the BPF scheduler is being loaded -- * or by inhering the parent's policy during fork, the task's policy is -- * rejected and forcefully reverted to %SCHED_NORMAL. The number of such -- * events are reported through /sys/kernel/debug/sched_ext::nr_rejected. -- */ -- bool disallow; /* reject switching into SCX */ -- -- /* cold fields */ -- struct list_head tasks_node; -- struct task_struct *task; -- struct scx_sched_task_stats sts; --}; -- --void sched_ext_free(struct task_struct *p); -- --#else /* !CONFIG_SCHED_CLASS_EXT */ -- --static inline void sched_ext_free(struct task_struct *p) {} -- --#endif /* CONFIG_SCHED_CLASS_EXT */ --#endif /* _LINUX_SCHED_EXT_H */ -diff --git b/include/linux/sched/hmbird_proc_val.h a/include/linux/sched/hmbird_proc_val.h -new file mode 100755 -index 000000000000..e59708b16ea3 ---- /dev/null -+++ a/include/linux/sched/hmbird_proc_val.h -@@ -0,0 +1,22 @@ -+#ifndef __HMBORD_PROC_VAL_H__ -+#define __HMBORD_PROC_VAL_H__ -+ -+extern int scx_enable; -+extern int partial_enable; -+extern int cpuctrl_high_ratio; -+extern int cpuctrl_low_ratio; -+extern int slim_stats; -+extern int hmbirdcore_debug; -+extern int slim_for_app; -+extern int misfit_ds; -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+extern int cpu7_tl; -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int slim_gov_debug; -+extern int scx_gov_ctrl; -+extern int sched_ravg_window_frame_per_sec; -+ -+#endif -\ No newline at end of file -diff --git b/include/linux/sched/hmbird_version.h a/include/linux/sched/hmbird_version.h -new file mode 100755 -index 000000000000..b2843881c26f ---- /dev/null -+++ a/include/linux/sched/hmbird_version.h -@@ -0,0 +1,52 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#ifndef _OPLUS_HMBIRD_VERSION_H_ -+#define _OPLUS_HMBIRD_VERSION_H_ -+#include -+#include -+#include -+ -+enum hmbird_version { -+ HMBIRD_UNINIT, -+ HMBIRD_GKI_VERSION, -+ HMBIRD_OGKI_VERSION, -+ HMBIRD_UNKNOW_VERSION, -+}; -+ -+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT; -+ -+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type" -+ -+static inline enum hmbird_version get_hmbird_version_type(void) -+{ -+ struct device_node *np = NULL; -+ const char *hmbird_version_str = NULL; -+ if (HMBIRD_UNINIT != hmbird_version_type) -+ return hmbird_version_type; -+ np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH); -+ if (np) { -+ of_property_read_string(np, "type", &hmbird_version_str); -+ if (NULL != hmbird_version_str) { -+ if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) { -+ hmbird_version_type = HMBIRD_OGKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi"); -+ } else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) { -+ hmbird_version_type = HMBIRD_GKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi"); -+ } else { -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi"); -+ } -+ return hmbird_version_type; -+ } -+ } -+ -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION"); -+ return hmbird_version_type; -+} -+ -+#endif /*_OPLUS_HMBIRD_VERSION_H_ */ -diff --git b/include/linux/sched/task.h a/include/linux/sched/task.h -index 03d35e3eda3c..6a5f0c0d74bd 100644 ---- b/include/linux/sched/task.h -+++ a/include/linux/sched/task.h -@@ -61,8 +61,10 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); - extern void init_idle(struct task_struct *idle, int cpu); - - extern int sched_fork(unsigned long clone_flags, struct task_struct *p); --extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); --extern void sched_cancel_fork(struct task_struct *p); -+extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); -+#ifdef CONFIG_HMBIRD_SCHED -+extern void hmbird_cancel_fork(struct task_struct *p); -+#endif - extern void sched_post_fork(struct task_struct *p); - extern void sched_dead(struct task_struct *p); - -diff --git b/include/uapi/linux/sched.h a/include/uapi/linux/sched.h -index 359a14cc76a4..969716ab4320 100755 ---- b/include/uapi/linux/sched.h -+++ a/include/uapi/linux/sched.h -@@ -118,7 +118,7 @@ struct clone_args { - /* SCHED_ISO: reserved but not implemented yet */ - #define SCHED_IDLE 5 - #define SCHED_DEADLINE 6 --#define SCHED_EXT 7 -+#define SCHED_HMBIRD 7 - - /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ - #define SCHED_RESET_ON_FORK 0x40000000 -diff --git b/init/Kconfig a/init/Kconfig -index 337276cbc7f8..bf052ca532b4 100644 ---- b/init/Kconfig -+++ a/init/Kconfig -@@ -1030,10 +1030,6 @@ config RT_GROUP_SCHED - realtime bandwidth for them. - See Documentation/scheduler/sched-rt-group.rst for more information. - --config EXT_GROUP_SCHED -- bool -- depends on SCHED_CLASS_EXT && CGROUP_SCHED -- default y - endif #CGROUP_SCHED - - config SCHED_MM_CID -diff --git b/init/init_task.c a/init/init_task.c -index 69a046388995..31ceb0e469f7 100755 ---- b/init/init_task.c -+++ a/init/init_task.c -@@ -6,7 +6,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -102,9 +101,6 @@ struct task_struct init_task - #endif - #ifdef CONFIG_CGROUP_SCHED - .sched_task_group = &root_task_group, --#endif --#ifdef CONFIG_SLIM_SCHED -- .scx = NULL, - #endif - .ptraced = LIST_HEAD_INIT(init_task.ptraced), - .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), -diff --git b/kernel/Kconfig.preempt a/kernel/Kconfig.preempt -index cf22ef6107b8..ca8de6eb1e16 100755 ---- b/kernel/Kconfig.preempt -+++ a/kernel/Kconfig.preempt -@@ -133,12 +133,8 @@ config SCHED_CORE - which is the likely usage by Linux distributions, there should - be no measurable impact on performance. - --config SLIM_SCHED -- bool "slim sched" -- default n --config SCHED_CLASS_EXT -- bool "Extensible Scheduling Class" -- depends on BPF_SYSCALL && BPF_JIT -+config HMBIRD_SCHED -+ bool "hmbird Scheduling Class(base on ext scheduler)" - help - This option enables a new scheduler class sched_ext (SCX), which - allows scheduling policies to be implemented as BPF programs to -diff --git b/kernel/bpf/bpf_struct_ops_types.h a/kernel/bpf/bpf_struct_ops_types.h -index 3618769d853d..5678a9ddf817 100644 ---- b/kernel/bpf/bpf_struct_ops_types.h -+++ a/kernel/bpf/bpf_struct_ops_types.h -@@ -9,8 +9,4 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) - #include - BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) - #endif --#ifdef CONFIG_SCHED_CLASS_EXT --#include --BPF_STRUCT_OPS_TYPE(sched_ext_ops) --#endif - #endif -diff --git b/kernel/fork.c a/kernel/fork.c -index a43f97302332..7a91505b9378 100755 ---- b/kernel/fork.c -+++ a/kernel/fork.c -@@ -23,7 +23,9 @@ - #include - #include - #include --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - #include - #include - #include -@@ -999,8 +1001,9 @@ void __put_task_struct(struct task_struct *tsk) - WARN_ON(!tsk->exit_state); - WARN_ON(refcount_read(&tsk->usage)); - WARN_ON(tsk == current); -- -- sched_ext_free(tsk); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_free(tsk); -+#endif - io_uring_free(tsk); - cgroup_free(tsk); - task_numa_free(tsk, true); -@@ -2517,7 +2520,11 @@ __latent_entropy struct task_struct *copy_process( - - retval = perf_event_init_task(p, clone_flags); - if (retval) -- goto bad_fork_sched_cancel_fork; -+#ifdef CONFIG_HMBIRD_SCHED -+ goto bad_fork_hmbird_cancel_fork; -+#else -+ goto bad_fork_cleanup_policy; -+#endif - retval = audit_alloc(p); - if (retval) - goto bad_fork_cleanup_perf; -@@ -2649,9 +2656,7 @@ __latent_entropy struct task_struct *copy_process( - * cgroup specific, it unconditionally needs to place the task on a - * runqueue. - */ -- retval = sched_cgroup_fork(p, args); -- if (retval) -- goto bad_fork_cancel_cgroup; -+ sched_cgroup_fork(p, args); - - /* - * From this point on we must avoid any synchronous user-space -@@ -2697,13 +2702,13 @@ __latent_entropy struct task_struct *copy_process( - /* Don't start children in a dying pid namespace */ - if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { - retval = -ENOMEM; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* Let kill terminate clone/fork in the middle */ - if (fatal_signal_pending(current)) { - retval = -EINTR; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* No more failure paths after this point. */ -@@ -2780,11 +2785,10 @@ __latent_entropy struct task_struct *copy_process( - - return p; - --bad_fork_core_free: -+bad_fork_cancel_cgroup: - sched_core_free(p); - spin_unlock(¤t->sighand->siglock); - write_unlock_irq(&tasklist_lock); --bad_fork_cancel_cgroup: - cgroup_cancel_fork(p, args); - bad_fork_put_pidfd: - if (clone_flags & CLONE_PIDFD) { -@@ -2823,8 +2827,10 @@ __latent_entropy struct task_struct *copy_process( - audit_free(p); - bad_fork_cleanup_perf: - perf_event_free_task(p); --bad_fork_sched_cancel_fork: -- sched_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+bad_fork_hmbird_cancel_fork: -+ hmbird_cancel_fork(p); -+#endif - bad_fork_cleanup_policy: - lockdep_free_task(p); - #ifdef CONFIG_NUMA -diff --git b/kernel/sched/build_policy.c a/kernel/sched/build_policy.c -index 005025f55bea..c091539a0f60 100755 ---- b/kernel/sched/build_policy.c -+++ a/kernel/sched/build_policy.c -@@ -28,8 +28,10 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED - #include - #include -+#endif - - #include - -@@ -54,6 +56,10 @@ - #include "cputime.c" - #include "deadline.c" - --#ifdef CONFIG_SCHED_CLASS_EXT --# include "ext.c" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/hmbird_util_track.c" -+#include "hmbird/hmbird_sched_proc.c" -+#include "hmbird/hmbird_shadow_tick.c" -+#include "hmbird/hmbird.c" -+#include "hmbird/hmbird_misc.c" - #endif -diff --git b/kernel/sched/core.c a/kernel/sched/core.c -index 25d5703df992..b894417ce7df 100755 ---- b/kernel/sched/core.c -+++ a/kernel/sched/core.c -@@ -96,6 +96,12 @@ - #include "../../io_uring/io-wq.h" - #include "../smpboot.h" - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/slim.h" -+#include "hmbird/hmbird_shadow_tick.h" -+#include "hmbird.h" -+#endif -+ - #include - #include - #include -@@ -170,7 +176,6 @@ __read_mostly int scheduler_running; - - DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); - -- - /* kernel prio, less is more */ - static inline int __task_prio(const struct task_struct *p) - { -@@ -183,11 +188,10 @@ static inline int __task_prio(const struct task_struct *p) - if (p->sched_class == &idle_sched_class) - return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - --#ifdef CONFIG_SCHED_CLASS_EXT -- if (p->sched_class == &ext_sched_class) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (p->sched_class == &hmbird_sched_class) - return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ - #endif -- - return MAX_RT_PRIO + MAX_NICE; /* 119, squash fair */ - } - -@@ -217,9 +221,9 @@ static inline bool prio_less(const struct task_struct *a, - if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ - return cfs_prio_less(a, b, in_fi); - --#ifdef CONFIG_SCHED_CLASS_EXT -+#ifdef CONFIG_HMBIRD_SCHED - if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ -- return scx_prio_less(a, b, in_fi); -+ return hmbird_prio_less(a, b, in_fi); - #endif - - return false; -@@ -1279,17 +1283,26 @@ bool sched_can_stop_tick(struct rq *rq) - if (fifo_nr_running) - return true; - -+#ifdef CONFIG_HMBIRD_SCHED - /* -- * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks -+ * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks - * left. For CFS, if there's more than one we need the tick for -- * involuntary preemption. For SCX, ask. -+ * involuntary preemption. For HMBIRD, ask. - */ -- if (!scx_switched_all() && rq->nr_running > 1) -+ if (!hmbird_enabled() && rq->nr_running > 1) - return false; - -- if (scx_enabled() && !scx_can_stop_tick(rq)) -+ if (hmbird_enabled() && !hmbird_can_stop_tick(rq)) - return false; -- -+#else -+ /* -+ * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; -+ * if there's more than one we need the tick for involuntary -+ * preemption. -+ */ -+ if (rq->nr_running > 1) -+ return false; -+#endif - /* - * If there is one task and it has CFS runtime bandwidth constraints - * and it's on the cpu now we don't want to stop the tick. -@@ -2222,10 +2235,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) - } - EXPORT_SYMBOL_GPL(deactivate_task); - --struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) -+#ifdef CONFIG_HMBIRD_SCHED -+struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - { -- struct sched_change_guard cg = { -+ struct hmbird_sched_change_guard cg = { - .rq = rq, - .p = p, - .queued = task_on_rq_queued(p), -@@ -2247,7 +2261,7 @@ sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - return cg; - } - --void sched_change_guard_fini(struct sched_change_guard *cg, int flags) -+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags) - { - if (cg->queued) - enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); -@@ -2255,6 +2269,7 @@ void sched_change_guard_fini(struct sched_change_guard *cg, int flags) - set_next_task(cg->rq, cg->p); - cg->done = true; - } -+#endif - - static inline int __normal_prio(int policy, int rt_prio, int nice) - { -@@ -2320,9 +2335,9 @@ inline int task_curr(const struct task_struct *p) - * this means any call to check_class_changed() must be followed by a call to - * balance_callback(). - */ --void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio) -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) - { - if (prev_class != p->sched_class) { - if (prev_class->switched_from) -@@ -2885,6 +2900,7 @@ static void - __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - { - struct rq *rq = task_rq(p); -+ bool queued, running; - - /* - * This here violates the locking rules for affinity, since we're only -@@ -2903,9 +2919,26 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - else - lockdep_assert_held(&p->pi_lock); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->sched_class->set_cpus_allowed(p, ctx); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ -+ if (queued) { -+ /* -+ * Because __kthread_bind() calls this on blocked tasks without -+ * holding rq->lock. -+ */ -+ lockdep_assert_rq_held(rq); -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); - } -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->sched_class->set_cpus_allowed(p, ctx); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - } - - /* -@@ -3772,7 +3805,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class)) -+#else - if (unlikely(!is_cpu_allowed(p, cpu))) -+#endif - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -@@ -4891,8 +4928,9 @@ late_initcall(sched_core_sysctl_init); - */ - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { -- -+#ifdef CONFIG_HMBIRD_SCHED - int ret; -+#endif - - trace_android_rvh_sched_fork(p); - -@@ -4933,21 +4971,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_reset_on_fork = 0; - } - -- scx_pre_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ ret = hmbird_pre_fork(p); -+ if (ret) -+ goto out_cancel; - - if (dl_prio(p->prio)) { - ret = -EAGAIN; - goto out_cancel; --#ifdef CONFIG_SCHED_CLASS_EXT -- } else if (task_on_scx(p)) { -- p->sched_class = &ext_sched_class; --#endif -+ } else if (task_on_hmbird(p)) { -+ p->sched_class = &hmbird_sched_class; - } else if (rt_prio(p->prio)) { - p->sched_class = &rt_sched_class; - } else { - p->sched_class = &fair_sched_class; - } -- -+#else -+ if (dl_prio(p->prio)) -+ return -EAGAIN; -+ else if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - init_entity_runnable_average(&p->se); - trace_android_rvh_finish_prio_fork(p); - -@@ -4966,12 +5012,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - #endif - return 0; - -+#ifdef CONFIG_HMBIRD_SCHED - out_cancel: -- scx_cancel_fork(p); -+ hmbird_cancel_fork(p); - return ret; -+#endif - } - --int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) -+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - { - unsigned long flags; - -@@ -4998,20 +5046,14 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -- -- return scx_fork(p); --} -- --void sched_cancel_fork(struct task_struct *p) --{ -- scx_cancel_fork(p); - } - - void sched_post_fork(struct task_struct *p) - { - uclamp_post_fork(p); -- -- scx_post_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_post_fork(p); -+#endif - } - - unsigned long to_ratio(u64 period, u64 runtime) -@@ -5875,21 +5917,28 @@ void scheduler_tick(void) - - if (sched_feat(LATENCY_WARN) && resched_latency) - resched_latency_warn(cpu, resched_latency); -- -- scx_notify_sched_tick(); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_notify_sched_tick(); -+#endif - perf_event_task_tick(); - - if (curr->flags & PF_WQ_WORKER) - wq_worker_tick(curr); - - #ifdef CONFIG_SMP -- if (!scx_switched_all()) { -+#ifdef CONFIG_HMBIRD_SCHED -+ if (!hmbird_enabled()) { -+#endif - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); -+#ifdef CONFIG_HMBIRD_SCHED - } -+#endif - #endif - trace_android_vh_scheduler_tick(rq); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ scheduler_tick_handler(NULL, NULL); -+#endif - } - - #ifdef CONFIG_NO_HZ_FULL -@@ -6191,7 +6240,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ -+#ifdef CONFIG_HMBIRD_SCHED - for_balance_class_range(class, prev->sched_class, &idle_sched_class) { -+#else -+ for_class_range(class, prev->sched_class, &idle_sched_class) { -+#endif - if (class->balance(rq, prev, rf)) - break; - } -@@ -6209,9 +6262,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - const struct sched_class *class; - struct task_struct *p; - -- if (scx_enabled()) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (hmbird_enabled()) - goto restart; -- -+#endif - /* - * Optimization: we know that if all tasks are in the fair class we can - * call that function directly, but only if the @prev task wasn't of a -@@ -6237,13 +6291,21 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - restart: - put_prev_task_balance(rq, prev, rf); - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { - p = class->pick_next_task(rq); - if (p) { -- scx_notify_pick_next_task(rq, p, class); -+ hmbird_notify_pick_next_task(rq, p, class); - return p; - } - } -+#else -+ for_each_class(class) { -+ p = class->pick_next_task(rq); -+ if (p) -+ return p; -+ } -+#endif - - BUG(); /* The idle class should always have a runnable task. */ - } -@@ -6272,7 +6334,11 @@ static inline struct task_struct *pick_task(struct rq *rq) - const struct sched_class *class; - struct task_struct *p; - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { -+#else -+ for_each_class(class) { -+#endif - p = class->pick_task(rq); - if (p) - return p; -@@ -6916,7 +6982,9 @@ static void __sched notrace __schedule(unsigned int sched_mode) - psi_sched_switch(prev, next, !task_on_rq_queued(prev)); - - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#endif - /* Also unlocks the rq: */ - rq = context_switch(rq, prev, next, &rf); - } else { -@@ -7244,24 +7312,31 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag - } - EXPORT_SYMBOL(default_wake_function); - --void __setscheduler_prio(struct task_struct *p, int prio) -+static void __setscheduler_prio(struct task_struct *p, int prio) - { -- /* -- * After switching all rt and fair class to ext, -- * stop class is switched to ext class too. Just -- * skip it. */ -+#ifdef CONFIG_HMBIRD_SCHED -+ bool on_hmbird = task_on_hmbird(p); -+ - if (p->sched_class == &stop_sched_class) -- ;/* do nothing */ -+ ; - else if (dl_prio(prio)) - p->sched_class = &dl_sched_class; --#ifdef CONFIG_SCHED_CLASS_EXT -- else if (task_on_scx(p)) -- p->sched_class = &ext_sched_class; --#endif -+ else if (rt_prio(prio) && on_hmbird) -+ p->sched_class = &hmbird_sched_class; - else if (rt_prio(prio)) - p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; - else - p->sched_class = &fair_sched_class; -+#else -+ if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - - p->prio = prio; - trace_android_rvh_setscheduler_prio(p); -@@ -7297,7 +7372,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - */ - void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - { -- int prio, oldprio, queue_flag = -+ int prio, oldprio, queued, running, queue_flag = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *prev_class; - struct rq_flags rf; -@@ -7360,42 +7435,52 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - queue_flag &= ~DEQUEUE_MOVE; - - prev_class = p->sched_class; -- SCHED_CHANGE_BLOCK(rq, p, queue_flag) { -- /* -- * Boosting condition are: -- * 1. -rt task is running and holds mutex A -- * --> -dl task blocks on mutex A -- * -- * 2. -dl task is running and holds mutex A -- * --> -dl task blocks on mutex A and could preempt the -- * running task -- */ -- if (dl_prio(prio)) { -- if (!dl_prio(p->normal_prio) || -- (pi_task && dl_prio(pi_task->prio) && -- dl_entity_preempt(&pi_task->dl, &p->dl))) { -- p->dl.pi_se = pi_task->dl.pi_se; -- queue_flag |= ENQUEUE_REPLENISH; -- } else { -- p->dl.pi_se = &p->dl; -- } -- } else if (rt_prio(prio)) { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- if (oldprio < prio) -- queue_flag |= ENQUEUE_HEAD; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flag); -+ if (running) -+ put_prev_task(rq, p); -+ -+ /* -+ * Boosting condition are: -+ * 1. -rt task is running and holds mutex A -+ * --> -dl task blocks on mutex A -+ * -+ * 2. -dl task is running and holds mutex A -+ * --> -dl task blocks on mutex A and could preempt the -+ * running task -+ */ -+ if (dl_prio(prio)) { -+ if (!dl_prio(p->normal_prio) || -+ (pi_task && dl_prio(pi_task->prio) && -+ dl_entity_preempt(&pi_task->dl, &p->dl))) { -+ p->dl.pi_se = pi_task->dl.pi_se; -+ queue_flag |= ENQUEUE_REPLENISH; - } else { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- else if (rt_prio(oldprio)) -- p->rt.timeout = 0; -- else if (!task_has_idle_policy(p)) -- reweight_task(p, prio - MAX_RT_PRIO); -+ p->dl.pi_se = &p->dl; - } -- -- __setscheduler_prio(p, prio); -+ } else if (rt_prio(prio)) { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ if (oldprio < prio) -+ queue_flag |= ENQUEUE_HEAD; -+ } else { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ else if (rt_prio(oldprio)) -+ p->rt.timeout = 0; -+ else if (!task_has_idle_policy(p)) -+ reweight_task(p, prio - MAX_RT_PRIO); - } - -+ __setscheduler_prio(p, prio); -+ -+ if (queued) -+ enqueue_task(rq, p, queue_flag); -+ if (running) -+ set_next_task(rq, p); -+ - check_class_changed(rq, p, prev_class, oldprio); - out_unlock: - /* Avoid rq from going away on us: */ -@@ -7416,7 +7501,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - - void set_user_nice(struct task_struct *p, long nice) - { -- bool allowed = false; -+ bool queued, running, allowed = false; - int old_prio; - struct rq_flags rf; - struct rq *rq; -@@ -7445,13 +7530,22 @@ void set_user_nice(struct task_struct *p, long nice) - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); -+ if (running) -+ put_prev_task(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->static_prio = NICE_TO_PRIO(nice); -- set_load_weight(p, true); -- old_prio = p->prio; -- p->prio = effective_prio(p); -- } -+ p->static_prio = NICE_TO_PRIO(nice); -+ set_load_weight(p, true); -+ old_prio = p->prio; -+ p->prio = effective_prio(p); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - - /* - * If the task increased its priority or is running and -@@ -7868,7 +7962,7 @@ static int __sched_setscheduler(struct task_struct *p, - bool user, bool pi) - { - int oldpolicy = -1, policy = attr->sched_policy; -- int retval, oldprio, newprio; -+ int retval, oldprio, newprio, queued, running; - const struct sched_class *prev_class; - struct balance_callback *head; - struct rq_flags rf; -@@ -7876,7 +7970,9 @@ static int __sched_setscheduler(struct task_struct *p, - int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct rq *rq; - bool cpuset_locked = false; -- -+#ifdef CONFIG_HMBIRD_SCHED -+ unsigned long flags; -+#endif - - /* The pi code expects interrupts enabled */ - BUG_ON(pi && in_interrupt()); -@@ -7942,7 +8038,9 @@ static int __sched_setscheduler(struct task_struct *p, - * To be able to change p->policy safely, the appropriate - * runqueue lock must be held. - */ -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+#endif - rq = task_rq_lock(p, &rf); - update_rq_clock(rq); - -@@ -7954,10 +8052,11 @@ static int __sched_setscheduler(struct task_struct *p, - goto unlock; - } - -- retval = scx_check_setscheduler(p, policy); -+#ifdef CONFIG_HMBIRD_SCHED -+ retval = hmbird_check_setscheduler(p, policy); - if (retval) - goto unlock; -- -+#endif - /* - * If not changing anything there's no need to proceed further, - * but store a possible modification of reset_on_fork. -@@ -8014,7 +8113,9 @@ static int __sched_setscheduler(struct task_struct *p, - if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { - policy = oldpolicy = -1; - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - goto recheck; -@@ -8047,16 +8148,23 @@ static int __sched_setscheduler(struct task_struct *p, - queue_flags &= ~DEQUEUE_MOVE; - } - -- SCHED_CHANGE_BLOCK(rq, p, queue_flags) { -- prev_class = p->sched_class; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flags); -+ if (running) -+ put_prev_task(rq, p); -+ -+ prev_class = p->sched_class; - -- if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -- __setscheduler_params(p, attr); -- __setscheduler_prio(p, newprio); -- trace_android_rvh_setscheduler(p); -- } -- __setscheduler_uclamp(p, attr); -+ if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -+ __setscheduler_params(p, attr); -+ __setscheduler_prio(p, newprio); -+ trace_android_rvh_setscheduler(p); -+ } -+ __setscheduler_uclamp(p, attr); - -+ if (queued) { - /* - * We enqueue to tail when the priority of a task is - * increased (user space view). -@@ -8064,7 +8172,10 @@ static int __sched_setscheduler(struct task_struct *p, - if (oldprio < p->prio) - queue_flags |= ENQUEUE_HEAD; - -+ enqueue_task(rq, p, queue_flags); - } -+ if (running) -+ set_next_task(rq, p); - - check_class_changed(rq, p, prev_class, oldprio); - -@@ -8072,7 +8183,9 @@ static int __sched_setscheduler(struct task_struct *p, - preempt_disable(); - head = splice_balance_callbacks(rq); - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (pi) { - if (cpuset_locked) - cpuset_unlock(); -@@ -8087,7 +8200,9 @@ static int __sched_setscheduler(struct task_struct *p, - - unlock: - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - return retval; -@@ -8785,6 +8900,9 @@ static void do_sched_yield(void) - long skip = 0; - - trace_android_rvh_before_do_sched_yield(&skip); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_skip_yield(&skip); -+#endif - if (skip) - return; - -@@ -9309,7 +9427,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - break; - } -@@ -9337,7 +9457,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - } - return ret; -@@ -9638,15 +9760,25 @@ int migrate_task_to(struct task_struct *p, int target_cpu) - */ - void sched_setnuma(struct task_struct *p, int nid) - { -+ bool queued, running; - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) { -- p->numa_preferred_nid = nid; -- } -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE); -+ if (running) -+ put_prev_task(rq, p); - -+ p->numa_preferred_nid = nid; -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - task_rq_unlock(rq, p, &rf); - } - #endif /* CONFIG_NUMA_BALANCING */ -@@ -10212,17 +10344,23 @@ void __init sched_init(void) - int i; - - /* Make sure the linker didn't screw up */ -+#ifdef CONFIG_HMBIRD_SCHED - #ifdef CONFIG_SMP -- BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+#endif -+ WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class)); -+#else -+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 || -+ &fair_sched_class != &rt_sched_class + 1 || -+ &rt_sched_class != &dl_sched_class + 1); -+#ifdef CONFIG_SMP -+ BUG_ON(&dl_sched_class != &stop_sched_class + 1); - #endif -- BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); -- BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); -- BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); --#ifdef CONFIG_SCHED_CLASS_EXT -- BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); -- BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); - #endif -- - wait_bit_init(); - - #ifdef CONFIG_FAIR_GROUP_SCHED -@@ -10392,8 +10530,9 @@ void __init sched_init(void) - balance_push_set(smp_processor_id(), false); - #endif - init_sched_fair_class(); -- init_sched_ext_class(); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ init_sched_hmbird_class(); -+#endif - psi_init(); - - init_uclamp(); -@@ -10863,6 +11002,11 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) - struct task_group *tg = css_tg(css); - struct task_group *parent = css_tg(css->parent); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_tg_online(tg); -+#endif -+ -+ - if (parent) - sched_online_group(tg, parent); - -@@ -10912,13 +11056,77 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) - } - #endif - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 dl) -+{ -+ int i; -+ -+ for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) { -+ if (dl < HMBIRD_BPF_DSQS_DEADLINE[i]) -+ break; -+ } -+ -+ i = max_t(int, i-1, MIN_CGROUP_DL_IDX); -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return 0; -+ update_cgroup_ids_table(css->cgroup->kn->id, i); -+ -+ return 0; -+} -+ -+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ u8 i; -+ -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX]; -+ i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1); -+ if (i < 0) { -+ pr_err(" <%s> i is %d, less than 0, name is %s\n", -+ __func__, i, css->cgroup->kn->name); -+ i = DEFAULT_CGROUP_DL_IDX; -+ } -+ -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[i]; -+} -+ -+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css, -+ struct task_struct *p, int prio) -+{ -+ if (!css || !rt_prio(prio)) -+ return; -+ -+ if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) { -+ if (!strcmp(css->cgroup->kn->name, "display")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3); -+ else if (!strcmp(css->cgroup->kn->name, "touch")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2); -+ else if (!strcmp(css->cgroup->kn->name, "multimedia")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+ } -+} -+#endif -+ - static void cpu_cgroup_attach(struct cgroup_taskset *tset) - { - struct task_struct *task; - struct cgroup_subsys_state *css; - - cgroup_taskset_for_each(task, css, tset) { -- -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_change_rt_sched_prop(css, task, task->prio); -+#endif - sched_move_task(task); - } - -@@ -11600,7 +11808,13 @@ static struct cftype cpu_legacy_files[] = { - .write_u64 = cpu_uclamp_ls_write_u64, - }, - #endif -- -+#ifdef CONFIG_HMBIRD_SCHED -+ { -+ .name = "hmbird.deadline", -+ .read_u64 = cgroup_read_hmbird_deadline, -+ .write_u64 = cgroup_write_hmbird_deadline, -+ }, -+#endif - { } /* Terminate */ - }; - -@@ -11649,7 +11863,6 @@ static int cpu_local_stat_show(struct seq_file *sf, - } - - #ifdef CONFIG_FAIR_GROUP_SCHED -- - static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, - struct cftype *cft) - { -diff --git b/kernel/sched/debug.c a/kernel/sched/debug.c -index a6c99df9edf9..e09904f4d6e5 100755 ---- b/kernel/sched/debug.c -+++ a/kernel/sched/debug.c -@@ -375,9 +375,8 @@ static __init int sched_init_debug(void) - #endif - - debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); -- --#ifdef CONFIG_SCHED_CLASS_EXT -- debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops); -+#ifdef CONFIG_HMBIRD_SCHED -+ debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops); - #endif - return 0; - } -@@ -1006,9 +1005,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - SEQ_printf(m, - "---------------------------------------------------------" - "----------\n"); --#ifdef CONFIG_SLIM_SCHED -- SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop); --#endif - - #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) - #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) -@@ -1104,8 +1100,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - } else if (fair_policy(p->policy)) { - P(se.slice); - } --#ifdef CONFIG_SCHED_CLASS_EXT -- __PS("ext.enabled", p->sched_class == &ext_sched_class); -+#ifdef CONFIG_HMBIRD_SCHED -+ __PS("hmbird.enabled", p->sched_class == &hmbird_sched_class); -+ __PS("sched_prop", get_hmbird_ts(p)->sched_prop); -+ __PS("top_task_prop", get_hmbird_ts(p)->top_task_prop); - #endif - #undef PN_SCHEDSTAT - #undef P_SCHEDSTAT -diff --git b/kernel/sched/ext.c a/kernel/sched/ext.c -deleted file mode 100755 -index 4845d441df2b..000000000000 ---- b/kernel/sched/ext.c -+++ /dev/null -@@ -1,3978 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -- --enum scx_internal_consts { -- SCX_NR_ONLINE_OPS = SCX_OP_IDX(init), -- SCX_DSP_DFL_MAX_BATCH = 32, -- SCX_DSP_MAX_LOOPS = 32, -- SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, --}; -- --enum scx_ops_enable_state { -- SCX_OPS_PREPPING, -- SCX_OPS_ENABLING, -- SCX_OPS_ENABLED, -- SCX_OPS_DISABLING, -- SCX_OPS_DISABLED, --}; -- --static inline struct task_group *css_tg(struct cgroup_subsys_state *css) --{ -- return css ? container_of(css, struct task_group, css) : NULL; --} -- --/* -- * sched_ext_entity->ops_state -- * -- * Used to track the task ownership between the SCX core and the BPF scheduler. -- * State transitions look as follows: -- * -- * NONE -> QUEUEING -> QUEUED -> DISPATCHING -- * ^ | | -- * | v v -- * \-------------------------------/ -- * -- * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -- * sites for explanations on the conditions being waited upon and why they are -- * safe. Transitions out of them into NONE or QUEUED must store_release and the -- * waiters should load_acquire. -- * -- * Tracking scx_ops_state enables sched_ext core to reliably determine whether -- * any given task can be dispatched by the BPF scheduler at all times and thus -- * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -- * to try to dispatch any task anytime regardless of its state as the SCX core -- * can safely reject invalid dispatches. -- */ --enum scx_ops_state { -- SCX_OPSS_NONE, /* owned by the SCX core */ -- SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -- SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ -- SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ -- -- /* -- * QSEQ brands each QUEUED instance so that, when dispatch races -- * dequeue/requeue, the dispatcher can tell whether it still has a claim -- * on the task being dispatched. -- */ -- SCX_OPSS_QSEQ_SHIFT = 2, -- SCX_OPSS_STATE_MASK = (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1, -- SCX_OPSS_QSEQ_MASK = ~SCX_OPSS_STATE_MASK, --}; -- --/* -- * During exit, a task may schedule after losing its PIDs. When disabling the -- * BPF scheduler, we need to be able to iterate tasks in every state to -- * guarantee system safety. Maintain a dedicated task list which contains every -- * task between its fork and eventual free. -- */ --static DEFINE_SPINLOCK(scx_tasks_lock); --static LIST_HEAD(scx_tasks); -- --/* ops enable/disable */ --static struct kthread_worker *scx_ops_helper; --static DEFINE_MUTEX(scx_ops_enable_mutex); --DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); --DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); --static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); --static bool scx_switch_all_req; --static bool scx_switching_all; --DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -- --static struct sched_ext_ops scx_ops; --static bool scx_warned_zero_slice; -- --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); --DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); --static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); -- --struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] = -- { [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT }; -- --static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE); --static struct scx_exit_info scx_exit_info; -- --static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0); -- --/* -- * The maximum amount of time in jiffies that a task may be runnable without -- * being scheduled on a CPU. If this timeout is exceeded, it will trigger -- * scx_ops_error(). -- */ --unsigned long scx_watchdog_timeout; -- --/* -- * The last time the delayed work was run. This delayed work relies on -- * ksoftirqd being able to run to service timer interrupts, so it's possible -- * that this work itself could get wedged. To account for this, we check that -- * it's not stalled in the timer tick, and trigger an error if it is. -- */ --unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; -- --static struct delayed_work scx_watchdog_work; -- --/* idle tracking */ --#ifdef CONFIG_SMP --#ifdef CONFIG_CPUMASK_OFFSTACK --#define CL_ALIGNED_IF_ONSTACK --#else --#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp --#endif -- --static struct { -- cpumask_var_t cpu; -- cpumask_var_t smt; --} idle_masks CL_ALIGNED_IF_ONSTACK; -- --static bool __cacheline_aligned_in_smp scx_has_idle_cpus; --#endif /* CONFIG_SMP */ -- --/* for %SCX_KICK_WAIT */ --static u64 __percpu *scx_kick_cpus_pnt_seqs; -- --/* -- * Direct dispatch marker. -- * -- * Non-NULL values are used for direct dispatch from enqueue path. A valid -- * pointer points to the task currently being enqueued. An ERR_PTR value is used -- * to indicate that direct dispatch has already happened. -- */ --static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); -- --/* dispatch queues */ --static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; -- --static LLIST_HEAD(dsqs_to_free); -- --/* dispatch buf */ --struct scx_dsp_buf_ent { -- struct task_struct *task; -- u64 qseq; -- u64 dsq_id; -- u64 enq_flags; --}; -- --static u32 scx_dsp_max_batch; --static struct scx_dsp_buf_ent __percpu *scx_dsp_buf; -- --struct scx_dsp_ctx { -- struct rq *rq; -- struct rq_flags *rf; -- u32 buf_cursor; -- u32 nr_tasks; --}; -- --static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx); -- --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags); --void scx_bpf_kick_cpu(s32 cpu, u64 flags); -- --struct scx_task_iter { -- struct sched_ext_entity cursor; -- struct task_struct *locked; -- struct rq *rq; -- struct rq_flags rf; --}; -- --#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) -- --/* if the highest set bit is N, return a mask with bits [N+1, 31] set */ --static u32 higher_bits(u32 flags) --{ -- return ~((1 << fls(flags)) - 1); --} -- --/* return the mask with only the highest bit set */ --static u32 highest_bit(u32 flags) --{ -- int bit = fls(flags); -- return bit ? 1 << (bit - 1) : 0; --} -- --/* -- * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX -- * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate -- * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check -- * whether it's running from an allowed context. -- * -- * @mask is constant, always inline to cull the mask calculations. -- */ --static __always_inline void scx_kf_allow(u32 mask) --{ -- /* nesting is allowed only in increasing scx_kf_mask order */ -- WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask, -- "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n", -- current->scx->kf_mask, mask); -- current->scx->kf_mask |= mask; --} -- --static void scx_kf_disallow(u32 mask) --{ -- current->scx->kf_mask &= ~mask; --} -- --#define SCX_CALL_OP(mask, op, args...) \ --do { \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- scx_ops.op(args); \ -- } \ --} while (0) -- --#define SCX_CALL_OP_RET(mask, op, args...) \ --({ \ -- __typeof__(scx_ops.op(args)) __ret; \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- __ret = scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- __ret = scx_ops.op(args); \ -- } \ -- __ret; \ --}) -- --/* -- * Some kfuncs are allowed only on the tasks that are subjects of the -- * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such -- * restrictions, the following SCX_CALL_OP_*() variants should be used when -- * invoking scx_ops operations that take task arguments. These can only be used -- * for non-nesting operations due to the way the tasks are tracked. -- * -- * kfuncs which can only operate on such tasks can in turn use -- * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on -- * the specific task. -- */ --#define SCX_CALL_OP_TASK(mask, op, task, args...) \ --do { \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- SCX_CALL_OP(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ --} while (0) -- --#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ --({ \ -- __typeof__(scx_ops.op(task, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- __ret; \ --}) -- --#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ --({ \ -- __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task0; \ -- current->scx->kf_tasks[1] = task1; \ -- __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- current->scx->kf_tasks[1] = NULL; \ -- __ret; \ --}) -- --//#include "./slim_walt.c" -- --/* @mask is constant, always inline to cull unnecessary branches */ --static __always_inline bool scx_kf_allowed(u32 mask) --{ -- if (unlikely(!(current->scx->kf_mask & mask))) { -- scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", -- mask, current->scx->kf_mask); -- return false; -- } -- -- if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) && -- in_interrupt())) { -- scx_ops_error("sleepable kfunc called from non-sleepable context"); -- return false; -- } -- -- /* -- * Enforce nesting boundaries. e.g. A kfunc which can be called from -- * DISPATCH must not be called if we're running DEQUEUE which is nested -- * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE -- * boundary thanks to the above in_interrupt() check. -- */ -- if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && -- (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { -- scx_ops_error("cpu_release kfunc called from a nested operation"); -- return false; -- } -- -- if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && -- (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) { -- scx_ops_error("dispatch kfunc called from a nested operation"); -- return false; -- } -- -- return true; --} -- --/* see SCX_CALL_OP_TASK() */ --static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, -- struct task_struct *p) --{ -- if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED)) -- return false; -- -- if (unlikely((p != current->scx->kf_tasks[0] && -- p != current->scx->kf_tasks[1]))) { -- scx_ops_error("called on a task not being operated on"); -- return false; -- } -- -- return true; --} -- --/** -- * scx_task_iter_init - Initialize a task iterator -- * @iter: iterator to init -- * -- * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized, -- * @iter must eventually be exited with scx_task_iter_exit(). -- * -- * scx_tasks_lock may be released between this and the first next() call or -- * between any two next() calls. If scx_tasks_lock is released between two -- * next() calls, the caller is responsible for ensuring that the task being -- * iterated remains accessible either through RCU read lock or obtaining a -- * reference count. -- * -- * All tasks which existed when the iteration started are guaranteed to be -- * visited as long as they still exist. -- */ --static void scx_task_iter_init(struct scx_task_iter *iter) --{ -- lockdep_assert_held(&scx_tasks_lock); -- -- iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; -- list_add(&iter->cursor.tasks_node, &scx_tasks); -- iter->locked = NULL; --} -- --/** -- * scx_task_iter_exit - Exit a task iterator -- * @iter: iterator to exit -- * -- * Exit a previously initialized @iter. Must be called with scx_tasks_lock held. -- * If the iterator holds a task's rq lock, that rq lock is released. See -- * scx_task_iter_init() for details. -- */ --static void scx_task_iter_exit(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- if (list_empty(cursor)) -- return; -- -- list_del_init(cursor); --} -- --/** -- * scx_task_iter_next - Next task -- * @iter: iterator to walk -- * -- * Visit the next task. See scx_task_iter_init() for details. -- */ --static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- struct sched_ext_entity *pos; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- list_for_each_entry(pos, cursor, tasks_node) { -- if (&pos->tasks_node == &scx_tasks) -- return NULL; -- if (!(pos->flags & SCX_TASK_CURSOR)) { -- list_move(cursor, &pos->tasks_node); -- return pos->task; -- } -- } -- -- /* can't happen, should always terminate at scx_tasks above */ -- BUG(); --} -- --/** -- * scx_task_iter_next_filtered - Next non-idle task -- * @iter: iterator to walk -- * -- * Visit the next non-idle task. See scx_task_iter_init() for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- while ((p = scx_task_iter_next(iter))) { -- if (!is_idle_task(p)) -- return p; -- } -- return NULL; --} -- --/** -- * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked -- * @iter: iterator to walk -- * -- * Visit the next non-idle task with its rq lock held. See scx_task_iter_init() -- * for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered_locked(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- p = scx_task_iter_next_filtered(iter); -- if (!p) -- return NULL; -- -- iter->rq = task_rq_lock(p, &iter->rf); -- iter->locked = p; -- return p; --} -- --static enum scx_ops_enable_state scx_ops_enable_state(void) --{ -- return atomic_read(&scx_ops_enable_state_var); --} -- --static enum scx_ops_enable_state --scx_ops_set_enable_state(enum scx_ops_enable_state to) --{ -- return atomic_xchg(&scx_ops_enable_state_var, to); --} -- --static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, -- enum scx_ops_enable_state from) --{ -- int from_v = from; -- -- return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); --} -- --static bool scx_ops_disabling(void) --{ -- return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING); --} -- --/** -- * wait_ops_state - Busy-wait the specified ops state to end -- * @p: target task -- * @opss: state to wait the end of -- * -- * Busy-wait for @p to transition out of @opss. This can only be used when the -- * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also -- * has load_acquire semantics to ensure that the caller can see the updates made -- * in the enqueueing and dispatching paths. -- */ --static void wait_ops_state(struct task_struct *p, u64 opss) --{ -- do { -- cpu_relax(); -- } while (atomic64_read_acquire(&p->scx->ops_state) == opss); --} -- --/** -- * ops_cpu_valid - Verify a cpu number -- * @cpu: cpu number which came from a BPF ops -- * -- * @cpu is a cpu number which came from the BPF scheduler and can be any value. -- * Verify that it is in range and one of the possible cpus. -- */ --static bool ops_cpu_valid(s32 cpu) --{ -- return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); --} -- --/** -- * ops_sanitize_err - Sanitize a -errno value -- * @ops_name: operation to blame on failure -- * @err: -errno value to sanitize -- * -- * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return -- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can -- * cause misbehaviors. For an example, a large negative return from -- * ops.prep_enable() triggers an oops when passed up the call chain because the -- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is -- * handled as a pointer. -- */ --static int ops_sanitize_err(const char *ops_name, s32 err) --{ -- if (err < 0 && err >= -MAX_ERRNO) -- return err; -- -- scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); -- return -EPROTO; --} -- --/** -- * touch_core_sched - Update timestamp used for core-sched task ordering -- * @rq: rq to read clock from, must be locked -- * @p: task to update the timestamp for -- * -- * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to -- * implement global or local-DSQ FIFO ordering for core-sched. Should be called -- * when a task becomes runnable and its turn on the CPU ends (e.g. slice -- * exhaustion). -- */ --static void touch_core_sched(struct rq *rq, struct task_struct *p) --{ --#ifdef CONFIG_SCHED_CORE -- /* -- * It's okay to update the timestamp spuriously. Use -- * sched_core_disabled() which is cheaper than enabled(). -- */ -- if (!sched_core_disabled()) -- p->scx->core_sched_at = rq_clock_task(rq); --#endif --} -- --/** -- * touch_core_sched_dispatch - Update core-sched timestamp on dispatch -- * @rq: rq to read clock from, must be locked -- * @p: task being dispatched -- * -- * If the BPF scheduler implements custom core-sched ordering via -- * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO -- * ordering within each local DSQ. This function is called from dispatch paths -- * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect. -- */ --static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- assert_clock_updated(rq); -- --#ifdef CONFIG_SCHED_CORE -- if (SCX_HAS_OP(core_sched_before)) -- touch_core_sched(rq, p); --#endif --} -- --static void update_curr_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- u64 now = rq_clock_task(rq); -- u64 delta_exec; -- -- if (time_before_eq64(now, curr->se.exec_start)) -- return; -- -- delta_exec = now - curr->se.exec_start; -- curr->se.exec_start = now; -- curr->se.sum_exec_runtime += delta_exec; -- account_group_exec_runtime(curr, delta_exec); -- cgroup_account_cputime(curr, delta_exec); -- -- if (curr->scx->slice != SCX_SLICE_INF) { -- curr->scx->slice -= min(curr->scx->slice, delta_exec); -- if (!curr->scx->slice) -- touch_core_sched(rq, curr); -- } --} -- --static bool scx_dsq_priq_less(struct rb_node *node_a, -- const struct rb_node *node_b) --{ -- const struct sched_ext_entity *a = -- container_of(node_a, struct sched_ext_entity, dsq_node.priq); -- const struct sched_ext_entity *b = -- container_of(node_b, struct sched_ext_entity, dsq_node.priq); -- -- return time_before64(a->dsq_vtime, b->dsq_vtime); --} -- --static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, -- u64 enq_flags) --{ -- bool is_local = dsq->id == SCX_DSQ_LOCAL; -- -- WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo)); -- WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq)); -- -- if (!is_local) { -- raw_spin_lock(&dsq->lock); -- if (unlikely(dsq->id == SCX_DSQ_INVALID)) { -- scx_ops_error("attempting to dispatch to a destroyed dsq"); -- /* fall back to the global dsq */ -- raw_spin_unlock(&dsq->lock); -- dsq = &scx_dsq_global; -- raw_spin_lock(&dsq->lock); -- } -- } -- -- if (enq_flags & SCX_ENQ_DSQ_PRIQ) { -- p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; -- rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq, -- scx_dsq_priq_less); -- } else { -- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) -- list_add(&p->scx->dsq_node.fifo, &dsq->fifo); -- else -- list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo); -- } -- dsq->nr++; -- p->scx->dsq = dsq; -- -- /* -- * We're transitioning out of QUEUEING or DISPATCHING. store_release to -- * match waiters' load_acquire. -- */ -- if (enq_flags & SCX_ENQ_CLEAR_OPSS) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- if (is_local) { -- struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq); -- struct rq *rq = scx->rq; -- bool preempt = false; -- -- if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && -- rq->curr->sched_class == &ext_sched_class) { -- rq->curr->scx->slice = 0; -- preempt = true; -- } -- -- if (preempt || sched_class_above(&ext_sched_class, -- rq->curr->sched_class)) -- resched_curr(rq); -- } else { -- raw_spin_unlock(&dsq->lock); -- } --} -- --static void task_unlink_from_dsq(struct task_struct *p, -- struct scx_dispatch_q *dsq) --{ -- if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { -- rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; -- } else { -- list_del_init(&p->scx->dsq_node.fifo); -- } -- --} -- --static bool task_linked_on_dsq(struct task_struct *p) --{ -- return !list_empty(&p->scx->dsq_node.fifo) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq); --} -- --static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p) --{ -- struct scx_dispatch_q *dsq = p->scx->dsq; -- bool is_local = dsq == &scx_rq->local_dsq; -- -- if (!dsq) { -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- /* -- * When dispatching directly from the BPF scheduler to a local -- * DSQ, the task isn't associated with any DSQ but -- * @p->scx->holding_cpu may be set under the protection of -- * %SCX_OPSS_DISPATCHING. -- */ -- if (p->scx->holding_cpu >= 0) -- p->scx->holding_cpu = -1; -- return; -- } -- -- if (!is_local) -- raw_spin_lock(&dsq->lock); -- -- /* -- * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node -- * can't change underneath us. -- */ -- if (p->scx->holding_cpu < 0) { -- /* @p must still be on @dsq, dequeue */ -- WARN_ON_ONCE(!task_linked_on_dsq(p)); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- } else { -- /* -- * We're racing against dispatch_to_local_dsq() which already -- * removed @p from @dsq and set @p->scx->holding_cpu. Clear the -- * holding_cpu which tells dispatch_to_local_dsq() that it lost -- * the race. -- */ -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- p->scx->holding_cpu = -1; -- } -- p->scx->dsq = NULL; -- -- if (!is_local) -- raw_spin_unlock(&dsq->lock); --} -- --static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) --{ -- lockdep_assert(rcu_read_lock_any_held()); -- -- return &scx_dsq_global; --} -- --static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, -- struct task_struct *p) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id == SCX_DSQ_LOCAL) -- return &rq->scx->local_dsq; -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", -- dsq_id, p->comm, p->pid); -- return &scx_dsq_global; -- } -- -- return dsq; --} -- --static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- -- /* @p must match the task which is being enqueued */ -- if (unlikely(p != ddsp_task)) { -- if (IS_ERR(ddsp_task)) -- scx_ops_error("%s[%d] already direct-dispatched", -- p->comm, p->pid); -- else -- scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]", -- ddsp_task->comm, ddsp_task->pid, -- p->comm, p->pid); -- return; -- } -- -- /* -- * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because -- * dispatching to the local DSQ of a different CPU requires unlocking -- * the current rq which isn't allowed in the enqueue path. Use -- * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL. -- */ -- if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) { -- scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch"); -- return; -- } -- -- touch_core_sched_dispatch(task_rq(p), p); -- -- dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- -- /* -- * Mark that dispatch already happened by spoiling direct_dispatch_task -- * with a non-NULL value which can never match a valid task pointer. -- */ -- __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); --} -- --static bool test_rq_online(struct rq *rq) --{ --#ifdef CONFIG_SMP -- return rq->online; --#else -- return true; --#endif --} -- --static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- int sticky_cpu) --{ -- struct task_struct **ddsp_taskp; -- u64 qseq; -- -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- if (p->scx->flags & SCX_TASK_ENQ_LOCAL) { -- enq_flags |= SCX_ENQ_LOCAL; -- p->scx->flags &= ~SCX_TASK_ENQ_LOCAL; -- } -- -- /* rq migration */ -- if (sticky_cpu == cpu_of(rq)) -- goto local_norefill; -- -- /* -- * If !rq->online, we already told the BPF scheduler that the CPU is -- * offline. We're just trying to on/offline the CPU. Don't bother the -- * BPF scheduler. -- */ -- if (unlikely(!test_rq_online(rq))) -- goto local; -- -- /* see %SCX_OPS_ENQ_EXITING */ -- if (!static_branch_unlikely(&scx_ops_enq_exiting) && -- unlikely(p->flags & PF_EXITING)) -- goto local; -- -- /* see %SCX_OPS_ENQ_LAST */ -- if (!static_branch_unlikely(&scx_ops_enq_last) && -- (enq_flags & SCX_ENQ_LAST)) -- goto local; -- -- if (!SCX_HAS_OP(enqueue)) { -- if (enq_flags & SCX_ENQ_LOCAL) -- goto local; -- else -- goto global; -- } -- -- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ -- qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; -- -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq); -- -- ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); -- WARN_ON_ONCE(*ddsp_taskp); -- *ddsp_taskp = p; -- -- SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); -- -- /* -- * If not directly dispatched, QUEUEING isn't clear yet and dispatch or -- * dequeue may be waiting. The store_release matches their load_acquire. -- */ -- if (*ddsp_taskp == p) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq); -- *ddsp_taskp = NULL; -- return; -- --local: -- /* -- * For task-ordering, slice refill must be treated as implying the end -- * of the current slice. Otherwise, the longer @p stays on the CPU, the -- * higher priority it becomes from scx_prio_less()'s POV. -- */ -- touch_core_sched(rq, p); -- p->scx->slice = SCX_SLICE_DFL; --local_norefill: -- dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags); -- return; -- --global: -- touch_core_sched(rq, p); /* see the comment in local: */ -- p->scx->slice = SCX_SLICE_DFL; -- dispatch_enqueue(&scx_dsq_global, p, enq_flags); --} -- --static bool watchdog_task_watched(const struct task_struct *p) --{ -- return !list_empty(&p->scx->watchdog_node); --} -- --static void watchdog_watch_task(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- if (p->scx->flags & SCX_TASK_WATCHDOG_RESET) -- p->scx->runnable_at = jiffies; -- p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET; -- list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list); --} -- --static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) --{ -- list_del_init(&p->scx->watchdog_node); -- if (reset_timeout) -- p->scx->flags |= SCX_TASK_WATCHDOG_RESET; --} -- --static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) --{ -- int sticky_cpu = p->scx->sticky_cpu; -- -- enq_flags |= rq->scx->extra_enq_flags; -- -- if (sticky_cpu >= 0) -- p->scx->sticky_cpu = -1; -- -- /* -- * Restoring a running task will be immediately followed by -- * set_next_task_scx() which expects the task to not be on the BPF -- * scheduler as tasks can only start running through local DSQs. Force -- * direct-dispatch into the local DSQ by setting the sticky_cpu. -- */ -- if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -- sticky_cpu = cpu_of(rq); -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- WARN_ON_ONCE(!watchdog_task_watched(p)); -- return; -- } -- -- watchdog_watch_task(rq, p); -- p->scx->flags |= SCX_TASK_QUEUED; -- rq->scx->nr_running++; -- add_nr_running(rq, 1); -- -- if (SCX_HAS_OP(runnable)) -- SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); -- -- if (enq_flags & SCX_ENQ_WAKEUP) -- touch_core_sched(rq, p); -- -- do_enqueue_task(rq, p, enq_flags, sticky_cpu); --} -- --static void ops_dequeue(struct task_struct *p, u64 deq_flags) --{ -- u64 opss; -- -- watchdog_unwatch_task(p, false); -- -- /* acquire ensures that we see the preceding updates on QUEUED */ -- opss = atomic64_read_acquire(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_NONE: -- break; -- case SCX_OPSS_QUEUEING: -- /* -- * QUEUEING is started and finished while holding @p's rq lock. -- * As we're holding the rq lock now, we shouldn't see QUEUEING. -- */ -- BUG(); -- case SCX_OPSS_QUEUED: -- if (SCX_HAS_OP(dequeue)) -- SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); -- -- if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_NONE)) -- break; -- fallthrough; -- case SCX_OPSS_DISPATCHING: -- /* -- * If @p is being dispatched from the BPF scheduler to a DSQ, -- * wait for the transfer to complete so that @p doesn't get -- * added to its DSQ after dequeueing is complete. -- * -- * As we're waiting on DISPATCHING with the rq locked, the -- * dispatching side shouldn't try to lock the rq while -- * DISPATCHING is set. See dispatch_to_local_dsq(). -- * -- * DISPATCHING shouldn't have qseq set and control can reach -- * here with NONE @opss from the above QUEUED case block. -- * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. -- */ -- wait_ops_state(p, SCX_OPSS_DISPATCHING); -- BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- break; -- } --} -- --static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) --{ -- struct scx_rq *scx_rq = rq->scx; -- -- if (!(p->scx->flags & SCX_TASK_QUEUED)) { -- WARN_ON_ONCE(watchdog_task_watched(p)); -- return; -- } -- -- ops_dequeue(p, deq_flags); -- -- /* -- * A currently running task which is going off @rq first gets dequeued -- * and then stops running. As we want running <-> stopping transitions -- * to be contained within runnable <-> quiescent transitions, trigger -- * ->stopping() early here instead of in put_prev_task_scx(). -- * -- * @p may go through multiple stopping <-> running transitions between -- * here and put_prev_task_scx() if task attribute changes occur while -- * balance_scx() leaves @rq unlocked. However, they don't contain any -- * information meaningful to the BPF scheduler and can be suppressed by -- * skipping the callbacks if the task is !QUEUED. -- */ -- if (SCX_HAS_OP(stopping) && task_current(rq, p)) { -- update_curr_scx(rq); -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); -- } -- -- //if(task_current(rq, p)) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- if (SCX_HAS_OP(quiescent)) -- SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); -- -- if (deq_flags & SCX_DEQ_SLEEP) -- p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP; -- else -- p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP; -- -- p->scx->flags &= ~SCX_TASK_QUEUED; -- BUG_ON(!scx_rq->nr_running); -- scx_rq->nr_running--; -- sub_nr_running(rq, 1); -- -- dispatch_dequeue(scx_rq, p); --} -- --static void yield_task_scx(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); -- else -- p->scx->slice = 0; --} -- --static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) --{ -- struct task_struct *from = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); -- else -- return false; --} -- --#ifdef CONFIG_SMP --/** -- * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -- * @rq: rq to move the task into, currently locked -- * @p: task to move -- * @enq_flags: %SCX_ENQ_* -- * -- * Move @p which is currently on a different rq to @rq's local DSQ. The caller -- * must: -- * -- * 1. Start with exclusive access to @p either through its DSQ lock or -- * %SCX_OPSS_DISPATCHING flag. -- * -- * 2. Set @p->scx->holding_cpu to raw_smp_processor_id(). -- * -- * 3. Remember task_rq(@p). Release the exclusive access so that we don't -- * deadlock with dequeue. -- * -- * 4. Lock @rq and the task_rq from #3. -- * -- * 5. Call this function. -- * -- * Returns %true if @p was successfully moved. %false after racing dequeue and -- * losing. -- */ --static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -- u64 enq_flags) --{ -- struct rq *task_rq; -- -- lockdep_assert_rq_held(rq); -- -- /* -- * If dequeue got to @p while we were trying to lock both rq's, it'd -- * have cleared @p->scx->holding_cpu to -1. While other cpus may have -- * updated it to different values afterwards, as this operation can't be -- * preempted or recurse, @p->scx->holding_cpu can never become -- * raw_smp_processor_id() again before we're done. Thus, we can tell -- * whether we lost to dequeue by testing whether @p->scx->holding_cpu is -- * still raw_smp_processor_id(). -- * -- * See dispatch_dequeue() for the counterpart. -- */ -- if (unlikely(p->scx->holding_cpu != raw_smp_processor_id())) -- return false; -- -- /* @p->rq couldn't have changed if we're still the holding cpu */ -- task_rq = task_rq(p); -- lockdep_assert_rq_held(task_rq); -- -- WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)); -- deactivate_task(task_rq, p, 0); -- set_task_cpu(p, cpu_of(rq)); -- p->scx->sticky_cpu = cpu_of(rq); -- -- /* -- * We want to pass scx-specific enq_flags but activate_task() will -- * truncate the upper 32 bit. As we own @rq, we can pass them through -- * @rq->scx->extra_enq_flags instead. -- */ -- WARN_ON_ONCE(rq->scx->extra_enq_flags); -- rq->scx->extra_enq_flags = enq_flags; -- activate_task(rq, p, 0); -- rq->scx->extra_enq_flags = 0; -- -- return true; --} -- --/** -- * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * We're holding @rq lock and trying to dispatch a task from @src_rq to -- * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether -- * @rq stays locked isn't important as long as the state is restored after -- * dispatch_to_local_dsq_unlock(). -- */ --static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- rq_unpin_lock(rq, rf); -- -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(rq); -- raw_spin_rq_lock(dst_rq); -- } else if (rq == src_rq) { -- double_lock_balance(rq, dst_rq); -- rq_repin_lock(rq, rf); -- } else if (rq == dst_rq) { -- double_lock_balance(rq, src_rq); -- rq_repin_lock(rq, rf); -- } else { -- raw_spin_rq_unlock(rq); -- double_rq_lock(src_rq, dst_rq); -- } --} -- --/** -- * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock() -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return. -- */ --static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } else if (rq == src_rq) { -- double_unlock_balance(rq, dst_rq); -- } else if (rq == dst_rq) { -- double_unlock_balance(rq, src_rq); -- } else { -- double_rq_unlock(src_rq, dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } --} --#endif /* CONFIG_SMP */ -- -- --static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq) --{ -- return likely(test_rq_online(rq)) && !is_migration_disabled(p) && -- cpumask_test_cpu(cpu_of(rq), p->cpus_ptr); --} -- --static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -- struct scx_dispatch_q *dsq) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rb_node *rb_node; -- struct rq *task_rq; -- bool moved = false; --retry: -- if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -- return false; -- -- raw_spin_lock(&dsq->lock); -- -- list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- for (rb_node = rb_first_cached(&dsq->priq); rb_node; -- rb_node = rb_next(rb_node)) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- raw_spin_unlock(&dsq->lock); -- return false; -- --this_rq: -- /* @dsq is locked and @p is on this rq */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo); -- dsq->nr--; -- scx_rq->local_dsq.nr++; -- p->scx->dsq = &scx_rq->local_dsq; -- raw_spin_unlock(&dsq->lock); -- return true; -- --remote_rq: --#ifdef CONFIG_SMP -- /* -- * @dsq is locked and @p is on a remote rq. @p is currently protected by -- * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -- * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -- * rq lock or fail, do a little dancing from our side. See -- * move_task_to_local_dsq(). -- */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- p->scx->holding_cpu = raw_smp_processor_id(); -- raw_spin_unlock(&dsq->lock); -- -- rq_unpin_lock(rq, rf); -- double_lock_balance(rq, task_rq); -- rq_repin_lock(rq, rf); -- -- moved = move_task_to_local_dsq(rq, p, 0); -- -- double_unlock_balance(rq, task_rq); --#endif /* CONFIG_SMP */ -- if (likely(moved)) -- return true; -- goto retry; --} -- --enum dispatch_to_local_dsq_ret { -- DTL_DISPATCHED, /* successfully dispatched */ -- DTL_LOST, /* lost race to dequeue */ -- DTL_NOT_LOCAL, /* destination is not a local DSQ */ -- DTL_INVALID, /* invalid local dsq_id */ --}; -- --/** -- * dispatch_to_local_dsq - Dispatch a task to a local dsq -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @dsq_id: destination dsq ID -- * @p: task to dispatch -- * @enq_flags: %SCX_ENQ_* -- * -- * We're holding @rq lock and want to dispatch @p to the local DSQ identified by -- * @dsq_id. This function performs all the synchronization dancing needed -- * because local DSQs are protected with rq locks. -- * -- * The caller must have exclusive ownership of @p (e.g. through -- * %SCX_OPSS_DISPATCHING). -- */ --static enum dispatch_to_local_dsq_ret --dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id, -- struct task_struct *p, u64 enq_flags) --{ -- struct rq *src_rq = task_rq(p); -- struct rq *dst_rq; -- -- /* -- * We're synchronized against dequeue through DISPATCHING. As @p can't -- * be dequeued, its task_rq and cpus_allowed are stable too. -- */ -- if (dsq_id == SCX_DSQ_LOCAL) { -- dst_rq = rq; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]", -- cpu, p->comm, p->pid); -- return DTL_INVALID; -- } -- dst_rq = cpu_rq(cpu); -- } else { -- return DTL_NOT_LOCAL; -- } -- -- /* if dispatching to @rq that @p is already on, no lock dancing needed */ -- if (rq == src_rq && rq == dst_rq) { -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags | SCX_ENQ_CLEAR_OPSS); -- return DTL_DISPATCHED; -- } -- --#ifdef CONFIG_SMP -- if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { -- struct rq *locked_dst_rq = dst_rq; -- bool dsp; -- -- /* -- * @p is on a possibly remote @src_rq which we need to lock to -- * move the task. If dequeue is in progress, it'd be locking -- * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq -- * lock while holding DISPATCHING. -- * -- * As DISPATCHING guarantees that @p is wholly ours, we can -- * pretend that we're moving from a DSQ and use the same -- * mechanism - mark the task under transfer with holding_cpu, -- * release DISPATCHING and then follow the same protocol. -- */ -- p->scx->holding_cpu = raw_smp_processor_id(); -- -- /* store_release ensures that dequeue sees the above */ -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq); -- -- /* -- * We don't require the BPF scheduler to avoid dispatching to -- * offline CPUs mostly for convenience but also because CPUs can -- * go offline between scx_bpf_dispatch() calls and here. If @p -- * is destined to an offline CPU, queue it on its current CPU -- * instead, which should always be safe. As this is an allowed -- * behavior, don't trigger an ops error. -- */ -- if (unlikely(!test_rq_online(dst_rq))) -- dst_rq = src_rq; -- -- if (src_rq == dst_rq) { -- /* -- * As @p is staying on the same rq, there's no need to -- * go through the full deactivate/activate cycle. -- * Optimize by abbreviating the operations in -- * move_task_to_local_dsq(). -- */ -- dsp = p->scx->holding_cpu == raw_smp_processor_id(); -- if (likely(dsp)) { -- p->scx->holding_cpu = -1; -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags); -- } -- } else { -- dsp = move_task_to_local_dsq(dst_rq, p, enq_flags); -- } -- -- /* if the destination CPU is idle, wake it up */ -- if (dsp && p->sched_class > dst_rq->curr->sched_class) -- resched_curr(dst_rq); -- -- dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq); -- -- return dsp ? DTL_DISPATCHED : DTL_LOST; -- } --#endif /* CONFIG_SMP */ -- -- scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", -- cpu_of(dst_rq), p->comm, p->pid); -- return DTL_INVALID; --} -- --/** -- * finish_dispatch - Asynchronously finish dispatching a task -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @p: task to finish dispatching -- * @qseq_at_dispatch: qseq when @p started getting dispatched -- * @dsq_id: destination DSQ ID -- * @enq_flags: %SCX_ENQ_* -- * -- * Dispatching to local DSQs may need to wait for queueing to complete or -- * require rq lock dancing. As we don't wanna do either while inside -- * ops.dispatch() to avoid locking order inversion, we split dispatching into -- * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the -- * task and its qseq. Once ops.dispatch() returns, this function is called to -- * finish up. -- * -- * There is no guarantee that @p is still valid for dispatching or even that it -- * was valid in the first place. Make sure that the task is still owned by the -- * BPF scheduler and claim the ownership before dispatching. -- */ --static void finish_dispatch(struct rq *rq, struct rq_flags *rf, -- struct task_struct *p, u64 qseq_at_dispatch, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- u64 opss; -- -- touch_core_sched_dispatch(rq, p); --retry: -- /* -- * No need for _acquire here. @p is accessed only after a successful -- * try_cmpxchg to DISPATCHING. -- */ -- opss = atomic64_read(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_DISPATCHING: -- case SCX_OPSS_NONE: -- /* someone else already got to it */ -- return; -- case SCX_OPSS_QUEUED: -- /* -- * If qseq doesn't match, @p has gone through at least one -- * dispatch/dequeue and re-enqueue cycle between -- * scx_bpf_dispatch() and here and we have no claim on it. -- */ -- if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) -- return; -- -- /* -- * While we know @p is accessible, we don't yet have a claim on -- * it - the BPF scheduler is allowed to dispatch tasks -- * spuriously and there can be a racing dequeue attempt. Let's -- * claim @p by atomically transitioning it from QUEUED to -- * DISPATCHING. -- */ -- if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_DISPATCHING))) -- break; -- goto retry; -- case SCX_OPSS_QUEUEING: -- /* -- * do_enqueue_task() is in the process of transferring the task -- * to the BPF scheduler while holding @p's rq lock. As we aren't -- * holding any kernel or BPF resource that the enqueue path may -- * depend upon, it's safe to wait. -- */ -- wait_ops_state(p, opss); -- goto retry; -- } -- -- BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) { -- case DTL_DISPATCHED: -- break; -- case DTL_LOST: -- break; -- case DTL_INVALID: -- dsq_id = SCX_DSQ_GLOBAL; -- fallthrough; -- case DTL_NOT_LOCAL: -- dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()), -- dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- break; -- } --} -- --static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- u32 u; -- -- for (u = 0; u < dspc->buf_cursor; u++) { -- struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u]; -- -- finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id, -- ent->enq_flags); -- } -- -- dspc->nr_tasks += dspc->buf_cursor; -- dspc->buf_cursor = 0; --} -- --static int balance_one(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf, bool local) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- bool prev_on_scx = prev->sched_class == &ext_sched_class; -- int nr_loops = SCX_DSP_MAX_LOOPS; -- -- lockdep_assert_rq_held(rq); -- -- if (static_branch_unlikely(&scx_ops_cpu_preempt) && -- unlikely(rq->scx->cpu_released)) { -- /* -- * If the previous sched_class for the current CPU was not SCX, -- * notify the BPF scheduler that it again has control of the -- * core. This callback complements ->cpu_release(), which is -- * emitted in scx_notify_pick_next_task(). -- */ -- if (SCX_HAS_OP(cpu_acquire)) -- SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq), -- NULL); -- rq->scx->cpu_released = false; -- } -- -- if (prev_on_scx) { -- WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP)); -- update_curr_scx(rq); -- -- /* -- * If @prev is runnable & has slice left, it has priority and -- * fetching more just increases latency for the fetched tasks. -- * Tell put_prev_task_scx() to put @prev on local_dsq. If the -- * BPF scheduler wants to handle this explicitly, it should -- * implement ->cpu_released(). -- * -- * See scx_ops_disable_workfn() for the explanation on the -- * disabling() test. -- * -- * When balancing a remote CPU for core-sched, there won't be a -- * following put_prev_task_scx() call and we don't own -- * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the -- * same conditions later and pick @rq->curr accordingly. -- */ -- if ((prev->scx->flags & SCX_TASK_QUEUED) && -- prev->scx->slice && !scx_ops_disabling()) { -- if (local) -- prev->scx->flags |= SCX_TASK_BAL_KEEP; -- return 1; -- } -- } -- -- /* if there already are tasks to run, nothing to do */ -- if (scx_rq->local_dsq.nr) -- return 1; -- -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- if (!SCX_HAS_OP(dispatch)) -- return 0; -- -- dspc->rq = rq; -- dspc->rf = rf; -- -- /* -- * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, -- * the local DSQ might still end up empty after a successful -- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() -- * produced some tasks, retry. The BPF scheduler may depend on this -- * looping behavior to simplify its implementation. -- */ -- do { -- dspc->nr_tasks = 0; -- -- SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), -- prev_on_scx ? prev : NULL); -- -- flush_dispatch_buf(rq, rf); -- -- if (scx_rq->local_dsq.nr) -- return 1; -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- /* -- * ops.dispatch() can trap us in this loop by repeatedly -- * dispatching ineligible tasks. Break out once in a while to -- * allow the watchdog to run. As IRQ can't be enabled in -- * balance(), we want to complete this scheduling cycle and then -- * start a new one. IOW, we want to call resched_curr() on the -- * next, most likely idle, task, not the current one. Use -- * scx_bpf_kick_cpu() for deferred kicking. -- */ -- if (unlikely(!--nr_loops)) { -- scx_bpf_kick_cpu(cpu_of(rq), 0); -- break; -- } -- } while (dspc->nr_tasks); -- -- return 0; --} -- --static int balance_scx(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf) --{ -- int ret; -- -- ret = balance_one(rq, prev, rf, true); --#ifdef CONFIG_SCHED_SMT -- /* -- * When core-sched is enabled, this ops.balance() call will be followed -- * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx() -- * on the SMT siblings. Balance the siblings too. -- */ -- if (sched_core_enabled(rq)) { -- const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); -- int scpu; -- -- for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { -- struct rq *srq = cpu_rq(scpu); -- struct rq_flags srf; -- struct task_struct *sprev = srq->curr; -- -- /* -- * While core-scheduling, rq lock is shared among -- * siblings but the debug annotations and rq clock -- * aren't. Do pinning dance to transfer the ownership. -- */ -- WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); -- rq_unpin_lock(rq, rf); -- rq_pin_lock(srq, &srf); -- -- update_rq_clock(srq); -- balance_one(srq, sprev, &srf, false); -- -- rq_unpin_lock(srq, &srf); -- rq_repin_lock(rq, rf); -- } -- } --#endif -- return ret; --} -- --static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) --{ -- if (p->scx->flags & SCX_TASK_QUEUED) { -- /* -- * Core-sched might decide to execute @p before it is -- * dispatched. Call ops_dequeue() to notify the BPF scheduler. -- */ -- ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); -- dispatch_dequeue(rq->scx, p); -- } -- -- p->se.exec_start = rq_clock_task(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, running, p); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock); -- -- watchdog_unwatch_task(p, true); -- -- /* -- * @p is getting newly scheduled or got kicked after someone updated its -- * slice. Refresh whether tick can be stopped. See can_stop_tick_scx(). -- */ -- if ((p->scx->slice == SCX_SLICE_INF) != -- (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) { -- if (p->scx->slice == SCX_SLICE_INF) -- rq->scx->flags |= SCX_RQ_CAN_STOP_TICK; -- else -- rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK; -- -- sched_update_tick_dependency(rq); -- } --} -- --static void put_prev_task_scx(struct rq *rq, struct task_struct *p) --{ --#ifndef CONFIG_SMP -- /* -- * UP workaround. -- * -- * Because SCX may transfer tasks across CPUs during dispatch, dispatch -- * is performed from its balance operation which isn't called in UP. -- * Let's work around by calling it from the operations which come right -- * after. -- * -- * 1. If the prev task is on SCX, pick_next_task() calls -- * .put_prev_task() right after. As .put_prev_task() is also called -- * from other places, we need to distinguish the calls which can be -- * done by looking at the previous task's state - if still queued or -- * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task(). -- * This case is handled here. -- * -- * 2. If the prev task is not on SCX, the first following call into SCX -- * will be .pick_next_task(), which is covered by calling -- * balance_scx() from pick_next_task_scx(). -- * -- * Note that we can't merge the first case into the second as -- * balance_scx() must be called before the previous SCX task goes -- * through put_prev_task_scx(). -- * -- * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf. -- * Pass in %NULL. -- */ -- if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP)) -- balance_scx(rq, p, NULL); --#endif -- -- update_curr_scx(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- -- /* -- * If we're being called from put_prev_task_balance(), balance_scx() may -- * have decided that @p should keep running. -- */ -- if (p->scx->flags & SCX_TASK_BAL_KEEP) { -- p->scx->flags &= ~SCX_TASK_BAL_KEEP; -- watchdog_watch_task(rq, p); -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- watchdog_watch_task(rq, p); -- -- /* -- * If @p has slice left and balance_scx() didn't tag it for -- * keeping, @p is getting preempted by a higher priority -- * scheduler class or core-sched forcing a different task. Leave -- * it at the head of the local DSQ. -- */ -- if (p->scx->slice && !scx_ops_disabling()) { -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- /* -- * If we're in the pick_next_task path, balance_scx() should -- * have already populated the local DSQ if there are any other -- * available tasks. If empty, tell ops.enqueue() that @p is the -- * only one available for this cpu. ops.enqueue() should put it -- * on the local DSQ so that the subsequent pick_next_task_scx() -- * can find the task unless it wants to trigger a separate -- * follow-up scheduling event. -- */ -- if (list_empty(&rq->scx->local_dsq.fifo)) -- do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1); -- else -- do_enqueue_task(rq, p, 0, -1); -- } --} -- --static struct task_struct *first_local_task(struct rq *rq) --{ -- struct rb_node *rb_node; -- struct sched_ext_entity *entity; -- -- if (!list_empty(&rq->scx->local_dsq.fifo)) { -- entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo); -- return entity->task; -- } -- -- rb_node = rb_first_cached(&rq->scx->local_dsq.priq); -- if (rb_node) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- return entity->task; -- } -- -- return NULL; --} -- --static struct task_struct *pick_next_task_scx(struct rq *rq) --{ -- struct task_struct *p; -- --#ifndef CONFIG_SMP -- /* UP workaround - see the comment at the head of put_prev_task_scx() */ -- if (unlikely(rq->curr->sched_class != &ext_sched_class)) -- balance_scx(rq, rq->curr, NULL); --#endif -- -- p = first_local_task(rq); -- if (!p) -- return NULL; -- -- if (unlikely(!p->scx->slice)) { -- if (!scx_ops_disabling() && !scx_warned_zero_slice) { -- printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n", -- p->comm, p->pid); -- scx_warned_zero_slice = true; -- } -- p->scx->slice = SCX_SLICE_DFL; -- } -- -- set_next_task_scx(rq, p, true); -- -- return p; --} -- --#ifdef CONFIG_SCHED_CORE --/** -- * scx_prio_less - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Core-sched is implemented as an additional scheduling layer on top of the -- * usual sched_class'es and needs to find out the expected task ordering. For -- * SCX, core-sched calls this function to interrogate the task ordering. -- * -- * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used -- * to implement the default task ordering. The older the timestamp, the higher -- * prority the task - the global FIFO ordering matching the default scheduling -- * behavior. -- * -- * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to -- * implement FIFO ordering within each local DSQ. See pick_task_scx(). -- */ --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi) --{ -- /* -- * The const qualifiers are dropped from task_struct pointers when -- * calling ops.core_sched_before(). Accesses are controlled by the -- * verifier. -- */ -- if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling()) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, -- (struct task_struct *)a, -- (struct task_struct *)b); -- else -- return time_after64(a->scx->core_sched_at, b->scx->core_sched_at); --} -- --/** -- * pick_task_scx - Pick a candidate task for core-sched -- * @rq: rq to pick the candidate task from -- * -- * Core-sched calls this function on each SMT sibling to determine the next -- * tasks to run on the SMT siblings. balance_one() has been called on all -- * siblings and put_prev_task_scx() has been called only for the current CPU. -- * -- * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look -- * at the first task in the local dsq. @rq->curr has to be considered explicitly -- * to mimic %SCX_TASK_BAL_KEEP. -- */ --static struct task_struct *pick_task_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- struct task_struct *first = first_local_task(rq); -- -- if (curr->scx->flags & SCX_TASK_QUEUED) { -- /* is curr the only runnable task? */ -- if (!first) -- return curr; -- -- /* -- * Does curr trump first? We can always go by core_sched_at for -- * this comparison as it represents global FIFO ordering when -- * the default core-sched ordering is used and local-DSQ FIFO -- * ordering otherwise. -- * -- * We can have a task with an earlier timestamp on the DSQ. For -- * example, when a current task is preempted by a sibling -- * picking a different cookie, the task would be requeued at the -- * head of the local DSQ with an earlier timestamp than the -- * core-sched picked next task. Besides, the BPF scheduler may -- * dispatch any tasks to the local DSQ anytime. -- */ -- if (curr->scx->slice && time_before64(curr->scx->core_sched_at, -- first->scx->core_sched_at)) -- return curr; -- } -- -- return first; /* this may be %NULL */ --} --#endif /* CONFIG_SCHED_CORE */ -- --static enum scx_cpu_preempt_reason --preempt_reason_from_class(const struct sched_class *class) --{ --#ifdef CONFIG_SMP -- if (class == &stop_sched_class) -- return SCX_CPU_PREEMPT_STOP; --#endif -- if (class == &dl_sched_class) -- return SCX_CPU_PREEMPT_DL; -- if (class == &rt_sched_class) -- return SCX_CPU_PREEMPT_RT; -- return SCX_CPU_PREEMPT_UNKNOWN; --} -- --void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task, -- const struct sched_class *active) --{ -- lockdep_assert_rq_held(rq); -- -- /* -- * The callback is conceptually meant to convey that the CPU is no -- * longer under the control of SCX. Therefore, don't invoke the -- * callback if the CPU is is staying on SCX, or going idle (in which -- * case the SCX scheduler has actively decided not to schedule any -- * tasks on the CPU). -- */ -- if (likely(active >= &ext_sched_class)) -- return; -- -- /* -- * At this point we know that SCX was preempted by a higher priority -- * sched_class, so invoke the ->cpu_release() callback if we have not -- * done so already. We only send the callback once between SCX being -- * preempted, and it regaining control of the CPU. -- * -- * ->cpu_release() complements ->cpu_acquire(), which is emitted the -- * next time that balance_scx() is invoked. -- */ -- if (!rq->scx->cpu_released) { -- if (SCX_HAS_OP(cpu_release)) { -- struct scx_cpu_release_args args = { -- .reason = preempt_reason_from_class(active), -- .task = task, -- }; -- -- SCX_CALL_OP(SCX_KF_CPU_RELEASE, -- cpu_release, cpu_of(rq), &args); -- } -- rq->scx->cpu_released = true; -- } --} -- --#ifdef CONFIG_SMP -- --static bool test_and_clear_cpu_idle(int cpu) --{ -- if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -- if (cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- return true; -- } else { -- return false; -- } --} -- --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- int cpu; -- -- do { -- cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); -- if (cpu < nr_cpu_ids) { -- const struct cpumask *sbm = topology_sibling_cpumask(cpu); -- -- /* -- * If offline, @cpu is not its own sibling and we can -- * get caught in an infinite loop as @cpu is never -- * cleared from idle_masks.smt. Clear @cpu directly in -- * such cases. -- */ -- if (likely(cpumask_test_cpu(cpu, sbm))) -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm); -- else -- cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu)); -- } else { -- cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -- if (cpu >= nr_cpu_ids) -- return -EBUSY; -- } -- } while (!test_and_clear_cpu_idle(cpu)); -- -- return cpu; --} -- --static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) --{ -- s32 cpu; -- -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return prev_cpu; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local DSQ of the waker. -- */ -- if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 && -- scx_has_idle_cpus && !(current->flags & PF_EXITING)) { -- cpu = smp_processor_id(); -- if (cpumask_test_cpu(cpu, p->cpus_ptr)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (test_and_clear_cpu_idle(prev_cpu)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return prev_cpu; -- } -- -- if (p->nr_cpus_allowed == 1) -- return prev_cpu; -- -- cpu = scx_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- -- return prev_cpu; --} -- --static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) --{ -- if (SCX_HAS_OP(select_cpu)) { -- s32 cpu; -- -- cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu, -- wake_flags); -- if (ops_cpu_valid(cpu)) { -- return cpu; -- } else { -- scx_ops_error("select_cpu returned invalid cpu %d", cpu); -- return prev_cpu; -- } -- } else { -- return scx_select_cpu_dfl(p, prev_cpu, wake_flags); -- } --} -- --static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx) --{ -- set_cpus_allowed_common(p, ctx); -- -- /* -- * The effective cpumask is stored in @p->cpus_ptr which may temporarily -- * differ from the configured one in @p->cpus_mask. Always tell the bpf -- * scheduler the effective one. -- * -- * Fine-grained memory write control is enforced by BPF making the const -- * designation pointless. Cast it away when calling the operation. -- */ -- if (SCX_HAS_OP(set_cpumask)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, -- (struct cpumask *)p->cpus_ptr); --} -- --static void reset_idle_masks(void) --{ -- /* consider all cpus idle, should converge to the actual state quickly */ -- cpumask_setall(idle_masks.cpu); -- cpumask_setall(idle_masks.smt); -- scx_has_idle_cpus = true; --} -- --void __scx_update_idle(struct rq *rq, bool idle) --{ -- int cpu = cpu_of(rq); -- struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -- -- if (SCX_HAS_OP(update_idle)) { -- SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); -- if (!static_branch_unlikely(&scx_builtin_idle_enabled)) -- return; -- } -- -- if (idle) { -- cpumask_set_cpu(cpu, idle_masks.cpu); -- if (!scx_has_idle_cpus) -- scx_has_idle_cpus = true; -- -- /* -- * idle_masks.smt handling is racy but that's fine as it's only -- * for optimization and self-correcting. -- */ -- for_each_cpu(cpu, sib_mask) { -- if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -- return; -- } -- cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -- } else { -- cpumask_clear_cpu(cpu, idle_masks.cpu); -- if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -- } --} -- --#else /* !CONFIG_SMP */ -- --static bool test_and_clear_cpu_idle(int cpu) { return false; } --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } --static void reset_idle_masks(void) {} -- --#endif /* CONFIG_SMP */ -- --static bool check_rq_for_timeouts(struct rq *rq) --{ -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rq_flags rf; -- bool timed_out = false; -- -- rq_lock_irqsave(rq, &rf); -- list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) { -- unsigned long last_runnable; -- -- p = entity->task; -- last_runnable = p->scx->runnable_at; -- -- if (unlikely(time_after(jiffies, -- last_runnable + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "%s[%d] failed to run for %u.%03us", -- p->comm, p->pid, -- dur_ms / 1000, dur_ms % 1000); -- timed_out = true; -- break; -- } -- } -- rq_unlock_irqrestore(rq, &rf); -- -- return timed_out; --} -- --static void scx_watchdog_workfn(struct work_struct *work) --{ -- int cpu; -- -- scx_watchdog_timestamp = jiffies; -- -- for_each_online_cpu(cpu) { -- if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) -- break; -- -- cond_resched(); -- } -- queue_delayed_work(system_unbound_wq, to_delayed_work(work), -- scx_watchdog_timeout / 2); --} -- --static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) --{ -- update_curr_scx(rq); -- //scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock); -- /* -- * While disabling, always resched and refresh core-sched timestamp as -- * we can't trust the slice management or ops.core_sched_before(). -- */ -- if (scx_ops_disabling()) { -- curr->scx->slice = 0; -- touch_core_sched(rq, curr); -- } -- -- if (!curr->scx->slice) -- resched_curr(rq); --} -- --#define SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- --static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg) --{ -- int ret; -- -- WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED); -- -- p->scx->disallow = false; -- -- if (SCX_HAS_OP(prep_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- }; -- -- ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args); -- if (unlikely(ret)) { -- ret = ops_sanitize_err("prep_enable", ret); -- return ret; -- } -- } -- //scx_sched_init_task(p); -- -- if (p->scx->disallow) { -- struct rq *rq; -- struct rq_flags rf; -- -- rq = task_rq_lock(p, &rf); -- -- /* -- * We're either in fork or load path and @p->policy will be -- * applied right after. Reverting @p->policy here and rejecting -- * %SCHED_EXT transitions from scx_check_setscheduler() -- * guarantees that if ops.prep_enable() sets @p->disallow, @p -- * can never be in SCX. -- */ -- if (p->policy == SCHED_EXT) { -- p->policy = SCHED_NORMAL; -- atomic64_inc(&scx_nr_rejected); -- } -- -- task_rq_unlock(rq, p, &rf); -- } -- -- p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET); -- return 0; --} -- --static void scx_ops_enable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED)); -- -- if (SCX_HAS_OP(enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- p->scx->flags |= SCX_TASK_OPS_ENABLED; --} -- --static void scx_ops_disable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- if (p->scx->flags & SCX_TASK_OPS_PREPPED) { -- if (SCX_HAS_OP(cancel_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- } else if (p->scx->flags & SCX_TASK_OPS_ENABLED) { -- if (SCX_HAS_OP(disable)) -- SCX_CALL_OP(SCX_KF_REST, disable, p); -- p->scx->flags &= ~SCX_TASK_OPS_ENABLED; -- } --} -- --static void set_task_scx_weight(struct task_struct *p) --{ -- u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -- -- p->scx->weight = sched_weight_to_cgroup(weight); --} -- --/** -- * refresh_scx_weight - Refresh a task's ext weight -- * @p: task to refresh ext weight for -- * -- * @p->scx->weight carries the task's static priority in cgroup weight scale to -- * enable easy access from the BPF scheduler. To keep it synchronized with the -- * current task priority, this function should be called when a new task is -- * created, priority is changed for a task on sched_ext, and a task is switched -- * to sched_ext from other classes. -- */ --static void refresh_scx_weight(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- set_task_scx_weight(p); -- if (SCX_HAS_OP(set_weight)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight); --} -- --void scx_pre_fork(struct task_struct *p) --{ -- p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL); -- if (!p->scx) { -- goto lock; -- } -- -- p->scx->dsq = NULL; -- INIT_LIST_HEAD(&p->scx->dsq_node.fifo); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- INIT_LIST_HEAD(&p->scx->watchdog_node); -- p->scx->flags = 0; -- p->scx->weight = 0; -- p->scx->sticky_cpu = -1; -- p->scx->holding_cpu = -1; -- p->scx->kf_mask = 0; -- atomic64_set(&p->scx->ops_state, 0); -- p->scx->runnable_at = INITIAL_JIFFIES; -- p->scx->slice = SCX_SLICE_DFL; -- p->scx->task = p; -- p->sched_prop = 0; -- -- /* -- * BPF scheduler enable/disable paths want to be able to iterate and -- * update all tasks which can become complex when racing forks. As -- * enable/disable are very cold paths, let's use a percpu_rwsem to -- * exclude forks. -- */ --lock: -- percpu_down_read(&scx_fork_rwsem); --} -- --int scx_fork(struct task_struct *p) --{ -- percpu_rwsem_assert_held(&scx_fork_rwsem); -- -- if (scx_enabled()) -- return scx_ops_prepare_task(p, task_group(p)); -- else -- return 0; --} -- --void scx_post_fork(struct task_struct *p) --{ -- if (scx_enabled()) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- /* -- * Set the weight manually before calling ops.enable() so that -- * the scheduler doesn't see a stale value if they inspect the -- * task struct. We'll invoke ops.set_weight() afterwards, as it -- * would be odd to receive a callback on the task before we -- * tell the scheduler that it's been fully enabled. -- */ -- set_task_scx_weight(p); -- scx_ops_enable_task(p); -- refresh_scx_weight(p); -- task_rq_unlock(rq, p, &rf); -- } -- -- spin_lock_irq(&scx_tasks_lock); -- list_add_tail(&p->scx->tasks_node, &scx_tasks); -- spin_unlock_irq(&scx_tasks_lock); -- -- percpu_up_read(&scx_fork_rwsem); --} -- --void scx_cancel_fork(struct task_struct *p) --{ -- if (scx_enabled()) -- scx_ops_disable_task(p); -- percpu_up_read(&scx_fork_rwsem); --} -- --void sched_ext_free(struct task_struct *p) --{ -- unsigned long flags; -- -- spin_lock_irqsave(&scx_tasks_lock, flags); -- list_del_init(&p->scx->tasks_node); -- spin_unlock_irqrestore(&scx_tasks_lock, flags); -- -- /* -- * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED -> -- * ENABLED transitions can't race us. Disable ops for @p. -- */ -- if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- scx_ops_disable_task(p); -- task_rq_unlock(rq, p, &rf); -- } --} -- --static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) --{ --} -- --static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} --static void switched_to_scx(struct rq *rq, struct task_struct *p) {} -- --int scx_check_setscheduler(struct task_struct *p, int policy) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- /* if disallow, reject transitioning into SCX */ -- if (scx_enabled() && READ_ONCE(p->scx->disallow) && -- p->policy != policy && policy == SCHED_EXT) -- return -EACCES; -- -- return 0; --} -- --#ifdef CONFIG_NO_HZ_FULL --bool scx_can_stop_tick(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (scx_ops_disabling()) -- return false; -- -- if (p->sched_class != &ext_sched_class) -- return true; -- -- /* -- * @rq can dispatch from different DSQs, so we can't tell whether it -- * needs the tick or not by looking at nr_running. Allow stopping ticks -- * iff the BPF scheduler indicated so. See set_next_task_scx(). -- */ -- return rq->scx->flags & SCX_RQ_CAN_STOP_TICK; --} --#endif -- --static inline void scx_cgroup_lock(void) {} --static inline void scx_cgroup_unlock(void) {} -- --/* -- * Omitted operations: -- * -- * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -- * task isn't tied to the CPU at that point. Preemption is implemented by -- * resetting the victim task's slice to 0 and triggering reschedule on the -- * target CPU. -- * -- * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -- * -- * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -- * their current sched_class. Call them directly from sched core instead. -- * -- * - task_woken, switched_from: Unnecessary. -- */ --DEFINE_SCHED_CLASS(ext) = { -- .enqueue_task = enqueue_task_scx, -- .dequeue_task = dequeue_task_scx, -- .yield_task = yield_task_scx, -- .yield_to_task = yield_to_task_scx, -- -- .check_preempt_curr = check_preempt_curr_scx, -- -- .pick_next_task = pick_next_task_scx, -- -- .put_prev_task = put_prev_task_scx, -- .set_next_task = set_next_task_scx, -- --#ifdef CONFIG_SMP -- .balance = balance_scx, -- .select_task_rq = select_task_rq_scx, -- .set_cpus_allowed = set_cpus_allowed_scx, --#endif -- --#ifdef CONFIG_SCHED_CORE -- .pick_task = pick_task_scx, --#endif -- -- .task_tick = task_tick_scx, -- -- .switched_to = switched_to_scx, -- .prio_changed = prio_changed_scx, -- -- .update_curr = update_curr_scx, -- --#ifdef CONFIG_UCLAMP_TASK -- .uclamp_enabled = 0, --#endif --}; -- --static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) --{ -- memset(dsq, 0, sizeof(*dsq)); -- -- raw_spin_lock_init(&dsq->lock); -- INIT_LIST_HEAD(&dsq->fifo); -- dsq->id = dsq_id; --} -- --static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id & SCX_DSQ_FLAG_BUILTIN) -- return ERR_PTR(-EINVAL); -- -- dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node); -- if (!dsq) -- return ERR_PTR(-ENOMEM); -- -- init_dsq(dsq, dsq_id); -- -- return dsq; --} -- --static void free_dsq_irq_workfn(struct irq_work *irq_work) --{ -- struct llist_node *to_free = llist_del_all(&dsqs_to_free); -- struct scx_dispatch_q *dsq, *tmp_dsq; -- -- llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) -- kfree_rcu(dsq, rcu); --} -- --static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); -- --static void destroy_dsq(u64 dsq_id) --{ --} -- --static void scx_cgroup_exit(void) {} --static int scx_cgroup_init(void) { return 0; } --static void scx_cgroup_config_knobs(void) {} -- --/* -- * Used by sched_fork() and __setscheduler_prio() to pick the matching -- * sched_class. dl/rt are already handled. -- */ --bool task_on_scx(struct task_struct *p) --{ -- if (!scx_enabled() || scx_ops_disabling()) -- return false; -- if (READ_ONCE(scx_switching_all)) -- return true; -- return p->policy == SCHED_EXT; --} -- --static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags) --{ -- if (enq_flags & SCX_ENQ_LAST) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); --} -- --static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {} -- --static void scx_ops_disable_workfn(struct kthread_work *work) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- struct scx_task_iter sti; -- struct task_struct *p; -- const char *reason; -- int i, cpu, type; -- -- type = atomic_read(&scx_exit_type); -- while (true) { -- /* -- * NONE indicates that a new scx_ops has been registered since -- * disable was scheduled - don't kill the new ops. DONE -- * indicates that the ops has already been disabled. -- */ -- if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE) -- return; -- if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE)) -- break; -- } -- -- cancel_delayed_work_sync(&scx_watchdog_work); -- -- switch (type) { -- case SCX_EXIT_UNREG: -- reason = "BPF scheduler unregistered"; -- break; -- case SCX_EXIT_SYSRQ: -- reason = "disabled by sysrq-S"; -- break; -- case SCX_EXIT_ERROR: -- reason = "runtime error"; -- break; -- case SCX_EXIT_ERROR_BPF: -- reason = "scx_bpf_error"; -- break; -- case SCX_EXIT_ERROR_STALL: -- reason = "runnable task stall"; -- break; -- default: -- reason = ""; -- } -- -- ei->type = type; -- strlcpy(ei->reason, reason, sizeof(ei->reason)); -- -- switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { -- case SCX_OPS_DISABLED: -- pr_warn("sched_ext: ops error detected without ops (%s)\n", -- scx_exit_info.msg); -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- return; -- case SCX_OPS_PREPPING: -- goto forward_progress_guaranteed; -- case SCX_OPS_DISABLING: -- /* shouldn't happen but handle it like ENABLING if it does */ -- WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); -- fallthrough; -- case SCX_OPS_ENABLING: -- case SCX_OPS_ENABLED: -- break; -- } -- -- /* -- * DISABLING is set and ops was either ENABLING or ENABLED indicating -- * that the ops and static branches are set. -- * -- * We must guarantee that all runnable tasks make forward progress -- * without trusting the BPF scheduler. We can't grab any mutexes or -- * rwsems as they might be held by tasks that the BPF scheduler is -- * forgetting to run, which unfortunately also excludes toggling the -- * static branches. -- * -- * Let's work around by overriding a couple ops and modifying behaviors -- * based on the DISABLING state and then cycling the tasks through -- * dequeue/enqueue to force global FIFO scheduling. -- * -- * a. ops.enqueue() and .dispatch() are overridden for simple global -- * FIFO scheduling. -- * -- * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value -- * can't be trusted. Whenever a tick triggers, the running task is -- * rotated to the tail of the queue with core_sched_at touched. -- * -- * c. pick_next_task() suppresses zero slice warning. -- * -- * d. scx_prio_less() reverts to the default core_sched_at order. -- */ -- scx_ops.enqueue = scx_ops_fallback_enqueue; -- scx_ops.dispatch = scx_ops_fallback_dispatch; -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- SCHED_CHANGE_BLOCK(task_rq(p), p, -- DEQUEUE_SAVE | DEQUEUE_MOVE) { -- /* cycling deq/enq is enough, see above */ -- } -- } -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* kick all CPUs to restore ticks */ -- for_each_possible_cpu(cpu) -- resched_cpu(cpu); -- --forward_progress_guaranteed: -- /* -- * Here, every runnable task is guaranteed to make forward progress and -- * we can safely use blocking synchronization constructs. Actually -- * disable ops. -- */ -- mutex_lock(&scx_ops_enable_mutex); -- -- static_branch_disable(&__scx_switched_all); -- WRITE_ONCE(scx_switching_all, false); -- -- /* avoid racing against fork and cgroup changes */ -- cpus_read_lock(); -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- bool alive = READ_ONCE(p->__state) != TASK_DEAD; -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL); -- -- __setscheduler_prio(p, p->prio); -- } -- -- if (alive) -- check_class_changed(task_rq(p), p, old_class, p->prio); -- -- scx_ops_disable_task(p); -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* no task is on scx, turn off all the switches and flush in-progress calls */ -- static_branch_disable_cpuslocked(&__scx_ops_enabled); -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- static_branch_disable_cpuslocked(&scx_has_op[i]); -- static_branch_disable_cpuslocked(&scx_ops_enq_last); -- static_branch_disable_cpuslocked(&scx_ops_enq_exiting); -- static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- synchronize_rcu(); -- -- scx_cgroup_exit(); -- -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- cpus_read_unlock(); -- -- if (ei->type >= SCX_EXIT_ERROR) { -- printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name); -- -- if (ei->msg[0] == '\0') -- printk(KERN_ERR "sched_ext: %s\n", ei->reason); -- else -- printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg); -- -- stack_trace_print(ei->bt, ei->bt_len, 2); -- } -- -- if (scx_ops.exit) -- SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); -- -- memset(&scx_ops, 0, sizeof(scx_ops)); -- -- free_percpu(scx_dsp_buf); -- scx_dsp_buf = NULL; -- scx_dsp_max_batch = 0; -- -- mutex_unlock(&scx_ops_enable_mutex); -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- -- scx_cgroup_config_knobs(); --} -- --static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); -- --static void schedule_scx_ops_disable_work(void) --{ -- struct kthread_worker *helper = READ_ONCE(scx_ops_helper); -- -- /* -- * We may be called spuriously before the first bpf_sched_ext_reg(). If -- * scx_ops_helper isn't set up yet, there's nothing to do. -- */ -- if (helper) -- kthread_queue_work(helper, &scx_ops_disable_work); --} -- --static void scx_ops_disable(enum scx_exit_type type) --{ -- int none = SCX_EXIT_NONE; -- -- if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)) -- type = SCX_EXIT_ERROR; -- -- atomic_try_cmpxchg(&scx_exit_type, &none, type); -- -- schedule_scx_ops_disable_work(); --} -- --static void scx_ops_error_irq_workfn(struct irq_work *irq_work) --{ -- schedule_scx_ops_disable_work(); --} -- --static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- int none = SCX_EXIT_NONE; -- va_list args; -- -- if (!atomic_try_cmpxchg(&scx_exit_type, &none, type)) -- return; -- -- ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1); -- -- va_start(args, fmt); -- vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args); -- va_end(args); -- -- irq_work_queue(&scx_ops_error_irq_work); --} -- --static struct kthread_worker *scx_create_rt_helper(const char *name) --{ -- struct kthread_worker *helper; -- -- helper = kthread_create_worker(0, name); -- if (helper) -- sched_set_fifo(helper->task); -- return helper; --} -- --static int scx_ops_enable(struct sched_ext_ops *ops) --{ -- struct scx_task_iter sti; -- struct task_struct *p; -- int i, ret; -- int tcnt = 0; -- unsigned long long start = 0; -- unsigned long long total_start = sched_clock(); -- -- mutex_lock(&scx_ops_enable_mutex); -- -- if (!scx_ops_helper) { -- WRITE_ONCE(scx_ops_helper, -- scx_create_rt_helper("sched_ext_ops_helper")); -- if (!scx_ops_helper) { -- ret = -ENOMEM; -- goto err_unlock; -- } -- } -- -- if (scx_ops_enable_state() != SCX_OPS_DISABLED) { -- ret = -EBUSY; -- goto err_unlock; -- } -- -- //slim_walt_enable(true); -- -- /* -- * Set scx_ops, transition to PREPPING and clear exit info to arm the -- * disable path. Failure triggers full disabling from here on. -- */ -- scx_ops = *ops; -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != -- SCX_OPS_DISABLED); -- -- memset(&scx_exit_info, 0, sizeof(scx_exit_info)); -- atomic_set(&scx_exit_type, SCX_EXIT_NONE); -- scx_warned_zero_slice = false; -- -- atomic64_set(&scx_nr_rejected, 0); -- -- /* -- * Keep CPUs stable during enable so that the BPF scheduler can track -- * online CPUs by watching ->on/offline_cpu() after ->init(). -- */ -- cpus_read_lock(); -- -- scx_switch_all_req = true; -- if (scx_ops.init) { -- ret = SCX_CALL_OP_RET(SCX_KF_INIT, init); -- if (ret) { -- ret = ops_sanitize_err("init", ret); -- goto err_disable; -- } -- -- /* -- * Exit early if ops.init() triggered scx_bpf_error(). Not -- * strictly necessary as we'll fail transitioning into ENABLING -- * later but that'd be after calling ops.prep_enable() on all -- * tasks and with -EBUSY which isn't very intuitive. Let's exit -- * early with success so that the condition is notified through -- * ops.exit() like other scx_bpf_error() invocations. -- */ -- if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE) -- goto err_disable; -- } -- -- WARN_ON_ONCE(scx_dsp_buf); -- scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; -- scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch, -- __alignof__(scx_dsp_buf[0])); -- if (!scx_dsp_buf) { -- ret = -ENOMEM; -- goto err_disable; -- } -- -- scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; -- if (ops->timeout_ms) -- scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); -- -- scx_watchdog_timestamp = jiffies; -- queue_delayed_work(system_unbound_wq, &scx_watchdog_work, -- scx_watchdog_timeout / 2); -- -- /* -- * Lock out forks, cgroup on/offlining and moves before opening the -- * floodgate so that they don't wander into the operations prematurely. -- */ -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- if (((void (**)(void))ops)[i]) -- static_branch_enable_cpuslocked(&scx_has_op[i]); -- -- if (ops->flags & SCX_OPS_ENQ_LAST) -- static_branch_enable_cpuslocked(&scx_ops_enq_last); -- -- if (ops->flags & SCX_OPS_ENQ_EXITING) -- static_branch_enable_cpuslocked(&scx_ops_enq_exiting); -- if (scx_ops.cpu_acquire || scx_ops.cpu_release) -- static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); -- -- if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { -- reset_idle_masks(); -- static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); -- } else { -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- } -- -- /* -- * All cgroups should be initialized before letting in tasks. cgroup -- * on/offlining and task migrations are already locked out. -- */ -- ret = scx_cgroup_init(); -- if (ret) -- goto err_disable_unlock; -- -- static_branch_enable_cpuslocked(&__scx_ops_enabled); -- -- /* -- * Enable ops for every task. Fork is excluded by scx_fork_rwsem -- * preventing new tasks from being added. No need to exclude tasks -- * leaving as sched_ext_free() can handle both prepped and enabled -- * tasks. Prep all tasks first and then enable them with preemption -- * disabled. -- */ -- spin_lock_irq(&scx_tasks_lock); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered(&sti))) { -- get_task_struct(p); -- spin_unlock_irq(&scx_tasks_lock); -- -- ret = scx_ops_prepare_task(p, task_group(p)); -- if (ret) { -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n", -- ret, p->comm, p->pid); -- goto err_disable_unlock; -- } -- -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- } -- scx_task_iter_exit(&sti); -- -- /* -- * All tasks are prepped but are still ops-disabled. Ensure that -- * %current can't be scheduled out and switch everyone. -- * preempt_disable() is necessary because we can't guarantee that -- * %current won't be starved if scheduled out while switching. -- */ -- preempt_disable(); -- -- /* -- * From here on, the disable path must assume that tasks have ops -- * enabled and need to be recovered. -- */ -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { -- preempt_enable(); -- spin_unlock_irq(&scx_tasks_lock); -- ret = -EBUSY; -- goto err_disable_unlock; -- } -- -- /* -- * We're fully committed and can't fail. The PREPPED -> ENABLED -- * transitions here are synchronized against sched_ext_free() through -- * scx_tasks_lock. -- */ -- WRITE_ONCE(scx_switching_all, scx_switch_all_req); -- start = sched_clock(); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- tcnt++; -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- scx_ops_enable_task(p); -- __setscheduler_prio(p, p->prio); -- } -- -- check_class_changed(task_rq(p), p, old_class, p->prio); -- } else { -- scx_ops_disable_task(p); -- } -- } -- printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id()); -- scx_task_iter_exit(&sti); -- -- spin_unlock_irq(&scx_tasks_lock); -- preempt_enable(); -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { -- ret = -EBUSY; -- goto err_disable; -- } -- -- if (scx_switch_all_req) -- static_branch_enable_cpuslocked(&__scx_switched_all); -- -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- -- scx_cgroup_config_knobs(); -- printk("\n total duration = %llu , current cpu = %d\n", sched_clock() - total_start, smp_processor_id()); -- -- return 0; -- --err_unlock: -- mutex_unlock(&scx_ops_enable_mutex); -- return ret; -- --err_disable_unlock: -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); --err_disable: -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- /* must be fully disabled before returning */ -- scx_ops_disable(SCX_EXIT_ERROR); -- kthread_flush_work(&scx_ops_disable_work); -- return ret; --} -- --#ifdef CONFIG_SCHED_DEBUG --static const char *scx_ops_enable_state_str[] = { -- [SCX_OPS_PREPPING] = "prepping", -- [SCX_OPS_ENABLING] = "enabling", -- [SCX_OPS_ENABLED] = "enabled", -- [SCX_OPS_DISABLING] = "disabling", -- [SCX_OPS_DISABLED] = "disabled", --}; -- --static int scx_debug_show(struct seq_file *m, void *v) --{ -- mutex_lock(&scx_ops_enable_mutex); -- seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name); -- seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled()); -- seq_printf(m, "%-30s: %d\n", "switching_all", -- READ_ONCE(scx_switching_all)); -- seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all()); -- seq_printf(m, "%-30s: %s\n", "enable_state", -- scx_ops_enable_state_str[scx_ops_enable_state()]); -- seq_printf(m, "%-30s: %llu\n", "nr_rejected", -- atomic64_read(&scx_nr_rejected)); -- mutex_unlock(&scx_ops_enable_mutex); -- return 0; --} -- --static int scx_debug_open(struct inode *inode, struct file *file) --{ -- return single_open(file, scx_debug_show, NULL); --} -- --const struct file_operations sched_ext_fops = { -- .open = scx_debug_open, -- .read = seq_read, -- .llseek = seq_lseek, -- .release = single_release, --}; --#endif -- --/******************************************************************************** -- * bpf_struct_ops plumbing. -- */ --#include --#include --#include -- --extern struct btf *btf_vmlinux; --static const struct btf_type *task_struct_type; -- --static bool bpf_scx_is_valid_access(int off, int size, -- enum bpf_access_type type, -- const struct bpf_prog *prog, -- struct bpf_insn_access_aux *info) --{ -- if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) -- return false; -- if (type != BPF_READ) -- return false; -- if (off % size != 0) -- return false; -- -- return btf_ctx_access(off, size, type, prog, info); --} -- --static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, -- const struct bpf_reg_state *reg, -- int off, int size) --{ -- return 0; --} -- --static const struct bpf_func_proto * --bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) --{ -- switch (func_id) { -- case BPF_FUNC_task_storage_get: -- return &bpf_task_storage_get_proto; -- case BPF_FUNC_task_storage_delete: -- return &bpf_task_storage_delete_proto; -- default: -- return bpf_base_func_proto(func_id); -- } --} -- --const struct bpf_verifier_ops bpf_scx_verifier_ops = { -- .get_func_proto = bpf_scx_get_func_proto, -- .is_valid_access = bpf_scx_is_valid_access, -- .btf_struct_access = bpf_scx_btf_struct_access, --}; -- --static int bpf_scx_init_member(const struct btf_type *t, -- const struct btf_member *member, -- void *kdata, const void *udata) --{ -- const struct sched_ext_ops *uops = udata; -- struct sched_ext_ops *ops = kdata; -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- int ret; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, dispatch_max_batch): -- if (*(u32 *)(udata + moff) > INT_MAX) -- return -E2BIG; -- ops->dispatch_max_batch = *(u32 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, flags): -- if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) -- return -EINVAL; -- ops->flags = *(u64 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, name): -- ret = bpf_obj_name_cpy(ops->name, uops->name, -- sizeof(ops->name)); -- if (ret < 0) -- return ret; -- if (ret == 0) -- return -EINVAL; -- return 1; -- case offsetof(struct sched_ext_ops, timeout_ms): -- if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT) -- return -E2BIG; -- ops->timeout_ms = *(u32 *)(udata + moff); -- return 1; -- } -- -- return 0; --} -- --static int bpf_scx_check_member(const struct btf_type *t, -- const struct btf_member *member, -- const struct bpf_prog *prog) --{ -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, prep_enable): -- case offsetof(struct sched_ext_ops, init): -- case offsetof(struct sched_ext_ops, exit): -- break; -- default: -- /* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */ -- /* if (prog->aux->sleepable) -- return -EINVAL; */ -- break; -- } -- -- return 0; --} -- --static int bpf_scx_reg(void *kdata) --{ -- return scx_ops_enable(kdata); --} -- --static void bpf_scx_unreg(void *kdata) --{ -- scx_ops_disable(SCX_EXIT_UNREG); -- kthread_flush_work(&scx_ops_disable_work); --} -- --static int bpf_scx_init(struct btf *btf) --{ -- u32 type_id; -- -- type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); -- if (type_id < 0) -- return -EINVAL; -- task_struct_type = btf_type_by_id(btf, type_id); -- -- return 0; --} -- --/* "extern" to avoid sparse warning, only used in this file */ --extern struct bpf_struct_ops bpf_sched_ext_ops; -- --struct bpf_struct_ops bpf_sched_ext_ops = { -- .verifier_ops = &bpf_scx_verifier_ops, -- .reg = bpf_scx_reg, -- .unreg = bpf_scx_unreg, -- .check_member = bpf_scx_check_member, -- .init_member = bpf_scx_init_member, -- .init = bpf_scx_init, -- .name = "sched_ext_ops", --}; -- --static void sysrq_handle_sched_ext_reset(u8 key) --{ -- if (scx_ops_helper) -- scx_ops_disable(SCX_EXIT_SYSRQ); -- else -- pr_info("sched_ext: BPF scheduler not yet used\n"); --} -- --static const struct sysrq_key_op sysrq_sched_ext_reset_op = { -- .handler = sysrq_handle_sched_ext_reset, -- .help_msg = "reset-sched-ext(S)", -- .action_msg = "Disable sched_ext and revert all tasks to CFS", -- .enable_mask = SYSRQ_ENABLE_RTNICE, --}; -- --static void kick_cpus_irq_workfn(struct irq_work *irq_work) --{ -- struct rq *this_rq = this_rq(); -- u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); -- int this_cpu = cpu_of(this_rq); -- int cpu; -- -- for_each_cpu(cpu, this_rq->scx->cpus_to_kick) { -- struct rq *rq = cpu_rq(cpu); -- unsigned long flags; -- -- raw_spin_rq_lock_irqsave(rq, flags); -- -- if (cpu_online(cpu) || cpu == this_cpu) { -- if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) && -- rq->curr->sched_class == &ext_sched_class) -- rq->curr->scx->slice = 0; -- pseqs[cpu] = rq->scx->pnt_seq; -- resched_curr(rq); -- } else { -- cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait); -- } -- -- raw_spin_rq_unlock_irqrestore(rq, flags); -- } -- -- for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait, -- cpumask_of(this_cpu)) { -- /* -- * Pairs with smp_store_release() issued by this CPU in -- * scx_notify_pick_next_task() on the resched path. -- * -- * We busy-wait here to guarantee that no other task can be -- * scheduled on our core before the target CPU has entered the -- * resched path. -- */ -- while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu]) -- cpu_relax(); -- } -- -- cpumask_clear(this_rq->scx->cpus_to_kick); -- cpumask_clear(this_rq->scx->cpus_to_preempt); -- cpumask_clear(this_rq->scx->cpus_to_wait); --} -- --void __init init_sched_ext_class(void) --{ -- int cpu; -- u32 v; -- -- /* -- * The following is to prevent the compiler from optimizing out the enum -- * definitions so that BPF scheduler implementations can use them -- * through the generated vmlinux.h. -- */ -- WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | -- SCX_TG_ONLINE | SCX_KICK_PREEMPT); -- -- init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); --#ifdef CONFIG_SMP -- BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -- BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); --#endif -- scx_kick_cpus_pnt_seqs = -- __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * -- num_possible_cpus(), -- __alignof__(scx_kick_cpus_pnt_seqs[0])); -- BUG_ON(!scx_kick_cpus_pnt_seqs); -- -- for_each_possible_cpu(cpu) { -- struct rq *rq = cpu_rq(cpu); -- -- rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL); -- if (rq->scx) -- rq->scx->rq = rq; -- else -- pr_err("fatal error : alloc rq->scx failed!!!\n"); -- -- init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL); -- INIT_LIST_HEAD(&rq->scx->watchdog_list); -- -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL)); -- init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn); -- } -- -- register_sysrq_key('S', &sysrq_sched_ext_reset_op); -- INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); -- scx_cgroup_config_knobs(); --} -- -- --/******************************************************************************** -- * Helpers that can be called from the BPF scheduler. -- */ --#include -- --/* Disables missing prototype warnings for kfuncs */ --__diag_push(); --__diag_ignore_all("-Wmissing-prototypes", -- "Global functions as their definitions will be in vmlinux BTF"); -- --/** -- * scx_bpf_switch_all - Switch all tasks into SCX -- * @into_scx: switch direction -- * -- * If @into_scx is %true, all existing and future non-dl/rt tasks are switched -- * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on -- * SCX. The actual switching is asynchronous. Can be called from ops.init(). -- */ --void scx_bpf_switch_all(void) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return; -- -- scx_switch_all_req = true; --} -- --BTF_SET8_START(scx_kfunc_ids_init) --BTF_ID_FLAGS(func, scx_bpf_switch_all) --BTF_SET8_END(scx_kfunc_ids_init) -- --static const struct btf_kfunc_id_set scx_kfunc_set_init = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_init, --}; -- --/** -- * scx_bpf_create_dsq - Create a custom DSQ -- * @dsq_id: DSQ to create -- * @node: NUMA node to allocate from -- * -- * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(), -- * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move(). -- */ --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return -EINVAL; -- -- if (unlikely(node >= (int)nr_node_ids || -- (node < 0 && node != NUMA_NO_NODE))) -- return -EINVAL; -- return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); --} -- --BTF_SET8_START(scx_kfunc_ids_sleepable) --BTF_ID_FLAGS(func, scx_bpf_create_dsq) --BTF_SET8_END(scx_kfunc_ids_sleepable) -- --static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_sleepable, --}; -- --static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) --{ -- if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) -- return false; -- -- lockdep_assert_irqs_disabled(); -- -- if (unlikely(!p)) { -- scx_ops_error("called with NULL task"); -- return false; -- } -- -- if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { -- scx_ops_error("invalid enq_flags 0x%llx", enq_flags); -- return false; -- } -- -- return true; --} -- --static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) --{ -- struct task_struct *ddsp_task; -- int idx; -- -- ddsp_task = __this_cpu_read(direct_dispatch_task); -- if (ddsp_task) { -- direct_dispatch(ddsp_task, p, dsq_id, enq_flags); -- return; -- } -- -- idx = __this_cpu_read(scx_dsp_ctx.buf_cursor); -- if (unlikely(idx >= scx_dsp_max_batch)) { -- scx_ops_error("dispatch buffer overflow"); -- return; -- } -- -- this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){ -- .task = p, -- .qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK, -- .dsq_id = dsq_id, -- .enq_flags = enq_flags, -- }; -- __this_cpu_inc(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe -- * to call this function spuriously. Can be called from ops.enqueue() and -- * ops.dispatch(). -- * -- * When called from ops.enqueue(), it's for direct dispatch and @p must match -- * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the -- * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be -- * on the target CPU in the first place. -- * -- * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id -- * and this function can be called upto ops.dispatch_max_batch times to dispatch -- * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the -- * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. -- * -- * This function doesn't have any locking restrictions and may be called under -- * BPF locks (in the future when BPF introduces more flexible locking). -- * -- * @p is allowed to run for @slice. The scheduling path is triggered on slice -- * exhaustion. If zero, the current residual slice is maintained. If -- * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with -- * scx_bpf_kick_cpu() to trigger scheduling. -- */ --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- scx_dispatch_commit(p, dsq_id, enq_flags); --} -- --/** -- * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. -- * Tasks queued into the priority queue are ordered by @vtime and always -- * consumed after the tasks in the FIFO queue. All other aspects are identical -- * to scx_bpf_dispatch(). -- * -- * @vtime ordering is according to time_before64() which considers wrapping. A -- * numerically larger vtime may indicate an earlier position in the ordering and -- * vice-versa. -- */ --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 vtime, u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- p->scx->dsq_vtime = vtime; -- -- scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); --} -- --BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime) --BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_enqueue_dispatch, --}; -- --/** -- * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots -- * -- * Can only be called from ops.dispatch(). -- */ --u32 scx_bpf_dispatch_nr_slots(void) --{ -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return 0; -- -- return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ -- * @dsq_id: DSQ to consume -- * -- * Consume a task from the non-local DSQ identified by @dsq_id and transfer it -- * to the current CPU's local DSQ for execution. Can only be called from -- * ops.dispatch(). -- * -- * This function flushes the in-flight dispatches from scx_bpf_dispatch() before -- * trying to consume the specified DSQ. It may also grab rq locks and thus can't -- * be called under any BPF locks. -- * -- * Returns %true if a task has been consumed, %false if there isn't any task to -- * consume. -- */ --bool scx_bpf_consume(u64 dsq_id) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- struct scx_dispatch_q *dsq; -- -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return false; -- -- flush_dispatch_buf(dspc->rq, dspc->rf); -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); -- return false; -- } -- -- if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) { -- /* -- * A successfully consumed task can be dequeued before it starts -- * running while the CPU is trying to migrate other dispatched -- * tasks. Bump nr_tasks to tell balance_scx() to retry on empty -- * local DSQ. -- */ -- dspc->nr_tasks++; -- return true; -- } else { -- return false; -- } --} -- --BTF_SET8_START(scx_kfunc_ids_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) --BTF_ID_FLAGS(func, scx_bpf_consume) --BTF_SET8_END(scx_kfunc_ids_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_dispatch, --}; -- --/** -- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ -- * -- * Iterate over all of the tasks currently enqueued on the local DSQ of the -- * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of -- * processed tasks. Can only be called from ops.cpu_release(). -- */ --u32 scx_bpf_reenqueue_local(void) --{ -- u32 nr_enqueued, i; -- struct rq *rq; -- struct scx_rq *scx_rq; -- -- if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) -- return 0; -- -- rq = cpu_rq(smp_processor_id()); -- lockdep_assert_rq_held(rq); -- scx_rq = rq->scx; -- -- /* -- * Get the number of tasks on the local DSQ before iterating over it to -- * pull off tasks. The enqueue callback below can signal that it wants -- * the task to stay on the local DSQ, and we want to prevent the BPF -- * scheduler from causing us to loop indefinitely. -- */ -- nr_enqueued = scx_rq->local_dsq.nr; -- for (i = 0; i < nr_enqueued; i++) { -- struct task_struct *p; -- -- p = first_local_task(rq); -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- WARN_ON_ONCE(p->scx->holding_cpu != -1); -- dispatch_dequeue(scx_rq, p); -- do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); -- } -- -- return nr_enqueued; --} -- --BTF_SET8_START(scx_kfunc_ids_cpu_release) --BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) --BTF_SET8_END(scx_kfunc_ids_cpu_release) -- --static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_cpu_release, --}; -- --/** -- * scx_bpf_kick_cpu - Trigger reschedule on a CPU -- * @cpu: cpu to kick -- * @flags: SCX_KICK_* flags -- * -- * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or -- * trigger rescheduling on a busy CPU. This can be called from any online -- * scx_ops operation and the actual kicking is performed asynchronously through -- * an irq work. -- */ --void scx_bpf_kick_cpu(s32 cpu, u64 flags) --{ -- struct rq *rq; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d", cpu); -- return; -- } -- -- preempt_disable(); -- rq = this_rq(); -- -- /* -- * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting -- * rq locks. We can probably be smarter and avoid bouncing if called -- * from ops which don't hold a rq lock. -- */ -- cpumask_set_cpu(cpu, rq->scx->cpus_to_kick); -- if (flags & SCX_KICK_PREEMPT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt); -- if (flags & SCX_KICK_WAIT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_wait); -- -- irq_work_queue(&rq->scx->kick_cpus_irq_work); -- preempt_enable(); --} -- --/** -- * scx_bpf_dsq_nr_queued - Return the number of queued tasks -- * @dsq_id: id of the DSQ -- * -- * Return the number of tasks in the DSQ matching @dsq_id. If not found, -- * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops -- * operations. -- */ --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) --{ -- struct scx_dispatch_q *dsq; -- -- lockdep_assert(rcu_read_lock_any_held()); -- -- if (dsq_id == SCX_DSQ_LOCAL) { -- return this_rq()->scx->local_dsq.nr; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (ops_cpu_valid(cpu)) -- return cpu_rq(cpu)->scx->local_dsq.nr; -- } else { -- dsq = find_non_local_dsq(dsq_id); -- if (dsq) -- return dsq->nr; -- } -- return -ENOENT; --} -- --/** -- * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state -- * @cpu: cpu to test and clear idle for -- * -- * Returns %true if @cpu was idle and its idle state was successfully cleared. -- * %false otherwise. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return false; -- } -- -- if (ops_cpu_valid(cpu)) -- return test_and_clear_cpu_idle(cpu); -- else -- return false; --} -- --/** -- * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu -- * @cpus_allowed: Allowed cpumask -- * -- * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked -- * idle cpu number on success. -%EBUSY if no matching cpu was found. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return -EBUSY; -- } -- -- return scx_pick_idle_cpu(cpus_allowed); --} -- --/** -- * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking -- * per-CPU cpumask. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_cpumask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.cpu; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, -- * per-physical-core cpumask. Can be used to determine if an entire physical -- * core is free. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_smtmask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.smt; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to -- * either the percpu, or SMT idle-tracking cpumask. -- */ --void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) --{ -- /* -- * Empty function body because we aren't actually acquiring or -- * releasing a reference to a global idle cpumask, which is read-only -- * in the caller and is never released. The acquire / release semantics -- * here are just used to make the cpumask is a trusted pointer in the -- * caller. -- */ --} -- --struct scx_bpf_error_bstr_bufs { -- u64 data[MAX_BPRINTF_VARARGS]; -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs); -- --/** -- * scx_bpf_error_bstr - Indicate fatal error -- * @fmt: error message format string -- * @data: format string parameters packaged using ___bpf_fill() macro -- * @data__sz: @data len, must end in '__sz' for the verifier -- * -- * Indicate that the BPF scheduler encountered a fatal error and initiate ops -- * disabling. -- */ --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) --{ -- struct scx_bpf_error_bstr_bufs *bufs; -- unsigned long flags; -- struct bpf_bprintf_data args = {}; -- int ret; -- -- local_irq_save(flags); -- bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs); -- -- if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || -- (data__sz && !data)) { -- scx_ops_error("invalid data=%p and data__sz=%u", -- (void *)data, data__sz); -- goto out_restore; -- } -- -- ret = copy_from_kernel_nofault(bufs->data, data, data__sz); -- if (ret) { -- scx_ops_error("failed to read data fields (%d)", ret); -- goto out_restore; -- } -- -- ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args); -- if (ret < 0) { -- scx_ops_error("failed to format prepration (%d)", ret); -- goto out_restore; -- } -- -- ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt, -- args.bin_args); -- bpf_bprintf_cleanup(&args); -- if (ret < 0) { -- scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format", -- fmt, data, data__sz); -- goto out_restore; -- } -- -- scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg); --out_restore: -- local_irq_restore(flags); --} -- --/** -- * scx_bpf_destroy_dsq - Destroy a custom DSQ -- * @dsq_id: DSQ to destroy -- * -- * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with -- * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is -- * empty and no further tasks are dispatched to it. Ignored if called on a DSQ -- * which doesn't exist. Can be called from any online scx_ops operations. -- */ --void scx_bpf_destroy_dsq(u64 dsq_id) --{ -- destroy_dsq(dsq_id); --} -- --/** -- * scx_bpf_task_running - Is task currently running? -- * @p: task of interest -- */ --bool scx_bpf_task_running(const struct task_struct *p) --{ -- return task_rq(p)->curr == p; --} -- --/** -- * scx_bpf_task_cpu - CPU a task is currently associated with -- * @p: task of interest -- */ --s32 scx_bpf_task_cpu(const struct task_struct *p) --{ -- return task_cpu(p); --} -- --/** -- * scx_bpf_task_cgroup - Return the sched cgroup of a task -- * @p: task of interest -- * -- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with -- * from the scheduler's POV. SCX operations should use this function to -- * determine @p's current cgroup as, unlike following @p->cgroups, -- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all -- * rq-locked operations. Can be called on the parameter tasks of rq-locked -- * operations. The restriction guarantees that @p's rq is locked by the caller. -- */ --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) --{ -- struct task_group *tg = p->sched_task_group; -- struct cgroup *cgrp = &cgrp_dfl_root.cgrp; -- -- if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) -- goto out; -- -- /* -- * A task_group may either be a cgroup or an autogroup. In the latter -- * case, @tg->css.cgroup is %NULL. A task_group can't become the other -- * kind once created. -- */ -- if (tg && tg->css.cgroup) -- cgrp = tg->css.cgroup; -- else -- cgrp = &cgrp_dfl_root.cgrp; --out: -- cgroup_get(cgrp); -- return cgrp; --} -- --BTF_SET8_START(scx_kfunc_ids_any) --BTF_ID_FLAGS(func, scx_bpf_kick_cpu) --BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) --BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) --BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu) --BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) --BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) --BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) --BTF_ID_FLAGS(func, scx_bpf_task_running) --BTF_ID_FLAGS(func, scx_bpf_task_cpu) --BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE) --BTF_SET8_END(scx_kfunc_ids_any) -- --static const struct btf_kfunc_id_set scx_kfunc_set_any = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_any, --}; -- --__diag_pop(); -- --/* -- * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set() -- * needs most of the system to be up. -- */ --static int __init register_ext_kfuncs(void) --{ -- int ret; -- -- /* -- * Some kfuncs are context-sensitive and can only be called from -- * specific SCX ops. They are grouped into BTF sets accordingly. -- * Unfortunately, BPF currently doesn't have a way of enforcing such -- * restrictions. Eventually, the verifier should be able to enforce -- * them. For now, register them the same and make each kfunc explicitly -- * check using scx_kf_allowed(). -- */ -- if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_init)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_sleepable)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_enqueue_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_cpu_release)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_any))) { -- pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret); -- return ret; -- } -- -- return 0; --} --__initcall(register_ext_kfuncs); -diff --git b/kernel/sched/ext.h a/kernel/sched/ext.h -deleted file mode 100755 -index fba8ed845f99..000000000000 ---- b/kernel/sched/ext.h -+++ /dev/null -@@ -1,257 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ -- -- --/* -- * Tag marking a kernel function as a kfunc. This is meant to minimize the -- * amount of copy-paste that kfunc authors have to include for correctness so -- * as to avoid issues such as the compiler inlining or eliding either a static -- * kfunc, or a global kfunc in an LTO build. -- */ --#define __bpf_kfunc __used noinline -- --enum scx_wake_flags { -- /* expose select WF_* flags as enums */ -- SCX_WAKE_EXEC = WF_EXEC, -- SCX_WAKE_FORK = WF_FORK, -- SCX_WAKE_TTWU = WF_TTWU, -- SCX_WAKE_SYNC = WF_SYNC, --}; -- --enum scx_enq_flags { -- /* expose select ENQUEUE_* flags as enums */ -- SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, -- SCX_ENQ_HEAD = ENQUEUE_HEAD, -- -- /* high 32bits are SCX specific */ -- -- /* -- * Set the following to trigger preemption when calling -- * scx_bpf_dispatch() with a local dsq as the target. The slice of the -- * current task is cleared to zero and the CPU is kicked into the -- * scheduling path. Implies %SCX_ENQ_HEAD. -- */ -- SCX_ENQ_PREEMPT = 1LLU << 32, -- -- /* -- * The task being enqueued was previously enqueued on the current CPU's -- * %SCX_DSQ_LOCAL, but was removed from it in a call to the -- * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was -- * invoked in a ->cpu_release() callback, and the task is again -- * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the -- * task will not be scheduled on the CPU until at least the next invocation -- * of the ->cpu_acquire() callback. -- */ -- SCX_ENQ_REENQ = 1LLU << 40, -- -- /* -- * The task being enqueued is the only task available for the cpu. By -- * default, ext core keeps executing such tasks but when -- * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with -- * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set. -- * -- * If the BPF scheduler wants to continue executing the task, -- * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately. -- * If the task gets queued on a different dsq or the BPF side, the BPF -- * scheduler is responsible for triggering a follow-up scheduling event. -- * Otherwise, Execution may stall. -- */ -- SCX_ENQ_LAST = 1LLU << 41, -- -- /* -- * A hint indicating that it's advisable to enqueue the task on the -- * local dsq of the currently selected CPU. Currently used by -- * select_cpu_dfl() and together with %SCX_ENQ_LAST. -- */ -- SCX_ENQ_LOCAL = 1LLU << 42, -- -- /* high 8 bits are internal */ -- __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, -- -- SCX_ENQ_CLEAR_OPSS = 1LLU << 56, -- SCX_ENQ_DSQ_PRIQ = 1LLU << 57, --}; -- --enum scx_deq_flags { -- /* expose select DEQUEUE_* flags as enums */ -- SCX_DEQ_SLEEP = DEQUEUE_SLEEP, -- -- /* high 32bits are SCX specific */ -- -- /* -- * The generic core-sched layer decided to execute the task even though -- * it hasn't been dispatched yet. Dequeue from the BPF side. -- */ -- SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, --}; -- --enum scx_tg_flags { -- SCX_TG_ONLINE = 1U << 0, -- SCX_TG_INITED = 1U << 1, --}; -- --enum scx_kick_flags { -- SCX_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -- SCX_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ --}; -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --extern const struct sched_class ext_sched_class; --extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops; --extern const struct file_operations sched_ext_fops; --extern unsigned long scx_watchdog_timeout; --extern unsigned long scx_watchdog_timestamp; -- --DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); --DECLARE_STATIC_KEY_FALSE(__scx_switched_all); --#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) --#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) -- --DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); -- --bool task_on_scx(struct task_struct *p); --void scx_pre_fork(struct task_struct *p); --int scx_fork(struct task_struct *p); --void scx_post_fork(struct task_struct *p); --void scx_cancel_fork(struct task_struct *p); --int scx_check_setscheduler(struct task_struct *p, int policy); --bool scx_can_stop_tick(struct rq *rq); --void init_sched_ext_class(void); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...); --#define scx_ops_error(fmt, args...) \ -- scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args) -- --void __scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active); -- --static inline void scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active) --{ -- if (!scx_enabled()) -- return; --#ifdef CONFIG_SMP -- /* -- * Pairs with the smp_load_acquire() issued by a CPU in -- * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -- * resched. -- */ -- smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1); --#endif -- if (!static_branch_unlikely(&scx_ops_cpu_preempt)) -- return; -- __scx_notify_pick_next_task(rq, p, active); --} -- --//extern void scx_scheduler_tick(void); --static inline void scx_notify_sched_tick(void) --{ -- unsigned long last_check; -- -- if (!scx_enabled()) -- return; -- //scx_scheduler_tick(); -- -- last_check = scx_watchdog_timestamp; -- if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "watchdog failed to check in for %u.%03us", -- dur_ms / 1000, dur_ms % 1000); -- } --} -- --static inline const struct sched_class *next_active_class(const struct sched_class *class) --{ -- class++; -- if (scx_switched_all() && class == &fair_sched_class) -- class++; -- if (!scx_enabled() && class == &ext_sched_class) -- class++; -- return class; --} -- --#define for_active_class_range(class, _from, _to) \ -- for (class = (_from); class != (_to); class = next_active_class(class)) -- --#define for_each_active_class(class) \ -- for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -- --/* -- * SCX requires a balance() call before every pick_next_task() call including -- * when waking up from idle. -- */ --#define for_balance_class_range(class, prev_class, end_class) \ -- for_active_class_range(class, (prev_class) > &ext_sched_class ? \ -- &ext_sched_class : (prev_class), (end_class)) -- --#ifdef CONFIG_SCHED_CORE --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi); --#endif -- --#else /* CONFIG_SCHED_CLASS_EXT */ -- --#define scx_enabled() false --#define scx_switched_all() false -- --static inline void scx_pre_fork(struct task_struct *p) {} --static inline int scx_fork(struct task_struct *p) { return 0; } --static inline void scx_post_fork(struct task_struct *p) {} --static inline void scx_cancel_fork(struct task_struct *p) {} --static inline int scx_check_setscheduler(struct task_struct *p, -- int policy) { return 0; } --static inline bool scx_can_stop_tick(struct rq *rq) { return true; } --static inline void init_sched_ext_class(void) {} --static inline void scx_notify_pick_next_task(struct rq *rq, -- const struct task_struct *p, -- const struct sched_class *active) {} --static inline void scx_notify_sched_tick(void) {} -- --#define for_each_active_class for_each_class --#define for_balance_class_range for_class_range -- --#endif /* CONFIG_SCHED_CLASS_EXT */ -- --#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) --void __scx_update_idle(struct rq *rq, bool idle); -- --static inline void scx_update_idle(struct rq *rq, bool idle) --{ -- if (scx_enabled()) -- __scx_update_idle(rq, idle); --} --#else --static inline void scx_update_idle(struct rq *rq, bool idle) {} --#endif -- --#ifdef CONFIG_CGROUP_SCHED --#ifdef CONFIG_EXT_GROUP_SCHED --int scx_tg_online(struct task_group *tg); --void scx_tg_offline(struct task_group *tg); --int scx_cgroup_can_attach(struct cgroup_taskset *tset); --void scx_move_task(struct task_struct *p); --void scx_cgroup_finish_attach(void); --void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); --void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); --#else /* CONFIG_EXT_GROUP_SCHED */ --static inline int scx_tg_online(struct task_group *tg) { return 0; } --static inline void scx_tg_offline(struct task_group *tg) {} --static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } --static inline void scx_move_task(struct task_struct *p) {} --static inline void scx_cgroup_finish_attach(void) {} --static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} --static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} --#endif /* CONFIG_EXT_GROUP_SCHED */ --#endif /* CONFIG_CGROUP_SCHED */ -diff --git b/kernel/sched/hmbird.h a/kernel/sched/hmbird.h -new file mode 100755 -index 000000000000..fa76c7f09977 ---- /dev/null -+++ a/kernel/sched/hmbird.h -@@ -0,0 +1,342 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#ifndef _HMBIRD_H_ -+#define _HMBIRD_H_ -+ -+/* -+ * Tag marking a kernel function as a kfunc. This is meant to minimize the -+ * amount of copy-paste that kfunc authors have to include for correctness so -+ * as to avoid issues such as the compiler inlining or eliding either a static -+ * kfunc, or a global kfunc in an LTO build. -+ */ -+ -+#define DEBUG_INTERNAL (1 << 0) -+#define DEBUG_INFO_TRACE (1 << 1) -+#define DEBUG_INFO_SYSTRACE (1 << 2) -+ -+#define NUMS_CGROUP_KINDS (512) -+#define SLIM_FOR_SGAME (1 << 0) -+#define SLIM_FOR_GENSHIN (1 << 1) -+ -+#ifdef MAX_TASK_NR -+#define MAX_KEY_THREAD_RECORD ((MAX_TASK_NR + 1) >> 1) -+#else -+#define MAX_KEY_THREAD_RECORD (1 << 3) -+#endif /* MAX_TASK_NR */ -+ -+#define TOP_TASK_SHIFT (8) -+#define TOP_TASK_MAX (1 << TOP_TASK_SHIFT) -+#ifndef TOP_TASK_BITS_MASK -+#define TOP_TASK_BITS_MASK (TOP_TASK_MAX - 1) -+#endif /* TOP_TASK_BITS_MASK */ -+ -+extern struct md_info_t *md_info; -+ -+#define debug_enabled() \ -+ (unlikely(hmbirdcore_debug)) -+ -+#define hmbird_debug(fmt, ...) \ -+ pr_info(":"fmt, ##__VA_ARGS__) -+ -+#define hmbird_err(id, fmt, ...) \ -+do { \ -+ pr_err(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_deferred_err(id, fmt, ...) \ -+do { \ -+ printk_deferred(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \ -+do { \ -+ if (unlikely(cond)) { \ -+ hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+ } \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_info_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_info_trace(fmt, ...) -+#endif -+ -+#define hmbird_info_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+ } \ -+} while (0) -+ -+#define hmbird_output_systrace(fmt, ...) \ -+do { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_internal_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_internal_trace(fmt, ...) -+#endif -+ -+#define hmbird_internal_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) { \ -+ hmbird_output_systrace(fmt, ##__VA_ARGS__); \ -+ } \ -+} while (0) -+ -+enum hmbird_wake_flags { -+ /* expose select WF_* flags as enums */ -+ HMBIRD_WAKE_EXEC = WF_EXEC, -+ HMBIRD_WAKE_FORK = WF_FORK, -+ HMBIRD_WAKE_TTWU = WF_TTWU, -+ HMBIRD_WAKE_SYNC = WF_SYNC, -+}; -+ -+enum hmbird_enq_flags { -+ /* expose select ENQUEUE_* flags as enums */ -+ HMBIRD_ENQ_WAKEUP = ENQUEUE_WAKEUP, -+ HMBIRD_ENQ_HEAD = ENQUEUE_HEAD, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * Set the following to trigger preemption when calling -+ * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the -+ * current task is cleared to zero and the CPU is kicked into the -+ * scheduling path. Implies %HMBIRD_ENQ_HEAD. -+ */ -+ HMBIRD_ENQ_PREEMPT = 1LLU << 32, -+ HMBIRD_ENQ_REENQ = 1LLU << 40, -+ HMBIRD_ENQ_LAST = 1LLU << 41, -+ -+ /* -+ * A hint indicating that it's advisable to enqueue the task on the -+ * local dsq of the currently selected CPU. Currently used by -+ * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST. -+ */ -+ HMBIRD_ENQ_LOCAL = 1LLU << 42, -+ -+ /* high 8 bits are internal */ -+ __HMBIRD_ENQ_INTERNAL_MASK = 0xffLLU << 56, -+ -+ HMBIRD_ENQ_CLEAR_OPSS = 1LLU << 56, -+ HMBIRD_ENQ_DSQ_PRIQ = 1LLU << 57, -+}; -+ -+enum hmbird_deq_flags { -+ /* expose select DEQUEUE_* flags as enums */ -+ HMBIRD_DEQ_SLEEP = DEQUEUE_SLEEP, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * The generic core-sched layer decided to execute the task even though -+ * it hasn't been dispatched yet. Dequeue from the BPF side. -+ */ -+ HMBIRD_DEQ_CORE_SCHED_EXEC = 1LLU << 32, -+}; -+ -+enum hmbird_kick_flags { -+ HMBIRD_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -+ HMBIRD_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ -+}; -+ -+enum hmbird_task_prop_type { -+ HMBIRD_TASK_PROP_COMMON, -+ HMBIRD_TASK_PROP_PIPELINE, -+ HMBIRD_TASK_PROP_DEBUG_OR_LOG, -+ HMBIRD_TASK_PROP_HIGH_LOAD, -+ HMBIRD_TASK_PROP_IO, -+ HMBIRD_TASK_PROP_NETWORK, -+ HMBIRD_TASK_PROP_PERIODIC, -+ HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL, -+ HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL, -+ HMBIRD_TASK_PROP_ISOLATE, -+ HMBIRD_TASK_PROP_MAX, -+}; -+ -+enum hmbird_preempt_policy_id { -+ HMBIRD_PREEMPT_POLICY_NONE, -+ HMBIRD_PREEMPT_POLICY_PRIO_BASED, -+}; -+ -+extern const struct sched_class hmbird_sched_class; -+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops; -+extern const struct file_operations sched_hmbird_fops; -+extern unsigned long hmbird_watchdog_timeout; -+extern unsigned long hmbird_watchdog_timestamp; -+ -+enum scx_rq_flags { -+ HMBIRD_RQ_CAN_STOP_TICK = 1 << 0, -+}; -+ -+struct hmbird_rq { -+ struct hmbird_dispatch_q local_dsq; -+ struct list_head watchdog_list; -+ u64 ops_qseq; -+ u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -+ u32 nr_running; -+ u32 flags; -+ bool cpu_released; -+ cpumask_var_t cpus_to_kick; -+ cpumask_var_t cpus_to_preempt; -+ cpumask_var_t cpus_to_wait; -+ u64 pnt_seq; -+ u64 *prev_runnable_sum_fixed; -+ u32 *prev_window_size; -+ struct irq_work kick_cpus_irq_work; -+ bool pipeline; -+ bool exclusive; -+ bool period_disallow; -+ bool nonperiod_disallow; -+ struct rq *rq; -+ struct hmbird_sched_rq_stats *srq; -+}; -+ -+struct hmbird_sched_change_guard { -+ struct task_struct *p; -+ struct rq *rq; -+ bool queued; -+ bool running; -+ bool done; -+}; -+ -+extern struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -+ -+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags); -+ -+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -+ for (struct hmbird_sched_change_guard __cg = \ -+ hmbird_sched_change_guard_init(__rq, __p, __flags); \ -+ !__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags)) -+ -+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+bool task_on_hmbird(struct task_struct *p); -+int hmbird_pre_fork(struct task_struct *p); -+void hmbird_post_fork(struct task_struct *p); -+void hmbird_cancel_fork(struct task_struct *p); -+int hmbird_check_setscheduler(struct task_struct *p, int policy); -+bool hmbird_can_stop_tick(struct rq *rq); -+void init_sched_hmbird_class(void); -+ -+void hmbird_ops_exit(void); -+#define hmbird_ops_error(fmt, ...) \ -+do { \ -+ hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__); \ -+ hmbird_ops_exit(); \ -+} while (0) -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active); -+ -+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight) -+{ -+ return clamp_t(unsigned long, -+ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -+ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); -+} -+ -+static inline void hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active) -+{ -+ if (!hmbird_enabled()) -+ return; -+#ifdef CONFIG_SMP -+ /* -+ * Pairs with the smp_load_acquire() issued by a CPU in -+ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -+ * resched. -+ */ -+ smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1); -+#endif -+ if (!static_branch_unlikely(&hmbird_ops_cpu_preempt)) -+ return; -+ __hmbird_notify_pick_next_task(rq, p, active); -+} -+ -+extern void hmbird_scheduler_tick(void); -+void scan_timeout(struct rq *rq); -+void hmbird_notify_sched_tick(void); -+ -+static inline const struct sched_class *next_active_class(const struct sched_class *class) -+{ -+ class++; -+ -+ if (hmbird_enabled() && class == &fair_sched_class) -+ class++; -+ if (!hmbird_enabled() && class == &hmbird_sched_class) -+ class++; -+ return class; -+} -+ -+#define for_active_class_range(class, _from, _to) \ -+ for (class = (_from); class != (_to); class = next_active_class(class)) -+ -+#define for_each_active_class(class) \ -+ for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -+ -+/* -+ * HMBIRD requires a balance() call before every pick_next_task() call including -+ * when waking up from idle. -+ */ -+#define for_balance_class_range(class, prev_class, end_class) \ -+ for_active_class_range(class, (prev_class) > &hmbird_sched_class ? \ -+ &hmbird_sched_class : (prev_class), (end_class)) -+ -+#define MIN_CGROUP_DL_IDX (5) /* 8ms */ -+#define DEFAULT_CGROUP_DL_IDX (8) /* 64ms */ -+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS]; -+ -+ -+void __hmbird_update_idle(struct rq *rq, bool idle); -+ -+static inline void hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ if (hmbird_enabled()) -+ __hmbird_update_idle(rq, idle); -+} -+ -+int hmbird_ctrl(bool enable); -+ -+int hmbird_tg_online(struct task_group *tg); -+ -+ -+static inline u16 hmbird_task_util(struct task_struct *p) -+{ -+ return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0; -+} -+u16 hmbird_cpu_util(int cpu); -+ -+bool get_hmbird_ops_enabled(void); -+bool get_non_hmbird_task(void); -+void set_cpu_cluster(u64 cpu_cluster); -+#endif /*_EXT_H_*/ -diff --git b/kernel/sched/hmbird/hmbird.c a/kernel/sched/hmbird/hmbird.c -new file mode 100755 -index 000000000000..86f9fd092424 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird.c -@@ -0,0 +1,4354 @@ -+// SPDX-License-Identifier: GPL-2.0 -+ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#include -+#include -+ -+#include "slim.h" -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+#include -+ -+#define CLUSTER_SEPARATE -+ -+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0); -+atomic_t non_hmbird_task; -+atomic_t hmbird_module_loaded = ATOMIC_INIT(0); -+int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0}; -+ -+enum hmbird_internal_consts { -+ HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ, -+}; -+ -+enum hmbird_ops_enable_state { -+ HMBIRD_OPS_PREPPING, -+ HMBIRD_OPS_ENABLING, -+ HMBIRD_OPS_ENABLED, -+ HMBIRD_OPS_DISABLING, -+ HMBIRD_OPS_DISABLED, -+}; -+ -+static inline void put_hmbird_ts(struct task_struct *p) -+{ -+ kfree((void *)p->android_oem_data1[HMBIRD_TS_IDX]); -+ p->android_oem_data1[HMBIRD_TS_IDX] = 0; -+} -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+/* -+ * hmbird_entity->ops_state -+ * -+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler. -+ * State transitions look as follows: -+ * -+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING -+ * ^ | | -+ * | v v -+ * \-------------------------------/ -+ * -+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -+ * sites for explanations on the conditions being waited upon and why they are -+ * safe. Transitions out of them into NONE or QUEUED must store_release and the -+ * waiters should load_acquire. -+ * -+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether -+ * any given task can be dispatched by the BPF scheduler at all times and thus -+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core -+ * can safely reject invalid dispatches. -+ */ -+enum hmbird_ops_state { -+ HMBIRD_OPSS_NONE, /* owned by the HMBIRD core */ -+ HMBIRD_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -+ HMBIRD_OPSS_QUEUED, /* owned by the BPF scheduler */ -+ HMBIRD_OPSS_DISPATCHING, /* in transit back to the HMBIRD core */ -+ -+ /* -+ * QSEQ brands each QUEUED instance so that, when dispatch races -+ * dequeue/requeue, the dispatcher can tell whether it still has a claim -+ * on the task being dispatched. -+ */ -+ HMBIRD_OPSS_QSEQ_SHIFT = 2, -+ HMBIRD_OPSS_STATE_MASK = (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1, -+ HMBIRD_OPSS_QSEQ_MASK = ~HMBIRD_OPSS_STATE_MASK, -+}; -+ -+enum switch_stat { -+ HMBIRD_DISABLED, -+ HMBIRD_SWITCH_PREP, -+ HMBIRD_RQ_SWITCH_BEGIN, -+ HMBIRD_RQ_SWITCH_DONE, -+ HMBIRD_ENABLED, -+}; -+enum switch_stat curr_ss; -+ -+/* -+ * During exit, a task may schedule after losing its PIDs. When disabling the -+ * BPF scheduler, we need to be able to iterate tasks in every state to -+ * guarantee system safety. Maintain a dedicated task list which contains every -+ * task between its fork and eventual free. -+ */ -+DEFINE_SPINLOCK(hmbird_tasks_lock); -+static LIST_HEAD(hmbird_tasks); -+ -+/* ops enable/disable */ -+static struct kthread_worker *hmbird_ops_helper; -+static DEFINE_MUTEX(hmbird_ops_enable_mutex); -+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem); -+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED); -+ -+static bool hmbird_warned_zero_slice; -+enum hmbird_switch_type sw_type; -+static int skip_num[MAX_GLOBAL_DSQS]; -+static int big_distribute_mask_prev; -+static int little_distribute_mask_prev; -+ -+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0); -+ -+/* -+ * The maximum amount of time in jiffies that a task may be runnable without -+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger -+ * hmbird_ops_error(). -+ */ -+unsigned long hmbird_watchdog_timeout; -+ -+/* -+ * The last time the delayed work was run. This delayed work relies on -+ * ksoftirqd being able to run to service timer interrupts, so it's possible -+ * that this work itself could get wedged. To account for this, we check that -+ * it's not stalled in the timer tick, and trigger an error if it is. -+ */ -+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES; -+ -+static struct delayed_work hmbird_watchdog_work; -+static struct work_struct hmbird_err_exit_work; -+ -+/* idle tracking */ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_CPUMASK_OFFSTACK -+#define CL_ALIGNED_IF_ONSTACK -+#else -+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp -+#endif -+ -+static struct { -+ cpumask_var_t cpu; -+ cpumask_var_t smt; -+} idle_masks CL_ALIGNED_IF_ONSTACK; -+ -+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus; -+#endif /* CONFIG_SMP */ -+ -+/* dispatch queues */ -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global; -+ -+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128}; -+u32 pcp_dsq_deadline = 20; -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS]; -+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq); -+ -+static u64 max_hmbird_dsq_internal_id; -+ -+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/ -+#define CLUSTER_SEPARATE_IDX (8) -+#define GDSQS_ID_BASE (3) -+#define UX_COMPATIBLE_IDX (4) -+#define NON_PERIOD_START (5) -+#define NON_PERIOD_END (MAX_GLOBAL_DSQS) -+#define CREATE_DSQ_LEVEL_WITHIN (1) -+ -+struct hmbird_sched_info { -+ spinlock_t lock; -+ int curr_idx[2]; -+ int rtime[MAX_GLOBAL_DSQS]; -+}; -+ -+struct pcp_sched_info { -+ s64 pcp_seq; -+ int rtime; -+ bool pcp_round; -+}; -+ -+/* -+ * pcp_info may rw by another cpu. -+ * protected by rq->lock. -+ */ -+atomic64_t pcp_dsq_round; -+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info); -+ -+struct md_info_t *md_info; -+ -+static int b_rescue_l, l_rescue_b; -+static struct hmbird_sched_info sinfo; -+ -+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC; -+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = { -+ 0, 0, 0, 0, 0, -+ 32 * NSEC_PER_MSEC, -+ 20 * NSEC_PER_MSEC, -+ 14 * NSEC_PER_MSEC, -+ 8 * NSEC_PER_MSEC, -+ 6 * NSEC_PER_MSEC -+}; -+ -+struct cluster_ctx { -+ /* cpu-dsq map must within [lower, upper) */ -+ int upper; -+ int lower; -+ int tidx; -+}; -+ -+enum stat_items { -+ GLOBAL_STAT, -+ CPU_ALLOW_FAIL, -+ RT_CNT, -+ KEY_TASK_CNT, -+ SWITCH_IDX, -+ TIMEOUT_CNT, -+ -+ TOTAL_DSP_CNT, -+ MOVE_RQ_CNT, -+ SELECT_CPU, -+ -+ DWORD_STAT_END = SELECT_CPU, -+ -+ GDSQ_CNT, -+ ERR_IDX, -+ PCP_TIMEOUT_CNT, -+ PCP_LDSQ_CNT, -+ PCP_ENQL_CNT, -+ -+ MAX_ITEMS, -+}; -+static DEFINE_SPINLOCK(stats_lock); -+static char *stats_str[MAX_ITEMS] = { -+ "global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt", -+ "switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt", -+ "select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt", -+ "pcp_ldsq_cnt", "pcp_enql_cnt" -+}; -+ -+ -+struct stats_struct { -+ u64 global_stat[2]; -+ u64 cpu_allow_fail[2]; -+ u64 rt_cnt[2]; -+ u64 key_task_cnt[2]; -+ u64 switch_idx[2]; -+ u64 timeout_cnt[2]; -+ -+ u64 total_dsp_cnt[2]; -+ u64 move_rq_cnt[2]; -+ u64 select_cpu[2]; -+ -+ u64 gdsq_count[MAX_GLOBAL_DSQS][2]; -+ u64 err_idx[5]; -+ u64 pcp_timeout_cnt[NR_CPUS]; -+ u64 pcp_ldsq_count[NR_CPUS][2]; -+ u64 pcp_enql_cnt[NR_CPUS]; -+} stats_data; -+ -+static struct { -+ cpumask_var_t ex_free; -+ cpumask_var_t exclusive; -+ cpumask_var_t partial; -+ cpumask_var_t big; -+ cpumask_var_t little; -+} iso_masks __read_mostly; -+ -+/* -+ * Need more synchronization for these two variables? -+ * I choose not to. -+ */ -+static int l_need_rescue, b_need_rescue; -+ -+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...) \ -+{ \ -+ char buf[MAX_FATAL_INFO]; \ -+ \ -+ scnprintf(buf, MAX_FATAL_INFO, fmt, ##args); \ -+ hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf); \ -+ trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable), \ -+ READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf); \ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); \ -+} \ -+ -+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2) -+{ -+ int c1, c2; -+ struct cpumask mask = {.bits = {0}}; -+ struct cpumask tmp = {.bits = {0}}; -+ -+ if (!slim_stats) -+ return false; -+ -+ if (!rq1 || !rq2) -+ return false; -+ -+ c1 = cpu_of(rq1); -+ c2 = cpu_of(rq2); -+ cpumask_set_cpu(c1, &mask); -+ cpumask_set_cpu(c2, &mask); -+ -+ if (cpumask_and(&tmp, iso_masks.little, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.big, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.partial, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ return false; -+} -+ -+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu) -+{ -+ unsigned long flags; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ if (!slim_stats) -+ return; -+ -+ switch (item) { -+ case GLOBAL_STAT: -+ fallthrough; -+ case CPU_ALLOW_FAIL: -+ fallthrough; -+ case RT_CNT: -+ fallthrough; -+ case KEY_TASK_CNT: -+ fallthrough; -+ case SWITCH_IDX: -+ fallthrough; -+ case TIMEOUT_CNT: -+ fallthrough; -+ case TOTAL_DSP_CNT: -+ fallthrough; -+ case MOVE_RQ_CNT: -+ fallthrough; -+ case SELECT_CPU: -+ pval = pbase + item * 2 + idx; -+ break; -+ case GDSQ_CNT: -+ pval = &stats_data.gdsq_count[dsq_id][idx]; -+ break; -+ case ERR_IDX: -+ pval = &stats_data.err_idx[idx]; -+ break; -+ case PCP_TIMEOUT_CNT: -+ pval = &stats_data.pcp_timeout_cnt[cpu]; -+ break; -+ case PCP_LDSQ_CNT: -+ pval = &stats_data.pcp_ldsq_count[cpu][idx]; -+ break; -+ case PCP_ENQL_CNT: -+ pval = &stats_data.pcp_enql_cnt[cpu]; -+ break; -+ default: -+ return; -+ } -+ -+ spin_lock_irqsave(&stats_lock, flags); -+ *pval += 1; -+ spin_unlock_irqrestore(&stats_lock, flags); -+} -+ -+static inline bool handle_ret(int ret, int *idx, int len) -+{ -+ if (ret < 0 || ret >= len - *idx) -+ return true; -+ *idx += ret; -+ return false; -+} -+ -+#define PRINT_INTV (5 * HZ) -+void stats_print(char *buf, int len) -+{ -+ int idx = 0, i, j, ret; -+ int item = 0; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ for (item = 0; item < MAX_ITEMS; item++) { -+ if (item <= DWORD_STAT_END) { -+ pval = pbase + item * 2; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n", -+ stats_str[item], pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == GDSQ_CNT) { -+ for (j = 0; j < MAX_GLOBAL_DSQS; j++) { -+ pval = (u64 *)&stats_data.gdsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == ERR_IDX) { -+ pval = (u64 *)&stats_data.err_idx; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n", -+ stats_str[item], pval[0], -+ pval[1], pval[2], pval[3], pval[4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == PCP_TIMEOUT_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_timeout_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_LDSQ_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_ldsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_ENQL_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_enql_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } -+ } -+ -+ if (!md_info) { -+ buf[idx] = '\0'; -+ return; -+ } -+ -+ ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_SWITCHS; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "sw_rec[%d] = %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.sw_rec[i].switch_at, -+ md_info->kern_dump.sw_rec[i].is_success, -+ md_info->kern_dump.sw_rec[i].end_state, -+ md_info->kern_dump.sw_rec[i].switch_reason); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_EXCEP_ID; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "excep[%d] = %llu %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.excep_rec[i][0], -+ md_info->kern_dump.excep_rec[i][1], -+ md_info->kern_dump.excep_rec[i][2], -+ md_info->kern_dump.excep_rec[i][3], -+ md_info->kern_dump.excep_rec[i][4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n", -+ i, md_info->kern_dump.excep_idx[i]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ -+ buf[idx] = '\0'; -+} -+ -+enum cpu_type { -+ LITTLE, -+ BIG, -+ PARTIAL, -+ EXCLUSIVE, -+ EX_FREE, -+ INVALID -+}; -+ -+enum dsq_type { -+ GLOBAL_DSQ, -+ PCP_DSQ, -+ OTHER, -+ MAX_DSQ_TYPE, -+}; -+ -+static enum cpu_type cpu_cluster(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (get_hmbird_rq(rq)->exclusive) { -+ return EXCLUSIVE; -+ } else { -+ if (cpumask_test_cpu(cpu, iso_masks.little)) -+ return LITTLE; -+ else if (cpumask_test_cpu(cpu, iso_masks.big)) -+ return BIG; -+ else if (cpumask_test_cpu(cpu, iso_masks.partial)) -+ return PARTIAL; -+ else if (cpumask_test_cpu(cpu, iso_masks.exclusive)) -+ return EXCLUSIVE; -+ else if (cpumask_test_cpu(cpu, iso_masks.ex_free)) -+ return EX_FREE; -+ } -+ return INVALID; -+} -+ -+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq) -+{ -+ if (!dsq) -+ return OTHER; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= GDSQS_ID_BASE) && -+ ((dsq->id & 0xff) < MAX_GLOBAL_DSQS)) -+ return GLOBAL_DSQ; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) && -+ ((dsq->id & 0xff) < max_hmbird_dsq_internal_id)) -+ return PCP_DSQ; -+ -+ return OTHER; -+} -+ -+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq) -+{ -+ enum dsq_type type; -+ -+ type = get_dsq_type(dsq); -+ switch (type) { -+ case GLOBAL_DSQ: -+ case PCP_DSQ: -+ return (dsq->id & 0xff) - GDSQS_ID_BASE; -+ default: -+ return -1; -+ } -+ return -1; -+} -+ -+static void update_cpus_idle(bool set, struct cpumask *mask) -+{ -+ int cpu; -+ struct rq *rq; -+ -+ if (set) { -+ for_each_cpu(cpu, mask) { -+ rq = cpu_rq(cpu); -+ if (is_idle_task(rq->curr)) -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ } -+ } else -+ cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask); -+} -+ -+static void set_partial_status(bool enable, bool little, bool big) -+{ -+ WRITE_ONCE(partial_enable, enable); -+ WRITE_ONCE(l_need_rescue, little); -+ WRITE_ONCE(b_need_rescue, big); -+} -+ -+static bool is_little_need_rescue(void) -+{ -+ return READ_ONCE(l_need_rescue); -+} -+ -+static bool is_big_need_rescue(void) -+{ -+ return READ_ONCE(b_need_rescue); -+} -+ -+static bool is_partial_enabled(void) -+{ -+ return READ_ONCE(partial_enable); -+} -+ -+static bool is_partial_cpu(int cpu) -+{ -+ return cpumask_test_cpu(cpu, iso_masks.partial); -+} -+ -+static void set_iso_par_free(bool enable) -+{ -+ WRITE_ONCE(isolate_ctrl, enable); -+} -+ -+static bool is_iso_par_free(void) -+{ -+ return READ_ONCE(isolate_ctrl); -+} -+ -+static bool skip_update_idle(void) -+{ -+ int cpu = smp_processor_id(); -+ enum cpu_type type = cpu_cluster(cpu); -+ -+ if ((type == EXCLUSIVE && !is_iso_par_free()) || -+ /* partial enable may changed during idle, it doesn't matter. */ -+ (!is_partial_enabled() && type == PARTIAL)) -+ return true; -+ -+ return false; -+} -+ -+static void init_isolate_cpus(void) -+{ -+ WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -+ cpumask_set_cpu(0, iso_masks.big); -+ cpumask_set_cpu(1, iso_masks.big); -+ cpumask_set_cpu(2, iso_masks.big); -+ cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(4, iso_masks.big); -+ cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(6, iso_masks.partial); -+ cpumask_set_cpu(7, iso_masks.ex_free); -+} -+ -+extern spinlock_t css_set_lock; -+ -+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp) -+{ -+ int i; -+ struct cgroup *anc; -+ -+ spin_lock_irq(&css_set_lock); -+ for (i = 0; i < cgrp->level; i++) { -+ anc = cgrp->ancestors[i]; -+ if (anc->level != CREATE_DSQ_LEVEL_WITHIN) -+ continue; -+ cgroup_get(anc); -+ spin_unlock_irq(&css_set_lock); -+ return anc; -+ } -+ spin_unlock_irq(&css_set_lock); -+ hmbird_err(NO_CGROUP_L1, ":error cgroup = %s\n", cgrp->kn->name); -+ return NULL; -+} -+ -+#define PCP_IDX_BIT (1 << 31) -+ -+static bool is_pcp_rt(struct task_struct *p) -+{ -+ return rt_prio(p->prio) && (p->nr_cpus_allowed == 1); -+} -+ -+static bool is_pcp_idx(int idx) -+{ -+ return idx >= MAX_GLOBAL_DSQS; -+} -+ -+static bool is_critical_system_task(struct task_struct *p) -+{ -+ int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK; -+ -+ return (p->prio < (MAX_RT_PRIO >> 1) && -+ (sp_dl < SCHED_PROP_DEADLINE_LEVEL3)); -+} -+ -+#define ISOLATE_TASK_TOP_BIT (1 << 17) -+#define PIPELINE_TASK_TOP_BIT (1 << 9) -+static bool is_pipeline_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT); -+} -+ -+static bool is_isolate_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT); -+} -+ -+static bool is_critical_app_task_without_isolate(struct task_struct *p) -+{ -+ return task_is_top_task(p) && !is_isolate_task(p); -+} -+ -+static int find_idx_from_task(struct task_struct *p) -+{ -+ int idx, cpu; -+ int sp_dl; -+ struct task_group *tg = p->sched_task_group; -+ -+ if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) { -+ cpu = cpumask_any(p->cpus_ptr); -+ idx = cpu + MAX_GLOBAL_DSQS; -+ return idx; -+ } -+ -+ if (is_critical_system_task(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL0; -+ goto done; -+ } -+ -+ if (is_critical_app_task_without_isolate(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL1; -+ goto done; -+ } -+ -+ if (p->pid == scx_systemui_pid) { -+ idx = SCHED_PROP_DEADLINE_LEVEL4; -+ goto done; -+ } -+ -+ sp_dl = hmbird_get_dsq_id(p); -+ if (sp_dl) { -+ idx = sp_dl; -+ goto done; -+ } -+ -+ if (rt_prio(p->prio)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL3; -+ goto done; -+ } -+ -+ if (tg && tg->css.cgroup && tg->css.cgroup->kn) { -+ if (likely(tg->css.cgroup->kn->id >= 0 && -+ tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS)) -+ idx = cgroup_ids_table[tg->css.cgroup->kn->id]; -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ -+done: -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) { -+ hmbird_err(DSQ_ID_ERR, " : idx error, idx = %d-----\n", idx); -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } -+ return idx; -+} -+ -+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p) -+{ -+ int idx; -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq; -+ -+ if (!p) -+ return NULL; -+ -+ idx = find_idx_from_task(p); -+ if (is_pcp_idx(idx)) { -+ idx -= MAX_GLOBAL_DSQS; -+ dsq = &per_cpu(pcp_ldsq, idx); -+ get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq); -+ slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx); -+ } else { -+ dsq = &gdsqs[idx]; -+ get_hmbird_ts(p)->gdsq_idx = idx; -+ slim_stats_record(GDSQ_CNT, 0, idx, 0); -+ } -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ dsq->last_consume_at = jiffies; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ return dsq; -+} -+ -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq); -+ -+static void set_partial_rescue(bool p_state, bool l_over, bool b_over) -+{ -+ set_partial_status(p_state, l_over, b_over); -+ update_cpus_idle(p_state, iso_masks.partial); -+ hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled()); -+ hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue()); -+ hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue()); -+} -+ -+static void free_isocpu(bool enable) -+{ -+ set_iso_par_free(enable); -+ update_cpus_idle(enable, iso_masks.ex_free); -+ hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free()); -+} -+ -+inline u64 get_hmbird_cpu_util(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed) -+ return 0; -+ u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed); -+ u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size); -+ -+ do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT); -+ -+ return prev_runnable_sum_fixed; -+} -+ -+static inline unsigned int get_scaling_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->max; -+} -+ -+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static u64 get_cpus_max_util(struct cpumask *mask) -+{ -+ int cpu; -+ u64 max = 0; -+ u64 util, ratio; -+ unsigned long effective_cap = 0; -+ -+ for_each_cpu(cpu, mask) { -+ if (slim_walt_ctrl) -+ slim_get_cpu_util(cpu, &util); -+ else -+ util = get_hmbird_cpu_util(cpu); -+ -+ /* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/ -+ if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu))) -+ effective_cap = arch_scale_cpu_capacity(cpu); -+ else -+ effective_cap = arch_scale_cpu_capacity(cpu) * -+ get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu); -+ -+ ratio = util * 100 / effective_cap; -+ hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util); -+ hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n", -+ cpu, (u64)arch_scale_cpu_capacity(cpu)); -+ hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n", -+ cpu, (u64)effective_cap); -+ -+ if (ratio > 100) -+ ratio = 100; -+ -+ if (ratio > max) -+ max = ratio; -+ } -+ return max; -+} -+ -+static bool cluster_need_rescue(struct cpumask *mask, int hres) -+{ -+ return get_cpus_max_util(mask) > hres; -+} -+ -+void partial_dynamic_ctrl(void) -+{ -+ u64 lmax = 0, bmax = 0; -+ bool l_over, l_under, b_over, b_under; -+ static bool last_l_over, last_b_over; -+ static unsigned long last_check; -+ -+ /* Check partial every jiffies. need lock? */ -+ if (time_before_eq(jiffies, READ_ONCE(last_check))) -+ return; -+ -+ WRITE_ONCE(last_check, jiffies); -+ -+ lmax = get_cpus_max_util(iso_masks.little); -+ l_over = lmax > parctrl_high_ratio_l; -+ l_under = lmax < parctrl_low_ratio_l; -+ -+ bmax = get_cpus_max_util(iso_masks.big); -+ b_over = bmax > parctrl_high_ratio; -+ b_under = bmax < parctrl_low_ratio; -+ -+ if (is_partial_enabled() && (l_over || b_over)) { -+ if (last_l_over != l_over || last_b_over != b_over) -+ set_partial_rescue(true, l_over, b_over); -+ } else if (!is_partial_enabled() && (l_over || b_over)) { -+ set_partial_rescue(true, l_over, b_over); -+ } else if (is_partial_enabled() && l_under && b_under) { -+ set_partial_rescue(false, false, false); -+ } -+ last_l_over = l_over; -+ last_b_over = b_over; -+ -+ if (is_partial_enabled()) { -+ if (cpumask_empty(iso_masks.partial)) { -+ bmax = bmax > lmax ? bmax : lmax; -+ } else { -+ bmax = get_cpus_max_util(iso_masks.partial); -+ } -+ if (!is_iso_par_free() && bmax > isoctrl_high_ratio) { -+ hmbird_info_trace("partial max = %llu\n", bmax); -+ free_isocpu(true); -+ } else if (is_iso_par_free() && bmax < isoctrl_low_ratio) -+ free_isocpu(false); -+ } else if (is_iso_par_free()) { -+ free_isocpu(false); -+ } -+} -+ -+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx) -+{ -+ hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx); -+} -+ -+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx) -+{ -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[idx])) { -+ slim_stats_record(GDSQ_CNT, 1, idx, 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ int i; -+ -+ for (i = 0; i < UX_COMPATIBLE_IDX; i++) { -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ slim_stats_record(GDSQ_CNT, 1, i, 0); -+ return 1; -+ } -+ } -+ return 0; -+} -+ -+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) { -+ slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ goto clear_timeout; -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) -+ goto clear_timeout; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ hmbird_info_trace( -+ "dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n", -+ dsq_id_to_internal(dsq), entity->task->comm, -+ jiffies, msecs_to_jiffies(deadline), entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1); -+ return; -+ -+clear_timeout: -+ hmbird_info_trace("dsq[%d] clear timeout\n", -+ dsq_id_to_internal(dsq)); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0); -+ dsq->is_timeout = false; -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq)); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime) -+{ -+ hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime); -+} -+ -+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any) -+{ -+ bool is_timeout; -+ int cpu = cpu_of(rq); -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu); -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ is_timeout = dsq->is_timeout; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ /* -+ * dsq->is_timeout may change here, let it be. -+ * it won't cause serious problems. -+ * the same for consume_dispatch_q later. -+ */ -+ if (!is_timeout && !any) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, dsq)) { -+ if (is_timeout) { -+ hmbird_info_trace("dsq[%d] consume a pcp timeout task\n", -+ dsq_id_to_internal(dsq)); -+ update_timeout_stats(rq, dsq, pcp_dsq_deadline); -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu); -+ } -+ slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu); -+ return 1; -+ } -+ /* -+ * No pcp task, clear quota. -+ */ -+ if (any) { -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+ } -+ return 0; -+} -+ -+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf) -+{ -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ } -+ return 0; -+} -+ -+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf, -+ int tmp, int cidx, int tidx) -+{ -+ unsigned long flags; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) { -+ if (tmp != cidx) { -+ spin_lock(&sinfo.lock); -+ sinfo.curr_idx[tidx] = tmp; -+ spin_unlock(&sinfo.lock); -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]); -+ slim_stats_record(SWITCH_IDX, 0, 0, 0); -+ } -+ slim_stats_record(GDSQ_CNT, 1, tmp, 0); -+ -+ raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags); -+ gdsqs[tmp].last_consume_at = jiffies; -+ raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags); -+ return 1; -+ } -+ return 0; -+ -+} -+ -+static int get_cidx(struct cluster_ctx *ctx) -+{ -+ int cidx; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ if (cidx < ctx->lower || cidx >= ctx->upper) { -+ sinfo.curr_idx[ctx->tidx] = ctx->lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]); -+ slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ } -+ spin_unlock(&sinfo.lock); -+ -+ return cidx; -+} -+ -+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = CLUSTER_SEPARATE_IDX; -+ if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) { -+ pr_debug(" %s iso_masks.little first is %d\n", -+ __func__, cpumask_first(iso_masks.little)); -+ ctx->upper = NON_PERIOD_END; -+ } -+ ctx->tidx = 0; -+ break; -+ case LITTLE: -+ ctx->lower = CLUSTER_SEPARATE_IDX; -+ ctx->upper = NON_PERIOD_END; -+ if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) { -+ pr_debug(" %s iso_masks.big first is %d", -+ __func__, cpumask_first(iso_masks.big)); -+ ctx->lower = NON_PERIOD_START; -+ } -+ ctx->tidx = 1; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ case LITTLE: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = NON_PERIOD_END; -+ ctx->tidx = 0; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ if (cluster_separate) { -+ return gen_cluster_ctx_separate(ctx, type); -+ } else { -+ return gen_cluster_ctx_common(ctx, type); -+ } -+} -+ -+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ int i; -+ bool is_timeout; -+ unsigned long flags; -+ struct cluster_ctx ctx; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ /* Third param means only consume timeout pcp dsq. */ -+ if (consume_pcp_dsq(rq, rf, false)) -+ return 1; -+ -+ for (i = ctx.lower; i < ctx.upper; i++) { -+ raw_spin_lock_irqsave(&gdsqs[i].lock, flags); -+ is_timeout = gdsqs[i].is_timeout; -+ raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags); -+ /* gdsqs[i].is_timeout may change here, let it be... */ -+ if (is_timeout) { -+ /* -+ * consume_dispatch_q will acquire dsq-lock, -+ * So cannot keep lock here, annoy enough. -+ * may rewrite a consume_dispatch_q_locked, TODO. -+ */ -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ hmbird_info_trace("dsq[%d] consume a timeout task\n", i); -+ slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0); -+ update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]); -+ return 1; -+ } -+ } -+ } -+ return 0; -+} -+ -+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ struct cluster_ctx ctx; -+ int cidx; -+ int tmp; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ cidx = get_cidx(&ctx); -+ tmp = cidx; -+ do { -+ if (check_pcp_dsq_round(rq, rf)) -+ return 1; -+ if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx)) -+ return 1; -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[tmp] = 0; -+ systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]); -+ tmp++; -+ if (tmp >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ tmp = ctx.lower; -+ } -+ spin_unlock(&sinfo.lock); -+ } while (tmp != cidx); -+ -+ return consume_pcp_dsq(rq, rf, true); -+} -+ -+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ enum cpu_type type = cpu_cluster(cpu_of(rq)); -+ bool period_allowed = !get_hmbird_rq(rq)->period_disallow; -+ bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow; -+ -+ switch (type) { -+ case EX_FREE: -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ break; -+ case EXCLUSIVE: -+ if (!is_iso_par_free()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ } -+ /* Only non-period task can run on isolate cpus */ -+ if (!READ_ONCE(iso_free_rescue)) { -+ if (is_big_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ } else { -+ /* Free run, can run on any task. */ -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ case PARTIAL: -+ if (!is_partial_enabled()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ return 0; -+ } -+ if (is_big_need_rescue()) { -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ -+ case BIG: -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ if (is_iso_par_free() || (is_little_need_rescue() && -+ !cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) { -+ hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++); -+ return 1; -+ } -+ } -+ break; -+ -+ case LITTLE: -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ -+ if (is_iso_par_free() || (is_big_need_rescue() && -+ !cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) { -+ hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++); -+ return 1; -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return 0; -+} -+ -+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf) -+{ -+ return consume_hmbird_global_dsq(rq, rf); -+} -+ -+ -+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time) -+{ -+ int idx; -+ -+ /* which dsq belongs to while task enqueue, task will consume its running time. */ -+ idx = get_hmbird_ts(p)->gdsq_idx; -+ /* Only non-period dsq share running time between each other. */ -+ if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id) -+ return; -+ -+ if (idx >= MAX_GLOBAL_DSQS) { -+ per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } else { -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[idx] += exec_time; -+ spin_unlock(&sinfo.lock); -+ systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]); -+ } -+} -+ -+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type) -+{ -+ int cidx; -+ struct cluster_ctx ctx; -+ int cpu = cpu_of(rq); -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ if (cidx < ctx.lower || cidx >= ctx.upper) { -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ slim_stats_record(ERR_IDX, ctx.tidx, 0, 0); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ } -+ -+ while (1) { -+ if (per_cpu(pcp_info, cpu).pcp_round) { -+ if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) { -+ hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n", -+ cpu, per_cpu(pcp_info, cpu).rtime); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } -+ } -+ if (sinfo.rtime[cidx] < dsq_quota[cidx]) -+ break; -+ -+ /* clear current dsq rtime */ -+ sinfo.rtime[cidx] = 0; -+ systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]); -+ -+ sinfo.curr_idx[ctx.tidx]++; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", -+ ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ } -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ slim_stats_record(SWITCH_IDX, 1, 0, 0); -+ } -+ spin_unlock(&sinfo.lock); -+} -+ -+ -+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p) -+{ -+ enum cpu_type type; -+ -+ if (!rq || !p) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ switch (type) { -+ case PARTIAL: -+ return; -+ case EXCLUSIVE: -+ return; -+ case EX_FREE: -+ return; -+ default: -+ break; -+ } -+ update_dsq_idx(rq, p, type); -+} -+ -+ -+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ int dsq_id; -+ -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo) || dsq->is_timeout) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ hmbird_deferred_err(SCAN_ENTITY_NULL, -+ " : entity is NULL, dsq->id = %llu\n", dsq->id); -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ dsq->is_timeout = true; -+ dsq_id = dsq_id_to_internal(dsq); -+ hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n", -+ dsq_id, entity->task->comm, jiffies, entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1); -+ raw_spin_unlock(&dsq->lock); -+ -+ return true; -+} -+ -+void scan_timeout(struct rq *rq) -+{ -+ int i; -+ int cpu = cpu_of(rq); -+ struct hmbird_dispatch_q *dsq; -+ static u64 last_scan_at; -+ static DEFINE_PER_CPU(u64, pcp_last_scan_at); -+ -+ if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu))) -+ return; -+ per_cpu(pcp_last_scan_at, cpu) = jiffies; -+ -+ dsq = &per_cpu(pcp_ldsq, cpu); -+ scan_dsq_timeout(rq, dsq, pcp_dsq_deadline); -+ -+ if (time_before_eq(jiffies, (unsigned long)last_scan_at)) -+ return; -+ last_scan_at = jiffies; -+ -+ for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) { -+ dsq = &gdsqs[i]; -+ scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]); -+ } -+} -+ -+/*******************************Initialize***********************************/ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id); -+static void init_dsq_at_boot(void) -+{ -+ int i, cpu; -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i)); -+ } -+ for_each_possible_cpu(cpu) -+ init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i + cpu)); -+ -+ max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu; -+ spin_lock_init(&sinfo.lock); -+} -+ -+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgrp_name_to_idx(struct cgroup *cgrp) -+{ -+ int idx; -+ -+ if (!cgrp) -+ return -1; -+ -+ if (!strcmp(cgrp->kn->name, "display") -+ || !strcmp(cgrp->kn->name, "multimedia")) -+ idx = 5; /* 8ms */ -+ else if (!strcmp(cgrp->kn->name, "top-app") -+ || !strcmp(cgrp->kn->name, "ss-top")) -+ idx = 6; /* 16ms */ -+ else if (!strcmp(cgrp->kn->name, "ssfg") -+ || !strcmp(cgrp->kn->name, "foreground")) -+ idx = 7; /* 32ms */ -+ else if (!strcmp(cgrp->kn->name, "bg") -+ || !strcmp(cgrp->kn->name, "log") -+ || !strcmp(cgrp->kn->name, "dex2oat") -+ || !strcmp(cgrp->kn->name, "background")) -+ idx = 9; /* 128ms */ -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */ -+ -+ return idx; -+} -+ -+static void init_root_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX); -+} -+ -+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) { -+ pr_err("%s idx err!\n", __func__); -+ return; -+ } -+ -+ if (cgroup_ids_table[cgrp->kn->id] == -1) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp)); -+} -+ -+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ struct cgroup *l1cgrp; -+ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ l1cgrp = cgroup_ancestor_l1(cgrp); -+ if (l1cgrp) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp)); -+ cgroup_put(l1cgrp); -+} -+ -+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg) -+{ -+ switch (cgrp->level) { -+ case 0: -+ init_root_tg(cgrp, tg); -+ break; -+ case 1: -+ init_level1_tg(cgrp, tg); -+ break; -+ default: -+ init_child_tg(cgrp, tg); -+ break; -+ } -+} -+ -+/**************************************************************************/ -+ -+struct hmbird_task_iter { -+ struct hmbird_entity cursor; -+ struct task_struct *locked; -+ struct rq *rq; -+ struct rq_flags rf; -+}; -+ -+/** -+ * hmbird_task_iter_init - Initialize a task iterator -+ * @iter: iterator to init -+ * -+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized, -+ * @iter must eventually be exited with hmbird_task_iter_exit(). -+ * -+ * hmbird_tasks_lock may be released between this and the first next() call or -+ * between any two next() calls. If hmbird_tasks_lock is released between two -+ * next() calls, the caller is responsible for ensuring that the task being -+ * iterated remains accessible either through RCU read lock or obtaining a -+ * reference count. -+ * -+ * All tasks which existed when the iteration started are guaranteed to be -+ * visited as long as they still exist. -+ */ -+static void hmbird_task_iter_init(struct hmbird_task_iter *iter) -+{ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR }; -+ list_add(&iter->cursor.tasks_node, &hmbird_tasks); -+ iter->locked = NULL; -+} -+ -+/** -+ * hmbird_task_iter_exit - Exit a task iterator -+ * @iter: iterator to exit -+ * -+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held. -+ * If the iterator holds a task's rq lock, that rq lock is released. See -+ * hmbird_task_iter_init() for details. -+ */ -+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ if (list_empty(cursor)) -+ return; -+ -+ list_del_init(cursor); -+} -+ -+/** -+ * hmbird_task_iter_next - Next task -+ * @iter: iterator to walk -+ * -+ * Visit the next task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ struct hmbird_entity *pos; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ list_for_each_entry(pos, cursor, tasks_node) { -+ if (&pos->tasks_node == &hmbird_tasks) -+ return NULL; -+ if (!(pos->flags & HMBIRD_TASK_CURSOR)) { -+ list_move(cursor, &pos->tasks_node); -+ return pos->task; -+ } -+ } -+ -+ /* can't happen, should always terminate at hmbird_tasks above */ -+ hmbird_deferred_err(ITER_RET_NULL, " : unreachable path in scx_task_iter_next\n"); -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered - Next non-idle task -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ while ((p = hmbird_task_iter_next(iter))) { -+ if (!is_idle_task(p)) -+ return p; -+ } -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init() -+ * for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ p = hmbird_task_iter_next_filtered(iter); -+ if (!p) -+ return NULL; -+ -+ iter->rq = task_rq_lock(p, &iter->rf); -+ iter->locked = p; -+ return p; -+} -+ -+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void) -+{ -+ return atomic_read(&hmbird_ops_enable_state_var); -+} -+ -+static enum hmbird_ops_enable_state -+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to) -+{ -+ return atomic_xchg(&hmbird_ops_enable_state_var, to); -+} -+ -+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to, -+ enum hmbird_ops_enable_state from) -+{ -+ int from_v = from; -+ -+ return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to); -+} -+ -+static bool hmbird_ops_disabling(void) -+{ -+ return false; -+} -+ -+/** -+ * wait_ops_state - Busy-wait the specified ops state to end -+ * @p: target task -+ * @opss: state to wait the end of -+ * -+ * Busy-wait for @p to transition out of @opss. This can only be used when the -+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also -+ * has load_acquire semantics to ensure that the caller can see the updates made -+ * in the enqueueing and dispatching paths. -+ */ -+static void wait_ops_state(struct task_struct *p, u64 opss) -+{ -+ do { -+ cpu_relax(); -+ } while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss); -+} -+ -+ -+static void update_curr_hmbird(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ u64 now = rq_clock_task(rq); -+ u64 delta_exec; -+ -+ if (time_before_eq64(now, curr->se.exec_start)) -+ return; -+ -+ delta_exec = now - curr->se.exec_start; -+ curr->se.exec_start = now; -+ update_runningtime(rq, curr, delta_exec); -+ curr->se.sum_exec_runtime += delta_exec; -+ account_group_exec_runtime(curr, delta_exec); -+ cgroup_account_cputime(curr, delta_exec); -+ -+ if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF) -+ get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec); -+ -+ trace_sched_stat_runtime(curr, delta_exec, 0); -+} -+ -+static bool hmbird_dsq_priq_less(struct rb_node *node_a, -+ const struct rb_node *node_b) -+{ -+ const struct hmbird_entity *a = -+ container_of(node_a, struct hmbird_entity, dsq_node.priq); -+ const struct hmbird_entity *b = -+ container_of(node_b, struct hmbird_entity, dsq_node.priq); -+ -+ return time_before64(a->dsq_vtime, b->dsq_vtime); -+} -+ -+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ bool is_local = dsq->id == HMBIRD_DSQ_LOCAL; -+ unsigned long flags; -+ -+ hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq || -+ !list_empty(&get_hmbird_ts(p)->dsq_node.fifo), -+ "task = %s, dsq->id = %llu\n", p->comm, dsq->id); -+ hmbird_cond_deferred_err(ENQ_EXIST2, -+ (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq), -+ "task = %s\n", p->comm); -+ -+ if (!is_local) { -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ); -+ hmbird_ops_error(": %s\n", -+ "attempting to dispatch to a destroyed dsq"); -+ /* fall back to the global dsq */ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ dsq = &hmbird_dsq_global; -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ } -+ } -+ -+ if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) { -+ get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ; -+ rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq, -+ hmbird_dsq_priq_less); -+ } else { -+ if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT)) -+ list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ else -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ } -+ dsq->nr++; -+ get_hmbird_ts(p)->dsq = dsq; -+ -+ /* -+ * We're transitioning out of QUEUEING or DISPATCHING. store_release to -+ * match waiters' load_acquire. -+ */ -+ if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS) -+ atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE); -+ -+ if (is_local) { -+ struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq); -+ struct rq *rq = hmbird->rq; -+ bool preempt = false; -+ -+ if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr && -+ rq->curr->sched_class == &hmbird_sched_class) { -+ get_hmbird_ts(rq->curr)->slice = 0; -+ preempt = true; -+ } -+ -+ if (preempt || sched_class_above(&hmbird_sched_class, -+ rq->curr->sched_class)) -+ resched_curr(rq); -+ } else { -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ } -+} -+ -+static void task_unlink_from_dsq(struct task_struct *p, -+ struct hmbird_dispatch_q *dsq) -+{ -+ if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) { -+ rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ; -+ } else { -+ list_del_init(&get_hmbird_ts(p)->dsq_node.fifo); -+ } -+} -+ -+static bool task_linked_on_dsq(struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+} -+ -+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq; -+ bool is_local = dsq == &hmbird_rq->local_dsq; -+ -+ if (!dsq) { -+ hmbird_cond_deferred_err(TASK_LINKED1, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ /* -+ * When dispatching directly from the BPF scheduler to a local -+ * DSQ, the task isn't associated with any DSQ but -+ * @get_hmbird_ts(p)->holding_cpu may be set under the protection of -+ * %HMBIRD_OPSS_DISPATCHING. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu >= 0) -+ get_hmbird_ts(p)->holding_cpu = -1; -+ return; -+ } -+ -+ if (!is_local) -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ /* -+ * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node -+ * can't change underneath us. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu < 0) { -+ /* @p must still be on @dsq, dequeue */ -+ hmbird_cond_deferred_err(TASK_UNLINKED, -+ !task_linked_on_dsq(p), "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ } else { -+ /* -+ * We're racing against dispatch_to_local_dsq() which already -+ * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the -+ * holding_cpu which tells dispatch_to_local_dsq() that it lost -+ * the race. -+ */ -+ hmbird_cond_deferred_err(TASK_LINKED2, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ get_hmbird_ts(p)->holding_cpu = -1; -+ } -+ get_hmbird_ts(p)->dsq = NULL; -+ -+ if (!is_local) -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+ -+static bool test_rq_online(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ return rq->online; -+#else -+ return true; -+#endif -+} -+ -+static void refill_task_slice(struct task_struct *p) -+{ -+ if (is_isolate_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO; -+ else if (is_pipeline_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2; -+ else -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+} -+ -+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -+ int sticky_cpu) -+{ -+ struct hmbird_dispatch_q *d; -+ s32 cpu = -1; -+ -+ hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), -+ "task = %s\n", p->comm); -+ -+ if (is_pcp_rt(p)) { -+ /* Enqueue percpu rt task to local directly. */ -+ /* Or cause a bug when disable dispatch. */ -+ if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ } -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (cpu >= 0) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ -+ if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ /* rq migration */ -+ if (sticky_cpu == cpu_of(rq)) -+ goto local_norefill; -+ /* -+ * If !rq->online, we already told the BPF scheduler that the CPU is -+ * offline. We're just trying to on/offline the CPU. Don't bother the -+ * BPF scheduler. -+ */ -+ if (unlikely(!test_rq_online(rq))) -+ goto local; -+ -+ /* see %HMBIRD_OPS_ENQ_LAST */ -+ if (enq_flags & HMBIRD_ENQ_LAST) -+ goto local; -+ -+ if (enq_flags & HMBIRD_ENQ_LOCAL) -+ goto local; -+ else -+ goto global; -+local: -+ /* -+ * For task-ordering, slice refill must be treated as implying the end -+ * of the current slice. Otherwise, the longer @p stays on the CPU, the -+ * higher priority it becomes from hmbird_prio_less()'s POV. -+ */ -+ refill_task_slice(p); -+local_norefill: -+ dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags); -+ slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq)); -+ return; -+ -+global: -+ d = find_dsq_from_task(p); -+ if (d) { -+ refill_task_slice(p); -+ dispatch_enqueue(d, p, enq_flags); -+ return; -+ } -+ slim_stats_record(GLOBAL_STAT, 0, 0, 0); -+ refill_task_slice(p); -+ dispatch_enqueue(&hmbird_dsq_global, p, enq_flags); -+} -+ -+static bool watchdog_task_watched(const struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->watchdog_node); -+} -+ -+static void watchdog_watch_task(struct rq *rq, struct task_struct *p) -+{ -+ lockdep_assert_rq_held(rq); -+ if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ get_hmbird_ts(p)->runnable_at = jiffies; -+ clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list); -+} -+ -+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) -+{ -+ list_del_init(&get_hmbird_ts(p)->watchdog_node); -+ if (reset_timeout) -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags) -+{ -+ int sticky_cpu = get_hmbird_ts(p)->sticky_cpu; -+ -+ enq_flags |= get_hmbird_rq(rq)->extra_enq_flags; -+ -+ if (sticky_cpu >= 0) -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ -+ /* -+ * Restoring a running task will be immediately followed by -+ * set_next_task_hmbird() which expects the task to not be on the BPF -+ * scheduler as tasks can only start running through local DSQs. Force -+ * direct-dispatch into the local DSQ by setting the sticky_cpu. -+ */ -+ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -+ sticky_cpu = cpu_of(rq); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_UNWATCHED, -+ !watchdog_task_watched(p), "task = %s\n", p->comm); -+ return; -+ } -+ -+ watchdog_watch_task(rq, p); -+ set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ get_hmbird_rq(rq)->nr_running++; -+ add_nr_running(rq, 1); -+ -+ do_enqueue_task(rq, p, enq_flags, sticky_cpu); -+} -+ -+static void ops_dequeue(struct task_struct *p, u64 deq_flags) -+{ -+ u64 opss; -+ -+ watchdog_unwatch_task(p, false); -+ -+ /* acquire ensures that we see the preceding updates on QUEUED */ -+ opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state); -+ -+ switch (opss & HMBIRD_OPSS_STATE_MASK) { -+ case HMBIRD_OPSS_NONE: -+ break; -+ case HMBIRD_OPSS_QUEUEING: -+ /* -+ * QUEUEING is started and finished while holding @p's rq lock. -+ * As we're holding the rq lock now, we shouldn't see QUEUEING. -+ */ -+ hmbird_deferred_err(DEQ_DEQING, " : unreachable path in %s\n", __func__); -+ break; -+ case HMBIRD_OPSS_QUEUED: -+ if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss, -+ HMBIRD_OPSS_NONE)) -+ break; -+ fallthrough; -+ case HMBIRD_OPSS_DISPATCHING: -+ /* -+ * If @p is being dispatched from the BPF scheduler to a DSQ, -+ * wait for the transfer to complete so that @p doesn't get -+ * added to its DSQ after dequeueing is complete. -+ * -+ * As we're waiting on DISPATCHING with the rq locked, the -+ * dispatching side shouldn't try to lock the rq while -+ * DISPATCHING is set. See dispatch_to_local_dsq(). -+ * -+ * DISPATCHING shouldn't have qseq set and control can reach -+ * here with NONE @opss from the above QUEUED case block. -+ * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss. -+ */ -+ wait_ops_state(p, HMBIRD_OPSS_DISPATCHING); -+ hmbird_cond_deferred_err(HMBIRD_OPN, -+ atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE, -+ "task = %s\n", p->comm); -+ break; -+ } -+} -+ -+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ -+ if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p), -+ "task = %s\n", p->comm); -+ return; -+ } -+ -+ ops_dequeue(p, deq_flags); -+ -+ if (slim_walt_ctrl) { -+ if (task_current(rq, p)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ if (deq_flags & HMBIRD_DEQ_SLEEP) -+ set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else -+ clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), -+ (unsigned long *)&get_hmbird_ts(p)->flags); -+ -+ clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm); -+ hmbird_rq->nr_running--; -+ sub_nr_running(rq, 1); -+ dispatch_dequeue(hmbird_rq, p); -+} -+ -+static void yield_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ get_hmbird_ts(p)->slice = 0; -+} -+ -+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to) -+{ -+ return false; -+} -+ -+#ifdef CONFIG_SMP -+/** -+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -+ * @rq: rq to move the task into, currently locked -+ * @p: task to move -+ * @enq_flags: %HMBIRD_ENQ_* -+ * -+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller -+ * must: -+ * -+ * 1. Start with exclusive access to @p either through its DSQ lock or -+ * %HMBIRD_OPSS_DISPATCHING flag. -+ * -+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id(). -+ * -+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't -+ * deadlock with dequeue. -+ * -+ * 4. Lock @rq and the task_rq from #3. -+ * -+ * 5. Call this function. -+ * -+ * Returns %true if @p was successfully moved. %false after racing dequeue and -+ * losing. -+ */ -+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ struct rq *task_rq; -+ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * If dequeue got to @p while we were trying to lock both rq's, it'd -+ * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have -+ * updated it to different values afterwards, as this operation can't be -+ * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become -+ * raw_smp_processor_id() again before we're done. Thus, we can tell -+ * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is -+ * still raw_smp_processor_id(). -+ * -+ * See dispatch_dequeue() for the counterpart. -+ */ -+ if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id())) -+ return false; -+ -+ /* @p->rq couldn't have changed if we're still the holding cpu */ -+ task_rq = task_rq(p); -+ lockdep_assert_rq_held(task_rq); -+ deactivate_task(task_rq, p, 0); -+ set_task_cpu(p, cpu_of(rq)); -+ get_hmbird_ts(p)->sticky_cpu = cpu_of(rq); -+ -+ /* -+ * We want to pass hmbird-specific enq_flags but activate_task() will -+ * truncate the upper 32 bit. As we own @rq, we can pass them through -+ * @get_hmbird_rq(rq)->extra_enq_flags instead. -+ */ -+ hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags, -+ "task = %s\n", p->comm); -+ get_hmbird_rq(rq)->extra_enq_flags = enq_flags; -+ activate_task(rq, p, 0); -+ get_hmbird_rq(rq)->extra_enq_flags = 0; -+ -+ return true; -+} -+ -+#endif /* CONFIG_SMP */ -+ -+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu) -+{ -+ int fitable = 1; -+ -+ return fitable; -+} -+ -+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq, -+ struct hmbird_dispatch_q *dsq) -+{ -+ bool dsq_misfit; -+ int cpu = cpu_of(rq); -+ u64 task_util = 0; -+ struct cluster_ctx ctx; -+ int dsq_int = dsq_id_to_internal(dsq); -+ -+ if (!cpumask_test_cpu(cpu, iso_masks.little)) -+ return false; -+ if (p->pid == scx_systemui_pid) -+ return true; -+ -+ gen_cluster_ctx(&ctx, BIG); -+ dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 && -+ dsq_int <= SCHED_PROP_DEADLINE_LEVEL4); -+#ifdef CLUSTER_SEPARATE -+ /* In rescue mode, little will consume big cluster's dsq.*/ -+ dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper); -+#endif -+ if (!dsq_misfit) -+ return false; -+ -+ if (p) { -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &task_util); -+ else -+ task_util = get_hmbird_ts(p)->demand_scaled; -+ } -+ -+ if (task_util <= misfit_ds) -+ return false; -+ -+ hmbird_info_trace(":task %s can't run on cpu%d, util = %llu\n", -+ p->comm, cpu, task_util); -+ return true; -+} -+ -+static int check_misfit_task_on_fake_big(struct task_struct *p, struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (likely(p->pid != scx_systemui_pid)) -+ return false; -+ -+ if (topology_cluster_id(num_possible_cpus() - 1) > 1 && -+ arch_scale_cpu_capacity(cpu) == arch_scale_cpu_capacity(0) && -+ (parctrl_high_ratio <= 0 || parctrl_high_ratio_l <= 0)) -+ return true; -+ -+ return false; -+} -+ -+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq) -+{ -+ if (!cpumask_test_cpu(cpu_of(rq), task_cpu_possible_mask(p))) -+ return false; -+ -+ if (!task_fits_cpu_hmbird(p, cpu_of(rq))) -+ return false; -+ -+ if (check_misfit_task_on_little(p, rq, dsq)) -+ return false; -+ -+ if (check_misfit_task_on_fake_big(p, rq)) -+ return false; -+ -+ return likely(test_rq_online(rq)); -+} -+ -+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return; -+ -+ if (add) -+ skipn[idx]++; -+ else -+ skipn[idx] = 0; -+} -+ -+static bool skip_too_much(struct hmbird_dispatch_q *dsq) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return false; -+ -+ if (skip_num[idx] > 3) { -+ skip_num[idx] = 0; -+ return true; -+ } -+ -+ return false; -+} -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rb_node *rb_node; -+ struct rq *task_rq; -+ unsigned long flags; -+ bool moved = false; -+ struct task_struct *may_fit = NULL; -+ int skip = 0; -+ -+retry: -+ if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -+ return false; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) { -+ set_skip_num(dsq, skip_num, (bool)may_fit); -+ goto this_rq; -+ } -+ if (skip_too_much(dsq)) -+ goto remote_rq; -+ if (!may_fit) -+ may_fit = p; -+ if (++skip <= 3) -+ continue; -+ /* -+ * If the recent 3 tasks not fit, use the first one. -+ * and clear the skip, because the first one is consumed. -+ */ -+ set_skip_num(dsq, skip_num, false); -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ /* No more task, use the first may fit task.*/ -+ if (may_fit) { -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ -+ for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) -+ goto this_rq; -+ goto remote_rq; -+ } -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ return false; -+ -+this_rq: -+ /* @dsq is locked and @p is on this rq */ -+ hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo); -+ dsq->nr--; -+ hmbird_rq->local_dsq.nr++; -+ get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ -+remote_rq: -+#ifdef CONFIG_SMP -+ if (cpu_same_cluster_stat(p, rq, task_rq)) -+ slim_stats_record(MOVE_RQ_CNT, 0, 0, 0); -+ else -+ slim_stats_record(MOVE_RQ_CNT, 1, 0, 0); -+ /* -+ * @dsq is locked and @p is on a remote rq. @p is currently protected by -+ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -+ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -+ * rq lock or fail, do a little dancing from our side. See -+ * move_task_to_local_dsq(). -+ */ -+ hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id(); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ rq_unpin_lock(rq, rf); -+ double_lock_balance(rq, task_rq); -+ rq_repin_lock(rq, rf); -+ -+ moved = move_task_to_local_dsq(rq, p, 0); -+ -+ double_unlock_balance(rq, task_rq); -+#endif /* CONFIG_SMP */ -+ if (likely(moved)) { -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ } -+ may_fit = NULL; -+ goto retry; -+} -+ -+ -+static int balance_one(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf, bool local) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class; -+ -+ if (!hmbird_rq) -+ return 1; -+ -+ lockdep_assert_rq_held(rq); -+ -+ if (static_branch_unlikely(&hmbird_ops_cpu_preempt) && -+ unlikely(get_hmbird_rq(rq)->cpu_released)) { -+ /* -+ * If the previous sched_class for the current CPU was not HMBIRD, -+ * notify the BPF scheduler that it again has control of the -+ * core. This callback complements ->cpu_release(), which is -+ * emitted in hmbird_notify_pick_next_task(). -+ */ -+ get_hmbird_rq(rq)->cpu_released = false; -+ } -+ -+ if (prev_on_hmbird) -+ update_curr_hmbird(rq); -+ /* if there already are tasks to run, nothing to do */ -+ if (hmbird_rq->local_dsq.nr) -+ return 1; -+ -+ if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) { -+ slim_stats_record(GLOBAL_STAT, 1, 0, 0); -+ return 1; -+ } -+ -+ if (consume_dispatch_global(rq, rf)) -+ return 1; -+ -+ return 0; -+} -+ -+static int balance_hmbird(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf) -+{ -+ return balance_one(rq, prev, rf, true); -+} -+ -+/* -+ * output task util to systrace, only for debug mode. -+ * we can not output too many logs to systrace buffer even in debug mode -+ * only output debug-info while it exceed misfit_ds. -+ */ -+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p) -+{ -+ static DEFINE_PER_CPU(int, is_last_exceed); -+ int cpu = cpu_of(rq); -+ u64 util = 0; -+ -+ if (likely(!debug_enabled())) -+ return; -+ -+ if (!p) -+ return; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ util = uclamp_rq_util_with(rq, util, p); -+ -+ if (util >= misfit_ds) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util); -+ per_cpu(is_last_exceed, cpu) = true; -+ } else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0); -+ per_cpu(is_last_exceed, cpu) = false; -+ } else { -+ } -+} -+ -+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first) -+{ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ /* -+ * Core-sched might decide to execute @p before it is -+ * dispatched. Call ops_dequeue() to notify the BPF scheduler. -+ */ -+ ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC); -+ dispatch_dequeue(get_hmbird_rq(rq), p); -+ } -+ -+ p->se.exec_start = rq_clock_task(rq); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK); -+ } -+ -+ watchdog_unwatch_task(p, true); -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx); -+ systrace_output_cpu_ds(rq, p); -+ /* -+ * @p is getting newly scheduled or got kicked after someone updated its -+ * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird(). -+ */ -+ if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) != -+ (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) { -+ if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) -+ get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK; -+ else -+ get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK; -+ -+ sched_update_tick_dependency(rq); -+ } -+ -+ p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime; -+} -+ -+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p) -+{ -+ update_curr_hmbird(rq); -+ -+ update_dispatch_dsq_info(rq, p); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ watchdog_watch_task(rq, p); -+ -+ if (is_pipeline_task(p)) { -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1); -+ return; -+ } -+ -+ /* -+ * If we're in the pick_next_task path, balance_hmbird() should -+ * have already populated the local DSQ if there are any other -+ * available tasks. If empty, tell ops.enqueue() that @p is the -+ * only one available for this cpu. ops.enqueue() should put it -+ * on the local DSQ so that the subsequent pick_next_task_hmbird() -+ * can find the task unless it wants to trigger a separate -+ * follow-up scheduling event. -+ */ -+ if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1); -+ else -+ do_enqueue_task(rq, p, 0, -1); -+ } -+} -+ -+static struct task_struct *first_local_task(struct rq *rq) -+{ -+ struct rb_node *rb_node; -+ struct hmbird_entity *entity; -+ -+ if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) { -+ entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo, -+ struct hmbird_entity, dsq_node.fifo); -+ return entity->task; -+ } -+ -+ rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq); -+ if (rb_node) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ return entity->task; -+ } -+ return NULL; -+} -+ -+static struct task_struct *pick_next_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p; -+ -+ p = first_local_task(rq); -+ if (!p) -+ return NULL; -+ -+ if (unlikely(!get_hmbird_ts(p)->slice)) { -+ if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice) -+ hmbird_warned_zero_slice = true; -+ -+ refill_task_slice(p); -+ } -+ -+ set_next_task_hmbird(rq, p, true); -+ -+ return p; -+} -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task, -+ const struct sched_class *active) -+{ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * The callback is conceptually meant to convey that the CPU is no -+ * longer under the control of HMBIRD. Therefore, don't invoke the -+ * callback if the CPU is staying on HMBIRD, or going idle (in which -+ * case the HMBIRD scheduler has actively decided not to schedule any -+ * tasks on the CPU). -+ */ -+ if (likely(active >= &hmbird_sched_class)) -+ return; -+ -+ /* -+ * At this point we know that HMBIRD was preempted by a higher priority -+ * sched_class, so invoke the ->cpu_release() callback if we have not -+ * done so already. We only send the callback once between HMBIRD being -+ * preempted, and it regaining control of the CPU. -+ * -+ * ->cpu_release() complements ->cpu_acquire(), which is emitted the -+ * next time that balance_hmbird() is invoked. -+ */ -+ if (!get_hmbird_rq(rq)->cpu_released) -+ get_hmbird_rq(rq)->cpu_released = true; -+} -+ -+#ifdef CONFIG_SMP -+ -+static bool test_and_clear_cpu_idle(int cpu) -+{ -+ if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -+ if (cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ return true; -+ } else { -+ return false; -+ } -+} -+ -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) -+{ -+ int cpu; -+ -+ do { -+ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -+ if (cpu >= nr_cpu_ids) -+ return -EBUSY; -+ } while (!test_and_clear_cpu_idle(cpu)); -+ -+ return cpu; -+} -+ -+ -+static bool prev_cpu_misfit(int prev) -+{ -+ if (!is_partial_enabled() && is_partial_cpu(prev)) -+ return true; -+ -+ return false; -+} -+ -+static int heavy_rt_placement(struct task_struct *p, int prev) -+{ -+ struct cpumask tmp = {.bits = {0}}; -+ int cpu; -+ u64 util = 0; -+ -+ if (!rt_prio(p->prio)) -+ return -EFAULT; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ -+ if (util < misfit_ds) -+ return -EFAULT; -+ -+ if (is_partial_enabled()) -+ cpumask_or(&tmp, iso_masks.big, iso_masks.partial); -+ else -+ cpumask_copy(&tmp, iso_masks.big); -+ -+ cpu = hmbird_pick_idle_cpu(&tmp); -+ if (cpu >= 0) -+ return cpu; -+ -+ if (cpumask_test_cpu(prev, iso_masks.big) || -+ (is_partial_enabled() && is_partial_cpu(prev))) -+ return prev; -+ -+ return cpumask_first(&tmp); -+} -+ -+static int spec_task_before_pick_idle(struct task_struct *p, int prev) -+{ -+ int cpu; -+ -+ cpu = heavy_rt_placement(p, prev); -+ if (cpu >= 0) -+ return cpu; -+ return -EFAULT; -+} -+ -+static int cpumask_distribute_next(struct cpumask *mask, int *prev) -+{ -+ int p = *prev, n; -+ -+ n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1); -+ if (n < nr_cpu_ids) -+ WRITE_ONCE(*prev, n); -+ return n; -+} -+ -+static int repick_fallback_cpu(void) -+{ -+ /* -+ * partial cpu follow big cluster's Scheduling policy, -+ * simply return first bit cpu. -+ */ -+ return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev); -+} -+ -+/* -+ * Must return a valid cpu num, as this task's cpu. -+ */ -+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu, -+ struct cpumask *mask, int *prev_mask) -+{ -+ int cpu; -+ -+ cpu = hmbird_pick_idle_cpu(mask); -+ if (cpu >= 0) -+ return cpu; -+ return cpumask_distribute_next(mask, prev_mask); -+} -+ -+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type) -+{ -+ int idx; -+ struct cluster_ctx ctx; -+ -+ idx = find_idx_from_task(p); -+ if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS) -+ return false; -+ -+ gen_cluster_ctx(&ctx, type); -+ if (idx >= ctx.lower && idx < ctx.upper) -+ return true; -+ return false; -+} -+ -+static bool is_valid_cpu(int cpu) -+{ -+ return (cpu >= 0) && (cpu < nr_cpu_ids); -+} -+ -+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) -+{ -+ s32 cpu = -1; -+ struct cpumask mask = {.bits = {0}}; -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (is_valid_cpu(cpu)) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ partial_dynamic_ctrl(); -+ -+ if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ iso_masks.big, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ if (p->nr_cpus_allowed == 1) { -+ cpu = cpumask_any(p->cpus_ptr); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* For non-period global dsq, not contain pcp task. */ -+ if (task_only_blongs_to_cluster(p, LITTLE)) { -+ cpumask_copy(&mask, iso_masks.little); -+ if (unlikely(l_need_rescue)) { -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ cpumask_or(&mask, iso_masks.ex_free, &mask); -+ } -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &little_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ if (task_only_blongs_to_cluster(p, BIG)) { -+ cpumask_copy(&mask, iso_masks.big); -+ if (unlikely(b_need_rescue)) { -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ cpumask_or(&mask, iso_masks.ex_free, &mask); -+ } -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ cpu = spec_task_before_pick_idle(p, prev_cpu); -+ if (is_valid_cpu(cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* if the previous CPU is idle, dispatch directly to it */ -+ if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+ } -+ -+ cpu = hmbird_pick_idle_cpu(cpu_possible_mask); -+ if (is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ if (prev_cpu_misfit(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ cpu = repick_fallback_cpu(); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+} -+ -+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags) -+{ -+ return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags); -+} -+ -+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx) -+{ -+ set_cpus_allowed_common(p, ctx); -+} -+ -+static void reset_idle_masks(void) -+{ -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little); -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big); -+ if (is_partial_enabled()) -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial); -+ hmbird_has_idle_cpus = true; -+} -+ -+void __hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ int cpu = cpu_of(rq); -+ struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -+ -+ if (skip_update_idle()) -+ return; -+ -+ if (idle) { -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ if (!hmbird_has_idle_cpus) -+ hmbird_has_idle_cpus = true; -+ -+ /* -+ * idle_masks.smt handling is racy but that's fine as it's only -+ * for optimization and self-correcting. -+ */ -+ for_each_cpu(cpu, sib_mask) { -+ if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -+ return; -+ } -+ cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -+ } else { -+ cpumask_clear_cpu(cpu, idle_masks.cpu); -+ if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ -+ cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -+ } -+} -+ -+#else /* !CONFIG_SMP */ -+ -+static bool test_and_clear_cpu_idle(int cpu) { return false; } -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } -+static void reset_idle_masks(void) {} -+ -+#endif /* CONFIG_SMP */ -+ -+static bool check_rq_for_timeouts(struct rq *rq) -+{ -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rq_flags rf; -+ -+ rq_lock_irqsave(rq, &rf); -+ list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) { -+ unsigned long last_runnable; -+ -+ p = entity->task; -+ last_runnable = get_hmbird_ts(p)->runnable_at; -+ -+ if (unlikely(time_after(jiffies, -+ last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -+ -+ rq_unlock_irqrestore(rq, &rf); -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb", -+ p->comm, p->pid, -+ dur_ms / 1000, dur_ms % 1000, -+ get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0, -+ cpumask_pr_args(&p->cpus_mask)); -+ return 1; -+ } -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ return 0; -+} -+ -+static void hmbird_watchdog_workfn(struct work_struct *work) -+{ -+ int cpu; -+ bool timeout; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ -+ for_each_online_cpu(cpu) { -+ timeout = check_rq_for_timeouts(cpu_rq(cpu)); -+ if (unlikely(timeout)) -+ break; -+ -+ cond_resched(); -+ } -+ if (!timeout) -+ queue_delayed_work(system_unbound_wq, to_delayed_work(work), -+ hmbird_watchdog_timeout / 2); -+} -+ -+static void set_pcp_round(struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) { -+ per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round); -+ per_cpu(pcp_info, cpu).pcp_round = true; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+} -+ -+ -+/* -+ * Just for debug: output hmbird on/off state per 10s. -+ */ -+#define OUTPUT_INTVAL (msecs_to_jiffies(10 * 1000)) -+static void inform_hmbird_onoff_from_systrace(void) -+{ -+ static unsigned long __read_mostly next_print; -+ -+ if (time_before(jiffies, READ_ONCE(next_print))) -+ return; -+ -+ WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL); -+ hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio); -+} -+ -+void hmbird_notify_sched_tick(void) -+{ -+ unsigned long last_check; -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ hmbird_scheduler_tick(); -+ -+ if (!hmbird_enabled()) -+ return; -+ -+ last_check = hmbird_watchdog_timestamp; -+ if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -+ -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "watchdog failed to check in for %u.%03us", -+ dur_ms / 1000, dur_ms % 1000); -+ } -+ scan_timeout(rq); -+} -+ -+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued) -+{ -+ update_curr_hmbird(rq); -+ -+ set_pcp_round(rq); -+ -+ if (slim_walt_ctrl) -+ hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE); -+ /* -+ * While disabling, always resched and refresh core-sched timestamp as -+ * we can't trust the slice management or ops.core_sched_before(). -+ */ -+ if (hmbird_ops_disabling()) -+ get_hmbird_ts(curr)->slice = 0; -+ -+ if (!get_hmbird_ts(curr)->slice) -+ resched_curr(rq); -+ -+ inform_hmbird_onoff_from_systrace(); -+} -+ -+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg) -+{ -+ hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ get_hmbird_ts(p)->disallow = false; -+ -+ hmbird_sched_init_task(p); -+ -+ set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ return 0; -+} -+ -+static void hmbird_ops_enable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void hmbird_ops_disable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void set_task_hmbird_weight(struct task_struct *p) -+{ -+ u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -+ -+ get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight); -+} -+ -+/** -+ * refresh_hmbird_weight - Refresh a task's hmbird weight -+ * @p: task to refresh hmbird weight for -+ * -+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to -+ * enable easy access from the BPF scheduler. To keep it synchronized with the -+ * current task priority, this function should be called when a new task is -+ * created, priority is changed for a task on hmbird, and a task is switched -+ * to hmbird from other classes. -+ */ -+static void refresh_hmbird_weight(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ set_task_hmbird_weight(p); -+} -+ -+int hmbird_pre_fork(struct task_struct *p) -+{ -+ int ret = 0; -+ -+ p->android_oem_data1[HMBIRD_TS_IDX] = -+ (u64)(kzalloc(sizeof(struct hmbird_entity), GFP_KERNEL)); -+ if (!get_hmbird_ts(p)) -+ return -1; -+ -+ get_hmbird_ts(p)->dsq = NULL; -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node); -+ get_hmbird_ts(p)->flags = 0; -+ get_hmbird_ts(p)->weight = 0; -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ get_hmbird_ts(p)->holding_cpu = -1; -+ get_hmbird_ts(p)->kf_mask = 0; -+ atomic64_set(&get_hmbird_ts(p)->ops_state, 0); -+ get_hmbird_ts(p)->runnable_at = INITIAL_JIFFIES; -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+ get_hmbird_ts(p)->task = p; -+ hmbird_set_sched_prop(p, 0); -+ -+ get_hmbird_ts(p)->critical_affinity_cpu = -1; -+ get_hmbird_ts(p)->sched_class = &hmbird_sched_class; -+ get_hmbird_ts(p)->tick_hit_count = 0; -+ get_hmbird_ts(p)->start_jiffies = 0; -+ -+ /* -+ * BPF scheduler enable/disable paths want to be able to iterate and -+ * update all tasks which can become complex when racing forks. As -+ * enable/disable are very cold paths, let's use a percpu_rwsem to -+ * exclude forks. -+ */ -+ -+ return ret; -+} -+ -+void hmbird_post_fork(struct task_struct *p) -+{ -+ percpu_down_read(&hmbird_fork_rwsem); -+ -+ if (hmbird_enabled()) { -+ struct rq_flags rf; -+ struct rq *rq; -+ p->sched_class = &hmbird_sched_class; -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ rq = task_rq_lock(p, &rf); -+ /* -+ * Set the weight manually before calling ops.enable() so that -+ * the scheduler doesn't see a stale value if they inspect the -+ * task struct. We'll invoke ops.set_weight() afterwards, as it -+ * would be odd to receive a callback on the task before we -+ * tell the scheduler that it's been fully enabled. -+ */ -+ set_task_hmbird_weight(p); -+ hmbird_ops_enable_task(p); -+ refresh_hmbird_weight(p); -+ task_rq_unlock(rq, p, &rf); -+ } else { -+ if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ } -+ -+ spin_lock_irq(&hmbird_tasks_lock); -+ list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_cancel_fork(struct task_struct *p) -+{ -+ if (hmbird_enabled()) -+ hmbird_ops_disable_task(p); -+ put_hmbird_ts(p); -+} -+ -+void hmbird_free(struct task_struct *p) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+ list_del_init(&get_hmbird_ts(p)->tasks_node); -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+ -+ /* -+ * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED -> -+ * ENABLED transitions can't race us. Disable ops for @p. -+ */ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) || -+ test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ hmbird_ops_disable_task(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ put_hmbird_ts(p); -+} -+ -+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio) -+{ -+} -+ -+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type) -+{ -+ return (prop >> TOP_TASK_SHIFT) & (1 << type); -+} -+ -+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p) -+{ -+ uint32_t prop = get_top_task_prop(p); -+ -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE)) -+ return HMBIRD_TASK_PROP_PIPELINE; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) || -+ !task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) { -+ return HMBIRD_TASK_PROP_COMMON; -+ } -+ return HMBIRD_TASK_PROP_DEBUG_OR_LOG; -+} -+ -+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b) -+{ -+ int type_a = hmbird_get_task_type(a); -+ int type_b = hmbird_get_task_type(b); -+ -+ return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b]; -+} -+ -+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ enum cpu_type type; -+ int sp_dl; -+ struct task_struct *curr = NULL; -+ -+ switch (hmbird_preempt_policy) { -+ case HMBIRD_PREEMPT_POLICY_PRIO_BASED: -+ curr = rq->curr; -+ if (curr && hmbird_prio_higher(p, curr)) -+ goto preempt; -+ break; -+ default: -+ break; -+ } -+ if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) || -+ (is_critical_system_task(p) && !is_critical_system_task(rq->curr))) -+ goto preempt; -+ -+ sp_dl = find_idx_from_task(p); -+ if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled())) -+ return; -+ -+ if (rq->curr->prio > p->prio) -+ goto preempt; -+ -+ return; -+ -+preempt: -+ resched_curr(rq); -+} -+ -+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {} -+ -+int hmbird_check_setscheduler(struct task_struct *p, int policy) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ /* if disallow, reject transitioning into HMBIRD */ -+ if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) && -+ p->policy != policy && policy == SCHED_HMBIRD) -+ return -EACCES; -+ -+ return 0; -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool hmbird_can_stop_tick(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ if (hmbird_ops_disabling()) -+ return false; -+ -+ if (p->sched_class != &hmbird_sched_class) -+ return true; -+ -+ return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK; -+} -+#endif -+ -+int hmbird_tg_online(struct task_group *tg) -+{ -+ struct cgroup *cgrp; -+ -+ if (!tg) -+ return 0; -+ cgrp = tg->css.cgroup; -+ if (!cgrp || !(cgrp->kn)) -+ return 0; -+ update_cgroup_ids_table(cgrp->kn->id, -1); -+ return 0; -+} -+ -+/* -+ * Omitted operations: -+ * -+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -+ * task isn't tied to the CPU at that point. Preemption is implemented by -+ * resetting the victim task's slice to 0 and triggering reschedule on the -+ * target CPU. -+ * -+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -+ * -+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -+ * their current sched_class. Call them directly from sched core instead. -+ * -+ * - task_woken, switched_from: Unnecessary. -+ */ -+DEFINE_SCHED_CLASS(hmbird) = { -+ .enqueue_task = enqueue_task_hmbird, -+ .dequeue_task = dequeue_task_hmbird, -+ .yield_task = yield_task_hmbird, -+ .yield_to_task = yield_to_task_hmbird, -+ -+ .check_preempt_curr = check_preempt_curr_hmbird, -+ -+ .pick_next_task = pick_next_task_hmbird, -+ -+ .put_prev_task = put_prev_task_hmbird, -+ .set_next_task = set_next_task_hmbird, -+ -+#ifdef CONFIG_SMP -+ .balance = balance_hmbird, -+ .select_task_rq = select_task_rq_hmbird, -+ .set_cpus_allowed = set_cpus_allowed_hmbird, -+#endif -+ .task_tick = task_tick_hmbird, -+ -+ .switched_to = switched_to_hmbird, -+ .prio_changed = prio_changed_hmbird, -+ -+ .update_curr = update_curr_hmbird, -+ -+#ifdef CONFIG_UCLAMP_TASK -+ .uclamp_enabled = 0, -+#endif -+}; -+ -+/* -+ * Must with rq lock held. -+ */ -+bool task_is_hmbird(struct task_struct *p) -+{ -+ return p->sched_class == &hmbird_sched_class; -+} -+ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id) -+{ -+ memset(dsq, 0, sizeof(*dsq)); -+ -+ raw_spin_lock_init(&dsq->lock); -+ INIT_LIST_HEAD(&dsq->fifo); -+ dsq->id = dsq_id; -+} -+ -+static int hmbird_cgroup_init(void) -+{ -+ struct cgroup_subsys_state *css; -+ -+ css_for_each_descendant_pre(css, &root_task_group.css) { -+ struct task_group *tg = css_tg(css); -+ -+ cgrp_dsq_idx_init(css->cgroup, tg); -+ } -+ return 0; -+} -+ -+/* -+ * Used by sched_fork() and __setscheduler_prio() to pick the matching -+ * sched_class. dl/rt are already handled. -+ */ -+bool task_on_hmbird(struct task_struct *p) -+{ -+ return hmbird_enabled(); -+} -+ -+static void __setscheduler_prio(struct task_struct *p, int prio) -+{ -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) { -+ p->prio = prio; -+ return; -+ } else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (on_hmbird && rt_prio(prio)) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ -+ p->prio = prio; -+} -+ -+/* -+ * Heartbeat, avoid humbird keep running while APP already exit. -+ * Check whether APP send alive-signal periodly. -+ */ -+#define HEARTBEAT_TIMEOUT (msecs_to_jiffies(2500)) -+#define HEARTBEAT_CHECK_INTERVAL (msecs_to_jiffies(1000)) -+static struct timer_list hb_timer; -+static unsigned long next_hb; -+ -+void hb_timer_handler(struct timer_list *timer) -+{ -+ if (!heartbeat_enable) -+ goto refill; -+ -+ pr_info(": enter timer.\n"); -+ if (heartbeat) { -+ heartbeat = 0; -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ } -+ -+ /* can't detect heartbeat, disable ext. */ -+ if (time_after(jiffies, READ_ONCE(next_hb))) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT, -+ "can't detect heartbeat, disable ext\n"); -+ } -+refill: -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_start(void) -+{ -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_init(void) -+{ -+ timer_setup(&hb_timer, hb_timer_handler, 0); -+} -+ -+static void hb_timer_exit(void) -+{ -+ del_timer(&hb_timer); -+} -+ -+static bool check_and_disable_cpuhp(void) -+{ -+ struct rq *rq; -+ struct rq_flags rf; -+ int cpu; -+ -+ cpu_hotplug_disable(); -+ cpus_read_lock(); -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ rq_lock_irqsave(rq, &rf); -+ if (!rq->online) { -+ rq_unlock_irqrestore(rq, &rf); -+ goto err_offline; -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ cpus_read_unlock(); -+ return true; -+ -+err_offline: -+ cpus_read_unlock(); -+ cpu_hotplug_enable(); -+ return false; -+} -+ -+static void reenable_cpuhp(void) -+{ -+ cpu_hotplug_enable(); -+} -+ -+static void scheduler_switch_done(bool final_state) -+{ -+ /* set scx_enable again, switch may fail. */ -+ scx_enable = final_state; -+ /* reset heartbeat*/ -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ -+ if (final_state) -+ hb_timer_start(); -+ else -+ hb_timer_exit(); -+} -+ -+/** -+ * ss : curr switch state -+ * finish : success or fail -+ * enable : curr operation is diabling or enabling? -+ * fail_reason : string output when failed, empty string when success. -+ */ -+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason) -+{ -+ char *s1 = finish ? "finished" : "failed"; -+ char *s2 = enable ? "enabled" : "disabled"; -+ -+ hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss); -+ if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) { -+ sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type)); -+ hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n", -+ s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason); -+ } -+ curr_ss = ss; -+} -+ -+bool get_hmbird_ops_enabled(void) -+{ -+ return atomic_read(&__hmbird_ops_enabled); -+} -+ -+bool get_non_hmbird_task(void) -+{ -+ return atomic_read(&non_hmbird_task); -+} -+ -+static void hmbird_ops_disable_workfn(struct kthread_work *work) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int cpu; -+ -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, ""); -+ cancel_delayed_work_sync(&hmbird_watchdog_work); -+ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) { -+ case HMBIRD_OPS_DISABLED: -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled"); -+ scheduler_switch_done(false); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return; -+ case HMBIRD_OPS_PREPPING: -+ fallthrough; -+ case HMBIRD_OPS_DISABLING: -+ /* shouldn't happen but handle it like ENABLING if it does */ -+ WARN_ONCE(true, "hmbird: duplicate disabling instance?"); -+ fallthrough; -+ case HMBIRD_OPS_ENABLING: -+ case HMBIRD_OPS_ENABLED: -+ break; -+ } -+ -+ /* kick all CPUs to restore ticks */ -+ for_each_possible_cpu(cpu) -+ resched_cpu(cpu); -+ -+ /* avoid racing against fork and cgroup changes */ -+ cpus_read_lock(); -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, ""); -+ spin_lock_irq(&hmbird_tasks_lock); -+ atomic_set(&__hmbird_ops_enabled, false); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ bool alive = READ_ONCE(p->__state) != TASK_DEAD; -+ -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ get_hmbird_ts(p)->slice = min_t(u64, -+ get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL); -+ -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ if (alive) -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ -+ hmbird_ops_disable_task(p); -+ } -+ hmbird_task_iter_exit(&sti); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, ""); -+ -+ atomic_set(&non_hmbird_task, true); -+ /* no task is on hmbird, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt); -+ synchronize_rcu(); -+ -+ percpu_up_write(&hmbird_fork_rwsem); -+ cpus_read_unlock(); -+ -+ if (slim_walt_ctrl) -+ slim_walt_enable(false); -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ -+ hmbird_switch_log(HMBIRD_DISABLED, 1, 0, ""); -+ scheduler_switch_done(false); -+ reenable_cpuhp(); -+} -+ -+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn); -+ -+static void schedule_hmbird_ops_disable_work(void) -+{ -+ struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper); -+ -+ /* -+ * We may be called spuriously before the first bpf_hmbird_reg(). If -+ * hmbird_ops_helper isn't set up yet, there's nothing to do. -+ */ -+ if (helper) -+ kthread_queue_work(helper, &hmbird_ops_disable_work); -+} -+ -+static void hmbird_ops_disable(void) -+{ -+ schedule_hmbird_ops_disable_work(); -+} -+ -+static void hmbird_err_exit_workfn(struct work_struct *work) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ hmbird_ctrl(false); -+ -+ for_each_present_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy) -+ continue; -+ if (cpu != policy->cpu) -+ goto put; -+ down_write(&policy->rwsem); -+ WARN_ON(store_scaling_governor(policy, -+ saved_gov[cpu], strlen(saved_gov[cpu])) <= 0); -+ up_write(&policy->rwsem); -+ hmbird_info_trace(":restore origin gov : %s\n", saved_gov[cpu]); -+put: -+ cpufreq_cpu_put(policy); -+ } -+ memset((char *)saved_gov, 0, sizeof(saved_gov)); -+} -+ -+void hmbird_ops_exit(void) -+{ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); -+} -+ -+static struct kthread_worker *hmbird_create_rt_helper(const char *name) -+{ -+ struct kthread_worker *helper; -+ -+ helper = kthread_create_worker(KTW_FREEZABLE, name); -+ if (helper) -+ sched_set_fifo(helper->task); -+ return helper; -+} -+ -+static inline void set_audio_thread_sched_prop(struct task_struct *p) -+{ -+ struct cgroup_subsys_state *css; -+ -+ if (likely(p->prio >= MAX_RT_PRIO)) -+ return; -+ -+ rcu_read_lock(); -+ css = task_css(p, cpuset_cgrp_id); -+ if (!css) { -+ rcu_read_unlock(); -+ return; -+ } -+ rcu_read_unlock(); -+ -+ if (!strcmp(css->cgroup->kn->name, "audio-app")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+} -+ -+int scx_systemui_pid = -1; -+void set_systemui_thread_pid(struct task_struct *p) -+{ -+ if ((strcmp(p->comm, "ndroid.systemui") == 0) && (p->pid == p->tgid)) -+ scx_systemui_pid = p->pid; -+} -+ -+static int hmbird_ops_enable(void *unused) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int ret; -+ int tcnt = 0; -+ unsigned long long start = 0; -+ -+ if (!check_and_disable_cpuhp()) { -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline"); -+ return -EBUSY; -+ } -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, ""); -+ mutex_lock(&hmbird_ops_enable_mutex); -+ -+ if (!hmbird_ops_helper) { -+ WRITE_ONCE(hmbird_ops_helper, -+ hmbird_create_rt_helper("hmbird_ops_helper")); -+ if (!hmbird_ops_helper) { -+ ret = -ENOMEM; -+ goto err_unlock; -+ } -+ } -+ -+ if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) { -+ ret = -EBUSY; -+ goto err_unlock; -+ } -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) != -+ HMBIRD_OPS_DISABLED); -+ -+ hmbird_warned_zero_slice = false; -+ -+ atomic64_set(&hmbird_nr_rejected, 0); -+ -+ /* -+ * Keep CPUs stable during enable so that the BPF scheduler can track -+ * online CPUs by watching ->on/offline_cpu() after ->init(). -+ */ -+ cpus_read_lock(); -+ -+ hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work, -+ hmbird_watchdog_timeout / 2); -+ -+ /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ */ -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ reset_idle_masks(); -+ -+ /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. -+ */ -+ ret = hmbird_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock; -+ -+ /* -+ * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem -+ * preventing new tasks from being added. No need to exclude tasks -+ * leaving as hmbird_free() can handle both prepped and enabled -+ * tasks. Prep all tasks first and then enable them with preemption -+ * disabled. -+ */ -+ spin_lock_irq(&hmbird_tasks_lock); -+ -+ atomic_set(&non_hmbird_task, false); -+ atomic_set(&__hmbird_ops_enabled, true); -+ -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered(&sti))) -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ hmbird_task_iter_exit(&sti); -+ -+ /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. -+ */ -+ preempt_disable(); -+ -+ /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ */ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) { -+ atomic_set(&non_hmbird_task, true); -+ atomic_set(&__hmbird_ops_enabled, false); -+ preempt_enable(); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ ret = -EBUSY; -+ goto err_disable_unlock; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED -+ * transitions here are synchronized against hmbird_free() through -+ * hmbird_tasks_lock. -+ */ -+ start = sched_clock(); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, ""); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ tcnt++; -+ if (READ_ONCE(p->__state) != TASK_DEAD) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ -+ set_audio_thread_sched_prop(p); -+ set_systemui_thread_pid(p); -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ hmbird_ops_enable_task(p); -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ } else { -+ hmbird_ops_disable_task(p); -+ } -+ } -+ hmbird_task_iter_exit(&sti); -+ -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, ""); -+ preempt_enable(); -+ percpu_up_write(&hmbird_fork_rwsem); -+ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) { -+ ret = -EBUSY; -+ goto err_disable; -+ } -+ -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_ENABLED, 1, 1, ""); -+ scheduler_switch_done(true); -+ -+ return 0; -+ -+err_unlock: -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock"); -+ scheduler_switch_done(false); -+ return ret; -+ -+err_disable_unlock: -+ percpu_up_write(&hmbird_fork_rwsem); -+err_disable: -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ /* must be fully disabled before returning */ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable"); -+ scheduler_switch_done(false); -+ return ret; -+} -+ -+#ifdef CONFIG_SCHED_DEBUG -+static const char *hmbird_ops_enable_state_str[] = { -+ [HMBIRD_OPS_PREPPING] = "prepping", -+ [HMBIRD_OPS_ENABLING] = "enabling", -+ [HMBIRD_OPS_ENABLED] = "enabled", -+ [HMBIRD_OPS_DISABLING] = "disabling", -+ [HMBIRD_OPS_DISABLED] = "disabled", -+}; -+ -+static int hmbird_debug_show(struct seq_file *m, void *v) -+{ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled()); -+ seq_printf(m, "%-30s: %s\n", "enable_state", -+ hmbird_ops_enable_state_str[hmbird_ops_enable_state()]); -+ seq_printf(m, "%-30s: %llu\n", "nr_rejected", -+ atomic64_read(&hmbird_nr_rejected)); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return 0; -+} -+ -+static int hmbird_debug_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_debug_show, NULL); -+} -+ -+const struct file_operations sched_hmbird_fops = { -+ .open = hmbird_debug_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif -+ -+ -+static int bpf_hmbird_reg(void *kdata) -+{ -+ return hmbird_ops_enable(kdata); -+} -+ -+static int bpf_hmbird_unreg(void *kdata) -+{ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ return 0; -+} -+ -+void set_hmbird_module_loaded(int is_loaded) -+{ -+ atomic_set(&hmbird_module_loaded, is_loaded); -+} -+ -+/* -+ * MUST load hmbird module before enable hmbird scheduler -+ * load track & hmbird gover implemented in hmbird module -+ */ -+int hmbird_ctrl(bool enable) -+{ -+ if (!atomic_read(&hmbird_module_loaded)) { -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "ext module unloaded\n"); -+ return -EINVAL; -+ } -+ -+ if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED -+ || hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING -+ || hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already enabled(ing)\n"); -+ hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING || -+ hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already disabled(ing)\n"); -+ hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (enable) -+ return bpf_hmbird_reg(NULL); -+ else -+ return bpf_hmbird_unreg(NULL); -+} -+ -+void set_cpu_isomask(int cpu, cpumask_var_t *mask) -+{ -+ cpumask_clear_cpu(cpu, iso_masks.ex_free); -+ cpumask_clear_cpu(cpu, iso_masks.exclusive); -+ cpumask_clear_cpu(cpu, iso_masks.partial); -+ cpumask_clear_cpu(cpu, iso_masks.big); -+ cpumask_clear_cpu(cpu, iso_masks.little); -+ cpumask_set_cpu(cpu, *mask); -+} -+ -+void set_cpu_cluster(u64 cpu_cluster) -+{ -+ int cpu; -+ -+ for_each_present_cpu(cpu) { -+ u64 pos = 1 << cpu; -+ -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.ex_free)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.exclusive)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.partial)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.big)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) -+ set_cpu_isomask(cpu, &(iso_masks.little)); -+ } -+} -+ -+static void get_hmbird_snapshot(struct panic_snapshot_t *p) -+{ -+ struct hmbird_dispatch_q *dsq; -+ struct rq *rq; -+ int cpu, i; -+ -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ p->rq_nr[cpu] = rq->nr_running; -+ p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running; -+ } -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ struct hmbird_entity *entity; -+ -+ dsq = &gdsqs[i]; -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo)) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ p->runnable_at[i] = entity->runnable_at; -+ raw_spin_unlock(&dsq->lock); -+ -+ } -+ -+ p->snap_misc.hmbird_enabled = hmbird_enabled(); -+ p->snap_misc.curr_ss = curr_ss; -+ p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state(); -+ p->snap_misc.parctrl_high_ratio = parctrl_high_ratio; -+ p->snap_misc.parctrl_low_ratio = parctrl_low_ratio; -+ p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l; -+ p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l; -+ p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio; -+ p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio; -+ p->snap_misc.misfit_ds = misfit_ds; -+ p->snap_misc.partial_enable = partial_enable; -+ p->snap_misc.iso_free_rescue = iso_free_rescue; -+ p->snap_misc.isolate_ctrl = isolate_ctrl; -+ p->snap_misc.snap_jiffies = jiffies; -+ p->snap_misc.snap_time = local_clock(); -+} -+ -+// MTK minidump begin -+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3) -+{ -+ strscpy(m->desc_str, str, DESC_STR_LEN); -+ m->len = d1 * d2 * d3; -+ m->parse[0] = d1; -+ m->parse[1] = d2; -+ m->parse[2] = d3; -+} -+ -+static void init_desc_metas(struct md_info_t *m) -+{ -+ init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :", -+ 1, MAX_SWITCHS, SWITCH_ITEMS); -+ -+ init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1); -+ -+ init_desc_meta(&m->kern_dump.excep_rec_meta, -+ "excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS); -+ -+ init_desc_meta(&m->kern_dump.excep_idx_meta, -+ "excep idx :", 1, 1, MAX_EXCEP_ID); -+ -+ init_desc_meta(&m->kern_dump.snap.runnable_at_meta, -+ "each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS); -+ -+ init_desc_meta(&m->kern_dump.snap.rq_nr_meta, -+ "rq runnable task nr:", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta, -+ "scxrq runnable task nr :", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.snap_misc_meta, -+ "misc snap params :", 1, 1, SNAP_ITEMS); -+} -+ -+static void init_md_meta(struct md_info_t *m) -+{ -+ m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64); -+ m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64); -+ m->meta.unit_size = sizeof(u64); -+ m->meta.switches = MAX_SWITCHS; -+ m->meta.exceps = MAX_EXCEPS; -+ m->meta.global_dsqs = MAX_GLOBAL_DSQS; -+ m->meta.parse_dimens = PARSE_DIMENS; -+ m->meta.nr_cpus = num_possible_cpus(); -+ m->meta.real_cpus = nr_cpu_ids; -+ m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64); -+ m->meta.nr_meta_desc = 8; -+ m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64); -+ -+ init_desc_metas(m); -+} -+ -+#define MINIDUMP_DFL_SIZE (4 * 1024) -+ -+struct notifier_block hmbird_panic_blk; -+static int hmbird_panic_handler(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ if (!md_info) -+ return NOTIFY_DONE; -+ -+ get_hmbird_snapshot(&md_info->kern_dump.snap); -+ -+ return NOTIFY_DONE; -+} -+ -+static void panic_blk_init(void) -+{ -+ int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE); -+ -+ md_info = kzalloc(dump_size, GFP_KERNEL); -+ if (!md_info) -+ return; -+ init_md_meta(md_info); -+ -+ hmbird_panic_blk.notifier_call = hmbird_panic_handler; -+ /* make sure to execute before minidump. */ -+ hmbird_panic_blk.priority = INT_MAX; -+ atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk); -+ -+ hmbird_debug("register minidump.\n"); -+} -+ -+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size) -+{ -+ *vaddr = (unsigned long)md_info; -+ *size = sizeof(struct md_info_t); -+} -+// MTK minidump end -+ -+static inline void init_sched_prop_to_preempt_prio(void) -+{ -+ for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) { -+ switch (i) { -+ case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 5; -+ break; -+ -+ case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 4; -+ break; -+ -+ case HMBIRD_TASK_PROP_PIPELINE: -+ case HMBIRD_TASK_PROP_ISOLATE: -+ sched_prop_to_preempt_prio[i] = 3; -+ break; -+ -+ case HMBIRD_TASK_PROP_COMMON: -+ sched_prop_to_preempt_prio[i] = 1; -+ break; -+ -+ case HMBIRD_TASK_PROP_DEBUG_OR_LOG: -+ sched_prop_to_preempt_prio[i] = 0; -+ break; -+ -+ default: -+ sched_prop_to_preempt_prio[i] = 2; -+ break; -+ } -+ } -+} -+ -+void __init init_sched_hmbird_class(void) -+{ -+ int cpu; -+ u32 v; -+ struct hmbird_entity *init_hmbird; -+ -+ /* -+ * The following is to prevent the compiler from optimizing out the enum -+ * definitions so that BPF scheduler implementations can use them -+ * through the generated vmlinux.h. -+ */ -+ WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP | -+ HMBIRD_KICK_PREEMPT); -+ -+ init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL); -+ init_dsq_at_boot(); -+ init_isolate_cpus(); -+ hb_timer_init(); -+ init_sched_prop_to_preempt_prio(); -+#ifdef CONFIG_SMP -+ WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); -+#endif -+ -+ /* -+ * we can't static init init_task's hmbird struct, init here. -+ * init_task->hmbird would not use during boot. -+ */ -+ init_hmbird = kzalloc(sizeof(struct hmbird_entity), GFP_KERNEL); -+ init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird; -+ if (init_hmbird) { -+ INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo); -+ INIT_LIST_HEAD(&init_hmbird->watchdog_node); -+ init_hmbird->sticky_cpu = -1; -+ init_hmbird->holding_cpu = -1; -+ atomic64_set(&init_hmbird->ops_state, 0); -+ init_hmbird->runnable_at = jiffies; -+ init_hmbird->slice = HMBIRD_SLICE_DFL; -+ init_hmbird->task = &init_task; -+ hmbird_set_sched_prop(&init_task, 0); -+ } else { -+ hmbird_err(INIT_TASK_FAIL, ":alloc init_task.scx failed!!!\n"); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ /* -+ * exec during boot phase, no need to care about alloc failed. -+ * lifecycle same to rq, no need to free. -+ */ -+ rq->android_oem_data1[HMBIRD_RQ_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_rq), GFP_KERNEL); -+ if (get_hmbird_rq(rq)) { -+ get_hmbird_rq(rq)->rq = rq; -+ get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window; -+ } else { -+ hmbird_err(ALLOC_RQSCX_FAIL, ":alloc rq->scx failed!!!\n"); -+ } -+ -+ rq->android_oem_data1[HMBIRD_OPS_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL); -+ if (!get_hmbird_ops(rq)) -+ pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n"); -+ init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL); -+ INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list); -+ -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL)); -+ -+ hmbird_ops_init(get_hmbird_ops(rq)); -+ } -+ -+ INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn); -+ INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn); -+ hmbird_misc_init(); -+ -+ panic_blk_init(); -+} -+ -diff --git b/kernel/sched/hmbird/hmbird_misc.c a/kernel/sched/hmbird/hmbird_misc.c -new file mode 100755 -index 000000000000..895e8a29df33 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_misc.c -@@ -0,0 +1,148 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * @Description: -+ * @Version: -+ * @Author: wangrui8 -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include "hmbird_sched.h" -+#include -+#include -+#include "slim.h" -+ -+noinline int tracing_mark_write(const char *buf) -+{ -+ trace_printk(buf); -+ return 0; -+} -+ -+struct yield_opt_params yield_opt_params = { -+ .enable = 0, -+ .frame_per_sec = 120, -+ .frame_time_ns = NSEC_PER_SEC / 120, -+ .yield_headroom = 10, -+}; -+ -+DEFINE_PER_CPU(struct sched_yield_state, ystate); -+ -+static inline void hmbird_yield_state_update(struct sched_yield_state *ys) -+{ -+ if (!raw_spin_is_locked(&ys->lock)) -+ return; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ -+ if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1 -+ || ys->yield_cnt_after_sleep > yield_headroom) { -+ ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP); -+ } else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) { -+ ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP); -+ } -+ ys->yield_cnt = 0; -+ ys->sleep_times = 0; -+ ys->yield_cnt_after_sleep = 0; -+} -+ -+void hmbird_skip_yield(long *skip) -+{ -+ if (!get_hmbird_ops_enabled() || !yield_opt_params.enable) -+ return; -+ unsigned long flags, sleep_now = 0; -+ struct sched_yield_state *ys; -+ int cpu = raw_smp_processor_id(), cont_yield, new_frame; -+ int frame_time_ns = yield_opt_params.frame_time_ns; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ u64 wc; -+ -+ if (!(*skip)) { -+ wc = sched_clock(); -+ ys = &per_cpu(ystate, cpu); -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ -+ cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP; -+ new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1); -+ -+ if (!cont_yield && new_frame) { -+ hmbird_yield_state_update(ys); -+ ys->last_update_time = wc; -+ ys->sleep_end = ys->last_yield_time + frame_time_ns -+ - yield_headroom * YIELD_DURATION; -+ } -+ -+ if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) { -+ *skip = true; -+ -+ sleep_now = ys->sleep_times ? -+ max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep; -+ if (wc + sleep_now > ys->sleep_end) { -+ u64 delta = ys->sleep_end - wc; -+ -+ if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION) -+ sleep_now = delta; -+ else -+ sleep_now = 0; -+ } -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ if (sleep_now) { -+ sleep_now = div64_u64(sleep_now, 1000); -+ usleep_range_state(sleep_now, sleep_now, TASK_IDLE); -+ } -+ ys->sleep_times++; -+ ys->last_yield_time = sched_clock(); -+ return; -+ } -+ if (ys->sleep_times) -+ ys->yield_cnt_after_sleep++; -+ else -+ (ys->yield_cnt)++; -+ ys->last_yield_time = wc; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+} -+ -+struct boost_policy_params boost_policy_params = { -+ .enable = 0, -+ .bottom_freq = 1200000, -+ .boost_weight = 120, -+}; -+ -+int hmbird_get_boost_enable(void) -+{ -+ return boost_policy_params.enable; -+} -+ -+unsigned int hmbird_get_boost_bottom_freq(void) -+{ -+ return boost_policy_params.bottom_freq; -+} -+ -+int hmbird_get_boost_weight(void) -+{ -+ return boost_policy_params.boost_weight; -+} -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops) -+{ -+ hmbird_ops->scx_enable = get_hmbird_ops_enabled; -+ hmbird_ops->check_non_task = get_non_hmbird_task; -+ hmbird_ops->hmbird_get_md_info = hmbird_get_md_info; -+ hmbird_ops->hmbird_get_boost_enable = hmbird_get_boost_enable; -+ hmbird_ops->hmbird_get_boost_bottom_freq = hmbird_get_boost_bottom_freq; -+ hmbird_ops->hmbird_get_boost_weight = hmbird_get_boost_weight; -+} -+ -+void hmbird_misc_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_init(&ys->lock); -+ } -+ -+ set_hmbird_module_loaded(1); -+} -+ -diff --git b/kernel/sched/hmbird/hmbird_sched.h a/kernel/sched/hmbird/hmbird_sched.h -new file mode 100755 -index 000000000000..76acbe9685e4 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef __HMBIRD_SCHED__ -+#define __HMBIRD_SCHED__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include <../../kernel/time/tick-sched.h> -+#include <../../kernel/sched/sched.h> -+#include -+#include -+#include -+ -+#define REGISTER_TRACE_VH(vender_hook, handler) \ -+{ \ -+ ret = register_trace_##vender_hook(handler, NULL); \ -+ if (ret) { \ -+ pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \ -+ } \ -+} -+#define REGISTER_TRACE(vendor_hook, handler, data, err) \ -+do { \ -+ ret = register_trace_##vendor_hook(handler, data); \ -+ if (ret) { \ -+ pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret); \ -+ goto err; \ -+ } \ -+} while (0) -+ -+#define UNREGISTER_TRACE(vendor_hook, handler, data) \ -+ unregister_trace_##vendor_hook(handler, data) \ -+ -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+ -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int sched_ravg_window_frame_per_sec; -+extern int slim_gov_debug; -+extern int cpu7_tl; -+extern int scx_gov_ctrl; -+extern spinlock_t new_sched_ravg_window_lock; -+extern int cluster_separate; -+ -+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER BIT(31) -+#define MAX_YIELD_SLEEP (2000000ULL) -+#define MIN_YIELD_SLEEP (200000ULL) -+#define YIELD_DURATION (5000ULL) -+#define DEFAULT_YIELD_SLEEP_TH (10) -+ -+struct sched_yield_state { -+ raw_spinlock_t lock; -+ u64 last_yield_time; -+ u64 last_update_time; -+ u64 sleep_end; -+ unsigned long yield_cnt; -+ unsigned long yield_cnt_after_sleep; -+ unsigned long sleep; -+ int sleep_times; -+}; -+ -+DECLARE_PER_CPU(struct sched_yield_state, ystate); -+ -+void hmbird_window_rollover_run_once(struct rq *rq); -+void hmbird_yield_state_update_per_frame(void); -+void hmbird_misc_init(void); -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops); -+#endif /*__HMBIRD_SCHED__*/ -diff --git b/kernel/sched/hmbird/hmbird_sched_proc.c a/kernel/sched/hmbird/hmbird_sched_proc.c -new file mode 100755 -index 000000000000..295d45404608 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched_proc.c -@@ -0,0 +1,640 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "hmbird_sched_proc.h" -+#include -+ -+#include "hmbird_util_track.h" -+#include "slim.h" -+ -+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched" -+#define SLIM_FREQ_GOV_DIR "slim_freq_gov" -+#define LOAD_TRACK_DIR "slim_walt" -+#define HMBIRD_PROC_PERMISSION 0666 -+ -+int scx_enable; -+int partial_enable; -+int cpuctrl_high_ratio = 55; -+int cpuctrl_low_ratio = 40; -+int slim_stats; -+int hmbirdcore_debug; -+int slim_for_app; -+int misfit_ds = 90; -+unsigned int highres_tick_ctrl; -+unsigned int highres_tick_ctrl_dbg; -+int cpu7_tl = 70; -+int slim_walt_ctrl; -+int slim_walt_dump; -+int slim_walt_policy; -+int slim_gov_debug; -+int scx_gov_ctrl = 1; -+int sched_ravg_window_frame_per_sec = 125; -+int parctrl_high_ratio = 55; -+int parctrl_low_ratio = 40; -+int parctrl_high_ratio_l = 65; -+int parctrl_low_ratio_l = 50; -+int isoctrl_high_ratio = 75; -+int isoctrl_low_ratio = 60; -+int isolate_ctrl; -+int iso_free_rescue; -+int heartbeat; -+int heartbeat_enable = 1; -+int watchdog_enable; -+int save_gov; -+u64 cpu_cluster_masks; -+int hmbird_preempt_policy; -+int cluster_separate; -+ -+char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= 32) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtoint(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoint\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+static int set_proc_buf_val_u64(struct file *file, const char __user *buf, -+ size_t count, u64 *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= sizeof(kbuf)) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtou64(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoul\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/* common ops begin */ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ return count; -+} -+ -+static int hmbird_common_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%d\n", *(int *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_show, pde_data(inode)); -+} -+ -+static int hmbird_common_ul_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%lu\n", *(unsigned long *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_ul_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_ul_show, pde_data(inode)); -+} -+ -+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write); -+/* common ops end */ -+ -+/* scx_enable ops begin */ -+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC); -+ if (hmbird_ctrl(*pval)) -+ return -EFAULT; -+ -+ return count; -+} -+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write); -+/* scx_enable ops end */ -+ -+/* hmbird_stats ops begin */ -+#define MAX_STATS_BUF (4096) -+static int hmbird_stats_proc_show(struct seq_file *m, void *v) -+{ -+ char *buf; -+ -+ buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ stats_print(buf, MAX_STATS_BUF); -+ -+ seq_printf(m, "%s\n", buf); -+ -+ kfree(buf); -+ return 0; -+} -+ -+static int hmbird_stats_proc_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_stats_proc_show, inode); -+} -+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL); -+/* hmbird_stats ops end */ -+ -+/* sched_ravg_window_frame_per_sec ops begin */ -+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file, -+ const char __user *buf, size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ sched_ravg_window_change(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open, -+ sched_ravg_window_frame_per_sec_proc_write); -+/* sched_ravg_window_frame_per_sec ops end */ -+ -+static ssize_t save_gov_str(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ for_each_possible_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy || (cpu != policy->cpu)) -+ continue; -+ WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0); -+ hmbird_info_systrace(":save origin gov : %s\n", saved_gov[cpu]); -+ } -+ return count; -+} -+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str); -+ -+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ u64 *pval = (u64 *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val_u64(file, buf, count, pval)) -+ return -EFAULT; -+ -+ if (scx_enable == 0) -+ set_cpu_cluster(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write); -+ -+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ int tmp_val; -+ -+ if (set_proc_buf_val(file, buf, count, &tmp_val)) -+ return -EFAULT; -+ -+ slim_walt_enable(tmp_val); -+ *pval = tmp_val; -+ -+ return count; -+} -+ -+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write); -+ -+/* yield_opt ops begin */ -+static int yield_opt_show(struct seq_file *m, void *v) -+{ -+ struct yield_opt_params *data = m->private; -+ -+ seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n", -+ data->enable, data->frame_per_sec, data->yield_headroom); -+ return 0; -+} -+ -+static int yield_opt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, yield_opt_show, pde_data(inode)); -+} -+ -+static ssize_t yield_opt_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu; -+ unsigned long flags; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if (copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp -+ != 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) || -+ (yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp; -+ yield_opt_params.frame_per_sec = frame_per_sec_tmp; -+ yield_opt_params.yield_headroom = yield_headroom_tmp; -+ yield_opt_params.enable = enable_tmp; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ ys->last_yield_time = 0; -+ ys->last_update_time = 0; -+ ys->sleep_end = 0; -+ ys->yield_cnt = 0; -+ ys->yield_cnt_after_sleep = 0; -+ ys->sleep = 0; -+ ys->sleep_times = 0; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write); -+ -+/* boost_policy ops begin */ -+static int boost_policy_show(struct seq_file *m, void *v) -+{ -+ struct boost_policy_params *data = m->private; -+ -+ seq_printf(m, "boost_policy:{\"enable\":%d; \"bottom_freq\":%u; \"boost_weight\":%d}\n", -+ data->enable, data->bottom_freq, data->boost_weight); -+ return 0; -+} -+ -+static int boost_policy_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, boost_policy_show, pde_data(inode)); -+} -+ -+static ssize_t boost_policy_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, bottom_freq_tmp, boost_weight_tmp; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if(copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &bottom_freq_tmp, &boost_weight_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || -+ (boost_weight_tmp < 50 || boost_weight_tmp > 300) || -+ (bottom_freq_tmp < 400000 || bottom_freq_tmp > 2200000)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ boost_policy_params.bottom_freq = bottom_freq_tmp; -+ boost_policy_params.boost_weight = boost_weight_tmp; -+ boost_policy_params.enable = enable_tmp; -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(boost_policy, boost_policy_open, boost_policy_write); -+ -+/* tick_hit ops begin */ -+static int tick_hit_show(struct seq_file *m, void *v) -+{ -+ struct tick_hit_params *data = m->private; -+ -+ seq_printf(m, "tick_hit:{\"enable\":%d; \"hit_count_thres\":%d; \"jiffies_num\":%lu}\n", -+ data->enable, data->hit_count_thres, data->jiffies_num); -+ return 0; -+} -+ -+static int tick_hit_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, tick_hit_show, pde_data(inode)); -+} -+ -+static ssize_t tick_hit_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, hit_count_thres_tmp, jiffies_num_tmp; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if(copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &hit_count_thres_tmp, &jiffies_num_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ tick_hit_params.hit_count_thres = hit_count_thres_tmp; -+ tick_hit_params.jiffies_num = jiffies_num_tmp; -+ tick_hit_params.enable = enable_tmp; -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(tick_hit, tick_hit_open, tick_hit_write); -+ -+static int __init hmbird_proc_init(void) -+{ -+ struct proc_dir_entry *hmbird_dir; -+ struct proc_dir_entry *load_track_dir; -+ struct proc_dir_entry *freq_gov_dir; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ -+ /* mkdir /proc/hmbird_sched */ -+ hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL); -+ if (!hmbird_dir) { -+ pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &scx_enable_proc_ops, -+ &scx_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &partial_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_stats); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbirdcore_debug); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_for_app); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &misfit_ds); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl_dbg); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpu7_tl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &cpu_cluster_masks_proc_ops, -+ &cpu_cluster_masks); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &save_gov_proc_ops, -+ &save_gov); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &watchdog_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isolate_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &iso_free_rescue); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_stats_proc_ops); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &yield_opt_proc_ops, -+ &yield_opt_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("boost_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &boost_policy_proc_ops, -+ &boost_policy_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("tick_hit", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &tick_hit_proc_ops, -+ &tick_hit_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbird_preempt_policy); -+ /* /proc/hmbird_sched--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_walt */ -+ load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir); -+ if (!load_track_dir) { -+ pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_walt--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &slim_walt_ctrl_proc_ops, -+ &slim_walt_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_dump); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_policy); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &sched_ravg_window_frame_per_sec_proc_ops, -+ &sched_ravg_window_frame_per_sec); -+ /* /proc/hmbird_sched/slim_walt--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_freq_gov */ -+ freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir); -+ if (!freq_gov_dir) { -+ pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_freq_gov--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &slim_gov_debug); -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &scx_gov_ctrl); -+ /* /proc/hmbird_sched/slim_freq_gov--end */ -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cluster_separate); -+ -+ return 0; -+} -+ -+device_initcall(hmbird_proc_init); -diff --git b/kernel/sched/hmbird/hmbird_sched_proc.h a/kernel/sched/hmbird/hmbird_sched_proc.h -new file mode 100755 -index 000000000000..e22bc75f1dd7 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched_proc.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SCHED_PROC_H__ -+#define __HMBIRD_SCHED_PROC_H__ -+ -+#include -+#include -+ -+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \ -+ do { \ -+ if (!proc_create(name, mode, parent, proc_ops)) { \ -+ pr_err("Error creating proc entry %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \ -+ do { \ -+ if (!proc_create_data(name, mode, parent, proc_ops, data)) { \ -+ pr_err("Error creating proc entry with data %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_PROC_OPS(name, open_func, write_func) \ -+ static const struct proc_ops name##_proc_ops = { \ -+ .proc_open = open_func, \ -+ .proc_write = write_func, \ -+ .proc_read = seq_read, \ -+ .proc_lseek = seq_lseek, \ -+ .proc_release = single_release, \ -+ } -+ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos); -+static int hmbird_common_show(struct seq_file *m, void *v); -+static int hmbird_common_open(struct inode *inode, struct file *file); -+ -+#endif -diff --git b/kernel/sched/hmbird/hmbird_shadow_tick.c a/kernel/sched/hmbird/hmbird_shadow_tick.c -new file mode 100755 -index 000000000000..2744cd42bbfd ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_shadow_tick.c -@@ -0,0 +1,198 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include -+#include <../../kernel/time/tick-sched.h> -+#include -+ -+#include "hmbird_shadow_tick.h" -+#include "slim.h" -+ -+#define HIGHRES_WATCH_CPU 0 -+ -+#include -+static bool shadow_tick_enable(void) {return highres_tick_ctrl; } -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; } -+#endif -+static bool shadow_tick_timer_init_flag; -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define shadow_tick_printk(fmt, args...) \ -+do { \ -+ int cpu = smp_processor_id(); \ -+ if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU) \ -+ trace_printk("hmbird shadow tick :"fmt, args); \ -+} while (0) -+#else -+#define shadow_tick_printk(fmt, args...) -+#endif -+ -+DEFINE_PER_CPU(struct hrtimer, stt); -+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu))) -+#define STOP_IDLE_TRIGGER (1) -+#define PERIODIC_TICK_TRIGGER (2) -+#define TICK_INTVAL (1000000ULL) -+#define HMBIRD_TICK_HIT_BOOST BIT(29) -+/* -+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms, -+ * so we can't restart hrtimer in scheduler tick. -+ */ -+static DEFINE_PER_CPU(u8, trigger_event); -+ -+/* -+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick. -+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler. -+ */ -+static DEFINE_PER_CPU(u8, tick_phase); -+ -+static inline void highres_timer_ctrl(bool enable, int cpu) -+{ -+ if (enable && hmbird_enabled()) { -+ if (!hrtimer_active(shadow_tick_timer(cpu))) -+ hrtimer_start(shadow_tick_timer(cpu), -+ ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED); -+ } else { -+ if (!enable) -+ hrtimer_cancel(shadow_tick_timer(cpu)); -+ } -+} -+ -+static inline void high_res_clear_phase(int cpu) -+{ -+ per_cpu(tick_phase, cpu) = 0; -+} -+ -+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer) -+{ -+ per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3; -+ if (per_cpu(tick_phase, cpu)) { -+ hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) { -+ per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+struct tick_hit_params tick_hit_params = { -+ .enable = 0, -+ .jiffies_num = 2, -+ .hit_count_thres = 6, -+}; -+ -+static void tick_hit_critical_task(struct task_struct *curr, struct rq *rq) -+{ -+ struct hmbird_entity *see = get_hmbird_ts(curr); -+ if (!tick_hit_params.enable || !see) -+ return; -+ -+ if ((!task_is_top_task(curr) || curr->pid != curr->tgid) && (curr->pid != scx_systemui_pid)) -+ return; -+ -+ if (see->tick_hit_count == 0) { -+ if (see->start_jiffies == 0) { -+ see->start_jiffies = jiffies; -+ see->tick_hit_count = 1; -+ } else { -+ see->start_jiffies = 0; /* status reset */ -+ see->tick_hit_count = 0; -+ } -+ } else { -+ if (time_before_eq(jiffies, see->start_jiffies + tick_hit_params.jiffies_num)) { -+ see->tick_hit_count++; -+ if (see->tick_hit_count >= tick_hit_params.hit_count_thres) { -+ cpufreq_update_util(rq, HMBIRD_TICK_HIT_BOOST); -+ } -+ } else { -+ see->tick_hit_count = 1; -+ see->start_jiffies = jiffies; -+ } -+ } -+} -+ -+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ struct rq_flags rf; -+ -+ rq_lock(rq, &rf); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ tick_hit_critical_task(curr, rq); -+ rq_unlock(rq, &rf); -+ -+ return highres_next_phase(cpu, timer); -+} -+ -+void shadow_tick_timer_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance; -+ } -+} -+ -+void start_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable()) { -+ if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER) -+ highres_timer_ctrl(false, cpu); -+ per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static void stop_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ per_cpu(trigger_event, cpu) = 0; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(false, cpu); -+} -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ stop_shadow_tick_timer(); -+} -+ -+void scheduler_tick_handler(void *unused, struct rq *rq) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ start_shadow_tick_timer(); -+} -+ -+static int __init hmbird_shadow_tick_init(void) -+{ -+ int ret = 0; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ shadow_tick_timer_init(); -+ shadow_tick_timer_init_flag = true; -+ return ret; -+} -+ -+device_initcall(hmbird_shadow_tick_init); -diff --git b/kernel/sched/hmbird/hmbird_shadow_tick.h a/kernel/sched/hmbird/hmbird_shadow_tick.h -new file mode 100755 -index 000000000000..0c26fd984f0a ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_shadow_tick.h -@@ -0,0 +1,11 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SHADOW_TICK_H__ -+#define __HMBIRD_SHADOW_TICK_H__ -+ -+#include -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+void scheduler_tick_handler(void *unused, struct rq *rq); -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state); -+#endif -diff --git b/kernel/sched/hmbird/hmbird_trace.h a/kernel/sched/hmbird/hmbird_trace.h -new file mode 100755 -index 000000000000..8468b406f347 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_trace.h -@@ -0,0 +1,92 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hmbird -+ -+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HMBIRD_H -+ -+#include -+#include -+#include -+ -+#define MAX_FATAL_INFO (64) -+ -+TRACE_EVENT(hmbird_fatal_info, -+ -+ TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info), -+ -+ TP_ARGS(type, pe, lnr, bnr, info), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, type) -+ __field(int, pe) -+ __field(int, lnr) -+ __field(int, bnr) -+ __array(char, info, MAX_FATAL_INFO)), -+ -+ TP_fast_assign( -+ __entry->type = type; -+ __entry->pe = pe; -+ __entry->lnr = lnr; -+ __entry->bnr = bnr; -+ memcpy(__entry->info, info, MAX_FATAL_INFO);), -+ -+ TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s", -+ __entry->type, __entry->pe, __entry->lnr, __entry->bnr, -+ __entry->info) -+); -+ -+TRACE_EVENT(hmbird_update_history, -+ -+ TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq, -+ struct task_struct *p, u32 runtime, int samples, int event), -+ -+ TP_ARGS(hmbird, rq, p, runtime, samples, event), -+ -+ TP_STRUCT__entry( -+ __array(char, comm, TASK_COMM_LEN) -+ __field(pid_t, pid) -+ __field(unsigned int, runtime) -+ __field(int, samples) -+ __field(int, event) -+ __field(unsigned int, demand) -+ __array(u32, hist, RAVG_HIST_SIZE) -+ __field(u16, task_util) -+ __field(int, cpu)), -+ -+ TP_fast_assign( -+ memcpy(__entry->comm, p->comm, TASK_COMM_LEN); -+ __entry->pid = p->pid; -+ __entry->runtime = runtime; -+ __entry->samples = samples; -+ __entry->event = event; -+ __entry->demand = hmbird->sts.demand; -+ memcpy(__entry->hist, hmbird->sts.sum_history, -+ RAVG_HIST_SIZE * sizeof(u32)); -+ __entry->task_util = hmbird->sts.demand_scaled; -+ __entry->cpu = rq->cpu;), -+ -+ TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d", -+ __entry->comm, __entry->pid, -+ __entry->runtime, __entry->samples, -+ __entry->event, -+ __entry->demand, -+ __entry->hist[0], __entry->hist[1], -+ __entry->hist[2], __entry->hist[3], -+ __entry->hist[4], -+ __entry->task_util, -+ __entry->cpu) -+); -+ -+#endif /*_TRACE_HMBIRD_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird -+ -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE hmbird_trace -+/* This part must be outside protection */ -+#include -diff --git b/kernel/sched/hmbird/hmbird_util_track.c a/kernel/sched/hmbird/hmbird_util_track.c -new file mode 100755 -index 000000000000..d520a8403401 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_util_track.c -@@ -0,0 +1,626 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+ -+#define CREATE_TRACE_POINTS -+#include "hmbird_trace.h" -+#undef CREATE_TRACE_POINTS -+ -+#define HMBIRD_DEBUG_PANIC (1 << 3) -+static bool init_irq_work_inited; -+ -+extern noinline int tracing_mark_write(const char *buf); -+ -+int hmbird_sched_ravg_window = 8000000; -+int new_hmbird_sched_ravg_window = 8000000; -+DEFINE_SPINLOCK(new_sched_ravg_window_lock); -+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+ -+static struct irq_work hmbird_slim_walt_irq_work; -+ -+/*Sysctl related interface*/ -+#define WINDOW_STATS_RECENT 0 -+#define WINDOW_STATS_MAX 1 -+#define WINDOW_STATS_MAX_RECENT_AVG 2 -+#define WINDOW_STATS_AVG 3 -+#define WINDOW_STATS_INVALID_POLICY 4 -+ -+ -+#define SCX_SCHED_CAPACITY_SHIFT 10 -+#define SCHED_ACCOUNT_WAIT_TIME 0 -+ -+atomic64_t hmbird_irq_work_lastq_ws; -+static u64 tick_sched_clock; -+ -+static inline u64 scale_exec_time(u64 delta, struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT; -+} -+ -+static inline u64 scale_time_to_util(u64 d) -+{ -+ do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT); -+ return d; -+} -+ -+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ delta = scale_exec_time(delta, rq); -+ sts->sum += delta; -+ if (unlikely(sts->sum > hmbird_sched_ravg_window)) -+ sts->sum = hmbird_sched_ravg_window; -+ -+ return delta; -+} -+ -+ -+static int -+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) -+{ -+ /* -+ * No need to bother updating task demand for the idle task. -+ */ -+ if (is_idle_task(p)) -+ return 0; -+ -+ /* -+ * When a task is waking up it is completing a segment of non-busy -+ * time. Likewise, if wait time is not treated as busy time, then -+ * when a task begins to run or is migrated, it is not running and -+ * is completing a segment of non-busy time. -+ */ -+ if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && -+ (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) -+ return 0; -+ -+ /* -+ * The idle exit time is not accounted for the first task _picked_ up to -+ * run on the idle CPU. -+ */ -+ if (event == PICK_NEXT_TASK && rq->curr == rq->idle) -+ return 0; -+ -+ /* -+ * TASK_UPDATE can be called on sleeping task, when its moved between -+ * related groups -+ */ -+ if (event == TASK_UPDATE) { -+ if (rq->curr == p) -+ return 1; -+ -+ return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; -+ } -+ -+ return 1; -+} -+ -+static void rollover_cpu_window(struct rq *rq, bool full_window) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 curr_sum = srq->curr_runnable_sum; -+ -+ if (unlikely(full_window)) -+ curr_sum = 0; -+ -+ srq->prev_runnable_sum = curr_sum; -+ srq->curr_runnable_sum = 0; -+} -+ -+static u64 -+update_window_start(struct rq *rq, u64 wallclock, int event) -+{ -+ s64 delta; -+ int nr_windows; -+ bool full_window; -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start = srq->window_start; -+ -+ if (wallclock < srq->latest_clock) -+ wallclock = srq->latest_clock; -+ delta = wallclock - srq->window_start; -+ if (delta < 0) { -+ delta = 0; -+ wallclock = srq->window_start; -+ } -+ srq->latest_clock = wallclock; -+ if (delta < hmbird_sched_ravg_window) -+ return old_window_start; -+ -+ nr_windows = div64_u64(delta, hmbird_sched_ravg_window); -+ srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window; -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ full_window = nr_windows > 1; -+ rollover_cpu_window(rq, full_window); -+ -+ return old_window_start; -+} -+ -+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -+ -+static inline unsigned int get_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static inline unsigned int cpu_cur_freq(int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cur; -+} -+ -+static void -+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, -+ u64 wallclock) -+{ -+ int cpu = cpu_of(rq); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * -+ arch_scale_cpu_capacity(cpu), get_max_freq(cpu)); -+} -+ -+/* -+ * Called when new window is starting for a task, to record cpu usage over -+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 -+ * when, say, a real-time task runs without preemption for several windows at a -+ * stretch. -+ */ -+static void update_history(struct rq *rq, struct task_struct *p, -+ u32 runtime, int samples, int event) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u32 *hist = &sts->sum_history[0]; -+ int i; -+ u32 max = 0, avg, demand; -+ u64 sum = 0; -+ u16 demand_scaled; -+ -+ /* Ignore windows where task had no activity */ -+ if (!runtime || is_idle_task(p) || !samples) -+ goto done; -+ -+ /* Push new 'runtime' value onto stack */ -+ for (; samples > 0; samples--) { -+ hist[sts->cidx] = runtime; -+ sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE; -+ } -+ -+ for (i = 0; i < RAVG_HIST_SIZE; i++) { -+ sum += hist[i]; -+ if (hist[i] > max) -+ max = hist[i]; -+ } -+ -+ sts->sum = 0; -+ avg = div64_u64(sum, RAVG_HIST_SIZE); -+ -+ switch (slim_walt_policy) { -+ case WINDOW_STATS_RECENT: -+ demand = runtime; -+ break; -+ case WINDOW_STATS_MAX: -+ demand = max; -+ break; -+ case WINDOW_STATS_AVG: -+ demand = avg; -+ break; -+ default: -+ demand = max(avg, runtime); -+ } -+ -+ demand_scaled = scale_time_to_util(demand); -+ -+ sts->demand = demand; -+ sts->demand_scaled = demand_scaled; -+ -+done: -+ return; -+} -+ -+ -+static u64 update_task_demand(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ -+ u64 delta, window_start = srq->window_start; -+ int new_window, nr_full_windows; -+ u32 window_size = hmbird_sched_ravg_window; -+ u64 runtime; -+ -+ new_window = mark_start < window_start; -+ if (!account_busy_for_task_demand(rq, p, event)) { -+ if (new_window) -+ /* -+ * If the time accounted isn't being accounted as -+ * busy time, and a new window started, only the -+ * previous window need be closed out with the -+ * pre-existing demand. Multiple windows may have -+ * elapsed, but since empty windows are dropped, -+ * it is not necessary to account those. -+ */ -+ update_history(rq, p, sts->sum, 1, event); -+ return 0; -+ } -+ -+ if (!new_window) { -+ /* -+ * The simple case - busy time contained within the existing -+ * window. -+ */ -+ return add_to_task_demand(rq, p, wallclock - mark_start); -+ } -+ -+ /* -+ * Busy time spans at least two windows. Temporarily rewind -+ * window_start to first window boundary after mark_start. -+ */ -+ delta = window_start - mark_start; -+ nr_full_windows = div64_u64(delta, window_size); -+ window_start -= (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (window_start - mark_start) first */ -+ runtime = add_to_task_demand(rq, p, window_start - mark_start); -+ -+ /* Push new sample(s) into task's demand history */ -+ update_history(rq, p, sts->sum, 1, event); -+ if (nr_full_windows) { -+ u64 scaled_window = scale_exec_time(window_size, rq); -+ -+ update_history(rq, p, scaled_window, nr_full_windows, event); -+ runtime += nr_full_windows * scaled_window; -+ } -+ -+ /* -+ * Roll window_start back to current to process any remainder -+ * in current window. -+ */ -+ window_start += (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (wallclock - window_start) next */ -+ mark_start = window_start; -+ runtime += add_to_task_demand(rq, p, wallclock - mark_start); -+ -+ return runtime; -+} -+ -+u16 slim_walt_cpu_util(int cpu) -+{ -+ u64 prev_runnable_sum; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ prev_runnable_sum = srq->prev_runnable_sum; -+ do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT); -+ -+ return (u16)prev_runnable_sum; -+} -+ -+static DEFINE_PER_CPU(u16, prev_cpu_util); -+static inline void cpu_util_update_systrace_c(int cpu) -+{ -+ char buf[256]; -+ u16 cpu_util = slim_walt_cpu_util(cpu); -+ -+ if (cpu_util != per_cpu(prev_cpu_util, cpu)) { -+ snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n", -+ cpu, cpu_util); -+ tracing_mark_write(buf); -+ per_cpu(prev_cpu_util, cpu) = cpu_util; -+ } -+} -+ -+ -+static inline int account_busy_for_cpu_time(struct rq *rq, -+ struct task_struct *p, -+ int event) -+{ -+ return !is_idle_task(p) && (event == PUT_PREV_TASK -+ || event == TASK_UPDATE); -+} -+ -+ -+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ int new_window, full_window = 0; -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 window_start = srq->window_start; -+ u32 window_size = srq->prev_window_size; -+ u64 delta; -+ u64 *curr_runnable_sum = &srq->curr_runnable_sum; -+ u64 *prev_runnable_sum = &srq->prev_runnable_sum; -+ -+ new_window = mark_start < window_start; -+ if (new_window) -+ full_window = (window_start - mark_start) >= window_size; -+ -+ -+ if (!account_busy_for_cpu_time(rq, p, event)) -+ goto done; -+ -+ -+ if (!new_window) { -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. No rollover -+ * since we didn't start a new window. An example of this is -+ * when a task starts execution and then sleeps within the -+ * same window. -+ */ -+ delta = wallclock - mark_start; -+ -+ delta = scale_exec_time(delta, rq); -+ *curr_runnable_sum += delta; -+ -+ goto done; -+ } -+ -+ /* -+ * situations below this need window rollover, -+ * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done -+ * in update_window_start() -+ * -+ * For task counters curr/prev_window[_cpu] are rolled over in the early part of -+ * this function. If full_window(s) have expired and time since last update needs -+ * to be accounted as busy time, set the prev to a complete window size time, else -+ * add the prev window portion. -+ * -+ * For task curr counters a new window has begun, always assign -+ */ -+ -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. A new window -+ * must have been started in udpate_window_start() -+ * If any of these three above conditions are true -+ * then this busy time can't be accounted as irqtime. -+ * -+ * Busy time for the idle task need not be accounted. -+ * -+ * An example of this would be a task that starts execution -+ * and then sleeps once a new window has begun. -+ */ -+ -+ /* -+ * A full window hasn't elapsed, account partial -+ * contribution to previous completed window. -+ */ -+ -+ -+ delta = full_window ? scale_exec_time(window_size, rq) : -+ scale_exec_time(window_start - mark_start, rq); -+ -+ *prev_runnable_sum += delta; -+ -+ /* Account piece of busy time in the current window. */ -+ delta = scale_exec_time(wallclock - window_start, rq); -+ *curr_runnable_sum += delta; -+ -+done: -+ if (slim_walt_dump && new_window) -+ cpu_util_update_systrace_c(rq->cpu); -+} -+ -+ -+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq) -+{ -+ u64 result; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 new_window_start = srq->window_start; -+ -+ if (old_window_start == new_window_start) -+ return; -+ -+ result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws, -+ old_window_start, new_window_start); -+ if (result != old_window_start) -+ return; -+ -+ if (likely(cpu_online(raw_smp_processor_id()))) -+ irq_work_queue(&hmbird_slim_walt_irq_work); -+ else -+ irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask)); -+} -+ -+/* -+ * In the core scheduler, most of the load update points update the rq_clock after -+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of -+ * obtaining the time, but to prevent the subsequent migration of the load update -+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock -+ * update here. -+ */ -+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p, -+ struct rq *rq, int event) -+{ -+ if (!(rq->clock_update_flags & RQCF_UPDATED)) -+ update_rq_clock(rq); -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock)); -+} -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start; -+ -+ if (!slim_walt_ctrl) -+ return; -+ -+ if (!srq->window_start || sts->mark_start == wallclock) -+ return; -+ -+ old_window_start = update_window_start(rq, wallclock, event); -+ -+ if (!sts->window_start) -+ sts->window_start = srq->window_start; -+ -+ if (!sts->mark_start) -+ goto done; -+ -+ update_task_rq_cpu_cycles(p, rq, event, wallclock); -+ update_task_demand(p, rq, event, wallclock); -+ update_cpu_busy_time(p, rq, event, wallclock); -+ -+ sts->window_start = srq->window_start; -+ -+done: -+ sts->mark_start = wallclock; -+ slim_walt_window_rollover_run_once(old_window_start, rq); -+} -+ -+static void slim_walt_irq_work(struct irq_work *irq_work) -+{ -+ cpumask_t lock_cpus; -+ struct hmbird_sched_rq_stats *srq; -+ struct rq *rq; -+ int cpu; -+ int level = 0; -+ u64 wc; -+ unsigned long flags; -+ -+ cpumask_copy(&lock_cpus, cpu_possible_mask); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ if (level == 0) -+ raw_spin_lock(&cpu_rq(cpu)->__lock); -+ else -+ raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); -+ level++; -+ } -+ -+ wc = sched_clock(); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ rq = cpu_rq(cpu); -+ hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc); -+ } -+ -+ cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER); -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) { -+ srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id()); -+ if (wc < srq->window_start + new_hmbird_sched_ravg_window) -+ hmbird_sched_ravg_window = new_hmbird_sched_ravg_window; -+ } -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ raw_spin_unlock(&cpu_rq(cpu)->__lock); -+ } -+} -+static void hmbird_sched_init_rq(struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ srq->task_exec_scale = 1024; -+ srq->window_start = 0; -+} -+ -+void hmbird_sched_init_task(struct task_struct *p) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ memset(sts, 0, sizeof(struct hmbird_sched_task_stats)); -+} -+ -+static void hmbird_sched_stats_init(void) -+{ -+ unsigned long flags; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ raw_spin_lock_irqsave(&rq->__lock, flags); -+ hmbird_sched_init_rq(rq); -+ raw_spin_unlock_irqrestore(&rq->__lock, flags); -+ } -+ slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG; -+ -+ if (false == init_irq_work_inited) { -+ init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work); -+ init_irq_work_inited = true; -+ } -+} -+ -+void hmbird_scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (unlikely(!tick_sched_clock)) { -+ /* -+ * Let the window begin 20us prior to the tick, -+ * that way we are guaranteed a rollover when the tick occurs. -+ * Use rq->clock directly instead of rq_clock() since -+ * we do not have the rq lock and -+ * rq->clock was updated in the tick callpath. -+ */ -+ if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000)) -+ return; -+ for_each_possible_cpu(cpu) { -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ srq->window_start = tick_sched_clock; -+ } -+ atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock); -+ } -+} -+ -+void slim_walt_enable(int enable) -+{ -+ if (1 == !!enable) { -+ hmbird_sched_stats_init(); -+ WRITE_ONCE(tick_sched_clock, 0); -+ } else -+ slim_walt_ctrl = 0; -+} -+ -+void slim_get_cpu_util(int cpu, u64 *util) -+{ -+ if (cpu < 0) -+ return; -+ -+ *util = slim_walt_cpu_util(cpu); -+} -+ -+void slim_get_task_util(struct task_struct *p, u64 *util) -+{ -+ if (p == NULL) -+ return; -+ -+ *util = get_hmbird_ts(p)->sts.demand_scaled; -+} -+ -+void sched_ravg_window_change(int frame_per_sec) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec; -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+} -diff --git b/kernel/sched/hmbird/hmbird_util_track.h a/kernel/sched/hmbird/hmbird_util_track.h -new file mode 100644 -index 000000000000..17b85df7f83b ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_util_track.h -@@ -0,0 +1,33 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef __HMBIRD_UTIL_TRACK_H__ -+#define __HMBIRD_UTIL_TRACK_H__ -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock); -+void hmbird_sched_init_task(struct task_struct *p); -+void slim_walt_enable(int enable); -+void slim_get_cpu_util(int cpu, u64 *util); -+void slim_get_task_util(struct task_struct *p, u64 *util); -+ -+extern atomic64_t hmbird_irq_work_lastq_ws; -+ -+enum task_event { -+ PUT_PREV_TASK = 0, -+ PICK_NEXT_TASK = 1, -+ TASK_WAKE = 2, -+ TASK_MIGRATE = 3, -+ TASK_UPDATE = 4, -+ IRQ_UPDATE = 5, -+}; -+ -+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+#endif /* __HMBIRD_UTIL_TRACK_H__ */ -diff --git b/kernel/sched/hmbird/slim.h a/kernel/sched/hmbird/slim.h -new file mode 100755 -index 000000000000..dffe6506658e ---- /dev/null -+++ a/kernel/sched/hmbird/slim.h -@@ -0,0 +1,56 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+ -+#ifndef __SLIM_H -+#define __SLIM_H -+ -+extern atomic_t __hmbird_ops_enabled; -+extern atomic_t non_hmbird_task; -+extern int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+extern int heartbeat; -+extern int heartbeat_enable; -+extern int watchdog_enable; -+extern int isolate_ctrl; -+extern int parctrl_high_ratio; -+extern int parctrl_low_ratio; -+extern int isoctrl_high_ratio; -+extern int isoctrl_low_ratio; -+extern int iso_free_rescue; -+extern int yield_opt; -+ -+extern enum hmbird_switch_type sw_type; -+extern noinline int tracing_mark_write(const char *buf); -+int task_top_id(struct task_struct *p); -+void stats_print(char *buf, int len); -+void hmbird_skip_yield(long *skip); -+extern spinlock_t hmbird_tasks_lock; -+extern int scx_systemui_pid; -+ -+struct yield_opt_params { -+ int enable; -+ int frame_per_sec; -+ u64 frame_time_ns; -+ int yield_headroom; -+}; -+ -+extern struct yield_opt_params yield_opt_params; -+ -+struct tick_hit_params { -+ int enable; -+ unsigned long jiffies_num; -+ int hit_count_thres; -+}; -+ -+extern struct tick_hit_params tick_hit_params; -+ -+struct boost_policy_params { -+ int enable; -+ unsigned int bottom_freq; -+ int boost_weight; -+}; -+ -+extern struct boost_policy_params boost_policy_params; -+ -+#define MAX_GOV_LEN (16) -+extern char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+#endif -diff --git b/kernel/sched/idle.c a/kernel/sched/idle.c -index b33cefeb4188..487255ac6832 100755 ---- b/kernel/sched/idle.c -+++ a/kernel/sched/idle.c -@@ -408,13 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl - - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) - { -- scx_update_idle(rq, false); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, false); -+#endif - } - - static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) - { - update_idle_core(rq); -- scx_update_idle(rq, true); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, true); -+#endif - schedstat_inc(rq->sched_goidle); - } - -diff --git b/kernel/sched/sched.h a/kernel/sched/sched.h -index 509e8dc616ab..a7e9caea1efe 100755 ---- b/kernel/sched/sched.h -+++ a/kernel/sched/sched.h -@@ -188,18 +188,13 @@ static inline int idle_policy(int policy) - return policy == SCHED_IDLE; - } - --static inline int normal_policy(int policy) --{ --#ifdef CONFIG_SCHED_CLASS_EXT -- if (policy == SCHED_EXT) -- return true; --#endif -- return policy == SCHED_NORMAL; --} -- - static inline int fair_policy(int policy) - { -- return normal_policy(policy) || policy == SCHED_BATCH; -+#ifdef CONFIG_HMBIRD_SCHED -+ return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#else -+ return policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#endif - } - - static inline int rt_policy(int policy) -@@ -247,25 +242,7 @@ static inline void update_avg(u64 *avg, u64 sample) - #define shr_bound(val, shift) \ - (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) - --/* -- * cgroup weight knobs should use the common MIN, DFL and MAX values which are -- * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it -- * maps pretty well onto the shares value used by scheduler and the round-trip -- * conversions preserve the original value over the entire range. -- */ --static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) --{ -- return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); --} -- --static inline unsigned long sched_weight_to_cgroup(unsigned long weight) --{ -- return clamp_t(unsigned long, -- DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -- CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); --} -- --/* -+/* - * !! For sched_setattr_nocheck() (kernel) only !! - * - * This is actually gross. :( -@@ -532,10 +509,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) { } - #endif /* CONFIG_SMP */ - #else /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_HMBIRD_SCHED - static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) - { - return 0; - } -+#endif - #endif /* CONFIG_FAIR_GROUP_SCHED */ - - #else /* CONFIG_CGROUP_SCHED */ -@@ -700,29 +679,6 @@ struct cfs_rq { - #endif /* CONFIG_FAIR_GROUP_SCHED */ - }; - --#ifdef CONFIG_SCHED_CLASS_EXT --/* scx_rq->flags, protected by the rq lock */ --enum scx_rq_flags { -- SCX_RQ_CAN_STOP_TICK = 1 << 0, --}; -- --struct scx_rq { -- struct scx_dispatch_q local_dsq; -- struct list_head watchdog_list; -- u64 ops_qseq; -- u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -- u32 nr_running; -- u32 flags; -- bool cpu_released; -- cpumask_var_t cpus_to_kick; -- cpumask_var_t cpus_to_preempt; -- cpumask_var_t cpus_to_wait; -- u64 pnt_seq; -- struct irq_work kick_cpus_irq_work; -- struct rq *rq; --}; --#endif /* CONFIG_SCHED_CLASS_EXT */ -- - static inline int rt_bandwidth_enabled(void) - { - return sysctl_sched_rt_runtime >= 0; -@@ -1258,11 +1214,7 @@ struct rq { - - ANDROID_OEM_DATA_ARRAY(1, 16); - --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, struct scx_rq *scx); --#else - ANDROID_KABI_RESERVE(1); --#endif - ANDROID_KABI_RESERVE(2); - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); -@@ -2340,11 +2292,6 @@ struct affinity_context { - unsigned int flags; - }; - --enum rq_onoff_reason { -- RQ_ONOFF_HOTPLUG, /* CPU is going on/offline */ -- RQ_ONOFF_TOPOLOGY, /* sched domain topology update */ --}; -- - struct sched_class { - - #ifdef CONFIG_UCLAMP_TASK -@@ -2551,7 +2498,6 @@ extern void init_sched_rt_class(void); - extern void init_sched_fair_class(void); - - extern void reweight_task(struct task_struct *p, int prio); --extern void __setscheduler_prio(struct task_struct *p, int prio); - - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); -@@ -2632,53 +2578,6 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) - extern void activate_task(struct rq *rq, struct task_struct *p, int flags); - extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); - --struct sched_change_guard { -- struct task_struct *p; -- struct rq *rq; -- bool queued; -- bool running; -- bool done; --}; -- --extern struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -- --extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags); -- --/** -- * SCHED_CHANGE_BLOCK - Nested block for task attribute updates -- * @__rq: Runqueue the target task belongs to -- * @__p: Target task -- * @__flags: DEQUEUE/ENQUEUE_* flags -- * -- * A task may need to be dequeued and put_prev_task'd for attribute updates and -- * set_next_task'd and re-enqueued afterwards. This helper defines a nested -- * block which automatically handles these preparation and cleanup operations. -- * -- * SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- * update_attribute(p); -- * ... -- * } -- * -- * If @__flags is a variable, the variable may be updated in the block body and -- * the updated value will be used when re-enqueueing @p. -- * -- * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling -- * update_rq_clock() beforehand. Otherwise, the rq clock is automatically -- * updated iff the task needs to be dequeued and re-enqueued. Only the former -- * case guarantees that the rq clock is up-to-date inside and after the block. -- */ --#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -- for (struct sched_change_guard __cg = \ -- sched_change_guard_init(__rq, __p, __flags); \ -- !__cg.done; sched_change_guard_fini(&__cg, __flags)) -- --extern void check_class_changing(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class); --extern void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio); -- - extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); - - #ifdef CONFIG_PREEMPT_RT -@@ -3710,6 +3609,8 @@ static inline bool cpu_busy_with_softirqs(int cpu) - } - #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ - --#include "ext.h" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird.h" -+#endif - - #endif /* _KERNEL_SCHED_SCHED_H */ -diff --git b/kernel/time/tick-sched.c a/kernel/time/tick-sched.c -index 998c2eefe173..c13772ee823c 100755 ---- b/kernel/time/tick-sched.c -+++ a/kernel/time/tick-sched.c -@@ -34,6 +34,10 @@ - - #include - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "../sched/hmbird/hmbird_shadow_tick.h" -+#endif -+ - /* - * Per-CPU nohz control structure - */ -@@ -1124,6 +1128,9 @@ void tick_nohz_idle_stop_tick(void) - ktime_t expires; - - trace_android_vh_tick_nohz_idle_stop_tick(NULL); -+#ifdef CONFIG_HMBIRD_SCHED -+ android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL); -+#endif - /* - * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the - * tick timer expiration time is known already. -diff --git b/tools/sched_ext/.gitignore a/tools/sched_ext/.gitignore -deleted file mode 100644 -index a3240f9f7eba..000000000000 ---- b/tools/sched_ext/.gitignore -+++ /dev/null -@@ -1,9 +0,0 @@ --scx_example_simple --scx_example_qmap --scx_example_central --scx_example_pair --scx_example_flatcg --scx_example_userland --*.skel.h --*.subskel.h --/tools/ -diff --git b/tools/sched_ext/Makefile a/tools/sched_ext/Makefile -deleted file mode 100644 -index 73c43782837d..000000000000 ---- b/tools/sched_ext/Makefile -+++ /dev/null -@@ -1,216 +0,0 @@ --# SPDX-License-Identifier: GPL-2.0 --# Copyright (c) 2022 Meta Platforms, Inc. and affiliates. --include ../build/Build.include --include ../scripts/Makefile.arch --include ../scripts/Makefile.include -- --ifneq ($(LLVM),) --ifneq ($(filter %/,$(LLVM)),) --LLVM_PREFIX := $(LLVM) --else ifneq ($(filter -%,$(LLVM)),) --LLVM_SUFFIX := $(LLVM) --endif -- --CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi --CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu --CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl --CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu --CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu --CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu --CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu --CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu --CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu --CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) -- --ifeq ($(CROSS_COMPILE),) --ifeq ($(CLANG_TARGET_FLAGS),) --$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk --else --CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) --endif # CLANG_TARGET_FLAGS --else --CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) --endif # CROSS_COMPILE -- --CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as --else --CC := $(CROSS_COMPILE)gcc --endif # LLVM -- --CURDIR := $(abspath .) --TOOLSDIR := $(abspath ..) --LIBDIR := $(TOOLSDIR)/lib --BPFDIR := $(LIBDIR)/bpf --TOOLSINCDIR := $(TOOLSDIR)/include --BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool --APIDIR := $(TOOLSINCDIR)/uapi --GENDIR := $(abspath ../../include/generated) --GENHDR := $(GENDIR)/autoconf.h -- --SCRATCH_DIR := $(CURDIR)/tools --BUILD_DIR := $(SCRATCH_DIR)/build --INCLUDE_DIR := $(SCRATCH_DIR)/include --BPFOBJ_DIR := $(BUILD_DIR)/libbpf --BPFOBJ := $(BPFOBJ_DIR)/libbpf.a --ifneq ($(CROSS_COMPILE),) --HOST_BUILD_DIR := $(BUILD_DIR)/host --HOST_SCRATCH_DIR := host-tools --HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include --else --HOST_BUILD_DIR := $(BUILD_DIR) --HOST_SCRATCH_DIR := $(SCRATCH_DIR) --HOST_INCLUDE_DIR := $(INCLUDE_DIR) --endif --HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a --RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids --DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -- --VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ -- $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ -- ../../vmlinux \ -- /sys/kernel/btf/vmlinux \ -- /boot/vmlinux-$(shell uname -r) --VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) --ifeq ($(VMLINUX_BTF),) --$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") --endif -- --BPFTOOL ?= $(DEFAULT_BPFTOOL) -- --ifneq ($(wildcard $(GENHDR)),) -- GENFLAGS := -DHAVE_GENHDR --endif -- --CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ -- -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -- -I$(TOOLSINCDIR) -I$(APIDIR) -- --CARGOFLAGS := --release -- --# Silence some warnings when compiled with clang --ifneq ($(LLVM),) --CFLAGS += -Wno-unused-command-line-argument --endif -- --LDFLAGS = -lelf -lz -lpthread -- --IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - &1 \ -- | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ --$(shell $(1) -dM -E - $@ --else -- $(call msg,CP,,$@) -- $(Q)cp "$(VMLINUX_H)" $@ --endif -- --%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \ -- | $(BPFOBJ) -- $(call msg,CLNG-BPF,,$@) -- $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ -- --%.skel.h: %.bpf.o $(BPFTOOL) -- $(call msg,GEN-SKEL,,$@) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) -- $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) -- $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@ -- $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h) -- --scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_userland: scx_example_userland.c scx_example_userland.skel.h \ -- scx_example_userland_common.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR) --atropos: export ATROPOS_CLANG = $(CLANG) --atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS) --atropos: $(INCLUDE_DIR)/vmlinux.h -- cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS) -- --clean: -- cargo clean --manifest-path=atropos/Cargo.toml -- rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) -- rm -f *.o *.bpf.o *.skel.h *.subskel.h -- rm -f scx_example_simple scx_example_qmap scx_example_central \ -- scx_example_pair scx_example_flatcg scx_example_userland -- --.PHONY: all atropos clean -- --# delete failed targets --.DELETE_ON_ERROR: -- --# keep intermediate (.skel.h, .bpf.o, etc) targets --.SECONDARY: -diff --git b/tools/sched_ext/atropos/.gitignore a/tools/sched_ext/atropos/.gitignore -deleted file mode 100644 -index 186dba259ec2..000000000000 ---- b/tools/sched_ext/atropos/.gitignore -+++ /dev/null -@@ -1,3 +0,0 @@ --src/bpf/.output --Cargo.lock --target -diff --git b/tools/sched_ext/atropos/Cargo.toml a/tools/sched_ext/atropos/Cargo.toml -deleted file mode 100644 -index 7462a836d53d..000000000000 ---- b/tools/sched_ext/atropos/Cargo.toml -+++ /dev/null -@@ -1,28 +0,0 @@ --[package] --name = "scx_atropos" --version = "0.5.0" --authors = ["Dan Schatzberg ", "Meta"] --edition = "2021" --description = "Userspace scheduling with BPF" --license = "GPL-2.0-only" -- --[dependencies] --anyhow = "1.0.65" --bitvec = { version = "1.0", features = ["serde"] } --clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } --ctrlc = { version = "3.1", features = ["termination"] } --fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"} --hex = "0.4.3" --libbpf-rs = "0.19.1" --libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] } --libc = "0.2.137" --log = "0.4.17" --ordered-float = "3.4.0" --simplelog = "0.12.0" -- --[build-dependencies] --bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false } --libbpf-cargo = "0.13.0" -- --[features] --enable_backtrace = [] -diff --git b/tools/sched_ext/atropos/build.rs a/tools/sched_ext/atropos/build.rs -deleted file mode 100644 -index 26e792c5e17e..000000000000 ---- b/tools/sched_ext/atropos/build.rs -+++ /dev/null -@@ -1,70 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --extern crate bindgen; -- --use std::env; --use std::fs::create_dir_all; --use std::path::Path; --use std::path::PathBuf; -- --use libbpf_cargo::SkeletonBuilder; -- --const HEADER_PATH: &str = "src/bpf/atropos.h"; -- --fn bindgen_atropos() { -- // Tell cargo to invalidate the built crate whenever the wrapper changes -- println!("cargo:rerun-if-changed={}", HEADER_PATH); -- -- // The bindgen::Builder is the main entry point -- // to bindgen, and lets you build up options for -- // the resulting bindings. -- let bindings = bindgen::Builder::default() -- // The input header we would like to generate -- // bindings for. -- .header(HEADER_PATH) -- // Tell cargo to invalidate the built crate whenever any of the -- // included header files changed. -- .parse_callbacks(Box::new(bindgen::CargoCallbacks)) -- // Finish the builder and generate the bindings. -- .generate() -- // Unwrap the Result and panic on failure. -- .expect("Unable to generate bindings"); -- -- // Write the bindings to the $OUT_DIR/bindings.rs file. -- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); -- bindings -- .write_to_file(out_path.join("atropos-sys.rs")) -- .expect("Couldn't write bindings!"); --} -- --fn gen_bpf_sched(name: &str) { -- let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap(); -- let clang = env::var("ATROPOS_CLANG").unwrap(); -- eprintln!("{}", clang); -- let outpath = format!("./src/bpf/.output/{}.skel.rs", name); -- let skel = Path::new(&outpath); -- let src = format!("./src/bpf/{}.bpf.c", name); -- SkeletonBuilder::new() -- .source(src.clone()) -- .clang(clang) -- .clang_args(bpf_cflags) -- .build_and_generate(&skel) -- .unwrap(); -- println!("cargo:rerun-if-changed={}", src); --} -- --fn main() { -- bindgen_atropos(); -- // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton. -- // Reasons are because the generated skeleton contains compiler attributes -- // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]` -- // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside -- // the path attribute either (see https://github.com/rust-lang/rust/pull/83366). -- // -- // However, there is hope! When the above feature stabilizes we can clean this -- // all up. -- create_dir_all("./src/bpf/.output").unwrap(); -- gen_bpf_sched("atropos"); --} -diff --git b/tools/sched_ext/atropos/rustfmt.toml a/tools/sched_ext/atropos/rustfmt.toml -deleted file mode 100644 -index b7258ed0a8d8..000000000000 ---- b/tools/sched_ext/atropos/rustfmt.toml -+++ /dev/null -@@ -1,8 +0,0 @@ --# Get help on options with `rustfmt --help=config` --# Please keep these in alphabetical order. --edition = "2021" --group_imports = "StdExternalCrate" --imports_granularity = "Item" --merge_derives = false --use_field_init_shorthand = true --version = "Two" -diff --git b/tools/sched_ext/atropos/src/atropos_sys.rs a/tools/sched_ext/atropos/src/atropos_sys.rs -deleted file mode 100644 -index bbeaf856d40e..000000000000 ---- b/tools/sched_ext/atropos/src/atropos_sys.rs -+++ /dev/null -@@ -1,10 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#![allow(non_upper_case_globals)] --#![allow(non_camel_case_types)] --#![allow(non_snake_case)] --#![allow(dead_code)] -- --include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs")); -diff --git b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -deleted file mode 100644 -index 3905a403e940..000000000000 ---- b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -+++ /dev/null -@@ -1,743 +0,0 @@ --/* Copyright (c) Meta Platforms, Inc. and affiliates. */ --/* -- * This software may be used and distributed according to the terms of the -- * GNU General Public License version 2. -- * -- * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF -- * part does simple round robin in each domain and the userspace part -- * calculates the load factor of each domain and tells the BPF part how to load -- * balance the domains. -- * -- * Every task has an entry in the task_data map which lists which domain the -- * task belongs to. When a task first enters the system (atropos_prep_enable), -- * they are round-robined to a domain. -- * -- * atropos_select_cpu is the primary scheduling logic, invoked when a task -- * becomes runnable. The lb_data map is populated by userspace to inform the BPF -- * scheduler that a task should be migrated to a new domain. Otherwise, the task -- * is scheduled in priority order as follows: -- * * The current core if the task was woken up synchronously and there are idle -- * cpus in the system -- * * The previous core, if idle -- * * The pinned-to core if the task is pinned to a specific core -- * * Any idle cpu in the domain -- * -- * If none of the above conditions are met, then the task is enqueued to a -- * dispatch queue corresponding to the domain (atropos_enqueue). -- * -- * atropos_dispatch will attempt to consume a task from its domain's -- * corresponding dispatch queue (this occurs after scheduling any tasks directly -- * assigned to it due to the logic in atropos_select_cpu). If no task is found, -- * then greedy load stealing will attempt to find a task on another dispatch -- * queue to run. -- * -- * Load balancing is almost entirely handled by userspace. BPF populates the -- * task weight, dom mask and current dom in the task_data map and executes the -- * load balance based on userspace populating the lb_data map. -- */ --#include "../../../scx_common.bpf.h" --#include "atropos.h" -- --#include --#include --#include --#include --#include --#include -- --char _license[] SEC("license") = "GPL"; -- --/* -- * const volatiles are set during initialization and treated as consts by the -- * jit compiler. -- */ -- --/* -- * Domains and cpus -- */ --const volatile __u32 nr_doms = 32; /* !0 for veristat, set during init */ --const volatile __u32 nr_cpus = 64; /* !0 for veristat, set during init */ --const volatile __u32 cpu_dom_id_map[MAX_CPUS]; --const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; -- --const volatile bool kthreads_local; --const volatile bool fifo_sched; --const volatile bool switch_partial; --const volatile __u32 greedy_threshold; -- --/* base slice duration */ --const volatile __u64 slice_us = 20000; -- --/* -- * Exit info -- */ --int exit_type = SCX_EXIT_NONE; --char exit_msg[SCX_EXIT_MSG_LEN]; -- --struct pcpu_ctx { -- __u32 dom_rr_cur; /* used when scanning other doms */ -- -- /* libbpf-rs does not respect the alignment, so pad out the struct explicitly */ -- __u8 _padding[CACHELINE_SIZE - sizeof(u64)]; --} __attribute__((aligned(CACHELINE_SIZE))); -- --struct pcpu_ctx pcpu_ctx[MAX_CPUS]; -- --/* -- * Domain context -- */ --struct dom_ctx { -- struct bpf_cpumask __kptr *cpumask; -- u64 vtime_now; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __type(key, u32); -- __type(value, struct dom_ctx); -- __uint(max_entries, MAX_DOMS); -- __uint(map_flags, 0); --} dom_ctx SEC(".maps"); -- --/* -- * Statistics -- */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, ATROPOS_NR_STATS); --} stats SEC(".maps"); -- --static inline void stat_add(enum stat_idx idx, u64 addend) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p) += addend; --} -- --/* Map pid -> task_ctx */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, struct task_ctx); -- __uint(max_entries, 1000000); -- __uint(map_flags, 0); --} task_data SEC(".maps"); -- --/* -- * This is populated from userspace to indicate which pids should be reassigned -- * to new doms. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, u32); -- __uint(max_entries, 1000); -- __uint(map_flags, 0); --} lb_data SEC(".maps"); -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p, -- u32 new_dom_id) --{ -- struct dom_ctx *old_domc, *new_domc; -- struct bpf_cpumask *d_cpumask, *t_cpumask; -- u32 old_dom_id = task_ctx->dom_id; -- s64 vtime_delta; -- -- old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id); -- if (!old_domc) { -- scx_bpf_error("No dom%u", old_dom_id); -- return false; -- } -- -- vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now; -- -- new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id); -- if (!new_domc) { -- scx_bpf_error("No dom%u", new_dom_id); -- return false; -- } -- -- d_cpumask = new_domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to get domain %u cpumask kptr", -- new_dom_id); -- return false; -- } -- -- t_cpumask = task_ctx->cpumask; -- if (!t_cpumask) { -- scx_bpf_error("Failed to look up task cpumask"); -- return false; -- } -- -- /* -- * set_cpumask might have happened between userspace requesting LB and -- * here and @p might not be able to run in @dom_id anymore. Verify. -- */ -- if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- p->cpus_ptr)) { -- p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta; -- task_ctx->dom_id = new_dom_id; -- bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask, -- p->cpus_ptr); -- } -- -- return task_ctx->dom_id == new_dom_id; --} -- --s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu, -- u32 wake_flags) --{ -- s32 cpu; -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- struct bpf_cpumask *p_cpumask; -- -- if (!task_ctx) -- return -ENOENT; -- -- if (kthreads_local && -- (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- cpu = prev_cpu; -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local dsq of the waker. -- */ -- if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) { -- struct task_struct *current = (void *)bpf_get_current_task(); -- -- if (!(BPF_CORE_READ(current, flags) & PF_EXITING) && -- task_ctx->dom_id < MAX_DOMS) { -- struct dom_ctx *domc; -- struct bpf_cpumask *d_cpumask; -- const struct cpumask *idle_cpumask; -- bool has_idle; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id); -- if (!domc) { -- scx_bpf_error("Failed to find dom%u", -- task_ctx->dom_id); -- return prev_cpu; -- } -- d_cpumask = domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to acquire domain %u cpumask kptr", -- task_ctx->dom_id); -- return prev_cpu; -- } -- -- idle_cpumask = scx_bpf_get_idle_cpumask(); -- -- has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- idle_cpumask); -- -- scx_bpf_put_idle_cpumask(idle_cpumask); -- -- if (has_idle) { -- cpu = bpf_get_smp_processor_id(); -- if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- stat_add(ATROPOS_STAT_WAKE_SYNC, 1); -- goto local; -- } -- } -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- stat_add(ATROPOS_STAT_PREV_IDLE, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- /* If only one core is allowed, dispatch */ -- if (p->nr_cpus_allowed == 1) { -- stat_add(ATROPOS_STAT_PINNED, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) -- return -ENOENT; -- -- /* If there is an eligible idle CPU, dispatch directly */ -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- if (cpu >= 0) { -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * @prev_cpu may be in a different domain. Returning an out-of-domain -- * CPU can lead to stalls as all in-domain CPUs may be idle by the time -- * @p gets enqueued. -- */ -- if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask)) -- cpu = prev_cpu; -- else -- cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask); -- -- return cpu; -- --local: -- task_ctx->dispatch_local = true; -- return cpu; --} -- --void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- u32 *new_dom; -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- new_dom = bpf_map_lookup_elem(&lb_data, &pid); -- if (new_dom && *new_dom != task_ctx->dom_id && -- task_set_dsq(task_ctx, p, *new_dom)) { -- struct bpf_cpumask *p_cpumask; -- s32 cpu; -- -- stat_add(ATROPOS_STAT_LOAD_BALANCE, 1); -- -- /* -- * If dispatch_local is set, We own @p's idle state but we are -- * not gonna put the task in the associated local dsq which can -- * cause the CPU to stall. Kick it. -- */ -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0); -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) { -- scx_bpf_error("Failed to get task_ctx->cpumask"); -- return; -- } -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- } -- -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags); -- return; -- } -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000, -- enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- u32 dom_id = task_ctx->dom_id; -- struct dom_ctx *domc; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, domc->vtime_now - slice_us * 1000)) -- vtime = domc->vtime_now - slice_us * 1000; -- -- scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --static u32 cpu_to_dom_id(s32 cpu) --{ -- const volatile u32 *dom_idp; -- -- if (nr_doms <= 1) -- return 0; -- -- dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]); -- if (!dom_idp) -- return MAX_DOMS; -- -- return *dom_idp; --} -- --static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) --{ -- s32 cpu; -- -- if (dom_id >= MAX_DOMS) -- return false; -- -- bpf_for(cpu, 0, nr_cpus) { -- if (bpf_cpumask_test_cpu(cpu, cpumask) && -- (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64)))) -- return true; -- } -- return false; --} -- --static u32 dom_rr_next(s32 cpu) --{ -- struct pcpu_ctx *pcpuc; -- u32 dom_id; -- -- pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]); -- if (!pcpuc) -- return 0; -- -- dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms; -- -- if (dom_id == cpu_to_dom_id(cpu)) -- dom_id = (dom_id + 1) % nr_doms; -- -- pcpuc->dom_rr_cur = dom_id; -- return dom_id; --} -- --void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 dom = cpu_to_dom_id(cpu); -- -- if (scx_bpf_consume(dom)) { -- stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1); -- return; -- } -- -- if (!greedy_threshold) -- return; -- -- bpf_repeat(nr_doms - 1) { -- u32 dom_id = dom_rr_next(cpu); -- -- if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold && -- scx_bpf_consume(dom_id)) { -- stat_add(ATROPOS_STAT_GREEDY, 1); -- break; -- } -- } --} -- --void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_at = bpf_ktime_get_ns(); --} -- --void BPF_STRUCT_OPS(atropos_running, struct task_struct *p) --{ -- struct task_ctx *taskc; -- struct dom_ctx *domc; -- pid_t pid = p->pid; -- u32 dom_id; -- -- if (fifo_sched) -- return; -- -- taskc = bpf_map_lookup_elem(&task_data, &pid); -- if (!taskc) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- dom_id = taskc->dom_id; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(domc->vtime_now, p->scx.dsq_vtime)) -- domc->vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at; -- task_ctx->runnable_at = 0; --} -- --void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->weight = weight; --} -- --struct pick_task_domain_loop_ctx { -- struct task_struct *p; -- const struct cpumask *cpumask; -- u64 dom_mask; -- u32 dom_rr_base; -- u32 dom_id; --}; -- --static int pick_task_domain_loopfn(u32 idx, void *data) --{ -- struct pick_task_domain_loop_ctx *lctx = data; -- u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms; -- -- if (dom_id >= MAX_DOMS) -- return 1; -- -- if (cpumask_intersects_domain(lctx->cpumask, dom_id)) { -- lctx->dom_mask |= 1LLU << dom_id; -- if (lctx->dom_id == MAX_DOMS) -- lctx->dom_id = dom_id; -- } -- return 0; --} -- --static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- struct pick_task_domain_loop_ctx lctx = { -- .p = p, -- .cpumask = cpumask, -- .dom_id = MAX_DOMS, -- }; -- s32 cpu = bpf_get_smp_processor_id(); -- -- if (cpu < 0 || cpu >= MAX_CPUS) -- return MAX_DOMS; -- -- lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur); -- -- bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0); -- task_ctx->dom_mask = lctx.dom_mask; -- -- return lctx.dom_id; --} -- --static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- u32 dom_id = 0; -- -- if (nr_doms > 1) -- dom_id = pick_task_domain(task_ctx, p, cpumask); -- -- if (!task_set_dsq(task_ctx, p, dom_id)) -- scx_bpf_error("Failed to set domain %d for %s[%d]", -- dom_id, p->comm, p->pid); --} -- --void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_set_domain(task_ctx, p, cpumask); --} -- --s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct bpf_cpumask *cpumask; -- struct task_ctx task_ctx, *map_value; -- long ret; -- pid_t pid; -- -- memset(&task_ctx, 0, sizeof(task_ctx)); -- -- pid = p->pid; -- ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return ret; -- } -- -- /* -- * Read the entry from the map immediately so we can add the cpumask -- * with bpf_kptr_xchg(). -- */ -- map_value = bpf_map_lookup_elem(&task_data, &pid); -- if (!map_value) -- /* Should never happen -- it was just inserted above. */ -- return -EINVAL; -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- bpf_map_delete_elem(&task_data, &pid); -- return -ENOMEM; -- } -- -- cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask); -- if (cpumask) { -- /* Should never happen as we just inserted it above. */ -- bpf_cpumask_release(cpumask); -- bpf_map_delete_elem(&task_data, &pid); -- return -EINVAL; -- } -- -- task_set_domain(map_value, p, p->cpus_ptr); -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p) --{ -- pid_t pid = p->pid; -- long ret = bpf_map_delete_elem(&task_data, &pid); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return; -- } --} -- --static int create_dom_dsq(u32 idx, void *data) --{ -- struct dom_ctx domc_init = {}, *domc; -- struct bpf_cpumask *cpumask; -- u32 cpu, dom_id = idx; -- s32 ret; -- -- ret = scx_bpf_create_dsq(dom_id, -1); -- if (ret < 0) { -- scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret); -- return 1; -- } -- -- ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0); -- if (ret) { -- scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret); -- return 1; -- } -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- /* Should never happen, we just inserted it above. */ -- scx_bpf_error("No dom%u", dom_id); -- return 1; -- } -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id); -- return 1; -- } -- -- for (cpu = 0; cpu < MAX_CPUS; cpu++) { -- const volatile __u64 *dmask; -- -- dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]); -- if (!dmask) { -- scx_bpf_error("array index error"); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- if (*dmask & (1LLU << (cpu % 64))) -- bpf_cpumask_set_cpu(cpu, cpumask); -- } -- -- cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask); -- if (cpumask) { -- scx_bpf_error("Domain %u was already present", dom_id); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(atropos_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- bpf_loop(nr_doms, create_dom_dsq, NULL, 0); -- -- for (u32 i = 0; i < nr_cpus; i++) -- pcpu_ctx[i].dom_rr_cur = i; -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg); -- exit_type = ei->type; --} -- --SEC(".struct_ops") --struct sched_ext_ops atropos = { -- .select_cpu = (void *)atropos_select_cpu, -- .enqueue = (void *)atropos_enqueue, -- .dispatch = (void *)atropos_dispatch, -- .runnable = (void *)atropos_runnable, -- .running = (void *)atropos_running, -- .stopping = (void *)atropos_stopping, -- .quiescent = (void *)atropos_quiescent, -- .set_weight = (void *)atropos_set_weight, -- .set_cpumask = (void *)atropos_set_cpumask, -- .prep_enable = (void *)atropos_prep_enable, -- .disable = (void *)atropos_disable, -- .init = (void *)atropos_init, -- .exit = (void *)atropos_exit, -- .flags = 0, -- .name = "atropos", --}; -diff --git b/tools/sched_ext/atropos/src/bpf/atropos.h a/tools/sched_ext/atropos/src/bpf/atropos.h -deleted file mode 100644 -index addf29ca104a..000000000000 ---- b/tools/sched_ext/atropos/src/bpf/atropos.h -+++ /dev/null -@@ -1,44 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#ifndef __ATROPOS_H --#define __ATROPOS_H -- --#include --#ifndef __kptr --#ifdef __KERNEL__ --#error "__kptr_ref not defined in the kernel" --#endif --#define __kptr --#endif -- --#define MAX_CPUS 512 --#define MAX_DOMS 64 /* limited to avoid complex bitmask ops */ --#define CACHELINE_SIZE 64 -- --/* Statistics */ --enum stat_idx { -- ATROPOS_STAT_TASK_GET_ERR, -- ATROPOS_STAT_WAKE_SYNC, -- ATROPOS_STAT_PREV_IDLE, -- ATROPOS_STAT_PINNED, -- ATROPOS_STAT_DIRECT_DISPATCH, -- ATROPOS_STAT_DSQ_DISPATCH, -- ATROPOS_STAT_GREEDY, -- ATROPOS_STAT_LOAD_BALANCE, -- ATROPOS_STAT_LAST_TASK, -- ATROPOS_NR_STATS, --}; -- --struct task_ctx { -- unsigned long long dom_mask; /* the domains this task can run on */ -- struct bpf_cpumask __kptr *cpumask; -- unsigned int dom_id; -- unsigned int weight; -- unsigned long long runnable_at; -- unsigned long long runnable_for; -- bool dispatch_local; --}; -- --#endif /* __ATROPOS_H */ -diff --git b/tools/sched_ext/atropos/src/main.rs a/tools/sched_ext/atropos/src/main.rs -deleted file mode 100644 -index 0d313662f713..000000000000 ---- b/tools/sched_ext/atropos/src/main.rs -+++ /dev/null -@@ -1,942 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#[path = "bpf/.output/atropos.skel.rs"] --mod atropos; --pub use atropos::*; --pub mod atropos_sys; -- --use std::cell::Cell; --use std::collections::{BTreeMap, BTreeSet}; --use std::ffi::CStr; --use std::ops::Bound::{Included, Unbounded}; --use std::sync::atomic::{AtomicBool, Ordering}; --use std::sync::Arc; --use std::time::{Duration, SystemTime}; -- --use ::fb_procfs as procfs; --use anyhow::{anyhow, bail, Context, Result}; --use bitvec::prelude::*; --use clap::Parser; --use log::{info, trace, warn}; --use ordered_float::OrderedFloat; -- --/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF --/// part does simple round robin in each domain and the userspace part --/// calculates the load factor of each domain and tells the BPF part how to load --/// balance the domains. -- --/// This scheduler demonstrates dividing scheduling logic between BPF and --/// userspace and using rust to build the userspace part. An earlier variant of --/// this scheduler was used to balance across six domains, each representing a --/// chiplet in a six-chiplet AMD processor, and could match the performance of --/// production setup using CFS. --#[derive(Debug, Parser)] --struct Opts { -- /// Scheduling slice duration in microseconds. -- #[clap(short, long, default_value = "20000")] -- slice_us: u64, -- -- /// Monitoring and load balance interval in seconds. -- #[clap(short, long, default_value = "2.0")] -- interval: f64, -- -- /// Build domains according to how CPUs are grouped at this cache level -- /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id. -- #[clap(short = 'c', long, default_value = "3")] -- cache_level: u32, -- -- /// Instead of using cache locality, set the cpumask for each domain -- /// manually, provide multiple --cpumasks, one for each domain. E.g. -- /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with -- /// the corresponding CPUs belonging to each domain. Each CPU must -- /// belong to precisely one domain. -- #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")] -- cpumasks: Vec, -- -- /// When non-zero, enable greedy task stealing. When a domain is idle, a -- /// cpu will attempt to steal tasks from a domain with at least -- /// greedy_threshold tasks enqueued. These tasks aren't permanently -- /// stolen from the domain. -- #[clap(short, long, default_value = "4")] -- greedy_threshold: u32, -- -- /// The load decay factor. Every interval, the existing load is decayed -- /// by this factor and new load is added. Must be in the range [0.0, -- /// 0.99]. The smaller the value, the more sensitive load calculation -- /// is to recent changes. When 0.0, history is ignored and the load -- /// value from the latest period is used directly. -- #[clap(short, long, default_value = "0.5")] -- load_decay_factor: f64, -- -- /// Disable load balancing. Unless disabled, periodically userspace will -- /// calculate the load factor of each domain and instruct BPF which -- /// processes to move. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- no_load_balance: bool, -- -- /// Put per-cpu kthreads directly into local dsq's. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- kthreads_local: bool, -- -- /// Use FIFO scheduling instead of weighted vtime scheduling. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- fifo_sched: bool, -- -- /// If specified, only tasks which have their scheduling policy set to -- /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all -- /// tasks are switched. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- partial: bool, -- -- /// Enable verbose output including libbpf details. Specify multiple -- /// times to increase verbosity. -- #[clap(short, long, action = clap::ArgAction::Count)] -- verbose: u8, --} -- --fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result { -- Ok(reader -- .read_stat() -- .context("Failed to read procfs")? -- .total_cpu -- .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?) --} -- --fn now_monotonic() -> u64 { -- let mut time = libc::timespec { -- tv_sec: 0, -- tv_nsec: 0, -- }; -- let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) }; -- assert!(ret == 0); -- time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 --} -- --fn clear_map(map: &mut libbpf_rs::Map) { -- // XXX: libbpf_rs has some design flaw that make it impossible to -- // delete while iterating despite it being safe so we alias it here -- let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) }; -- for key in map.keys() { -- let _ = deleter.delete(&key); -- } --} -- --#[derive(Debug)] --struct TaskLoad { -- runnable_for: u64, -- load: f64, --} -- --#[derive(Debug)] --struct TaskInfo { -- pid: i32, -- dom_mask: u64, -- migrated: Cell, --} -- --struct LoadBalancer<'a, 'b, 'c> { -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- -- tasks_by_load: Vec, TaskInfo>>, -- load_avg: f64, -- dom_loads: Vec, -- -- imbal: Vec, -- doms_to_push: BTreeMap, u32>, -- doms_to_pull: BTreeMap, u32>, -- -- nr_lb_data_errors: &'c mut u64, --} -- --impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> { -- const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10; -- const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1; -- const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50; -- -- fn new( -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- nr_lb_data_errors: &'c mut u64, -- ) -> Self { -- Self { -- maps, -- task_loads, -- nr_doms, -- load_decay_factor, -- -- tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(), -- load_avg: 0f64, -- dom_loads: vec![0.0; nr_doms], -- -- imbal: vec![0.0; nr_doms], -- doms_to_pull: BTreeMap::new(), -- doms_to_push: BTreeMap::new(), -- -- nr_lb_data_errors, -- } -- } -- -- fn read_task_loads(&mut self, period: Duration) -> Result<()> { -- let now_mono = now_monotonic(); -- let task_data = self.maps.task_data(); -- let mut this_task_loads = BTreeMap::::new(); -- let mut load_sum = 0.0f64; -- self.dom_loads = vec![0f64; self.nr_doms]; -- -- for key in task_data.keys() { -- if let Some(task_ctx_vec) = task_data -- .lookup(&key, libbpf_rs::MapFlags::ANY) -- .context("Failed to lookup task_data")? -- { -- let task_ctx = -- unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) }; -- let pid = i32::from_ne_bytes( -- key.as_slice() -- .try_into() -- .context("Invalid key length in task_data map")?, -- ); -- -- let (this_at, this_for, weight) = unsafe { -- ( -- std::ptr::read_volatile(&task_ctx.runnable_at as *const u64), -- std::ptr::read_volatile(&task_ctx.runnable_for as *const u64), -- std::ptr::read_volatile(&task_ctx.weight as *const u32), -- ) -- }; -- -- let (mut delta, prev_load) = match self.task_loads.get(&pid) { -- Some(prev) => (this_for - prev.runnable_for, Some(prev.load)), -- None => (this_for, None), -- }; -- -- // Non-zero this_at indicates that the task is currently -- // runnable. Note that we read runnable_at and runnable_for -- // without any synchronization and there is a small window -- // where we end up misaccounting. While this can cause -- // temporary error, it's unlikely to cause any noticeable -- // misbehavior especially given the load value clamping. -- if this_at > 0 && this_at < now_mono { -- delta += now_mono - this_at; -- } -- -- delta = delta.min(period.as_nanos() as u64); -- let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64) -- .clamp(0.0, weight as f64); -- -- let this_load = match prev_load { -- Some(prev_load) => { -- prev_load * self.load_decay_factor -- + this_load * (1.0 - self.load_decay_factor) -- } -- None => this_load, -- }; -- -- this_task_loads.insert( -- pid, -- TaskLoad { -- runnable_for: this_for, -- load: this_load, -- }, -- ); -- -- load_sum += this_load; -- self.dom_loads[task_ctx.dom_id as usize] += this_load; -- // Only record pids that are eligible for load balancing -- if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) { -- continue; -- } -- self.tasks_by_load[task_ctx.dom_id as usize].insert( -- OrderedFloat(this_load), -- TaskInfo { -- pid, -- dom_mask: task_ctx.dom_mask, -- migrated: Cell::new(false), -- }, -- ); -- } -- } -- -- self.load_avg = load_sum / self.nr_doms as f64; -- *self.task_loads = this_task_loads; -- Ok(()) -- } -- -- // To balance dom loads we identify doms with lower and higher load than average -- fn calculate_dom_load_balance(&mut self) -> Result<()> { -- for (dom, dom_load) in self.dom_loads.iter().enumerate() { -- let imbal = dom_load - self.load_avg; -- if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO { -- if imbal > 0f64 { -- self.doms_to_push.insert(OrderedFloat(imbal), dom as u32); -- } else { -- self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32); -- } -- self.imbal[dom] = imbal; -- } -- } -- Ok(()) -- } -- -- // Find the first candidate pid which hasn't already been migrated and -- // can run in @pull_dom. -- fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)> -- where -- I: IntoIterator, &'d TaskInfo)>, -- { -- match tasks_by_load -- .into_iter() -- .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0) -- .next() -- { -- Some((OrderedFloat(load), task)) => Some((*load, task)), -- None => None, -- } -- } -- -- fn pick_victim( -- &self, -- (push_dom, to_push): (u32, f64), -- (pull_dom, to_pull): (u32, f64), -- ) -> Option<(&TaskInfo, f64)> { -- let to_xfer = to_pull.min(to_push); -- -- trace!( -- "considering dom {}@{:.2} -> {}@{:.2}", -- push_dom, -- to_push, -- pull_dom, -- to_pull -- ); -- -- let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs(); -- -- trace!( -- "to_xfer={:.2} tasks_by_load={:?}", -- to_xfer, -- &self.tasks_by_load[push_dom as usize] -- ); -- -- // We want to pick a task to transfer from push_dom to pull_dom to -- // maximize the reduction of load imbalance between the two. IOW, -- // pick a task which has the closest load value to $to_xfer that can -- // be migrated. Find such task by locating the first migratable task -- // while scanning left from $to_xfer and the counterpart while -- // scanning right and picking the better of the two. -- let (load, task, new_imbal) = match ( -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Unbounded, Included(&OrderedFloat(to_xfer)))) -- .rev(), -- pull_dom, -- ), -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Included(&OrderedFloat(to_xfer)), Unbounded)), -- pull_dom, -- ), -- ) { -- (None, None) => return None, -- (Some((load, task)), None) | (None, Some((load, task))) => { -- (load, task, calc_new_imbal(load)) -- } -- (Some((load0, task0)), Some((load1, task1))) => { -- let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1)); -- if new_imbal0 <= new_imbal1 { -- (load0, task0, new_imbal0) -- } else { -- (load1, task1, new_imbal1) -- } -- } -- }; -- -- // If the best candidate can't reduce the imbalance, there's nothing -- // to do for this pair. -- let old_imbal = to_push + to_pull; -- if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal { -- trace!( -- "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal -- ); -- return None; -- } -- -- trace!( -- "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal, -- ); -- -- Some((task, load)) -- } -- -- // Actually execute the load balancing. Concretely this writes pid -> dom -- // entries into the lb_data map for bpf side to consume. -- fn load_balance(&mut self) -> Result<()> { -- clear_map(self.maps.lb_data()); -- -- trace!("imbal={:?}", &self.imbal); -- trace!("doms_to_push={:?}", &self.doms_to_push); -- trace!("doms_to_pull={:?}", &self.doms_to_pull); -- -- // Push from the most imbalanced to least. -- while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() { -- let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO; -- let mut pushed = 0f64; -- -- // Transfer tasks from push_dom to reduce imbalance. -- loop { -- let last_pushed = pushed; -- -- // Pull from the most imbalaned to least. -- let mut doms_to_pull = BTreeMap::<_, _>::new(); -- std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull); -- let mut pull_doms = doms_to_pull.into_iter().rev().collect::>(); -- -- for (to_pull, pull_dom) in pull_doms.iter_mut() { -- if let Some((task, load)) = -- self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull))) -- { -- // Execute migration. -- task.migrated.set(true); -- to_push -= load; -- *to_pull -= load; -- pushed += load; -- -- // Ask BPF code to execute the migration. -- let pid = task.pid; -- let cpid = (pid as libc::pid_t).to_ne_bytes(); -- if let Err(e) = self.maps.lb_data().update( -- &cpid, -- &pull_dom.to_ne_bytes(), -- libbpf_rs::MapFlags::NO_EXIST, -- ) { -- warn!( -- "Failed to update lb_data map for pid={} error={:?}", -- pid, &e -- ); -- *self.nr_lb_data_errors += 1; -- } -- -- // Always break after a successful migration so that -- // the pulling domains are always considered in the -- // descending imbalance order. -- break; -- } -- } -- -- pull_doms -- .into_iter() -- .map(|(k, v)| self.doms_to_pull.insert(k, v)) -- .count(); -- -- // Stop repeating if nothing got transferred or pushed enough. -- if pushed == last_pushed || pushed >= push_max { -- break; -- } -- } -- } -- Ok(()) -- } --} -- --struct Scheduler<'a> { -- skel: AtroposSkel<'a>, -- struct_ops: Option, -- -- nr_cpus: usize, -- nr_doms: usize, -- load_decay_factor: f64, -- balance_load: bool, -- -- proc_reader: procfs::ProcReader, -- -- prev_at: SystemTime, -- prev_total_cpu: procfs::CpuStat, -- task_loads: BTreeMap, -- -- nr_lb_data_errors: u64, --} -- --impl<'a> Scheduler<'a> { -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn parse_cpusets( -- cpumasks: &[String], -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- if cpumasks.len() > atropos_sys::MAX_DOMS as usize { -- bail!( -- "Number of requested DSQs ({}) is greater than MAX_DOMS ({})", -- cpumasks.len(), -- atropos_sys::MAX_DOMS -- ); -- } -- let mut cpus = vec![-1i32; nr_cpus]; -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()]; -- for (dq, cpumask) in cpumasks.iter().enumerate() { -- let hex_str = { -- let mut tmp_str = cpumask -- .strip_prefix("0x") -- .unwrap_or(cpumask) -- .replace('_', ""); -- if tmp_str.len() % 2 != 0 { -- tmp_str = "0".to_string() + &tmp_str; -- } -- tmp_str -- }; -- let byte_vec = hex::decode(&hex_str) -- .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?; -- -- for (index, &val) in byte_vec.iter().rev().enumerate() { -- let mut v = val; -- while v != 0 { -- let lsb = v.trailing_zeros() as usize; -- v &= !(1 << lsb); -- let cpu = index * 8 + lsb; -- if cpu > nr_cpus { -- bail!( -- concat!( -- "Found cpu ({}) in cpumask ({}) which is larger", -- " than the number of cpus on the machine ({})" -- ), -- cpu, -- cpumask, -- nr_cpus -- ); -- } -- if cpus[cpu] != -1 { -- bail!( -- "Found cpu ({}) with dq ({}) but also in cpumask ({})", -- cpu, -- cpus[cpu], -- cpumask -- ); -- } -- cpus[cpu] = dq as i32; -- cpusets[dq].set(cpu, true); -- } -- } -- cpusets[dq].set_uninitialized(false); -- } -- -- for (cpu, &dq) in cpus.iter().enumerate() { -- if dq < 0 { -- bail!( -- "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.", -- cpu -- ); -- } -- } -- -- Ok((cpusets, cpus)) -- } -- -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn cpusets_from_cache( -- level: u32, -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- let mut cpu_to_cache = vec![]; // (cpu_id, cache_id) -- let mut cache_ids = BTreeSet::::new(); -- let mut nr_not_found = 0; -- -- // Build cpu -> cache ID mapping. -- for cpu in 0..nr_cpus { -- let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level); -- let id = match std::fs::read_to_string(&path) { -- Ok(val) => val -- .trim() -- .parse::() -- .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?, -- Err(e) if e.kind() == std::io::ErrorKind::NotFound => { -- nr_not_found += 1; -- 0 -- } -- Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)), -- }; -- -- cpu_to_cache.push(id); -- cache_ids.insert(id); -- } -- -- if nr_not_found > 1 { -- warn!( -- "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0", -- level, nr_not_found, nr_cpus -- ); -- } -- -- // Cache IDs may have holes. Assign consecutive domain IDs to -- // existing cache IDs. -- let mut cache_to_dom = BTreeMap::::new(); -- let mut nr_doms = 0; -- for cache_id in cache_ids.iter() { -- cache_to_dom.insert(*cache_id, nr_doms); -- nr_doms += 1; -- } -- -- if nr_doms > atropos_sys::MAX_DOMS { -- bail!( -- "Total number of doms {} is greater than MAX_DOMS ({})", -- nr_doms, -- atropos_sys::MAX_DOMS -- ); -- } -- -- // Build and return dom -> cpumask and cpu -> dom mappings. -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize]; -- let mut cpu_to_dom = vec![]; -- -- for cpu in 0..nr_cpus { -- let dom_id = cache_to_dom[&cpu_to_cache[cpu]]; -- cpusets[dom_id as usize].set(cpu, true); -- cpu_to_dom.push(dom_id as i32); -- } -- -- Ok((cpusets, cpu_to_dom)) -- } -- -- fn init(opts: &Opts) -> Result { -- // Open the BPF prog first for verification. -- let mut skel_builder = AtroposSkelBuilder::default(); -- skel_builder.obj_builder.debug(opts.verbose > 0); -- let mut skel = skel_builder.open().context("Failed to open BPF program")?; -- -- let nr_cpus = libbpf_rs::num_possible_cpus().unwrap(); -- if nr_cpus > atropos_sys::MAX_CPUS as usize { -- bail!( -- "nr_cpus ({}) is greater than MAX_CPUS ({})", -- nr_cpus, -- atropos_sys::MAX_CPUS -- ); -- } -- -- // Initialize skel according to @opts. -- let (cpusets, cpus) = if opts.cpumasks.len() > 0 { -- Self::parse_cpusets(&opts.cpumasks, nr_cpus)? -- } else { -- Self::cpusets_from_cache(opts.cache_level, nr_cpus)? -- }; -- let nr_doms = cpusets.len(); -- skel.rodata().nr_doms = nr_doms as u32; -- skel.rodata().nr_cpus = nr_cpus as u32; -- -- for (cpu, dom) in cpus.iter().enumerate() { -- skel.rodata().cpu_dom_id_map[cpu] = *dom as u32; -- } -- -- for (dom, cpuset) in cpusets.iter().enumerate() { -- let raw_cpuset_slice = cpuset.as_raw_slice(); -- let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom]; -- let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len()); -- left.clone_from_slice(cpuset.as_raw_slice()); -- let cpumask_str = dom_cpumask_slice -- .iter() -- .take((nr_cpus + 63) / 64) -- .rev() -- .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x)); -- info!( -- "DOM[{:02}] cpumask{} ({} cpus)", -- dom, -- &cpumask_str, -- cpuset.count_ones() -- ); -- } -- -- skel.rodata().slice_us = opts.slice_us; -- skel.rodata().kthreads_local = opts.kthreads_local; -- skel.rodata().fifo_sched = opts.fifo_sched; -- skel.rodata().switch_partial = opts.partial; -- skel.rodata().greedy_threshold = opts.greedy_threshold; -- -- // Attach. -- let mut skel = skel.load().context("Failed to load BPF program")?; -- skel.attach().context("Failed to attach BPF program")?; -- let struct_ops = Some( -- skel.maps_mut() -- .atropos() -- .attach_struct_ops() -- .context("Failed to attach atropos struct ops")?, -- ); -- info!("Atropos Scheduler Attached"); -- -- // Other stuff. -- let mut proc_reader = procfs::ProcReader::new(); -- let prev_total_cpu = read_total_cpu(&mut proc_reader)?; -- -- Ok(Self { -- skel, -- struct_ops, // should be held to keep it attached -- -- nr_cpus, -- nr_doms, -- load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99), -- balance_load: !opts.no_load_balance, -- -- proc_reader, -- -- prev_at: SystemTime::now(), -- prev_total_cpu, -- task_loads: BTreeMap::new(), -- -- nr_lb_data_errors: 0, -- }) -- } -- -- fn get_cpu_busy(&mut self) -> Result { -- let total_cpu = read_total_cpu(&mut self.proc_reader)?; -- let busy = match (&self.prev_total_cpu, &total_cpu) { -- ( -- procfs::CpuStat { -- user_usec: Some(prev_user), -- nice_usec: Some(prev_nice), -- system_usec: Some(prev_system), -- idle_usec: Some(prev_idle), -- iowait_usec: Some(prev_iowait), -- irq_usec: Some(prev_irq), -- softirq_usec: Some(prev_softirq), -- stolen_usec: Some(prev_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- procfs::CpuStat { -- user_usec: Some(curr_user), -- nice_usec: Some(curr_nice), -- system_usec: Some(curr_system), -- idle_usec: Some(curr_idle), -- iowait_usec: Some(curr_iowait), -- irq_usec: Some(curr_irq), -- softirq_usec: Some(curr_softirq), -- stolen_usec: Some(curr_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- ) => { -- let idle_usec = curr_idle - prev_idle; -- let iowait_usec = curr_iowait - prev_iowait; -- let user_usec = curr_user - prev_user; -- let system_usec = curr_system - prev_system; -- let nice_usec = curr_nice - prev_nice; -- let irq_usec = curr_irq - prev_irq; -- let softirq_usec = curr_softirq - prev_softirq; -- let stolen_usec = curr_stolen - prev_stolen; -- -- let busy_usec = -- user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec; -- let total_usec = idle_usec + busy_usec + iowait_usec; -- busy_usec as f64 / total_usec as f64 -- } -- _ => { -- bail!("Some procfs stats are not populated!"); -- } -- }; -- -- self.prev_total_cpu = total_cpu; -- Ok(busy) -- } -- -- fn read_bpf_stats(&mut self) -> Result> { -- let mut maps = self.skel.maps_mut(); -- let stats_map = maps.stats(); -- let mut stats: Vec = Vec::new(); -- let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus]; -- -- for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS { -- let cpu_stat_vec = stats_map -- .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY) -- .with_context(|| format!("Failed to lookup stat {}", stat))? -- .expect("per-cpu stat should exist"); -- let sum = cpu_stat_vec -- .iter() -- .map(|val| { -- u64::from_ne_bytes( -- val.as_slice() -- .try_into() -- .expect("Invalid value length in stat map"), -- ) -- }) -- .sum(); -- stats_map -- .update_percpu( -- &(stat as u32).to_ne_bytes(), -- &zero_vec, -- libbpf_rs::MapFlags::ANY, -- ) -- .context("Failed to zero stat")?; -- stats.push(sum); -- } -- Ok(stats) -- } -- -- fn report( -- &self, -- stats: &Vec, -- cpu_busy: f64, -- processing_dur: Duration, -- load_avg: f64, -- dom_loads: &Vec, -- imbal: &Vec, -- ) { -- let stat = |idx| stats[idx as usize]; -- let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK); -- -- info!( -- "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms", -- cpu_busy * 100.0, -- load_avg, -- stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize], -- stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize], -- self.nr_lb_data_errors, -- processing_dur.as_millis(), -- ); -- -- let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0; -- -- info!( -- "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}", -- total, -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY), -- ); -- -- for i in 0..self.nr_doms { -- info!( -- "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}", -- i, -- dom_loads[i], -- if imbal[i] < 0.0 { -imbal[i] } else { 0.0 }, -- if imbal[i] > 0.0 { imbal[i] } else { 0.0 }, -- ); -- } -- } -- -- fn step(&mut self) -> Result<()> { -- let started_at = std::time::SystemTime::now(); -- let bpf_stats = self.read_bpf_stats()?; -- let cpu_busy = self.get_cpu_busy()?; -- -- let mut lb = LoadBalancer::new( -- self.skel.maps_mut(), -- &mut self.task_loads, -- self.nr_doms, -- self.load_decay_factor, -- &mut self.nr_lb_data_errors, -- ); -- -- lb.read_task_loads(started_at.duration_since(self.prev_at)?)?; -- lb.calculate_dom_load_balance()?; -- -- if self.balance_load { -- lb.load_balance()?; -- } -- -- // Extract fields needed for reporting and drop lb to release -- // mutable borrows. -- let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal); -- -- self.report( -- &bpf_stats, -- cpu_busy, -- std::time::SystemTime::now().duration_since(started_at)?, -- load_avg, -- &dom_loads, -- &imbal, -- ); -- -- self.prev_at = started_at; -- Ok(()) -- } -- -- fn read_bpf_exit_type(&mut self) -> i32 { -- unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) } -- } -- -- fn report_bpf_exit_type(&mut self) -> Result<()> { -- // Report msg if EXT_OPS_EXIT_ERROR. -- match self.read_bpf_exit_type() { -- 0 => Ok(()), -- etype if etype == 2 => { -- let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) }; -- let msg = cstr -- .to_str() -- .context("Failed to convert exit msg to string") -- .unwrap(); -- bail!("BPF exit_type={} msg={}", etype, msg); -- } -- etype => { -- info!("BPF exit_type={}", etype); -- Ok(()) -- } -- } -- } --} -- --impl<'a> Drop for Scheduler<'a> { -- fn drop(&mut self) { -- if let Some(struct_ops) = self.struct_ops.take() { -- drop(struct_ops); -- } -- } --} -- --fn main() -> Result<()> { -- let opts = Opts::parse(); -- -- let llv = match opts.verbose { -- 0 => simplelog::LevelFilter::Info, -- 1 => simplelog::LevelFilter::Debug, -- _ => simplelog::LevelFilter::Trace, -- }; -- let mut lcfg = simplelog::ConfigBuilder::new(); -- lcfg.set_time_level(simplelog::LevelFilter::Error) -- .set_location_level(simplelog::LevelFilter::Off) -- .set_target_level(simplelog::LevelFilter::Off) -- .set_thread_level(simplelog::LevelFilter::Off); -- simplelog::TermLogger::init( -- llv, -- lcfg.build(), -- simplelog::TerminalMode::Stderr, -- simplelog::ColorChoice::Auto, -- )?; -- -- let shutdown = Arc::new(AtomicBool::new(false)); -- let shutdown_clone = shutdown.clone(); -- ctrlc::set_handler(move || { -- shutdown_clone.store(true, Ordering::Relaxed); -- }) -- .context("Error setting Ctrl-C handler")?; -- -- let mut sched = Scheduler::init(&opts)?; -- -- while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 { -- std::thread::sleep(Duration::from_secs_f64(opts.interval)); -- sched.step()?; -- } -- -- sched.report_bpf_exit_type() --} -diff --git b/tools/sched_ext/gnu/stubs.h a/tools/sched_ext/gnu/stubs.h -deleted file mode 100644 -index 719225b16626..000000000000 ---- b/tools/sched_ext/gnu/stubs.h -+++ /dev/null -@@ -1 +0,0 @@ --/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ -diff --git b/tools/sched_ext/scx_common.bpf.h a/tools/sched_ext/scx_common.bpf.h -deleted file mode 100644 -index e56de9dc86f2..000000000000 ---- b/tools/sched_ext/scx_common.bpf.h -+++ /dev/null -@@ -1,288 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __SCHED_EXT_COMMON_BPF_H --#define __SCHED_EXT_COMMON_BPF_H -- --#include "vmlinux.h" --#include --#include --#include --#include "user_exit_info.h" -- --#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ --#define PF_EXITING 0x00000004 --#define CLOCK_MONOTONIC 1 -- --/* -- * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can -- * lead to really confusing misbehaviors. Let's trigger a build failure. -- */ --static inline void ___vmlinux_h_sanity_check___(void) --{ -- _Static_assert(SCX_DSQ_FLAG_BUILTIN, -- "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); --} -- --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; -- --static inline __attribute__((format(printf, 1, 2))) --void ___scx_bpf_error_format_checker(const char *fmt, ...) {} -- --/* -- * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments -- * instead of an array of u64. Note that __param[] must have at least one -- * element to keep the verifier happy. -- */ --#define scx_bpf_error(fmt, args...) \ --({ \ -- static char ___fmt[] = fmt; \ -- unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ -- \ -- _Pragma("GCC diagnostic push") \ -- _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ -- ___bpf_fill(___param, args); \ -- _Pragma("GCC diagnostic pop") \ -- \ -- scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ -- \ -- ___scx_bpf_error_format_checker(fmt, ##args); \ --}) -- --void scx_bpf_switch_all(void) __ksym; --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; --bool scx_bpf_consume(u64 dsq_id) __ksym; --u32 scx_bpf_dispatch_nr_slots(void) __ksym; --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym; --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym; --void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; --s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym; --const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; --const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; --void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; --void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; --bool scx_bpf_task_running(const struct task_struct *p) __ksym; --s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym; --u32 scx_bpf_reenqueue_local(void) __ksym; -- --#define BPF_STRUCT_OPS(name, args...) \ --SEC("struct_ops/"#name) \ --BPF_PROG(name, ##args) -- --#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ --SEC("struct_ops.s/"#name) \ --BPF_PROG(name, ##args) -- --/** -- * MEMBER_VPTR - Obtain the verified pointer to a struct or array member -- * @base: struct or array to index -- * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...) -- * -- * The verifier often gets confused by the instruction sequence the compiler -- * generates for indexing struct fields or arrays. This macro forces the -- * compiler to generate a code sequence which first calculates the byte offset, -- * checks it against the struct or array size and add that byte offset to -- * generate the pointer to the member to help the verifier. -- * -- * Ideally, we want to abort if the calculated offset is out-of-bounds. However, -- * BPF currently doesn't support abort, so evaluate to NULL instead. The caller -- * must check for NULL and take appropriate action to appease the verifier. To -- * avoid confusing the verifier, it's best to check for NULL and dereference -- * immediately. -- * -- * vptr = MEMBER_VPTR(my_array, [i][j]); -- * if (!vptr) -- * return error; -- * *vptr = new_value; -- */ --#define MEMBER_VPTR(base, member) (typeof(base member) *)({ \ -- u64 __base = (u64)base; \ -- u64 __addr = (u64)&(base member) - __base; \ -- asm volatile ( \ -- "if %0 <= %[max] goto +2\n" \ -- "%0 = 0\n" \ -- "goto +1\n" \ -- "%0 += %1\n" \ -- : "+r"(__addr) \ -- : "r"(__base), \ -- [max]"i"(sizeof(base) - sizeof(base member))); \ -- __addr; \ --}) -- --/* -- * BPF core and other generic helpers -- */ -- --/* list and rbtree */ --#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) --#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) -- --void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; --void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; -- --#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) --#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) -- --void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; --struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; --struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, -- struct bpf_rb_node *node) __ksym; --void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, -- bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym; --struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; -- --/* task */ --struct task_struct *bpf_task_from_pid(s32 pid) __ksym; --struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; --void bpf_task_release(struct task_struct *p) __ksym; -- --/* cgroup */ --struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; --void bpf_cgroup_release(struct cgroup *cgrp) __ksym; --struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; -- --/* cpumask */ --struct bpf_cpumask *bpf_cpumask_create(void) __ksym; --struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; --u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; --u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym; -- --/* rcu */ --void bpf_rcu_read_lock(void) __ksym; --void bpf_rcu_read_unlock(void) __ksym; -- --/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */ --struct bpf_iter_num; -- --extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym; --extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym; --extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym; -- --#ifndef bpf_for_each --/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for -- * using BPF open-coded iterators without having to write mundane explicit -- * low-level loop logic. Instead, it provides for()-like generic construct -- * that can be used pretty naturally. E.g., for some hypothetical cgroup -- * iterator, you'd write: -- * -- * struct cgroup *cg, *parent_cg = <...>; -- * -- * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { -- * bpf_printk("Child cgroup id = %d", cg->cgroup_id); -- * if (cg->cgroup_id == 123) -- * break; -- * } -- * -- * I.e., it looks almost like high-level for each loop in other languages, -- * supports continue/break, and is verifiable by BPF verifier. -- * -- * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) -- * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to -- * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int -- * *`, not just `int`. So for integers bpf_for() is more convenient. -- * -- * Note: this macro relies on C99 feature of allowing to declare variables -- * inside for() loop, bound to for() loop lifetime. It also utilizes GCC -- * extension: __attribute__((cleanup())), supported by both GCC and -- * Clang. -- */ --#define bpf_for_each(type, cur, args...) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ -- cleanup(bpf_iter_##type##_destroy))), \ -- /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_##type##_new(&___it, ##args), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_##type##_destroy, (void *)0); \ -- /* iteration and termination check */ \ -- (((cur) = bpf_iter_##type##_next(&___it))); \ --) --#endif /* bpf_for_each */ -- --#ifndef bpf_for --/* bpf_for(i, start, end) implements a for()-like looping construct that sets -- * provided integer variable *i* to values starting from *start* through, -- * but not including, *end*. It also proves to BPF verifier that *i* belongs -- * to range [start, end), so this can be used for accessing arrays without -- * extra checks. -- * -- * Note: *start* and *end* are assumed to be expressions with no side effects -- * and whose values do not change throughout bpf_for() loop execution. They do -- * not have to be statically known or constant, though. -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_for(i, start, end) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, (start), (end)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- ({ \ -- /* iteration step */ \ -- int *___t = bpf_iter_num_next(&___it); \ -- /* termination and bounds check */ \ -- (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ -- }); \ --) --#endif /* bpf_for */ -- --#ifndef bpf_repeat --/* bpf_repeat(N) performs N iterations without exposing iteration number -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_repeat(N) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, 0, (N)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- bpf_iter_num_next(&___it); \ -- /* nothing here */ \ --) --#endif /* bpf_repeat */ -- --#endif /* __SCHED_EXT_COMMON_BPF_H */ -diff --git b/tools/sched_ext/scx_example_central.bpf.c a/tools/sched_ext/scx_example_central.bpf.c -deleted file mode 100644 -index 4cec04b4c2ed..000000000000 ---- b/tools/sched_ext/scx_example_central.bpf.c -+++ /dev/null -@@ -1,334 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A central FIFO sched_ext scheduler which demonstrates the followings: -- * -- * a. Making all scheduling decisions from one CPU: -- * -- * The central CPU is the only one making scheduling decisions. All other -- * CPUs kick the central CPU when they run out of tasks to run. -- * -- * There is one global BPF queue and the central CPU schedules all CPUs by -- * dispatching from the global queue to each CPU's local dsq from dispatch(). -- * This isn't the most straightforward. e.g. It'd be easier to bounce -- * through per-CPU BPF queues. The current design is chosen to maximally -- * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching. -- * -- * b. Tickless operation -- * -- * All tasks are dispatched with the infinite slice which allows stopping the -- * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full -- * parameter. The tickless operation can be observed through -- * /proc/interrupts. -- * -- * Periodic switching is enforced by a periodic timer checking all CPUs and -- * preempting them as necessary. Unfortunately, BPF timer currently doesn't -- * have a way to pin to a specific CPU, so the periodic timer isn't pinned to -- * the central CPU. -- * -- * c. Preemption -- * -- * Kthreads are unconditionally queued to the head of a matching local dsq -- * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always -- * prioritized over user threads, which is required for ensuring forward -- * progress as e.g. the periodic timer may run on a ksoftirqd and if the -- * ksoftirqd gets starved by a user thread, there may not be anything else to -- * vacate that user thread. -- * -- * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the -- * next tasks. -- * -- * This scheduler is designed to maximize usage of various SCX mechanisms. A -- * more practical implementation would likely put the scheduling loop outside -- * the central CPU's dispatch() path and add some form of priority mechanism. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --enum { -- FALLBACK_DSQ_ID = 0, -- MAX_CPUS = 4096, -- MS_TO_NS = 1000LLU * 1000, -- TIMER_INTERVAL_NS = 1 * MS_TO_NS, --}; -- --const volatile bool switch_partial; --const volatile s32 central_cpu; --const volatile u32 nr_cpu_ids = 64; /* !0 for veristat, set during init */ -- --u64 nr_total, nr_locals, nr_queued, nr_lost_pids; --u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries; --u64 nr_overflows; -- --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, s32); --} central_q SEC(".maps"); -- --/* can't use percpu map due to bad lookups */ --static bool cpu_gimme_task[MAX_CPUS]; --static u64 cpu_started_at[MAX_CPUS]; -- --struct central_timer { -- struct bpf_timer timer; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, 1); -- __type(key, u32); -- __type(value, struct central_timer); --} central_timer SEC(".maps"); -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- /* -- * Steer wakeups to the central CPU as much as possible to avoid -- * disturbing other CPUs. It's safe to blindly return the central cpu as -- * select_cpu() is a hint and if @p can't be on it, the kernel will -- * automatically pick a fallback CPU. -- */ -- return central_cpu; --} -- --void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags) --{ -- s32 pid = p->pid; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- /* -- * Push per-cpu kthreads at the head of local dsq's and preempt the -- * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked -- * behind other threads which is necessary for forward progress -- * guarantee as we depend on the BPF timer which may run from ksoftirqd. -- */ -- if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- __sync_fetch_and_add(&nr_locals, 1); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF, -- enq_flags | SCX_ENQ_PREEMPT); -- return; -- } -- -- if (bpf_map_push_elem(¢ral_q, &pid, 0)) { -- __sync_fetch_and_add(&nr_overflows, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_queued, 1); -- -- if (!scx_bpf_task_running(p)) -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); --} -- --static bool dispatch_to_cpu(s32 cpu) --{ -- struct task_struct *p; -- s32 pid; -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (bpf_map_pop_elem(¢ral_q, &pid)) -- break; -- -- __sync_fetch_and_sub(&nr_queued, 1); -- -- p = bpf_task_from_pid(pid); -- if (!p) { -- __sync_fetch_and_add(&nr_lost_pids, 1); -- continue; -- } -- -- /* -- * If we can't run the task at the top, do the dumb thing and -- * bounce it to the fallback dsq. -- */ -- if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- __sync_fetch_and_add(&nr_mismatches, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0); -- bpf_task_release(p); -- continue; -- } -- -- /* dispatch to local and mark that @cpu doesn't need more */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0); -- -- if (cpu != central_cpu) -- scx_bpf_kick_cpu(cpu, 0); -- -- bpf_task_release(p); -- return true; -- } -- -- return false; --} -- --void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (cpu == central_cpu) { -- /* dispatch for all other CPUs first */ -- __sync_fetch_and_add(&nr_dispatches, 1); -- -- bpf_for(cpu, 0, nr_cpu_ids) { -- bool *gimme; -- -- if (!scx_bpf_dispatch_nr_slots()) -- break; -- -- /* central's gimme is never set */ -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme && !*gimme) -- continue; -- -- if (dispatch_to_cpu(cpu)) -- *gimme = false; -- } -- -- /* -- * Retry if we ran out of dispatch buffer slots as we might have -- * skipped some CPUs and also need to dispatch for self. The ext -- * core automatically retries if the local dsq is empty but we -- * can't rely on that as we're dispatching for other CPUs too. -- * Kick self explicitly to retry. -- */ -- if (!scx_bpf_dispatch_nr_slots()) { -- __sync_fetch_and_add(&nr_retries, 1); -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- return; -- } -- -- /* look for a task to run on the central CPU */ -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- dispatch_to_cpu(central_cpu); -- } else { -- bool *gimme; -- -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme) -- *gimme = true; -- -- /* -- * Force dispatch on the scheduling CPU so that it finds a task -- * to run for us. -- */ -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- } --} -- --void BPF_STRUCT_OPS(central_running, struct task_struct *p) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ --} -- --void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = 0; --} -- --static int central_timerfn(void *map, int *key, struct bpf_timer *timer) --{ -- u64 now = bpf_ktime_get_ns(); -- u64 nr_to_kick = nr_queued; -- s32 i; -- -- bpf_for(i, 0, nr_cpu_ids) { -- s32 cpu = (nr_timers + i) % nr_cpu_ids; -- u64 *started_at; -- -- if (cpu == central_cpu) -- continue; -- -- /* kick iff the current one exhausted its slice */ -- started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at && *started_at && -- vtime_before(now, *started_at + SCX_SLICE_DFL)) -- continue; -- -- /* and there's something pending */ -- if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) || -- scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu)) -- ; -- else if (nr_to_kick) -- nr_to_kick--; -- else -- continue; -- -- scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); -- } -- -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- -- bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- __sync_fetch_and_add(&nr_timers, 1); -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(central_init) --{ -- u32 key = 0; -- struct bpf_timer *timer; -- int ret; -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1); -- if (ret) -- return ret; -- -- timer = bpf_map_lookup_elem(¢ral_timer, &key); -- if (!timer) -- return -ESRCH; -- -- bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); -- bpf_timer_set_callback(timer, central_timerfn); -- ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- return ret; --} -- --void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops central_ops = { -- /* -- * We are offloading all scheduling decisions to the central CPU and -- * thus being the last task on a given CPU doesn't mean anything -- * special. Enqueue the last tasks like any other tasks. -- */ -- .flags = SCX_OPS_ENQ_LAST, -- -- .select_cpu = (void *)central_select_cpu, -- .enqueue = (void *)central_enqueue, -- .dispatch = (void *)central_dispatch, -- .running = (void *)central_running, -- .stopping = (void *)central_stopping, -- .init = (void *)central_init, -- .exit = (void *)central_exit, -- .name = "central", --}; -diff --git b/tools/sched_ext/scx_example_central.c a/tools/sched_ext/scx_example_central.c -deleted file mode 100644 -index 7ad591cbdc65..000000000000 ---- b/tools/sched_ext/scx_example_central.c -+++ /dev/null -@@ -1,94 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_central.skel.h" -- --const char help_fmt[] = --"A central FIFO sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-c CPU] [-p]\n" --"\n" --" -c CPU Override the central CPU (default: 0)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_central *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_central__open(); -- assert(skel); -- -- skel->rodata->central_cpu = 0; -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "c:ph")) != -1) { -- switch (opt) { -- case 'c': -- skel->rodata->central_cpu = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_central__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.central_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf("total :%10lu local:%10lu queued:%10lu lost:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_locals, -- skel->bss->nr_queued, -- skel->bss->nr_lost_pids); -- printf("timer :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n", -- skel->bss->nr_timers, -- skel->bss->nr_dispatches, -- skel->bss->nr_mismatches, -- skel->bss->nr_retries); -- printf("overflow:%10lu\n", -- skel->bss->nr_overflows); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_central__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_flatcg.bpf.c a/tools/sched_ext/scx_example_flatcg.bpf.c -deleted file mode 100644 -index f6078b9a681f..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.bpf.c -+++ /dev/null -@@ -1,872 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext flattened cgroup hierarchy scheduler. It implements -- * hierarchical weight-based cgroup CPU control by flattening the cgroup -- * hierarchy into a single layer by compounding the active weight share at each -- * level. Consider the following hierarchy with weights in parentheses: -- * -- * R + A (100) + B (100) -- * | \ C (100) -- * \ D (200) -- * -- * Ignoring the root and threaded cgroups, only B, C and D can contain tasks. -- * Let's say all three have runnable tasks. The total share that each of these -- * three cgroups is entitled to can be calculated by compounding its share at -- * each level. -- * -- * For example, B is competing against C and in that competition its share is -- * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's -- * share in that competition is 200/(200+100) == 1/3. B's eventual share in the -- * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's -- * eventual shaer is the same at 1/6. D is only competing at the top level and -- * its share is 200/(100+200) == 2/3. -- * -- * So, instead of hierarchically scheduling level-by-level, we can consider it -- * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3 -- * and keep updating the eventual shares as the cgroups' runnable states change. -- * -- * This flattening of hierarchy can bring a substantial performance gain when -- * the cgroup hierarchy is nested multiple levels. in a simple benchmark using -- * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it -- * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two -- * apache instances competing with 2:1 weight ratio nested four level deep. -- * -- * However, the gain comes at the cost of not being able to properly handle -- * thundering herd of cgroups. For example, if many cgroups which are nested -- * behind a low priority parent cgroup wake up around the same time, they may be -- * able to consume more CPU cycles than they are entitled to. In many use cases, -- * this isn't a real concern especially given the performance gain. Also, there -- * are ways to mitigate the problem further by e.g. introducing an extra -- * scheduling layer on cgroup delegation boundaries. -- * -- * The scheduler first picks the cgroup to run and then schedule the tasks -- * within by using nested weighted vtime scheduling by default. The -- * cgroup-internal scheduling can be switched to FIFO with the -f option. -- */ --#include "scx_common.bpf.h" --#include "user_exit_info.h" --#include "scx_example_flatcg.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */ --const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL; --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --u64 cvtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, u64); -- __uint(max_entries, FCG_NR_STATS); --} stats SEC(".maps"); -- --static void stat_inc(enum fcg_stat_idx idx) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p)++; --} -- --struct fcg_cpu_ctx { -- u64 cur_cgid; -- u64 cur_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, struct fcg_cpu_ctx); -- __uint(max_entries, 1); --} cpu_ctx SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_cgrp_ctx); --} cgrp_ctx SEC(".maps"); -- --struct cgv_node { -- struct bpf_rb_node rb_node; -- __u64 cvtime; -- __u64 cgid; --}; -- --private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock; --private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node); -- --struct cgv_node_stash { -- struct cgv_node __kptr *node; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, 16384); -- __type(key, __u64); -- __type(value, struct cgv_node_stash); --} cgv_node_stash SEC(".maps"); -- --struct fcg_task_ctx { -- u64 bypassed_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_task_ctx); --} task_ctx SEC(".maps"); -- --/* gets inc'd on weight tree changes to expire the cached hweights */ --unsigned long hweight_gen = 1; -- --static u64 div_round_up(u64 dividend, u64 divisor) --{ -- return (dividend + divisor - 1) / divisor; --} -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) --{ -- struct cgv_node *cgc_a, *cgc_b; -- -- cgc_a = container_of(a, struct cgv_node, rb_node); -- cgc_b = container_of(b, struct cgv_node, rb_node); -- -- return cgc_a->cvtime < cgc_b->cvtime; --} -- --static struct fcg_cpu_ctx *find_cpu_ctx(void) --{ -- struct fcg_cpu_ctx *cpuc; -- u32 idx = 0; -- -- cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx); -- if (!cpuc) { -- scx_bpf_error("cpu_ctx lookup failed"); -- return NULL; -- } -- return cpuc; --} -- --static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id); -- return NULL; -- } -- return cgc; --} -- --static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgrp = bpf_cgroup_ancestor(cgrp, level); -- if (!cgrp) { -- scx_bpf_error("ancestor cgroup lookup failed"); -- return NULL; -- } -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- scx_bpf_error("ancestor cgrp_ctx lookup failed"); -- bpf_cgroup_release(cgrp); -- return cgc; --} -- --static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- int level; -- -- if (!cgc->nr_active) { -- stat_inc(FCG_STAT_HWT_SKIP); -- return; -- } -- -- if (cgc->hweight_gen == hweight_gen) { -- stat_inc(FCG_STAT_HWT_CACHE); -- return; -- } -- -- stat_inc(FCG_STAT_HWT_UPDATES); -- bpf_for(level, 0, cgrp->level + 1) { -- struct fcg_cgrp_ctx *cgc; -- bool is_active; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- -- if (!level) { -- cgc->hweight = FCG_HWEIGHT_ONE; -- cgc->hweight_gen = hweight_gen; -- } else { -- struct fcg_cgrp_ctx *pcgc; -- -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- -- /* -- * We can be oppotunistic here and not grab the -- * cgv_tree_lock and deal with the occasional races. -- * However, hweight updates are already cached and -- * relatively low-frequency. Let's just do the -- * straightforward thing. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- is_active = cgc->nr_active; -- if (is_active) { -- cgc->hweight_gen = pcgc->hweight_gen; -- cgc->hweight = -- div_round_up(pcgc->hweight * cgc->weight, -- pcgc->child_weight_sum); -- } -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!is_active) { -- stat_inc(FCG_STAT_HWT_RACE); -- break; -- } -- } -- } --} -- --static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) --{ -- u64 delta, cvtime, max_budget; -- -- /* -- * A node which is on the rbtree can't be pointed to from elsewhere yet -- * and thus can't be updated and repositioned. Instead, we collect the -- * vtime deltas separately and apply it asynchronously here. -- */ -- delta = cgc->cvtime_delta; -- __sync_fetch_and_sub(&cgc->cvtime_delta, delta); -- cvtime = cgv_node->cvtime + delta; -- -- /* -- * Allow a cgroup to carry the maximum budget proportional to its -- * hweight such that a full-hweight cgroup can immediately take up half -- * of the CPUs at the most while staying at the front of the rbtree. -- */ -- max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / -- (2 * FCG_HWEIGHT_ONE); -- if (vtime_before(cvtime, cvtime_now - max_budget)) -- cvtime = cvtime_now - max_budget; -- -- cgv_node->cvtime = cvtime; --} -- --static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- u64 cgid = cgrp->kn->id; -- -- /* paired with cmpxchg in try_pick_next_cgroup() */ -- if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { -- stat_inc(FCG_STAT_ENQ_SKIP); -- return; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid); -- return; -- } -- -- /* NULL if the node is already on the rbtree */ -- cgv_node = bpf_kptr_xchg(&stash->node, NULL); -- if (!cgv_node) { -- stat_inc(FCG_STAT_ENQ_RACE); -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); --} -- --void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * If select_cpu_dfl() is recommending local enqueue, the target CPU is -- * idle. Follow it and charge the cgroup later in fcg_stopping() after -- * the fact. Use the same mechanism to deal with tasks with custom -- * affinities so that we don't have to worry about per-cgroup dq's -- * containing tasks that can't be executed from some CPUs. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) { -- /* -- * Tell fcg_stopping() that this bypassed the regular scheduling -- * path and should be force charged to the cgroup. 0 is used to -- * indicate that the task isn't bypassing, so if the current -- * runtime is 0, go back by one nanosecond. -- */ -- taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1; -- -- /* -- * The global dq is deprioritized as we don't want to let tasks -- * to boost themselves by constraining its cpumask. The -- * deprioritization is rather severe, so let's not apply that to -- * per-cpu kernel threads. This is ham-fisted. We probably wanna -- * implement per-cgroup fallback dq's instead so that we have -- * more control over when tasks with custom cpumask get issued. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || -- (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) { -- stat_inc(FCG_STAT_LOCAL); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- } else { -- stat_inc(FCG_STAT_GLOBAL); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } -- return; -- } -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- goto out_release; -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 tvtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) -- tvtime = cgc->tvtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, -- tvtime, enq_flags); -- } -- -- cgrp_enqueued(cgrp, cgc); --out_release: -- bpf_cgroup_release(cgrp); --} -- --/* -- * Walk the cgroup tree to update the active weight sums as tasks wake up and -- * sleep. The weight sums are used as the base when calculating the proportion a -- * given cgroup or task is entitled to at each level. -- */ --static void update_active_weight_sums(struct cgroup *cgrp, bool runnable) --{ -- struct fcg_cgrp_ctx *cgc; -- bool updated = false; -- int idx; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- /* -- * In most cases, a hot cgroup would have multiple threads going to -- * sleep and waking up while the whole cgroup stays active. In leaf -- * cgroups, ->nr_runnable which is updated with __sync operations gates -- * ->nr_active updates, so that we don't have to grab the cgv_tree_lock -- * repeatedly for a busy cgroup which is staying active. -- */ -- if (runnable) { -- if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_ACT); -- } else { -- if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_DEACT); -- } -- -- /* -- * If @cgrp is becoming runnable, its hweight should be refreshed after -- * it's added to the weight tree so that enqueue has the up-to-date -- * value. If @cgrp is becoming quiescent, the hweight should be -- * refreshed before it's removed from the weight tree so that the usage -- * charging which happens afterwards has access to the latest value. -- */ -- if (!runnable) -- cgrp_refresh_hweight(cgrp, cgc); -- -- /* propagate upwards */ -- bpf_for(idx, 0, cgrp->level) { -- int level = cgrp->level - idx; -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- bool propagate = false; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- if (level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- } -- -- /* -- * We need the propagation protected by a lock to synchronize -- * against weight changes. There's no reason to drop the lock at -- * each level but bpf_spin_lock() doesn't want any function -- * calls while locked. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- -- if (runnable) { -- if (!cgc->nr_active++) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum += cgc->weight; -- } -- } -- } else { -- if (!--cgc->nr_active) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum -= cgc->weight; -- } -- } -- } -- -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!propagate) -- break; -- } -- -- if (updated) -- __sync_fetch_and_add(&hweight_gen, 1); -- -- if (runnable) -- cgrp_refresh_hweight(cgrp, cgc); --} -- --void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, true); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) --{ -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- if (fifo_sched) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- /* -- * @cgc->tvtime_now always progresses forward as tasks start -- * executing. The test and update can be performed concurrently -- * from multiple CPUs and thus racy. Any error should be -- * contained and temporary. Let's just live with it. -- */ -- if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) -- cgc->tvtime_now = p->scx.dsq_vtime; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- /* scale the execution time by the inverse of the weight and charge */ -- if (!fifo_sched) -- p->scx.dsq_vtime += -- (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- if (!taskc->bypassed_at) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- __sync_fetch_and_add(&cgc->cvtime_delta, -- p->se.sum_exec_runtime - taskc->bypassed_at); -- taskc->bypassed_at = 0; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, false); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) --{ -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- if (cgrp->level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1); -- if (!pcgc) -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- if (pcgc && cgc->nr_active) -- pcgc->child_weight_sum += (s64)weight - cgc->weight; -- cgc->weight = weight; -- bpf_spin_unlock(&cgv_tree_lock); --} -- --static bool try_pick_next_cgroup(u64 *cgidp) --{ -- struct bpf_rb_node *rb_node; -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 cgid; -- -- /* pop the front cgroup and wind cvtime_now accordingly */ -- bpf_spin_lock(&cgv_tree_lock); -- -- rb_node = bpf_rbtree_first(&cgv_tree); -- if (!rb_node) { -- bpf_spin_unlock(&cgv_tree_lock); -- stat_inc(FCG_STAT_PNC_NO_CGRP); -- *cgidp = 0; -- return true; -- } -- -- rb_node = bpf_rbtree_remove(&cgv_tree, rb_node); -- bpf_spin_unlock(&cgv_tree_lock); -- -- cgv_node = container_of(rb_node, struct cgv_node, rb_node); -- cgid = cgv_node->cgid; -- -- if (vtime_before(cvtime_now, cgv_node->cvtime)) -- cvtime_now = cgv_node->cvtime; -- -- /* -- * If lookup fails, the cgroup's gone. Free and move on. See -- * fcg_cgroup_exit(). -- */ -- cgrp = bpf_cgroup_from_id(cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- if (!scx_bpf_consume(cgid)) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_EMPTY); -- goto out_stash; -- } -- -- /* -- * Successfully consumed from the cgroup. This will be our current -- * cgroup for the new slice. Refresh its hweight. -- */ -- cgrp_refresh_hweight(cgrp, cgc); -- -- bpf_cgroup_release(cgrp); -- -- /* -- * As the cgroup may have more tasks, add it back to the rbtree. Note -- * that here we charge the full slice upfront and then exact later -- * according to the actual consumption. This prevents lowpri thundering -- * herd from saturating the machine. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- -- *cgidp = cgid; -- stat_inc(FCG_STAT_PNC_NEXT); -- return true; -- --out_stash: -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- /* -- * Paired with cmpxchg in cgrp_enqueued(). If they see the following -- * transition, they'll enqueue the cgroup. If they are earlier, we'll -- * see their task in the dq below and requeue the cgroup. -- */ -- __sync_val_compare_and_swap(&cgc->queued, 1, 0); -- -- if (scx_bpf_dsq_nr_queued(cgid)) { -- bpf_spin_lock(&cgv_tree_lock); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- goto out_free; -- } -- } -- -- return false; -- --out_free: -- bpf_obj_drop(cgv_node); -- return false; --} -- --void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) --{ -- struct fcg_cpu_ctx *cpuc; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 now = bpf_ktime_get_ns(); -- -- cpuc = find_cpu_ctx(); -- if (!cpuc) -- return; -- -- if (!cpuc->cur_cgid) -- goto pick_next_cgroup; -- -- if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { -- if (scx_bpf_consume(cpuc->cur_cgid)) { -- stat_inc(FCG_STAT_CNS_KEEP); -- return; -- } -- stat_inc(FCG_STAT_CNS_EMPTY); -- } else { -- stat_inc(FCG_STAT_CNS_EXPIRE); -- } -- -- /* -- * The current cgroup is expiring. It was already charged a full slice. -- * Calculate the actual usage and accumulate the delta. -- */ -- cgrp = bpf_cgroup_from_id(cpuc->cur_cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_CNS_GONE); -- goto pick_next_cgroup; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (cgc) { -- /* -- * We want to update the vtime delta and then look for the next -- * cgroup to execute but the latter needs to be done in a loop -- * and we can't keep the lock held. Oh well... -- */ -- bpf_spin_lock(&cgv_tree_lock); -- __sync_fetch_and_add(&cgc->cvtime_delta, -- (cpuc->cur_at + cgrp_slice_ns - now) * -- FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- stat_inc(FCG_STAT_CNS_GONE); -- } -- -- bpf_cgroup_release(cgrp); -- --pick_next_cgroup: -- cpuc->cur_at = now; -- -- if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { -- cpuc->cur_cgid = 0; -- return; -- } -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_pick_next_cgroup(&cpuc->cur_cgid)) -- break; -- } --} -- --s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct fcg_task_ctx *taskc; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- taskc = bpf_task_storage_get(&task_ctx, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!taskc) -- return -ENOMEM; -- -- taskc->bypassed_at = 0; -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp, -- struct scx_cgroup_init_args *args) --{ -- struct fcg_cgrp_ctx *cgc; -- struct cgv_node *cgv_node; -- struct cgv_node_stash empty_stash = {}, *stash; -- u64 cgid = cgrp->kn->id; -- int ret; -- -- /* -- * Technically incorrect as cgroup ID is full 64bit while dq ID is -- * 63bit. Should not be a problem in practice and easy to spot in the -- * unlikely case that it breaks. -- */ -- ret = scx_bpf_create_dsq(cgid, -1); -- if (ret) -- return ret; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!cgc) { -- ret = -ENOMEM; -- goto err_destroy_dsq; -- } -- -- cgc->weight = args->weight; -- cgc->hweight = FCG_HWEIGHT_ONE; -- -- ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash, -- BPF_NOEXIST); -- if (ret) { -- if (ret != -ENOMEM) -- scx_bpf_error("unexpected stash creation error (%d)", -- ret); -- goto err_destroy_dsq; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("unexpected cgv_node stash lookup failure"); -- ret = -ENOENT; -- goto err_destroy_dsq; -- } -- -- cgv_node = bpf_obj_new(struct cgv_node); -- if (!cgv_node) { -- ret = -ENOMEM; -- goto err_del_cgv_node; -- } -- -- cgv_node->cgid = cgid; -- cgv_node->cvtime = cvtime_now; -- -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- ret = -EBUSY; -- goto err_drop; -- } -- -- return 0; -- --err_drop: -- bpf_obj_drop(cgv_node); --err_del_cgv_node: -- bpf_map_delete_elem(&cgv_node_stash, &cgid); --err_destroy_dsq: -- scx_bpf_destroy_dsq(cgid); -- return ret; --} -- --void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- -- /* -- * For now, there's no way find and remove the cgv_node if it's on the -- * cgv_tree. Let's drain them in the dispatch path as they get popped -- * off the front of the tree. -- */ -- bpf_map_delete_elem(&cgv_node_stash, &cgid); -- scx_bpf_destroy_dsq(cgid); --} -- --s32 BPF_STRUCT_OPS(fcg_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops flatcg_ops = { -- .enqueue = (void *)fcg_enqueue, -- .dispatch = (void *)fcg_dispatch, -- .runnable = (void *)fcg_runnable, -- .running = (void *)fcg_running, -- .stopping = (void *)fcg_stopping, -- .quiescent = (void *)fcg_quiescent, -- .prep_enable = (void *)fcg_prep_enable, -- .cgroup_set_weight = (void *)fcg_cgroup_set_weight, -- .cgroup_init = (void *)fcg_cgroup_init, -- .cgroup_exit = (void *)fcg_cgroup_exit, -- .init = (void *)fcg_init, -- .exit = (void *)fcg_exit, -- .flags = SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING, -- .name = "flatcg", --}; -diff --git b/tools/sched_ext/scx_example_flatcg.c a/tools/sched_ext/scx_example_flatcg.c -deleted file mode 100644 -index f9c8a5b84a70..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.c -+++ /dev/null -@@ -1,232 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2023 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2023 Tejun Heo -- * Copyright (c) 2023 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_flatcg.h" --#include "scx_example_flatcg.skel.h" -- --#ifndef FILEID_KERNFS --#define FILEID_KERNFS 0xfe --#endif -- --const char help_fmt[] = --"A flattened cgroup hierarchy sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -i INTERVAL Report interval\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --static float read_cpu_util(__u64 *last_sum, __u64 *last_idle) --{ -- FILE *fp; -- char buf[4096]; -- char *line, *cur = NULL, *tok; -- __u64 sum = 0, idle = 0; -- __u64 delta_sum, delta_idle; -- int idx; -- -- fp = fopen("/proc/stat", "r"); -- if (!fp) { -- perror("fopen(\"/proc/stat\")"); -- return 0.0; -- } -- -- if (!fgets(buf, sizeof(buf), fp)) { -- perror("fgets(\"/proc/stat\")"); -- fclose(fp); -- return 0.0; -- } -- fclose(fp); -- -- line = buf; -- for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) { -- char *endp = NULL; -- __u64 v; -- -- if (idx == 0) { -- line = NULL; -- continue; -- } -- v = strtoull(tok, &endp, 0); -- if (!endp || *endp != '\0') { -- fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n", -- idx, tok); -- continue; -- } -- sum += v; -- if (idx == 4) -- idle = v; -- } -- -- delta_sum = sum - *last_sum; -- delta_idle = idle - *last_idle; -- *last_sum = sum; -- *last_idle = idle; -- -- return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0; --} -- --static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats) --{ -- __u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus]; -- __u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); -- -- for (idx = 0; idx < FCG_NR_STATS; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_flatcg *skel; -- struct bpf_link *link; -- struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 }; -- bool dump_cgrps = false; -- __u64 last_cpu_sum = 0, last_cpu_idle = 0; -- __u64 last_stats[FCG_NR_STATS] = {}; -- unsigned long seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_flatcg__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open: %s\n", strerror(errno)); -- return 1; -- } -- -- skel->rodata->nr_cpus = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) { -- double v; -- -- switch (opt) { -- case 's': -- v = strtod(optarg, NULL); -- skel->rodata->cgrp_slice_ns = v * 1000; -- break; -- case 'i': -- v = strtod(optarg, NULL); -- intv_ts.tv_sec = v; -- intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000; -- break; -- case 'd': -- dump_cgrps = true; -- break; -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- case 'h': -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d", -- (double)skel->rodata->cgrp_slice_ns / 1000000.0, -- (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0, -- dump_cgrps); -- -- if (scx_example_flatcg__load(skel)) { -- fprintf(stderr, "Failed to load: %s\n", strerror(errno)); -- return 1; -- } -- -- link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops); -- if (!link) { -- fprintf(stderr, "Failed to attach_struct_ops: %s\n", -- strerror(errno)); -- return 1; -- } -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- __u64 acc_stats[FCG_NR_STATS]; -- __u64 stats[FCG_NR_STATS]; -- float cpu_util; -- int i; -- -- cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle); -- -- fcg_read_stats(skel, acc_stats); -- for (i = 0; i < FCG_NR_STATS; i++) -- stats[i] = acc_stats[i] - last_stats[i]; -- -- memcpy(last_stats, acc_stats, sizeof(acc_stats)); -- -- printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n", -- seq++, cpu_util * 100.0, skel->data->hweight_gen); -- printf(" act:%6llu deact:%6llu local:%6llu global:%6llu\n", -- stats[FCG_STAT_ACT], -- stats[FCG_STAT_DEACT], -- stats[FCG_STAT_LOCAL], -- stats[FCG_STAT_GLOBAL]); -- printf("HWT skip:%6llu race:%6llu cache:%6llu update:%6llu\n", -- stats[FCG_STAT_HWT_SKIP], -- stats[FCG_STAT_HWT_RACE], -- stats[FCG_STAT_HWT_CACHE], -- stats[FCG_STAT_HWT_UPDATES]); -- printf("ENQ skip:%6llu race:%6llu\n", -- stats[FCG_STAT_ENQ_SKIP], -- stats[FCG_STAT_ENQ_RACE]); -- printf("CNS keep:%6llu expire:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_CNS_KEEP], -- stats[FCG_STAT_CNS_EXPIRE], -- stats[FCG_STAT_CNS_EMPTY], -- stats[FCG_STAT_CNS_GONE]); -- printf("PNC nocgrp:%6llu next:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_PNC_NO_CGRP], -- stats[FCG_STAT_PNC_NEXT], -- stats[FCG_STAT_PNC_EMPTY], -- stats[FCG_STAT_PNC_GONE]); -- printf("BAD remove:%6llu\n", -- acc_stats[FCG_STAT_BAD_REMOVAL]); -- -- nanosleep(&intv_ts, NULL); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_flatcg__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_flatcg.h a/tools/sched_ext/scx_example_flatcg.h -deleted file mode 100644 -index 490758ed41f0..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.h -+++ /dev/null -@@ -1,49 +0,0 @@ --#ifndef __SCX_EXAMPLE_FLATCG_H --#define __SCX_EXAMPLE_FLATCG_H -- --enum { -- FCG_HWEIGHT_ONE = 1LLU << 16, --}; -- --enum fcg_stat_idx { -- FCG_STAT_ACT, -- FCG_STAT_DEACT, -- FCG_STAT_LOCAL, -- FCG_STAT_GLOBAL, -- -- FCG_STAT_HWT_UPDATES, -- FCG_STAT_HWT_CACHE, -- FCG_STAT_HWT_SKIP, -- FCG_STAT_HWT_RACE, -- -- FCG_STAT_ENQ_SKIP, -- FCG_STAT_ENQ_RACE, -- -- FCG_STAT_CNS_KEEP, -- FCG_STAT_CNS_EXPIRE, -- FCG_STAT_CNS_EMPTY, -- FCG_STAT_CNS_GONE, -- -- FCG_STAT_PNC_NO_CGRP, -- FCG_STAT_PNC_NEXT, -- FCG_STAT_PNC_EMPTY, -- FCG_STAT_PNC_GONE, -- -- FCG_STAT_BAD_REMOVAL, -- -- FCG_NR_STATS, --}; -- --struct fcg_cgrp_ctx { -- u32 nr_active; -- u32 nr_runnable; -- u32 queued; -- u32 weight; -- u32 hweight; -- u64 child_weight_sum; -- u64 hweight_gen; -- s64 cvtime_delta; -- u64 tvtime_now; --}; -- --#endif /* __SCX_EXAMPLE_FLATCG_H */ -diff --git b/tools/sched_ext/scx_example_pair.bpf.c a/tools/sched_ext/scx_example_pair.bpf.c -deleted file mode 100644 -index 279efe58b777..000000000000 ---- b/tools/sched_ext/scx_example_pair.bpf.c -+++ /dev/null -@@ -1,627 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext core-scheduler which always makes every sibling CPU pair -- * execute from the same CPU cgroup. -- * -- * This scheduler is a minimal implementation and would need some form of -- * priority handling both inside each cgroup and across the cgroups to be -- * practically useful. -- * -- * Each CPU in the system is paired with exactly one other CPU, according to a -- * "stride" value that can be specified when the BPF scheduler program is first -- * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee -- * that they will only ever schedule tasks that belong to the same CPU cgroup. -- * -- * Scheduler Initialization -- * ------------------------ -- * -- * The scheduler BPF program is first initialized from user space, before it is -- * enabled. During this initialization process, each CPU on the system is -- * assigned several values that are constant throughout its runtime: -- * -- * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling -- * decisions. Paired CPUs always schedule tasks from the same -- * CPU cgroup, and synchronize with each other to guarantee -- * that this constraint is not violated. -- * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access -- * a struct pair_ctx object that is shared between the pair. -- * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the -- * pair. Each struct pair_ctx has an active_mask field, -- * which is a bitmap used to indicate whether each core -- * in the pair currently has an actively running task. -- * This index specifies which entry in the bitmap corresponds -- * to each CPU in the pair. -- * -- * During this initialization, the CPUs are paired according to a "stride" that -- * may be specified when invoking the user space program that initializes and -- * loads the scheduler. By default, the stride is 1/2 the total number of CPUs. -- * -- * Tasks and cgroups -- * ----------------- -- * -- * Every cgroup in the system is registered with the scheduler using the -- * pair_cgroup_init() callback, and every task in the system is associated with -- * exactly one cgroup. At a high level, the idea with the pair scheduler is to -- * always schedule tasks from the same cgroup within a given CPU pair. When a -- * task is enqueued (i.e. passed to the pair_enqueue() callback function), its -- * cgroup ID is read from its task struct, and then a corresponding queue map -- * is used to FIFO-enqueue the task for that cgroup. -- * -- * If you look through the implementation of the scheduler, you'll notice that -- * there is quite a bit of complexity involved with looking up the per-cgroup -- * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash -- * BPF hash map that is used to map a cgroup ID to a globally unique ID that's -- * allocated in the BPF program. This is done because we use separate maps to -- * store the FIFO queue of tasks, and the length of that map, per cgroup. This -- * complexity is only present because of current deficiencies in BPF that will -- * soon be addressed. The main point to keep in mind is that newly enqueued -- * tasks are added to their cgroup's FIFO queue. -- * -- * Dispatching tasks -- * ----------------- -- * -- * This section will describe how enqueued tasks are dispatched and scheduled. -- * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is -- * as follows: -- * -- * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is -- * the structure that's used to synchronize amongst the two pair CPUs in their -- * scheduling decisions. After any of the following events have occurred: -- * -- * - The cgroup's slice run has expired, or -- * - The cgroup becomes empty, or -- * - Either CPU in the pair is preempted by a higher priority scheduling class -- * -- * The cgroup transitions to the draining state and stops executing new tasks -- * from the cgroup. -- * -- * 2. If the pair is still executing a task, mark the pair_ctx as draining, and -- * wait for the pair CPU to be preempted. -- * -- * 3. Otherwise, if the pair CPU is not running a task, we can move onto -- * scheduling new tasks. Pop the next cgroup id from the top_q queue. -- * -- * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it. -- * -- * Note again that this scheduling behavior is simple, but the implementation -- * is complex mostly because this it hits several BPF shortcomings and has to -- * work around in often awkward ways. Most of the shortcomings are expected to -- * be resolved in the near future which should allow greatly simplifying this -- * scheduler. -- * -- * Dealing with preemption -- * ----------------------- -- * -- * SCX is the lowest priority sched_class, and could be preempted by them at -- * any time. To address this, the scheduler implements pair_cpu_release() and -- * pair_cpu_acquire() callbacks which are invoked by the core scheduler when -- * the scheduler loses and gains control of the CPU respectively. -- * -- * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and -- * then invoke: -- * -- * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- * -- * This preempts the pair CPU, and waits until it has re-entered the scheduler -- * before returning. This is necessary to ensure that the higher priority -- * sched_class that preempted our scheduler does not schedule a task -- * concurrently with our pair CPU. -- * -- * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption -- * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable -- * pair scheduling. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include "scx_example_pair.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; -- --/* !0 for veristat, set during init */ --const volatile u32 nr_cpu_ids = 64; -- --/* a pair of CPUs stay on a cgroup for this duration */ --const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL; -- --/* cpu ID -> pair cpu ID */ --const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 }; -- --/* cpu ID -> pair_id */ --const volatile u32 pair_id[MAX_CPUS]; -- --/* CPU ID -> CPU # in the pair (0 or 1) */ --const volatile u32 in_pair_idx[MAX_CPUS]; -- --struct pair_ctx { -- struct bpf_spin_lock lock; -- -- /* the cgroup the pair is currently executing */ -- u64 cgid; -- -- /* the pair started executing the current cgroup at */ -- u64 started_at; -- -- /* whether the current cgroup is draining */ -- bool draining; -- -- /* the CPUs that are currently active on the cgroup */ -- u32 active_mask; -- -- /* -- * the CPUs that are currently preempted and running tasks in a -- * different scheduler. -- */ -- u32 preempted_mask; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, MAX_CPUS / 2); -- __type(key, u32); -- __type(value, struct pair_ctx); --} pair_ctx SEC(".maps"); -- --/* queue of cgrp_q's possibly with tasks on them */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- /* -- * Because it's difficult to build strong synchronization encompassing -- * multiple non-trivial operations in BPF, this queue is managed in an -- * opportunistic way so that we guarantee that a cgroup w/ active tasks -- * is always on it but possibly multiple times. Once we have more robust -- * synchronization constructs and e.g. linked list, we should be able to -- * do this in a prettier way but for now just size it big enough. -- */ -- __uint(max_entries, 4 * MAX_CGRPS); -- __type(value, u64); --} top_q SEC(".maps"); -- --/* per-cgroup q which FIFOs the tasks from the cgroup */ --struct cgrp_q { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, MAX_QUEUED); -- __type(value, u32); --}; -- --/* -- * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local -- * storage; however, a cgroup local storage can only be accessed from the BPF -- * progs attached to the cgroup. For now, work around by allocating array of -- * cgrp_q's and then allocating per-cgroup indices. -- * -- * Another caveat: It's difficult to populate a large array of maps statically -- * or from BPF. Initialize it from userland. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, MAX_CGRPS); -- __type(key, s32); -- __array(values, struct cgrp_q); --} cgrp_q_arr SEC(".maps"); -- --static u64 cgrp_q_len[MAX_CGRPS]; -- --/* -- * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be -- * useful to have as a map type. -- */ --static u32 cgrp_q_idx_cursor; --static u64 cgrp_q_idx_busy[MAX_CGRPS]; -- --/* -- * All added up, the following is what we do: -- * -- * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking -- * for a free ID. If not found, fail cgroup creation with -EBUSY. -- * -- * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following -- * cgrp_q_idx_hash. -- * -- * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from -- * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr. -- * -- * This is sadly complicated for something pretty simple. Hopefully, we should -- * be able to simplify in the future. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, MAX_CGRPS); -- __uint(key_size, sizeof(u64)); /* cgrp ID */ -- __uint(value_size, sizeof(s32)); /* cgrp_q idx */ --} cgrp_q_idx_hash SEC(".maps"); -- --/* statistics */ --u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions; --u64 nr_exps, nr_exp_waits, nr_exp_empty; --u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty; -- --struct user_exit_info uei; -- --static bool time_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- struct cgrp_q *cgq; -- s32 pid = p->pid; -- u64 cgid; -- u32 *q_idx; -- u64 *cgq_len; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- cgrp = scx_bpf_task_cgroup(p); -- cgid = cgrp->kn->id; -- bpf_cgroup_release(cgrp); -- -- /* find the cgroup's q and push @p into it */ -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!q_idx) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return; -- } -- -- cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx); -- if (!cgq) { -- scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]", -- cgid, *q_idx); -- return; -- } -- -- if (bpf_map_push_elem(cgq, &pid, 0)) { -- scx_bpf_error("cgroup[%llu] queue overflow", cgid); -- return; -- } -- -- /* bump q len, if going 0 -> 1, queue cgroup into the top_q */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len) { -- scx_bpf_error("MEMBER_VTPR malfunction"); -- return; -- } -- -- if (!__sync_fetch_and_add(cgq_len, 1) && -- bpf_map_push_elem(&top_q, &cgid, 0)) { -- scx_bpf_error("top_q overflow"); -- return; -- } --} -- --static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) --{ -- u32 *vptr; -- -- vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *pairc = bpf_map_lookup_elem(&pair_ctx, vptr); -- if (!(*pairc)) -- return -EINVAL; -- -- vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *mask = 1U << *vptr; -- -- return 0; --} -- --static int try_dispatch(s32 cpu) --{ -- struct pair_ctx *pairc; -- struct bpf_map *cgq_map; -- struct task_struct *p; -- u64 now = bpf_ktime_get_ns(); -- bool kick_pair = false; -- bool expired, pair_preempted; -- u32 *vptr, in_pair_mask; -- s32 pid, q_idx; -- u64 cgid; -- int ret; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) { -- scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]", -- cpu); -- return -ENOENT; -- } -- -- bpf_spin_lock(&pairc->lock); -- pairc->active_mask &= ~in_pair_mask; -- -- expired = time_before(pairc->started_at + pair_batch_dur_ns, now); -- if (expired || pairc->draining) { -- u64 new_cgid = 0; -- -- __sync_fetch_and_add(&nr_exps, 1); -- -- /* -- * We're done with the current cgid. An obvious optimization -- * would be not draining if the next cgroup is the current one. -- * For now, be dumb and always expire. -- */ -- pairc->draining = true; -- -- pair_preempted = pairc->preempted_mask; -- if (pairc->active_mask || pair_preempted) { -- /* -- * The other CPU is still active, or is no longer under -- * our control due to e.g. being preempted by a higher -- * priority sched_class. We want to wait until this -- * cgroup expires, or until control of our pair CPU has -- * been returned to us. -- * -- * If the pair controls its CPU, and the time already -- * expired, kick. When the other CPU arrives at -- * dispatch and clears its active mask, it'll push the -- * pair to the next cgroup and kick this CPU. -- */ -- __sync_fetch_and_add(&nr_exp_waits, 1); -- bpf_spin_unlock(&pairc->lock); -- if (expired && !pair_preempted) -- kick_pair = true; -- goto out_maybe_kick; -- } -- -- bpf_spin_unlock(&pairc->lock); -- -- /* -- * Pick the next cgroup. It'd be easier / cleaner to not drop -- * pairc->lock and use stronger synchronization here especially -- * given that we'll be switching cgroups significantly less -- * frequently than tasks. Unfortunately, bpf_spin_lock can't -- * really protect anything non-trivial. Let's do opportunistic -- * operations instead. -- */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u32 *q_idx; -- u64 *cgq_len; -- -- if (bpf_map_pop_elem(&top_q, &new_cgid)) { -- /* no active cgroup, go idle */ -- __sync_fetch_and_add(&nr_exp_empty, 1); -- return 0; -- } -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid); -- if (!q_idx) -- continue; -- -- /* -- * This is the only place where empty cgroups are taken -- * off the top_q. -- */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len || !*cgq_len) -- continue; -- -- /* -- * If it has any tasks, requeue as we may race and not -- * execute it. -- */ -- bpf_map_push_elem(&top_q, &new_cgid, 0); -- break; -- } -- -- bpf_spin_lock(&pairc->lock); -- -- /* -- * The other CPU may already have started on a new cgroup while -- * we dropped the lock. Make sure that we're still draining and -- * start on the new cgroup. -- */ -- if (pairc->draining && !pairc->active_mask) { -- __sync_fetch_and_add(&nr_cgrp_next, 1); -- pairc->cgid = new_cgid; -- pairc->started_at = now; -- pairc->draining = false; -- kick_pair = true; -- } else { -- __sync_fetch_and_add(&nr_cgrp_coll, 1); -- } -- } -- -- cgid = pairc->cgid; -- pairc->active_mask |= in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- -- /* again, it'd be better to do all these with the lock held, oh well */ -- vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!vptr) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return -ENOENT; -- } -- q_idx = *vptr; -- -- /* claim one task from cgrp_q w/ q_idx */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u64 *cgq_len, len; -- -- cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]); -- if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) { -- /* the cgroup must be empty, expire and repeat */ -- __sync_fetch_and_add(&nr_cgrp_empty, 1); -- bpf_spin_lock(&pairc->lock); -- pairc->draining = true; -- pairc->active_mask &= ~in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- return -EAGAIN; -- } -- -- if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len) -- continue; -- -- break; -- } -- -- cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx); -- if (!cgq_map) { -- scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- if (bpf_map_pop_elem(cgq_map, &pid)) { -- scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- p = bpf_task_from_pid(pid); -- if (p) { -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } else { -- /* we don't handle dequeues, retry on lost tasks */ -- __sync_fetch_and_add(&nr_missing, 1); -- return -EAGAIN; -- } -- --out_maybe_kick: -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } -- return 0; --} -- --void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) --{ -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_dispatch(cpu) != -EAGAIN) -- break; -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask &= ~in_pair_mask; -- /* Kick the pair CPU, unless it was also preempted. */ -- kick_pair = !pairc->preempted_mask; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask |= in_pair_mask; -- pairc->active_mask &= ~in_pair_mask; -- /* Kick the pair CPU if it's still running. */ -- kick_pair = pairc->active_mask; -- pairc->draining = true; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- } -- } -- __sync_fetch_and_add(&nr_preemptions, 1); --} -- --s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 i, q_idx; -- -- bpf_for(i, 0, MAX_CGRPS) { -- q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS; -- if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1)) -- break; -- } -- if (i == MAX_CGRPS) -- return -EBUSY; -- -- if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]); -- if (busy) -- *busy = 0; -- return -EBUSY; -- } -- -- return 0; --} -- --void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 *q_idx; -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (q_idx) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]); -- if (busy) -- *busy = 0; -- bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid); -- } --} -- --s32 BPF_STRUCT_OPS(pair_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops pair_ops = { -- .enqueue = (void *)pair_enqueue, -- .dispatch = (void *)pair_dispatch, -- .cpu_acquire = (void *)pair_cpu_acquire, -- .cpu_release = (void *)pair_cpu_release, -- .cgroup_init = (void *)pair_cgroup_init, -- .cgroup_exit = (void *)pair_cgroup_exit, -- .init = (void *)pair_init, -- .exit = (void *)pair_exit, -- .name = "pair", --}; -diff --git b/tools/sched_ext/scx_example_pair.c a/tools/sched_ext/scx_example_pair.c -deleted file mode 100644 -index 18e032bbc173..000000000000 ---- b/tools/sched_ext/scx_example_pair.c -+++ /dev/null -@@ -1,143 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_pair.h" --#include "scx_example_pair.skel.h" -- --const char help_fmt[] = --"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n" --"execute from the same CPU cgroup.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-S STRIDE] [-p]\n" --"\n" --" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_pair *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 stride, i, opt, outer_fd; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_pair__open(); -- assert(skel); -- -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- /* pair up the earlier half to the latter by default, override with -s */ -- stride = skel->rodata->nr_cpu_ids / 2; -- -- while ((opt = getopt(argc, argv, "S:ph")) != -1) { -- switch (opt) { -- case 'S': -- stride = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- for (i = 0; i < skel->rodata->nr_cpu_ids; i++) { -- if (skel->rodata->pair_cpu[i] < 0) { -- skel->rodata->pair_cpu[i] = i + stride; -- skel->rodata->pair_cpu[i + stride] = i; -- skel->rodata->pair_id[i] = i; -- skel->rodata->pair_id[i + stride] = i; -- skel->rodata->in_pair_idx[i] = 0; -- skel->rodata->in_pair_idx[i + stride] = 1; -- } -- } -- -- assert(!scx_example_pair__load(skel)); -- -- /* -- * Populate the cgrp_q_arr map which is an array containing per-cgroup -- * queues. It'd probably be better to do this from BPF but there are too -- * many to initialize statically and there's no way to dynamically -- * populate from BPF. -- */ -- outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr); -- assert(outer_fd >= 0); -- -- printf("Initializing"); -- for (i = 0; i < MAX_CGRPS; i++) { -- s32 inner_fd; -- -- if (exit_req) -- break; -- -- inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, -- sizeof(u32), MAX_QUEUED, NULL); -- assert(inner_fd >= 0); -- assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY)); -- close(inner_fd); -- -- if (!(i % 10)) -- printf("."); -- fflush(stdout); -- } -- printf("\n"); -- -- /* -- * Fully initialized, attach and run. -- */ -- link = bpf_map__attach_struct_ops(skel->maps.pair_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf(" total:%10lu dispatch:%10lu missing:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_dispatched, -- skel->bss->nr_missing); -- printf(" kicks:%10lu preemptions:%7lu\n", -- skel->bss->nr_kicks, -- skel->bss->nr_preemptions); -- printf(" exp:%10lu exp_wait:%10lu exp_empty:%10lu\n", -- skel->bss->nr_exps, -- skel->bss->nr_exp_waits, -- skel->bss->nr_exp_empty); -- printf("cgnext:%10lu cgcoll:%10lu cgempty:%10lu\n", -- skel->bss->nr_cgrp_next, -- skel->bss->nr_cgrp_coll, -- skel->bss->nr_cgrp_empty); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_pair__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_pair.h a/tools/sched_ext/scx_example_pair.h -deleted file mode 100644 -index f60b824272f7..000000000000 ---- b/tools/sched_ext/scx_example_pair.h -+++ /dev/null -@@ -1,10 +0,0 @@ --#ifndef __SCX_EXAMPLE_PAIR_H --#define __SCX_EXAMPLE_PAIR_H -- --enum { -- MAX_CPUS = 4096, -- MAX_QUEUED = 4096, -- MAX_CGRPS = 4096, --}; -- --#endif /* __SCX_EXAMPLE_PAIR_H */ -diff --git b/tools/sched_ext/scx_example_qmap.bpf.c a/tools/sched_ext/scx_example_qmap.bpf.c -deleted file mode 100644 -index 579ab21ae403..000000000000 ---- b/tools/sched_ext/scx_example_qmap.bpf.c -+++ /dev/null -@@ -1,401 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple five-level FIFO queue scheduler. -- * -- * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets -- * assigned to one depending on its compound weight. Each CPU round robins -- * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from -- * queue0, 2 from queue1, 4 from queue2 and so on. -- * -- * This scheduler demonstrates: -- * -- * - BPF-side queueing using PIDs. -- * - Sleepable per-task storage allocation using ops.prep_enable(). -- * - Using ops.cpu_release() to handle a higher priority scheduling class taking -- * the CPU away. -- * - Core-sched support. -- * -- * This scheduler is primarily for demonstration and testing of sched_ext -- * features and unlikely to be useful for actual workloads. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include -- --char _license[] SEC("license") = "GPL"; -- --const volatile u64 slice_ns = SCX_SLICE_DFL; --const volatile bool switch_partial; --const volatile u32 stall_user_nth; --const volatile u32 stall_kernel_nth; --const volatile u32 dsp_inf_loop_after; --const volatile s32 disallow_tgid; -- --u32 test_error_cnt; -- --struct user_exit_info uei; -- --struct qmap { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, u32); --} queue0 SEC(".maps"), -- queue1 SEC(".maps"), -- queue2 SEC(".maps"), -- queue3 SEC(".maps"), -- queue4 SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, 5); -- __type(key, int); -- __array(values, struct qmap); --} queue_arr SEC(".maps") = { -- .values = { -- [0] = &queue0, -- [1] = &queue1, -- [2] = &queue2, -- [3] = &queue3, -- [4] = &queue4, -- }, --}; -- --/* -- * Per-queue sequence numbers to implement core-sched ordering. -- * -- * Tail seq is assigned to each queued task and incremented. Head seq tracks the -- * sequence number of the latest dispatched task. The distance between the a -- * task's seq and the associated queue's head seq is called the queue distance -- * and used when comparing two tasks for ordering. See qmap_core_sched_before(). -- */ --static u64 core_sched_head_seqs[5]; --static u64 core_sched_tail_seqs[5]; -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local_dsq */ -- u64 core_sched_seq; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --/* Per-cpu dispatch index and remaining count */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(max_entries, 2); -- __type(key, u32); -- __type(value, u64); --} dispatch_idx_cnt SEC(".maps"); -- --/* Statistics */ --unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued; --unsigned long nr_core_sched_execed; -- --s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- struct task_ctx *tctx; -- s32 cpu; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- return cpu; -- -- return prev_cpu; --} -- --static int weight_to_idx(u32 weight) --{ -- /* Coarsely map the compound weight to a FIFO. */ -- if (weight <= 25) -- return 0; -- else if (weight <= 50) -- return 1; -- else if (weight < 200) -- return 2; -- else if (weight < 400) -- return 3; -- else -- return 4; --} -- --void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) --{ -- static u32 user_cnt, kernel_cnt; -- struct task_ctx *tctx; -- u32 pid = p->pid; -- int idx = weight_to_idx(p->scx.weight); -- void *ring; -- -- if (p->flags & PF_KTHREAD) { -- if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth)) -- return; -- } else { -- if (stall_user_nth && !(++user_cnt % stall_user_nth)) -- return; -- } -- -- if (test_error_cnt && !--test_error_cnt) -- scx_bpf_error("test triggering error"); -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * All enqueued tasks must have their core_sched_seq updated for correct -- * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in -- * qmap_ops.flags. -- */ -- tctx->core_sched_seq = core_sched_tail_seqs[idx]++; -- -- /* -- * If qmap_select_cpu() is telling us to or this is the last runnable -- * task on the CPU, enqueue locally. -- */ -- if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) { -- tctx->force_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); -- return; -- } -- -- /* -- * If the task was re-enqueued due to the CPU being preempted by a -- * higher priority scheduling class, just re-enqueue the task directly -- * on the global DSQ. As we want another CPU to pick it up, find and -- * kick an idle CPU. -- */ -- if (enq_flags & SCX_ENQ_REENQ) { -- s32 cpu; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags); -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- return; -- } -- -- ring = bpf_map_lookup_elem(&queue_arr, &idx); -- if (!ring) { -- scx_bpf_error("failed to find ring %d", idx); -- return; -- } -- -- /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */ -- if (bpf_map_push_elem(ring, &pid, 0)) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_enqueued, 1); --} -- --/* -- * The BPF queue map doesn't support removal and sched_ext can handle spurious -- * dispatches. qmap_dequeue() is only used to collect statistics. -- */ --void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags) --{ -- __sync_fetch_and_add(&nr_dequeued, 1); -- if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) -- __sync_fetch_and_add(&nr_core_sched_execed, 1); --} -- --static void update_core_sched_head_seq(struct task_struct *p) --{ -- struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- int idx = weight_to_idx(p->scx.weight); -- -- if (tctx) -- core_sched_head_seqs[idx] = tctx->core_sched_seq; -- else -- scx_bpf_error("task_ctx lookup failed"); --} -- --void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 zero = 0, one = 1; -- u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero); -- u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one); -- void *fifo; -- s32 pid; -- int i; -- -- if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) { -- struct task_struct *p; -- -- /* -- * PID 2 should be kthreadd which should mostly be idle and off -- * the scheduler. Let's keep dispatching it to force the kernel -- * to call this function over and over again. -- */ -- p = bpf_task_from_pid(2); -- if (p) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- if (!idx || !cnt) { -- scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt); -- return; -- } -- -- for (i = 0; i < 5; i++) { -- /* Advance the dispatch cursor and pick the fifo. */ -- if (!*cnt) { -- *idx = (*idx + 1) % 5; -- *cnt = 1 << *idx; -- } -- (*cnt)--; -- -- fifo = bpf_map_lookup_elem(&queue_arr, idx); -- if (!fifo) { -- scx_bpf_error("failed to find ring %llu", *idx); -- return; -- } -- -- /* Dispatch or advance. */ -- if (!bpf_map_pop_elem(fifo, &pid)) { -- struct task_struct *p; -- -- p = bpf_task_from_pid(pid); -- if (p) { -- update_core_sched_head_seq(p); -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- *cnt = 0; -- } --} -- --/* -- * The distance from the head of the queue scaled by the weight of the queue. -- * The lower the number, the older the task and the higher the priority. -- */ --static s64 task_qdist(struct task_struct *p) --{ -- int idx = weight_to_idx(p->scx.weight); -- struct task_ctx *tctx; -- s64 qdist; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return 0; -- } -- -- qdist = tctx->core_sched_seq - core_sched_head_seqs[idx]; -- -- /* -- * As queue index increments, the priority doubles. The queue w/ index 3 -- * is dispatched twice more frequently than 2. Reflect the difference by -- * scaling qdists accordingly. Note that the shift amount needs to be -- * flipped depending on the sign to avoid flipping priority direction. -- */ -- if (qdist >= 0) -- return qdist << (4 - idx); -- else -- return qdist << idx; --} -- --/* -- * This is called to determine the task ordering when core-sched is picking -- * tasks to execute on SMT siblings and should encode about the same ordering as -- * the regular scheduling path. Use the priority-scaled distances from the head -- * of the queues to compare the two tasks which should be consistent with the -- * dispatch path behavior. -- */ --bool BPF_STRUCT_OPS(qmap_core_sched_before, -- struct task_struct *a, struct task_struct *b) --{ -- return task_qdist(a) > task_qdist(b); --} -- --void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- u32 cnt; -- -- /* -- * Called when @cpu is taken by a higher priority scheduling class. This -- * makes @cpu no longer available for executing sched_ext tasks. As we -- * don't want the tasks in @cpu's local dsq to sit there until @cpu -- * becomes available again, re-enqueue them into the global dsq. See -- * %SCX_ENQ_REENQ handling in qmap_enqueue(). -- */ -- cnt = scx_bpf_reenqueue_local(); -- if (cnt) -- __sync_fetch_and_add(&nr_reenqueued, cnt); --} -- --s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (p->tgid == disallow_tgid) -- p->scx.disallow = true; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(qmap_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops qmap_ops = { -- .select_cpu = (void *)qmap_select_cpu, -- .enqueue = (void *)qmap_enqueue, -- .dequeue = (void *)qmap_dequeue, -- .dispatch = (void *)qmap_dispatch, -- .core_sched_before = (void *)qmap_core_sched_before, -- .cpu_release = (void *)qmap_cpu_release, -- .prep_enable = (void *)qmap_prep_enable, -- .init = (void *)qmap_init, -- .exit = (void *)qmap_exit, -- .flags = SCX_OPS_ENQ_LAST, -- .timeout_ms = 5000U, -- .name = "qmap", --}; -diff --git b/tools/sched_ext/scx_example_qmap.c a/tools/sched_ext/scx_example_qmap.c -deleted file mode 100644 -index ccb4814ee61b..000000000000 ---- b/tools/sched_ext/scx_example_qmap.c -+++ /dev/null -@@ -1,107 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_qmap.skel.h" -- --const char help_fmt[] = --"A simple five-level FIFO queue sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" --" -t COUNT Stall every COUNT'th user thread\n" --" -T COUNT Stall every COUNT'th kernel thread\n" --" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n" --" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_qmap *skel; -- struct bpf_link *link; -- int opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_qmap__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) { -- switch (opt) { -- case 's': -- skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; -- break; -- case 'e': -- skel->bss->test_error_cnt = strtoul(optarg, NULL, 0); -- break; -- case 't': -- skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0); -- break; -- case 'T': -- skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0); -- break; -- case 'l': -- skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0); -- break; -- case 'd': -- skel->rodata->disallow_tgid = strtol(optarg, NULL, 0); -- if (skel->rodata->disallow_tgid < 0) -- skel->rodata->disallow_tgid = getpid(); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_qmap__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.qmap_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- long nr_enqueued = skel->bss->nr_enqueued; -- long nr_dispatched = skel->bss->nr_dispatched; -- -- printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n", -- nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, -- skel->bss->nr_reenqueued, skel->bss->nr_dequeued, -- skel->bss->nr_core_sched_execed); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_qmap__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_simple.bpf.c a/tools/sched_ext/scx_example_simple.bpf.c -deleted file mode 100644 -index 4bccca3e2047..000000000000 ---- b/tools/sched_ext/scx_example_simple.bpf.c -+++ /dev/null -@@ -1,128 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple scheduler. -- * -- * By default, it operates as a simple global weighted vtime scheduler and can -- * be switched to FIFO scheduling. It also demonstrates the following niceties. -- * -- * - Statistics tracking how many tasks are queued to local and global dsq's. -- * - Termination notification for userspace. -- * -- * While very simple, this scheduler should work reasonably well on CPUs with a -- * uniform L3 cache topology. While preemption is not implemented, the fact that -- * the scheduling queue is shared across all CPUs means that whatever is at the -- * front of the queue is likely to be executed fairly quickly given enough -- * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads -- * but comes with the usual problems with FIFO scheduling where saturating -- * threads can easily drown out interactive ones. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --static u64 vtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, 2); /* [local, global] */ --} stats SEC(".maps"); -- --static void stat_inc(u32 idx) --{ -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); -- if (cnt_p) -- (*cnt_p)++; --} -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) --{ -- /* -- * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that -- * running @p on its CPU directly shouldn't affect fairness. Just queue -- * it on the local FIFO. -- */ -- if (enq_flags & SCX_ENQ_LOCAL) { -- stat_inc(0); /* count local queueing */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- return; -- } -- -- stat_inc(1); /* count global queueing */ -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) -- vtime = vtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --void BPF_STRUCT_OPS(simple_running, struct task_struct *p) --{ -- if (fifo_sched) -- return; -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(vtime_now, p->scx.dsq_vtime)) -- vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --s32 BPF_STRUCT_OPS(simple_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .running = (void *)simple_running, -- .stopping = (void *)simple_stopping, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", --}; -diff --git b/tools/sched_ext/scx_example_simple.c a/tools/sched_ext/scx_example_simple.c -deleted file mode 100644 -index 486b401f7c95..000000000000 ---- b/tools/sched_ext/scx_example_simple.c -+++ /dev/null -@@ -1,101 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_simple.skel.h" -- --const char help_fmt[] = --"A simple sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-f] [-p]\n" --"\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int simple) --{ -- exit_req = 1; --} -- --static void read_stats(struct scx_example_simple *skel, u64 *stats) --{ -- int nr_cpus = libbpf_num_possible_cpus(); -- u64 cnts[2][nr_cpus]; -- u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * 2); -- -- for (idx = 0; idx < 2; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_simple *skel; -- struct bpf_link *link; -- u32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_simple__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "fph")) != -1) { -- switch (opt) { -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_simple__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.simple_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- u64 stats[2]; -- -- read_stats(skel, stats); -- printf("local=%lu global=%lu\n", stats[0], stats[1]); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_simple__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_userland.bpf.c a/tools/sched_ext/scx_example_userland.bpf.c -deleted file mode 100644 -index a089bc6bbe86..000000000000 ---- b/tools/sched_ext/scx_example_userland.bpf.c -+++ /dev/null -@@ -1,269 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A minimal userland scheduler. -- * -- * In terms of scheduling, this provides two different types of behaviors: -- * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity. -- * All such tasks are direct-dispatched from the kernel, and are never -- * enqueued in user space. -- * 2. A primitive vruntime scheduler that is implemented in user space, for all -- * other tasks. -- * -- * Some parts of this example user space scheduler could be implemented more -- * efficiently using more complex and sophisticated data structures. For -- * example, rather than using BPF_MAP_TYPE_QUEUE's, -- * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between -- * user space and kernel space. Similarly, we use a simple vruntime-sorted list -- * in user space, but an rbtree could be used instead. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include --#include "scx_common.bpf.h" --#include "scx_example_userland_common.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; --const volatile s32 usersched_pid; -- --/* !0 for veristat, set during init */ --const volatile u32 num_possible_cpus = 64; -- --/* Stats that are printed by user space. */ --u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues; -- --struct user_exit_info uei; -- --/* -- * Whether the user space scheduler needs to be scheduled due to a task being -- * enqueued in user space. -- */ --static bool usersched_needed; -- --/* -- * The map containing tasks that are enqueued in user space from the kernel. -- * -- * This map is drained by the user space scheduler. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, struct scx_userland_enqueued_task); --} enqueued SEC(".maps"); -- --/* -- * The map containing tasks that are dispatched to the kernel from user space. -- * -- * Drained by the kernel in userland_dispatch(). -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, s32); --} dispatched SEC(".maps"); -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local DSQ */ --}; -- --/* Map that contains task-local storage. */ --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --static bool is_usersched_task(const struct task_struct *p) --{ -- return p->pid == usersched_pid; --} -- --static bool keep_in_kernel(const struct task_struct *p) --{ -- return p->nr_cpus_allowed < num_possible_cpus; --} -- --static struct task_struct *usersched_task(void) --{ -- struct task_struct *p; -- -- p = bpf_task_from_pid(usersched_pid); -- /* -- * Should never happen -- the usersched task should always be managed -- * by sched_ext. -- */ -- if (!p) { -- scx_bpf_error("Failed to find usersched task %d", usersched_pid); -- /* -- * We should never hit this path, and we error out of the -- * scheduler above just in case, so the scheduler will soon be -- * be evicted regardless. So as to simplify the logic in the -- * caller to not have to check for NULL, return an acquired -- * reference to the current task here rather than NULL. -- */ -- return bpf_task_acquire(bpf_get_current_task_btf()); -- } -- -- return p; --} -- --s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- if (keep_in_kernel(p)) { -- s32 cpu; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to look up task-local storage for %s", p->comm); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- tctx->force_local = true; -- return cpu; -- } -- } -- -- return prev_cpu; --} -- --static void dispatch_user_scheduler(void) --{ -- struct task_struct *p; -- -- usersched_needed = false; -- p = usersched_task(); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); --} -- --static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags) --{ -- struct scx_userland_enqueued_task task; -- -- memset(&task, 0, sizeof(task)); -- task.pid = p->pid; -- task.sum_exec_runtime = p->se.sum_exec_runtime; -- task.weight = p->scx.weight; -- -- if (bpf_map_push_elem(&enqueued, &task, 0)) { -- /* -- * If we fail to enqueue the task in user space, put it -- * directly on the global DSQ. -- */ -- __sync_fetch_and_add(&nr_failed_enqueues, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- __sync_fetch_and_add(&nr_user_enqueues, 1); -- usersched_needed = true; -- } --} -- --void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags) --{ -- if (keep_in_kernel(p)) { -- u64 dsq_id = SCX_DSQ_GLOBAL; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to lookup task ctx for %s", p->comm); -- return; -- } -- -- if (tctx->force_local) -- dsq_id = SCX_DSQ_LOCAL; -- tctx->force_local = false; -- scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags); -- __sync_fetch_and_add(&nr_kernel_enqueues, 1); -- return; -- } else if (!is_usersched_task(p)) { -- enqueue_task_in_user_space(p, enq_flags); -- } --} -- --void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (usersched_needed) -- dispatch_user_scheduler(); -- -- bpf_repeat(4096) { -- s32 pid; -- struct task_struct *p; -- -- if (bpf_map_pop_elem(&dispatched, &pid)) -- break; -- -- /* -- * The task could have exited by the time we get around to -- * dispatching it. Treat this as a normal occurrence, and simply -- * move onto the next iteration. -- */ -- p = bpf_task_from_pid(pid); -- if (!p) -- continue; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } --} -- --s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(userland_init) --{ -- if (num_possible_cpus == 0) { -- scx_bpf_error("User scheduler # CPUs uninitialized (%d)", -- num_possible_cpus); -- return -EINVAL; -- } -- -- if (usersched_pid <= 0) { -- scx_bpf_error("User scheduler pid uninitialized (%d)", -- usersched_pid); -- return -EINVAL; -- } -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops userland_ops = { -- .select_cpu = (void *)userland_select_cpu, -- .enqueue = (void *)userland_enqueue, -- .dispatch = (void *)userland_dispatch, -- .prep_enable = (void *)userland_prep_enable, -- .init = (void *)userland_init, -- .exit = (void *)userland_exit, -- .timeout_ms = 3000, -- .name = "userland", --}; -diff --git b/tools/sched_ext/scx_example_userland.c a/tools/sched_ext/scx_example_userland.c -deleted file mode 100644 -index 4152b1e65fe1..000000000000 ---- b/tools/sched_ext/scx_example_userland.c -+++ /dev/null -@@ -1,402 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext user space scheduler which provides vruntime semantics -- * using a simple ordered-list implementation. -- * -- * Each CPU in the system resides in a single, global domain. This precludes -- * the need to do any load balancing between domains. The scheduler could -- * easily be extended to support multiple domains, with load balancing -- * happening in user space. -- * -- * Any task which has any CPU affinity is scheduled entirely in BPF. This -- * program only schedules tasks which may run on any CPU. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include "user_exit_info.h" --#include "scx_example_userland_common.h" --#include "scx_example_userland.skel.h" -- --const char help_fmt[] = --"A minimal userland sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-b BATCH] [-p]\n" --"\n" --" -b BATCH The number of tasks to batch when dispatching (default: 8)\n" --" -p Don't switch all, switch only tasks on SCHED_EXT policy\n" --" -h Display this help and exit\n"; -- --/* Defined in UAPI */ --#define SCHED_EXT 7 -- --/* Number of tasks to batch when dispatching to user space. */ --static __u32 batch_size = 8; -- --static volatile int exit_req; --static int enqueued_fd, dispatched_fd; -- --static struct scx_example_userland *skel; --static struct bpf_link *ops_link; -- --/* Stats collected in user space. */ --static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches; -- --/* The data structure containing tasks that are enqueued in user space. */ --struct enqueued_task { -- LIST_ENTRY(enqueued_task) entries; -- __u64 sum_exec_runtime; -- double vruntime; --}; -- --/* -- * Use a vruntime-sorted list to store tasks. This could easily be extended to -- * a more optimal data structure, such as an rbtree as is done in CFS. We -- * currently elect to use a sorted list to simplify the example for -- * illustrative purposes. -- */ --LIST_HEAD(listhead, enqueued_task); -- --/* -- * A vruntime-sorted list of tasks. The head of the list contains the task with -- * the lowest vruntime. That is, the task that has the "highest" claim to be -- * scheduled. -- */ --static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head); -- --/* -- * The statically allocated array of tasks. We use a statically allocated list -- * here to avoid having to allocate on the enqueue path, which could cause a -- * deadlock. A more substantive user space scheduler could e.g. provide a hook -- * for newly enabled tasks that are passed to the scheduler from the -- * .prep_enable() callback to allows the scheduler to allocate on safe paths. -- */ --struct enqueued_task tasks[USERLAND_MAX_TASKS]; -- --static double min_vruntime; -- --static void sigint_handler(int userland) --{ -- exit_req = 1; --} -- --static __u32 task_pid(const struct enqueued_task *task) --{ -- return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task); --} -- --static int dispatch_task(s32 pid) --{ -- int err; -- -- err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d\n", pid); -- exit_req = 1; -- } else { -- nr_vruntime_dispatches++; -- } -- -- return err; --} -- --static struct enqueued_task *get_enqueued_task(__s32 pid) --{ -- if (pid >= USERLAND_MAX_TASKS) -- return NULL; -- -- return &tasks[pid]; --} -- --static double calc_vruntime_delta(__u64 weight, __u64 delta) --{ -- double weight_f = (double)weight / 100.0; -- double delta_f = (double)delta; -- -- return delta_f / weight_f; --} -- --static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task) --{ -- __u64 delta; -- -- delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime; -- -- enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta); -- if (min_vruntime > enqueued->vruntime) -- enqueued->vruntime = min_vruntime; -- enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime; --} -- --static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task) --{ -- struct enqueued_task *curr, *enqueued, *prev; -- -- curr = get_enqueued_task(bpf_task->pid); -- if (!curr) -- return ENOENT; -- -- update_enqueued(curr, bpf_task); -- nr_vruntime_enqueues++; -- -- /* -- * Enqueue the task in a vruntime-sorted list. A more optimal data -- * structure such as an rbtree could easily be used as well. We elect -- * to use a list here simply because it's less code, and thus the -- * example is less convoluted and better serves to illustrate what a -- * user space scheduler could look like. -- */ -- -- if (LIST_EMPTY(&vruntime_head)) { -- LIST_INSERT_HEAD(&vruntime_head, curr, entries); -- return 0; -- } -- -- LIST_FOREACH(enqueued, &vruntime_head, entries) { -- if (curr->vruntime <= enqueued->vruntime) { -- LIST_INSERT_BEFORE(enqueued, curr, entries); -- return 0; -- } -- prev = enqueued; -- } -- -- LIST_INSERT_AFTER(prev, curr, entries); -- -- return 0; --} -- --static void drain_enqueued_map(void) --{ -- while (1) { -- struct scx_userland_enqueued_task task; -- int err; -- -- if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) -- return; -- -- err = vruntime_enqueue(&task); -- if (err) { -- fprintf(stderr, "Failed to enqueue task %d: %s\n", -- task.pid, strerror(err)); -- exit_req = 1; -- return; -- } -- } --} -- --static void dispatch_batch(void) --{ -- __u32 i; -- -- for (i = 0; i < batch_size; i++) { -- struct enqueued_task *task; -- int err; -- __s32 pid; -- -- task = LIST_FIRST(&vruntime_head); -- if (!task) -- return; -- -- min_vruntime = task->vruntime; -- pid = task_pid(task); -- LIST_REMOVE(task, entries); -- err = dispatch_task(pid); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d in %u\n", -- pid, i); -- return; -- } -- } --} -- --static void *run_stats_printer(void *arg) --{ -- while (!exit_req) { -- __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; -- -- nr_failed_enqueues = skel->bss->nr_failed_enqueues; -- nr_kernel_enqueues = skel->bss->nr_kernel_enqueues; -- nr_user_enqueues = skel->bss->nr_user_enqueues; -- total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues; -- -- printf("o-----------------------o\n"); -- printf("| BPF ENQUEUES |\n"); -- printf("|-----------------------|\n"); -- printf("| kern: %10llu |\n", nr_kernel_enqueues); -- printf("| user: %10llu |\n", nr_user_enqueues); -- printf("| failed: %10llu |\n", nr_failed_enqueues); -- printf("| -------------------- |\n"); -- printf("| total: %10llu |\n", total); -- printf("| |\n"); -- printf("|-----------------------|\n"); -- printf("| VRUNTIME / USER |\n"); -- printf("|-----------------------|\n"); -- printf("| enq: %10llu |\n", nr_vruntime_enqueues); -- printf("| disp: %10llu |\n", nr_vruntime_dispatches); -- printf("o-----------------------o\n"); -- printf("\n\n"); -- sleep(1); -- } -- -- return NULL; --} -- --static int spawn_stats_thread(void) --{ -- pthread_t stats_printer; -- -- return pthread_create(&stats_printer, NULL, run_stats_printer, NULL); --} -- --static int bootstrap(int argc, char **argv) --{ -- int err; -- __u32 opt; -- struct sched_param sched_param = { -- .sched_priority = sched_get_priority_max(SCHED_EXT), -- }; -- bool switch_partial = false; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- /* -- * Enforce that the user scheduler task is managed by sched_ext. The -- * task eagerly drains the list of enqueued tasks in its main work -- * loop, and then yields the CPU. The BPF scheduler only schedules the -- * user space scheduler task when at least one other task in the system -- * needs to be scheduled. -- */ -- err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param); -- if (err) { -- fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err)); -- return err; -- } -- -- while ((opt = getopt(argc, argv, "b:ph")) != -1) { -- switch (opt) { -- case 'b': -- batch_size = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- exit(opt != 'h'); -- } -- } -- -- /* -- * It's not always safe to allocate in a user space scheduler, as an -- * enqueued task could hold a lock that we require in order to be able -- * to allocate. -- */ -- err = mlockall(MCL_CURRENT | MCL_FUTURE); -- if (err) { -- fprintf(stderr, "Failed to prefault and lock address space: %s\n", -- strerror(err)); -- return err; -- } -- -- skel = scx_example_userland__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno)); -- return errno; -- } -- skel->rodata->num_possible_cpus = libbpf_num_possible_cpus(); -- assert(skel->rodata->num_possible_cpus > 0); -- skel->rodata->usersched_pid = getpid(); -- assert(skel->rodata->usersched_pid > 0); -- skel->rodata->switch_partial = switch_partial; -- -- err = scx_example_userland__load(skel); -- if (err) { -- fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- enqueued_fd = bpf_map__fd(skel->maps.enqueued); -- dispatched_fd = bpf_map__fd(skel->maps.dispatched); -- assert(enqueued_fd > 0); -- assert(dispatched_fd > 0); -- -- err = spawn_stats_thread(); -- if (err) { -- fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops); -- if (!ops_link) { -- fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno)); -- err = errno; -- goto destroy_skel; -- } -- -- return 0; -- --destroy_skel: -- scx_example_userland__destroy(skel); -- exit_req = 1; -- return err; --} -- --static void sched_main_loop(void) --{ -- while (!exit_req) { -- /* -- * Perform the following work in the main user space scheduler -- * loop: -- * -- * 1. Drain all tasks from the enqueued map, and enqueue them -- * to the vruntime sorted list. -- * -- * 2. Dispatch a batch of tasks from the vruntime sorted list -- * down to the kernel. -- * -- * 3. Yield the CPU back to the system. The BPF scheduler will -- * reschedule the user space scheduler once another task has -- * been enqueued to user space. -- */ -- drain_enqueued_map(); -- dispatch_batch(); -- sched_yield(); -- } --} -- --int main(int argc, char **argv) --{ -- int err; -- -- err = bootstrap(argc, argv); -- if (err) { -- fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err)); -- return err; -- } -- -- sched_main_loop(); -- -- exit_req = 1; -- bpf_link__destroy(ops_link); -- uei_print(&skel->bss->uei); -- scx_example_userland__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_userland_common.h a/tools/sched_ext/scx_example_userland_common.h -deleted file mode 100644 -index 639c6809c5ff..000000000000 ---- b/tools/sched_ext/scx_example_userland_common.h -+++ /dev/null -@@ -1,19 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* Copyright (c) 2022 Meta, Inc */ -- --#ifndef __SCX_USERLAND_COMMON_H --#define __SCX_USERLAND_COMMON_H -- --#define USERLAND_MAX_TASKS 8192 -- --/* -- * An instance of a task that has been enqueued by the kernel for consumption -- * by a user space global scheduler thread. -- */ --struct scx_userland_enqueued_task { -- __s32 pid; -- u64 sum_exec_runtime; -- u64 weight; --}; -- --#endif // __SCX_USERLAND_COMMON_H -diff --git b/tools/sched_ext/user_exit_info.h a/tools/sched_ext/user_exit_info.h -deleted file mode 100644 -index e701ef0e0b86..000000000000 ---- b/tools/sched_ext/user_exit_info.h -+++ /dev/null -@@ -1,50 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Define struct user_exit_info which is shared between BPF and userspace parts -- * to communicate exit status and other information. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __USER_EXIT_INFO_H --#define __USER_EXIT_INFO_H -- --struct user_exit_info { -- int type; -- char reason[128]; -- char msg[1024]; --}; -- --#ifdef __bpf__ -- --#include "vmlinux.h" --#include -- --static inline void uei_record(struct user_exit_info *uei, -- const struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason); -- bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg); -- /* use __sync to force memory barrier */ -- __sync_val_compare_and_swap(&uei->type, uei->type, ei->type); --} -- --#else /* !__bpf__ */ -- --static inline bool uei_exited(struct user_exit_info *uei) --{ -- /* use __sync to force memory barrier */ -- return __sync_val_compare_and_swap(&uei->type, -1, -1); --} -- --static inline void uei_print(const struct user_exit_info *uei) --{ -- fprintf(stderr, "EXIT: %s", uei->reason); -- if (uei->msg[0] != '\0') -- fprintf(stderr, " (%s)", uei->msg); -- fputs("\n", stderr); --} -- --#endif /* __bpf__ */ --#endif /* __USER_EXIT_INFO_H */ diff --git a/oneplus/hmbird/fengchi_OP-PAD-2-MT6991_A16.patch b/oneplus/hmbird/fengchi_OP-PAD-2-MT6991_A16.patch deleted file mode 100644 index 42ddfa7e..00000000 --- a/oneplus/hmbird/fengchi_OP-PAD-2-MT6991_A16.patch +++ /dev/null @@ -1,20969 +0,0 @@ -diff --git a/Documentation/scheduler/index.rst b/Documentation/scheduler/index.rst -index 0b650bb550e6..3170747226f6 100644 ---- a/Documentation/scheduler/index.rst -+++ b/Documentation/scheduler/index.rst -@@ -19,7 +19,6 @@ Scheduler - sched-nice-design - sched-rt-group - sched-stats -- sched-ext - sched-debug - - text_files -diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst -deleted file mode 100644 -index 84c30b44f104..000000000000 ---- a/Documentation/scheduler/sched-ext.rst -+++ /dev/null -@@ -1,230 +0,0 @@ --========================== --Extensible Scheduler Class --========================== -- --sched_ext is a scheduler class whose behavior can be defined by a set of BPF --programs - the BPF scheduler. -- --* sched_ext exports a full scheduling interface so that any scheduling -- algorithm can be implemented on top. -- --* The BPF scheduler can group CPUs however it sees fit and schedule them -- together, as tasks aren't tied to specific CPUs at the time of wakeup. -- --* The BPF scheduler can be turned on and off dynamically anytime. -- --* The system integrity is maintained no matter what the BPF scheduler does. -- The default scheduling behavior is restored anytime an error is detected, -- a runnable task stalls, or on invoking the SysRq key sequence -- :kbd:`SysRq-S`. -- --Switching to and from sched_ext --=============================== -- --``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and --``tools/sched_ext`` contains the example schedulers. -- --sched_ext is used only when the BPF scheduler is loaded and running. -- --If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be --treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is --loaded. On load, such tasks will be switched to and scheduled by sched_ext. -- --The BPF scheduler can choose to schedule all normal and lower class tasks by --calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this --case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and --``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers, --this mode can be selected with the ``-a`` option. -- --Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or --detection of any internal error including stalled runnable tasks aborts the --BPF scheduler and reverts all tasks back to CFS. -- --.. code-block:: none -- -- # make -j16 -C tools/sched_ext -- # tools/sched_ext/scx_example_simple -- local=0 global=3 -- local=5 global=24 -- local=9 global=44 -- local=13 global=56 -- local=17 global=72 -- ^CEXIT: BPF scheduler unregistered -- --If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler --and whether a given task is on sched_ext can be determined as follows: -- --.. code-block:: none -- -- # cat /sys/kernel/debug/sched/ext -- ops : simple -- enabled : 1 -- switching_all : 1 -- switched_all : 1 -- enable_state : enabled -- -- # grep ext /proc/self/sched -- ext.enabled : 1 -- --The Basics --========== -- --Userspace can implement an arbitrary BPF scheduler by loading a set of BPF --programs that implement ``struct sched_ext_ops``. The only mandatory field --is ``ops.name`` which must be a valid BPF object name. All operations are --optional. The following modified excerpt is from --``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO --scheduler. -- --.. code-block:: c -- -- s32 BPF_STRUCT_OPS(simple_init) -- { -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; -- } -- -- void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) -- { -- if (enq_flags & SCX_ENQ_LOCAL) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags); -- } -- -- void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) -- { -- exit_type = ei->type; -- } -- -- SEC(".struct_ops") -- struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", -- }; -- --Dispatch Queues ----------------- -- --To match the impedance between the scheduler core and the BPF scheduler, --sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a --priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``), --and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage --an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and --``scx_bpf_destroy_dsq()``. -- --A CPU always executes a task from its local DSQ. A task is "dispatched" to a --DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's --local DSQ. -- --When a CPU is looking for the next task to run, if the local DSQ is not --empty, the first task is picked. Otherwise, the CPU tries to consume the --global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()`` --is invoked. -- --Scheduling Cycle ------------------ -- --The following briefly shows how a waking task is scheduled and executed. -- --1. When a task is waking up, ``ops.select_cpu()`` is the first operation -- invoked. This serves two purposes. First, CPU selection optimization -- hint. Second, waking up the selected CPU if idle. -- -- The CPU selected by ``ops.select_cpu()`` is an optimization hint and not -- binding. The actual decision is made at the last step of scheduling. -- However, there is a small performance gain if the CPU -- ``ops.select_cpu()`` returns matches the CPU the task eventually runs on. -- -- A side-effect of selecting a CPU is waking it up from idle. While a BPF -- scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper, -- using ``ops.select_cpu()`` judiciously can be simpler and more efficient. -- -- Note that the scheduler core will ignore an invalid CPU selection, for -- example, if it's outside the allowed cpumask of the task. -- --2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can -- make one of the following decisions: -- -- * Immediately dispatch the task to either the global or local DSQ by -- calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or -- ``SCX_DSQ_LOCAL``, respectively. -- -- * Immediately dispatch the task to a custom DSQ by calling -- ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63. -- -- * Queue the task on the BPF side. -- --3. When a CPU is ready to schedule, it first looks at its local DSQ. If -- empty, it then looks at the global DSQ. If there still isn't a task to -- run, ``ops.dispatch()`` is invoked which can use the following two -- functions to populate the local DSQ. -- -- * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can -- be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``, -- ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()`` -- currently can't be called with BPF locks held, this is being worked on -- and will be supported. ``scx_bpf_dispatch()`` schedules dispatching -- rather than performing them immediately. There can be up to -- ``ops.dispatch_max_batch`` pending tasks. -- -- * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ -- to the dispatching DSQ. This function cannot be called with any BPF -- locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks -- before trying to consume the specified DSQ. -- --4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ, -- the CPU runs the first one. If empty, the following steps are taken: -- -- * Try to consume the global DSQ. If successful, run the task. -- -- * If ``ops.dispatch()`` has dispatched any tasks, retry #3. -- -- * If the previous task is an SCX task and still runnable, keep executing -- it (see ``SCX_OPS_ENQ_LAST``). -- -- * Go idle. -- --Note that the BPF scheduler can always choose to dispatch tasks immediately --in ``ops.enqueue()`` as illustrated in the above simple example. If only the --built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as --a task is never queued on the BPF scheduler and both the local and global --DSQs are consumed automatically. -- --``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use --``scx_bpf_dispatch_vtime()`` for the priority queue. See the function --documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for --more information. -- --Where to Look --============= -- --* ``include/linux/sched/ext.h`` defines the core data structures, ops table -- and constants. -- --* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. -- The functions prefixed with ``scx_bpf_`` can be called from the BPF -- scheduler. -- --* ``tools/sched_ext/`` hosts example BPF scheduler implementations. -- -- * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example -- using a custom DSQ. -- -- * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting -- five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. -- --ABI Instability --=============== -- --The APIs provided by sched_ext to BPF schedulers programs have no stability --guarantees. This includes the ops table callbacks and constants defined in --``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in --``kernel/sched/ext.c``. -- --While we will attempt to provide a relatively stable API surface when --possible, they are subject to change without warning between kernel --versions. -diff --git a/MAINTAINERS b/MAINTAINERS -index 9fc6108503c3..2384b55682ee 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -19132,8 +19132,6 @@ R: Ben Segall (CONFIG_CFS_BANDWIDTH) - R: Mel Gorman (CONFIG_NUMA_BALANCING) - R: Daniel Bristot de Oliveira (SCHED_DEADLINE) - R: Valentin Schneider (TOPOLOGY) --R: Tejun Heo (SCHED_EXT) --R: David Vernet (SCHED_EXT) - L: linux-kernel@vger.kernel.org - S: Maintained - T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core -@@ -19142,7 +19140,6 @@ F: include/linux/sched.h - F: include/linux/wait.h - F: include/uapi/linux/sched.h - F: kernel/sched/ --F: tools/sched_ext/ - - SCSI LIBSAS SUBSYSTEM - R: John Garry -diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig -index f93980c09d7f..5ba590235a8c 100644 ---- a/arch/arm64/configs/defconfig -+++ b/arch/arm64/configs/defconfig -@@ -6,8 +6,7 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_BPF_SYSCALL=y - CONFIG_BPF_JIT=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_BSD_PROCESS_ACCT=y - CONFIG_BSD_PROCESS_ACCT_V3=y -diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig -index 4f756dca5e05..6c1bb94f875d 100644 ---- a/arch/arm64/configs/gki_defconfig -+++ b/arch/arm64/configs/gki_defconfig -@@ -10,8 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y - # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set - CONFIG_BPF_LSM=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_TASKSTATS=y - CONFIG_TASK_DELAY_ACCT=y -diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c -index 4bd028fa7500..5825ce9d6d7b 100644 ---- a/drivers/gpu/drm/msm/msm_drv.c -+++ b/drivers/gpu/drm/msm/msm_drv.c -@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) - } - - sched_set_fifo(ev_thread->worker->task); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3); -+#endif - } - - ret = drm_vblank_init(ddev, priv->num_crtcs); -diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c -index bd001445815e..dcf2e1ebf9c5 100644 ---- a/drivers/tty/sysrq.c -+++ b/drivers/tty/sysrq.c -@@ -524,7 +524,6 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { - NULL, /* P */ - NULL, /* Q */ - NULL, /* R */ -- /* S: May be registered by sched_ext for resetting */ - NULL, /* S */ - NULL, /* T */ - NULL, /* U */ -diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h -index c45078a445c8..6c15659f874e 100644 ---- a/include/asm-generic/vmlinux.lds.h -+++ b/include/asm-generic/vmlinux.lds.h -@@ -131,7 +131,7 @@ - *(__dl_sched_class) \ - *(__rt_sched_class) \ - *(__fair_sched_class) \ -- *(__ext_sched_class) \ -+ *(__hmbird_sched_class) \ - *(__idle_sched_class) \ - __sched_class_lowest = .; - -diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h -index 63452733a2f9..3a2f906ed3ce 100644 ---- a/include/linux/cpufreq.h -+++ b/include/linux/cpufreq.h -@@ -44,6 +44,10 @@ enum cpufreq_table_sorting { - CPUFREQ_TABLE_SORTED_DESCENDING - }; - -+ssize_t store_scaling_governor(struct cpufreq_policy *policy, -+ const char *buf, size_t count); -+ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf); -+ - struct cpufreq_cpuinfo { - unsigned int max_freq; - unsigned int min_freq; -diff --git a/include/linux/sched.h b/include/linux/sched.h -index 2cbc342db12d..dd6b1d58af01 100644 ---- a/include/linux/sched.h -+++ b/include/linux/sched.h -@@ -73,7 +73,9 @@ struct task_delay_info; - struct task_group; - struct user_event_mm; - --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - - /* - * Task state bitmask. NOTE! These bits are also -@@ -298,11 +300,6 @@ enum { - - extern void scheduler_tick(void); - --#ifdef CONFIG_SLIM_SCHED --extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer); --extern void stop_shadow_tick_timer(void); --#endif -- - #define MAX_SCHEDULE_TIMEOUT LONG_MAX - - extern long schedule_timeout(long timeout); -@@ -1523,13 +1520,8 @@ struct task_struct { - */ - struct callback_head l1d_flush_kill; - #endif --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, unsigned long sched_prop); -- ANDROID_KABI_USE(2, struct sched_ext_entity *scx); --#else - ANDROID_KABI_RESERVE(1); - ANDROID_KABI_RESERVE(2); --#endif - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); - ANDROID_KABI_RESERVE(5); -@@ -1933,6 +1925,92 @@ static inline int task_nice(const struct task_struct *p) - return PRIO_TO_NICE((p)->static_prio); - } - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline bool task_is_top_task(struct task_struct *p) -+{ -+ return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ ->top_task_prop & TOP_TASK_BITS_MASK); -+} -+ -+static inline int get_top_task_prop(struct task_struct *p) -+{ -+ return get_hmbird_ts(p)->top_task_prop; -+} -+ -+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear) -+{ -+ if (set) -+ get_hmbird_ts(p)->top_task_prop |= set; -+ if (clear) -+ get_hmbird_ts(p)->top_task_prop &= ~clear; -+ return 0; -+} -+ -+static inline void reset_top_task_prop(struct task_struct *p) -+{ -+ get_hmbird_ts(p)->top_task_prop = 0; -+} -+ -+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ entity->sched_prop = sp; -+ } -+ return 0; -+} -+ -+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->sched_prop; -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq) -+{ -+ unsigned long new_dsq; -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq; -+ entity->sched_prop = new_dsq; -+ } -+} -+ -+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK); -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ entity->dsq_sync_ux = val; -+} -+ -+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->dsq_sync_ux; -+ else -+ return 0; -+} -+#endif -+ - extern int can_nice(const struct task_struct *p, const int nice); - extern int task_curr(const struct task_struct *p); - extern int idle_cpu(int cpu); -diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h -deleted file mode 100755 -index b737a00c1373..000000000000 ---- a/include/linux/sched/ext.h -+++ /dev/null -@@ -1,647 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef _LINUX_SCHED_EXT_H --#define _LINUX_SCHED_EXT_H -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --#include -- --enum scx_consts { -- SCX_OPS_NAME_LEN = 128, -- SCX_EXIT_REASON_LEN = 128, -- SCX_EXIT_BT_LEN = 64, -- SCX_EXIT_MSG_LEN = 1024, -- -- SCX_SLICE_DFL = 20 * NSEC_PER_MSEC, -- SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */ --}; -- --/* -- * DSQ (dispatch queue) IDs are 64bit of the format: -- * -- * Bits: [63] [62 .. 0] -- * [ B] [ ID ] -- * -- * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -- * ID: 63 bit ID -- * -- * Built-in IDs: -- * -- * Bits: [63] [62] [61..32] [31 .. 0] -- * [ 1] [ L] [ R ] [ V ] -- * -- * 1: 1 for built-in DSQs. -- * L: 1 for LOCAL_ON DSQ IDs, 0 for others -- * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -- */ --enum scx_dsq_id_flags { -- SCX_DSQ_FLAG_BUILTIN = 1LLU << 63, -- SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -- -- SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0, -- SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, -- SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, -- SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, -- SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, --}; -- --enum scx_exit_type { -- SCX_EXIT_NONE, -- SCX_EXIT_DONE, -- -- SCX_EXIT_UNREG = 64, /* BPF unregistration */ -- SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -- SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ -- SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ --}; -- --/* -- * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is -- * being disabled. -- */ --struct scx_exit_info { -- /* %SCX_EXIT_* - broad category of the exit reason */ -- enum scx_exit_type type; -- /* textual representation of the above */ -- char reason[SCX_EXIT_REASON_LEN]; -- /* number of entries in the backtrace */ -- u32 bt_len; -- /* backtrace if exiting due to an error */ -- unsigned long bt[SCX_EXIT_BT_LEN]; -- /* extra message */ -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --/* sched_ext_ops.flags */ --enum scx_ops_flags { -- /* -- * Keep built-in idle tracking even if ops.update_idle() is implemented. -- */ -- SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, -- -- /* -- * By default, if there are no other task to run on the CPU, ext core -- * keeps running the current task even after its slice expires. If this -- * flag is specified, such tasks are passed to ops.enqueue() with -- * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. -- */ -- SCX_OPS_ENQ_LAST = 1LLU << 1, -- -- /* -- * An exiting task may schedule after PF_EXITING is set. In such cases, -- * bpf_task_from_pid() may not be able to find the task and if the BPF -- * scheduler depends on pid lookup for dispatching, the task will be -- * lost leading to various issues including RCU grace period stalls. -- * -- * To mask this problem, by default, unhashed tasks are automatically -- * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't -- * depend on pid lookups and wants to handle these tasks directly, the -- * following flag can be used. -- */ -- SCX_OPS_ENQ_EXITING = 1LLU << 2, -- -- /* -- * CPU cgroup knob enable flags -- */ -- SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16, /* cpu.weight */ -- -- SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | -- SCX_OPS_ENQ_LAST | -- SCX_OPS_ENQ_EXITING | -- SCX_OPS_CGROUP_KNOB_WEIGHT, --}; -- --/* argument container for ops.enable() and friends */ --struct scx_enable_args { --}; -- --/* argument container for ops->cgroup_init() */ --struct scx_cgroup_init_args { -- /* the weight of the cgroup [1..10000] */ -- u32 weight; --}; -- --enum scx_cpu_preempt_reason { -- /* next task is being scheduled by &sched_class_rt */ -- SCX_CPU_PREEMPT_RT, -- /* next task is being scheduled by &sched_class_dl */ -- SCX_CPU_PREEMPT_DL, -- /* next task is being scheduled by &sched_class_stop */ -- SCX_CPU_PREEMPT_STOP, -- /* unknown reason for SCX being preempted */ -- SCX_CPU_PREEMPT_UNKNOWN, --}; -- --/* -- * Argument container for ops->cpu_acquire(). Currently empty, but may be -- * expanded in the future. -- */ --struct scx_cpu_acquire_args {}; -- --/* argument container for ops->cpu_release() */ --struct scx_cpu_release_args { -- /* the reason the CPU was preempted */ -- enum scx_cpu_preempt_reason reason; -- -- /* the task that's going to be scheduled on the CPU */ -- struct task_struct *task; --}; -- --/** -- * struct sched_ext_ops - Operation table for BPF scheduler implementation -- * -- * Userland can implement an arbitrary scheduling policy by implementing and -- * loading operations in this table. -- */ --struct sched_ext_ops { -- /** -- * select_cpu - Pick the target CPU for a task which is being woken up -- * @p: task being woken up -- * @prev_cpu: the cpu @p was on before sleeping -- * @wake_flags: SCX_WAKE_* -- * -- * Decision made here isn't final. @p may be moved to any CPU while it -- * is getting dispatched for execution later. However, as @p is not on -- * the rq at this point, getting the eventual execution CPU right here -- * saves a small bit of overhead down the line. -- * -- * If an idle CPU is returned, the CPU is kicked and will try to -- * dispatch. While an explicit custom mechanism can be added, -- * select_cpu() serves as the default way to wake up idle CPUs. -- */ -- s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); -- -- /** -- * enqueue - Enqueue a task on the BPF scheduler -- * @p: task being enqueued -- * @enq_flags: %SCX_ENQ_* -- * -- * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() -- * or enqueue on the BPF scheduler. If not directly dispatched, the bpf -- * scheduler owns @p and if it fails to dispatch @p, the task will -- * stall. -- */ -- void (*enqueue)(struct task_struct *p, u64 enq_flags); -- -- /** -- * dequeue - Remove a task from the BPF scheduler -- * @p: task being dequeued -- * @deq_flags: %SCX_DEQ_* -- * -- * Remove @p from the BPF scheduler. This is usually called to isolate -- * the task while updating its scheduling properties (e.g. priority). -- * -- * The ext core keeps track of whether the BPF side owns a given task or -- * not and can gracefully ignore spurious dispatches from BPF side, -- * which makes it safe to not implement this method. However, depending -- * on the scheduling logic, this can lead to confusing behaviors - e.g. -- * scheduling position not being updated across a priority change. -- */ -- void (*dequeue)(struct task_struct *p, u64 deq_flags); -- -- /** -- * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs -- * @cpu: CPU to dispatch tasks for -- * @prev: previous task being switched out -- * -- * Called when a CPU's local dsq is empty. The operation should dispatch -- * one or more tasks from the BPF scheduler into the DSQs using -- * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using -- * scx_bpf_consume(). -- * -- * The maximum number of times scx_bpf_dispatch() can be called without -- * an intervening scx_bpf_consume() is specified by -- * ops.dispatch_max_batch. See the comments on top of the two functions -- * for more details. -- * -- * When not %NULL, @prev is an SCX task with its slice depleted. If -- * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in -- * @prev->scx.flags, it is not enqueued yet and will be enqueued after -- * ops.dispatch() returns. To keep executing @prev, return without -- * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. -- */ -- void (*dispatch)(s32 cpu, struct task_struct *prev); -- -- /** -- * runnable - A task is becoming runnable on its associated CPU -- * @p: task becoming runnable -- * @enq_flags: %SCX_ENQ_* -- * -- * This and the following three functions can be used to track a task's -- * execution state transitions. A task becomes ->runnable() on a CPU, -- * and then goes through one or more ->running() and ->stopping() pairs -- * as it runs on the CPU, and eventually becomes ->quiescent() when it's -- * done running on the CPU. -- * -- * @p is becoming runnable on the CPU because it's -- * -- * - waking up (%SCX_ENQ_WAKEUP) -- * - being moved from another CPU -- * - being restored after temporarily taken off the queue for an -- * attribute change. -- * -- * This and ->enqueue() are related but not coupled. This operation -- * notifies @p's state transition and may not be followed by ->enqueue() -- * e.g. when @p is being dispatched to a remote CPU. Likewise, a task -- * may be ->enqueue()'d without being preceded by this operation e.g. -- * after exhausting its slice. -- */ -- void (*runnable)(struct task_struct *p, u64 enq_flags); -- -- /** -- * running - A task is starting to run on its associated CPU -- * @p: task starting to run -- * -- * See ->runnable() for explanation on the task state notifiers. -- */ -- void (*running)(struct task_struct *p); -- -- /** -- * stopping - A task is stopping execution -- * @p: task stopping to run -- * @runnable: is task @p still runnable? -- * -- * See ->runnable() for explanation on the task state notifiers. If -- * !@runnable, ->quiescent() will be invoked after this operation -- * returns. -- */ -- void (*stopping)(struct task_struct *p, bool runnable); -- -- /** -- * quiescent - A task is becoming not runnable on its associated CPU -- * @p: task becoming not runnable -- * @deq_flags: %SCX_DEQ_* -- * -- * See ->runnable() for explanation on the task state notifiers. -- * -- * @p is becoming quiescent on the CPU because it's -- * -- * - sleeping (%SCX_DEQ_SLEEP) -- * - being moved to another CPU -- * - being temporarily taken off the queue for an attribute change -- * (%SCX_DEQ_SAVE) -- * -- * This and ->dequeue() are related but not coupled. This operation -- * notifies @p's state transition and may not be preceded by ->dequeue() -- * e.g. when @p is being dispatched to a remote CPU. -- */ -- void (*quiescent)(struct task_struct *p, u64 deq_flags); -- -- /** -- * yield - Yield CPU -- * @from: yielding task -- * @to: optional yield target task -- * -- * If @to is NULL, @from is yielding the CPU to other runnable tasks. -- * The BPF scheduler should ensure that other available tasks are -- * dispatched before the yielding task. Return value is ignored in this -- * case. -- * -- * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf -- * scheduler can implement the request, return %true; otherwise, %false. -- */ -- bool (*yield)(struct task_struct *from, struct task_struct *to); -- -- /** -- * core_sched_before - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Used by core-sched to determine the ordering between two tasks. See -- * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on -- * core-sched. -- * -- * Both @a and @b are runnable and may or may not currently be queued on -- * the BPF scheduler. Should return %true if @a should run before @b. -- * %false if there's no required ordering or @b should run before @a. -- * -- * If not specified, the default is ordering them according to when they -- * became runnable. -- */ -- bool (*core_sched_before)(struct task_struct *a,struct task_struct *b); -- -- /** -- * set_weight - Set task weight -- * @p: task to set weight for -- * @weight: new eight [1..10000] -- * -- * Update @p's weight to @weight. -- */ -- void (*set_weight)(struct task_struct *p, u32 weight); -- -- /** -- * set_cpumask - Set CPU affinity -- * @p: task to set CPU affinity for -- * @cpumask: cpumask of cpus that @p can run on -- * -- * Update @p's CPU affinity to @cpumask. -- */ -- void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask); -- -- /** -- * update_idle - Update the idle state of a CPU -- * @cpu: CPU to udpate the idle state for -- * @idle: whether entering or exiting the idle state -- * -- * This operation is called when @rq's CPU goes or leaves the idle -- * state. By default, implementing this operation disables the built-in -- * idle CPU tracking and the following helpers become unavailable: -- * -- * - scx_bpf_select_cpu_dfl() -- * - scx_bpf_test_and_clear_cpu_idle() -- * - scx_bpf_pick_idle_cpu() -- * - scx_bpf_any_idle_cpu() -- * -- * The user also must implement ops.select_cpu() as the default -- * implementation relies on scx_bpf_select_cpu_dfl(). -- * -- * If you keep the built-in idle tracking, specify the -- * %SCX_OPS_KEEP_BUILTIN_IDLE flag. -- */ -- void (*update_idle)(s32 cpu, bool idle); -- -- /** -- * cpu_acquire - A CPU is becoming available to the BPF scheduler -- * @cpu: The CPU being acquired by the BPF scheduler. -- * @args: Acquire arguments, see the struct definition. -- * -- * A CPU that was previously released from the BPF scheduler is now once -- * again under its control. -- */ -- void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); -- -- /** -- * cpu_release - A CPU is taken away from the BPF scheduler -- * @cpu: The CPU being released by the BPF scheduler. -- * @args: Release arguments, see the struct definition. -- * -- * The specified CPU is no longer under the control of the BPF -- * scheduler. This could be because it was preempted by a higher -- * priority sched_class, though there may be other reasons as well. The -- * caller should consult @args->reason to determine the cause. -- */ -- void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -- -- /** -- * cpu_online - A CPU became online -- * @cpu: CPU which just came up -- * -- * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs beforehand. -- */ -- void (*cpu_online)(s32 cpu); -- -- /** -- * cpu_offline - A CPU is going offline -- * @cpu: CPU which is going offline -- * -- * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs afterwards. -- */ -- void (*cpu_offline)(s32 cpu); -- -- /** -- * prep_enable - Prepare to enable BPF scheduling for a task -- * @p: task to prepare BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Either we're loading a BPF scheduler or a new task is being forked. -- * Prepare BPF scheduling for @p. This operation may block and can be -- * used for allocations. -- * -- * Return 0 for success, -errno for failure. An error return while -- * loading will abort loading of the BPF scheduler. During a fork, will -- * abort the specific fork. -- */ -- s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * enable - Enable BPF scheduling for a task -- * @p: task to enable BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Enable @p for BPF scheduling. @p is now in the cgroup specified for -- * the preceding prep_enable() and will start running soon. -- */ -- void (*enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * cancel_enable - Cancel prep_enable() -- * @p: task being canceled -- * @args: enable arguments, see the struct definition -- * -- * @p was prep_enable()'d but failed before reaching enable(). Undo the -- * preparation. -- */ -- void (*cancel_enable)(struct task_struct *p, -- struct scx_enable_args *args); -- -- /** -- * disable - Disable BPF scheduling for a task -- * @p: task to disable BPF scheduling for -- * -- * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. -- * Disable BPF scheduling for @p. -- */ -- void (*disable)(struct task_struct *p); -- -- /* -- * All online ops must come before ops.init(). -- */ -- -- /** -- * init - Initialize the BPF scheduler -- */ -- s32 (*init)(void); -- -- /** -- * exit - Clean up after the BPF scheduler -- * @info: Exit info -- */ -- void (*exit)(struct scx_exit_info *info); -- -- /** -- * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch -- */ -- u32 dispatch_max_batch; -- -- /** -- * flags - %SCX_OPS_* flags -- */ -- u64 flags; -- -- /** -- * timeout_ms - The maximum amount of time, in milliseconds, that a -- * runnable task should be able to wait before being scheduled. The -- * maximum timeout may not exceed the default timeout of 30 seconds. -- * -- * Defaults to the maximum allowed timeout value of 30 seconds. -- */ -- u32 timeout_ms; -- -- /** -- * name - BPF scheduler's name -- * -- * Must be a non-zero valid BPF object name including only isalnum(), -- * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the -- * BPF scheduler is enabled. -- */ -- char name[SCX_OPS_NAME_LEN]; --}; -- --/* -- * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -- * scheduler core and the BPF scheduler. See the documentation for more details. -- */ --struct scx_dispatch_q { -- raw_spinlock_t lock; -- struct list_head fifo; /* processed in dispatching order */ -- struct rb_root_cached priq; /* processed in p->scx.dsq_vtime order */ -- u32 nr; -- u64 id; -- struct llist_node free_node; -- struct rcu_head rcu; --}; -- --/* scx_entity.flags */ --enum scx_ent_flags { -- SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ -- SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -- SCX_TASK_ENQ_LOCAL = 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */ -- -- SCX_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -- SCX_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -- -- SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -- SCX_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -- -- SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ --}; -- --/* scx_entity.dsq_flags */ --enum scx_ent_dsq_flags { -- SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ --}; -- --/* -- * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from -- * everywhere and the following bits track which kfunc sets are currently -- * allowed for %current. This simple per-task tracking works because SCX ops -- * nest in a limited way. BPF will likely implement a way to allow and disallow -- * kfuncs depending on the calling context which will replace this manual -- * mechanism. See scx_kf_allow(). -- */ --enum scx_kf_mask { -- SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ -- /* all non-sleepables may be nested inside INIT and SLEEPABLE */ -- SCX_KF_INIT = 1 << 0, /* running ops.init() */ -- SCX_KF_SLEEPABLE = 1 << 1, /* other sleepable init operations */ -- /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ -- SCX_KF_CPU_RELEASE = 1 << 2, /* ops.cpu_release() */ -- /* ops.dequeue (in REST) may be nested inside DISPATCH */ -- SCX_KF_DISPATCH = 1 << 3, /* ops.dispatch() */ -- SCX_KF_ENQUEUE = 1 << 4, /* ops.enqueue() */ -- SCX_KF_REST = 1 << 5, /* other rq-locked operations */ -- -- __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | -- SCX_KF_ENQUEUE | SCX_KF_REST, -- __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_REST, --}; -- --#define RAVG_HIST_SIZE 5 --struct scx_sched_task_stats { -- u64 mark_start; -- u64 window_start; -- u32 sum; -- u32 sum_history[RAVG_HIST_SIZE]; -- int cidx; -- -- u32 demand; -- u16 demand_scaled; -- void *sdsq; --}; -- -- -- --/* -- * The following is embedded in task_struct and contains all fields necessary -- * for a task to be scheduled by SCX. -- */ --struct sched_ext_entity { -- struct scx_dispatch_q *dsq; -- struct { -- struct list_head fifo; /* dispatch order */ -- struct rb_node priq; /* p->scx.dsq_vtime order */ -- } dsq_node; -- struct list_head watchdog_node; -- u32 flags; /* protected by rq lock */ -- u32 dsq_flags; /* protected by dsq lock */ -- u32 weight; -- s32 sticky_cpu; -- s32 holding_cpu; -- u32 kf_mask; /* see scx_kf_mask above */ -- struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ -- atomic64_t ops_state; -- unsigned long runnable_at; --#ifdef CONFIG_SCHED_CORE -- u64 core_sched_at; /* see scx_prio_less() */ --#endif -- -- /* BPF scheduler modifiable fields */ -- -- /* -- * Runtime budget in nsecs. This is usually set through -- * scx_bpf_dispatch() but can also be modified directly by the BPF -- * scheduler. Automatically decreased by SCX as the task executes. On -- * depletion, a scheduling event is triggered. -- * -- * This value is cleared to zero if the task is preempted by -- * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the -- * task ran. Use p->se.sum_exec_runtime instead. -- */ -- u64 slice; -- -- /* -- * Used to order tasks when dispatching to the vtime-ordered priority -- * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime() -- * but can also be modified directly by the BPF scheduler. Modifying it -- * while a task is queued on a dsq may mangle the ordering and is not -- * recommended. -- */ -- u64 dsq_vtime; -- -- /* -- * If set, reject future sched_setscheduler(2) calls updating the policy -- * to %SCHED_EXT with -%EACCES. -- * -- * If set from ops.prep_enable() and the task's policy is already -- * %SCHED_EXT, which can happen while the BPF scheduler is being loaded -- * or by inhering the parent's policy during fork, the task's policy is -- * rejected and forcefully reverted to %SCHED_NORMAL. The number of such -- * events are reported through /sys/kernel/debug/sched_ext::nr_rejected. -- */ -- bool disallow; /* reject switching into SCX */ -- -- /* cold fields */ -- struct list_head tasks_node; -- struct task_struct *task; -- struct scx_sched_task_stats sts; --}; -- --void sched_ext_free(struct task_struct *p); -- --#else /* !CONFIG_SCHED_CLASS_EXT */ -- --static inline void sched_ext_free(struct task_struct *p) {} -- --#endif /* CONFIG_SCHED_CLASS_EXT */ --#endif /* _LINUX_SCHED_EXT_H */ -diff --git a/include/linux/sched/hmbird_proc_val.h b/include/linux/sched/hmbird_proc_val.h -new file mode 100755 -index 000000000000..e59708b16ea3 ---- /dev/null -+++ b/include/linux/sched/hmbird_proc_val.h -@@ -0,0 +1,22 @@ -+#ifndef __HMBORD_PROC_VAL_H__ -+#define __HMBORD_PROC_VAL_H__ -+ -+extern int scx_enable; -+extern int partial_enable; -+extern int cpuctrl_high_ratio; -+extern int cpuctrl_low_ratio; -+extern int slim_stats; -+extern int hmbirdcore_debug; -+extern int slim_for_app; -+extern int misfit_ds; -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+extern int cpu7_tl; -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int slim_gov_debug; -+extern int scx_gov_ctrl; -+extern int sched_ravg_window_frame_per_sec; -+ -+#endif -\ No newline at end of file -diff --git a/include/linux/sched/hmbird_version.h b/include/linux/sched/hmbird_version.h -new file mode 100755 -index 000000000000..b2843881c26f ---- /dev/null -+++ b/include/linux/sched/hmbird_version.h -@@ -0,0 +1,52 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#ifndef _OPLUS_HMBIRD_VERSION_H_ -+#define _OPLUS_HMBIRD_VERSION_H_ -+#include -+#include -+#include -+ -+enum hmbird_version { -+ HMBIRD_UNINIT, -+ HMBIRD_GKI_VERSION, -+ HMBIRD_OGKI_VERSION, -+ HMBIRD_UNKNOW_VERSION, -+}; -+ -+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT; -+ -+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type" -+ -+static inline enum hmbird_version get_hmbird_version_type(void) -+{ -+ struct device_node *np = NULL; -+ const char *hmbird_version_str = NULL; -+ if (HMBIRD_UNINIT != hmbird_version_type) -+ return hmbird_version_type; -+ np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH); -+ if (np) { -+ of_property_read_string(np, "type", &hmbird_version_str); -+ if (NULL != hmbird_version_str) { -+ if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) { -+ hmbird_version_type = HMBIRD_OGKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi"); -+ } else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) { -+ hmbird_version_type = HMBIRD_GKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi"); -+ } else { -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi"); -+ } -+ return hmbird_version_type; -+ } -+ } -+ -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION"); -+ return hmbird_version_type; -+} -+ -+#endif /*_OPLUS_HMBIRD_VERSION_H_ */ -diff --git a/include/linux/sched/sa_common.h b/include/linux/sched/sa_common.h -new file mode 100755 -index 000000000000..6f76d7cfd7bf ---- /dev/null -+++ b/include/linux/sched/sa_common.h -@@ -0,0 +1,797 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_COMMON_H_ -+#define _OPLUS_SA_COMMON_H_ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+#include -+#endif -+#include -+#include -+#include -+#include -+#include -+ -+#include "sa_oemdata.h" -+#include "sa_common_struct.h" -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) -+#define OPLUS_UX_EEVDF_COMPATIBLE 1 -+#endif -+ -+#define SA_DEBUG_ON 0 -+ -+#if (SA_DEBUG_ON >= 1) -+#define DEBUG_BUG_ON(x) BUG_ON(x) -+#define DEBUG_WARN_ON(x) WARN_ON(x) -+#else -+#define DEBUG_BUG_ON(x) -+#define DEBUG_WARN_ON(x) -+#endif -+ -+#define ux_err(fmt, ...) \ -+ pr_err("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_warn(fmt, ...) \ -+ pr_warn("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_debug(fmt, ...) \ -+ pr_info("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+ -+#define UX_MSG_LEN 64 -+#define UX_DEPTH_MAX 5 -+ -+/* define for debug */ -+#define DEBUG_SYSTRACE (1 << 0) -+#define DEBUG_FTRACE (1 << 1) -+#define DEBUG_KMSG (1 << 2) /* used for frameboost */ -+#define DEBUG_PIPELINE (1 << 3) -+#define DEBUG_DYNAMIC_HZ (1 << 4) -+#define DEBUG_DYNAMIC_PREEMPT (1 << 5) -+#define DEBUG_AMU_INSTRUCTION (1 << 6) -+#define DEBUG_VERBOSE (1 << 10) /* used for frameboost */ -+#define DEBUG_SET_DSQ_ID (1 << 11) /* used for hmbird */ -+ -+/* define for sched assist feature */ -+#define FEATURE_COMMON (1 << 0) -+#define FEATURE_SPREAD (1 << 1) -+ -+#define UX_EXEC_SLICE (4000000U) -+ -+/* define for sched assist thread type, keep same as the define in java file */ -+#define SA_OPT_CLEAR (0) -+#define SA_TYPE_LIGHT (1 << 0) -+#define SA_TYPE_HEAVY (1 << 1) -+#define SA_TYPE_ANIMATOR (1 << 2) -+/* SA_TYPE_LISTPICK for camera */ -+#define SA_TYPE_LISTPICK (1 << 3) -+#define SA_TYPE_MQ_VIP (1 << 4) -+#define SA_OPT_SET (1 << 7) -+#define SA_OPT_RESET (1 << 8) -+#define SA_OPT_SET_PRIORITY (1 << 9) -+ -+/* The following ux value only used in kernel */ -+#define SA_TYPE_SWIFT (1 << 14) -+/* clear ux type when dequeue */ -+#define SA_TYPE_ONCE (1 << 15) -+#define SA_TYPE_INHERIT (1 << 16) -+/* SA_TYPE_COMBINED isn't marked in ux_state, it only exists in return value of function */ -+#define SA_TYPE_COMBINED (1 << 17) -+#define SA_TYPE_URGENT_MASK (SA_TYPE_LIGHT|SA_TYPE_ANIMATOR|SA_TYPE_SWIFT) -+#define SCHED_ASSIST_UX_MASK (SA_TYPE_LIGHT|SA_TYPE_HEAVY|SA_TYPE_ANIMATOR|SA_TYPE_LISTPICK|SA_TYPE_SWIFT) -+ -+/* load balance operation is performed on the following ux types */ -+#define SCHED_ASSIST_LB_UX (SA_TYPE_SWIFT | SA_TYPE_ANIMATOR | SA_TYPE_LIGHT | SA_TYPE_HEAVY) -+#define POSSIBLE_UX_MASK (SA_TYPE_SWIFT | \ -+ SA_TYPE_LIGHT | \ -+ SA_TYPE_HEAVY | \ -+ SA_TYPE_ANIMATOR | \ -+ SA_TYPE_LISTPICK | \ -+ SA_TYPE_ONCE | \ -+ SA_TYPE_INHERIT) -+ -+#define SCHED_ASSIST_UX_PRIORITY_MASK (0xFF000000) -+#define SCHED_ASSIST_UX_PRIORITY_SHIFT 24 -+ -+#define UX_PRIORITY_TOP_APP 0x0A000000 -+#define UX_PRIORITY_AUDIO 0x0A000000 -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+#define UX_PRIORITY_PIPELINE_UI 0x06000000 -+#define UX_PRIORITY_PIPELINE 0x05000000 -+#endif -+ -+/* define for sched assist scene type, keep same as the define in java file */ -+#define SA_SCENE_OPT_CLEAR (0) -+#define SA_LAUNCH (1 << 0) -+#define SA_SLIDE (1 << 1) -+#define SA_CAMERA (1 << 2) -+#define SA_ANIM_START (1 << 3) /* we care about both launcher and top app */ -+#define SA_ANIM (1 << 4) /* we only care about launcher */ -+#define SA_INPUT (1 << 5) -+#define SA_LAUNCHER_SI (1 << 6) -+#define SA_SCENE_OPT_SET (1 << 7) -+ -+#define ROOT_UID 0 -+#define SYSTEM_UID 1000 -+#define FIRST_APPLICATION_UID 10000 -+#define LAST_APPLICATION_UID 19999 -+#define PER_USER_RANGE 100000 -+/* define for clear IM_FLAG -+if im_flag is 70, it should clear im_flag_audio (70 - 64 = 6) -+eg: gerrit patchset "30438485" -+ */ -+#define IM_FLAG_CLEAR 64 -+ -+extern pid_t save_audio_tgid; -+extern pid_t save_top_app_tgid; -+extern unsigned int top_app_type; -+extern int global_lowend_plat_opt; -+ -+#ifdef CONFIG_OPLUS_SCHED_HALT_MASK_PRT -+/* This must be the same as the definition of pause_type in walt_halt.c */ -+enum oplus_pause_type { -+ OPLUS_HALT, -+ OPLUS_PARTIAL_HALT, -+ -+ OPLUS_MAX_PAUSE_TYPE -+}; -+extern cpumask_t cur_cpus_halt_mask; -+extern cpumask_t cur_cpus_phalt_mask; -+DECLARE_PER_CPU(int[OPLUS_MAX_PAUSE_TYPE], oplus_cur_pause_client); -+extern void sa_corectl_systrace_c(void); -+#endif /* CONFIG_OPLUS_SCHED_HALT_MASK_PRT */ -+ -+/* define for boost threshold unit */ -+#define BOOST_THRESHOLD_UNIT (51) -+ -+ -+enum UX_STATE_TYPE { -+ UX_STATE_INVALID = 0, -+ UX_STATE_NONE, -+ UX_STATE_STATIC, -+ UX_STATE_INHERIT, -+ UX_STATE_COMBINED, -+ MAX_UX_STATE_TYPE, -+}; -+ -+enum INHERIT_UX_TYPE { -+ INHERIT_UX_BINDER = 0, -+ INHERIT_UX_RWSEM, -+ INHERIT_UX_MUTEX, -+ INHERIT_UX_FUTEX, -+ INHERIT_UX_PIFUTEX, -+ INHERIT_UX_MAX, -+}; -+ -+/* -+ * WANNING: -+ * new flag should be add before MAX_IM_FLAG_TYPE, never change -+ * the value of those existed flag type. -+ */ -+enum IM_FLAG_TYPE { -+ IM_FLAG_NONE = 0, -+ IM_FLAG_SURFACEFLINGER, -+ IM_FLAG_HWC, /* Discarded */ -+ IM_FLAG_RENDERENGINE, -+ IM_FLAG_WEBVIEW, -+ IM_FLAG_CAMERA_HAL, -+ IM_FLAG_AUDIO, -+ IM_FLAG_HWBINDER, -+ IM_FLAG_LAUNCHER, -+ IM_FLAG_LAUNCHER_NON_UX_RENDER, -+ IM_FLAG_SS_LOCK_OWNER, -+ IM_FLAG_FORBID_SET_CPU_AFFINITY = 11, /* forbid setting cpu affinity from app */ -+ IM_FLAG_SYSTEMSERVER_PID, -+ IM_FLAG_MIDASD, -+ IM_FLAG_AUDIO_CAMERA_HAL, /* audio mode disable camera hal ux */ -+ IM_FLAG_AFFINITY_THREAD, -+ IM_FLAG_TPD_SET_CPU_AFFINITY = 16, -+ IM_FLAG_COMPRESS_THREAD = 17, /* compress thread skips locking protect */ -+ IM_FLAG_RENDER_THREAD = 18, -+ MAX_IM_FLAG_TYPE, -+}; -+ -+#define MAX_IM_FLAG_PRIO MAX_IM_FLAG_TYPE -+ -+enum ots_state { -+ OTS_STATE_SET_AFFINITY, -+ OTS_STATE_DDL_ACTIVE, -+ OTS_STATE_DDL_ACTIVE_PREEMPTED, -+ OTS_STATE_MAX, -+}; -+ -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+DECLARE_PER_CPU(u64, retired_instrs); -+DECLARE_PER_CPU(u64, nvcsw); -+DECLARE_PER_CPU(u64, nivcsw); -+#endif -+ -+struct ux_sched_cluster { -+ struct cpumask cpus; -+ unsigned long capacity; -+}; -+ -+#define OPLUS_NR_CPUS (8) -+#define OPLUS_MAX_CLS (5) -+struct ux_sched_cputopo { -+ int cls_nr; -+ struct ux_sched_cluster sched_cls[OPLUS_NR_CPUS]; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ cpumask_t oplus_cpu_array[2*OPLUS_MAX_CLS][OPLUS_MAX_CLS]; -+#endif -+}; -+ -+struct oplus_rq { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_root_cached ux_list; -+ /* a tree to track minimum exec vruntime */ -+ struct rb_root_cached exec_timeline; -+ /* malloc this spinlock_t instead of built-in to shrank size of oplus_rq */ -+ spinlock_t *ux_list_lock; -+ int nr_running; -+ u64 min_vruntime; -+ u64 load_weight; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) -+ struct rb_root_cached ddl_root; -+ spinlock_t *ddl_lock; -+#endif -+ -+#ifdef CONFIG_LOCKING_PROTECT -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ struct list_head locking_thread_list; -+ spinlock_t *locking_list_lock; -+ int rq_locking_task; -+#else -+ struct sched_entity *last_entity; -+#endif -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ struct oplus_lb lb; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+ struct hrtimer *resched_timer; -+ int cpu; -+#endif -+}; -+ -+extern int global_debug_enabled; -+extern int global_sched_assist_enabled; -+extern int global_sched_assist_scene; -+extern int global_silver_perf_core; -+extern int global_sched_group_enabled; -+ -+struct rq; -+ -+#ifdef CONFIG_LOCKING_PROTECT -+struct sched_assist_locking_ops { -+ void (*check_preempt_tick)(struct task_struct *p, -+ unsigned long *ideal_runtime, bool *skip_preempt, -+ unsigned long delta_exec, struct cfs_rq *cfs_rq, -+ struct sched_entity *curr, unsigned int granularity); -+ void (*check_preempt_wakeup)(struct rq *rq, struct task_struct *p, bool *preempt, bool *nopreempt); -+ void (*state_systrace_c)(unsigned int cpu, struct task_struct *p); -+ void (*locking_tick_hit)(struct task_struct *prev, struct task_struct *next); -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ void (*replace_next_task_fair)(struct rq *rq, -+ struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*enqueue_entity)(struct rq *rq, struct task_struct *p); -+ void (*dequeue_entity)(struct rq *rq, struct task_struct *p); -+#else -+ void (*set_last_entity)(struct task_struct *prev, struct task_struct *next, struct rq *rq); -+ void (*pick_last_entity)(struct rq *rq, struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*clear_last_entity)(struct rq *rq, struct task_struct *p); -+#endif -+ void (*opt_ss_lock_contention)(struct task_struct *p, int old_im, int new_im); -+}; -+ -+extern struct sched_assist_locking_ops *locking_ops; -+ -+ -+#define LOCKING_CALL_OP(op, args...) \ -+do { \ -+ if (locking_ops && locking_ops->op) { \ -+ locking_ops->op(args); \ -+ } \ -+} while (0) -+ -+#define LOCKING_CALL_OP_RET(op, args...) \ -+({ \ -+ __typeof__(locking_ops->op(args)) __ret = 0; \ -+ if (locking_ops && locking_ops->op) \ -+ __ret = locking_ops->op(args); \ -+ __ret; \ -+}) -+ -+void register_sched_assist_locking_ops(struct sched_assist_locking_ops *ops); -+#endif -+ -+/* attention: before insert .ko, task's list->prev/next is init with 0 */ -+static inline bool oplus_list_empty(struct list_head *list) -+{ -+ return list_empty(list) || (list->prev == 0 && list->next == 0); -+} -+ -+static inline bool oplus_rbtree_empty(struct rb_root_cached *list) -+{ -+ return (rb_first_cached(list) == NULL); -+} -+ -+/** -+ * Check if there are ux tasks waiting to run on the specified cpu -+ */ -+static inline bool orq_has_ux_tasks(struct oplus_rq *orq) -+{ -+ return !oplus_rbtree_empty(&orq->ux_list); -+} -+ -+/* attention: before insert .ko, task's node->__rb_parent_color is init with 0 */ -+static inline bool oplus_rbnode_empty(struct rb_node *node) -+{ -+ return RB_EMPTY_NODE(node) || (node->__rb_parent_color == 0); -+} -+ -+static inline struct oplus_task_struct *ux_list_first_entry(struct rb_root_cached *list) -+{ -+ struct rb_node *leftmost = rb_first_cached(list); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, ux_entry); -+} -+ -+static inline struct oplus_task_struct *exec_timeline_first_entry(struct rb_root_cached *tree) -+{ -+ struct rb_node *leftmost = rb_first_cached(tree); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, exec_time_node); -+} -+ -+typedef bool (*migrate_task_callback_t)(struct task_struct *tsk, int src_cpu, int dst_cpu); -+extern migrate_task_callback_t fbg_migrate_task_callback; -+typedef void (*android_rvh_schedule_handler_t)(struct task_struct *prev, -+ struct task_struct *next, struct rq *rq); -+extern android_rvh_schedule_handler_t fbg_android_rvh_schedule_callback; -+ -+extern struct kmem_cache *oplus_task_struct_cachep; -+ -+#define ots_to_ts(ots) (ots->task) -+#define OTS_IDX 0 -+#define ORQ_IDX 0 -+ -+static inline bool test_task_is_fair(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid CFS priority is MAX_RT_PRIO..MAX_PRIO-1 */ -+ if ((task->prio >= MAX_RT_PRIO) && (task->prio <= MAX_PRIO-1)) -+ return true; -+ return false; -+} -+ -+static inline bool test_task_is_rt(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid RT priority is 0..MAX_RT_PRIO-1 */ -+ if (rt_prio(task->prio)) -+ return true; -+ -+ return false; -+} -+ -+static inline struct oplus_task_struct *get_oplus_task_struct(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = NULL; -+ -+ /* not Skip idle thread */ -+ if (!t) -+ return NULL; -+ -+ ots = (struct oplus_task_struct *) READ_ONCE(t->android_oem_data1[OTS_IDX]); -+ if (IS_ERR_OR_NULL(ots)) -+ return NULL; -+ -+ return ots; -+} -+ -+static inline unsigned long oplus_get_im_flag(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return IM_FLAG_NONE; -+ -+ return ots->im_flag; -+} -+ -+static inline bool is_optimized_audio_thread(struct task_struct *t) -+{ -+ unsigned long im_flag; -+ -+ im_flag = oplus_get_im_flag(t); -+ if (test_bit(IM_FLAG_AUDIO, &im_flag)) -+ return true; -+ -+ return false; -+} -+ -+static inline void oplus_set_im_flag(struct task_struct *t, int im_flag) -+{ -+ struct oplus_task_struct *ots = NULL; -+ ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ set_bit(im_flag, &ots->im_flag); -+} -+ -+static inline int get_ots_ux_state(struct oplus_task_struct *ots) -+{ -+ int ux_state; -+ int sub_ux_state; -+ int ux_prio; -+ int ret; -+ -+ ux_prio = ots->ux_state & SCHED_ASSIST_UX_PRIORITY_MASK; -+ ux_state = ots->ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ sub_ux_state = ots->sub_ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ -+ if (sub_ux_state) { -+ ret = ux_state | (sub_ux_state & ~SA_TYPE_INHERIT) | SA_TYPE_COMBINED; -+ } else { -+ ret = ux_state; -+ } -+ -+ if (ret & SCHED_ASSIST_UX_MASK) { -+ return ux_prio | ret; -+ } -+ -+ return 0; -+} -+ -+bool is_multiple_ux(struct oplus_task_struct *ots); -+ -+static inline int oplus_get_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return get_ots_ux_state(ots); -+} -+ -+static inline int oplus_get_static_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return 0; -+ } -+ return ots->ux_state; -+} -+ -+static inline int oplus_get_sub_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->sub_ux_state; -+} -+ -+static inline int oplus_get_inherited_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return ots->ux_state; -+ } -+ return ots->sub_ux_state; -+} -+ -+void oplus_set_ux_state_lock(struct task_struct *t, int ux_state, int inherit_type, bool need_lock_rq); -+ -+static inline s64 oplus_get_inherit_ux(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return atomic64_read(&ots->inherit_ux); -+} -+ -+static inline void oplus_set_inherit_ux(struct task_struct *t, s64 inherit_ux) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ atomic64_set(&ots->inherit_ux, inherit_ux); -+} -+ -+static inline int oplus_get_ux_depth(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->ux_depth; -+} -+ -+static inline void oplus_set_ux_depth(struct task_struct *t, int ux_depth) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->ux_depth = ux_depth; -+} -+ -+static inline u64 oplus_get_enqueue_time(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->enqueue_time; -+} -+ -+static inline void oplus_set_enqueue_time(struct task_struct *t, u64 enqueue_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->enqueue_time = enqueue_time; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* -+ * Record the number of task context switches -+ * and scheduling delays when enqueueing a task. -+ */ -+ ots->snap_pcount = t->sched_info.pcount; -+ ots->snap_run_delay = t->sched_info.run_delay; -+#endif -+} -+ -+static inline void oplus_set_inherit_ux_start(struct task_struct *t, u64 start_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->inherit_ux_start = t->se.sum_exec_runtime; -+} -+ -+static inline void init_task_ux_info(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ RB_CLEAR_NODE(&ots->ux_entry); -+ RB_CLEAR_NODE(&ots->exec_time_node); -+ ots->ux_state = 0; -+ ots->sub_ux_state = 0; -+ atomic64_set(&ots->inherit_ux, 0); -+ ots->ux_depth = 0; -+ ots->enqueue_time = 0; -+ ots->inherit_ux_start = 0; -+ ots->ux_priority = -1; -+ ots->ux_nice = -1; -+ ots->vruntime = 0; -+ ots->preset_vruntime = 0; -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG) -+ ots->abnormal_flag = 0; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_SCHED_SPREAD -+ ots->lb_state = 0; -+ ots->ld_flag = 0; -+#endif -+ ots->exec_calc_runtime = 0; -+ ots->is_update_runtime = 0; -+ ots->target_process = -1; -+ ots->wake_tid = 0; -+ ots->running_start_time = 0; -+ ots->update_running_start_time = false; -+ ots->last_wake_ts = 0; -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ memset(&ots->lkinfo, 0, sizeof(struct locking_info)); -+ INIT_LIST_HEAD(&ots->lkinfo.node); -+/*#endif*/ -+ ots->block_start_time = 0; -+#ifdef CONFIG_LOCKING_PROTECT -+ INIT_LIST_HEAD(&ots->locking_entry); -+ ots->locking_start_time = 0; -+ ots->locking_depth = 0; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ ots->snap_pcount = 0; -+ ots->snap_run_delay = 0; -+ plist_node_init(&ots->rtb, MAX_IM_FLAG_PRIO); -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+ atomic_set(&ots->pipeline_cpu, -1); -+#endif -+ -+#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO) -+ ots->amu_cycle = 0; -+ ots->amu_instruct = 0; -+#endif -+}; -+ -+static inline bool test_ux_type(struct task_struct *task, int ux_type) -+{ -+ return oplus_get_ux_state(task) & ux_type; -+} -+ -+static inline bool is_heavy_ux_task(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return false; -+ -+ return get_ots_ux_state(ots) & SA_TYPE_HEAVY; -+} -+ -+static inline bool sched_assist_scene(unsigned int scene) -+{ -+ if (unlikely(!global_sched_assist_enabled)) -+ return false; -+ -+ return global_sched_assist_scene & scene; -+} -+ -+static inline unsigned long oplus_task_util(struct task_struct *p) -+{ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+ struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; -+ -+ return wts->demand_scaled; -+#else -+ return READ_ONCE(p->se.avg.util_avg); -+#endif -+} -+ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+static inline u32 task_wts_sum(struct task_struct *tsk) -+{ -+ struct walt_task_struct *wts = (struct walt_task_struct *) tsk->android_vendor_data1; -+ -+ return wts->sum; -+} -+#endif -+ -+bool is_min_cluster(int cpu); -+bool is_max_cluster(int cpu); -+bool is_mid_cluster(int cpu); -+bool im_mali(const char *comm); -+bool is_top(struct task_struct *p); -+bool task_is_runnable(struct task_struct *task); -+struct oplus_rq *get_oplus_rq(struct rq *rq); -+ -+typedef unsigned long (*kallsyms_lookup_name_t)(const char *name); -+extern kallsyms_lookup_name_t _kallsyms_lookup_name; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+int ux_mask_to_prio(int ux_mask); -+int ux_prio_to_mask(int prio); -+#endif -+ -+noinline int tracing_mark_write(const char *buf); -+void hwbinder_systrace_c(unsigned int cpu, int flag); -+void sched_assist_init_oplus_rq(void); -+void queue_ux_thread(struct rq *rq, struct task_struct *p, int enqueue); -+ -+void inherit_ux_inc(struct task_struct *task, int type); -+void inherit_ux_sub(struct task_struct *task, int type, int value); -+void set_inherit_ux(struct task_struct *task, int type, int depth, int inherit_val); -+void reset_inherit_ux(struct task_struct *inherit_task, struct task_struct *ux_task, int reset_type); -+void unset_inherit_ux(struct task_struct *task, int type); -+void unset_inherit_ux_value(struct task_struct *task, int type, int value); -+void inc_inherit_ux_refs(struct task_struct *task, int type); -+void clear_all_inherit_type(struct task_struct *p); -+int get_max_inherit_gran(struct task_struct *p); -+ -+bool is_heavy_load_top_task(struct task_struct *p); -+bool test_task_is_fair(struct task_struct *task); -+bool test_task_is_rt(struct task_struct *task); -+ -+bool test_task_ux(struct task_struct *task); -+bool test_task_ux_depth(int ux_depth); -+bool test_inherit_ux(struct task_struct *task, int type); -+bool test_set_inherit_ux(struct task_struct *task); -+int get_ux_state_type(struct task_struct *task); -+void sched_assist_target_comm(struct task_struct *task, const char *comm); -+unsigned int ux_task_exec_limit(struct task_struct *p); -+ -+void update_ux_sched_cputopo(void); -+bool is_task_util_over(struct task_struct *tsk, int threshold); -+bool oplus_task_misfit(struct task_struct *tsk, int cpu); -+ssize_t oplus_show_cpus(const struct cpumask *mask, char *buf); -+void adjust_rt_lowest_mask(struct task_struct *p, struct cpumask *local_cpu_mask, int ret, bool force_adjust); -+bool sa_skip_rt_sync(struct rq *rq, struct task_struct *p, bool *sync); -+void ux_state_systrace_c(unsigned int cpu, struct task_struct *p); -+bool sa_rt_skip_ux_cpu(int cpu); -+int is_vip_mvp(struct task_struct *p); -+ -+/* s64 account_ux_runtime(struct rq *rq, struct task_struct *curr); */ -+void opt_ss_lock_contention(struct task_struct *p, unsigned long old_im, int new_im); -+ -+/* register vender hook in kernel/sched/topology.c */ -+void android_vh_build_sched_domains_handler(void *unused, bool has_asym); -+ -+/* register vender hook in kernel/sched/rt.c */ -+void android_rvh_select_task_rq_rt_handler(void *unused, struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags, int *new_cpu); -+void android_rvh_find_lowest_rq_handler(void *unused, struct task_struct *p, struct cpumask *local_cpu_mask, int ret, int *best_cpu); -+ -+/* register vender hook in kernel/sched/core.c */ -+void android_rvh_sched_fork_handler(void *unused, struct task_struct *p); -+void android_rvh_after_enqueue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_dequeue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_schedule_handler(void *unused, struct task_struct *prev, struct task_struct *next, struct rq *rq); -+void android_vh_scheduler_tick_handler(void *unused, struct rq *rq); -+ -+void android_vh_account_process_tick_gran_handler(void *unused, struct task_struct *p, struct rq *rq, int user_tick, int *ticks); -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+void sa_sched_switch_handler(void *unused, bool preempt, struct task_struct *prev, struct task_struct *next, unsigned int prev_state); -+#endif -+ -+void set_im_flag_with_bit(int im_flag, struct task_struct *task); -+ -+/* register vendor hook in kernel/cgroup/cgroup-v1.c */ -+void android_vh_cgroup_set_task_handler(void *unused, int ret, struct task_struct *task); -+/* register vendor hook in kernel/signal.c */ -+void android_vh_exit_signal_handler(void *unused, struct task_struct *p); -+void android_rvh_set_cpus_allowed_by_task_handler(void *unused, const struct cpumask *cpu_valid_mask, -+ const struct cpumask *new_mask, struct task_struct *task, unsigned int *dest_cpu); -+void android_rvh_setscheduler_handler(void *unused, struct task_struct *p); -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_BAN_APP_SET_AFFINITY) -+void android_vh_sched_setaffinity_early_handler(void *unused, struct task_struct *task, const struct cpumask *new_mask, bool *skip); -+#endif -+ -+#ifdef CONFIG_OPLUS_SCHED_GROUP_OPT -+void android_vh_reweight_entity_handler(void *unused, struct sched_entity *se); -+#endif -+ -+#ifdef CONFIG_BLOCKIO_UX_OPT -+int sa_blockio_init(void); -+void sa_blockio_exit(void); -+#endif -+extern struct notifier_block process_exit_notifier_block; -+#endif /* _OPLUS_SA_COMMON_H_ */ -diff --git a/include/linux/sched/sa_common_struct.h b/include/linux/sched/sa_common_struct.h -new file mode 100755 -index 000000000000..876feff48b36 ---- /dev/null -+++ b/include/linux/sched/sa_common_struct.h -@@ -0,0 +1,277 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+/* -+this file is splited from the sa_common.h to adapt the OKI, -+IS_ENABLED is not allowed in here, because the macro will not work in OKI. -+*/ -+#ifndef _OPLUS_SA_COMMON_STRUCT_H_ -+#define _OPLUS_SA_COMMON_STRUCT_H_ -+ -+#define MAX_CLUSTER (4) -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+/* hot-thread */ -+struct task_record { -+#define RECOED_WINSIZE (1 << 8) -+#define RECOED_WINIDX_MASK (RECOED_WINSIZE - 1) -+ u8 winidx; -+ u8 count; -+}; -+ -+#define MAX_TASK_COMM_LEN 256 -+struct uid_struct { -+ uid_t uid; -+ u64 uid_total_cycle; -+ u64 uid_total_inst; -+ spinlock_t lock; -+ char leader_comm[TASK_COMM_LEN]; -+ char cmdline[MAX_TASK_COMM_LEN]; -+}; -+ -+struct amu_uid_entry { -+ uid_t uid; -+ struct uid_struct *uid_struct; -+ struct hlist_node node; -+}; -+ -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+struct locking_info { -+ u64 waittime_stamp; -+ u64 holdtime_stamp; -+ /* Used in torture acquire latency statistic.*/ -+ u64 acquire_stamp; -+ /* -+ * mutex or rwsem optimistic spin start time. Because a task -+ * can't spin both on mutex and rwsem at one time, use one common -+ * threshold time is OK. -+ */ -+ u64 opt_spin_start_time; -+ struct task_struct *holder; -+ u32 waittype; -+ bool ux_contrib; -+ /* -+ * Whether task is ux when it's going to be added to mutex or -+ * rwsem waiter list. It helps us check whether there is ux -+ * task on mutex or rwsem waiter list. Also, a task can't be -+ * added to both mutex and rwsem at one time, so use one common -+ * field is OK. -+ */ -+ bool is_block_ux; -+ u32 kill_flag; -+ /* for cfs enqueue smoothly.*/ -+ struct list_head node; -+ struct task_struct *owner; -+ struct list_head lock_head; -+ u64 clear_seq; -+ atomic_t lock_depth; -+}; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+#define RAVG_HIST_SIZE 5 -+#define SCX_SLICE_DFL (1 * NSEC_PER_MSEC) -+#define SCX_SLICE_INF U64_MAX -+#define DEFAULT_CGROUP_DL_IDX (8) -+#define EXT_FLAG_RT_CHANGED (1 << 0) -+#define EXT_FLAG_CFS_CHANGED (1 << 1) -+struct scx_task_stats { -+ u64 mark_start; -+ u64 window_start; -+ u32 sum; -+ u32 sum_history[RAVG_HIST_SIZE]; -+ int cidx; -+ u32 demand; -+ u16 demand_scaled; -+ void *sdsq; -+}; -+/* -+ * The following is embedded in task_struct and contains all fields necessary -+ * for a task to be scheduled by SCX. -+ */ -+struct scx_entity { -+ struct scx_dispatch_q *dsq; -+ struct { -+ struct list_head fifo; /* dispatch order */ -+ struct rb_node priq; /* p->scx.dsq_vtime order */ -+ } dsq_node; -+ u32 flags; /* protected by rq lock */ -+ u32 dsq_flags; /* protected by dsq lock */ -+ s32 sticky_cpu; -+ unsigned long runnable_at; -+ u64 slice; -+ u64 dsq_vtime; -+ int gdsq_idx; -+ int ext_flags; -+ int prio_backup; -+ unsigned long sched_prop; -+ struct scx_task_stats sts; -+}; -+/*#endif*/ -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ -+#define MAX_CPU_FREQ_STATE 32 -+#define MAX_CPU_CNT 8 -+ -+#define STATE_IN_POOL 0 -+#define STATE_ACTIVE 1 -+ -+struct powermodel_freq_task_state { -+ u8 powermodel_enqueued:1; -+ u8 powermodel_index:7; -+}; -+ -+struct powermodel_cpu_task_state { -+ struct oplus_task_struct *ots; -+ struct powermodel_freq_task_state powermodel_freq_task_states[MAX_CPU_FREQ_STATE]; -+ u64 powermodel_last_seq; -+ u64 last_seq; -+ struct list_head node; -+ int state; -+ int pool_id; -+}; -+ -+#endif -+ -+ -+/* Please add your own members of task_struct here :) */ -+struct oplus_task_struct { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_node ux_entry; -+ struct rb_node exec_time_node; -+ struct task_struct *task; -+ atomic64_t inherit_ux; -+ u64 enqueue_time; -+ u64 inherit_ux_start; -+ /* u64 sum_exec_baseline; */ -+ u64 total_exec; -+ u64 vruntime; -+ u64 preset_vruntime; -+ /* contains ux state -+ 1. if static and inherited ux both exist, static ux stores in ux_state, inherited ux in sub_ux_state. -+ 2. if only static ux exists, static ux stores in ux_state. -+ 2. if only inherited ux exists, inherited ux stores in ux_state */ -+ int ux_state; -+ int sub_ux_state; -+ u8 ux_depth; -+ s8 ux_priority; -+ s8 ux_nice; -+ pid_t affinity_pid; -+ pid_t affinity_tgid; -+ unsigned long state; -+ unsigned long im_flag; -+ atomic_t is_vip_mvp; -+ -+/* #if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) */ -+ u64 ddl; -+ u64 ddl_active_ts; -+ u64 runnable_ts; -+ struct rb_node ddl_node; -+/* #endif */ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG)*/ -+ int abnormal_flag; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_SCHED_SPREAD */ -+ int lb_state; -+ int ld_flag:1; -+ /* CONFIG_OPLUS_FEATURE_TASK_LOAD */ -+ int is_update_runtime:1; -+ int target_process; -+ u64 wake_tid; -+ u64 running_start_time; -+ bool update_running_start_time; -+ u64 exec_calc_runtime; -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct task_record record[MAX_CLUSTER]; /* 2*u64 */ -+ u64 block_start_time; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_FRAME_BOOST */ -+ struct list_head fbg_list; -+ raw_spinlock_t fbg_list_entry_lock; -+ bool fbg_running; /* task belongs to a group, and in running */ -+ u16 fbg_state; -+ s8 preferred_cluster_id; -+ s8 fbg_depth; -+ u64 last_wake_ts; -+ int fbg_cur_group; -+/*#ifdef CONFIG_LOCKING_PROTECT*/ -+ unsigned long locking_start_time; -+ unsigned long last_jiffies; -+ struct list_head locking_entry; -+ int locking_depth; -+ int lk_tick_hit; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ struct locking_info lkinfo; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+ struct scx_entity scx; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_FDLEAK_CHECK)*/ -+ u8 fdleak_flag; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+ /* for loadbalance */ -+ struct plist_node rtb; /* rt boost task */ -+ -+ /* -+ * The following variables are used to calculate the time -+ * a task spends in the running/runnable state. -+ */ -+ u64 snap_run_delay; -+ unsigned long snap_pcount; -+/*#endif*/ -+#if IS_ENABLED(CONFIG_OPLUS_SCHED_TUNE) -+ int stune_idx; -+#endif -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE)*/ -+ atomic_t pipeline_cpu; -+/*#endif*/ -+ -+ /* for oplus secure guard */ -+ int sg_flag; -+ int sg_scno; -+ uid_t sg_uid; -+ uid_t sg_euid; -+ gid_t sg_gid; -+ gid_t sg_egid; -+/*#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct uid_struct *uid_struct; -+ u64 amu_instruct; -+ u64 amu_cycle; -+/*#endif*/ -+ /* for binder ux */ -+ int binder_async_ux_enable; -+ bool binder_async_ux_sts; -+ int binder_thread_mode; -+ struct binder_node *binder_thread_node; -+ -+/* for powermodel */ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ struct powermodel_cpu_task_state *powermodel_cpu_task_states[MAX_CPU_CNT]; -+ u64 exec_runtime; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_CFBT) -+ int cfbt_cur_group; -+ bool cfbt_running; -+#endif /* CONFIG_OPLUS_FEATURE_SCHED_CFBT */ -+} ____cacheline_aligned; -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+#define INVALID_PID (-1) -+struct oplus_lb { -+ /* used for active_balance to record the running task. */ -+ pid_t pid; -+}; -+/*#endif*/ -+ -+#endif /* _OPLUS_SA_COMMON_STRUCT_H_ */ -+ -diff --git a/include/linux/sched/sa_oemdata.h b/include/linux/sched/sa_oemdata.h -new file mode 100755 -index 000000000000..ebbbc754e391 ---- /dev/null -+++ b/include/linux/sched/sa_oemdata.h -@@ -0,0 +1,13 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_OEMDATA_H_ -+#define _OPLUS_SA_OEMDATA_H_ -+ -+int sa_oemdata_init(void); -+void sa_oemdata_deinit(void); -+ -+#endif /* _OPLUS_SA_OEMDATA_H_ */ -diff --git a/include/linux/sched/sched_ext.h b/include/linux/sched/sched_ext.h -new file mode 100755 -index 000000000000..9f1afdb8a04b ---- /dev/null -+++ b/include/linux/sched/sched_ext.h -@@ -0,0 +1,57 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef _OPLUS_SCHED_EXT_H -+#define _OPLUS_SCHED_EXT_H -+#include "sa_common.h" -+ -+#define SCHED_PROP_TOP_THREAD_SHIFT (8) -+#define SCHED_PROP_TOP_THREAD_MASK (0xf << SCHED_PROP_TOP_THREAD_SHIFT) -+#define SCHED_PROP_DEADLINE_MASK (0xFF) /* deadline for ext sched class */ -+#define SCHED_PROP_DEADLINE_LEVEL1 (1) /* 1ms for user-aware audio tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL2 (2) /* 2ms for user-aware touch tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL3 (3) /* 4ms for user aware dispaly tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL4 (4) /* 6ms */ -+#define SCHED_PROP_DEADLINE_LEVEL5 (5) /* 8ms */ -+#define SCHED_PROP_DEADLINE_LEVEL6 (6) /* 16ms */ -+#define SCHED_PROP_DEADLINE_LEVEL7 (7) /* 32ms */ -+#define SCHED_PROP_DEADLINE_LEVEL8 (8) /* 64ms */ -+#define SCHED_PROP_DEADLINE_LEVEL9 (9) /* 128ms */ -+ -+static inline int sched_prop_get_top_thread_id(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ return -EPERM; -+ } -+ -+ return ((ots->scx.sched_prop & SCHED_PROP_TOP_THREAD_MASK) >> SCHED_PROP_TOP_THREAD_SHIFT); -+} -+ -+static inline int sched_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_set_sched_prop failed! fn=%s\n", __func__); -+ return -EPERM; -+ } -+ -+ ots->scx.sched_prop = sp; -+ return 0; -+} -+ -+static inline unsigned long sched_get_sched_prop(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_get_sched_prop failed! fn=%s\n", __func__); -+ return (unsigned long)-1; -+ } -+ return ots->scx.sched_prop; -+} -+ -+#endif /*_OPLUS_SCHED_EXT_H */ -diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h -index 03d35e3eda3c..6a5f0c0d74bd 100644 ---- a/include/linux/sched/task.h -+++ b/include/linux/sched/task.h -@@ -61,8 +61,10 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); - extern void init_idle(struct task_struct *idle, int cpu); - - extern int sched_fork(unsigned long clone_flags, struct task_struct *p); --extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); --extern void sched_cancel_fork(struct task_struct *p); -+extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); -+#ifdef CONFIG_HMBIRD_SCHED -+extern void hmbird_cancel_fork(struct task_struct *p); -+#endif - extern void sched_post_fork(struct task_struct *p); - extern void sched_dead(struct task_struct *p); - -diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h -index 359a14cc76a4..969716ab4320 100644 ---- a/include/uapi/linux/sched.h -+++ b/include/uapi/linux/sched.h -@@ -118,7 +118,7 @@ struct clone_args { - /* SCHED_ISO: reserved but not implemented yet */ - #define SCHED_IDLE 5 - #define SCHED_DEADLINE 6 --#define SCHED_EXT 7 -+#define SCHED_HMBIRD 7 - - /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ - #define SCHED_RESET_ON_FORK 0x40000000 -diff --git a/init/Kconfig b/init/Kconfig -index 0ead65803ecf..fbbc4a970b00 100644 ---- a/init/Kconfig -+++ b/init/Kconfig -@@ -1030,10 +1030,6 @@ config RT_GROUP_SCHED - realtime bandwidth for them. - See Documentation/scheduler/sched-rt-group.rst for more information. - --config EXT_GROUP_SCHED -- bool -- depends on SCHED_CLASS_EXT && CGROUP_SCHED -- default y - endif #CGROUP_SCHED - - config SCHED_MM_CID -diff --git a/init/init_task.c b/init/init_task.c -index 69a046388995..31ceb0e469f7 100644 ---- a/init/init_task.c -+++ b/init/init_task.c -@@ -6,7 +6,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -102,9 +101,6 @@ struct task_struct init_task - #endif - #ifdef CONFIG_CGROUP_SCHED - .sched_task_group = &root_task_group, --#endif --#ifdef CONFIG_SLIM_SCHED -- .scx = NULL, - #endif - .ptraced = LIST_HEAD_INIT(init_task.ptraced), - .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), -diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt -index cf22ef6107b8..b131d5662b48 100644 ---- a/kernel/Kconfig.preempt -+++ b/kernel/Kconfig.preempt -@@ -133,12 +133,8 @@ config SCHED_CORE - which is the likely usage by Linux distributions, there should - be no measurable impact on performance. - --config SLIM_SCHED -- bool "slim sched" -- default n --config SCHED_CLASS_EXT -- bool "Extensible Scheduling Class" -- depends on BPF_SYSCALL && BPF_JIT -+config HMBIRD_SCHED -+ bool "hmbird Scheduling Class(base on ext scheduler)" - help - This option enables a new scheduler class sched_ext (SCX), which - allows scheduling policies to be implemented as BPF programs to -@@ -159,3 +155,10 @@ config SCHED_CLASS_EXT - similar to struct sched_class. - - See Documentation/scheduler/sched-ext.rst for more details. -+ -+config DYNAMIC_WALT_SUPPORT -+ bool "Dynamic WALT Support for Extensible Scheduling Class" -+ depends on HMBIRD_SCHED -+ default n -+ help -+ Enable dynamic WALT support for Extensible Scheduling Class. -diff --git a/kernel/bpf/bpf_struct_ops_types.h b/kernel/bpf/bpf_struct_ops_types.h -index 3618769d853d..5678a9ddf817 100644 ---- a/kernel/bpf/bpf_struct_ops_types.h -+++ b/kernel/bpf/bpf_struct_ops_types.h -@@ -9,8 +9,4 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) - #include - BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) - #endif --#ifdef CONFIG_SCHED_CLASS_EXT --#include --BPF_STRUCT_OPS_TYPE(sched_ext_ops) --#endif - #endif -diff --git a/kernel/fork.c b/kernel/fork.c -index d0a46a152428..7a91505b9378 100644 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -23,7 +23,9 @@ - #include - #include - #include --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - #include - #include - #include -@@ -999,7 +1001,9 @@ void __put_task_struct(struct task_struct *tsk) - WARN_ON(!tsk->exit_state); - WARN_ON(refcount_read(&tsk->usage)); - WARN_ON(tsk == current); -- sched_ext_free(tsk); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_free(tsk); -+#endif - io_uring_free(tsk); - cgroup_free(tsk); - task_numa_free(tsk, true); -@@ -2516,7 +2520,11 @@ __latent_entropy struct task_struct *copy_process( - - retval = perf_event_init_task(p, clone_flags); - if (retval) -- goto bad_fork_sched_cancel_fork; -+#ifdef CONFIG_HMBIRD_SCHED -+ goto bad_fork_hmbird_cancel_fork; -+#else -+ goto bad_fork_cleanup_policy; -+#endif - retval = audit_alloc(p); - if (retval) - goto bad_fork_cleanup_perf; -@@ -2648,9 +2656,7 @@ __latent_entropy struct task_struct *copy_process( - * cgroup specific, it unconditionally needs to place the task on a - * runqueue. - */ -- retval = sched_cgroup_fork(p, args); -- if (retval) -- goto bad_fork_cancel_cgroup; -+ sched_cgroup_fork(p, args); - - /* - * From this point on we must avoid any synchronous user-space -@@ -2696,13 +2702,13 @@ __latent_entropy struct task_struct *copy_process( - /* Don't start children in a dying pid namespace */ - if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { - retval = -ENOMEM; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* Let kill terminate clone/fork in the middle */ - if (fatal_signal_pending(current)) { - retval = -EINTR; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* No more failure paths after this point. */ -@@ -2779,11 +2785,10 @@ __latent_entropy struct task_struct *copy_process( - - return p; - --bad_fork_core_free: -+bad_fork_cancel_cgroup: - sched_core_free(p); - spin_unlock(¤t->sighand->siglock); - write_unlock_irq(&tasklist_lock); --bad_fork_cancel_cgroup: - cgroup_cancel_fork(p, args); - bad_fork_put_pidfd: - if (clone_flags & CLONE_PIDFD) { -@@ -2822,8 +2827,10 @@ __latent_entropy struct task_struct *copy_process( - audit_free(p); - bad_fork_cleanup_perf: - perf_event_free_task(p); --bad_fork_sched_cancel_fork: -- sched_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+bad_fork_hmbird_cancel_fork: -+ hmbird_cancel_fork(p); -+#endif - bad_fork_cleanup_policy: - lockdep_free_task(p); - #ifdef CONFIG_NUMA -diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c -index 005025f55bea..c091539a0f60 100644 ---- a/kernel/sched/build_policy.c -+++ b/kernel/sched/build_policy.c -@@ -28,8 +28,10 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED - #include - #include -+#endif - - #include - -@@ -54,6 +56,10 @@ - #include "cputime.c" - #include "deadline.c" - --#ifdef CONFIG_SCHED_CLASS_EXT --# include "ext.c" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/hmbird_util_track.c" -+#include "hmbird/hmbird_sched_proc.c" -+#include "hmbird/hmbird_shadow_tick.c" -+#include "hmbird/hmbird.c" -+#include "hmbird/hmbird_misc.c" - #endif -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index d486b2dd8240..692c9aa0ffa6 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -96,6 +96,12 @@ - #include "../../io_uring/io-wq.h" - #include "../smpboot.h" - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/slim.h" -+#include "hmbird/hmbird_shadow_tick.h" -+#include "hmbird.h" -+#endif -+ - #include - #include - #include -@@ -170,7 +176,6 @@ __read_mostly int scheduler_running; - - DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); - -- - /* kernel prio, less is more */ - static inline int __task_prio(const struct task_struct *p) - { -@@ -183,8 +188,8 @@ static inline int __task_prio(const struct task_struct *p) - if (p->sched_class == &idle_sched_class) - return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - --#ifdef CONFIG_SCHED_CLASS_EXT -- if (p->sched_class == &ext_sched_class) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (p->sched_class == &hmbird_sched_class) - return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ - #endif - -@@ -217,9 +222,9 @@ static inline bool prio_less(const struct task_struct *a, - if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ - return cfs_prio_less(a, b, in_fi); - --#ifdef CONFIG_SCHED_CLASS_EXT -+#ifdef CONFIG_HMBIRD_SCHED - if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ -- return scx_prio_less(a, b, in_fi); -+ return hmbird_prio_less(a, b, in_fi); - #endif - - return false; -@@ -1279,17 +1284,26 @@ bool sched_can_stop_tick(struct rq *rq) - if (fifo_nr_running) - return true; - -+#ifdef CONFIG_HMBIRD_SCHED - /* -- * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks -+ * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks - * left. For CFS, if there's more than one we need the tick for -- * involuntary preemption. For SCX, ask. -+ * involuntary preemption. For HMBIRD, ask. - */ -- if (!scx_switched_all() && rq->nr_running > 1) -+ if (!hmbird_enabled() && rq->nr_running > 1) - return false; - -- if (scx_enabled() && !scx_can_stop_tick(rq)) -+ if (hmbird_enabled() && !hmbird_can_stop_tick(rq)) - return false; -- -+#else -+ /* -+ * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; -+ * if there's more than one we need the tick for involuntary -+ * preemption. -+ */ -+ if (rq->nr_running > 1) -+ return false; -+#endif - /* - * If there is one task and it has CFS runtime bandwidth constraints - * and it's on the cpu now we don't want to stop the tick. -@@ -2222,10 +2236,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) - } - EXPORT_SYMBOL_GPL(deactivate_task); - --struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) -+#ifdef CONFIG_HMBIRD_SCHED -+struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - { -- struct sched_change_guard cg = { -+ struct hmbird_sched_change_guard cg = { - .rq = rq, - .p = p, - .queued = task_on_rq_queued(p), -@@ -2247,7 +2262,7 @@ sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - return cg; - } - --void sched_change_guard_fini(struct sched_change_guard *cg, int flags) -+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags) - { - if (cg->queued) - enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); -@@ -2255,6 +2270,7 @@ void sched_change_guard_fini(struct sched_change_guard *cg, int flags) - set_next_task(cg->rq, cg->p); - cg->done = true; - } -+#endif - - static inline int __normal_prio(int policy, int rt_prio, int nice) - { -@@ -2320,9 +2336,9 @@ inline int task_curr(const struct task_struct *p) - * this means any call to check_class_changed() must be followed by a call to - * balance_callback(). - */ --void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio) -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) - { - if (prev_class != p->sched_class) { - if (prev_class->switched_from) -@@ -2885,6 +2901,7 @@ static void - __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - { - struct rq *rq = task_rq(p); -+ bool queued, running; - - /* - * This here violates the locking rules for affinity, since we're only -@@ -2903,9 +2920,26 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - else - lockdep_assert_held(&p->pi_lock); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->sched_class->set_cpus_allowed(p, ctx); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ -+ if (queued) { -+ /* -+ * Because __kthread_bind() calls this on blocked tasks without -+ * holding rq->lock. -+ */ -+ lockdep_assert_rq_held(rq); -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); - } -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->sched_class->set_cpus_allowed(p, ctx); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - } - - /* -@@ -3772,7 +3806,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class)) -+#else - if (unlikely(!is_cpu_allowed(p, cpu))) -+#endif - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -@@ -4891,7 +4929,9 @@ late_initcall(sched_core_sysctl_init); - */ - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { -+#ifdef CONFIG_HMBIRD_SCHED - int ret; -+#endif - - trace_android_rvh_sched_fork(p); - -@@ -4932,21 +4972,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_reset_on_fork = 0; - } - -- scx_pre_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ ret = hmbird_pre_fork(p); -+ if (ret) -+ goto out_cancel; - - if (dl_prio(p->prio)) { - ret = -EAGAIN; - goto out_cancel; --#ifdef CONFIG_SCHED_CLASS_EXT -- } else if (task_on_scx(p)) { -- p->sched_class = &ext_sched_class; --#endif -+ } else if (task_on_hmbird(p)) { -+ p->sched_class = &hmbird_sched_class; - } else if (rt_prio(p->prio)) { - p->sched_class = &rt_sched_class; - } else { - p->sched_class = &fair_sched_class; - } -- -+#else -+ if (dl_prio(p->prio)) -+ return -EAGAIN; -+ else if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - init_entity_runnable_average(&p->se); - trace_android_rvh_finish_prio_fork(p); - -@@ -4965,12 +5013,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - #endif - return 0; - -+#ifdef CONFIG_HMBIRD_SCHED - out_cancel: -- scx_cancel_fork(p); -+ hmbird_cancel_fork(p); - return ret; -+#endif - } - --int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) -+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - { - unsigned long flags; - -@@ -4997,19 +5047,17 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -- -- return scx_fork(p); --} -- --void sched_cancel_fork(struct task_struct *p) --{ -- scx_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_fork(p); -+#endif - } - - void sched_post_fork(struct task_struct *p) - { - uclamp_post_fork(p); -- scx_post_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_post_fork(p); -+#endif - } - - unsigned long to_ratio(u64 period, u64 runtime) -@@ -5874,19 +5922,28 @@ void scheduler_tick(void) - if (sched_feat(LATENCY_WARN) && resched_latency) - resched_latency_warn(cpu, resched_latency); - -- scx_notify_sched_tick(); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_notify_sched_tick(); -+#endif - perf_event_task_tick(); - - if (curr->flags & PF_WQ_WORKER) - wq_worker_tick(curr); - - #ifdef CONFIG_SMP -- if (!scx_switched_all()) { -+#ifdef CONFIG_HMBIRD_SCHED -+ if (!hmbird_enabled()) { -+#endif - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); -+#ifdef CONFIG_HMBIRD_SCHED - } -+#endif - #endif - trace_android_vh_scheduler_tick(rq); -+#ifdef CONFIG_HMBIRD_SCHED -+ scheduler_tick_handler(NULL, NULL); -+#endif - } - - #ifdef CONFIG_NO_HZ_FULL -@@ -6188,7 +6245,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ -+#ifdef CONFIG_HMBIRD_SCHED - for_balance_class_range(class, prev->sched_class, &idle_sched_class) { -+#else -+ for_class_range(class, prev->sched_class, &idle_sched_class) { -+#endif - if (class->balance(rq, prev, rf)) - break; - } -@@ -6206,9 +6267,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - const struct sched_class *class; - struct task_struct *p; - -- if (scx_enabled()) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (hmbird_enabled()) - goto restart; -- -+#endif - /* - * Optimization: we know that if all tasks are in the fair class we can - * call that function directly, but only if the @prev task wasn't of a -@@ -6234,13 +6296,21 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - restart: - put_prev_task_balance(rq, prev, rf); - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { - p = class->pick_next_task(rq); - if (p) { -- scx_notify_pick_next_task(rq, p, class); -+ hmbird_notify_pick_next_task(rq, p, class); - return p; - } - } -+#else -+ for_each_class(class) { -+ p = class->pick_next_task(rq); -+ if (p) -+ return p; -+ } -+#endif - - BUG(); /* The idle class should always have a runnable task. */ - } -@@ -6269,7 +6339,11 @@ static inline struct task_struct *pick_task(struct rq *rq) - const struct sched_class *class; - struct task_struct *p; - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { -+#else -+ for_each_class(class) { -+#endif - p = class->pick_task(rq); - if (p) - return p; -@@ -6913,6 +6987,9 @@ static void __sched notrace __schedule(unsigned int sched_mode) - psi_sched_switch(prev, next, !task_on_rq_queued(prev)); - - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#ifdef CONFIG_HMBIRD_SCHED -+ sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#endif - - /* Also unlocks the rq: */ - rq = context_switch(rq, prev, next, &rf); -@@ -7241,24 +7318,31 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag - } - EXPORT_SYMBOL(default_wake_function); - --void __setscheduler_prio(struct task_struct *p, int prio) -+static void __setscheduler_prio(struct task_struct *p, int prio) - { -- /* -- * After switching all rt and fair class to ext, -- * stop class is switched to ext class too. Just -- * skip it. */ -+#ifdef CONFIG_HMBIRD_SCHED -+ bool on_hmbird = task_on_hmbird(p); -+ - if (p->sched_class == &stop_sched_class) -- ;/* do nothing */ -+ ; - else if (dl_prio(prio)) - p->sched_class = &dl_sched_class; --#ifdef CONFIG_SCHED_CLASS_EXT -- else if (task_on_scx(p)) -- p->sched_class = &ext_sched_class; --#endif -+ else if (rt_prio(prio) && on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#else -+ if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; - else if (rt_prio(prio)) - p->sched_class = &rt_sched_class; - else - p->sched_class = &fair_sched_class; -+#endif - - p->prio = prio; - trace_android_rvh_setscheduler_prio(p); -@@ -7294,7 +7378,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - */ - void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - { -- int prio, oldprio, queue_flag = -+ int prio, oldprio, queued, running, queue_flag = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *prev_class; - struct rq_flags rf; -@@ -7357,42 +7441,52 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - queue_flag &= ~DEQUEUE_MOVE; - - prev_class = p->sched_class; -- SCHED_CHANGE_BLOCK(rq, p, queue_flag) { -- /* -- * Boosting condition are: -- * 1. -rt task is running and holds mutex A -- * --> -dl task blocks on mutex A -- * -- * 2. -dl task is running and holds mutex A -- * --> -dl task blocks on mutex A and could preempt the -- * running task -- */ -- if (dl_prio(prio)) { -- if (!dl_prio(p->normal_prio) || -- (pi_task && dl_prio(pi_task->prio) && -- dl_entity_preempt(&pi_task->dl, &p->dl))) { -- p->dl.pi_se = pi_task->dl.pi_se; -- queue_flag |= ENQUEUE_REPLENISH; -- } else { -- p->dl.pi_se = &p->dl; -- } -- } else if (rt_prio(prio)) { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- if (oldprio < prio) -- queue_flag |= ENQUEUE_HEAD; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flag); -+ if (running) -+ put_prev_task(rq, p); -+ -+ /* -+ * Boosting condition are: -+ * 1. -rt task is running and holds mutex A -+ * --> -dl task blocks on mutex A -+ * -+ * 2. -dl task is running and holds mutex A -+ * --> -dl task blocks on mutex A and could preempt the -+ * running task -+ */ -+ if (dl_prio(prio)) { -+ if (!dl_prio(p->normal_prio) || -+ (pi_task && dl_prio(pi_task->prio) && -+ dl_entity_preempt(&pi_task->dl, &p->dl))) { -+ p->dl.pi_se = pi_task->dl.pi_se; -+ queue_flag |= ENQUEUE_REPLENISH; - } else { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- else if (rt_prio(oldprio)) -- p->rt.timeout = 0; -- else if (!task_has_idle_policy(p)) -- reweight_task(p, prio - MAX_RT_PRIO); -+ p->dl.pi_se = &p->dl; - } -- -- __setscheduler_prio(p, prio); -+ } else if (rt_prio(prio)) { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ if (oldprio < prio) -+ queue_flag |= ENQUEUE_HEAD; -+ } else { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ else if (rt_prio(oldprio)) -+ p->rt.timeout = 0; -+ else if (!task_has_idle_policy(p)) -+ reweight_task(p, prio - MAX_RT_PRIO); - } - -+ __setscheduler_prio(p, prio); -+ -+ if (queued) -+ enqueue_task(rq, p, queue_flag); -+ if (running) -+ set_next_task(rq, p); -+ - check_class_changed(rq, p, prev_class, oldprio); - out_unlock: - /* Avoid rq from going away on us: */ -@@ -7413,7 +7507,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - - void set_user_nice(struct task_struct *p, long nice) - { -- bool allowed = false; -+ bool queued, running, allowed = false; - int old_prio; - struct rq_flags rf; - struct rq *rq; -@@ -7442,13 +7536,22 @@ void set_user_nice(struct task_struct *p, long nice) - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); -+ if (running) -+ put_prev_task(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->static_prio = NICE_TO_PRIO(nice); -- set_load_weight(p, true); -- old_prio = p->prio; -- p->prio = effective_prio(p); -- } -+ p->static_prio = NICE_TO_PRIO(nice); -+ set_load_weight(p, true); -+ old_prio = p->prio; -+ p->prio = effective_prio(p); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - - /* - * If the task increased its priority or is running and -@@ -7865,7 +7968,7 @@ static int __sched_setscheduler(struct task_struct *p, - bool user, bool pi) - { - int oldpolicy = -1, policy = attr->sched_policy; -- int retval, oldprio, newprio; -+ int retval, oldprio, newprio, queued, running; - const struct sched_class *prev_class; - struct balance_callback *head; - struct rq_flags rf; -@@ -7873,6 +7976,9 @@ static int __sched_setscheduler(struct task_struct *p, - int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct rq *rq; - bool cpuset_locked = false; -+#ifdef CONFIG_HMBIRD_SCHED -+ unsigned long flags; -+#endif - - /* The pi code expects interrupts enabled */ - BUG_ON(pi && in_interrupt()); -@@ -7938,6 +8044,9 @@ static int __sched_setscheduler(struct task_struct *p, - * To be able to change p->policy safely, the appropriate - * runqueue lock must be held. - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+#endif - rq = task_rq_lock(p, &rf); - update_rq_clock(rq); - -@@ -7949,10 +8058,11 @@ static int __sched_setscheduler(struct task_struct *p, - goto unlock; - } - -- retval = scx_check_setscheduler(p, policy); -+#ifdef CONFIG_HMBIRD_SCHED -+ retval = hmbird_check_setscheduler(p, policy); - if (retval) - goto unlock; -- -+#endif - /* - * If not changing anything there's no need to proceed further, - * but store a possible modification of reset_on_fork. -@@ -8009,6 +8119,9 @@ static int __sched_setscheduler(struct task_struct *p, - if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { - policy = oldpolicy = -1; - task_rq_unlock(rq, p, &rf); -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - goto recheck; -@@ -8041,16 +8154,23 @@ static int __sched_setscheduler(struct task_struct *p, - queue_flags &= ~DEQUEUE_MOVE; - } - -- SCHED_CHANGE_BLOCK(rq, p, queue_flags) { -- prev_class = p->sched_class; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flags); -+ if (running) -+ put_prev_task(rq, p); - -- if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -- __setscheduler_params(p, attr); -- __setscheduler_prio(p, newprio); -- trace_android_rvh_setscheduler(p); -- } -- __setscheduler_uclamp(p, attr); -+ prev_class = p->sched_class; - -+ if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -+ __setscheduler_params(p, attr); -+ __setscheduler_prio(p, newprio); -+ trace_android_rvh_setscheduler(p); -+ } -+ __setscheduler_uclamp(p, attr); -+ -+ if (queued) { - /* - * We enqueue to tail when the priority of a task is - * increased (user space view). -@@ -8058,7 +8178,10 @@ static int __sched_setscheduler(struct task_struct *p, - if (oldprio < p->prio) - queue_flags |= ENQUEUE_HEAD; - -+ enqueue_task(rq, p, queue_flags); - } -+ if (running) -+ set_next_task(rq, p); - - check_class_changed(rq, p, prev_class, oldprio); - -@@ -8066,7 +8189,9 @@ static int __sched_setscheduler(struct task_struct *p, - preempt_disable(); - head = splice_balance_callbacks(rq); - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (pi) { - if (cpuset_locked) - cpuset_unlock(); -@@ -8081,6 +8206,9 @@ static int __sched_setscheduler(struct task_struct *p, - - unlock: - task_rq_unlock(rq, p, &rf); -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - return retval; -@@ -9302,7 +9430,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - break; - } -@@ -9330,7 +9460,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - } - return ret; -@@ -9631,15 +9763,25 @@ int migrate_task_to(struct task_struct *p, int target_cpu) - */ - void sched_setnuma(struct task_struct *p, int nid) - { -+ bool queued, running; - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) { -- p->numa_preferred_nid = nid; -- } -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE); -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->numa_preferred_nid = nid; - -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - task_rq_unlock(rq, p, &rf); - } - #endif /* CONFIG_NUMA_BALANCING */ -@@ -10205,15 +10347,22 @@ void __init sched_init(void) - int i; - - /* Make sure the linker didn't screw up */ -+#ifdef CONFIG_HMBIRD_SCHED - #ifdef CONFIG_SMP -- BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+#endif -+ WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class)); -+#else -+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 || -+ &fair_sched_class != &rt_sched_class + 1 || -+ &rt_sched_class != &dl_sched_class + 1); -+#ifdef CONFIG_SMP -+ BUG_ON(&dl_sched_class != &stop_sched_class + 1); - #endif -- BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); -- BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); -- BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); --#ifdef CONFIG_SCHED_CLASS_EXT -- BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); -- BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); - #endif - - wait_bit_init(); -@@ -10385,8 +10534,9 @@ void __init sched_init(void) - balance_push_set(smp_processor_id(), false); - #endif - init_sched_fair_class(); -- init_sched_ext_class(); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ init_sched_hmbird_class(); -+#endif - psi_init(); - - init_uclamp(); -@@ -10790,6 +10940,9 @@ static void sched_change_group(struct task_struct *tsk) - */ - void sched_move_task(struct task_struct *tsk) - { -+ int queued, running, queue_flags = -+ DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; -+ struct task_group *group; - struct rq_flags rf; - struct rq *rq; - -@@ -10797,18 +10950,27 @@ void sched_move_task(struct task_struct *tsk) - rq = task_rq_lock(tsk, &rf); - update_rq_clock(rq); - -- SCHED_CHANGE_BLOCK(rq, tsk, -- DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { -- sched_change_group(tsk); -- } -+ running = task_current(rq, tsk); -+ queued = task_on_rq_queued(tsk); - -- /* -- * After changing group, the running task may have joined a throttled -- * one but it's still the running task. Trigger a resched to make sure -- * that task can still run. -- */ -- if (task_current(rq, tsk)) -+ if (queued) -+ dequeue_task(rq, tsk, queue_flags); -+ if (running) -+ put_prev_task(rq, tsk); -+ -+ sched_change_group(tsk, group); -+ -+ if (queued) -+ enqueue_task(rq, tsk, queue_flags); -+ if (running) { -+ set_next_task(rq, tsk); -+ /* -+ * After changing group, the running task may have joined a -+ * throttled one but it's still the running task. Trigger a -+ * resched to make sure that task can still run. -+ */ - resched_curr(rq); -+ } - - task_rq_unlock(rq, tsk, &rf); - } -@@ -10845,6 +11007,10 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) - struct task_group *tg = css_tg(css); - struct task_group *parent = css_tg(css->parent); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_tg_online(tg); -+#endif -+ - if (parent) - sched_online_group(tg, parent); - -@@ -10894,13 +11060,79 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) - } - #endif - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 dl) -+{ -+ int i; -+ -+ for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) { -+ if (dl < HMBIRD_BPF_DSQS_DEADLINE[i]) -+ break; -+ } -+ -+ i = max_t(int, i-1, MIN_CGROUP_DL_IDX); -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return 0; -+ update_cgroup_ids_table(css->cgroup->kn->id, i); -+ -+ return 0; -+} -+ -+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ u8 i; -+ -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX]; -+ i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1); -+ if (i < 0) { -+ pr_err(" <%s> i is %d, less than 0, name is %s\n", -+ __func__, i, css->cgroup->kn->name); -+ i = DEFAULT_CGROUP_DL_IDX; -+ } -+ -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[i]; -+} -+ -+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css, -+ struct task_struct *p, int prio) -+{ -+ if (!css || !rt_prio(prio)) -+ return; -+ -+ if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) { -+ if (!strcmp(css->cgroup->kn->name, "display")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3); -+ else if (!strcmp(css->cgroup->kn->name, "touch")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2); -+ else if (!strcmp(css->cgroup->kn->name, "multimedia")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+ } -+} -+#endif -+ - static void cpu_cgroup_attach(struct cgroup_taskset *tset) - { - struct task_struct *task; - struct cgroup_subsys_state *css; - -- cgroup_taskset_for_each(task, css, tset) -+ cgroup_taskset_for_each(task, css, tset) { -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_change_rt_sched_prop(css, task, task->prio); -+#endif - sched_move_task(task); -+ } - - trace_android_rvh_cpu_cgroup_attach(tset); - } -@@ -11579,6 +11811,13 @@ static struct cftype cpu_legacy_files[] = { - .read_u64 = cpu_uclamp_ls_read_u64, - .write_u64 = cpu_uclamp_ls_write_u64, - }, -+#endif -+#ifdef CONFIG_HMBIRD_SCHED -+ { -+ .name = "hmbird.deadline", -+ .read_u64 = cgroup_read_hmbird_deadline, -+ .write_u64 = cgroup_write_hmbird_deadline, -+ }, - #endif - { } /* Terminate */ - }; -@@ -11628,7 +11867,6 @@ static int cpu_local_stat_show(struct seq_file *sf, - } - - #ifdef CONFIG_FAIR_GROUP_SCHED -- - static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, - struct cftype *cft) - { -diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c -index a6c99df9edf9..f647dacc22e2 100644 ---- a/kernel/sched/debug.c -+++ b/kernel/sched/debug.c -@@ -376,8 +376,8 @@ static __init int sched_init_debug(void) - - debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); - --#ifdef CONFIG_SCHED_CLASS_EXT -- debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops); -+#ifdef CONFIG_HMBIRD_SCHED -+ debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops); - #endif - return 0; - } -@@ -1006,9 +1006,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - SEQ_printf(m, - "---------------------------------------------------------" - "----------\n"); --#ifdef CONFIG_SLIM_SCHED -- SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop); --#endif - - #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) - #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) -@@ -1104,8 +1101,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - } else if (fair_policy(p->policy)) { - P(se.slice); - } --#ifdef CONFIG_SCHED_CLASS_EXT -- __PS("ext.enabled", p->sched_class == &ext_sched_class); -+#ifdef CONFIG_HMBIRD_SCHED -+ __PS("hmbird.enabled", p->sched_class == &hmbird_sched_class); -+ __PS("sched_prop", get_hmbird_ts(p)->sched_prop); -+ __PS("top_task_prop", get_hmbird_ts(p)->top_task_prop); - #endif - #undef PN_SCHEDSTAT - #undef P_SCHEDSTAT -diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c -deleted file mode 100755 -index 4845d441df2b..000000000000 ---- a/kernel/sched/ext.c -+++ /dev/null -@@ -1,3978 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -- --enum scx_internal_consts { -- SCX_NR_ONLINE_OPS = SCX_OP_IDX(init), -- SCX_DSP_DFL_MAX_BATCH = 32, -- SCX_DSP_MAX_LOOPS = 32, -- SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, --}; -- --enum scx_ops_enable_state { -- SCX_OPS_PREPPING, -- SCX_OPS_ENABLING, -- SCX_OPS_ENABLED, -- SCX_OPS_DISABLING, -- SCX_OPS_DISABLED, --}; -- --static inline struct task_group *css_tg(struct cgroup_subsys_state *css) --{ -- return css ? container_of(css, struct task_group, css) : NULL; --} -- --/* -- * sched_ext_entity->ops_state -- * -- * Used to track the task ownership between the SCX core and the BPF scheduler. -- * State transitions look as follows: -- * -- * NONE -> QUEUEING -> QUEUED -> DISPATCHING -- * ^ | | -- * | v v -- * \-------------------------------/ -- * -- * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -- * sites for explanations on the conditions being waited upon and why they are -- * safe. Transitions out of them into NONE or QUEUED must store_release and the -- * waiters should load_acquire. -- * -- * Tracking scx_ops_state enables sched_ext core to reliably determine whether -- * any given task can be dispatched by the BPF scheduler at all times and thus -- * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -- * to try to dispatch any task anytime regardless of its state as the SCX core -- * can safely reject invalid dispatches. -- */ --enum scx_ops_state { -- SCX_OPSS_NONE, /* owned by the SCX core */ -- SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -- SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ -- SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ -- -- /* -- * QSEQ brands each QUEUED instance so that, when dispatch races -- * dequeue/requeue, the dispatcher can tell whether it still has a claim -- * on the task being dispatched. -- */ -- SCX_OPSS_QSEQ_SHIFT = 2, -- SCX_OPSS_STATE_MASK = (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1, -- SCX_OPSS_QSEQ_MASK = ~SCX_OPSS_STATE_MASK, --}; -- --/* -- * During exit, a task may schedule after losing its PIDs. When disabling the -- * BPF scheduler, we need to be able to iterate tasks in every state to -- * guarantee system safety. Maintain a dedicated task list which contains every -- * task between its fork and eventual free. -- */ --static DEFINE_SPINLOCK(scx_tasks_lock); --static LIST_HEAD(scx_tasks); -- --/* ops enable/disable */ --static struct kthread_worker *scx_ops_helper; --static DEFINE_MUTEX(scx_ops_enable_mutex); --DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); --DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); --static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); --static bool scx_switch_all_req; --static bool scx_switching_all; --DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -- --static struct sched_ext_ops scx_ops; --static bool scx_warned_zero_slice; -- --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); --DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); --static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); -- --struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] = -- { [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT }; -- --static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE); --static struct scx_exit_info scx_exit_info; -- --static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0); -- --/* -- * The maximum amount of time in jiffies that a task may be runnable without -- * being scheduled on a CPU. If this timeout is exceeded, it will trigger -- * scx_ops_error(). -- */ --unsigned long scx_watchdog_timeout; -- --/* -- * The last time the delayed work was run. This delayed work relies on -- * ksoftirqd being able to run to service timer interrupts, so it's possible -- * that this work itself could get wedged. To account for this, we check that -- * it's not stalled in the timer tick, and trigger an error if it is. -- */ --unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; -- --static struct delayed_work scx_watchdog_work; -- --/* idle tracking */ --#ifdef CONFIG_SMP --#ifdef CONFIG_CPUMASK_OFFSTACK --#define CL_ALIGNED_IF_ONSTACK --#else --#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp --#endif -- --static struct { -- cpumask_var_t cpu; -- cpumask_var_t smt; --} idle_masks CL_ALIGNED_IF_ONSTACK; -- --static bool __cacheline_aligned_in_smp scx_has_idle_cpus; --#endif /* CONFIG_SMP */ -- --/* for %SCX_KICK_WAIT */ --static u64 __percpu *scx_kick_cpus_pnt_seqs; -- --/* -- * Direct dispatch marker. -- * -- * Non-NULL values are used for direct dispatch from enqueue path. A valid -- * pointer points to the task currently being enqueued. An ERR_PTR value is used -- * to indicate that direct dispatch has already happened. -- */ --static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); -- --/* dispatch queues */ --static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; -- --static LLIST_HEAD(dsqs_to_free); -- --/* dispatch buf */ --struct scx_dsp_buf_ent { -- struct task_struct *task; -- u64 qseq; -- u64 dsq_id; -- u64 enq_flags; --}; -- --static u32 scx_dsp_max_batch; --static struct scx_dsp_buf_ent __percpu *scx_dsp_buf; -- --struct scx_dsp_ctx { -- struct rq *rq; -- struct rq_flags *rf; -- u32 buf_cursor; -- u32 nr_tasks; --}; -- --static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx); -- --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags); --void scx_bpf_kick_cpu(s32 cpu, u64 flags); -- --struct scx_task_iter { -- struct sched_ext_entity cursor; -- struct task_struct *locked; -- struct rq *rq; -- struct rq_flags rf; --}; -- --#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) -- --/* if the highest set bit is N, return a mask with bits [N+1, 31] set */ --static u32 higher_bits(u32 flags) --{ -- return ~((1 << fls(flags)) - 1); --} -- --/* return the mask with only the highest bit set */ --static u32 highest_bit(u32 flags) --{ -- int bit = fls(flags); -- return bit ? 1 << (bit - 1) : 0; --} -- --/* -- * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX -- * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate -- * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check -- * whether it's running from an allowed context. -- * -- * @mask is constant, always inline to cull the mask calculations. -- */ --static __always_inline void scx_kf_allow(u32 mask) --{ -- /* nesting is allowed only in increasing scx_kf_mask order */ -- WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask, -- "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n", -- current->scx->kf_mask, mask); -- current->scx->kf_mask |= mask; --} -- --static void scx_kf_disallow(u32 mask) --{ -- current->scx->kf_mask &= ~mask; --} -- --#define SCX_CALL_OP(mask, op, args...) \ --do { \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- scx_ops.op(args); \ -- } \ --} while (0) -- --#define SCX_CALL_OP_RET(mask, op, args...) \ --({ \ -- __typeof__(scx_ops.op(args)) __ret; \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- __ret = scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- __ret = scx_ops.op(args); \ -- } \ -- __ret; \ --}) -- --/* -- * Some kfuncs are allowed only on the tasks that are subjects of the -- * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such -- * restrictions, the following SCX_CALL_OP_*() variants should be used when -- * invoking scx_ops operations that take task arguments. These can only be used -- * for non-nesting operations due to the way the tasks are tracked. -- * -- * kfuncs which can only operate on such tasks can in turn use -- * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on -- * the specific task. -- */ --#define SCX_CALL_OP_TASK(mask, op, task, args...) \ --do { \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- SCX_CALL_OP(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ --} while (0) -- --#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ --({ \ -- __typeof__(scx_ops.op(task, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- __ret; \ --}) -- --#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ --({ \ -- __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task0; \ -- current->scx->kf_tasks[1] = task1; \ -- __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- current->scx->kf_tasks[1] = NULL; \ -- __ret; \ --}) -- --//#include "./slim_walt.c" -- --/* @mask is constant, always inline to cull unnecessary branches */ --static __always_inline bool scx_kf_allowed(u32 mask) --{ -- if (unlikely(!(current->scx->kf_mask & mask))) { -- scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", -- mask, current->scx->kf_mask); -- return false; -- } -- -- if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) && -- in_interrupt())) { -- scx_ops_error("sleepable kfunc called from non-sleepable context"); -- return false; -- } -- -- /* -- * Enforce nesting boundaries. e.g. A kfunc which can be called from -- * DISPATCH must not be called if we're running DEQUEUE which is nested -- * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE -- * boundary thanks to the above in_interrupt() check. -- */ -- if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && -- (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { -- scx_ops_error("cpu_release kfunc called from a nested operation"); -- return false; -- } -- -- if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && -- (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) { -- scx_ops_error("dispatch kfunc called from a nested operation"); -- return false; -- } -- -- return true; --} -- --/* see SCX_CALL_OP_TASK() */ --static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, -- struct task_struct *p) --{ -- if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED)) -- return false; -- -- if (unlikely((p != current->scx->kf_tasks[0] && -- p != current->scx->kf_tasks[1]))) { -- scx_ops_error("called on a task not being operated on"); -- return false; -- } -- -- return true; --} -- --/** -- * scx_task_iter_init - Initialize a task iterator -- * @iter: iterator to init -- * -- * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized, -- * @iter must eventually be exited with scx_task_iter_exit(). -- * -- * scx_tasks_lock may be released between this and the first next() call or -- * between any two next() calls. If scx_tasks_lock is released between two -- * next() calls, the caller is responsible for ensuring that the task being -- * iterated remains accessible either through RCU read lock or obtaining a -- * reference count. -- * -- * All tasks which existed when the iteration started are guaranteed to be -- * visited as long as they still exist. -- */ --static void scx_task_iter_init(struct scx_task_iter *iter) --{ -- lockdep_assert_held(&scx_tasks_lock); -- -- iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; -- list_add(&iter->cursor.tasks_node, &scx_tasks); -- iter->locked = NULL; --} -- --/** -- * scx_task_iter_exit - Exit a task iterator -- * @iter: iterator to exit -- * -- * Exit a previously initialized @iter. Must be called with scx_tasks_lock held. -- * If the iterator holds a task's rq lock, that rq lock is released. See -- * scx_task_iter_init() for details. -- */ --static void scx_task_iter_exit(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- if (list_empty(cursor)) -- return; -- -- list_del_init(cursor); --} -- --/** -- * scx_task_iter_next - Next task -- * @iter: iterator to walk -- * -- * Visit the next task. See scx_task_iter_init() for details. -- */ --static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- struct sched_ext_entity *pos; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- list_for_each_entry(pos, cursor, tasks_node) { -- if (&pos->tasks_node == &scx_tasks) -- return NULL; -- if (!(pos->flags & SCX_TASK_CURSOR)) { -- list_move(cursor, &pos->tasks_node); -- return pos->task; -- } -- } -- -- /* can't happen, should always terminate at scx_tasks above */ -- BUG(); --} -- --/** -- * scx_task_iter_next_filtered - Next non-idle task -- * @iter: iterator to walk -- * -- * Visit the next non-idle task. See scx_task_iter_init() for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- while ((p = scx_task_iter_next(iter))) { -- if (!is_idle_task(p)) -- return p; -- } -- return NULL; --} -- --/** -- * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked -- * @iter: iterator to walk -- * -- * Visit the next non-idle task with its rq lock held. See scx_task_iter_init() -- * for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered_locked(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- p = scx_task_iter_next_filtered(iter); -- if (!p) -- return NULL; -- -- iter->rq = task_rq_lock(p, &iter->rf); -- iter->locked = p; -- return p; --} -- --static enum scx_ops_enable_state scx_ops_enable_state(void) --{ -- return atomic_read(&scx_ops_enable_state_var); --} -- --static enum scx_ops_enable_state --scx_ops_set_enable_state(enum scx_ops_enable_state to) --{ -- return atomic_xchg(&scx_ops_enable_state_var, to); --} -- --static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, -- enum scx_ops_enable_state from) --{ -- int from_v = from; -- -- return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); --} -- --static bool scx_ops_disabling(void) --{ -- return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING); --} -- --/** -- * wait_ops_state - Busy-wait the specified ops state to end -- * @p: target task -- * @opss: state to wait the end of -- * -- * Busy-wait for @p to transition out of @opss. This can only be used when the -- * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also -- * has load_acquire semantics to ensure that the caller can see the updates made -- * in the enqueueing and dispatching paths. -- */ --static void wait_ops_state(struct task_struct *p, u64 opss) --{ -- do { -- cpu_relax(); -- } while (atomic64_read_acquire(&p->scx->ops_state) == opss); --} -- --/** -- * ops_cpu_valid - Verify a cpu number -- * @cpu: cpu number which came from a BPF ops -- * -- * @cpu is a cpu number which came from the BPF scheduler and can be any value. -- * Verify that it is in range and one of the possible cpus. -- */ --static bool ops_cpu_valid(s32 cpu) --{ -- return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); --} -- --/** -- * ops_sanitize_err - Sanitize a -errno value -- * @ops_name: operation to blame on failure -- * @err: -errno value to sanitize -- * -- * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return -- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can -- * cause misbehaviors. For an example, a large negative return from -- * ops.prep_enable() triggers an oops when passed up the call chain because the -- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is -- * handled as a pointer. -- */ --static int ops_sanitize_err(const char *ops_name, s32 err) --{ -- if (err < 0 && err >= -MAX_ERRNO) -- return err; -- -- scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); -- return -EPROTO; --} -- --/** -- * touch_core_sched - Update timestamp used for core-sched task ordering -- * @rq: rq to read clock from, must be locked -- * @p: task to update the timestamp for -- * -- * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to -- * implement global or local-DSQ FIFO ordering for core-sched. Should be called -- * when a task becomes runnable and its turn on the CPU ends (e.g. slice -- * exhaustion). -- */ --static void touch_core_sched(struct rq *rq, struct task_struct *p) --{ --#ifdef CONFIG_SCHED_CORE -- /* -- * It's okay to update the timestamp spuriously. Use -- * sched_core_disabled() which is cheaper than enabled(). -- */ -- if (!sched_core_disabled()) -- p->scx->core_sched_at = rq_clock_task(rq); --#endif --} -- --/** -- * touch_core_sched_dispatch - Update core-sched timestamp on dispatch -- * @rq: rq to read clock from, must be locked -- * @p: task being dispatched -- * -- * If the BPF scheduler implements custom core-sched ordering via -- * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO -- * ordering within each local DSQ. This function is called from dispatch paths -- * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect. -- */ --static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- assert_clock_updated(rq); -- --#ifdef CONFIG_SCHED_CORE -- if (SCX_HAS_OP(core_sched_before)) -- touch_core_sched(rq, p); --#endif --} -- --static void update_curr_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- u64 now = rq_clock_task(rq); -- u64 delta_exec; -- -- if (time_before_eq64(now, curr->se.exec_start)) -- return; -- -- delta_exec = now - curr->se.exec_start; -- curr->se.exec_start = now; -- curr->se.sum_exec_runtime += delta_exec; -- account_group_exec_runtime(curr, delta_exec); -- cgroup_account_cputime(curr, delta_exec); -- -- if (curr->scx->slice != SCX_SLICE_INF) { -- curr->scx->slice -= min(curr->scx->slice, delta_exec); -- if (!curr->scx->slice) -- touch_core_sched(rq, curr); -- } --} -- --static bool scx_dsq_priq_less(struct rb_node *node_a, -- const struct rb_node *node_b) --{ -- const struct sched_ext_entity *a = -- container_of(node_a, struct sched_ext_entity, dsq_node.priq); -- const struct sched_ext_entity *b = -- container_of(node_b, struct sched_ext_entity, dsq_node.priq); -- -- return time_before64(a->dsq_vtime, b->dsq_vtime); --} -- --static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, -- u64 enq_flags) --{ -- bool is_local = dsq->id == SCX_DSQ_LOCAL; -- -- WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo)); -- WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq)); -- -- if (!is_local) { -- raw_spin_lock(&dsq->lock); -- if (unlikely(dsq->id == SCX_DSQ_INVALID)) { -- scx_ops_error("attempting to dispatch to a destroyed dsq"); -- /* fall back to the global dsq */ -- raw_spin_unlock(&dsq->lock); -- dsq = &scx_dsq_global; -- raw_spin_lock(&dsq->lock); -- } -- } -- -- if (enq_flags & SCX_ENQ_DSQ_PRIQ) { -- p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; -- rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq, -- scx_dsq_priq_less); -- } else { -- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) -- list_add(&p->scx->dsq_node.fifo, &dsq->fifo); -- else -- list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo); -- } -- dsq->nr++; -- p->scx->dsq = dsq; -- -- /* -- * We're transitioning out of QUEUEING or DISPATCHING. store_release to -- * match waiters' load_acquire. -- */ -- if (enq_flags & SCX_ENQ_CLEAR_OPSS) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- if (is_local) { -- struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq); -- struct rq *rq = scx->rq; -- bool preempt = false; -- -- if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && -- rq->curr->sched_class == &ext_sched_class) { -- rq->curr->scx->slice = 0; -- preempt = true; -- } -- -- if (preempt || sched_class_above(&ext_sched_class, -- rq->curr->sched_class)) -- resched_curr(rq); -- } else { -- raw_spin_unlock(&dsq->lock); -- } --} -- --static void task_unlink_from_dsq(struct task_struct *p, -- struct scx_dispatch_q *dsq) --{ -- if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { -- rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; -- } else { -- list_del_init(&p->scx->dsq_node.fifo); -- } -- --} -- --static bool task_linked_on_dsq(struct task_struct *p) --{ -- return !list_empty(&p->scx->dsq_node.fifo) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq); --} -- --static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p) --{ -- struct scx_dispatch_q *dsq = p->scx->dsq; -- bool is_local = dsq == &scx_rq->local_dsq; -- -- if (!dsq) { -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- /* -- * When dispatching directly from the BPF scheduler to a local -- * DSQ, the task isn't associated with any DSQ but -- * @p->scx->holding_cpu may be set under the protection of -- * %SCX_OPSS_DISPATCHING. -- */ -- if (p->scx->holding_cpu >= 0) -- p->scx->holding_cpu = -1; -- return; -- } -- -- if (!is_local) -- raw_spin_lock(&dsq->lock); -- -- /* -- * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node -- * can't change underneath us. -- */ -- if (p->scx->holding_cpu < 0) { -- /* @p must still be on @dsq, dequeue */ -- WARN_ON_ONCE(!task_linked_on_dsq(p)); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- } else { -- /* -- * We're racing against dispatch_to_local_dsq() which already -- * removed @p from @dsq and set @p->scx->holding_cpu. Clear the -- * holding_cpu which tells dispatch_to_local_dsq() that it lost -- * the race. -- */ -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- p->scx->holding_cpu = -1; -- } -- p->scx->dsq = NULL; -- -- if (!is_local) -- raw_spin_unlock(&dsq->lock); --} -- --static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) --{ -- lockdep_assert(rcu_read_lock_any_held()); -- -- return &scx_dsq_global; --} -- --static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, -- struct task_struct *p) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id == SCX_DSQ_LOCAL) -- return &rq->scx->local_dsq; -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", -- dsq_id, p->comm, p->pid); -- return &scx_dsq_global; -- } -- -- return dsq; --} -- --static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- -- /* @p must match the task which is being enqueued */ -- if (unlikely(p != ddsp_task)) { -- if (IS_ERR(ddsp_task)) -- scx_ops_error("%s[%d] already direct-dispatched", -- p->comm, p->pid); -- else -- scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]", -- ddsp_task->comm, ddsp_task->pid, -- p->comm, p->pid); -- return; -- } -- -- /* -- * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because -- * dispatching to the local DSQ of a different CPU requires unlocking -- * the current rq which isn't allowed in the enqueue path. Use -- * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL. -- */ -- if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) { -- scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch"); -- return; -- } -- -- touch_core_sched_dispatch(task_rq(p), p); -- -- dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- -- /* -- * Mark that dispatch already happened by spoiling direct_dispatch_task -- * with a non-NULL value which can never match a valid task pointer. -- */ -- __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); --} -- --static bool test_rq_online(struct rq *rq) --{ --#ifdef CONFIG_SMP -- return rq->online; --#else -- return true; --#endif --} -- --static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- int sticky_cpu) --{ -- struct task_struct **ddsp_taskp; -- u64 qseq; -- -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- if (p->scx->flags & SCX_TASK_ENQ_LOCAL) { -- enq_flags |= SCX_ENQ_LOCAL; -- p->scx->flags &= ~SCX_TASK_ENQ_LOCAL; -- } -- -- /* rq migration */ -- if (sticky_cpu == cpu_of(rq)) -- goto local_norefill; -- -- /* -- * If !rq->online, we already told the BPF scheduler that the CPU is -- * offline. We're just trying to on/offline the CPU. Don't bother the -- * BPF scheduler. -- */ -- if (unlikely(!test_rq_online(rq))) -- goto local; -- -- /* see %SCX_OPS_ENQ_EXITING */ -- if (!static_branch_unlikely(&scx_ops_enq_exiting) && -- unlikely(p->flags & PF_EXITING)) -- goto local; -- -- /* see %SCX_OPS_ENQ_LAST */ -- if (!static_branch_unlikely(&scx_ops_enq_last) && -- (enq_flags & SCX_ENQ_LAST)) -- goto local; -- -- if (!SCX_HAS_OP(enqueue)) { -- if (enq_flags & SCX_ENQ_LOCAL) -- goto local; -- else -- goto global; -- } -- -- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ -- qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; -- -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq); -- -- ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); -- WARN_ON_ONCE(*ddsp_taskp); -- *ddsp_taskp = p; -- -- SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); -- -- /* -- * If not directly dispatched, QUEUEING isn't clear yet and dispatch or -- * dequeue may be waiting. The store_release matches their load_acquire. -- */ -- if (*ddsp_taskp == p) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq); -- *ddsp_taskp = NULL; -- return; -- --local: -- /* -- * For task-ordering, slice refill must be treated as implying the end -- * of the current slice. Otherwise, the longer @p stays on the CPU, the -- * higher priority it becomes from scx_prio_less()'s POV. -- */ -- touch_core_sched(rq, p); -- p->scx->slice = SCX_SLICE_DFL; --local_norefill: -- dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags); -- return; -- --global: -- touch_core_sched(rq, p); /* see the comment in local: */ -- p->scx->slice = SCX_SLICE_DFL; -- dispatch_enqueue(&scx_dsq_global, p, enq_flags); --} -- --static bool watchdog_task_watched(const struct task_struct *p) --{ -- return !list_empty(&p->scx->watchdog_node); --} -- --static void watchdog_watch_task(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- if (p->scx->flags & SCX_TASK_WATCHDOG_RESET) -- p->scx->runnable_at = jiffies; -- p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET; -- list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list); --} -- --static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) --{ -- list_del_init(&p->scx->watchdog_node); -- if (reset_timeout) -- p->scx->flags |= SCX_TASK_WATCHDOG_RESET; --} -- --static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) --{ -- int sticky_cpu = p->scx->sticky_cpu; -- -- enq_flags |= rq->scx->extra_enq_flags; -- -- if (sticky_cpu >= 0) -- p->scx->sticky_cpu = -1; -- -- /* -- * Restoring a running task will be immediately followed by -- * set_next_task_scx() which expects the task to not be on the BPF -- * scheduler as tasks can only start running through local DSQs. Force -- * direct-dispatch into the local DSQ by setting the sticky_cpu. -- */ -- if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -- sticky_cpu = cpu_of(rq); -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- WARN_ON_ONCE(!watchdog_task_watched(p)); -- return; -- } -- -- watchdog_watch_task(rq, p); -- p->scx->flags |= SCX_TASK_QUEUED; -- rq->scx->nr_running++; -- add_nr_running(rq, 1); -- -- if (SCX_HAS_OP(runnable)) -- SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); -- -- if (enq_flags & SCX_ENQ_WAKEUP) -- touch_core_sched(rq, p); -- -- do_enqueue_task(rq, p, enq_flags, sticky_cpu); --} -- --static void ops_dequeue(struct task_struct *p, u64 deq_flags) --{ -- u64 opss; -- -- watchdog_unwatch_task(p, false); -- -- /* acquire ensures that we see the preceding updates on QUEUED */ -- opss = atomic64_read_acquire(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_NONE: -- break; -- case SCX_OPSS_QUEUEING: -- /* -- * QUEUEING is started and finished while holding @p's rq lock. -- * As we're holding the rq lock now, we shouldn't see QUEUEING. -- */ -- BUG(); -- case SCX_OPSS_QUEUED: -- if (SCX_HAS_OP(dequeue)) -- SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); -- -- if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_NONE)) -- break; -- fallthrough; -- case SCX_OPSS_DISPATCHING: -- /* -- * If @p is being dispatched from the BPF scheduler to a DSQ, -- * wait for the transfer to complete so that @p doesn't get -- * added to its DSQ after dequeueing is complete. -- * -- * As we're waiting on DISPATCHING with the rq locked, the -- * dispatching side shouldn't try to lock the rq while -- * DISPATCHING is set. See dispatch_to_local_dsq(). -- * -- * DISPATCHING shouldn't have qseq set and control can reach -- * here with NONE @opss from the above QUEUED case block. -- * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. -- */ -- wait_ops_state(p, SCX_OPSS_DISPATCHING); -- BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- break; -- } --} -- --static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) --{ -- struct scx_rq *scx_rq = rq->scx; -- -- if (!(p->scx->flags & SCX_TASK_QUEUED)) { -- WARN_ON_ONCE(watchdog_task_watched(p)); -- return; -- } -- -- ops_dequeue(p, deq_flags); -- -- /* -- * A currently running task which is going off @rq first gets dequeued -- * and then stops running. As we want running <-> stopping transitions -- * to be contained within runnable <-> quiescent transitions, trigger -- * ->stopping() early here instead of in put_prev_task_scx(). -- * -- * @p may go through multiple stopping <-> running transitions between -- * here and put_prev_task_scx() if task attribute changes occur while -- * balance_scx() leaves @rq unlocked. However, they don't contain any -- * information meaningful to the BPF scheduler and can be suppressed by -- * skipping the callbacks if the task is !QUEUED. -- */ -- if (SCX_HAS_OP(stopping) && task_current(rq, p)) { -- update_curr_scx(rq); -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); -- } -- -- //if(task_current(rq, p)) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- if (SCX_HAS_OP(quiescent)) -- SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); -- -- if (deq_flags & SCX_DEQ_SLEEP) -- p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP; -- else -- p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP; -- -- p->scx->flags &= ~SCX_TASK_QUEUED; -- BUG_ON(!scx_rq->nr_running); -- scx_rq->nr_running--; -- sub_nr_running(rq, 1); -- -- dispatch_dequeue(scx_rq, p); --} -- --static void yield_task_scx(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); -- else -- p->scx->slice = 0; --} -- --static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) --{ -- struct task_struct *from = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); -- else -- return false; --} -- --#ifdef CONFIG_SMP --/** -- * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -- * @rq: rq to move the task into, currently locked -- * @p: task to move -- * @enq_flags: %SCX_ENQ_* -- * -- * Move @p which is currently on a different rq to @rq's local DSQ. The caller -- * must: -- * -- * 1. Start with exclusive access to @p either through its DSQ lock or -- * %SCX_OPSS_DISPATCHING flag. -- * -- * 2. Set @p->scx->holding_cpu to raw_smp_processor_id(). -- * -- * 3. Remember task_rq(@p). Release the exclusive access so that we don't -- * deadlock with dequeue. -- * -- * 4. Lock @rq and the task_rq from #3. -- * -- * 5. Call this function. -- * -- * Returns %true if @p was successfully moved. %false after racing dequeue and -- * losing. -- */ --static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -- u64 enq_flags) --{ -- struct rq *task_rq; -- -- lockdep_assert_rq_held(rq); -- -- /* -- * If dequeue got to @p while we were trying to lock both rq's, it'd -- * have cleared @p->scx->holding_cpu to -1. While other cpus may have -- * updated it to different values afterwards, as this operation can't be -- * preempted or recurse, @p->scx->holding_cpu can never become -- * raw_smp_processor_id() again before we're done. Thus, we can tell -- * whether we lost to dequeue by testing whether @p->scx->holding_cpu is -- * still raw_smp_processor_id(). -- * -- * See dispatch_dequeue() for the counterpart. -- */ -- if (unlikely(p->scx->holding_cpu != raw_smp_processor_id())) -- return false; -- -- /* @p->rq couldn't have changed if we're still the holding cpu */ -- task_rq = task_rq(p); -- lockdep_assert_rq_held(task_rq); -- -- WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)); -- deactivate_task(task_rq, p, 0); -- set_task_cpu(p, cpu_of(rq)); -- p->scx->sticky_cpu = cpu_of(rq); -- -- /* -- * We want to pass scx-specific enq_flags but activate_task() will -- * truncate the upper 32 bit. As we own @rq, we can pass them through -- * @rq->scx->extra_enq_flags instead. -- */ -- WARN_ON_ONCE(rq->scx->extra_enq_flags); -- rq->scx->extra_enq_flags = enq_flags; -- activate_task(rq, p, 0); -- rq->scx->extra_enq_flags = 0; -- -- return true; --} -- --/** -- * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * We're holding @rq lock and trying to dispatch a task from @src_rq to -- * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether -- * @rq stays locked isn't important as long as the state is restored after -- * dispatch_to_local_dsq_unlock(). -- */ --static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- rq_unpin_lock(rq, rf); -- -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(rq); -- raw_spin_rq_lock(dst_rq); -- } else if (rq == src_rq) { -- double_lock_balance(rq, dst_rq); -- rq_repin_lock(rq, rf); -- } else if (rq == dst_rq) { -- double_lock_balance(rq, src_rq); -- rq_repin_lock(rq, rf); -- } else { -- raw_spin_rq_unlock(rq); -- double_rq_lock(src_rq, dst_rq); -- } --} -- --/** -- * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock() -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return. -- */ --static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } else if (rq == src_rq) { -- double_unlock_balance(rq, dst_rq); -- } else if (rq == dst_rq) { -- double_unlock_balance(rq, src_rq); -- } else { -- double_rq_unlock(src_rq, dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } --} --#endif /* CONFIG_SMP */ -- -- --static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq) --{ -- return likely(test_rq_online(rq)) && !is_migration_disabled(p) && -- cpumask_test_cpu(cpu_of(rq), p->cpus_ptr); --} -- --static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -- struct scx_dispatch_q *dsq) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rb_node *rb_node; -- struct rq *task_rq; -- bool moved = false; --retry: -- if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -- return false; -- -- raw_spin_lock(&dsq->lock); -- -- list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- for (rb_node = rb_first_cached(&dsq->priq); rb_node; -- rb_node = rb_next(rb_node)) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- raw_spin_unlock(&dsq->lock); -- return false; -- --this_rq: -- /* @dsq is locked and @p is on this rq */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo); -- dsq->nr--; -- scx_rq->local_dsq.nr++; -- p->scx->dsq = &scx_rq->local_dsq; -- raw_spin_unlock(&dsq->lock); -- return true; -- --remote_rq: --#ifdef CONFIG_SMP -- /* -- * @dsq is locked and @p is on a remote rq. @p is currently protected by -- * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -- * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -- * rq lock or fail, do a little dancing from our side. See -- * move_task_to_local_dsq(). -- */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- p->scx->holding_cpu = raw_smp_processor_id(); -- raw_spin_unlock(&dsq->lock); -- -- rq_unpin_lock(rq, rf); -- double_lock_balance(rq, task_rq); -- rq_repin_lock(rq, rf); -- -- moved = move_task_to_local_dsq(rq, p, 0); -- -- double_unlock_balance(rq, task_rq); --#endif /* CONFIG_SMP */ -- if (likely(moved)) -- return true; -- goto retry; --} -- --enum dispatch_to_local_dsq_ret { -- DTL_DISPATCHED, /* successfully dispatched */ -- DTL_LOST, /* lost race to dequeue */ -- DTL_NOT_LOCAL, /* destination is not a local DSQ */ -- DTL_INVALID, /* invalid local dsq_id */ --}; -- --/** -- * dispatch_to_local_dsq - Dispatch a task to a local dsq -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @dsq_id: destination dsq ID -- * @p: task to dispatch -- * @enq_flags: %SCX_ENQ_* -- * -- * We're holding @rq lock and want to dispatch @p to the local DSQ identified by -- * @dsq_id. This function performs all the synchronization dancing needed -- * because local DSQs are protected with rq locks. -- * -- * The caller must have exclusive ownership of @p (e.g. through -- * %SCX_OPSS_DISPATCHING). -- */ --static enum dispatch_to_local_dsq_ret --dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id, -- struct task_struct *p, u64 enq_flags) --{ -- struct rq *src_rq = task_rq(p); -- struct rq *dst_rq; -- -- /* -- * We're synchronized against dequeue through DISPATCHING. As @p can't -- * be dequeued, its task_rq and cpus_allowed are stable too. -- */ -- if (dsq_id == SCX_DSQ_LOCAL) { -- dst_rq = rq; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]", -- cpu, p->comm, p->pid); -- return DTL_INVALID; -- } -- dst_rq = cpu_rq(cpu); -- } else { -- return DTL_NOT_LOCAL; -- } -- -- /* if dispatching to @rq that @p is already on, no lock dancing needed */ -- if (rq == src_rq && rq == dst_rq) { -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags | SCX_ENQ_CLEAR_OPSS); -- return DTL_DISPATCHED; -- } -- --#ifdef CONFIG_SMP -- if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { -- struct rq *locked_dst_rq = dst_rq; -- bool dsp; -- -- /* -- * @p is on a possibly remote @src_rq which we need to lock to -- * move the task. If dequeue is in progress, it'd be locking -- * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq -- * lock while holding DISPATCHING. -- * -- * As DISPATCHING guarantees that @p is wholly ours, we can -- * pretend that we're moving from a DSQ and use the same -- * mechanism - mark the task under transfer with holding_cpu, -- * release DISPATCHING and then follow the same protocol. -- */ -- p->scx->holding_cpu = raw_smp_processor_id(); -- -- /* store_release ensures that dequeue sees the above */ -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq); -- -- /* -- * We don't require the BPF scheduler to avoid dispatching to -- * offline CPUs mostly for convenience but also because CPUs can -- * go offline between scx_bpf_dispatch() calls and here. If @p -- * is destined to an offline CPU, queue it on its current CPU -- * instead, which should always be safe. As this is an allowed -- * behavior, don't trigger an ops error. -- */ -- if (unlikely(!test_rq_online(dst_rq))) -- dst_rq = src_rq; -- -- if (src_rq == dst_rq) { -- /* -- * As @p is staying on the same rq, there's no need to -- * go through the full deactivate/activate cycle. -- * Optimize by abbreviating the operations in -- * move_task_to_local_dsq(). -- */ -- dsp = p->scx->holding_cpu == raw_smp_processor_id(); -- if (likely(dsp)) { -- p->scx->holding_cpu = -1; -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags); -- } -- } else { -- dsp = move_task_to_local_dsq(dst_rq, p, enq_flags); -- } -- -- /* if the destination CPU is idle, wake it up */ -- if (dsp && p->sched_class > dst_rq->curr->sched_class) -- resched_curr(dst_rq); -- -- dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq); -- -- return dsp ? DTL_DISPATCHED : DTL_LOST; -- } --#endif /* CONFIG_SMP */ -- -- scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", -- cpu_of(dst_rq), p->comm, p->pid); -- return DTL_INVALID; --} -- --/** -- * finish_dispatch - Asynchronously finish dispatching a task -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @p: task to finish dispatching -- * @qseq_at_dispatch: qseq when @p started getting dispatched -- * @dsq_id: destination DSQ ID -- * @enq_flags: %SCX_ENQ_* -- * -- * Dispatching to local DSQs may need to wait for queueing to complete or -- * require rq lock dancing. As we don't wanna do either while inside -- * ops.dispatch() to avoid locking order inversion, we split dispatching into -- * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the -- * task and its qseq. Once ops.dispatch() returns, this function is called to -- * finish up. -- * -- * There is no guarantee that @p is still valid for dispatching or even that it -- * was valid in the first place. Make sure that the task is still owned by the -- * BPF scheduler and claim the ownership before dispatching. -- */ --static void finish_dispatch(struct rq *rq, struct rq_flags *rf, -- struct task_struct *p, u64 qseq_at_dispatch, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- u64 opss; -- -- touch_core_sched_dispatch(rq, p); --retry: -- /* -- * No need for _acquire here. @p is accessed only after a successful -- * try_cmpxchg to DISPATCHING. -- */ -- opss = atomic64_read(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_DISPATCHING: -- case SCX_OPSS_NONE: -- /* someone else already got to it */ -- return; -- case SCX_OPSS_QUEUED: -- /* -- * If qseq doesn't match, @p has gone through at least one -- * dispatch/dequeue and re-enqueue cycle between -- * scx_bpf_dispatch() and here and we have no claim on it. -- */ -- if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) -- return; -- -- /* -- * While we know @p is accessible, we don't yet have a claim on -- * it - the BPF scheduler is allowed to dispatch tasks -- * spuriously and there can be a racing dequeue attempt. Let's -- * claim @p by atomically transitioning it from QUEUED to -- * DISPATCHING. -- */ -- if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_DISPATCHING))) -- break; -- goto retry; -- case SCX_OPSS_QUEUEING: -- /* -- * do_enqueue_task() is in the process of transferring the task -- * to the BPF scheduler while holding @p's rq lock. As we aren't -- * holding any kernel or BPF resource that the enqueue path may -- * depend upon, it's safe to wait. -- */ -- wait_ops_state(p, opss); -- goto retry; -- } -- -- BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) { -- case DTL_DISPATCHED: -- break; -- case DTL_LOST: -- break; -- case DTL_INVALID: -- dsq_id = SCX_DSQ_GLOBAL; -- fallthrough; -- case DTL_NOT_LOCAL: -- dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()), -- dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- break; -- } --} -- --static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- u32 u; -- -- for (u = 0; u < dspc->buf_cursor; u++) { -- struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u]; -- -- finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id, -- ent->enq_flags); -- } -- -- dspc->nr_tasks += dspc->buf_cursor; -- dspc->buf_cursor = 0; --} -- --static int balance_one(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf, bool local) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- bool prev_on_scx = prev->sched_class == &ext_sched_class; -- int nr_loops = SCX_DSP_MAX_LOOPS; -- -- lockdep_assert_rq_held(rq); -- -- if (static_branch_unlikely(&scx_ops_cpu_preempt) && -- unlikely(rq->scx->cpu_released)) { -- /* -- * If the previous sched_class for the current CPU was not SCX, -- * notify the BPF scheduler that it again has control of the -- * core. This callback complements ->cpu_release(), which is -- * emitted in scx_notify_pick_next_task(). -- */ -- if (SCX_HAS_OP(cpu_acquire)) -- SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq), -- NULL); -- rq->scx->cpu_released = false; -- } -- -- if (prev_on_scx) { -- WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP)); -- update_curr_scx(rq); -- -- /* -- * If @prev is runnable & has slice left, it has priority and -- * fetching more just increases latency for the fetched tasks. -- * Tell put_prev_task_scx() to put @prev on local_dsq. If the -- * BPF scheduler wants to handle this explicitly, it should -- * implement ->cpu_released(). -- * -- * See scx_ops_disable_workfn() for the explanation on the -- * disabling() test. -- * -- * When balancing a remote CPU for core-sched, there won't be a -- * following put_prev_task_scx() call and we don't own -- * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the -- * same conditions later and pick @rq->curr accordingly. -- */ -- if ((prev->scx->flags & SCX_TASK_QUEUED) && -- prev->scx->slice && !scx_ops_disabling()) { -- if (local) -- prev->scx->flags |= SCX_TASK_BAL_KEEP; -- return 1; -- } -- } -- -- /* if there already are tasks to run, nothing to do */ -- if (scx_rq->local_dsq.nr) -- return 1; -- -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- if (!SCX_HAS_OP(dispatch)) -- return 0; -- -- dspc->rq = rq; -- dspc->rf = rf; -- -- /* -- * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, -- * the local DSQ might still end up empty after a successful -- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() -- * produced some tasks, retry. The BPF scheduler may depend on this -- * looping behavior to simplify its implementation. -- */ -- do { -- dspc->nr_tasks = 0; -- -- SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), -- prev_on_scx ? prev : NULL); -- -- flush_dispatch_buf(rq, rf); -- -- if (scx_rq->local_dsq.nr) -- return 1; -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- /* -- * ops.dispatch() can trap us in this loop by repeatedly -- * dispatching ineligible tasks. Break out once in a while to -- * allow the watchdog to run. As IRQ can't be enabled in -- * balance(), we want to complete this scheduling cycle and then -- * start a new one. IOW, we want to call resched_curr() on the -- * next, most likely idle, task, not the current one. Use -- * scx_bpf_kick_cpu() for deferred kicking. -- */ -- if (unlikely(!--nr_loops)) { -- scx_bpf_kick_cpu(cpu_of(rq), 0); -- break; -- } -- } while (dspc->nr_tasks); -- -- return 0; --} -- --static int balance_scx(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf) --{ -- int ret; -- -- ret = balance_one(rq, prev, rf, true); --#ifdef CONFIG_SCHED_SMT -- /* -- * When core-sched is enabled, this ops.balance() call will be followed -- * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx() -- * on the SMT siblings. Balance the siblings too. -- */ -- if (sched_core_enabled(rq)) { -- const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); -- int scpu; -- -- for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { -- struct rq *srq = cpu_rq(scpu); -- struct rq_flags srf; -- struct task_struct *sprev = srq->curr; -- -- /* -- * While core-scheduling, rq lock is shared among -- * siblings but the debug annotations and rq clock -- * aren't. Do pinning dance to transfer the ownership. -- */ -- WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); -- rq_unpin_lock(rq, rf); -- rq_pin_lock(srq, &srf); -- -- update_rq_clock(srq); -- balance_one(srq, sprev, &srf, false); -- -- rq_unpin_lock(srq, &srf); -- rq_repin_lock(rq, rf); -- } -- } --#endif -- return ret; --} -- --static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) --{ -- if (p->scx->flags & SCX_TASK_QUEUED) { -- /* -- * Core-sched might decide to execute @p before it is -- * dispatched. Call ops_dequeue() to notify the BPF scheduler. -- */ -- ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); -- dispatch_dequeue(rq->scx, p); -- } -- -- p->se.exec_start = rq_clock_task(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, running, p); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock); -- -- watchdog_unwatch_task(p, true); -- -- /* -- * @p is getting newly scheduled or got kicked after someone updated its -- * slice. Refresh whether tick can be stopped. See can_stop_tick_scx(). -- */ -- if ((p->scx->slice == SCX_SLICE_INF) != -- (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) { -- if (p->scx->slice == SCX_SLICE_INF) -- rq->scx->flags |= SCX_RQ_CAN_STOP_TICK; -- else -- rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK; -- -- sched_update_tick_dependency(rq); -- } --} -- --static void put_prev_task_scx(struct rq *rq, struct task_struct *p) --{ --#ifndef CONFIG_SMP -- /* -- * UP workaround. -- * -- * Because SCX may transfer tasks across CPUs during dispatch, dispatch -- * is performed from its balance operation which isn't called in UP. -- * Let's work around by calling it from the operations which come right -- * after. -- * -- * 1. If the prev task is on SCX, pick_next_task() calls -- * .put_prev_task() right after. As .put_prev_task() is also called -- * from other places, we need to distinguish the calls which can be -- * done by looking at the previous task's state - if still queued or -- * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task(). -- * This case is handled here. -- * -- * 2. If the prev task is not on SCX, the first following call into SCX -- * will be .pick_next_task(), which is covered by calling -- * balance_scx() from pick_next_task_scx(). -- * -- * Note that we can't merge the first case into the second as -- * balance_scx() must be called before the previous SCX task goes -- * through put_prev_task_scx(). -- * -- * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf. -- * Pass in %NULL. -- */ -- if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP)) -- balance_scx(rq, p, NULL); --#endif -- -- update_curr_scx(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- -- /* -- * If we're being called from put_prev_task_balance(), balance_scx() may -- * have decided that @p should keep running. -- */ -- if (p->scx->flags & SCX_TASK_BAL_KEEP) { -- p->scx->flags &= ~SCX_TASK_BAL_KEEP; -- watchdog_watch_task(rq, p); -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- watchdog_watch_task(rq, p); -- -- /* -- * If @p has slice left and balance_scx() didn't tag it for -- * keeping, @p is getting preempted by a higher priority -- * scheduler class or core-sched forcing a different task. Leave -- * it at the head of the local DSQ. -- */ -- if (p->scx->slice && !scx_ops_disabling()) { -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- /* -- * If we're in the pick_next_task path, balance_scx() should -- * have already populated the local DSQ if there are any other -- * available tasks. If empty, tell ops.enqueue() that @p is the -- * only one available for this cpu. ops.enqueue() should put it -- * on the local DSQ so that the subsequent pick_next_task_scx() -- * can find the task unless it wants to trigger a separate -- * follow-up scheduling event. -- */ -- if (list_empty(&rq->scx->local_dsq.fifo)) -- do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1); -- else -- do_enqueue_task(rq, p, 0, -1); -- } --} -- --static struct task_struct *first_local_task(struct rq *rq) --{ -- struct rb_node *rb_node; -- struct sched_ext_entity *entity; -- -- if (!list_empty(&rq->scx->local_dsq.fifo)) { -- entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo); -- return entity->task; -- } -- -- rb_node = rb_first_cached(&rq->scx->local_dsq.priq); -- if (rb_node) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- return entity->task; -- } -- -- return NULL; --} -- --static struct task_struct *pick_next_task_scx(struct rq *rq) --{ -- struct task_struct *p; -- --#ifndef CONFIG_SMP -- /* UP workaround - see the comment at the head of put_prev_task_scx() */ -- if (unlikely(rq->curr->sched_class != &ext_sched_class)) -- balance_scx(rq, rq->curr, NULL); --#endif -- -- p = first_local_task(rq); -- if (!p) -- return NULL; -- -- if (unlikely(!p->scx->slice)) { -- if (!scx_ops_disabling() && !scx_warned_zero_slice) { -- printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n", -- p->comm, p->pid); -- scx_warned_zero_slice = true; -- } -- p->scx->slice = SCX_SLICE_DFL; -- } -- -- set_next_task_scx(rq, p, true); -- -- return p; --} -- --#ifdef CONFIG_SCHED_CORE --/** -- * scx_prio_less - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Core-sched is implemented as an additional scheduling layer on top of the -- * usual sched_class'es and needs to find out the expected task ordering. For -- * SCX, core-sched calls this function to interrogate the task ordering. -- * -- * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used -- * to implement the default task ordering. The older the timestamp, the higher -- * prority the task - the global FIFO ordering matching the default scheduling -- * behavior. -- * -- * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to -- * implement FIFO ordering within each local DSQ. See pick_task_scx(). -- */ --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi) --{ -- /* -- * The const qualifiers are dropped from task_struct pointers when -- * calling ops.core_sched_before(). Accesses are controlled by the -- * verifier. -- */ -- if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling()) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, -- (struct task_struct *)a, -- (struct task_struct *)b); -- else -- return time_after64(a->scx->core_sched_at, b->scx->core_sched_at); --} -- --/** -- * pick_task_scx - Pick a candidate task for core-sched -- * @rq: rq to pick the candidate task from -- * -- * Core-sched calls this function on each SMT sibling to determine the next -- * tasks to run on the SMT siblings. balance_one() has been called on all -- * siblings and put_prev_task_scx() has been called only for the current CPU. -- * -- * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look -- * at the first task in the local dsq. @rq->curr has to be considered explicitly -- * to mimic %SCX_TASK_BAL_KEEP. -- */ --static struct task_struct *pick_task_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- struct task_struct *first = first_local_task(rq); -- -- if (curr->scx->flags & SCX_TASK_QUEUED) { -- /* is curr the only runnable task? */ -- if (!first) -- return curr; -- -- /* -- * Does curr trump first? We can always go by core_sched_at for -- * this comparison as it represents global FIFO ordering when -- * the default core-sched ordering is used and local-DSQ FIFO -- * ordering otherwise. -- * -- * We can have a task with an earlier timestamp on the DSQ. For -- * example, when a current task is preempted by a sibling -- * picking a different cookie, the task would be requeued at the -- * head of the local DSQ with an earlier timestamp than the -- * core-sched picked next task. Besides, the BPF scheduler may -- * dispatch any tasks to the local DSQ anytime. -- */ -- if (curr->scx->slice && time_before64(curr->scx->core_sched_at, -- first->scx->core_sched_at)) -- return curr; -- } -- -- return first; /* this may be %NULL */ --} --#endif /* CONFIG_SCHED_CORE */ -- --static enum scx_cpu_preempt_reason --preempt_reason_from_class(const struct sched_class *class) --{ --#ifdef CONFIG_SMP -- if (class == &stop_sched_class) -- return SCX_CPU_PREEMPT_STOP; --#endif -- if (class == &dl_sched_class) -- return SCX_CPU_PREEMPT_DL; -- if (class == &rt_sched_class) -- return SCX_CPU_PREEMPT_RT; -- return SCX_CPU_PREEMPT_UNKNOWN; --} -- --void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task, -- const struct sched_class *active) --{ -- lockdep_assert_rq_held(rq); -- -- /* -- * The callback is conceptually meant to convey that the CPU is no -- * longer under the control of SCX. Therefore, don't invoke the -- * callback if the CPU is is staying on SCX, or going idle (in which -- * case the SCX scheduler has actively decided not to schedule any -- * tasks on the CPU). -- */ -- if (likely(active >= &ext_sched_class)) -- return; -- -- /* -- * At this point we know that SCX was preempted by a higher priority -- * sched_class, so invoke the ->cpu_release() callback if we have not -- * done so already. We only send the callback once between SCX being -- * preempted, and it regaining control of the CPU. -- * -- * ->cpu_release() complements ->cpu_acquire(), which is emitted the -- * next time that balance_scx() is invoked. -- */ -- if (!rq->scx->cpu_released) { -- if (SCX_HAS_OP(cpu_release)) { -- struct scx_cpu_release_args args = { -- .reason = preempt_reason_from_class(active), -- .task = task, -- }; -- -- SCX_CALL_OP(SCX_KF_CPU_RELEASE, -- cpu_release, cpu_of(rq), &args); -- } -- rq->scx->cpu_released = true; -- } --} -- --#ifdef CONFIG_SMP -- --static bool test_and_clear_cpu_idle(int cpu) --{ -- if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -- if (cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- return true; -- } else { -- return false; -- } --} -- --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- int cpu; -- -- do { -- cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); -- if (cpu < nr_cpu_ids) { -- const struct cpumask *sbm = topology_sibling_cpumask(cpu); -- -- /* -- * If offline, @cpu is not its own sibling and we can -- * get caught in an infinite loop as @cpu is never -- * cleared from idle_masks.smt. Clear @cpu directly in -- * such cases. -- */ -- if (likely(cpumask_test_cpu(cpu, sbm))) -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm); -- else -- cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu)); -- } else { -- cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -- if (cpu >= nr_cpu_ids) -- return -EBUSY; -- } -- } while (!test_and_clear_cpu_idle(cpu)); -- -- return cpu; --} -- --static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) --{ -- s32 cpu; -- -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return prev_cpu; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local DSQ of the waker. -- */ -- if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 && -- scx_has_idle_cpus && !(current->flags & PF_EXITING)) { -- cpu = smp_processor_id(); -- if (cpumask_test_cpu(cpu, p->cpus_ptr)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (test_and_clear_cpu_idle(prev_cpu)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return prev_cpu; -- } -- -- if (p->nr_cpus_allowed == 1) -- return prev_cpu; -- -- cpu = scx_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- -- return prev_cpu; --} -- --static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) --{ -- if (SCX_HAS_OP(select_cpu)) { -- s32 cpu; -- -- cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu, -- wake_flags); -- if (ops_cpu_valid(cpu)) { -- return cpu; -- } else { -- scx_ops_error("select_cpu returned invalid cpu %d", cpu); -- return prev_cpu; -- } -- } else { -- return scx_select_cpu_dfl(p, prev_cpu, wake_flags); -- } --} -- --static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx) --{ -- set_cpus_allowed_common(p, ctx); -- -- /* -- * The effective cpumask is stored in @p->cpus_ptr which may temporarily -- * differ from the configured one in @p->cpus_mask. Always tell the bpf -- * scheduler the effective one. -- * -- * Fine-grained memory write control is enforced by BPF making the const -- * designation pointless. Cast it away when calling the operation. -- */ -- if (SCX_HAS_OP(set_cpumask)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, -- (struct cpumask *)p->cpus_ptr); --} -- --static void reset_idle_masks(void) --{ -- /* consider all cpus idle, should converge to the actual state quickly */ -- cpumask_setall(idle_masks.cpu); -- cpumask_setall(idle_masks.smt); -- scx_has_idle_cpus = true; --} -- --void __scx_update_idle(struct rq *rq, bool idle) --{ -- int cpu = cpu_of(rq); -- struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -- -- if (SCX_HAS_OP(update_idle)) { -- SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); -- if (!static_branch_unlikely(&scx_builtin_idle_enabled)) -- return; -- } -- -- if (idle) { -- cpumask_set_cpu(cpu, idle_masks.cpu); -- if (!scx_has_idle_cpus) -- scx_has_idle_cpus = true; -- -- /* -- * idle_masks.smt handling is racy but that's fine as it's only -- * for optimization and self-correcting. -- */ -- for_each_cpu(cpu, sib_mask) { -- if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -- return; -- } -- cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -- } else { -- cpumask_clear_cpu(cpu, idle_masks.cpu); -- if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -- } --} -- --#else /* !CONFIG_SMP */ -- --static bool test_and_clear_cpu_idle(int cpu) { return false; } --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } --static void reset_idle_masks(void) {} -- --#endif /* CONFIG_SMP */ -- --static bool check_rq_for_timeouts(struct rq *rq) --{ -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rq_flags rf; -- bool timed_out = false; -- -- rq_lock_irqsave(rq, &rf); -- list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) { -- unsigned long last_runnable; -- -- p = entity->task; -- last_runnable = p->scx->runnable_at; -- -- if (unlikely(time_after(jiffies, -- last_runnable + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "%s[%d] failed to run for %u.%03us", -- p->comm, p->pid, -- dur_ms / 1000, dur_ms % 1000); -- timed_out = true; -- break; -- } -- } -- rq_unlock_irqrestore(rq, &rf); -- -- return timed_out; --} -- --static void scx_watchdog_workfn(struct work_struct *work) --{ -- int cpu; -- -- scx_watchdog_timestamp = jiffies; -- -- for_each_online_cpu(cpu) { -- if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) -- break; -- -- cond_resched(); -- } -- queue_delayed_work(system_unbound_wq, to_delayed_work(work), -- scx_watchdog_timeout / 2); --} -- --static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) --{ -- update_curr_scx(rq); -- //scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock); -- /* -- * While disabling, always resched and refresh core-sched timestamp as -- * we can't trust the slice management or ops.core_sched_before(). -- */ -- if (scx_ops_disabling()) { -- curr->scx->slice = 0; -- touch_core_sched(rq, curr); -- } -- -- if (!curr->scx->slice) -- resched_curr(rq); --} -- --#define SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- --static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg) --{ -- int ret; -- -- WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED); -- -- p->scx->disallow = false; -- -- if (SCX_HAS_OP(prep_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- }; -- -- ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args); -- if (unlikely(ret)) { -- ret = ops_sanitize_err("prep_enable", ret); -- return ret; -- } -- } -- //scx_sched_init_task(p); -- -- if (p->scx->disallow) { -- struct rq *rq; -- struct rq_flags rf; -- -- rq = task_rq_lock(p, &rf); -- -- /* -- * We're either in fork or load path and @p->policy will be -- * applied right after. Reverting @p->policy here and rejecting -- * %SCHED_EXT transitions from scx_check_setscheduler() -- * guarantees that if ops.prep_enable() sets @p->disallow, @p -- * can never be in SCX. -- */ -- if (p->policy == SCHED_EXT) { -- p->policy = SCHED_NORMAL; -- atomic64_inc(&scx_nr_rejected); -- } -- -- task_rq_unlock(rq, p, &rf); -- } -- -- p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET); -- return 0; --} -- --static void scx_ops_enable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED)); -- -- if (SCX_HAS_OP(enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- p->scx->flags |= SCX_TASK_OPS_ENABLED; --} -- --static void scx_ops_disable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- if (p->scx->flags & SCX_TASK_OPS_PREPPED) { -- if (SCX_HAS_OP(cancel_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- } else if (p->scx->flags & SCX_TASK_OPS_ENABLED) { -- if (SCX_HAS_OP(disable)) -- SCX_CALL_OP(SCX_KF_REST, disable, p); -- p->scx->flags &= ~SCX_TASK_OPS_ENABLED; -- } --} -- --static void set_task_scx_weight(struct task_struct *p) --{ -- u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -- -- p->scx->weight = sched_weight_to_cgroup(weight); --} -- --/** -- * refresh_scx_weight - Refresh a task's ext weight -- * @p: task to refresh ext weight for -- * -- * @p->scx->weight carries the task's static priority in cgroup weight scale to -- * enable easy access from the BPF scheduler. To keep it synchronized with the -- * current task priority, this function should be called when a new task is -- * created, priority is changed for a task on sched_ext, and a task is switched -- * to sched_ext from other classes. -- */ --static void refresh_scx_weight(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- set_task_scx_weight(p); -- if (SCX_HAS_OP(set_weight)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight); --} -- --void scx_pre_fork(struct task_struct *p) --{ -- p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL); -- if (!p->scx) { -- goto lock; -- } -- -- p->scx->dsq = NULL; -- INIT_LIST_HEAD(&p->scx->dsq_node.fifo); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- INIT_LIST_HEAD(&p->scx->watchdog_node); -- p->scx->flags = 0; -- p->scx->weight = 0; -- p->scx->sticky_cpu = -1; -- p->scx->holding_cpu = -1; -- p->scx->kf_mask = 0; -- atomic64_set(&p->scx->ops_state, 0); -- p->scx->runnable_at = INITIAL_JIFFIES; -- p->scx->slice = SCX_SLICE_DFL; -- p->scx->task = p; -- p->sched_prop = 0; -- -- /* -- * BPF scheduler enable/disable paths want to be able to iterate and -- * update all tasks which can become complex when racing forks. As -- * enable/disable are very cold paths, let's use a percpu_rwsem to -- * exclude forks. -- */ --lock: -- percpu_down_read(&scx_fork_rwsem); --} -- --int scx_fork(struct task_struct *p) --{ -- percpu_rwsem_assert_held(&scx_fork_rwsem); -- -- if (scx_enabled()) -- return scx_ops_prepare_task(p, task_group(p)); -- else -- return 0; --} -- --void scx_post_fork(struct task_struct *p) --{ -- if (scx_enabled()) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- /* -- * Set the weight manually before calling ops.enable() so that -- * the scheduler doesn't see a stale value if they inspect the -- * task struct. We'll invoke ops.set_weight() afterwards, as it -- * would be odd to receive a callback on the task before we -- * tell the scheduler that it's been fully enabled. -- */ -- set_task_scx_weight(p); -- scx_ops_enable_task(p); -- refresh_scx_weight(p); -- task_rq_unlock(rq, p, &rf); -- } -- -- spin_lock_irq(&scx_tasks_lock); -- list_add_tail(&p->scx->tasks_node, &scx_tasks); -- spin_unlock_irq(&scx_tasks_lock); -- -- percpu_up_read(&scx_fork_rwsem); --} -- --void scx_cancel_fork(struct task_struct *p) --{ -- if (scx_enabled()) -- scx_ops_disable_task(p); -- percpu_up_read(&scx_fork_rwsem); --} -- --void sched_ext_free(struct task_struct *p) --{ -- unsigned long flags; -- -- spin_lock_irqsave(&scx_tasks_lock, flags); -- list_del_init(&p->scx->tasks_node); -- spin_unlock_irqrestore(&scx_tasks_lock, flags); -- -- /* -- * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED -> -- * ENABLED transitions can't race us. Disable ops for @p. -- */ -- if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- scx_ops_disable_task(p); -- task_rq_unlock(rq, p, &rf); -- } --} -- --static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) --{ --} -- --static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} --static void switched_to_scx(struct rq *rq, struct task_struct *p) {} -- --int scx_check_setscheduler(struct task_struct *p, int policy) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- /* if disallow, reject transitioning into SCX */ -- if (scx_enabled() && READ_ONCE(p->scx->disallow) && -- p->policy != policy && policy == SCHED_EXT) -- return -EACCES; -- -- return 0; --} -- --#ifdef CONFIG_NO_HZ_FULL --bool scx_can_stop_tick(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (scx_ops_disabling()) -- return false; -- -- if (p->sched_class != &ext_sched_class) -- return true; -- -- /* -- * @rq can dispatch from different DSQs, so we can't tell whether it -- * needs the tick or not by looking at nr_running. Allow stopping ticks -- * iff the BPF scheduler indicated so. See set_next_task_scx(). -- */ -- return rq->scx->flags & SCX_RQ_CAN_STOP_TICK; --} --#endif -- --static inline void scx_cgroup_lock(void) {} --static inline void scx_cgroup_unlock(void) {} -- --/* -- * Omitted operations: -- * -- * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -- * task isn't tied to the CPU at that point. Preemption is implemented by -- * resetting the victim task's slice to 0 and triggering reschedule on the -- * target CPU. -- * -- * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -- * -- * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -- * their current sched_class. Call them directly from sched core instead. -- * -- * - task_woken, switched_from: Unnecessary. -- */ --DEFINE_SCHED_CLASS(ext) = { -- .enqueue_task = enqueue_task_scx, -- .dequeue_task = dequeue_task_scx, -- .yield_task = yield_task_scx, -- .yield_to_task = yield_to_task_scx, -- -- .check_preempt_curr = check_preempt_curr_scx, -- -- .pick_next_task = pick_next_task_scx, -- -- .put_prev_task = put_prev_task_scx, -- .set_next_task = set_next_task_scx, -- --#ifdef CONFIG_SMP -- .balance = balance_scx, -- .select_task_rq = select_task_rq_scx, -- .set_cpus_allowed = set_cpus_allowed_scx, --#endif -- --#ifdef CONFIG_SCHED_CORE -- .pick_task = pick_task_scx, --#endif -- -- .task_tick = task_tick_scx, -- -- .switched_to = switched_to_scx, -- .prio_changed = prio_changed_scx, -- -- .update_curr = update_curr_scx, -- --#ifdef CONFIG_UCLAMP_TASK -- .uclamp_enabled = 0, --#endif --}; -- --static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) --{ -- memset(dsq, 0, sizeof(*dsq)); -- -- raw_spin_lock_init(&dsq->lock); -- INIT_LIST_HEAD(&dsq->fifo); -- dsq->id = dsq_id; --} -- --static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id & SCX_DSQ_FLAG_BUILTIN) -- return ERR_PTR(-EINVAL); -- -- dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node); -- if (!dsq) -- return ERR_PTR(-ENOMEM); -- -- init_dsq(dsq, dsq_id); -- -- return dsq; --} -- --static void free_dsq_irq_workfn(struct irq_work *irq_work) --{ -- struct llist_node *to_free = llist_del_all(&dsqs_to_free); -- struct scx_dispatch_q *dsq, *tmp_dsq; -- -- llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) -- kfree_rcu(dsq, rcu); --} -- --static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); -- --static void destroy_dsq(u64 dsq_id) --{ --} -- --static void scx_cgroup_exit(void) {} --static int scx_cgroup_init(void) { return 0; } --static void scx_cgroup_config_knobs(void) {} -- --/* -- * Used by sched_fork() and __setscheduler_prio() to pick the matching -- * sched_class. dl/rt are already handled. -- */ --bool task_on_scx(struct task_struct *p) --{ -- if (!scx_enabled() || scx_ops_disabling()) -- return false; -- if (READ_ONCE(scx_switching_all)) -- return true; -- return p->policy == SCHED_EXT; --} -- --static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags) --{ -- if (enq_flags & SCX_ENQ_LAST) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); --} -- --static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {} -- --static void scx_ops_disable_workfn(struct kthread_work *work) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- struct scx_task_iter sti; -- struct task_struct *p; -- const char *reason; -- int i, cpu, type; -- -- type = atomic_read(&scx_exit_type); -- while (true) { -- /* -- * NONE indicates that a new scx_ops has been registered since -- * disable was scheduled - don't kill the new ops. DONE -- * indicates that the ops has already been disabled. -- */ -- if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE) -- return; -- if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE)) -- break; -- } -- -- cancel_delayed_work_sync(&scx_watchdog_work); -- -- switch (type) { -- case SCX_EXIT_UNREG: -- reason = "BPF scheduler unregistered"; -- break; -- case SCX_EXIT_SYSRQ: -- reason = "disabled by sysrq-S"; -- break; -- case SCX_EXIT_ERROR: -- reason = "runtime error"; -- break; -- case SCX_EXIT_ERROR_BPF: -- reason = "scx_bpf_error"; -- break; -- case SCX_EXIT_ERROR_STALL: -- reason = "runnable task stall"; -- break; -- default: -- reason = ""; -- } -- -- ei->type = type; -- strlcpy(ei->reason, reason, sizeof(ei->reason)); -- -- switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { -- case SCX_OPS_DISABLED: -- pr_warn("sched_ext: ops error detected without ops (%s)\n", -- scx_exit_info.msg); -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- return; -- case SCX_OPS_PREPPING: -- goto forward_progress_guaranteed; -- case SCX_OPS_DISABLING: -- /* shouldn't happen but handle it like ENABLING if it does */ -- WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); -- fallthrough; -- case SCX_OPS_ENABLING: -- case SCX_OPS_ENABLED: -- break; -- } -- -- /* -- * DISABLING is set and ops was either ENABLING or ENABLED indicating -- * that the ops and static branches are set. -- * -- * We must guarantee that all runnable tasks make forward progress -- * without trusting the BPF scheduler. We can't grab any mutexes or -- * rwsems as they might be held by tasks that the BPF scheduler is -- * forgetting to run, which unfortunately also excludes toggling the -- * static branches. -- * -- * Let's work around by overriding a couple ops and modifying behaviors -- * based on the DISABLING state and then cycling the tasks through -- * dequeue/enqueue to force global FIFO scheduling. -- * -- * a. ops.enqueue() and .dispatch() are overridden for simple global -- * FIFO scheduling. -- * -- * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value -- * can't be trusted. Whenever a tick triggers, the running task is -- * rotated to the tail of the queue with core_sched_at touched. -- * -- * c. pick_next_task() suppresses zero slice warning. -- * -- * d. scx_prio_less() reverts to the default core_sched_at order. -- */ -- scx_ops.enqueue = scx_ops_fallback_enqueue; -- scx_ops.dispatch = scx_ops_fallback_dispatch; -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- SCHED_CHANGE_BLOCK(task_rq(p), p, -- DEQUEUE_SAVE | DEQUEUE_MOVE) { -- /* cycling deq/enq is enough, see above */ -- } -- } -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* kick all CPUs to restore ticks */ -- for_each_possible_cpu(cpu) -- resched_cpu(cpu); -- --forward_progress_guaranteed: -- /* -- * Here, every runnable task is guaranteed to make forward progress and -- * we can safely use blocking synchronization constructs. Actually -- * disable ops. -- */ -- mutex_lock(&scx_ops_enable_mutex); -- -- static_branch_disable(&__scx_switched_all); -- WRITE_ONCE(scx_switching_all, false); -- -- /* avoid racing against fork and cgroup changes */ -- cpus_read_lock(); -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- bool alive = READ_ONCE(p->__state) != TASK_DEAD; -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL); -- -- __setscheduler_prio(p, p->prio); -- } -- -- if (alive) -- check_class_changed(task_rq(p), p, old_class, p->prio); -- -- scx_ops_disable_task(p); -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* no task is on scx, turn off all the switches and flush in-progress calls */ -- static_branch_disable_cpuslocked(&__scx_ops_enabled); -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- static_branch_disable_cpuslocked(&scx_has_op[i]); -- static_branch_disable_cpuslocked(&scx_ops_enq_last); -- static_branch_disable_cpuslocked(&scx_ops_enq_exiting); -- static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- synchronize_rcu(); -- -- scx_cgroup_exit(); -- -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- cpus_read_unlock(); -- -- if (ei->type >= SCX_EXIT_ERROR) { -- printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name); -- -- if (ei->msg[0] == '\0') -- printk(KERN_ERR "sched_ext: %s\n", ei->reason); -- else -- printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg); -- -- stack_trace_print(ei->bt, ei->bt_len, 2); -- } -- -- if (scx_ops.exit) -- SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); -- -- memset(&scx_ops, 0, sizeof(scx_ops)); -- -- free_percpu(scx_dsp_buf); -- scx_dsp_buf = NULL; -- scx_dsp_max_batch = 0; -- -- mutex_unlock(&scx_ops_enable_mutex); -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- -- scx_cgroup_config_knobs(); --} -- --static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); -- --static void schedule_scx_ops_disable_work(void) --{ -- struct kthread_worker *helper = READ_ONCE(scx_ops_helper); -- -- /* -- * We may be called spuriously before the first bpf_sched_ext_reg(). If -- * scx_ops_helper isn't set up yet, there's nothing to do. -- */ -- if (helper) -- kthread_queue_work(helper, &scx_ops_disable_work); --} -- --static void scx_ops_disable(enum scx_exit_type type) --{ -- int none = SCX_EXIT_NONE; -- -- if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)) -- type = SCX_EXIT_ERROR; -- -- atomic_try_cmpxchg(&scx_exit_type, &none, type); -- -- schedule_scx_ops_disable_work(); --} -- --static void scx_ops_error_irq_workfn(struct irq_work *irq_work) --{ -- schedule_scx_ops_disable_work(); --} -- --static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- int none = SCX_EXIT_NONE; -- va_list args; -- -- if (!atomic_try_cmpxchg(&scx_exit_type, &none, type)) -- return; -- -- ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1); -- -- va_start(args, fmt); -- vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args); -- va_end(args); -- -- irq_work_queue(&scx_ops_error_irq_work); --} -- --static struct kthread_worker *scx_create_rt_helper(const char *name) --{ -- struct kthread_worker *helper; -- -- helper = kthread_create_worker(0, name); -- if (helper) -- sched_set_fifo(helper->task); -- return helper; --} -- --static int scx_ops_enable(struct sched_ext_ops *ops) --{ -- struct scx_task_iter sti; -- struct task_struct *p; -- int i, ret; -- int tcnt = 0; -- unsigned long long start = 0; -- unsigned long long total_start = sched_clock(); -- -- mutex_lock(&scx_ops_enable_mutex); -- -- if (!scx_ops_helper) { -- WRITE_ONCE(scx_ops_helper, -- scx_create_rt_helper("sched_ext_ops_helper")); -- if (!scx_ops_helper) { -- ret = -ENOMEM; -- goto err_unlock; -- } -- } -- -- if (scx_ops_enable_state() != SCX_OPS_DISABLED) { -- ret = -EBUSY; -- goto err_unlock; -- } -- -- //slim_walt_enable(true); -- -- /* -- * Set scx_ops, transition to PREPPING and clear exit info to arm the -- * disable path. Failure triggers full disabling from here on. -- */ -- scx_ops = *ops; -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != -- SCX_OPS_DISABLED); -- -- memset(&scx_exit_info, 0, sizeof(scx_exit_info)); -- atomic_set(&scx_exit_type, SCX_EXIT_NONE); -- scx_warned_zero_slice = false; -- -- atomic64_set(&scx_nr_rejected, 0); -- -- /* -- * Keep CPUs stable during enable so that the BPF scheduler can track -- * online CPUs by watching ->on/offline_cpu() after ->init(). -- */ -- cpus_read_lock(); -- -- scx_switch_all_req = true; -- if (scx_ops.init) { -- ret = SCX_CALL_OP_RET(SCX_KF_INIT, init); -- if (ret) { -- ret = ops_sanitize_err("init", ret); -- goto err_disable; -- } -- -- /* -- * Exit early if ops.init() triggered scx_bpf_error(). Not -- * strictly necessary as we'll fail transitioning into ENABLING -- * later but that'd be after calling ops.prep_enable() on all -- * tasks and with -EBUSY which isn't very intuitive. Let's exit -- * early with success so that the condition is notified through -- * ops.exit() like other scx_bpf_error() invocations. -- */ -- if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE) -- goto err_disable; -- } -- -- WARN_ON_ONCE(scx_dsp_buf); -- scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; -- scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch, -- __alignof__(scx_dsp_buf[0])); -- if (!scx_dsp_buf) { -- ret = -ENOMEM; -- goto err_disable; -- } -- -- scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; -- if (ops->timeout_ms) -- scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); -- -- scx_watchdog_timestamp = jiffies; -- queue_delayed_work(system_unbound_wq, &scx_watchdog_work, -- scx_watchdog_timeout / 2); -- -- /* -- * Lock out forks, cgroup on/offlining and moves before opening the -- * floodgate so that they don't wander into the operations prematurely. -- */ -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- if (((void (**)(void))ops)[i]) -- static_branch_enable_cpuslocked(&scx_has_op[i]); -- -- if (ops->flags & SCX_OPS_ENQ_LAST) -- static_branch_enable_cpuslocked(&scx_ops_enq_last); -- -- if (ops->flags & SCX_OPS_ENQ_EXITING) -- static_branch_enable_cpuslocked(&scx_ops_enq_exiting); -- if (scx_ops.cpu_acquire || scx_ops.cpu_release) -- static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); -- -- if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { -- reset_idle_masks(); -- static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); -- } else { -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- } -- -- /* -- * All cgroups should be initialized before letting in tasks. cgroup -- * on/offlining and task migrations are already locked out. -- */ -- ret = scx_cgroup_init(); -- if (ret) -- goto err_disable_unlock; -- -- static_branch_enable_cpuslocked(&__scx_ops_enabled); -- -- /* -- * Enable ops for every task. Fork is excluded by scx_fork_rwsem -- * preventing new tasks from being added. No need to exclude tasks -- * leaving as sched_ext_free() can handle both prepped and enabled -- * tasks. Prep all tasks first and then enable them with preemption -- * disabled. -- */ -- spin_lock_irq(&scx_tasks_lock); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered(&sti))) { -- get_task_struct(p); -- spin_unlock_irq(&scx_tasks_lock); -- -- ret = scx_ops_prepare_task(p, task_group(p)); -- if (ret) { -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n", -- ret, p->comm, p->pid); -- goto err_disable_unlock; -- } -- -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- } -- scx_task_iter_exit(&sti); -- -- /* -- * All tasks are prepped but are still ops-disabled. Ensure that -- * %current can't be scheduled out and switch everyone. -- * preempt_disable() is necessary because we can't guarantee that -- * %current won't be starved if scheduled out while switching. -- */ -- preempt_disable(); -- -- /* -- * From here on, the disable path must assume that tasks have ops -- * enabled and need to be recovered. -- */ -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { -- preempt_enable(); -- spin_unlock_irq(&scx_tasks_lock); -- ret = -EBUSY; -- goto err_disable_unlock; -- } -- -- /* -- * We're fully committed and can't fail. The PREPPED -> ENABLED -- * transitions here are synchronized against sched_ext_free() through -- * scx_tasks_lock. -- */ -- WRITE_ONCE(scx_switching_all, scx_switch_all_req); -- start = sched_clock(); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- tcnt++; -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- scx_ops_enable_task(p); -- __setscheduler_prio(p, p->prio); -- } -- -- check_class_changed(task_rq(p), p, old_class, p->prio); -- } else { -- scx_ops_disable_task(p); -- } -- } -- printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id()); -- scx_task_iter_exit(&sti); -- -- spin_unlock_irq(&scx_tasks_lock); -- preempt_enable(); -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { -- ret = -EBUSY; -- goto err_disable; -- } -- -- if (scx_switch_all_req) -- static_branch_enable_cpuslocked(&__scx_switched_all); -- -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- -- scx_cgroup_config_knobs(); -- printk("\n total duration = %llu , current cpu = %d\n", sched_clock() - total_start, smp_processor_id()); -- -- return 0; -- --err_unlock: -- mutex_unlock(&scx_ops_enable_mutex); -- return ret; -- --err_disable_unlock: -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); --err_disable: -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- /* must be fully disabled before returning */ -- scx_ops_disable(SCX_EXIT_ERROR); -- kthread_flush_work(&scx_ops_disable_work); -- return ret; --} -- --#ifdef CONFIG_SCHED_DEBUG --static const char *scx_ops_enable_state_str[] = { -- [SCX_OPS_PREPPING] = "prepping", -- [SCX_OPS_ENABLING] = "enabling", -- [SCX_OPS_ENABLED] = "enabled", -- [SCX_OPS_DISABLING] = "disabling", -- [SCX_OPS_DISABLED] = "disabled", --}; -- --static int scx_debug_show(struct seq_file *m, void *v) --{ -- mutex_lock(&scx_ops_enable_mutex); -- seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name); -- seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled()); -- seq_printf(m, "%-30s: %d\n", "switching_all", -- READ_ONCE(scx_switching_all)); -- seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all()); -- seq_printf(m, "%-30s: %s\n", "enable_state", -- scx_ops_enable_state_str[scx_ops_enable_state()]); -- seq_printf(m, "%-30s: %llu\n", "nr_rejected", -- atomic64_read(&scx_nr_rejected)); -- mutex_unlock(&scx_ops_enable_mutex); -- return 0; --} -- --static int scx_debug_open(struct inode *inode, struct file *file) --{ -- return single_open(file, scx_debug_show, NULL); --} -- --const struct file_operations sched_ext_fops = { -- .open = scx_debug_open, -- .read = seq_read, -- .llseek = seq_lseek, -- .release = single_release, --}; --#endif -- --/******************************************************************************** -- * bpf_struct_ops plumbing. -- */ --#include --#include --#include -- --extern struct btf *btf_vmlinux; --static const struct btf_type *task_struct_type; -- --static bool bpf_scx_is_valid_access(int off, int size, -- enum bpf_access_type type, -- const struct bpf_prog *prog, -- struct bpf_insn_access_aux *info) --{ -- if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) -- return false; -- if (type != BPF_READ) -- return false; -- if (off % size != 0) -- return false; -- -- return btf_ctx_access(off, size, type, prog, info); --} -- --static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, -- const struct bpf_reg_state *reg, -- int off, int size) --{ -- return 0; --} -- --static const struct bpf_func_proto * --bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) --{ -- switch (func_id) { -- case BPF_FUNC_task_storage_get: -- return &bpf_task_storage_get_proto; -- case BPF_FUNC_task_storage_delete: -- return &bpf_task_storage_delete_proto; -- default: -- return bpf_base_func_proto(func_id); -- } --} -- --const struct bpf_verifier_ops bpf_scx_verifier_ops = { -- .get_func_proto = bpf_scx_get_func_proto, -- .is_valid_access = bpf_scx_is_valid_access, -- .btf_struct_access = bpf_scx_btf_struct_access, --}; -- --static int bpf_scx_init_member(const struct btf_type *t, -- const struct btf_member *member, -- void *kdata, const void *udata) --{ -- const struct sched_ext_ops *uops = udata; -- struct sched_ext_ops *ops = kdata; -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- int ret; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, dispatch_max_batch): -- if (*(u32 *)(udata + moff) > INT_MAX) -- return -E2BIG; -- ops->dispatch_max_batch = *(u32 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, flags): -- if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) -- return -EINVAL; -- ops->flags = *(u64 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, name): -- ret = bpf_obj_name_cpy(ops->name, uops->name, -- sizeof(ops->name)); -- if (ret < 0) -- return ret; -- if (ret == 0) -- return -EINVAL; -- return 1; -- case offsetof(struct sched_ext_ops, timeout_ms): -- if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT) -- return -E2BIG; -- ops->timeout_ms = *(u32 *)(udata + moff); -- return 1; -- } -- -- return 0; --} -- --static int bpf_scx_check_member(const struct btf_type *t, -- const struct btf_member *member, -- const struct bpf_prog *prog) --{ -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, prep_enable): -- case offsetof(struct sched_ext_ops, init): -- case offsetof(struct sched_ext_ops, exit): -- break; -- default: -- /* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */ -- /* if (prog->aux->sleepable) -- return -EINVAL; */ -- break; -- } -- -- return 0; --} -- --static int bpf_scx_reg(void *kdata) --{ -- return scx_ops_enable(kdata); --} -- --static void bpf_scx_unreg(void *kdata) --{ -- scx_ops_disable(SCX_EXIT_UNREG); -- kthread_flush_work(&scx_ops_disable_work); --} -- --static int bpf_scx_init(struct btf *btf) --{ -- u32 type_id; -- -- type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); -- if (type_id < 0) -- return -EINVAL; -- task_struct_type = btf_type_by_id(btf, type_id); -- -- return 0; --} -- --/* "extern" to avoid sparse warning, only used in this file */ --extern struct bpf_struct_ops bpf_sched_ext_ops; -- --struct bpf_struct_ops bpf_sched_ext_ops = { -- .verifier_ops = &bpf_scx_verifier_ops, -- .reg = bpf_scx_reg, -- .unreg = bpf_scx_unreg, -- .check_member = bpf_scx_check_member, -- .init_member = bpf_scx_init_member, -- .init = bpf_scx_init, -- .name = "sched_ext_ops", --}; -- --static void sysrq_handle_sched_ext_reset(u8 key) --{ -- if (scx_ops_helper) -- scx_ops_disable(SCX_EXIT_SYSRQ); -- else -- pr_info("sched_ext: BPF scheduler not yet used\n"); --} -- --static const struct sysrq_key_op sysrq_sched_ext_reset_op = { -- .handler = sysrq_handle_sched_ext_reset, -- .help_msg = "reset-sched-ext(S)", -- .action_msg = "Disable sched_ext and revert all tasks to CFS", -- .enable_mask = SYSRQ_ENABLE_RTNICE, --}; -- --static void kick_cpus_irq_workfn(struct irq_work *irq_work) --{ -- struct rq *this_rq = this_rq(); -- u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); -- int this_cpu = cpu_of(this_rq); -- int cpu; -- -- for_each_cpu(cpu, this_rq->scx->cpus_to_kick) { -- struct rq *rq = cpu_rq(cpu); -- unsigned long flags; -- -- raw_spin_rq_lock_irqsave(rq, flags); -- -- if (cpu_online(cpu) || cpu == this_cpu) { -- if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) && -- rq->curr->sched_class == &ext_sched_class) -- rq->curr->scx->slice = 0; -- pseqs[cpu] = rq->scx->pnt_seq; -- resched_curr(rq); -- } else { -- cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait); -- } -- -- raw_spin_rq_unlock_irqrestore(rq, flags); -- } -- -- for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait, -- cpumask_of(this_cpu)) { -- /* -- * Pairs with smp_store_release() issued by this CPU in -- * scx_notify_pick_next_task() on the resched path. -- * -- * We busy-wait here to guarantee that no other task can be -- * scheduled on our core before the target CPU has entered the -- * resched path. -- */ -- while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu]) -- cpu_relax(); -- } -- -- cpumask_clear(this_rq->scx->cpus_to_kick); -- cpumask_clear(this_rq->scx->cpus_to_preempt); -- cpumask_clear(this_rq->scx->cpus_to_wait); --} -- --void __init init_sched_ext_class(void) --{ -- int cpu; -- u32 v; -- -- /* -- * The following is to prevent the compiler from optimizing out the enum -- * definitions so that BPF scheduler implementations can use them -- * through the generated vmlinux.h. -- */ -- WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | -- SCX_TG_ONLINE | SCX_KICK_PREEMPT); -- -- init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); --#ifdef CONFIG_SMP -- BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -- BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); --#endif -- scx_kick_cpus_pnt_seqs = -- __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * -- num_possible_cpus(), -- __alignof__(scx_kick_cpus_pnt_seqs[0])); -- BUG_ON(!scx_kick_cpus_pnt_seqs); -- -- for_each_possible_cpu(cpu) { -- struct rq *rq = cpu_rq(cpu); -- -- rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL); -- if (rq->scx) -- rq->scx->rq = rq; -- else -- pr_err("fatal error : alloc rq->scx failed!!!\n"); -- -- init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL); -- INIT_LIST_HEAD(&rq->scx->watchdog_list); -- -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL)); -- init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn); -- } -- -- register_sysrq_key('S', &sysrq_sched_ext_reset_op); -- INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); -- scx_cgroup_config_knobs(); --} -- -- --/******************************************************************************** -- * Helpers that can be called from the BPF scheduler. -- */ --#include -- --/* Disables missing prototype warnings for kfuncs */ --__diag_push(); --__diag_ignore_all("-Wmissing-prototypes", -- "Global functions as their definitions will be in vmlinux BTF"); -- --/** -- * scx_bpf_switch_all - Switch all tasks into SCX -- * @into_scx: switch direction -- * -- * If @into_scx is %true, all existing and future non-dl/rt tasks are switched -- * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on -- * SCX. The actual switching is asynchronous. Can be called from ops.init(). -- */ --void scx_bpf_switch_all(void) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return; -- -- scx_switch_all_req = true; --} -- --BTF_SET8_START(scx_kfunc_ids_init) --BTF_ID_FLAGS(func, scx_bpf_switch_all) --BTF_SET8_END(scx_kfunc_ids_init) -- --static const struct btf_kfunc_id_set scx_kfunc_set_init = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_init, --}; -- --/** -- * scx_bpf_create_dsq - Create a custom DSQ -- * @dsq_id: DSQ to create -- * @node: NUMA node to allocate from -- * -- * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(), -- * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move(). -- */ --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return -EINVAL; -- -- if (unlikely(node >= (int)nr_node_ids || -- (node < 0 && node != NUMA_NO_NODE))) -- return -EINVAL; -- return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); --} -- --BTF_SET8_START(scx_kfunc_ids_sleepable) --BTF_ID_FLAGS(func, scx_bpf_create_dsq) --BTF_SET8_END(scx_kfunc_ids_sleepable) -- --static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_sleepable, --}; -- --static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) --{ -- if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) -- return false; -- -- lockdep_assert_irqs_disabled(); -- -- if (unlikely(!p)) { -- scx_ops_error("called with NULL task"); -- return false; -- } -- -- if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { -- scx_ops_error("invalid enq_flags 0x%llx", enq_flags); -- return false; -- } -- -- return true; --} -- --static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) --{ -- struct task_struct *ddsp_task; -- int idx; -- -- ddsp_task = __this_cpu_read(direct_dispatch_task); -- if (ddsp_task) { -- direct_dispatch(ddsp_task, p, dsq_id, enq_flags); -- return; -- } -- -- idx = __this_cpu_read(scx_dsp_ctx.buf_cursor); -- if (unlikely(idx >= scx_dsp_max_batch)) { -- scx_ops_error("dispatch buffer overflow"); -- return; -- } -- -- this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){ -- .task = p, -- .qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK, -- .dsq_id = dsq_id, -- .enq_flags = enq_flags, -- }; -- __this_cpu_inc(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe -- * to call this function spuriously. Can be called from ops.enqueue() and -- * ops.dispatch(). -- * -- * When called from ops.enqueue(), it's for direct dispatch and @p must match -- * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the -- * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be -- * on the target CPU in the first place. -- * -- * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id -- * and this function can be called upto ops.dispatch_max_batch times to dispatch -- * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the -- * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. -- * -- * This function doesn't have any locking restrictions and may be called under -- * BPF locks (in the future when BPF introduces more flexible locking). -- * -- * @p is allowed to run for @slice. The scheduling path is triggered on slice -- * exhaustion. If zero, the current residual slice is maintained. If -- * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with -- * scx_bpf_kick_cpu() to trigger scheduling. -- */ --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- scx_dispatch_commit(p, dsq_id, enq_flags); --} -- --/** -- * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. -- * Tasks queued into the priority queue are ordered by @vtime and always -- * consumed after the tasks in the FIFO queue. All other aspects are identical -- * to scx_bpf_dispatch(). -- * -- * @vtime ordering is according to time_before64() which considers wrapping. A -- * numerically larger vtime may indicate an earlier position in the ordering and -- * vice-versa. -- */ --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 vtime, u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- p->scx->dsq_vtime = vtime; -- -- scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); --} -- --BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime) --BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_enqueue_dispatch, --}; -- --/** -- * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots -- * -- * Can only be called from ops.dispatch(). -- */ --u32 scx_bpf_dispatch_nr_slots(void) --{ -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return 0; -- -- return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ -- * @dsq_id: DSQ to consume -- * -- * Consume a task from the non-local DSQ identified by @dsq_id and transfer it -- * to the current CPU's local DSQ for execution. Can only be called from -- * ops.dispatch(). -- * -- * This function flushes the in-flight dispatches from scx_bpf_dispatch() before -- * trying to consume the specified DSQ. It may also grab rq locks and thus can't -- * be called under any BPF locks. -- * -- * Returns %true if a task has been consumed, %false if there isn't any task to -- * consume. -- */ --bool scx_bpf_consume(u64 dsq_id) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- struct scx_dispatch_q *dsq; -- -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return false; -- -- flush_dispatch_buf(dspc->rq, dspc->rf); -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); -- return false; -- } -- -- if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) { -- /* -- * A successfully consumed task can be dequeued before it starts -- * running while the CPU is trying to migrate other dispatched -- * tasks. Bump nr_tasks to tell balance_scx() to retry on empty -- * local DSQ. -- */ -- dspc->nr_tasks++; -- return true; -- } else { -- return false; -- } --} -- --BTF_SET8_START(scx_kfunc_ids_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) --BTF_ID_FLAGS(func, scx_bpf_consume) --BTF_SET8_END(scx_kfunc_ids_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_dispatch, --}; -- --/** -- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ -- * -- * Iterate over all of the tasks currently enqueued on the local DSQ of the -- * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of -- * processed tasks. Can only be called from ops.cpu_release(). -- */ --u32 scx_bpf_reenqueue_local(void) --{ -- u32 nr_enqueued, i; -- struct rq *rq; -- struct scx_rq *scx_rq; -- -- if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) -- return 0; -- -- rq = cpu_rq(smp_processor_id()); -- lockdep_assert_rq_held(rq); -- scx_rq = rq->scx; -- -- /* -- * Get the number of tasks on the local DSQ before iterating over it to -- * pull off tasks. The enqueue callback below can signal that it wants -- * the task to stay on the local DSQ, and we want to prevent the BPF -- * scheduler from causing us to loop indefinitely. -- */ -- nr_enqueued = scx_rq->local_dsq.nr; -- for (i = 0; i < nr_enqueued; i++) { -- struct task_struct *p; -- -- p = first_local_task(rq); -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- WARN_ON_ONCE(p->scx->holding_cpu != -1); -- dispatch_dequeue(scx_rq, p); -- do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); -- } -- -- return nr_enqueued; --} -- --BTF_SET8_START(scx_kfunc_ids_cpu_release) --BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) --BTF_SET8_END(scx_kfunc_ids_cpu_release) -- --static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_cpu_release, --}; -- --/** -- * scx_bpf_kick_cpu - Trigger reschedule on a CPU -- * @cpu: cpu to kick -- * @flags: SCX_KICK_* flags -- * -- * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or -- * trigger rescheduling on a busy CPU. This can be called from any online -- * scx_ops operation and the actual kicking is performed asynchronously through -- * an irq work. -- */ --void scx_bpf_kick_cpu(s32 cpu, u64 flags) --{ -- struct rq *rq; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d", cpu); -- return; -- } -- -- preempt_disable(); -- rq = this_rq(); -- -- /* -- * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting -- * rq locks. We can probably be smarter and avoid bouncing if called -- * from ops which don't hold a rq lock. -- */ -- cpumask_set_cpu(cpu, rq->scx->cpus_to_kick); -- if (flags & SCX_KICK_PREEMPT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt); -- if (flags & SCX_KICK_WAIT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_wait); -- -- irq_work_queue(&rq->scx->kick_cpus_irq_work); -- preempt_enable(); --} -- --/** -- * scx_bpf_dsq_nr_queued - Return the number of queued tasks -- * @dsq_id: id of the DSQ -- * -- * Return the number of tasks in the DSQ matching @dsq_id. If not found, -- * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops -- * operations. -- */ --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) --{ -- struct scx_dispatch_q *dsq; -- -- lockdep_assert(rcu_read_lock_any_held()); -- -- if (dsq_id == SCX_DSQ_LOCAL) { -- return this_rq()->scx->local_dsq.nr; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (ops_cpu_valid(cpu)) -- return cpu_rq(cpu)->scx->local_dsq.nr; -- } else { -- dsq = find_non_local_dsq(dsq_id); -- if (dsq) -- return dsq->nr; -- } -- return -ENOENT; --} -- --/** -- * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state -- * @cpu: cpu to test and clear idle for -- * -- * Returns %true if @cpu was idle and its idle state was successfully cleared. -- * %false otherwise. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return false; -- } -- -- if (ops_cpu_valid(cpu)) -- return test_and_clear_cpu_idle(cpu); -- else -- return false; --} -- --/** -- * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu -- * @cpus_allowed: Allowed cpumask -- * -- * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked -- * idle cpu number on success. -%EBUSY if no matching cpu was found. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return -EBUSY; -- } -- -- return scx_pick_idle_cpu(cpus_allowed); --} -- --/** -- * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking -- * per-CPU cpumask. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_cpumask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.cpu; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, -- * per-physical-core cpumask. Can be used to determine if an entire physical -- * core is free. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_smtmask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.smt; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to -- * either the percpu, or SMT idle-tracking cpumask. -- */ --void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) --{ -- /* -- * Empty function body because we aren't actually acquiring or -- * releasing a reference to a global idle cpumask, which is read-only -- * in the caller and is never released. The acquire / release semantics -- * here are just used to make the cpumask is a trusted pointer in the -- * caller. -- */ --} -- --struct scx_bpf_error_bstr_bufs { -- u64 data[MAX_BPRINTF_VARARGS]; -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs); -- --/** -- * scx_bpf_error_bstr - Indicate fatal error -- * @fmt: error message format string -- * @data: format string parameters packaged using ___bpf_fill() macro -- * @data__sz: @data len, must end in '__sz' for the verifier -- * -- * Indicate that the BPF scheduler encountered a fatal error and initiate ops -- * disabling. -- */ --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) --{ -- struct scx_bpf_error_bstr_bufs *bufs; -- unsigned long flags; -- struct bpf_bprintf_data args = {}; -- int ret; -- -- local_irq_save(flags); -- bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs); -- -- if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || -- (data__sz && !data)) { -- scx_ops_error("invalid data=%p and data__sz=%u", -- (void *)data, data__sz); -- goto out_restore; -- } -- -- ret = copy_from_kernel_nofault(bufs->data, data, data__sz); -- if (ret) { -- scx_ops_error("failed to read data fields (%d)", ret); -- goto out_restore; -- } -- -- ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args); -- if (ret < 0) { -- scx_ops_error("failed to format prepration (%d)", ret); -- goto out_restore; -- } -- -- ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt, -- args.bin_args); -- bpf_bprintf_cleanup(&args); -- if (ret < 0) { -- scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format", -- fmt, data, data__sz); -- goto out_restore; -- } -- -- scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg); --out_restore: -- local_irq_restore(flags); --} -- --/** -- * scx_bpf_destroy_dsq - Destroy a custom DSQ -- * @dsq_id: DSQ to destroy -- * -- * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with -- * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is -- * empty and no further tasks are dispatched to it. Ignored if called on a DSQ -- * which doesn't exist. Can be called from any online scx_ops operations. -- */ --void scx_bpf_destroy_dsq(u64 dsq_id) --{ -- destroy_dsq(dsq_id); --} -- --/** -- * scx_bpf_task_running - Is task currently running? -- * @p: task of interest -- */ --bool scx_bpf_task_running(const struct task_struct *p) --{ -- return task_rq(p)->curr == p; --} -- --/** -- * scx_bpf_task_cpu - CPU a task is currently associated with -- * @p: task of interest -- */ --s32 scx_bpf_task_cpu(const struct task_struct *p) --{ -- return task_cpu(p); --} -- --/** -- * scx_bpf_task_cgroup - Return the sched cgroup of a task -- * @p: task of interest -- * -- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with -- * from the scheduler's POV. SCX operations should use this function to -- * determine @p's current cgroup as, unlike following @p->cgroups, -- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all -- * rq-locked operations. Can be called on the parameter tasks of rq-locked -- * operations. The restriction guarantees that @p's rq is locked by the caller. -- */ --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) --{ -- struct task_group *tg = p->sched_task_group; -- struct cgroup *cgrp = &cgrp_dfl_root.cgrp; -- -- if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) -- goto out; -- -- /* -- * A task_group may either be a cgroup or an autogroup. In the latter -- * case, @tg->css.cgroup is %NULL. A task_group can't become the other -- * kind once created. -- */ -- if (tg && tg->css.cgroup) -- cgrp = tg->css.cgroup; -- else -- cgrp = &cgrp_dfl_root.cgrp; --out: -- cgroup_get(cgrp); -- return cgrp; --} -- --BTF_SET8_START(scx_kfunc_ids_any) --BTF_ID_FLAGS(func, scx_bpf_kick_cpu) --BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) --BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) --BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu) --BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) --BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) --BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) --BTF_ID_FLAGS(func, scx_bpf_task_running) --BTF_ID_FLAGS(func, scx_bpf_task_cpu) --BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE) --BTF_SET8_END(scx_kfunc_ids_any) -- --static const struct btf_kfunc_id_set scx_kfunc_set_any = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_any, --}; -- --__diag_pop(); -- --/* -- * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set() -- * needs most of the system to be up. -- */ --static int __init register_ext_kfuncs(void) --{ -- int ret; -- -- /* -- * Some kfuncs are context-sensitive and can only be called from -- * specific SCX ops. They are grouped into BTF sets accordingly. -- * Unfortunately, BPF currently doesn't have a way of enforcing such -- * restrictions. Eventually, the verifier should be able to enforce -- * them. For now, register them the same and make each kfunc explicitly -- * check using scx_kf_allowed(). -- */ -- if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_init)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_sleepable)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_enqueue_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_cpu_release)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_any))) { -- pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret); -- return ret; -- } -- -- return 0; --} --__initcall(register_ext_kfuncs); -diff --git a/kernel/sched/ext.h b/kernel/sched/ext.h -deleted file mode 100755 -index fba8ed845f99..000000000000 ---- a/kernel/sched/ext.h -+++ /dev/null -@@ -1,257 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ -- -- --/* -- * Tag marking a kernel function as a kfunc. This is meant to minimize the -- * amount of copy-paste that kfunc authors have to include for correctness so -- * as to avoid issues such as the compiler inlining or eliding either a static -- * kfunc, or a global kfunc in an LTO build. -- */ --#define __bpf_kfunc __used noinline -- --enum scx_wake_flags { -- /* expose select WF_* flags as enums */ -- SCX_WAKE_EXEC = WF_EXEC, -- SCX_WAKE_FORK = WF_FORK, -- SCX_WAKE_TTWU = WF_TTWU, -- SCX_WAKE_SYNC = WF_SYNC, --}; -- --enum scx_enq_flags { -- /* expose select ENQUEUE_* flags as enums */ -- SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, -- SCX_ENQ_HEAD = ENQUEUE_HEAD, -- -- /* high 32bits are SCX specific */ -- -- /* -- * Set the following to trigger preemption when calling -- * scx_bpf_dispatch() with a local dsq as the target. The slice of the -- * current task is cleared to zero and the CPU is kicked into the -- * scheduling path. Implies %SCX_ENQ_HEAD. -- */ -- SCX_ENQ_PREEMPT = 1LLU << 32, -- -- /* -- * The task being enqueued was previously enqueued on the current CPU's -- * %SCX_DSQ_LOCAL, but was removed from it in a call to the -- * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was -- * invoked in a ->cpu_release() callback, and the task is again -- * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the -- * task will not be scheduled on the CPU until at least the next invocation -- * of the ->cpu_acquire() callback. -- */ -- SCX_ENQ_REENQ = 1LLU << 40, -- -- /* -- * The task being enqueued is the only task available for the cpu. By -- * default, ext core keeps executing such tasks but when -- * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with -- * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set. -- * -- * If the BPF scheduler wants to continue executing the task, -- * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately. -- * If the task gets queued on a different dsq or the BPF side, the BPF -- * scheduler is responsible for triggering a follow-up scheduling event. -- * Otherwise, Execution may stall. -- */ -- SCX_ENQ_LAST = 1LLU << 41, -- -- /* -- * A hint indicating that it's advisable to enqueue the task on the -- * local dsq of the currently selected CPU. Currently used by -- * select_cpu_dfl() and together with %SCX_ENQ_LAST. -- */ -- SCX_ENQ_LOCAL = 1LLU << 42, -- -- /* high 8 bits are internal */ -- __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, -- -- SCX_ENQ_CLEAR_OPSS = 1LLU << 56, -- SCX_ENQ_DSQ_PRIQ = 1LLU << 57, --}; -- --enum scx_deq_flags { -- /* expose select DEQUEUE_* flags as enums */ -- SCX_DEQ_SLEEP = DEQUEUE_SLEEP, -- -- /* high 32bits are SCX specific */ -- -- /* -- * The generic core-sched layer decided to execute the task even though -- * it hasn't been dispatched yet. Dequeue from the BPF side. -- */ -- SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, --}; -- --enum scx_tg_flags { -- SCX_TG_ONLINE = 1U << 0, -- SCX_TG_INITED = 1U << 1, --}; -- --enum scx_kick_flags { -- SCX_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -- SCX_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ --}; -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --extern const struct sched_class ext_sched_class; --extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops; --extern const struct file_operations sched_ext_fops; --extern unsigned long scx_watchdog_timeout; --extern unsigned long scx_watchdog_timestamp; -- --DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); --DECLARE_STATIC_KEY_FALSE(__scx_switched_all); --#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) --#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) -- --DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); -- --bool task_on_scx(struct task_struct *p); --void scx_pre_fork(struct task_struct *p); --int scx_fork(struct task_struct *p); --void scx_post_fork(struct task_struct *p); --void scx_cancel_fork(struct task_struct *p); --int scx_check_setscheduler(struct task_struct *p, int policy); --bool scx_can_stop_tick(struct rq *rq); --void init_sched_ext_class(void); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...); --#define scx_ops_error(fmt, args...) \ -- scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args) -- --void __scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active); -- --static inline void scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active) --{ -- if (!scx_enabled()) -- return; --#ifdef CONFIG_SMP -- /* -- * Pairs with the smp_load_acquire() issued by a CPU in -- * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -- * resched. -- */ -- smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1); --#endif -- if (!static_branch_unlikely(&scx_ops_cpu_preempt)) -- return; -- __scx_notify_pick_next_task(rq, p, active); --} -- --//extern void scx_scheduler_tick(void); --static inline void scx_notify_sched_tick(void) --{ -- unsigned long last_check; -- -- if (!scx_enabled()) -- return; -- //scx_scheduler_tick(); -- -- last_check = scx_watchdog_timestamp; -- if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "watchdog failed to check in for %u.%03us", -- dur_ms / 1000, dur_ms % 1000); -- } --} -- --static inline const struct sched_class *next_active_class(const struct sched_class *class) --{ -- class++; -- if (scx_switched_all() && class == &fair_sched_class) -- class++; -- if (!scx_enabled() && class == &ext_sched_class) -- class++; -- return class; --} -- --#define for_active_class_range(class, _from, _to) \ -- for (class = (_from); class != (_to); class = next_active_class(class)) -- --#define for_each_active_class(class) \ -- for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -- --/* -- * SCX requires a balance() call before every pick_next_task() call including -- * when waking up from idle. -- */ --#define for_balance_class_range(class, prev_class, end_class) \ -- for_active_class_range(class, (prev_class) > &ext_sched_class ? \ -- &ext_sched_class : (prev_class), (end_class)) -- --#ifdef CONFIG_SCHED_CORE --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi); --#endif -- --#else /* CONFIG_SCHED_CLASS_EXT */ -- --#define scx_enabled() false --#define scx_switched_all() false -- --static inline void scx_pre_fork(struct task_struct *p) {} --static inline int scx_fork(struct task_struct *p) { return 0; } --static inline void scx_post_fork(struct task_struct *p) {} --static inline void scx_cancel_fork(struct task_struct *p) {} --static inline int scx_check_setscheduler(struct task_struct *p, -- int policy) { return 0; } --static inline bool scx_can_stop_tick(struct rq *rq) { return true; } --static inline void init_sched_ext_class(void) {} --static inline void scx_notify_pick_next_task(struct rq *rq, -- const struct task_struct *p, -- const struct sched_class *active) {} --static inline void scx_notify_sched_tick(void) {} -- --#define for_each_active_class for_each_class --#define for_balance_class_range for_class_range -- --#endif /* CONFIG_SCHED_CLASS_EXT */ -- --#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) --void __scx_update_idle(struct rq *rq, bool idle); -- --static inline void scx_update_idle(struct rq *rq, bool idle) --{ -- if (scx_enabled()) -- __scx_update_idle(rq, idle); --} --#else --static inline void scx_update_idle(struct rq *rq, bool idle) {} --#endif -- --#ifdef CONFIG_CGROUP_SCHED --#ifdef CONFIG_EXT_GROUP_SCHED --int scx_tg_online(struct task_group *tg); --void scx_tg_offline(struct task_group *tg); --int scx_cgroup_can_attach(struct cgroup_taskset *tset); --void scx_move_task(struct task_struct *p); --void scx_cgroup_finish_attach(void); --void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); --void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); --#else /* CONFIG_EXT_GROUP_SCHED */ --static inline int scx_tg_online(struct task_group *tg) { return 0; } --static inline void scx_tg_offline(struct task_group *tg) {} --static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } --static inline void scx_move_task(struct task_struct *p) {} --static inline void scx_cgroup_finish_attach(void) {} --static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} --static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} --#endif /* CONFIG_EXT_GROUP_SCHED */ --#endif /* CONFIG_CGROUP_SCHED */ -diff --git a/kernel/sched/hmbird.h b/kernel/sched/hmbird.h -new file mode 100755 -index 000000000000..d0d84ddee13d ---- /dev/null -+++ b/kernel/sched/hmbird.h -@@ -0,0 +1,343 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#ifndef _HMBIRD_H_ -+#define _HMBIRD_H_ -+ -+/* -+ * Tag marking a kernel function as a kfunc. This is meant to minimize the -+ * amount of copy-paste that kfunc authors have to include for correctness so -+ * as to avoid issues such as the compiler inlining or eliding either a static -+ * kfunc, or a global kfunc in an LTO build. -+ */ -+ -+#define DEBUG_INTERNAL (1 << 0) -+#define DEBUG_INFO_TRACE (1 << 1) -+#define DEBUG_INFO_SYSTRACE (1 << 2) -+ -+#define NUMS_CGROUP_KINDS (512) -+#define SLIM_FOR_SGAME (1 << 0) -+#define SLIM_FOR_GENSHIN (1 << 1) -+ -+#ifdef MAX_TASK_NR -+#define MAX_KEY_THREAD_RECORD ((MAX_TASK_NR + 1) >> 1) -+#else -+#define MAX_KEY_THREAD_RECORD (1 << 3) -+#endif /* MAX_TASK_NR */ -+ -+#define TOP_TASK_SHIFT (8) -+#define TOP_TASK_MAX (1 << TOP_TASK_SHIFT) -+#ifndef TOP_TASK_BITS_MASK -+#define TOP_TASK_BITS_MASK (TOP_TASK_MAX - 1) -+#endif /* TOP_TASK_BITS_MASK */ -+ -+extern struct md_info_t *md_info; -+ -+#define debug_enabled() \ -+ (unlikely(hmbirdcore_debug)) -+ -+#define hmbird_debug(fmt, ...) \ -+ pr_info(":"fmt, ##__VA_ARGS__) -+ -+#define hmbird_err(id, fmt, ...) \ -+do { \ -+ pr_err(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_deferred_err(id, fmt, ...) \ -+do { \ -+ printk_deferred(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \ -+do { \ -+ if (unlikely(cond)) { \ -+ hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+ } \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_info_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_info_trace(fmt, ...) -+#endif -+ -+#define hmbird_info_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+ } \ -+} while (0) -+ -+#define hmbird_output_systrace(fmt, ...) \ -+do { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_internal_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_internal_trace(fmt, ...) -+#endif -+ -+#define hmbird_internal_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) { \ -+ hmbird_output_systrace(fmt, ##__VA_ARGS__); \ -+ } \ -+} while (0) -+ -+enum hmbird_wake_flags { -+ /* expose select WF_* flags as enums */ -+ HMBIRD_WAKE_EXEC = WF_EXEC, -+ HMBIRD_WAKE_FORK = WF_FORK, -+ HMBIRD_WAKE_TTWU = WF_TTWU, -+ HMBIRD_WAKE_SYNC = WF_SYNC, -+}; -+ -+enum hmbird_enq_flags { -+ /* expose select ENQUEUE_* flags as enums */ -+ HMBIRD_ENQ_WAKEUP = ENQUEUE_WAKEUP, -+ HMBIRD_ENQ_HEAD = ENQUEUE_HEAD, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * Set the following to trigger preemption when calling -+ * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the -+ * current task is cleared to zero and the CPU is kicked into the -+ * scheduling path. Implies %HMBIRD_ENQ_HEAD. -+ */ -+ HMBIRD_ENQ_PREEMPT = 1LLU << 32, -+ HMBIRD_ENQ_REENQ = 1LLU << 40, -+ HMBIRD_ENQ_LAST = 1LLU << 41, -+ -+ /* -+ * A hint indicating that it's advisable to enqueue the task on the -+ * local dsq of the currently selected CPU. Currently used by -+ * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST. -+ */ -+ HMBIRD_ENQ_LOCAL = 1LLU << 42, -+ -+ /* high 8 bits are internal */ -+ __HMBIRD_ENQ_INTERNAL_MASK = 0xffLLU << 56, -+ -+ HMBIRD_ENQ_CLEAR_OPSS = 1LLU << 56, -+ HMBIRD_ENQ_DSQ_PRIQ = 1LLU << 57, -+}; -+ -+enum hmbird_deq_flags { -+ /* expose select DEQUEUE_* flags as enums */ -+ HMBIRD_DEQ_SLEEP = DEQUEUE_SLEEP, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * The generic core-sched layer decided to execute the task even though -+ * it hasn't been dispatched yet. Dequeue from the BPF side. -+ */ -+ HMBIRD_DEQ_CORE_SCHED_EXEC = 1LLU << 32, -+}; -+ -+enum hmbird_kick_flags { -+ HMBIRD_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -+ HMBIRD_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ -+}; -+ -+enum hmbird_task_prop_type { -+ HMBIRD_TASK_PROP_COMMON, -+ HMBIRD_TASK_PROP_PIPELINE, -+ HMBIRD_TASK_PROP_DEBUG_OR_LOG, -+ HMBIRD_TASK_PROP_HIGH_LOAD, -+ HMBIRD_TASK_PROP_IO, -+ HMBIRD_TASK_PROP_NETWORK, -+ HMBIRD_TASK_PROP_PERIODIC, -+ HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL, -+ HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL, -+ HMBIRD_TASK_PROP_ISOLATE, -+ HMBIRD_TASK_PROP_MAX, -+}; -+ -+enum hmbird_preempt_policy_id { -+ HMBIRD_PREEMPT_POLICY_NONE, -+ HMBIRD_PREEMPT_POLICY_PRIO_BASED, -+}; -+ -+extern const struct sched_class hmbird_sched_class; -+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops; -+extern const struct file_operations sched_hmbird_fops; -+extern unsigned long hmbird_watchdog_timeout; -+extern unsigned long hmbird_watchdog_timestamp; -+ -+enum scx_rq_flags { -+ HMBIRD_RQ_CAN_STOP_TICK = 1 << 0, -+}; -+ -+struct hmbird_rq { -+ struct hmbird_dispatch_q local_dsq; -+ struct list_head watchdog_list; -+ u64 ops_qseq; -+ u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -+ u32 nr_running; -+ u32 flags; -+ bool cpu_released; -+ cpumask_var_t cpus_to_kick; -+ cpumask_var_t cpus_to_preempt; -+ cpumask_var_t cpus_to_wait; -+ u64 pnt_seq; -+ u64 *prev_runnable_sum_fixed; -+ u32 *prev_window_size; -+ struct irq_work kick_cpus_irq_work; -+ bool pipeline; -+ bool exclusive; -+ bool period_disallow; -+ bool nonperiod_disallow; -+ struct rq *rq; -+ struct hmbird_sched_rq_stats *srq; -+}; -+ -+struct hmbird_sched_change_guard { -+ struct task_struct *p; -+ struct rq *rq; -+ bool queued; -+ bool running; -+ bool done; -+}; -+ -+extern struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -+ -+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags); -+ -+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -+ for (struct hmbird_sched_change_guard __cg = \ -+ hmbird_sched_change_guard_init(__rq, __p, __flags); \ -+ !__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags)) -+ -+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+bool task_on_hmbird(struct task_struct *p); -+int hmbird_pre_fork(struct task_struct *p); -+int hmbird_fork(struct task_struct *p); -+void hmbird_post_fork(struct task_struct *p); -+void hmbird_cancel_fork(struct task_struct *p); -+int hmbird_check_setscheduler(struct task_struct *p, int policy); -+bool hmbird_can_stop_tick(struct rq *rq); -+void init_sched_hmbird_class(void); -+ -+void hmbird_ops_exit(void); -+#define hmbird_ops_error(fmt, ...) \ -+do { \ -+ hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__); \ -+ hmbird_ops_exit(); \ -+} while (0) -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active); -+ -+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight) -+{ -+ return clamp_t(unsigned long, -+ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -+ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); -+} -+ -+static inline void hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active) -+{ -+ if (!hmbird_enabled()) -+ return; -+#ifdef CONFIG_SMP -+ /* -+ * Pairs with the smp_load_acquire() issued by a CPU in -+ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -+ * resched. -+ */ -+ smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1); -+#endif -+ if (!static_branch_unlikely(&hmbird_ops_cpu_preempt)) -+ return; -+ __hmbird_notify_pick_next_task(rq, p, active); -+} -+ -+extern void hmbird_scheduler_tick(void); -+void scan_timeout(struct rq *rq); -+void hmbird_notify_sched_tick(void); -+ -+static inline const struct sched_class *next_active_class(const struct sched_class *class) -+{ -+ class++; -+ -+ if (hmbird_enabled() && class == &fair_sched_class) -+ class++; -+ if (!hmbird_enabled() && class == &hmbird_sched_class) -+ class++; -+ return class; -+} -+ -+#define for_active_class_range(class, _from, _to) \ -+ for (class = (_from); class != (_to); class = next_active_class(class)) -+ -+#define for_each_active_class(class) \ -+ for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -+ -+/* -+ * HMBIRD requires a balance() call before every pick_next_task() call including -+ * when waking up from idle. -+ */ -+#define for_balance_class_range(class, prev_class, end_class) \ -+ for_active_class_range(class, (prev_class) > &hmbird_sched_class ? \ -+ &hmbird_sched_class : (prev_class), (end_class)) -+ -+#define MIN_CGROUP_DL_IDX (5) /* 8ms */ -+#define DEFAULT_CGROUP_DL_IDX (8) /* 64ms */ -+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS]; -+ -+ -+void __hmbird_update_idle(struct rq *rq, bool idle); -+ -+static inline void hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ if (hmbird_enabled()) -+ __hmbird_update_idle(rq, idle); -+} -+ -+int hmbird_ctrl(bool enable); -+ -+int hmbird_tg_online(struct task_group *tg); -+ -+ -+static inline u16 hmbird_task_util(struct task_struct *p) -+{ -+ return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0; -+} -+u16 hmbird_cpu_util(int cpu); -+ -+bool get_hmbird_ops_enabled(void); -+bool get_non_hmbird_task(void); -+void set_cpu_cluster(u64 cpu_cluster); -+#endif /*_EXT_H_*/ -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -new file mode 100755 -index 000000000000..ce05928392bf ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird.c -@@ -0,0 +1,4313 @@ -+// SPDX-License-Identifier: GPL-2.0 -+ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#include -+#include -+ -+#include "slim.h" -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+#include -+ -+#define CLUSTER_SEPARATE -+ -+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0); -+atomic_t non_hmbird_task; -+atomic_t hmbird_module_loaded = ATOMIC_INIT(0); -+int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0}; -+ -+enum hmbird_internal_consts { -+ HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ, -+}; -+ -+enum hmbird_ops_enable_state { -+ HMBIRD_OPS_PREPPING, -+ HMBIRD_OPS_ENABLING, -+ HMBIRD_OPS_ENABLED, -+ HMBIRD_OPS_DISABLING, -+ HMBIRD_OPS_DISABLED, -+}; -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+/* -+ * hmbird_entity->ops_state -+ * -+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler. -+ * State transitions look as follows: -+ * -+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING -+ * ^ | | -+ * | v v -+ * \-------------------------------/ -+ * -+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -+ * sites for explanations on the conditions being waited upon and why they are -+ * safe. Transitions out of them into NONE or QUEUED must store_release and the -+ * waiters should load_acquire. -+ * -+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether -+ * any given task can be dispatched by the BPF scheduler at all times and thus -+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core -+ * can safely reject invalid dispatches. -+ */ -+enum hmbird_ops_state { -+ HMBIRD_OPSS_NONE, /* owned by the HMBIRD core */ -+ HMBIRD_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -+ HMBIRD_OPSS_QUEUED, /* owned by the BPF scheduler */ -+ HMBIRD_OPSS_DISPATCHING, /* in transit back to the HMBIRD core */ -+ -+ /* -+ * QSEQ brands each QUEUED instance so that, when dispatch races -+ * dequeue/requeue, the dispatcher can tell whether it still has a claim -+ * on the task being dispatched. -+ */ -+ HMBIRD_OPSS_QSEQ_SHIFT = 2, -+ HMBIRD_OPSS_STATE_MASK = (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1, -+ HMBIRD_OPSS_QSEQ_MASK = ~HMBIRD_OPSS_STATE_MASK, -+}; -+ -+enum switch_stat { -+ HMBIRD_DISABLED, -+ HMBIRD_SWITCH_PREP, -+ HMBIRD_RQ_SWITCH_BEGIN, -+ HMBIRD_RQ_SWITCH_DONE, -+ HMBIRD_ENABLED, -+}; -+enum switch_stat curr_ss; -+ -+/* -+ * During exit, a task may schedule after losing its PIDs. When disabling the -+ * BPF scheduler, we need to be able to iterate tasks in every state to -+ * guarantee system safety. Maintain a dedicated task list which contains every -+ * task between its fork and eventual free. -+ */ -+DEFINE_SPINLOCK(hmbird_tasks_lock); -+static LIST_HEAD(hmbird_tasks); -+ -+/* ops enable/disable */ -+static struct kthread_worker *hmbird_ops_helper; -+static DEFINE_MUTEX(hmbird_ops_enable_mutex); -+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem); -+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED); -+ -+static bool hmbird_warned_zero_slice; -+enum hmbird_switch_type sw_type; -+static int skip_num[MAX_GLOBAL_DSQS]; -+static int big_distribute_mask_prev; -+static int little_distribute_mask_prev; -+ -+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0); -+ -+/* -+ * The maximum amount of time in jiffies that a task may be runnable without -+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger -+ * hmbird_ops_error(). -+ */ -+unsigned long hmbird_watchdog_timeout; -+ -+/* -+ * The last time the delayed work was run. This delayed work relies on -+ * ksoftirqd being able to run to service timer interrupts, so it's possible -+ * that this work itself could get wedged. To account for this, we check that -+ * it's not stalled in the timer tick, and trigger an error if it is. -+ */ -+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES; -+ -+static struct delayed_work hmbird_watchdog_work; -+static struct work_struct hmbird_err_exit_work; -+ -+/* idle tracking */ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_CPUMASK_OFFSTACK -+#define CL_ALIGNED_IF_ONSTACK -+#else -+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp -+#endif -+ -+static struct { -+ cpumask_var_t cpu; -+ cpumask_var_t smt; -+} idle_masks CL_ALIGNED_IF_ONSTACK; -+ -+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus; -+#endif /* CONFIG_SMP */ -+ -+/* dispatch queues */ -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global; -+ -+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128}; -+u32 pcp_dsq_deadline = 20; -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS]; -+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq); -+ -+static u64 max_hmbird_dsq_internal_id; -+ -+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/ -+#define CLUSTER_SEPARATE_IDX (8) -+#define GDSQS_ID_BASE (3) -+#define UX_COMPATIBLE_IDX (4) -+#define NON_PERIOD_START (5) -+#define NON_PERIOD_END (MAX_GLOBAL_DSQS) -+#define CREATE_DSQ_LEVEL_WITHIN (1) -+ -+struct hmbird_sched_info { -+ spinlock_t lock; -+ int curr_idx[2]; -+ int rtime[MAX_GLOBAL_DSQS]; -+}; -+ -+struct pcp_sched_info { -+ s64 pcp_seq; -+ int rtime; -+ bool pcp_round; -+}; -+ -+/* -+ * pcp_info may rw by another cpu. -+ * protected by rq->lock. -+ */ -+atomic64_t pcp_dsq_round; -+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info); -+ -+struct md_info_t *md_info; -+ -+static int b_rescue_l, l_rescue_b; -+static struct hmbird_sched_info sinfo; -+ -+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC; -+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = { -+ 0, 0, 0, 0, 0, -+ 32 * NSEC_PER_MSEC, -+ 20 * NSEC_PER_MSEC, -+ 14 * NSEC_PER_MSEC, -+ 8 * NSEC_PER_MSEC, -+ 6 * NSEC_PER_MSEC -+}; -+ -+struct cluster_ctx { -+ /* cpu-dsq map must within [lower, upper) */ -+ int upper; -+ int lower; -+ int tidx; -+}; -+ -+enum stat_items { -+ GLOBAL_STAT, -+ CPU_ALLOW_FAIL, -+ RT_CNT, -+ KEY_TASK_CNT, -+ SWITCH_IDX, -+ TIMEOUT_CNT, -+ -+ TOTAL_DSP_CNT, -+ MOVE_RQ_CNT, -+ SELECT_CPU, -+ -+ DWORD_STAT_END = SELECT_CPU, -+ -+ GDSQ_CNT, -+ ERR_IDX, -+ PCP_TIMEOUT_CNT, -+ PCP_LDSQ_CNT, -+ PCP_ENQL_CNT, -+ -+ MAX_ITEMS, -+}; -+static DEFINE_SPINLOCK(stats_lock); -+static char *stats_str[MAX_ITEMS] = { -+ "global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt", -+ "switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt", -+ "select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt", -+ "pcp_ldsq_cnt", "pcp_enql_cnt" -+}; -+ -+ -+struct stats_struct { -+ u64 global_stat[2]; -+ u64 cpu_allow_fail[2]; -+ u64 rt_cnt[2]; -+ u64 key_task_cnt[2]; -+ u64 switch_idx[2]; -+ u64 timeout_cnt[2]; -+ -+ u64 total_dsp_cnt[2]; -+ u64 move_rq_cnt[2]; -+ u64 select_cpu[2]; -+ -+ u64 gdsq_count[MAX_GLOBAL_DSQS][2]; -+ u64 err_idx[5]; -+ u64 pcp_timeout_cnt[NR_CPUS]; -+ u64 pcp_ldsq_count[NR_CPUS][2]; -+ u64 pcp_enql_cnt[NR_CPUS]; -+} stats_data; -+ -+static struct { -+ cpumask_var_t ex_free; -+ cpumask_var_t exclusive; -+ cpumask_var_t partial; -+ cpumask_var_t big; -+ cpumask_var_t little; -+} iso_masks __read_mostly; -+ -+/* -+ * Need more synchronization for these two variables? -+ * I choose not to. -+ */ -+static int l_need_rescue, b_need_rescue; -+ -+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...) \ -+{ \ -+ char buf[MAX_FATAL_INFO]; \ -+ \ -+ scnprintf(buf, MAX_FATAL_INFO, fmt, ##args); \ -+ hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf); \ -+ trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable), \ -+ READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf); \ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); \ -+} \ -+ -+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2) -+{ -+ int c1, c2; -+ struct cpumask mask = {.bits = {0}}; -+ struct cpumask tmp = {.bits = {0}}; -+ -+ if (!slim_stats) -+ return false; -+ -+ if (!rq1 || !rq2) -+ return false; -+ -+ c1 = cpu_of(rq1); -+ c2 = cpu_of(rq2); -+ cpumask_set_cpu(c1, &mask); -+ cpumask_set_cpu(c2, &mask); -+ -+ if (cpumask_and(&tmp, iso_masks.little, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.big, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.partial, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ return false; -+} -+ -+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu) -+{ -+ unsigned long flags; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ if (!slim_stats) -+ return; -+ -+ switch (item) { -+ case GLOBAL_STAT: -+ fallthrough; -+ case CPU_ALLOW_FAIL: -+ fallthrough; -+ case RT_CNT: -+ fallthrough; -+ case KEY_TASK_CNT: -+ fallthrough; -+ case SWITCH_IDX: -+ fallthrough; -+ case TIMEOUT_CNT: -+ fallthrough; -+ case TOTAL_DSP_CNT: -+ fallthrough; -+ case MOVE_RQ_CNT: -+ fallthrough; -+ case SELECT_CPU: -+ pval = pbase + item * 2 + idx; -+ break; -+ case GDSQ_CNT: -+ pval = &stats_data.gdsq_count[dsq_id][idx]; -+ break; -+ case ERR_IDX: -+ pval = &stats_data.err_idx[idx]; -+ break; -+ case PCP_TIMEOUT_CNT: -+ pval = &stats_data.pcp_timeout_cnt[cpu]; -+ break; -+ case PCP_LDSQ_CNT: -+ pval = &stats_data.pcp_ldsq_count[cpu][idx]; -+ break; -+ case PCP_ENQL_CNT: -+ pval = &stats_data.pcp_enql_cnt[cpu]; -+ break; -+ default: -+ return; -+ } -+ -+ spin_lock_irqsave(&stats_lock, flags); -+ *pval += 1; -+ spin_unlock_irqrestore(&stats_lock, flags); -+} -+ -+static inline bool handle_ret(int ret, int *idx, int len) -+{ -+ if (ret < 0 || ret >= len - *idx) -+ return true; -+ *idx += ret; -+ return false; -+} -+ -+#define PRINT_INTV (5 * HZ) -+void stats_print(char *buf, int len) -+{ -+ int idx = 0, i, j, ret; -+ int item = 0; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ for (item = 0; item < MAX_ITEMS; item++) { -+ if (item <= DWORD_STAT_END) { -+ pval = pbase + item * 2; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n", -+ stats_str[item], pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == GDSQ_CNT) { -+ for (j = 0; j < MAX_GLOBAL_DSQS; j++) { -+ pval = (u64 *)&stats_data.gdsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == ERR_IDX) { -+ pval = (u64 *)&stats_data.err_idx; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n", -+ stats_str[item], pval[0], -+ pval[1], pval[2], pval[3], pval[4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == PCP_TIMEOUT_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_timeout_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_LDSQ_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_ldsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_ENQL_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_enql_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } -+ } -+ -+ if (!md_info) { -+ buf[idx] = '\0'; -+ return; -+ } -+ -+ ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_SWITCHS; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "sw_rec[%d] = %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.sw_rec[i].switch_at, -+ md_info->kern_dump.sw_rec[i].is_success, -+ md_info->kern_dump.sw_rec[i].end_state, -+ md_info->kern_dump.sw_rec[i].switch_reason); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_EXCEP_ID; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "excep[%d] = %llu %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.excep_rec[i][0], -+ md_info->kern_dump.excep_rec[i][1], -+ md_info->kern_dump.excep_rec[i][2], -+ md_info->kern_dump.excep_rec[i][3], -+ md_info->kern_dump.excep_rec[i][4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n", -+ i, md_info->kern_dump.excep_idx[i]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ -+ buf[idx] = '\0'; -+} -+ -+enum cpu_type { -+ LITTLE, -+ BIG, -+ PARTIAL, -+ EXCLUSIVE, -+ EX_FREE, -+ INVALID -+}; -+ -+enum dsq_type { -+ GLOBAL_DSQ, -+ PCP_DSQ, -+ OTHER, -+ MAX_DSQ_TYPE, -+}; -+ -+static enum cpu_type cpu_cluster(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (get_hmbird_rq(rq)->exclusive) { -+ return EXCLUSIVE; -+ } else { -+ if (cpumask_test_cpu(cpu, iso_masks.little)) -+ return LITTLE; -+ else if (cpumask_test_cpu(cpu, iso_masks.big)) -+ return BIG; -+ else if (cpumask_test_cpu(cpu, iso_masks.partial)) -+ return PARTIAL; -+ else if (cpumask_test_cpu(cpu, iso_masks.exclusive)) -+ return EXCLUSIVE; -+ else if (cpumask_test_cpu(cpu, iso_masks.ex_free)) -+ return EX_FREE; -+ } -+ return INVALID; -+} -+ -+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq) -+{ -+ if (!dsq) -+ return OTHER; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= GDSQS_ID_BASE) && -+ ((dsq->id & 0xff) < MAX_GLOBAL_DSQS)) -+ return GLOBAL_DSQ; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) && -+ ((dsq->id & 0xff) < max_hmbird_dsq_internal_id)) -+ return PCP_DSQ; -+ -+ return OTHER; -+} -+ -+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq) -+{ -+ enum dsq_type type; -+ -+ type = get_dsq_type(dsq); -+ switch (type) { -+ case GLOBAL_DSQ: -+ case PCP_DSQ: -+ return (dsq->id & 0xff) - GDSQS_ID_BASE; -+ default: -+ return -1; -+ } -+ return -1; -+} -+ -+static void update_cpus_idle(bool set, struct cpumask *mask) -+{ -+ int cpu; -+ struct rq *rq; -+ -+ if (set) { -+ for_each_cpu(cpu, mask) { -+ rq = cpu_rq(cpu); -+ if (is_idle_task(rq->curr)) -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ } -+ } else -+ cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask); -+} -+ -+static void set_partial_status(bool enable, bool little, bool big) -+{ -+ WRITE_ONCE(partial_enable, enable); -+ WRITE_ONCE(l_need_rescue, little); -+ WRITE_ONCE(b_need_rescue, big); -+} -+ -+static bool is_little_need_rescue(void) -+{ -+ return READ_ONCE(l_need_rescue); -+} -+ -+static bool is_big_need_rescue(void) -+{ -+ return READ_ONCE(b_need_rescue); -+} -+ -+static bool is_partial_enabled(void) -+{ -+ return READ_ONCE(partial_enable); -+} -+ -+static bool is_partial_cpu(int cpu) -+{ -+ return cpumask_test_cpu(cpu, iso_masks.partial); -+} -+ -+static void set_iso_par_free(bool enable) -+{ -+ WRITE_ONCE(isolate_ctrl, enable); -+} -+ -+static bool is_iso_par_free(void) -+{ -+ return READ_ONCE(isolate_ctrl); -+} -+ -+static bool skip_update_idle(void) -+{ -+ int cpu = smp_processor_id(); -+ enum cpu_type type = cpu_cluster(cpu); -+ -+ if ((type == EXCLUSIVE && !is_iso_par_free()) || -+ /* partial enable may changed during idle, it doesn't matter. */ -+ (!is_partial_enabled() && type == PARTIAL)) -+ return true; -+ -+ return false; -+} -+ -+static void init_isolate_cpus(void) -+{ -+ WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -+ cpumask_set_cpu(0, iso_masks.big); -+ cpumask_set_cpu(1, iso_masks.big); -+ cpumask_set_cpu(2, iso_masks.big); -+ cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(4, iso_masks.big); -+ cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(6, iso_masks.partial); -+ cpumask_set_cpu(7, iso_masks.ex_free); -+} -+ -+extern spinlock_t css_set_lock; -+ -+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp) -+{ -+ int i; -+ struct cgroup *anc; -+ -+ spin_lock_irq(&css_set_lock); -+ for (i = 0; i < cgrp->level; i++) { -+ anc = cgrp->ancestors[i]; -+ if (anc->level != CREATE_DSQ_LEVEL_WITHIN) -+ continue; -+ cgroup_get(anc); -+ spin_unlock_irq(&css_set_lock); -+ return anc; -+ } -+ spin_unlock_irq(&css_set_lock); -+ hmbird_err(NO_CGROUP_L1, ":error cgroup = %s\n", cgrp->kn->name); -+ return NULL; -+} -+ -+#define PCP_IDX_BIT (1 << 31) -+ -+static bool is_pcp_rt(struct task_struct *p) -+{ -+ return rt_prio(p->prio) && (p->nr_cpus_allowed == 1); -+} -+ -+static bool is_pcp_idx(int idx) -+{ -+ return idx >= MAX_GLOBAL_DSQS; -+} -+ -+static bool is_critical_system_task(struct task_struct *p) -+{ -+ int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK; -+ -+ return (p->prio < (MAX_RT_PRIO >> 1) && -+ (sp_dl < SCHED_PROP_DEADLINE_LEVEL3)); -+} -+ -+#define ISOLATE_TASK_TOP_BIT (1 << 17) -+#define PIPELINE_TASK_TOP_BIT (1 << 9) -+static bool is_pipeline_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT); -+} -+ -+static bool is_isolate_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT); -+} -+ -+static bool is_critical_app_task_without_isolate(struct task_struct *p) -+{ -+ return task_is_top_task(p) && !is_isolate_task(p); -+} -+ -+static int find_idx_from_task(struct task_struct *p) -+{ -+ int idx, cpu; -+ int sp_dl; -+ struct task_group *tg = p->sched_task_group; -+ -+ if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) { -+ cpu = cpumask_any(p->cpus_ptr); -+ idx = cpu + MAX_GLOBAL_DSQS; -+ return idx; -+ } -+ -+ if (is_critical_system_task(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL0; -+ goto done; -+ } -+ -+ if (is_critical_app_task_without_isolate(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL1; -+ goto done; -+ } -+ -+ sp_dl = hmbird_get_dsq_id(p); -+ if (sp_dl) { -+ idx = sp_dl; -+ goto done; -+ } -+ -+ if (rt_prio(p->prio)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL3; -+ goto done; -+ } -+ -+ if (tg && tg->css.cgroup && tg->css.cgroup->kn) { -+ if (likely(tg->css.cgroup->kn->id >= 0 && -+ tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS)) -+ idx = cgroup_ids_table[tg->css.cgroup->kn->id]; -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ -+done: -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) { -+ hmbird_err(DSQ_ID_ERR, " : idx error, idx = %d-----\n", idx); -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } -+ return idx; -+} -+ -+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p) -+{ -+ int idx; -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq; -+ -+ if (!p) -+ return NULL; -+ -+ idx = find_idx_from_task(p); -+ if (is_pcp_idx(idx)) { -+ idx -= MAX_GLOBAL_DSQS; -+ dsq = &per_cpu(pcp_ldsq, idx); -+ get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq); -+ slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx); -+ } else { -+ dsq = &gdsqs[idx]; -+ get_hmbird_ts(p)->gdsq_idx = idx; -+ slim_stats_record(GDSQ_CNT, 0, idx, 0); -+ } -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ dsq->last_consume_at = jiffies; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ return dsq; -+} -+ -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq); -+ -+static void set_partial_rescue(bool p_state, bool l_over, bool b_over) -+{ -+ set_partial_status(p_state, l_over, b_over); -+ update_cpus_idle(p_state, iso_masks.partial); -+ hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled()); -+ hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue()); -+ hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue()); -+} -+ -+static void free_isocpu(bool enable) -+{ -+ set_iso_par_free(enable); -+ update_cpus_idle(enable, iso_masks.ex_free); -+ hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free()); -+} -+ -+inline u64 get_hmbird_cpu_util(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed) -+ return 0; -+ u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed); -+ u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size); -+ -+ do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT); -+ -+ return prev_runnable_sum_fixed; -+} -+ -+static inline unsigned int get_scaling_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->max; -+} -+ -+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static u64 get_cpus_max_util(struct cpumask *mask) -+{ -+ int cpu; -+ u64 max = 0; -+ u64 util, ratio; -+ unsigned long effective_cap = 0; -+ for_each_cpu(cpu, mask) { -+ if (slim_walt_ctrl) -+ slim_get_cpu_util(cpu, &util); -+ else -+ util = get_hmbird_cpu_util(cpu); -+ -+ /* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/ -+ if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu))) -+ effective_cap = arch_scale_cpu_capacity(cpu); -+ else -+ effective_cap = arch_scale_cpu_capacity(cpu) * -+ get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu); -+ -+ ratio = util * 100 / effective_cap; -+ hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util); -+ hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n", -+ cpu, (u64)arch_scale_cpu_capacity(cpu)); -+ hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n", -+ cpu, (u64)effective_cap); -+ -+ if (ratio > 100) -+ ratio = 100; -+ -+ if (ratio > max) -+ max = ratio; -+ } -+ return max; -+} -+ -+static bool cluster_need_rescue(struct cpumask *mask, int hres) -+{ -+ return get_cpus_max_util(mask) > hres; -+} -+ -+void partial_dynamic_ctrl(void) -+{ -+ u64 lmax = 0, bmax = 0; -+ bool l_over, l_under, b_over, b_under; -+ static bool last_l_over, last_b_over; -+ static unsigned long last_check; -+ -+ /* Check partial every jiffies. need lock? */ -+ if (time_before_eq(jiffies, READ_ONCE(last_check))) -+ return; -+ -+ WRITE_ONCE(last_check, jiffies); -+ -+ lmax = get_cpus_max_util(iso_masks.little); -+ l_over = lmax > parctrl_high_ratio_l; -+ l_under = lmax < parctrl_low_ratio_l; -+ -+ bmax = get_cpus_max_util(iso_masks.big); -+ b_over = bmax > parctrl_high_ratio; -+ b_under = bmax < parctrl_low_ratio; -+ -+ if (is_partial_enabled() && (l_over || b_over)) { -+ if (last_l_over != l_over || last_b_over != b_over) -+ set_partial_rescue(true, l_over, b_over); -+ } else if (!is_partial_enabled() && (l_over || b_over)) { -+ set_partial_rescue(true, l_over, b_over); -+ } else if (is_partial_enabled() && l_under && b_under) { -+ set_partial_rescue(false, false, false); -+ } -+ last_l_over = l_over; -+ last_b_over = b_over; -+ -+ if (is_partial_enabled()) { -+ bmax = get_cpus_max_util(iso_masks.partial); -+ if (!is_iso_par_free() && bmax > isoctrl_high_ratio) { -+ hmbird_info_trace("partial max = %llu\n", bmax); -+ free_isocpu(true); -+ } else if (is_iso_par_free() && bmax < isoctrl_low_ratio) -+ free_isocpu(false); -+ } else if (is_iso_par_free()) { -+ free_isocpu(false); -+ } -+} -+ -+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx) -+{ -+ hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx); -+} -+ -+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx) -+{ -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[idx])) { -+ slim_stats_record(GDSQ_CNT, 1, idx, 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ int i; -+ -+ for (i = 0; i < UX_COMPATIBLE_IDX; i++) { -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ slim_stats_record(GDSQ_CNT, 1, i, 0); -+ return 1; -+ } -+ } -+ return 0; -+} -+ -+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) { -+ slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ goto clear_timeout; -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) -+ goto clear_timeout; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ hmbird_info_trace( -+ "dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n", -+ dsq_id_to_internal(dsq), entity->task->comm, -+ jiffies, msecs_to_jiffies(deadline), entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1); -+ return; -+ -+clear_timeout: -+ hmbird_info_trace("dsq[%d] clear timeout\n", -+ dsq_id_to_internal(dsq)); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0); -+ dsq->is_timeout = false; -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq)); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime) -+{ -+ hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime); -+} -+ -+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any) -+{ -+ bool is_timeout; -+ int cpu = cpu_of(rq); -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu); -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ is_timeout = dsq->is_timeout; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ /* -+ * dsq->is_timeout may change here, let it be. -+ * it won't cause serious problems. -+ * the same for consume_dispatch_q later. -+ */ -+ if (!is_timeout && !any) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, dsq)) { -+ if (is_timeout) { -+ hmbird_info_trace("dsq[%d] consume a pcp timeout task\n", -+ dsq_id_to_internal(dsq)); -+ update_timeout_stats(rq, dsq, pcp_dsq_deadline); -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu); -+ } -+ slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu); -+ return 1; -+ } -+ /* -+ * No pcp task, clear quota. -+ */ -+ if (any) { -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+ } -+ return 0; -+} -+ -+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf) -+{ -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ } -+ return 0; -+} -+ -+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf, -+ int tmp, int cidx, int tidx) -+{ -+ unsigned long flags; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) { -+ if (tmp != cidx) { -+ spin_lock(&sinfo.lock); -+ sinfo.curr_idx[tidx] = tmp; -+ spin_unlock(&sinfo.lock); -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]); -+ slim_stats_record(SWITCH_IDX, 0, 0, 0); -+ } -+ slim_stats_record(GDSQ_CNT, 1, tmp, 0); -+ -+ raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags); -+ gdsqs[tmp].last_consume_at = jiffies; -+ raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags); -+ return 1; -+ } -+ return 0; -+ -+} -+ -+static int get_cidx(struct cluster_ctx *ctx) -+{ -+ int cidx; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ if (cidx < ctx->lower || cidx >= ctx->upper) { -+ sinfo.curr_idx[ctx->tidx] = ctx->lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]); -+ slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ } -+ spin_unlock(&sinfo.lock); -+ -+ return cidx; -+} -+ -+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = CLUSTER_SEPARATE_IDX; -+ if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) { -+ pr_debug(" %s iso_masks.little first is %d\n", -+ __func__, cpumask_first(iso_masks.little)); -+ ctx->upper = NON_PERIOD_END; -+ } -+ ctx->tidx = 0; -+ break; -+ case LITTLE: -+ ctx->lower = CLUSTER_SEPARATE_IDX; -+ ctx->upper = NON_PERIOD_END; -+ if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) { -+ pr_debug(" %s iso_masks.big first is %d", -+ __func__, cpumask_first(iso_masks.big)); -+ ctx->lower = NON_PERIOD_START; -+ } -+ ctx->tidx = 1; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ case LITTLE: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = NON_PERIOD_END; -+ ctx->tidx = 0; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ if (cluster_separate) { -+ return gen_cluster_ctx_separate(ctx, type); -+ } else { -+ return gen_cluster_ctx_common(ctx, type); -+ } -+} -+ -+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ int i; -+ bool is_timeout; -+ unsigned long flags; -+ struct cluster_ctx ctx; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ /* Third param means only consume timeout pcp dsq. */ -+ if (consume_pcp_dsq(rq, rf, false)) -+ return 1; -+ -+ for (i = ctx.lower; i < ctx.upper; i++) { -+ raw_spin_lock_irqsave(&gdsqs[i].lock, flags); -+ is_timeout = gdsqs[i].is_timeout; -+ raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags); -+ /* gdsqs[i].is_timeout may change here, let it be... */ -+ if (is_timeout) { -+ /* -+ * consume_dispatch_q will acquire dsq-lock, -+ * So cannot keep lock here, annoy enough. -+ * may rewrite a consume_dispatch_q_locked, TODO. -+ */ -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ hmbird_info_trace("dsq[%d] consume a timeout task\n", i); -+ slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0); -+ update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]); -+ return 1; -+ } -+ } -+ } -+ return 0; -+} -+ -+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ struct cluster_ctx ctx; -+ int cidx; -+ int tmp; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ cidx = get_cidx(&ctx); -+ tmp = cidx; -+ do { -+ if (check_pcp_dsq_round(rq, rf)) -+ return 1; -+ if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx)) -+ return 1; -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[tmp] = 0; -+ systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]); -+ tmp++; -+ if (tmp >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ tmp = ctx.lower; -+ } -+ spin_unlock(&sinfo.lock); -+ } while (tmp != cidx); -+ -+ return consume_pcp_dsq(rq, rf, true); -+} -+ -+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ enum cpu_type type = cpu_cluster(cpu_of(rq)); -+ bool period_allowed = !get_hmbird_rq(rq)->period_disallow; -+ bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow; -+ -+ switch (type) { -+ case EX_FREE: -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ break; -+ case EXCLUSIVE: -+ if (!is_iso_par_free()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ } -+ /* Only non-period task can run on isolate cpus */ -+ if (!READ_ONCE(iso_free_rescue)) { -+ if (is_big_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ } else { -+ /* Free run, can run on any task. */ -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ case PARTIAL: -+ if (!is_partial_enabled()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ return 0; -+ } -+ if (is_big_need_rescue()) { -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ -+ case BIG: -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ if (is_iso_par_free() || (is_little_need_rescue() && -+ !cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) { -+ hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++); -+ return 1; -+ } -+ } -+ break; -+ -+ case LITTLE: -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ -+ if (is_iso_par_free() || (is_big_need_rescue() && -+ !cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) { -+ hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++); -+ return 1; -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return 0; -+} -+ -+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf) -+{ -+ return consume_hmbird_global_dsq(rq, rf); -+} -+ -+ -+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time) -+{ -+ int idx; -+ -+ /* which dsq belongs to while task enqueue, task will consume its running time. */ -+ idx = get_hmbird_ts(p)->gdsq_idx; -+ /* Only non-period dsq share running time between each other. */ -+ if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id) -+ return; -+ -+ if (idx >= MAX_GLOBAL_DSQS) { -+ per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } else { -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[idx] += exec_time; -+ spin_unlock(&sinfo.lock); -+ systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]); -+ } -+} -+ -+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type) -+{ -+ int cidx; -+ struct cluster_ctx ctx; -+ int cpu = cpu_of(rq); -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ if (cidx < ctx.lower || cidx >= ctx.upper) { -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ slim_stats_record(ERR_IDX, ctx.tidx, 0, 0); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ } -+ -+ while (1) { -+ if (per_cpu(pcp_info, cpu).pcp_round) { -+ if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) { -+ hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n", -+ cpu, per_cpu(pcp_info, cpu).rtime); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } -+ } -+ if (sinfo.rtime[cidx] < dsq_quota[cidx]) -+ break; -+ -+ /* clear current dsq rtime */ -+ sinfo.rtime[cidx] = 0; -+ systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]); -+ -+ sinfo.curr_idx[ctx.tidx]++; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", -+ ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ } -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ slim_stats_record(SWITCH_IDX, 1, 0, 0); -+ } -+ spin_unlock(&sinfo.lock); -+} -+ -+ -+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p) -+{ -+ enum cpu_type type; -+ -+ if (!rq || !p) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ switch (type) { -+ case PARTIAL: -+ return; -+ case EXCLUSIVE: -+ return; -+ case EX_FREE: -+ return; -+ default: -+ break; -+ } -+ update_dsq_idx(rq, p, type); -+} -+ -+ -+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ int dsq_id; -+ -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo) || dsq->is_timeout) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ hmbird_deferred_err(SCAN_ENTITY_NULL, -+ " : entity is NULL, dsq->id = %llu\n", dsq->id); -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ dsq->is_timeout = true; -+ dsq_id = dsq_id_to_internal(dsq); -+ hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n", -+ dsq_id, entity->task->comm, jiffies, entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1); -+ raw_spin_unlock(&dsq->lock); -+ -+ return true; -+} -+ -+void scan_timeout(struct rq *rq) -+{ -+ int i; -+ int cpu = cpu_of(rq); -+ struct hmbird_dispatch_q *dsq; -+ static u64 last_scan_at; -+ static DEFINE_PER_CPU(u64, pcp_last_scan_at); -+ -+ if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu))) -+ return; -+ per_cpu(pcp_last_scan_at, cpu) = jiffies; -+ -+ dsq = &per_cpu(pcp_ldsq, cpu); -+ scan_dsq_timeout(rq, dsq, pcp_dsq_deadline); -+ -+ if (time_before_eq(jiffies, (unsigned long)last_scan_at)) -+ return; -+ last_scan_at = jiffies; -+ -+ for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) { -+ dsq = &gdsqs[i]; -+ scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]); -+ } -+} -+ -+/*******************************Initialize***********************************/ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id); -+static void init_dsq_at_boot(void) -+{ -+ int i, cpu; -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i)); -+ } -+ for_each_possible_cpu(cpu) -+ init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i + cpu)); -+ -+ max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu; -+ spin_lock_init(&sinfo.lock); -+} -+ -+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgrp_name_to_idx(struct cgroup *cgrp) -+{ -+ int idx; -+ -+ if (!cgrp) -+ return -1; -+ -+ if (!strcmp(cgrp->kn->name, "display") -+ || !strcmp(cgrp->kn->name, "multimedia")) -+ idx = 5; /* 8ms */ -+ else if (!strcmp(cgrp->kn->name, "top-app") -+ || !strcmp(cgrp->kn->name, "ss-top")) -+ idx = 6; /* 16ms */ -+ else if (!strcmp(cgrp->kn->name, "ssfg") -+ || !strcmp(cgrp->kn->name, "foreground")) -+ idx = 7; /* 32ms */ -+ else if (!strcmp(cgrp->kn->name, "bg") -+ || !strcmp(cgrp->kn->name, "log") -+ || !strcmp(cgrp->kn->name, "dex2oat") -+ || !strcmp(cgrp->kn->name, "background")) -+ idx = 9; /* 128ms */ -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */ -+ -+ return idx; -+} -+ -+static void init_root_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX); -+} -+ -+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) { -+ pr_err("%s idx err!\n", __func__); -+ return; -+ } -+ -+ if (cgroup_ids_table[cgrp->kn->id] == -1) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp)); -+} -+ -+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ struct cgroup *l1cgrp; -+ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ l1cgrp = cgroup_ancestor_l1(cgrp); -+ if (l1cgrp) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp)); -+ cgroup_put(l1cgrp); -+} -+ -+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg) -+{ -+ switch (cgrp->level) { -+ case 0: -+ init_root_tg(cgrp, tg); -+ break; -+ case 1: -+ init_level1_tg(cgrp, tg); -+ break; -+ default: -+ init_child_tg(cgrp, tg); -+ break; -+ } -+} -+ -+/**************************************************************************/ -+ -+struct hmbird_task_iter { -+ struct hmbird_entity cursor; -+ struct task_struct *locked; -+ struct rq *rq; -+ struct rq_flags rf; -+}; -+ -+/** -+ * hmbird_task_iter_init - Initialize a task iterator -+ * @iter: iterator to init -+ * -+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized, -+ * @iter must eventually be exited with hmbird_task_iter_exit(). -+ * -+ * hmbird_tasks_lock may be released between this and the first next() call or -+ * between any two next() calls. If hmbird_tasks_lock is released between two -+ * next() calls, the caller is responsible for ensuring that the task being -+ * iterated remains accessible either through RCU read lock or obtaining a -+ * reference count. -+ * -+ * All tasks which existed when the iteration started are guaranteed to be -+ * visited as long as they still exist. -+ */ -+static void hmbird_task_iter_init(struct hmbird_task_iter *iter) -+{ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR }; -+ list_add(&iter->cursor.tasks_node, &hmbird_tasks); -+ iter->locked = NULL; -+} -+ -+/** -+ * hmbird_task_iter_exit - Exit a task iterator -+ * @iter: iterator to exit -+ * -+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held. -+ * If the iterator holds a task's rq lock, that rq lock is released. See -+ * hmbird_task_iter_init() for details. -+ */ -+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ if (list_empty(cursor)) -+ return; -+ -+ list_del_init(cursor); -+} -+ -+/** -+ * hmbird_task_iter_next - Next task -+ * @iter: iterator to walk -+ * -+ * Visit the next task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ struct hmbird_entity *pos; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ list_for_each_entry(pos, cursor, tasks_node) { -+ if (&pos->tasks_node == &hmbird_tasks) -+ return NULL; -+ if (!(pos->flags & HMBIRD_TASK_CURSOR)) { -+ list_move(cursor, &pos->tasks_node); -+ return pos->task; -+ } -+ } -+ -+ /* can't happen, should always terminate at hmbird_tasks above */ -+ hmbird_deferred_err(ITER_RET_NULL, " : unreachable path in scx_task_iter_next\n"); -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered - Next non-idle task -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ while ((p = hmbird_task_iter_next(iter))) { -+ if (!is_idle_task(p)) -+ return p; -+ } -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init() -+ * for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ p = hmbird_task_iter_next_filtered(iter); -+ if (!p) -+ return NULL; -+ -+ iter->rq = task_rq_lock(p, &iter->rf); -+ iter->locked = p; -+ return p; -+} -+ -+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void) -+{ -+ return atomic_read(&hmbird_ops_enable_state_var); -+} -+ -+static enum hmbird_ops_enable_state -+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to) -+{ -+ return atomic_xchg(&hmbird_ops_enable_state_var, to); -+} -+ -+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to, -+ enum hmbird_ops_enable_state from) -+{ -+ int from_v = from; -+ -+ return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to); -+} -+ -+static bool hmbird_ops_disabling(void) -+{ -+ return false; -+} -+ -+/** -+ * wait_ops_state - Busy-wait the specified ops state to end -+ * @p: target task -+ * @opss: state to wait the end of -+ * -+ * Busy-wait for @p to transition out of @opss. This can only be used when the -+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also -+ * has load_acquire semantics to ensure that the caller can see the updates made -+ * in the enqueueing and dispatching paths. -+ */ -+static void wait_ops_state(struct task_struct *p, u64 opss) -+{ -+ do { -+ cpu_relax(); -+ } while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss); -+} -+ -+ -+static void update_curr_hmbird(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ u64 now = rq_clock_task(rq); -+ u64 delta_exec; -+ -+ if (time_before_eq64(now, curr->se.exec_start)) -+ return; -+ -+ delta_exec = now - curr->se.exec_start; -+ curr->se.exec_start = now; -+ update_runningtime(rq, curr, delta_exec); -+ curr->se.sum_exec_runtime += delta_exec; -+ account_group_exec_runtime(curr, delta_exec); -+ cgroup_account_cputime(curr, delta_exec); -+ -+ if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF) -+ get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec); -+ -+ trace_sched_stat_runtime(curr, delta_exec, 0); -+} -+ -+static bool hmbird_dsq_priq_less(struct rb_node *node_a, -+ const struct rb_node *node_b) -+{ -+ const struct hmbird_entity *a = -+ container_of(node_a, struct hmbird_entity, dsq_node.priq); -+ const struct hmbird_entity *b = -+ container_of(node_b, struct hmbird_entity, dsq_node.priq); -+ -+ return time_before64(a->dsq_vtime, b->dsq_vtime); -+} -+ -+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ bool is_local = dsq->id == HMBIRD_DSQ_LOCAL; -+ unsigned long flags; -+ -+ hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq || -+ !list_empty(&get_hmbird_ts(p)->dsq_node.fifo), -+ "task = %s, dsq->id = %llu\n", p->comm, dsq->id); -+ hmbird_cond_deferred_err(ENQ_EXIST2, -+ (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq), -+ "task = %s\n", p->comm); -+ -+ if (!is_local) { -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ); -+ hmbird_ops_error(": %s\n", -+ "attempting to dispatch to a destroyed dsq"); -+ /* fall back to the global dsq */ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ dsq = &hmbird_dsq_global; -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ } -+ } -+ -+ if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) { -+ get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ; -+ rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq, -+ hmbird_dsq_priq_less); -+ } else { -+ if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT)) -+ list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ else -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ } -+ dsq->nr++; -+ get_hmbird_ts(p)->dsq = dsq; -+ -+ /* -+ * We're transitioning out of QUEUEING or DISPATCHING. store_release to -+ * match waiters' load_acquire. -+ */ -+ if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS) -+ atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE); -+ -+ if (is_local) { -+ struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq); -+ struct rq *rq = hmbird->rq; -+ bool preempt = false; -+ -+ if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr && -+ rq->curr->sched_class == &hmbird_sched_class) { -+ get_hmbird_ts(rq->curr)->slice = 0; -+ preempt = true; -+ } -+ -+ if (preempt || sched_class_above(&hmbird_sched_class, -+ rq->curr->sched_class)) -+ resched_curr(rq); -+ } else { -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ } -+} -+ -+static void task_unlink_from_dsq(struct task_struct *p, -+ struct hmbird_dispatch_q *dsq) -+{ -+ if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) { -+ rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ; -+ } else { -+ list_del_init(&get_hmbird_ts(p)->dsq_node.fifo); -+ } -+} -+ -+static bool task_linked_on_dsq(struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+} -+ -+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq; -+ bool is_local = dsq == &hmbird_rq->local_dsq; -+ -+ if (!dsq) { -+ hmbird_cond_deferred_err(TASK_LINKED1, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ /* -+ * When dispatching directly from the BPF scheduler to a local -+ * DSQ, the task isn't associated with any DSQ but -+ * @get_hmbird_ts(p)->holding_cpu may be set under the protection of -+ * %HMBIRD_OPSS_DISPATCHING. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu >= 0) -+ get_hmbird_ts(p)->holding_cpu = -1; -+ return; -+ } -+ -+ if (!is_local) -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ /* -+ * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node -+ * can't change underneath us. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu < 0) { -+ /* @p must still be on @dsq, dequeue */ -+ hmbird_cond_deferred_err(TASK_UNLINKED, -+ !task_linked_on_dsq(p), "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ } else { -+ /* -+ * We're racing against dispatch_to_local_dsq() which already -+ * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the -+ * holding_cpu which tells dispatch_to_local_dsq() that it lost -+ * the race. -+ */ -+ hmbird_cond_deferred_err(TASK_LINKED2, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ get_hmbird_ts(p)->holding_cpu = -1; -+ } -+ get_hmbird_ts(p)->dsq = NULL; -+ -+ if (!is_local) -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+ -+static bool test_rq_online(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ return rq->online; -+#else -+ return true; -+#endif -+} -+ -+static void refill_task_slice(struct task_struct *p) -+{ -+ if (is_isolate_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO; -+ else if (is_pipeline_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2; -+ else -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+} -+ -+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -+ int sticky_cpu) -+{ -+ struct hmbird_dispatch_q *d; -+ s32 cpu = -1; -+ -+ hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), -+ "task = %s\n", p->comm); -+ -+ if (is_pcp_rt(p)) { -+ /* Enqueue percpu rt task to local directly. */ -+ /* Or cause a bug when disable dispatch. */ -+ if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ } -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (cpu >= 0) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ -+ if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ /* rq migration */ -+ if (sticky_cpu == cpu_of(rq)) -+ goto local_norefill; -+ /* -+ * If !rq->online, we already told the BPF scheduler that the CPU is -+ * offline. We're just trying to on/offline the CPU. Don't bother the -+ * BPF scheduler. -+ */ -+ if (unlikely(!test_rq_online(rq))) -+ goto local; -+ -+ /* see %HMBIRD_OPS_ENQ_LAST */ -+ if (enq_flags & HMBIRD_ENQ_LAST) -+ goto local; -+ -+ if (enq_flags & HMBIRD_ENQ_LOCAL) -+ goto local; -+ else -+ goto global; -+local: -+ /* -+ * For task-ordering, slice refill must be treated as implying the end -+ * of the current slice. Otherwise, the longer @p stays on the CPU, the -+ * higher priority it becomes from hmbird_prio_less()'s POV. -+ */ -+ refill_task_slice(p); -+local_norefill: -+ dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags); -+ slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq)); -+ return; -+ -+global: -+ d = find_dsq_from_task(p); -+ if (d) { -+ refill_task_slice(p); -+ dispatch_enqueue(d, p, enq_flags); -+ return; -+ } -+ slim_stats_record(GLOBAL_STAT, 0, 0, 0); -+ refill_task_slice(p); -+ dispatch_enqueue(&hmbird_dsq_global, p, enq_flags); -+} -+ -+static bool watchdog_task_watched(const struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->watchdog_node); -+} -+ -+static void watchdog_watch_task(struct rq *rq, struct task_struct *p) -+{ -+ lockdep_assert_rq_held(rq); -+ if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ get_hmbird_ts(p)->runnable_at = jiffies; -+ clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list); -+} -+ -+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) -+{ -+ list_del_init(&get_hmbird_ts(p)->watchdog_node); -+ if (reset_timeout) -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags) -+{ -+ int sticky_cpu = get_hmbird_ts(p)->sticky_cpu; -+ -+ enq_flags |= get_hmbird_rq(rq)->extra_enq_flags; -+ -+ if (sticky_cpu >= 0) -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ -+ /* -+ * Restoring a running task will be immediately followed by -+ * set_next_task_hmbird() which expects the task to not be on the BPF -+ * scheduler as tasks can only start running through local DSQs. Force -+ * direct-dispatch into the local DSQ by setting the sticky_cpu. -+ */ -+ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -+ sticky_cpu = cpu_of(rq); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_UNWATCHED, -+ !watchdog_task_watched(p), "task = %s\n", p->comm); -+ return; -+ } -+ -+ watchdog_watch_task(rq, p); -+ set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ get_hmbird_rq(rq)->nr_running++; -+ add_nr_running(rq, 1); -+ -+ do_enqueue_task(rq, p, enq_flags, sticky_cpu); -+} -+ -+static void ops_dequeue(struct task_struct *p, u64 deq_flags) -+{ -+ u64 opss; -+ -+ watchdog_unwatch_task(p, false); -+ -+ /* acquire ensures that we see the preceding updates on QUEUED */ -+ opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state); -+ -+ switch (opss & HMBIRD_OPSS_STATE_MASK) { -+ case HMBIRD_OPSS_NONE: -+ break; -+ case HMBIRD_OPSS_QUEUEING: -+ /* -+ * QUEUEING is started and finished while holding @p's rq lock. -+ * As we're holding the rq lock now, we shouldn't see QUEUEING. -+ */ -+ hmbird_deferred_err(DEQ_DEQING, " : unreachable path in %s\n", __func__); -+ break; -+ case HMBIRD_OPSS_QUEUED: -+ if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss, -+ HMBIRD_OPSS_NONE)) -+ break; -+ fallthrough; -+ case HMBIRD_OPSS_DISPATCHING: -+ /* -+ * If @p is being dispatched from the BPF scheduler to a DSQ, -+ * wait for the transfer to complete so that @p doesn't get -+ * added to its DSQ after dequeueing is complete. -+ * -+ * As we're waiting on DISPATCHING with the rq locked, the -+ * dispatching side shouldn't try to lock the rq while -+ * DISPATCHING is set. See dispatch_to_local_dsq(). -+ * -+ * DISPATCHING shouldn't have qseq set and control can reach -+ * here with NONE @opss from the above QUEUED case block. -+ * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss. -+ */ -+ wait_ops_state(p, HMBIRD_OPSS_DISPATCHING); -+ hmbird_cond_deferred_err(HMBIRD_OPN, -+ atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE, -+ "task = %s\n", p->comm); -+ break; -+ } -+} -+ -+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ -+ if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p), -+ "task = %s\n", p->comm); -+ return; -+ } -+ -+ ops_dequeue(p, deq_flags); -+ -+ if (slim_walt_ctrl) { -+ if (task_current(rq, p)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ if (deq_flags & HMBIRD_DEQ_SLEEP) -+ set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else -+ clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), -+ (unsigned long *)&get_hmbird_ts(p)->flags); -+ -+ clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm); -+ hmbird_rq->nr_running--; -+ sub_nr_running(rq, 1); -+ dispatch_dequeue(hmbird_rq, p); -+} -+ -+static void yield_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ get_hmbird_ts(p)->slice = 0; -+} -+ -+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to) -+{ -+ return false; -+} -+ -+#ifdef CONFIG_SMP -+/** -+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -+ * @rq: rq to move the task into, currently locked -+ * @p: task to move -+ * @enq_flags: %HMBIRD_ENQ_* -+ * -+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller -+ * must: -+ * -+ * 1. Start with exclusive access to @p either through its DSQ lock or -+ * %HMBIRD_OPSS_DISPATCHING flag. -+ * -+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id(). -+ * -+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't -+ * deadlock with dequeue. -+ * -+ * 4. Lock @rq and the task_rq from #3. -+ * -+ * 5. Call this function. -+ * -+ * Returns %true if @p was successfully moved. %false after racing dequeue and -+ * losing. -+ */ -+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ struct rq *task_rq; -+ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * If dequeue got to @p while we were trying to lock both rq's, it'd -+ * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have -+ * updated it to different values afterwards, as this operation can't be -+ * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become -+ * raw_smp_processor_id() again before we're done. Thus, we can tell -+ * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is -+ * still raw_smp_processor_id(). -+ * -+ * See dispatch_dequeue() for the counterpart. -+ */ -+ if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id())) -+ return false; -+ -+ /* @p->rq couldn't have changed if we're still the holding cpu */ -+ task_rq = task_rq(p); -+ lockdep_assert_rq_held(task_rq); -+ deactivate_task(task_rq, p, 0); -+ set_task_cpu(p, cpu_of(rq)); -+ get_hmbird_ts(p)->sticky_cpu = cpu_of(rq); -+ -+ /* -+ * We want to pass hmbird-specific enq_flags but activate_task() will -+ * truncate the upper 32 bit. As we own @rq, we can pass them through -+ * @get_hmbird_rq(rq)->extra_enq_flags instead. -+ */ -+ hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags, -+ "task = %s\n", p->comm); -+ get_hmbird_rq(rq)->extra_enq_flags = enq_flags; -+ activate_task(rq, p, 0); -+ get_hmbird_rq(rq)->extra_enq_flags = 0; -+ -+ return true; -+} -+ -+#endif /* CONFIG_SMP */ -+ -+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu) -+{ -+ int fitable = 1; -+ -+ return fitable; -+} -+ -+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq, -+ struct hmbird_dispatch_q *dsq) -+{ -+ bool dsq_misfit; -+ int cpu = cpu_of(rq); -+ u64 task_util = 0; -+ struct cluster_ctx ctx; -+ int dsq_int = dsq_id_to_internal(dsq); -+ -+ if (!cpumask_test_cpu(cpu, iso_masks.little)) -+ return false; -+ -+ gen_cluster_ctx(&ctx, BIG); -+ dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 && -+ dsq_int <= SCHED_PROP_DEADLINE_LEVEL4); -+#ifdef CLUSTER_SEPARATE -+ /* In rescue mode, little will consume big cluster's dsq.*/ -+ dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper); -+#endif -+ if (!dsq_misfit) -+ return false; -+ -+ if (p) { -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &task_util); -+ else -+ task_util = get_hmbird_ts(p)->demand_scaled; -+ } -+ -+ if (task_util <= misfit_ds) -+ return false; -+ -+ hmbird_info_trace(":task %s can't run on cpu%d, util = %llu\n", -+ p->comm, cpu, task_util); -+ return true; -+} -+ -+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq) -+{ -+ if (!task_fits_cpu_hmbird(p, cpu_of(rq))) -+ return false; -+ -+ if (check_misfit_task_on_little(p, rq, dsq)) -+ return false; -+ -+ return likely(test_rq_online(rq)); -+} -+ -+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return; -+ -+ if (add) -+ skipn[idx]++; -+ else -+ skipn[idx] = 0; -+} -+ -+static bool skip_too_much(struct hmbird_dispatch_q *dsq) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return false; -+ -+ if (skip_num[idx] > 3) { -+ skip_num[idx] = 0; -+ return true; -+ } -+ -+ return false; -+} -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rb_node *rb_node; -+ struct rq *task_rq; -+ unsigned long flags; -+ bool moved = false; -+ struct task_struct *may_fit = NULL; -+ int skip = 0; -+ -+retry: -+ if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -+ return false; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) { -+ set_skip_num(dsq, skip_num, (bool)may_fit); -+ goto this_rq; -+ } -+ if (skip_too_much(dsq)) -+ goto remote_rq; -+ if (!may_fit) -+ may_fit = p; -+ if (++skip <= 3) -+ continue; -+ /* -+ * If the recent 3 tasks not fit, use the first one. -+ * and clear the skip, because the first one is consumed. -+ */ -+ set_skip_num(dsq, skip_num, false); -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ /* No more task, use the first may fit task.*/ -+ if (may_fit) { -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ -+ for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) -+ goto this_rq; -+ goto remote_rq; -+ } -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ return false; -+ -+this_rq: -+ /* @dsq is locked and @p is on this rq */ -+ hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo); -+ dsq->nr--; -+ hmbird_rq->local_dsq.nr++; -+ get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ -+remote_rq: -+#ifdef CONFIG_SMP -+ if (cpu_same_cluster_stat(p, rq, task_rq)) -+ slim_stats_record(MOVE_RQ_CNT, 0, 0, 0); -+ else -+ slim_stats_record(MOVE_RQ_CNT, 1, 0, 0); -+ /* -+ * @dsq is locked and @p is on a remote rq. @p is currently protected by -+ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -+ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -+ * rq lock or fail, do a little dancing from our side. See -+ * move_task_to_local_dsq(). -+ */ -+ hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id(); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ rq_unpin_lock(rq, rf); -+ double_lock_balance(rq, task_rq); -+ rq_repin_lock(rq, rf); -+ -+ moved = move_task_to_local_dsq(rq, p, 0); -+ -+ double_unlock_balance(rq, task_rq); -+#endif /* CONFIG_SMP */ -+ if (likely(moved)) { -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ } -+ may_fit = NULL; -+ goto retry; -+} -+ -+ -+static int balance_one(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf, bool local) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class; -+ -+ if (!hmbird_rq) -+ return 1; -+ -+ lockdep_assert_rq_held(rq); -+ -+ if (static_branch_unlikely(&hmbird_ops_cpu_preempt) && -+ unlikely(get_hmbird_rq(rq)->cpu_released)) { -+ /* -+ * If the previous sched_class for the current CPU was not HMBIRD, -+ * notify the BPF scheduler that it again has control of the -+ * core. This callback complements ->cpu_release(), which is -+ * emitted in hmbird_notify_pick_next_task(). -+ */ -+ get_hmbird_rq(rq)->cpu_released = false; -+ } -+ -+ if (prev_on_hmbird) -+ update_curr_hmbird(rq); -+ /* if there already are tasks to run, nothing to do */ -+ if (hmbird_rq->local_dsq.nr) -+ return 1; -+ -+ if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) { -+ slim_stats_record(GLOBAL_STAT, 1, 0, 0); -+ return 1; -+ } -+ -+ if (consume_dispatch_global(rq, rf)) -+ return 1; -+ -+ return 0; -+} -+ -+static int balance_hmbird(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf) -+{ -+ return balance_one(rq, prev, rf, true); -+} -+ -+/* -+ * output task util to systrace, only for debug mode. -+ * we can not output too many logs to systrace buffer even in debug mode -+ * only output debug-info while it exceed misfit_ds. -+ */ -+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p) -+{ -+ static DEFINE_PER_CPU(int, is_last_exceed); -+ int cpu = cpu_of(rq); -+ u64 util = 0; -+ -+ if (likely(!debug_enabled())) -+ return; -+ -+ if (!p) -+ return; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ util = uclamp_rq_util_with(rq, util, p); -+ -+ if (util >= misfit_ds) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util); -+ per_cpu(is_last_exceed, cpu) = true; -+ } else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0); -+ per_cpu(is_last_exceed, cpu) = false; -+ } else { -+ } -+} -+ -+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first) -+{ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ /* -+ * Core-sched might decide to execute @p before it is -+ * dispatched. Call ops_dequeue() to notify the BPF scheduler. -+ */ -+ ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC); -+ dispatch_dequeue(get_hmbird_rq(rq), p); -+ } -+ -+ p->se.exec_start = rq_clock_task(rq); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK); -+ } -+ -+ watchdog_unwatch_task(p, true); -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx); -+ systrace_output_cpu_ds(rq, p); -+ /* -+ * @p is getting newly scheduled or got kicked after someone updated its -+ * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird(). -+ */ -+ if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) != -+ (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) { -+ if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) -+ get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK; -+ else -+ get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK; -+ -+ sched_update_tick_dependency(rq); -+ } -+ -+ p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime; -+} -+ -+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p) -+{ -+ update_curr_hmbird(rq); -+ -+ update_dispatch_dsq_info(rq, p); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ watchdog_watch_task(rq, p); -+ -+ if (is_pipeline_task(p)) { -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1); -+ return; -+ } -+ -+ /* -+ * If we're in the pick_next_task path, balance_hmbird() should -+ * have already populated the local DSQ if there are any other -+ * available tasks. If empty, tell ops.enqueue() that @p is the -+ * only one available for this cpu. ops.enqueue() should put it -+ * on the local DSQ so that the subsequent pick_next_task_hmbird() -+ * can find the task unless it wants to trigger a separate -+ * follow-up scheduling event. -+ */ -+ if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1); -+ else -+ do_enqueue_task(rq, p, 0, -1); -+ } -+} -+ -+static struct task_struct *first_local_task(struct rq *rq) -+{ -+ struct rb_node *rb_node; -+ struct hmbird_entity *entity; -+ -+ if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) { -+ entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo, -+ struct hmbird_entity, dsq_node.fifo); -+ return entity->task; -+ } -+ -+ rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq); -+ if (rb_node) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ return entity->task; -+ } -+ return NULL; -+} -+ -+static struct task_struct *pick_next_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p; -+ -+ p = first_local_task(rq); -+ if (!p) -+ return NULL; -+ -+ if (unlikely(!get_hmbird_ts(p)->slice)) { -+ if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice) -+ hmbird_warned_zero_slice = true; -+ -+ refill_task_slice(p); -+ } -+ -+ set_next_task_hmbird(rq, p, true); -+ -+ return p; -+} -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task, -+ const struct sched_class *active) -+{ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * The callback is conceptually meant to convey that the CPU is no -+ * longer under the control of HMBIRD. Therefore, don't invoke the -+ * callback if the CPU is staying on HMBIRD, or going idle (in which -+ * case the HMBIRD scheduler has actively decided not to schedule any -+ * tasks on the CPU). -+ */ -+ if (likely(active >= &hmbird_sched_class)) -+ return; -+ -+ /* -+ * At this point we know that HMBIRD was preempted by a higher priority -+ * sched_class, so invoke the ->cpu_release() callback if we have not -+ * done so already. We only send the callback once between HMBIRD being -+ * preempted, and it regaining control of the CPU. -+ * -+ * ->cpu_release() complements ->cpu_acquire(), which is emitted the -+ * next time that balance_hmbird() is invoked. -+ */ -+ if (!get_hmbird_rq(rq)->cpu_released) -+ get_hmbird_rq(rq)->cpu_released = true; -+} -+ -+#ifdef CONFIG_SMP -+ -+static bool test_and_clear_cpu_idle(int cpu) -+{ -+ if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -+ if (cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ return true; -+ } else { -+ return false; -+ } -+} -+ -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) -+{ -+ int cpu; -+ -+ do { -+ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -+ if (cpu >= nr_cpu_ids) -+ return -EBUSY; -+ } while (!test_and_clear_cpu_idle(cpu)); -+ -+ return cpu; -+} -+ -+ -+static bool prev_cpu_misfit(int prev) -+{ -+ if (!is_partial_enabled() && is_partial_cpu(prev)) -+ return true; -+ -+ return false; -+} -+ -+static int heavy_rt_placement(struct task_struct *p, int prev) -+{ -+ struct cpumask tmp = {.bits = {0}}; -+ int cpu; -+ u64 util = 0; -+ -+ if (!rt_prio(p->prio)) -+ return -EFAULT; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ -+ if (util < misfit_ds) -+ return -EFAULT; -+ -+ if (is_partial_enabled()) -+ cpumask_or(&tmp, iso_masks.big, iso_masks.partial); -+ else -+ cpumask_copy(&tmp, iso_masks.big); -+ -+ cpu = hmbird_pick_idle_cpu(&tmp); -+ if (cpu >= 0) -+ return cpu; -+ -+ if (cpumask_test_cpu(prev, iso_masks.big) || -+ (is_partial_enabled() && is_partial_cpu(prev))) -+ return prev; -+ -+ return cpumask_first(&tmp); -+} -+ -+static int spec_task_before_pick_idle(struct task_struct *p, int prev) -+{ -+ int cpu; -+ -+ cpu = heavy_rt_placement(p, prev); -+ if (cpu >= 0) -+ return cpu; -+ return -EFAULT; -+} -+ -+static int cpumask_distribute_next(struct cpumask *mask, int *prev) -+{ -+ int p = *prev, n; -+ -+ n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1); -+ if (n < nr_cpu_ids) -+ WRITE_ONCE(*prev, n); -+ return n; -+} -+ -+static int repick_fallback_cpu(void) -+{ -+ /* -+ * partial cpu follow big cluster's Scheduling policy, -+ * simply return first bit cpu. -+ */ -+ return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev); -+} -+ -+/* -+ * Must return a valid cpu num, as this task's cpu. -+ */ -+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu, -+ struct cpumask *mask, int *prev_mask) -+{ -+ int cpu; -+ -+ cpu = hmbird_pick_idle_cpu(mask); -+ if (cpu >= 0) -+ return cpu; -+ return cpumask_distribute_next(mask, prev_mask); -+} -+ -+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type) -+{ -+ int idx; -+ struct cluster_ctx ctx; -+ -+ idx = find_idx_from_task(p); -+ if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS) -+ return false; -+ -+ gen_cluster_ctx(&ctx, type); -+ if (idx >= ctx.lower && idx < ctx.upper) -+ return true; -+ return false; -+} -+ -+static bool is_valid_cpu(int cpu) -+{ -+ return (cpu >= 0) && (cpu < nr_cpu_ids); -+} -+ -+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) -+{ -+ s32 cpu = -1; -+ struct cpumask mask = {.bits = {0}}; -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (is_valid_cpu(cpu)) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ partial_dynamic_ctrl(); -+ -+ if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ iso_masks.big, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ if (p->nr_cpus_allowed == 1) { -+ cpu = cpumask_any(p->cpus_ptr); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* For non-period global dsq, not contain pcp task. */ -+ if (task_only_blongs_to_cluster(p, LITTLE)) { -+ cpumask_copy(&mask, iso_masks.little); -+ if (unlikely(l_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &little_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ if (task_only_blongs_to_cluster(p, BIG)) { -+ cpumask_copy(&mask, iso_masks.big); -+ if (unlikely(b_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ cpu = spec_task_before_pick_idle(p, prev_cpu); -+ if (is_valid_cpu(cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* if the previous CPU is idle, dispatch directly to it */ -+ if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+ } -+ -+ cpu = hmbird_pick_idle_cpu(cpu_possible_mask); -+ if (is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ if (prev_cpu_misfit(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ cpu = repick_fallback_cpu(); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+} -+ -+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags) -+{ -+ return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags); -+} -+ -+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx) -+{ -+ set_cpus_allowed_common(p, ctx); -+} -+ -+static void reset_idle_masks(void) -+{ -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little); -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big); -+ if (is_partial_enabled()) -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial); -+ hmbird_has_idle_cpus = true; -+} -+ -+void __hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ int cpu = cpu_of(rq); -+ struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -+ -+ if (skip_update_idle()) -+ return; -+ -+ if (idle) { -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ if (!hmbird_has_idle_cpus) -+ hmbird_has_idle_cpus = true; -+ -+ /* -+ * idle_masks.smt handling is racy but that's fine as it's only -+ * for optimization and self-correcting. -+ */ -+ for_each_cpu(cpu, sib_mask) { -+ if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -+ return; -+ } -+ cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -+ } else { -+ cpumask_clear_cpu(cpu, idle_masks.cpu); -+ if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ -+ cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -+ } -+} -+ -+#else /* !CONFIG_SMP */ -+ -+static bool test_and_clear_cpu_idle(int cpu) { return false; } -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } -+static void reset_idle_masks(void) {} -+ -+#endif /* CONFIG_SMP */ -+ -+static bool check_rq_for_timeouts(struct rq *rq) -+{ -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rq_flags rf; -+ -+ rq_lock_irqsave(rq, &rf); -+ list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) { -+ unsigned long last_runnable; -+ -+ p = entity->task; -+ last_runnable = get_hmbird_ts(p)->runnable_at; -+ -+ if (unlikely(time_after(jiffies, -+ last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -+ -+ rq_unlock_irqrestore(rq, &rf); -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb", -+ p->comm, p->pid, -+ dur_ms / 1000, dur_ms % 1000, -+ get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0, -+ cpumask_pr_args(&p->cpus_mask)); -+ return 1; -+ } -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ return 0; -+} -+ -+static void hmbird_watchdog_workfn(struct work_struct *work) -+{ -+ int cpu; -+ bool timeout; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ -+ for_each_online_cpu(cpu) { -+ timeout = check_rq_for_timeouts(cpu_rq(cpu)); -+ if (unlikely(timeout)) -+ break; -+ -+ cond_resched(); -+ } -+ if (!timeout) -+ queue_delayed_work(system_unbound_wq, to_delayed_work(work), -+ hmbird_watchdog_timeout / 2); -+} -+ -+static void set_pcp_round(struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) { -+ per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round); -+ per_cpu(pcp_info, cpu).pcp_round = true; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+} -+ -+ -+/* -+ * Just for debug: output hmbird on/off state per 10s. -+ */ -+#define OUTPUT_INTVAL (msecs_to_jiffies(10 * 1000)) -+static void inform_hmbird_onoff_from_systrace(void) -+{ -+ static unsigned long __read_mostly next_print; -+ -+ if (time_before(jiffies, READ_ONCE(next_print))) -+ return; -+ -+ WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL); -+ hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio); -+} -+ -+void hmbird_notify_sched_tick(void) -+{ -+ unsigned long last_check; -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ hmbird_scheduler_tick(); -+ -+ if (!hmbird_enabled()) -+ return; -+ -+ last_check = hmbird_watchdog_timestamp; -+ if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -+ -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "watchdog failed to check in for %u.%03us", -+ dur_ms / 1000, dur_ms % 1000); -+ } -+ scan_timeout(rq); -+} -+ -+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued) -+{ -+ update_curr_hmbird(rq); -+ -+ set_pcp_round(rq); -+ -+ if (slim_walt_ctrl) -+ hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE); -+ /* -+ * While disabling, always resched and refresh core-sched timestamp as -+ * we can't trust the slice management or ops.core_sched_before(). -+ */ -+ if (hmbird_ops_disabling()) -+ get_hmbird_ts(curr)->slice = 0; -+ -+ if (!get_hmbird_ts(curr)->slice) -+ resched_curr(rq); -+ -+ inform_hmbird_onoff_from_systrace(); -+} -+ -+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg) -+{ -+ hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ get_hmbird_ts(p)->disallow = false; -+ -+ hmbird_sched_init_task(p); -+ -+ set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ return 0; -+} -+ -+static void hmbird_ops_enable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void hmbird_ops_disable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void set_task_hmbird_weight(struct task_struct *p) -+{ -+ u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -+ -+ get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight); -+} -+ -+/** -+ * refresh_hmbird_weight - Refresh a task's hmbird weight -+ * @p: task to refresh hmbird weight for -+ * -+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to -+ * enable easy access from the BPF scheduler. To keep it synchronized with the -+ * current task priority, this function should be called when a new task is -+ * created, priority is changed for a task on hmbird, and a task is switched -+ * to hmbird from other classes. -+ */ -+static void refresh_hmbird_weight(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ set_task_hmbird_weight(p); -+} -+ -+int hmbird_pre_fork(struct task_struct *p) -+{ -+ int ret = 0; -+ -+ p->android_oem_data1[HMBIRD_TS_IDX] = -+ (u64)(kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL)); -+ if (!get_hmbird_ts(p)) { -+ ret = -1; -+ goto lock; -+ } -+ -+ get_hmbird_ts(p)->dsq = NULL; -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node); -+ get_hmbird_ts(p)->flags = 0; -+ get_hmbird_ts(p)->weight = 0; -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ get_hmbird_ts(p)->holding_cpu = -1; -+ get_hmbird_ts(p)->kf_mask = 0; -+ atomic64_set(&get_hmbird_ts(p)->ops_state, 0); -+ get_hmbird_ts(p)->runnable_at = INITIAL_JIFFIES; -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+ get_hmbird_ts(p)->task = p; -+ hmbird_set_sched_prop(p, 0); -+ -+ get_hmbird_ts(p)->critical_affinity_cpu = -1; -+ get_hmbird_ts(p)->sched_class = &hmbird_sched_class; -+ -+ /* -+ * BPF scheduler enable/disable paths want to be able to iterate and -+ * update all tasks which can become complex when racing forks. As -+ * enable/disable are very cold paths, let's use a percpu_rwsem to -+ * exclude forks. -+ */ -+lock: -+ percpu_down_read(&hmbird_fork_rwsem); -+ -+ return ret; -+} -+ -+int hmbird_fork(struct task_struct *p) -+{ -+ percpu_rwsem_assert_held(&hmbird_fork_rwsem); -+ -+ if (hmbird_enabled()) -+ return hmbird_ops_prepare_task(p, task_group(p)); -+ else -+ return 0; -+} -+ -+void hmbird_post_fork(struct task_struct *p) -+{ -+ if (hmbird_enabled()) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ /* -+ * Set the weight manually before calling ops.enable() so that -+ * the scheduler doesn't see a stale value if they inspect the -+ * task struct. We'll invoke ops.set_weight() afterwards, as it -+ * would be odd to receive a callback on the task before we -+ * tell the scheduler that it's been fully enabled. -+ */ -+ set_task_hmbird_weight(p); -+ hmbird_ops_enable_task(p); -+ refresh_hmbird_weight(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ -+ spin_lock_irq(&hmbird_tasks_lock); -+ list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_cancel_fork(struct task_struct *p) -+{ -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ if (hmbird_enabled()) -+ hmbird_ops_disable_task(p); -+ -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_free(struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+ list_del_init(&get_hmbird_ts(p)->tasks_node); -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+ -+ /* -+ * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED -> -+ * ENABLED transitions can't race us. Disable ops for @p. -+ */ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) || -+ test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ hmbird_ops_disable_task(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+} -+ -+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio) -+{ -+} -+ -+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type) -+{ -+ return (prop >> TOP_TASK_SHIFT) & (1 << type); -+} -+ -+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p) -+{ -+ uint32_t prop = get_top_task_prop(p); -+ -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE)) -+ return HMBIRD_TASK_PROP_PIPELINE; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) || -+ !task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) { -+ return HMBIRD_TASK_PROP_COMMON; -+ } -+ return HMBIRD_TASK_PROP_DEBUG_OR_LOG; -+} -+ -+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b) -+{ -+ int type_a = hmbird_get_task_type(a); -+ int type_b = hmbird_get_task_type(b); -+ -+ return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b]; -+} -+ -+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ enum cpu_type type; -+ int sp_dl; -+ struct task_struct *curr = NULL; -+ -+ switch (hmbird_preempt_policy) { -+ case HMBIRD_PREEMPT_POLICY_PRIO_BASED: -+ curr = rq->curr; -+ if (curr && hmbird_prio_higher(p, curr)) -+ goto preempt; -+ break; -+ default: -+ break; -+ } -+ if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) || -+ (is_critical_system_task(p) && !is_critical_system_task(rq->curr))) -+ goto preempt; -+ -+ sp_dl = find_idx_from_task(p); -+ if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled())) -+ return; -+ -+ if (rq->curr->prio > p->prio) -+ goto preempt; -+ -+ return; -+ -+preempt: -+ resched_curr(rq); -+} -+ -+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {} -+ -+int hmbird_check_setscheduler(struct task_struct *p, int policy) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ /* if disallow, reject transitioning into HMBIRD */ -+ if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) && -+ p->policy != policy && policy == SCHED_HMBIRD) -+ return -EACCES; -+ -+ return 0; -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool hmbird_can_stop_tick(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ if (hmbird_ops_disabling()) -+ return false; -+ -+ if (p->sched_class != &hmbird_sched_class) -+ return true; -+ -+ return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK; -+} -+#endif -+ -+int hmbird_tg_online(struct task_group *tg) -+{ -+ struct cgroup *cgrp; -+ -+ if (!tg) -+ return 0; -+ cgrp = tg->css.cgroup; -+ if (!cgrp || !(cgrp->kn)) -+ return 0; -+ update_cgroup_ids_table(cgrp->kn->id, -1); -+ return 0; -+} -+ -+/* -+ * Omitted operations: -+ * -+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -+ * task isn't tied to the CPU at that point. Preemption is implemented by -+ * resetting the victim task's slice to 0 and triggering reschedule on the -+ * target CPU. -+ * -+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -+ * -+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -+ * their current sched_class. Call them directly from sched core instead. -+ * -+ * - task_woken, switched_from: Unnecessary. -+ */ -+DEFINE_SCHED_CLASS(hmbird) = { -+ .enqueue_task = enqueue_task_hmbird, -+ .dequeue_task = dequeue_task_hmbird, -+ .yield_task = yield_task_hmbird, -+ .yield_to_task = yield_to_task_hmbird, -+ -+ .check_preempt_curr = check_preempt_curr_hmbird, -+ -+ .pick_next_task = pick_next_task_hmbird, -+ -+ .put_prev_task = put_prev_task_hmbird, -+ .set_next_task = set_next_task_hmbird, -+ -+#ifdef CONFIG_SMP -+ .balance = balance_hmbird, -+ .select_task_rq = select_task_rq_hmbird, -+ .set_cpus_allowed = set_cpus_allowed_hmbird, -+#endif -+ .task_tick = task_tick_hmbird, -+ -+ .switched_to = switched_to_hmbird, -+ .prio_changed = prio_changed_hmbird, -+ -+ .update_curr = update_curr_hmbird, -+ -+#ifdef CONFIG_UCLAMP_TASK -+ .uclamp_enabled = 0, -+#endif -+}; -+ -+/* -+ * Must with rq lock held. -+ */ -+bool task_is_hmbird(struct task_struct *p) -+{ -+ return p->sched_class == &hmbird_sched_class; -+} -+ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id) -+{ -+ memset(dsq, 0, sizeof(*dsq)); -+ -+ raw_spin_lock_init(&dsq->lock); -+ INIT_LIST_HEAD(&dsq->fifo); -+ dsq->id = dsq_id; -+} -+ -+static int hmbird_cgroup_init(void) -+{ -+ struct cgroup_subsys_state *css; -+ -+ css_for_each_descendant_pre(css, &root_task_group.css) { -+ struct task_group *tg = css_tg(css); -+ -+ cgrp_dsq_idx_init(css->cgroup, tg); -+ } -+ return 0; -+} -+ -+/* -+ * Used by sched_fork() and __setscheduler_prio() to pick the matching -+ * sched_class. dl/rt are already handled. -+ */ -+bool task_on_hmbird(struct task_struct *p) -+{ -+ return hmbird_enabled(); -+} -+ -+static void __setscheduler_prio(struct task_struct *p, int prio) -+{ -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) { -+ p->prio = prio; -+ return; -+ } else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (on_hmbird && rt_prio(prio)) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ -+ p->prio = prio; -+} -+ -+/* -+ * Heartbeat, avoid humbird keep running while APP already exit. -+ * Check whether APP send alive-signal periodly. -+ */ -+#define HEARTBEAT_TIMEOUT (msecs_to_jiffies(2500)) -+#define HEARTBEAT_CHECK_INTERVAL (msecs_to_jiffies(1000)) -+static struct timer_list hb_timer; -+static unsigned long next_hb; -+ -+void hb_timer_handler(struct timer_list *timer) -+{ -+ if (!heartbeat_enable) -+ goto refill; -+ -+ pr_err(": enter timer.\n"); -+ if (heartbeat) { -+ heartbeat = 0; -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ } -+ -+ /* can't detect heartbeat, disable ext. */ -+ if (time_after(jiffies, READ_ONCE(next_hb))) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT, -+ "can't detect heartbeat, disable ext\n"); -+ } -+refill: -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_start(void) -+{ -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_init(void) -+{ -+ timer_setup(&hb_timer, hb_timer_handler, 0); -+} -+ -+static void hb_timer_exit(void) -+{ -+ del_timer(&hb_timer); -+} -+ -+static bool check_and_disable_cpuhp(void) -+{ -+ struct rq *rq; -+ struct rq_flags rf; -+ int cpu; -+ -+ cpu_hotplug_disable(); -+ cpus_read_lock(); -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ rq_lock_irqsave(rq, &rf); -+ if (!rq->online) { -+ rq_unlock_irqrestore(rq, &rf); -+ goto err_offline; -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ cpus_read_unlock(); -+ return true; -+ -+err_offline: -+ cpus_read_unlock(); -+ cpu_hotplug_enable(); -+ return false; -+} -+ -+static void reenable_cpuhp(void) -+{ -+ cpu_hotplug_enable(); -+} -+ -+static void scheduler_switch_done(bool final_state) -+{ -+ /* set scx_enable again, switch may fail. */ -+ scx_enable = final_state; -+ /* reset heartbeat*/ -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ -+ if (final_state) -+ hb_timer_start(); -+ else -+ hb_timer_exit(); -+} -+ -+/** -+ * ss : curr switch state -+ * finish : success or fail -+ * enable : curr operation is diabling or enabling? -+ * fail_reason : string output when failed, empty string when success. -+ */ -+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason) -+{ -+ char *s1 = finish ? "finished" : "failed"; -+ char *s2 = enable ? "enabled" : "disabled"; -+ -+ hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss); -+ if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) { -+ sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type)); -+ hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n", -+ s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason); -+ } -+ curr_ss = ss; -+} -+ -+bool get_hmbird_ops_enabled(void) -+{ -+ return atomic_read(&__hmbird_ops_enabled); -+} -+ -+bool get_non_hmbird_task(void) -+{ -+ return atomic_read(&non_hmbird_task); -+} -+ -+static void hmbird_ops_disable_workfn(struct kthread_work *work) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int cpu; -+ -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, ""); -+ cancel_delayed_work_sync(&hmbird_watchdog_work); -+ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) { -+ case HMBIRD_OPS_DISABLED: -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled"); -+ scheduler_switch_done(false); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return; -+ case HMBIRD_OPS_PREPPING: -+ fallthrough; -+ case HMBIRD_OPS_DISABLING: -+ /* shouldn't happen but handle it like ENABLING if it does */ -+ WARN_ONCE(true, "hmbird: duplicate disabling instance?"); -+ fallthrough; -+ case HMBIRD_OPS_ENABLING: -+ case HMBIRD_OPS_ENABLED: -+ break; -+ } -+ -+ /* kick all CPUs to restore ticks */ -+ for_each_possible_cpu(cpu) -+ resched_cpu(cpu); -+ -+ /* avoid racing against fork and cgroup changes */ -+ cpus_read_lock(); -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, ""); -+ spin_lock_irq(&hmbird_tasks_lock); -+ atomic_set(&__hmbird_ops_enabled, false); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ bool alive = READ_ONCE(p->__state) != TASK_DEAD; -+ -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ get_hmbird_ts(p)->slice = min_t(u64, -+ get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL); -+ -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ if (alive) -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ -+ hmbird_ops_disable_task(p); -+ } -+ hmbird_task_iter_exit(&sti); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, ""); -+ -+ atomic_set(&non_hmbird_task, true); -+ /* no task is on hmbird, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt); -+ synchronize_rcu(); -+ -+ percpu_up_write(&hmbird_fork_rwsem); -+ cpus_read_unlock(); -+ -+ if (slim_walt_ctrl) -+ slim_walt_enable(false); -+ -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 1, 0, ""); -+ scheduler_switch_done(false); -+ reenable_cpuhp(); -+} -+ -+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn); -+ -+static void schedule_hmbird_ops_disable_work(void) -+{ -+ struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper); -+ -+ /* -+ * We may be called spuriously before the first bpf_hmbird_reg(). If -+ * hmbird_ops_helper isn't set up yet, there's nothing to do. -+ */ -+ if (helper) -+ kthread_queue_work(helper, &hmbird_ops_disable_work); -+} -+ -+static void hmbird_ops_disable(void) -+{ -+ schedule_hmbird_ops_disable_work(); -+} -+ -+static void hmbird_err_exit_workfn(struct work_struct *work) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ hmbird_ctrl(false); -+ -+ for_each_present_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy) -+ continue; -+ if (cpu != policy->cpu) -+ goto put; -+ down_write(&policy->rwsem); -+ WARN_ON(store_scaling_governor(policy, -+ saved_gov[cpu], strlen(saved_gov[cpu])) <= 0); -+ up_write(&policy->rwsem); -+ hmbird_info_trace(":restore origin gov : %s\n", saved_gov[cpu]); -+put: -+ cpufreq_cpu_put(policy); -+ } -+ memset((char *)saved_gov, 0, sizeof(saved_gov)); -+} -+ -+void hmbird_ops_exit(void) -+{ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); -+} -+ -+static struct kthread_worker *hmbird_create_rt_helper(const char *name) -+{ -+ struct kthread_worker *helper; -+ -+ helper = kthread_create_worker(0, name); -+ if (helper) -+ sched_set_fifo(helper->task); -+ return helper; -+} -+ -+static inline void set_audio_thread_sched_prop(struct task_struct *p) -+{ -+ struct cgroup_subsys_state *css; -+ -+ if (likely(p->prio >= MAX_RT_PRIO)) -+ return; -+ -+ rcu_read_lock(); -+ css = task_css(p, cpuset_cgrp_id); -+ if (!css) { -+ rcu_read_unlock(); -+ return; -+ } -+ rcu_read_unlock(); -+ -+ if (!strcmp(css->cgroup->kn->name, "audio-app")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+} -+ -+static int hmbird_ops_enable(void *unused) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int ret; -+ int tcnt = 0; -+ unsigned long long start = 0; -+ -+ if (!check_and_disable_cpuhp()) { -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline"); -+ return -EBUSY; -+ } -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, ""); -+ mutex_lock(&hmbird_ops_enable_mutex); -+ -+ if (!hmbird_ops_helper) { -+ WRITE_ONCE(hmbird_ops_helper, -+ hmbird_create_rt_helper("hmbird_ops_helper")); -+ if (!hmbird_ops_helper) { -+ ret = -ENOMEM; -+ goto err_unlock; -+ } -+ } -+ -+ if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) { -+ ret = -EBUSY; -+ goto err_unlock; -+ } -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) != -+ HMBIRD_OPS_DISABLED); -+ -+ hmbird_warned_zero_slice = false; -+ -+ atomic64_set(&hmbird_nr_rejected, 0); -+ -+ /* -+ * Keep CPUs stable during enable so that the BPF scheduler can track -+ * online CPUs by watching ->on/offline_cpu() after ->init(). -+ */ -+ cpus_read_lock(); -+ -+ hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work, -+ hmbird_watchdog_timeout / 2); -+ -+ /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ */ -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ reset_idle_masks(); -+ -+ /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. -+ */ -+ ret = hmbird_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock; -+ -+ /* -+ * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem -+ * preventing new tasks from being added. No need to exclude tasks -+ * leaving as hmbird_free() can handle both prepped and enabled -+ * tasks. Prep all tasks first and then enable them with preemption -+ * disabled. -+ */ -+ spin_lock_irq(&hmbird_tasks_lock); -+ -+ atomic_set(&non_hmbird_task, false); -+ atomic_set(&__hmbird_ops_enabled, true); -+ -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered(&sti))) -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ hmbird_task_iter_exit(&sti); -+ -+ /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. -+ */ -+ preempt_disable(); -+ -+ /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ */ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) { -+ atomic_set(&non_hmbird_task, true); -+ atomic_set(&__hmbird_ops_enabled, false); -+ preempt_enable(); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ ret = -EBUSY; -+ goto err_disable_unlock; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED -+ * transitions here are synchronized against hmbird_free() through -+ * hmbird_tasks_lock. -+ */ -+ start = sched_clock(); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, ""); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ tcnt++; -+ if (READ_ONCE(p->__state) != TASK_DEAD) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ -+ set_audio_thread_sched_prop(p); -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ hmbird_ops_enable_task(p); -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ } else { -+ hmbird_ops_disable_task(p); -+ } -+ } -+ hmbird_task_iter_exit(&sti); -+ -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, ""); -+ preempt_enable(); -+ percpu_up_write(&hmbird_fork_rwsem); -+ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) { -+ ret = -EBUSY; -+ goto err_disable; -+ } -+ -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_ENABLED, 1, 1, ""); -+ scheduler_switch_done(true); -+ -+ return 0; -+ -+err_unlock: -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock"); -+ scheduler_switch_done(false); -+ return ret; -+ -+err_disable_unlock: -+ percpu_up_write(&hmbird_fork_rwsem); -+err_disable: -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ /* must be fully disabled before returning */ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable"); -+ scheduler_switch_done(false); -+ return ret; -+} -+ -+#ifdef CONFIG_SCHED_DEBUG -+static const char *hmbird_ops_enable_state_str[] = { -+ [HMBIRD_OPS_PREPPING] = "prepping", -+ [HMBIRD_OPS_ENABLING] = "enabling", -+ [HMBIRD_OPS_ENABLED] = "enabled", -+ [HMBIRD_OPS_DISABLING] = "disabling", -+ [HMBIRD_OPS_DISABLED] = "disabled", -+}; -+ -+static int hmbird_debug_show(struct seq_file *m, void *v) -+{ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled()); -+ seq_printf(m, "%-30s: %s\n", "enable_state", -+ hmbird_ops_enable_state_str[hmbird_ops_enable_state()]); -+ seq_printf(m, "%-30s: %llu\n", "nr_rejected", -+ atomic64_read(&hmbird_nr_rejected)); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return 0; -+} -+ -+static int hmbird_debug_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_debug_show, NULL); -+} -+ -+const struct file_operations sched_hmbird_fops = { -+ .open = hmbird_debug_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif -+ -+ -+static int bpf_hmbird_reg(void *kdata) -+{ -+ return hmbird_ops_enable(kdata); -+} -+ -+static int bpf_hmbird_unreg(void *kdata) -+{ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ return 0; -+} -+ -+void set_hmbird_module_loaded(int is_loaded) -+{ -+ atomic_set(&hmbird_module_loaded, is_loaded); -+} -+ -+/* -+ * MUST load hmbird module before enable hmbird scheduler -+ * load track & hmbird gover implemented in hmbird module -+ */ -+int hmbird_ctrl(bool enable) -+{ -+ if (!atomic_read(&hmbird_module_loaded)) { -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "ext module unloaded\n"); -+ return -EINVAL; -+ } -+ -+ if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED -+ || hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING -+ || hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already enabled(ing)\n"); -+ hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING || -+ hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already disabled(ing)\n"); -+ hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (enable) -+ return bpf_hmbird_reg(NULL); -+ else -+ return bpf_hmbird_unreg(NULL); -+} -+ -+void set_cpu_isomask(int cpu, cpumask_var_t *mask) -+{ -+ cpumask_clear_cpu(cpu, iso_masks.ex_free); -+ cpumask_clear_cpu(cpu, iso_masks.exclusive); -+ cpumask_clear_cpu(cpu, iso_masks.partial); -+ cpumask_clear_cpu(cpu, iso_masks.big); -+ cpumask_clear_cpu(cpu, iso_masks.little); -+ cpumask_set_cpu(cpu, *mask); -+} -+ -+void set_cpu_cluster(u64 cpu_cluster) -+{ -+ int cpu; -+ -+ for_each_present_cpu(cpu) { -+ u64 pos = 1 << cpu; -+ -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.ex_free)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.exclusive)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.partial)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.big)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) -+ set_cpu_isomask(cpu, &(iso_masks.little)); -+ } -+} -+ -+static void get_hmbird_snapshot(struct panic_snapshot_t *p) -+{ -+ struct hmbird_dispatch_q *dsq; -+ struct rq *rq; -+ int cpu, i; -+ -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ p->rq_nr[cpu] = rq->nr_running; -+ p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running; -+ } -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ struct hmbird_entity *entity; -+ -+ dsq = &gdsqs[i]; -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo)) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ p->runnable_at[i] = entity->runnable_at; -+ raw_spin_unlock(&dsq->lock); -+ -+ } -+ -+ p->snap_misc.hmbird_enabled = hmbird_enabled(); -+ p->snap_misc.curr_ss = curr_ss; -+ p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state(); -+ p->snap_misc.parctrl_high_ratio = parctrl_high_ratio; -+ p->snap_misc.parctrl_low_ratio = parctrl_low_ratio; -+ p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l; -+ p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l; -+ p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio; -+ p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio; -+ p->snap_misc.misfit_ds = misfit_ds; -+ p->snap_misc.partial_enable = partial_enable; -+ p->snap_misc.iso_free_rescue = iso_free_rescue; -+ p->snap_misc.isolate_ctrl = isolate_ctrl; -+ p->snap_misc.snap_jiffies = jiffies; -+ p->snap_misc.snap_time = local_clock(); -+} -+ -+// MTK minidump begin -+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3) -+{ -+ strscpy(m->desc_str, str, DESC_STR_LEN); -+ m->len = d1 * d2 * d3; -+ m->parse[0] = d1; -+ m->parse[1] = d2; -+ m->parse[2] = d3; -+} -+ -+static void init_desc_metas(struct md_info_t *m) -+{ -+ init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :", -+ 1, MAX_SWITCHS, SWITCH_ITEMS); -+ -+ init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1); -+ -+ init_desc_meta(&m->kern_dump.excep_rec_meta, -+ "excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS); -+ -+ init_desc_meta(&m->kern_dump.excep_idx_meta, -+ "excep idx :", 1, 1, MAX_EXCEP_ID); -+ -+ init_desc_meta(&m->kern_dump.snap.runnable_at_meta, -+ "each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS); -+ -+ init_desc_meta(&m->kern_dump.snap.rq_nr_meta, -+ "rq runnable task nr:", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta, -+ "scxrq runnable task nr :", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.snap_misc_meta, -+ "misc snap params :", 1, 1, SNAP_ITEMS); -+} -+ -+static void init_md_meta(struct md_info_t *m) -+{ -+ m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64); -+ m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64); -+ m->meta.unit_size = sizeof(u64); -+ m->meta.switches = MAX_SWITCHS; -+ m->meta.exceps = MAX_EXCEPS; -+ m->meta.global_dsqs = MAX_GLOBAL_DSQS; -+ m->meta.parse_dimens = PARSE_DIMENS; -+ m->meta.nr_cpus = num_possible_cpus(); -+ m->meta.real_cpus = nr_cpu_ids; -+ m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64); -+ m->meta.nr_meta_desc = 8; -+ m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64); -+ -+ init_desc_metas(m); -+} -+ -+#define MINIDUMP_DFL_SIZE (4 * 1024) -+ -+struct notifier_block hmbird_panic_blk; -+static int hmbird_panic_handler(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ if (!md_info) -+ return NOTIFY_DONE; -+ -+ get_hmbird_snapshot(&md_info->kern_dump.snap); -+ -+ return NOTIFY_DONE; -+} -+ -+static void panic_blk_init(void) -+{ -+ int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE); -+ -+ md_info = kzalloc(dump_size, GFP_KERNEL); -+ if (!md_info) -+ return; -+ init_md_meta(md_info); -+ -+ hmbird_panic_blk.notifier_call = hmbird_panic_handler; -+ /* make sure to execute before minidump. */ -+ hmbird_panic_blk.priority = INT_MAX; -+ atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk); -+ -+ hmbird_debug("register minidump.\n"); -+} -+ -+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size) -+{ -+ *vaddr = (unsigned long)md_info; -+ *size = sizeof(struct md_info_t); -+} -+// MTK minidump end -+ -+static inline void init_sched_prop_to_preempt_prio(void) -+{ -+ for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) { -+ switch (i) { -+ case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 5; -+ break; -+ -+ case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 4; -+ break; -+ -+ case HMBIRD_TASK_PROP_PIPELINE: -+ case HMBIRD_TASK_PROP_ISOLATE: -+ sched_prop_to_preempt_prio[i] = 3; -+ break; -+ -+ case HMBIRD_TASK_PROP_COMMON: -+ sched_prop_to_preempt_prio[i] = 1; -+ break; -+ -+ case HMBIRD_TASK_PROP_DEBUG_OR_LOG: -+ sched_prop_to_preempt_prio[i] = 0; -+ break; -+ -+ default: -+ sched_prop_to_preempt_prio[i] = 2; -+ break; -+ } -+ } -+} -+ -+void __init init_sched_hmbird_class(void) -+{ -+ int cpu; -+ u32 v; -+ struct hmbird_entity *init_hmbird; -+ -+ /* -+ * The following is to prevent the compiler from optimizing out the enum -+ * definitions so that BPF scheduler implementations can use them -+ * through the generated vmlinux.h. -+ */ -+ WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP | -+ HMBIRD_KICK_PREEMPT); -+ -+ init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL); -+ init_dsq_at_boot(); -+ init_isolate_cpus(); -+ hb_timer_init(); -+ init_sched_prop_to_preempt_prio(); -+#ifdef CONFIG_SMP -+ WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); -+#endif -+ -+ /* -+ * we can't static init init_task's hmbird struct, init here. -+ * init_task->hmbird would not use during boot. -+ */ -+ init_hmbird = kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL); -+ init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird; -+ if (init_hmbird) { -+ INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo); -+ INIT_LIST_HEAD(&init_hmbird->watchdog_node); -+ init_hmbird->sticky_cpu = -1; -+ init_hmbird->holding_cpu = -1; -+ atomic64_set(&init_hmbird->ops_state, 0); -+ init_hmbird->runnable_at = jiffies; -+ init_hmbird->slice = HMBIRD_SLICE_DFL; -+ init_hmbird->task = &init_task; -+ hmbird_set_sched_prop(&init_task, 0); -+ } else { -+ hmbird_err(INIT_TASK_FAIL, ":alloc init_task.scx failed!!!\n"); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ /* -+ * exec during boot phase, no need to care about alloc failed. -+ * lifecycle same to rq, no need to free. -+ */ -+ rq->android_oem_data1[HMBIRD_RQ_IDX] = -+ (u64)kmalloc(sizeof(struct hmbird_rq), GFP_KERNEL); -+ if (get_hmbird_rq(rq)) { -+ get_hmbird_rq(rq)->rq = rq; -+ get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window; -+ } else { -+ hmbird_err(ALLOC_RQSCX_FAIL, ":alloc rq->scx failed!!!\n"); -+ } -+ -+ rq->android_oem_data1[HMBIRD_OPS_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL); -+ if (!get_hmbird_ops(rq)) -+ pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n"); -+ init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL); -+ INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list); -+ -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL)); -+ -+ hmbird_ops_init(get_hmbird_ops(rq)); -+ } -+ -+ INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn); -+ INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn); -+ hmbird_misc_init(); -+ -+ panic_blk_init(); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_misc.c b/kernel/sched/hmbird/hmbird_misc.c -new file mode 100755 -index 000000000000..52582c22052c ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_misc.c -@@ -0,0 +1,124 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * @Description: -+ * @Version: -+ * @Author: wangrui8 -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include "hmbird_sched.h" -+#include -+#include -+#include "slim.h" -+ -+noinline int tracing_mark_write(const char *buf) -+{ -+ trace_printk(buf); -+ return 0; -+} -+ -+struct yield_opt_params yield_opt_params = { -+ .enable = 0, -+ .frame_per_sec = 120, -+ .frame_time_ns = NSEC_PER_SEC / 120, -+ .yield_headroom = 10, -+}; -+ -+DEFINE_PER_CPU(struct sched_yield_state, ystate); -+ -+static inline void hmbird_yield_state_update(struct sched_yield_state *ys) -+{ -+ if (!raw_spin_is_locked(&ys->lock)) -+ return; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ -+ if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1 -+ || ys->yield_cnt_after_sleep > yield_headroom) { -+ ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP); -+ } else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) { -+ ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP); -+ } -+ ys->yield_cnt = 0; -+ ys->sleep_times = 0; -+ ys->yield_cnt_after_sleep = 0; -+} -+ -+void hmbird_skip_yield(long *skip) -+{ -+ if (!get_hmbird_ops_enabled() || !yield_opt_params.enable) -+ return; -+ unsigned long flags, sleep_now = 0; -+ struct sched_yield_state *ys; -+ int cpu = raw_smp_processor_id(), cont_yield, new_frame; -+ int frame_time_ns = yield_opt_params.frame_time_ns; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ u64 wc; -+ -+ if (!(*skip)) { -+ wc = sched_clock(); -+ ys = &per_cpu(ystate, cpu); -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ -+ cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP; -+ new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1); -+ -+ if (!cont_yield && new_frame) { -+ hmbird_yield_state_update(ys); -+ ys->last_update_time = wc; -+ ys->sleep_end = ys->last_yield_time + frame_time_ns -+ - yield_headroom * YIELD_DURATION; -+ } -+ -+ if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) { -+ *skip = true; -+ -+ sleep_now = ys->sleep_times ? -+ max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep; -+ if (wc + sleep_now > ys->sleep_end) { -+ u64 delta = ys->sleep_end - wc; -+ -+ if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION) -+ sleep_now = delta; -+ else -+ sleep_now = 0; -+ } -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ if (sleep_now) { -+ sleep_now = div64_u64(sleep_now, 1000); -+ usleep_range_state(sleep_now, sleep_now, TASK_IDLE); -+ } -+ ys->sleep_times++; -+ ys->last_yield_time = sched_clock(); -+ return; -+ } -+ if (ys->sleep_times) -+ ys->yield_cnt_after_sleep++; -+ else -+ (ys->yield_cnt)++; -+ ys->last_yield_time = wc; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+} -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops) -+{ -+ hmbird_ops->scx_enable = get_hmbird_ops_enabled; -+ hmbird_ops->check_non_task = get_non_hmbird_task; -+ hmbird_ops->hmbird_get_md_info = hmbird_get_md_info; -+} -+ -+void hmbird_misc_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_init(&ys->lock); -+ } -+ -+ set_hmbird_module_loaded(1); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_sched.h b/kernel/sched/hmbird/hmbird_sched.h -new file mode 100755 -index 000000000000..6b5ef43cb827 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef __HMBIRD_SCHED__ -+#define __HMBIRD_SCHED__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include "../../time/tick-sched.h" -+#include "../../sched/sched.h" -+#include -+#include -+#include -+ -+#define REGISTER_TRACE_VH(vender_hook, handler) \ -+{ \ -+ ret = register_trace_##vender_hook(handler, NULL); \ -+ if (ret) { \ -+ pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \ -+ } \ -+} -+#define REGISTER_TRACE(vendor_hook, handler, data, err) \ -+do { \ -+ ret = register_trace_##vendor_hook(handler, data); \ -+ if (ret) { \ -+ pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret); \ -+ goto err; \ -+ } \ -+} while (0) -+ -+#define UNREGISTER_TRACE(vendor_hook, handler, data) \ -+ unregister_trace_##vendor_hook(handler, data) \ -+ -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+ -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int sched_ravg_window_frame_per_sec; -+extern int slim_gov_debug; -+extern int cpu7_tl; -+extern int scx_gov_ctrl; -+extern spinlock_t new_sched_ravg_window_lock; -+extern int cluster_separate; -+ -+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER BIT(31) -+#define MAX_YIELD_SLEEP (2000000ULL) -+#define MIN_YIELD_SLEEP (200000ULL) -+#define YIELD_DURATION (5000ULL) -+#define DEFAULT_YIELD_SLEEP_TH (10) -+ -+struct sched_yield_state { -+ raw_spinlock_t lock; -+ u64 last_yield_time; -+ u64 last_update_time; -+ u64 sleep_end; -+ unsigned long yield_cnt; -+ unsigned long yield_cnt_after_sleep; -+ unsigned long sleep; -+ int sleep_times; -+}; -+ -+DECLARE_PER_CPU(struct sched_yield_state, ystate); -+ -+void hmbird_window_rollover_run_once(struct rq *rq); -+void hmbird_yield_state_update_per_frame(void); -+void hmbird_misc_init(void); -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops); -+#endif /*__HMBIRD_SCHED__*/ -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.c b/kernel/sched/hmbird/hmbird_sched_proc.c -new file mode 100755 -index 000000000000..234768fc767a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.c -@@ -0,0 +1,527 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "hmbird_sched_proc.h" -+#include -+ -+#include "hmbird_util_track.h" -+#include "slim.h" -+ -+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched" -+#define SLIM_FREQ_GOV_DIR "slim_freq_gov" -+#define LOAD_TRACK_DIR "slim_walt" -+#define HMBIRD_PROC_PERMISSION 0666 -+ -+int scx_enable; -+int partial_enable; -+int cpuctrl_high_ratio = 55; -+int cpuctrl_low_ratio = 40; -+int slim_stats; -+int hmbirdcore_debug; -+int slim_for_app; -+int misfit_ds = 90; -+unsigned int highres_tick_ctrl; -+unsigned int highres_tick_ctrl_dbg; -+int cpu7_tl = 70; -+int slim_walt_ctrl; -+int slim_walt_dump; -+int slim_walt_policy; -+int slim_gov_debug; -+int scx_gov_ctrl = 1; -+int sched_ravg_window_frame_per_sec = 125; -+int parctrl_high_ratio = 55; -+int parctrl_low_ratio = 40; -+int parctrl_high_ratio_l = 65; -+int parctrl_low_ratio_l = 50; -+int isoctrl_high_ratio = 75; -+int isoctrl_low_ratio = 60; -+int isolate_ctrl; -+int iso_free_rescue; -+int heartbeat; -+int heartbeat_enable = 1; -+int watchdog_enable; -+int save_gov; -+u64 cpu_cluster_masks; -+int hmbird_preempt_policy; -+int cluster_separate; -+ -+char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= 32) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtoint(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoint\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+static int set_proc_buf_val_u64(struct file *file, const char __user *buf, -+ size_t count, u64 *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= sizeof(kbuf)) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtou64(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoul\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/* common ops begin */ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ return count; -+} -+ -+static int hmbird_common_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%d\n", *(int *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_show, pde_data(inode)); -+} -+ -+static int hmbird_common_ul_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%lu\n", *(unsigned long *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_ul_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_ul_show, pde_data(inode)); -+} -+ -+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write); -+/* common ops end */ -+ -+/* scx_enable ops begin */ -+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC); -+ if (hmbird_ctrl(*pval)) -+ return -EFAULT; -+ -+ return count; -+} -+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write); -+/* scx_enable ops end */ -+ -+/* hmbird_stats ops begin */ -+#define MAX_STATS_BUF (4096) -+static int hmbird_stats_proc_show(struct seq_file *m, void *v) -+{ -+ char *buf; -+ -+ buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ stats_print(buf, MAX_STATS_BUF); -+ -+ seq_printf(m, "%s\n", buf); -+ -+ kfree(buf); -+ return 0; -+} -+ -+static int hmbird_stats_proc_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_stats_proc_show, inode); -+} -+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL); -+/* hmbird_stats ops end */ -+ -+/* sched_ravg_window_frame_per_sec ops begin */ -+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file, -+ const char __user *buf, size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ sched_ravg_window_change(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open, -+ sched_ravg_window_frame_per_sec_proc_write); -+/* sched_ravg_window_frame_per_sec ops end */ -+ -+static ssize_t save_gov_str(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ for_each_possible_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy || (cpu != policy->cpu)) -+ continue; -+ WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0); -+ hmbird_info_systrace(":save origin gov : %s\n", saved_gov[cpu]); -+ } -+ return count; -+} -+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str); -+ -+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ u64 *pval = (u64 *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val_u64(file, buf, count, pval)) -+ return -EFAULT; -+ -+ if (scx_enable == 0) -+ set_cpu_cluster(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write); -+ -+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ int tmp_val; -+ -+ if (set_proc_buf_val(file, buf, count, &tmp_val)) -+ return -EFAULT; -+ -+ slim_walt_enable(tmp_val); -+ *pval = tmp_val; -+ -+ return count; -+} -+ -+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write); -+ -+/* yield_opt ops begin */ -+static int yield_opt_show(struct seq_file *m, void *v) -+{ -+ struct yield_opt_params *data = m->private; -+ -+ seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n", -+ data->enable, data->frame_per_sec, data->yield_headroom); -+ return 0; -+} -+ -+static int yield_opt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, yield_opt_show, pde_data(inode)); -+} -+ -+static ssize_t yield_opt_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu; -+ unsigned long flags; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if (copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp -+ != 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) || -+ (yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp; -+ yield_opt_params.frame_per_sec = frame_per_sec_tmp; -+ yield_opt_params.yield_headroom = yield_headroom_tmp; -+ yield_opt_params.enable = enable_tmp; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ ys->last_yield_time = 0; -+ ys->last_update_time = 0; -+ ys->sleep_end = 0; -+ ys->yield_cnt = 0; -+ ys->yield_cnt_after_sleep = 0; -+ ys->sleep = 0; -+ ys->sleep_times = 0; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write); -+ -+ -+static int __init hmbird_proc_init(void) -+{ -+ struct proc_dir_entry *hmbird_dir; -+ struct proc_dir_entry *load_track_dir; -+ struct proc_dir_entry *freq_gov_dir; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ -+ /* mkdir /proc/hmbird_sched */ -+ hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL); -+ if (!hmbird_dir) { -+ pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &scx_enable_proc_ops, -+ &scx_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &partial_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_stats); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbirdcore_debug); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_for_app); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &misfit_ds); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl_dbg); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpu7_tl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &cpu_cluster_masks_proc_ops, -+ &cpu_cluster_masks); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &save_gov_proc_ops, -+ &save_gov); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &watchdog_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isolate_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &iso_free_rescue); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_stats_proc_ops); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &yield_opt_proc_ops, -+ &yield_opt_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbird_preempt_policy); -+ /* /proc/hmbird_sched--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_walt */ -+ load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir); -+ if (!load_track_dir) { -+ pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_walt--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &slim_walt_ctrl_proc_ops, -+ &slim_walt_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_dump); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_policy); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &sched_ravg_window_frame_per_sec_proc_ops, -+ &sched_ravg_window_frame_per_sec); -+ /* /proc/hmbird_sched/slim_walt--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_freq_gov */ -+ freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir); -+ if (!freq_gov_dir) { -+ pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_freq_gov--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &slim_gov_debug); -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &scx_gov_ctrl); -+ /* /proc/hmbird_sched/slim_freq_gov--end */ -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cluster_separate); -+ -+ return 0; -+} -+ -+device_initcall(hmbird_proc_init); -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.h b/kernel/sched/hmbird/hmbird_sched_proc.h -new file mode 100755 -index 000000000000..e22bc75f1dd7 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SCHED_PROC_H__ -+#define __HMBIRD_SCHED_PROC_H__ -+ -+#include -+#include -+ -+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \ -+ do { \ -+ if (!proc_create(name, mode, parent, proc_ops)) { \ -+ pr_err("Error creating proc entry %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \ -+ do { \ -+ if (!proc_create_data(name, mode, parent, proc_ops, data)) { \ -+ pr_err("Error creating proc entry with data %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_PROC_OPS(name, open_func, write_func) \ -+ static const struct proc_ops name##_proc_ops = { \ -+ .proc_open = open_func, \ -+ .proc_write = write_func, \ -+ .proc_read = seq_read, \ -+ .proc_lseek = seq_lseek, \ -+ .proc_release = single_release, \ -+ } -+ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos); -+static int hmbird_common_show(struct seq_file *m, void *v); -+static int hmbird_common_open(struct inode *inode, struct file *file); -+ -+#endif -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.c b/kernel/sched/hmbird/hmbird_shadow_tick.c -new file mode 100755 -index 000000000000..21de551c5132 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.c -@@ -0,0 +1,159 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include -+#include "../../time/tick-sched.h" -+#include -+ -+#include "hmbird_shadow_tick.h" -+ -+#define HIGHRES_WATCH_CPU 0 -+ -+#include -+static bool shadow_tick_enable(void) {return highres_tick_ctrl; } -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; } -+#endif -+static bool shadow_tick_timer_init_flag; -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define shadow_tick_printk(fmt, args...) \ -+do { \ -+ int cpu = smp_processor_id(); \ -+ if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU) \ -+ trace_printk("hmbird shadow tick :"fmt, args); \ -+} while (0) -+#else -+#define shadow_tick_printk(fmt, args...) -+#endif -+ -+DEFINE_PER_CPU(struct hrtimer, stt); -+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu))) -+#define STOP_IDLE_TRIGGER (1) -+#define PERIODIC_TICK_TRIGGER (2) -+#define TICK_INTVAL (1000000ULL) -+/* -+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms, -+ * so we can't restart hrtimer in scheduler tick. -+ */ -+static DEFINE_PER_CPU(u8, trigger_event); -+ -+/* -+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick. -+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler. -+ */ -+static DEFINE_PER_CPU(u8, tick_phase); -+ -+static inline void highres_timer_ctrl(bool enable, int cpu) -+{ -+ if (enable && hmbird_enabled()) { -+ if (!hrtimer_active(shadow_tick_timer(cpu))) -+ hrtimer_start(shadow_tick_timer(cpu), -+ ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED); -+ } else { -+ if (!enable) -+ hrtimer_cancel(shadow_tick_timer(cpu)); -+ } -+} -+ -+static inline void high_res_clear_phase(int cpu) -+{ -+ per_cpu(tick_phase, cpu) = 0; -+} -+ -+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer) -+{ -+ per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3; -+ if (per_cpu(tick_phase, cpu)) { -+ hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) { -+ per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ struct rq_flags rf; -+ -+ rq_lock(rq, &rf); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ rq_unlock(rq, &rf); -+ -+ return highres_next_phase(cpu, timer); -+} -+ -+void shadow_tick_timer_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance; -+ } -+} -+ -+void start_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable()) { -+ if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER) -+ highres_timer_ctrl(false, cpu); -+ per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static void stop_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ per_cpu(trigger_event, cpu) = 0; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(false, cpu); -+} -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ stop_shadow_tick_timer(); -+} -+ -+void scheduler_tick_handler(void *unused, struct rq *rq) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ start_shadow_tick_timer(); -+} -+ -+static int __init hmbird_shadow_tick_init(void) -+{ -+ int ret = 0; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ shadow_tick_timer_init(); -+ shadow_tick_timer_init_flag = true; -+ return ret; -+} -+ -+device_initcall(hmbird_shadow_tick_init); -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.h b/kernel/sched/hmbird/hmbird_shadow_tick.h -new file mode 100755 -index 000000000000..0c26fd984f0a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.h -@@ -0,0 +1,11 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SHADOW_TICK_H__ -+#define __HMBIRD_SHADOW_TICK_H__ -+ -+#include -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+void scheduler_tick_handler(void *unused, struct rq *rq); -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state); -+#endif -diff --git a/kernel/sched/hmbird/hmbird_trace.h b/kernel/sched/hmbird/hmbird_trace.h -new file mode 100755 -index 000000000000..8468b406f347 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_trace.h -@@ -0,0 +1,92 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hmbird -+ -+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HMBIRD_H -+ -+#include -+#include -+#include -+ -+#define MAX_FATAL_INFO (64) -+ -+TRACE_EVENT(hmbird_fatal_info, -+ -+ TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info), -+ -+ TP_ARGS(type, pe, lnr, bnr, info), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, type) -+ __field(int, pe) -+ __field(int, lnr) -+ __field(int, bnr) -+ __array(char, info, MAX_FATAL_INFO)), -+ -+ TP_fast_assign( -+ __entry->type = type; -+ __entry->pe = pe; -+ __entry->lnr = lnr; -+ __entry->bnr = bnr; -+ memcpy(__entry->info, info, MAX_FATAL_INFO);), -+ -+ TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s", -+ __entry->type, __entry->pe, __entry->lnr, __entry->bnr, -+ __entry->info) -+); -+ -+TRACE_EVENT(hmbird_update_history, -+ -+ TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq, -+ struct task_struct *p, u32 runtime, int samples, int event), -+ -+ TP_ARGS(hmbird, rq, p, runtime, samples, event), -+ -+ TP_STRUCT__entry( -+ __array(char, comm, TASK_COMM_LEN) -+ __field(pid_t, pid) -+ __field(unsigned int, runtime) -+ __field(int, samples) -+ __field(int, event) -+ __field(unsigned int, demand) -+ __array(u32, hist, RAVG_HIST_SIZE) -+ __field(u16, task_util) -+ __field(int, cpu)), -+ -+ TP_fast_assign( -+ memcpy(__entry->comm, p->comm, TASK_COMM_LEN); -+ __entry->pid = p->pid; -+ __entry->runtime = runtime; -+ __entry->samples = samples; -+ __entry->event = event; -+ __entry->demand = hmbird->sts.demand; -+ memcpy(__entry->hist, hmbird->sts.sum_history, -+ RAVG_HIST_SIZE * sizeof(u32)); -+ __entry->task_util = hmbird->sts.demand_scaled; -+ __entry->cpu = rq->cpu;), -+ -+ TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d", -+ __entry->comm, __entry->pid, -+ __entry->runtime, __entry->samples, -+ __entry->event, -+ __entry->demand, -+ __entry->hist[0], __entry->hist[1], -+ __entry->hist[2], __entry->hist[3], -+ __entry->hist[4], -+ __entry->task_util, -+ __entry->cpu) -+); -+ -+#endif /*_TRACE_HMBIRD_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird -+ -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE hmbird_trace -+/* This part must be outside protection */ -+#include -diff --git a/kernel/sched/hmbird/hmbird_util_track.c b/kernel/sched/hmbird/hmbird_util_track.c -new file mode 100755 -index 000000000000..d520a8403401 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.c -@@ -0,0 +1,626 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+ -+#define CREATE_TRACE_POINTS -+#include "hmbird_trace.h" -+#undef CREATE_TRACE_POINTS -+ -+#define HMBIRD_DEBUG_PANIC (1 << 3) -+static bool init_irq_work_inited; -+ -+extern noinline int tracing_mark_write(const char *buf); -+ -+int hmbird_sched_ravg_window = 8000000; -+int new_hmbird_sched_ravg_window = 8000000; -+DEFINE_SPINLOCK(new_sched_ravg_window_lock); -+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+ -+static struct irq_work hmbird_slim_walt_irq_work; -+ -+/*Sysctl related interface*/ -+#define WINDOW_STATS_RECENT 0 -+#define WINDOW_STATS_MAX 1 -+#define WINDOW_STATS_MAX_RECENT_AVG 2 -+#define WINDOW_STATS_AVG 3 -+#define WINDOW_STATS_INVALID_POLICY 4 -+ -+ -+#define SCX_SCHED_CAPACITY_SHIFT 10 -+#define SCHED_ACCOUNT_WAIT_TIME 0 -+ -+atomic64_t hmbird_irq_work_lastq_ws; -+static u64 tick_sched_clock; -+ -+static inline u64 scale_exec_time(u64 delta, struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT; -+} -+ -+static inline u64 scale_time_to_util(u64 d) -+{ -+ do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT); -+ return d; -+} -+ -+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ delta = scale_exec_time(delta, rq); -+ sts->sum += delta; -+ if (unlikely(sts->sum > hmbird_sched_ravg_window)) -+ sts->sum = hmbird_sched_ravg_window; -+ -+ return delta; -+} -+ -+ -+static int -+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) -+{ -+ /* -+ * No need to bother updating task demand for the idle task. -+ */ -+ if (is_idle_task(p)) -+ return 0; -+ -+ /* -+ * When a task is waking up it is completing a segment of non-busy -+ * time. Likewise, if wait time is not treated as busy time, then -+ * when a task begins to run or is migrated, it is not running and -+ * is completing a segment of non-busy time. -+ */ -+ if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && -+ (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) -+ return 0; -+ -+ /* -+ * The idle exit time is not accounted for the first task _picked_ up to -+ * run on the idle CPU. -+ */ -+ if (event == PICK_NEXT_TASK && rq->curr == rq->idle) -+ return 0; -+ -+ /* -+ * TASK_UPDATE can be called on sleeping task, when its moved between -+ * related groups -+ */ -+ if (event == TASK_UPDATE) { -+ if (rq->curr == p) -+ return 1; -+ -+ return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; -+ } -+ -+ return 1; -+} -+ -+static void rollover_cpu_window(struct rq *rq, bool full_window) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 curr_sum = srq->curr_runnable_sum; -+ -+ if (unlikely(full_window)) -+ curr_sum = 0; -+ -+ srq->prev_runnable_sum = curr_sum; -+ srq->curr_runnable_sum = 0; -+} -+ -+static u64 -+update_window_start(struct rq *rq, u64 wallclock, int event) -+{ -+ s64 delta; -+ int nr_windows; -+ bool full_window; -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start = srq->window_start; -+ -+ if (wallclock < srq->latest_clock) -+ wallclock = srq->latest_clock; -+ delta = wallclock - srq->window_start; -+ if (delta < 0) { -+ delta = 0; -+ wallclock = srq->window_start; -+ } -+ srq->latest_clock = wallclock; -+ if (delta < hmbird_sched_ravg_window) -+ return old_window_start; -+ -+ nr_windows = div64_u64(delta, hmbird_sched_ravg_window); -+ srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window; -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ full_window = nr_windows > 1; -+ rollover_cpu_window(rq, full_window); -+ -+ return old_window_start; -+} -+ -+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -+ -+static inline unsigned int get_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static inline unsigned int cpu_cur_freq(int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cur; -+} -+ -+static void -+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, -+ u64 wallclock) -+{ -+ int cpu = cpu_of(rq); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * -+ arch_scale_cpu_capacity(cpu), get_max_freq(cpu)); -+} -+ -+/* -+ * Called when new window is starting for a task, to record cpu usage over -+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 -+ * when, say, a real-time task runs without preemption for several windows at a -+ * stretch. -+ */ -+static void update_history(struct rq *rq, struct task_struct *p, -+ u32 runtime, int samples, int event) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u32 *hist = &sts->sum_history[0]; -+ int i; -+ u32 max = 0, avg, demand; -+ u64 sum = 0; -+ u16 demand_scaled; -+ -+ /* Ignore windows where task had no activity */ -+ if (!runtime || is_idle_task(p) || !samples) -+ goto done; -+ -+ /* Push new 'runtime' value onto stack */ -+ for (; samples > 0; samples--) { -+ hist[sts->cidx] = runtime; -+ sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE; -+ } -+ -+ for (i = 0; i < RAVG_HIST_SIZE; i++) { -+ sum += hist[i]; -+ if (hist[i] > max) -+ max = hist[i]; -+ } -+ -+ sts->sum = 0; -+ avg = div64_u64(sum, RAVG_HIST_SIZE); -+ -+ switch (slim_walt_policy) { -+ case WINDOW_STATS_RECENT: -+ demand = runtime; -+ break; -+ case WINDOW_STATS_MAX: -+ demand = max; -+ break; -+ case WINDOW_STATS_AVG: -+ demand = avg; -+ break; -+ default: -+ demand = max(avg, runtime); -+ } -+ -+ demand_scaled = scale_time_to_util(demand); -+ -+ sts->demand = demand; -+ sts->demand_scaled = demand_scaled; -+ -+done: -+ return; -+} -+ -+ -+static u64 update_task_demand(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ -+ u64 delta, window_start = srq->window_start; -+ int new_window, nr_full_windows; -+ u32 window_size = hmbird_sched_ravg_window; -+ u64 runtime; -+ -+ new_window = mark_start < window_start; -+ if (!account_busy_for_task_demand(rq, p, event)) { -+ if (new_window) -+ /* -+ * If the time accounted isn't being accounted as -+ * busy time, and a new window started, only the -+ * previous window need be closed out with the -+ * pre-existing demand. Multiple windows may have -+ * elapsed, but since empty windows are dropped, -+ * it is not necessary to account those. -+ */ -+ update_history(rq, p, sts->sum, 1, event); -+ return 0; -+ } -+ -+ if (!new_window) { -+ /* -+ * The simple case - busy time contained within the existing -+ * window. -+ */ -+ return add_to_task_demand(rq, p, wallclock - mark_start); -+ } -+ -+ /* -+ * Busy time spans at least two windows. Temporarily rewind -+ * window_start to first window boundary after mark_start. -+ */ -+ delta = window_start - mark_start; -+ nr_full_windows = div64_u64(delta, window_size); -+ window_start -= (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (window_start - mark_start) first */ -+ runtime = add_to_task_demand(rq, p, window_start - mark_start); -+ -+ /* Push new sample(s) into task's demand history */ -+ update_history(rq, p, sts->sum, 1, event); -+ if (nr_full_windows) { -+ u64 scaled_window = scale_exec_time(window_size, rq); -+ -+ update_history(rq, p, scaled_window, nr_full_windows, event); -+ runtime += nr_full_windows * scaled_window; -+ } -+ -+ /* -+ * Roll window_start back to current to process any remainder -+ * in current window. -+ */ -+ window_start += (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (wallclock - window_start) next */ -+ mark_start = window_start; -+ runtime += add_to_task_demand(rq, p, wallclock - mark_start); -+ -+ return runtime; -+} -+ -+u16 slim_walt_cpu_util(int cpu) -+{ -+ u64 prev_runnable_sum; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ prev_runnable_sum = srq->prev_runnable_sum; -+ do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT); -+ -+ return (u16)prev_runnable_sum; -+} -+ -+static DEFINE_PER_CPU(u16, prev_cpu_util); -+static inline void cpu_util_update_systrace_c(int cpu) -+{ -+ char buf[256]; -+ u16 cpu_util = slim_walt_cpu_util(cpu); -+ -+ if (cpu_util != per_cpu(prev_cpu_util, cpu)) { -+ snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n", -+ cpu, cpu_util); -+ tracing_mark_write(buf); -+ per_cpu(prev_cpu_util, cpu) = cpu_util; -+ } -+} -+ -+ -+static inline int account_busy_for_cpu_time(struct rq *rq, -+ struct task_struct *p, -+ int event) -+{ -+ return !is_idle_task(p) && (event == PUT_PREV_TASK -+ || event == TASK_UPDATE); -+} -+ -+ -+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ int new_window, full_window = 0; -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 window_start = srq->window_start; -+ u32 window_size = srq->prev_window_size; -+ u64 delta; -+ u64 *curr_runnable_sum = &srq->curr_runnable_sum; -+ u64 *prev_runnable_sum = &srq->prev_runnable_sum; -+ -+ new_window = mark_start < window_start; -+ if (new_window) -+ full_window = (window_start - mark_start) >= window_size; -+ -+ -+ if (!account_busy_for_cpu_time(rq, p, event)) -+ goto done; -+ -+ -+ if (!new_window) { -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. No rollover -+ * since we didn't start a new window. An example of this is -+ * when a task starts execution and then sleeps within the -+ * same window. -+ */ -+ delta = wallclock - mark_start; -+ -+ delta = scale_exec_time(delta, rq); -+ *curr_runnable_sum += delta; -+ -+ goto done; -+ } -+ -+ /* -+ * situations below this need window rollover, -+ * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done -+ * in update_window_start() -+ * -+ * For task counters curr/prev_window[_cpu] are rolled over in the early part of -+ * this function. If full_window(s) have expired and time since last update needs -+ * to be accounted as busy time, set the prev to a complete window size time, else -+ * add the prev window portion. -+ * -+ * For task curr counters a new window has begun, always assign -+ */ -+ -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. A new window -+ * must have been started in udpate_window_start() -+ * If any of these three above conditions are true -+ * then this busy time can't be accounted as irqtime. -+ * -+ * Busy time for the idle task need not be accounted. -+ * -+ * An example of this would be a task that starts execution -+ * and then sleeps once a new window has begun. -+ */ -+ -+ /* -+ * A full window hasn't elapsed, account partial -+ * contribution to previous completed window. -+ */ -+ -+ -+ delta = full_window ? scale_exec_time(window_size, rq) : -+ scale_exec_time(window_start - mark_start, rq); -+ -+ *prev_runnable_sum += delta; -+ -+ /* Account piece of busy time in the current window. */ -+ delta = scale_exec_time(wallclock - window_start, rq); -+ *curr_runnable_sum += delta; -+ -+done: -+ if (slim_walt_dump && new_window) -+ cpu_util_update_systrace_c(rq->cpu); -+} -+ -+ -+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq) -+{ -+ u64 result; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 new_window_start = srq->window_start; -+ -+ if (old_window_start == new_window_start) -+ return; -+ -+ result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws, -+ old_window_start, new_window_start); -+ if (result != old_window_start) -+ return; -+ -+ if (likely(cpu_online(raw_smp_processor_id()))) -+ irq_work_queue(&hmbird_slim_walt_irq_work); -+ else -+ irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask)); -+} -+ -+/* -+ * In the core scheduler, most of the load update points update the rq_clock after -+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of -+ * obtaining the time, but to prevent the subsequent migration of the load update -+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock -+ * update here. -+ */ -+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p, -+ struct rq *rq, int event) -+{ -+ if (!(rq->clock_update_flags & RQCF_UPDATED)) -+ update_rq_clock(rq); -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock)); -+} -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start; -+ -+ if (!slim_walt_ctrl) -+ return; -+ -+ if (!srq->window_start || sts->mark_start == wallclock) -+ return; -+ -+ old_window_start = update_window_start(rq, wallclock, event); -+ -+ if (!sts->window_start) -+ sts->window_start = srq->window_start; -+ -+ if (!sts->mark_start) -+ goto done; -+ -+ update_task_rq_cpu_cycles(p, rq, event, wallclock); -+ update_task_demand(p, rq, event, wallclock); -+ update_cpu_busy_time(p, rq, event, wallclock); -+ -+ sts->window_start = srq->window_start; -+ -+done: -+ sts->mark_start = wallclock; -+ slim_walt_window_rollover_run_once(old_window_start, rq); -+} -+ -+static void slim_walt_irq_work(struct irq_work *irq_work) -+{ -+ cpumask_t lock_cpus; -+ struct hmbird_sched_rq_stats *srq; -+ struct rq *rq; -+ int cpu; -+ int level = 0; -+ u64 wc; -+ unsigned long flags; -+ -+ cpumask_copy(&lock_cpus, cpu_possible_mask); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ if (level == 0) -+ raw_spin_lock(&cpu_rq(cpu)->__lock); -+ else -+ raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); -+ level++; -+ } -+ -+ wc = sched_clock(); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ rq = cpu_rq(cpu); -+ hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc); -+ } -+ -+ cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER); -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) { -+ srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id()); -+ if (wc < srq->window_start + new_hmbird_sched_ravg_window) -+ hmbird_sched_ravg_window = new_hmbird_sched_ravg_window; -+ } -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ raw_spin_unlock(&cpu_rq(cpu)->__lock); -+ } -+} -+static void hmbird_sched_init_rq(struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ srq->task_exec_scale = 1024; -+ srq->window_start = 0; -+} -+ -+void hmbird_sched_init_task(struct task_struct *p) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ memset(sts, 0, sizeof(struct hmbird_sched_task_stats)); -+} -+ -+static void hmbird_sched_stats_init(void) -+{ -+ unsigned long flags; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ raw_spin_lock_irqsave(&rq->__lock, flags); -+ hmbird_sched_init_rq(rq); -+ raw_spin_unlock_irqrestore(&rq->__lock, flags); -+ } -+ slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG; -+ -+ if (false == init_irq_work_inited) { -+ init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work); -+ init_irq_work_inited = true; -+ } -+} -+ -+void hmbird_scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (unlikely(!tick_sched_clock)) { -+ /* -+ * Let the window begin 20us prior to the tick, -+ * that way we are guaranteed a rollover when the tick occurs. -+ * Use rq->clock directly instead of rq_clock() since -+ * we do not have the rq lock and -+ * rq->clock was updated in the tick callpath. -+ */ -+ if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000)) -+ return; -+ for_each_possible_cpu(cpu) { -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ srq->window_start = tick_sched_clock; -+ } -+ atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock); -+ } -+} -+ -+void slim_walt_enable(int enable) -+{ -+ if (1 == !!enable) { -+ hmbird_sched_stats_init(); -+ WRITE_ONCE(tick_sched_clock, 0); -+ } else -+ slim_walt_ctrl = 0; -+} -+ -+void slim_get_cpu_util(int cpu, u64 *util) -+{ -+ if (cpu < 0) -+ return; -+ -+ *util = slim_walt_cpu_util(cpu); -+} -+ -+void slim_get_task_util(struct task_struct *p, u64 *util) -+{ -+ if (p == NULL) -+ return; -+ -+ *util = get_hmbird_ts(p)->sts.demand_scaled; -+} -+ -+void sched_ravg_window_change(int frame_per_sec) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec; -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+} -diff --git a/kernel/sched/hmbird/hmbird_util_track.h b/kernel/sched/hmbird/hmbird_util_track.h -new file mode 100755 -index 000000000000..17b85df7f83b ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.h -@@ -0,0 +1,33 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef __HMBIRD_UTIL_TRACK_H__ -+#define __HMBIRD_UTIL_TRACK_H__ -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock); -+void hmbird_sched_init_task(struct task_struct *p); -+void slim_walt_enable(int enable); -+void slim_get_cpu_util(int cpu, u64 *util); -+void slim_get_task_util(struct task_struct *p, u64 *util); -+ -+extern atomic64_t hmbird_irq_work_lastq_ws; -+ -+enum task_event { -+ PUT_PREV_TASK = 0, -+ PICK_NEXT_TASK = 1, -+ TASK_WAKE = 2, -+ TASK_MIGRATE = 3, -+ TASK_UPDATE = 4, -+ IRQ_UPDATE = 5, -+}; -+ -+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+#endif /* __HMBIRD_UTIL_TRACK_H__ */ -diff --git a/kernel/sched/hmbird/slim.h b/kernel/sched/hmbird/slim.h -new file mode 100755 -index 000000000000..eb386260e950 ---- /dev/null -+++ b/kernel/sched/hmbird/slim.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+ -+#ifndef __SLIM_H -+#define __SLIM_H -+ -+extern atomic_t __hmbird_ops_enabled; -+extern atomic_t non_hmbird_task; -+extern int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+extern int heartbeat; -+extern int heartbeat_enable; -+extern int watchdog_enable; -+extern int isolate_ctrl; -+extern int parctrl_high_ratio; -+extern int parctrl_low_ratio; -+extern int isoctrl_high_ratio; -+extern int isoctrl_low_ratio; -+extern int iso_free_rescue; -+extern int yield_opt; -+ -+extern enum hmbird_switch_type sw_type; -+extern noinline int tracing_mark_write(const char *buf); -+int task_top_id(struct task_struct *p); -+void stats_print(char *buf, int len); -+void hmbird_skip_yield(long *skip); -+extern spinlock_t hmbird_tasks_lock; -+ -+struct yield_opt_params { -+ int enable; -+ int frame_per_sec; -+ u64 frame_time_ns; -+ int yield_headroom; -+}; -+ -+extern struct yield_opt_params yield_opt_params; -+ -+#define MAX_GOV_LEN (16) -+extern char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+#endif -diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c -index b33cefeb4188..487255ac6832 100644 ---- a/kernel/sched/idle.c -+++ b/kernel/sched/idle.c -@@ -408,13 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl - - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) - { -- scx_update_idle(rq, false); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, false); -+#endif - } - - static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) - { - update_idle_core(rq); -- scx_update_idle(rq, true); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, true); -+#endif - schedstat_inc(rq->sched_goidle); - } - -diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h -index cbc478992cc4..e35473a1f0ea 100644 ---- a/kernel/sched/sched.h -+++ b/kernel/sched/sched.h -@@ -187,18 +187,13 @@ static inline int idle_policy(int policy) - return policy == SCHED_IDLE; - } - --static inline int normal_policy(int policy) --{ --#ifdef CONFIG_SCHED_CLASS_EXT -- if (policy == SCHED_EXT) -- return true; --#endif -- return policy == SCHED_NORMAL; --} -- - static inline int fair_policy(int policy) - { -- return normal_policy(policy) || policy == SCHED_BATCH; -+#ifdef CONFIG_HMBIRD_SCHED -+ return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#else -+ return policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#endif - } - - static inline int rt_policy(int policy) -@@ -246,24 +241,6 @@ static inline void update_avg(u64 *avg, u64 sample) - #define shr_bound(val, shift) \ - (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) - --/* -- * cgroup weight knobs should use the common MIN, DFL and MAX values which are -- * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it -- * maps pretty well onto the shares value used by scheduler and the round-trip -- * conversions preserve the original value over the entire range. -- */ --static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) --{ -- return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); --} -- --static inline unsigned long sched_weight_to_cgroup(unsigned long weight) --{ -- return clamp_t(unsigned long, -- DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -- CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); --} -- - /* - * !! For sched_setattr_nocheck() (kernel) only !! - * -@@ -531,10 +508,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) { } - #endif /* CONFIG_SMP */ - #else /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_HMBIRD_SCHED - static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) - { - return 0; - } -+#endif - #endif /* CONFIG_FAIR_GROUP_SCHED */ - - #else /* CONFIG_CGROUP_SCHED */ -@@ -699,29 +678,6 @@ struct cfs_rq { - #endif /* CONFIG_FAIR_GROUP_SCHED */ - }; - --#ifdef CONFIG_SCHED_CLASS_EXT --/* scx_rq->flags, protected by the rq lock */ --enum scx_rq_flags { -- SCX_RQ_CAN_STOP_TICK = 1 << 0, --}; -- --struct scx_rq { -- struct scx_dispatch_q local_dsq; -- struct list_head watchdog_list; -- u64 ops_qseq; -- u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -- u32 nr_running; -- u32 flags; -- bool cpu_released; -- cpumask_var_t cpus_to_kick; -- cpumask_var_t cpus_to_preempt; -- cpumask_var_t cpus_to_wait; -- u64 pnt_seq; -- struct irq_work kick_cpus_irq_work; -- struct rq *rq; --}; --#endif /* CONFIG_SCHED_CLASS_EXT */ -- - static inline int rt_bandwidth_enabled(void) - { - return sysctl_sched_rt_runtime >= 0; -@@ -1257,11 +1213,7 @@ struct rq { - - ANDROID_OEM_DATA_ARRAY(1, 16); - --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, struct scx_rq *scx); --#else - ANDROID_KABI_RESERVE(1); --#endif - ANDROID_KABI_RESERVE(2); - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); -@@ -2339,11 +2291,6 @@ struct affinity_context { - unsigned int flags; - }; - --enum rq_onoff_reason { -- RQ_ONOFF_HOTPLUG, /* CPU is going on/offline */ -- RQ_ONOFF_TOPOLOGY, /* sched domain topology update */ --}; -- - struct sched_class { - - #ifdef CONFIG_UCLAMP_TASK -@@ -2550,7 +2497,6 @@ extern void init_sched_rt_class(void); - extern void init_sched_fair_class(void); - - extern void reweight_task(struct task_struct *p, int prio); --extern void __setscheduler_prio(struct task_struct *p, int prio); - - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); -@@ -2631,53 +2577,6 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) - extern void activate_task(struct rq *rq, struct task_struct *p, int flags); - extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); - --struct sched_change_guard { -- struct task_struct *p; -- struct rq *rq; -- bool queued; -- bool running; -- bool done; --}; -- --extern struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -- --extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags); -- --/** -- * SCHED_CHANGE_BLOCK - Nested block for task attribute updates -- * @__rq: Runqueue the target task belongs to -- * @__p: Target task -- * @__flags: DEQUEUE/ENQUEUE_* flags -- * -- * A task may need to be dequeued and put_prev_task'd for attribute updates and -- * set_next_task'd and re-enqueued afterwards. This helper defines a nested -- * block which automatically handles these preparation and cleanup operations. -- * -- * SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- * update_attribute(p); -- * ... -- * } -- * -- * If @__flags is a variable, the variable may be updated in the block body and -- * the updated value will be used when re-enqueueing @p. -- * -- * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling -- * update_rq_clock() beforehand. Otherwise, the rq clock is automatically -- * updated iff the task needs to be dequeued and re-enqueued. Only the former -- * case guarantees that the rq clock is up-to-date inside and after the block. -- */ --#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -- for (struct sched_change_guard __cg = \ -- sched_change_guard_init(__rq, __p, __flags); \ -- !__cg.done; sched_change_guard_fini(&__cg, __flags)) -- --extern void check_class_changing(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class); --extern void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio); -- - extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); - - #ifdef CONFIG_PREEMPT_RT -@@ -3709,6 +3608,8 @@ static inline bool cpu_busy_with_softirqs(int cpu) - } - #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ - --#include "ext.h" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird.h" -+#endif - - #endif /* _KERNEL_SCHED_SCHED_H */ -diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c -index c197c61d6ea9..919fed4ff08b 100644 ---- a/kernel/time/tick-sched.c -+++ b/kernel/time/tick-sched.c -@@ -34,6 +34,10 @@ - - #include - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "../sched/hmbird/hmbird_shadow_tick.h" -+#endif -+ - /* - * Per-CPU nohz control structure - */ -@@ -1112,6 +1116,9 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) - return true; - } - -+#ifdef CONFIG_HMBIRD_SCHED -+extern void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+#endif - /** - * tick_nohz_idle_stop_tick - stop the idle tick from the idle task - * -@@ -1124,7 +1131,9 @@ void tick_nohz_idle_stop_tick(void) - ktime_t expires; - - trace_android_vh_tick_nohz_idle_stop_tick(NULL); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL); -+#endif - /* - * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the - * tick timer expiration time is known already. -diff --git a/tools/sched_ext/.gitignore b/tools/sched_ext/.gitignore -deleted file mode 100644 -index a3240f9f7eba..000000000000 ---- a/tools/sched_ext/.gitignore -+++ /dev/null -@@ -1,9 +0,0 @@ --scx_example_simple --scx_example_qmap --scx_example_central --scx_example_pair --scx_example_flatcg --scx_example_userland --*.skel.h --*.subskel.h --/tools/ -diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile -deleted file mode 100644 -index 73c43782837d..000000000000 ---- a/tools/sched_ext/Makefile -+++ /dev/null -@@ -1,216 +0,0 @@ --# SPDX-License-Identifier: GPL-2.0 --# Copyright (c) 2022 Meta Platforms, Inc. and affiliates. --include ../build/Build.include --include ../scripts/Makefile.arch --include ../scripts/Makefile.include -- --ifneq ($(LLVM),) --ifneq ($(filter %/,$(LLVM)),) --LLVM_PREFIX := $(LLVM) --else ifneq ($(filter -%,$(LLVM)),) --LLVM_SUFFIX := $(LLVM) --endif -- --CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi --CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu --CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl --CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu --CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu --CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu --CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu --CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu --CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu --CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) -- --ifeq ($(CROSS_COMPILE),) --ifeq ($(CLANG_TARGET_FLAGS),) --$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk --else --CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) --endif # CLANG_TARGET_FLAGS --else --CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) --endif # CROSS_COMPILE -- --CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as --else --CC := $(CROSS_COMPILE)gcc --endif # LLVM -- --CURDIR := $(abspath .) --TOOLSDIR := $(abspath ..) --LIBDIR := $(TOOLSDIR)/lib --BPFDIR := $(LIBDIR)/bpf --TOOLSINCDIR := $(TOOLSDIR)/include --BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool --APIDIR := $(TOOLSINCDIR)/uapi --GENDIR := $(abspath ../../include/generated) --GENHDR := $(GENDIR)/autoconf.h -- --SCRATCH_DIR := $(CURDIR)/tools --BUILD_DIR := $(SCRATCH_DIR)/build --INCLUDE_DIR := $(SCRATCH_DIR)/include --BPFOBJ_DIR := $(BUILD_DIR)/libbpf --BPFOBJ := $(BPFOBJ_DIR)/libbpf.a --ifneq ($(CROSS_COMPILE),) --HOST_BUILD_DIR := $(BUILD_DIR)/host --HOST_SCRATCH_DIR := host-tools --HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include --else --HOST_BUILD_DIR := $(BUILD_DIR) --HOST_SCRATCH_DIR := $(SCRATCH_DIR) --HOST_INCLUDE_DIR := $(INCLUDE_DIR) --endif --HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a --RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids --DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -- --VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ -- $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ -- ../../vmlinux \ -- /sys/kernel/btf/vmlinux \ -- /boot/vmlinux-$(shell uname -r) --VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) --ifeq ($(VMLINUX_BTF),) --$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") --endif -- --BPFTOOL ?= $(DEFAULT_BPFTOOL) -- --ifneq ($(wildcard $(GENHDR)),) -- GENFLAGS := -DHAVE_GENHDR --endif -- --CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ -- -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -- -I$(TOOLSINCDIR) -I$(APIDIR) -- --CARGOFLAGS := --release -- --# Silence some warnings when compiled with clang --ifneq ($(LLVM),) --CFLAGS += -Wno-unused-command-line-argument --endif -- --LDFLAGS = -lelf -lz -lpthread -- --IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - &1 \ -- | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ --$(shell $(1) -dM -E - $@ --else -- $(call msg,CP,,$@) -- $(Q)cp "$(VMLINUX_H)" $@ --endif -- --%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \ -- | $(BPFOBJ) -- $(call msg,CLNG-BPF,,$@) -- $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ -- --%.skel.h: %.bpf.o $(BPFTOOL) -- $(call msg,GEN-SKEL,,$@) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) -- $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) -- $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@ -- $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h) -- --scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_userland: scx_example_userland.c scx_example_userland.skel.h \ -- scx_example_userland_common.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR) --atropos: export ATROPOS_CLANG = $(CLANG) --atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS) --atropos: $(INCLUDE_DIR)/vmlinux.h -- cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS) -- --clean: -- cargo clean --manifest-path=atropos/Cargo.toml -- rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) -- rm -f *.o *.bpf.o *.skel.h *.subskel.h -- rm -f scx_example_simple scx_example_qmap scx_example_central \ -- scx_example_pair scx_example_flatcg scx_example_userland -- --.PHONY: all atropos clean -- --# delete failed targets --.DELETE_ON_ERROR: -- --# keep intermediate (.skel.h, .bpf.o, etc) targets --.SECONDARY: -diff --git a/tools/sched_ext/atropos/.gitignore b/tools/sched_ext/atropos/.gitignore -deleted file mode 100644 -index 186dba259ec2..000000000000 ---- a/tools/sched_ext/atropos/.gitignore -+++ /dev/null -@@ -1,3 +0,0 @@ --src/bpf/.output --Cargo.lock --target -diff --git a/tools/sched_ext/atropos/Cargo.toml b/tools/sched_ext/atropos/Cargo.toml -deleted file mode 100644 -index 7462a836d53d..000000000000 ---- a/tools/sched_ext/atropos/Cargo.toml -+++ /dev/null -@@ -1,28 +0,0 @@ --[package] --name = "scx_atropos" --version = "0.5.0" --authors = ["Dan Schatzberg ", "Meta"] --edition = "2021" --description = "Userspace scheduling with BPF" --license = "GPL-2.0-only" -- --[dependencies] --anyhow = "1.0.65" --bitvec = { version = "1.0", features = ["serde"] } --clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } --ctrlc = { version = "3.1", features = ["termination"] } --fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"} --hex = "0.4.3" --libbpf-rs = "0.19.1" --libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] } --libc = "0.2.137" --log = "0.4.17" --ordered-float = "3.4.0" --simplelog = "0.12.0" -- --[build-dependencies] --bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false } --libbpf-cargo = "0.13.0" -- --[features] --enable_backtrace = [] -diff --git a/tools/sched_ext/atropos/build.rs b/tools/sched_ext/atropos/build.rs -deleted file mode 100644 -index 26e792c5e17e..000000000000 ---- a/tools/sched_ext/atropos/build.rs -+++ /dev/null -@@ -1,70 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --extern crate bindgen; -- --use std::env; --use std::fs::create_dir_all; --use std::path::Path; --use std::path::PathBuf; -- --use libbpf_cargo::SkeletonBuilder; -- --const HEADER_PATH: &str = "src/bpf/atropos.h"; -- --fn bindgen_atropos() { -- // Tell cargo to invalidate the built crate whenever the wrapper changes -- println!("cargo:rerun-if-changed={}", HEADER_PATH); -- -- // The bindgen::Builder is the main entry point -- // to bindgen, and lets you build up options for -- // the resulting bindings. -- let bindings = bindgen::Builder::default() -- // The input header we would like to generate -- // bindings for. -- .header(HEADER_PATH) -- // Tell cargo to invalidate the built crate whenever any of the -- // included header files changed. -- .parse_callbacks(Box::new(bindgen::CargoCallbacks)) -- // Finish the builder and generate the bindings. -- .generate() -- // Unwrap the Result and panic on failure. -- .expect("Unable to generate bindings"); -- -- // Write the bindings to the $OUT_DIR/bindings.rs file. -- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); -- bindings -- .write_to_file(out_path.join("atropos-sys.rs")) -- .expect("Couldn't write bindings!"); --} -- --fn gen_bpf_sched(name: &str) { -- let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap(); -- let clang = env::var("ATROPOS_CLANG").unwrap(); -- eprintln!("{}", clang); -- let outpath = format!("./src/bpf/.output/{}.skel.rs", name); -- let skel = Path::new(&outpath); -- let src = format!("./src/bpf/{}.bpf.c", name); -- SkeletonBuilder::new() -- .source(src.clone()) -- .clang(clang) -- .clang_args(bpf_cflags) -- .build_and_generate(&skel) -- .unwrap(); -- println!("cargo:rerun-if-changed={}", src); --} -- --fn main() { -- bindgen_atropos(); -- // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton. -- // Reasons are because the generated skeleton contains compiler attributes -- // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]` -- // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside -- // the path attribute either (see https://github.com/rust-lang/rust/pull/83366). -- // -- // However, there is hope! When the above feature stabilizes we can clean this -- // all up. -- create_dir_all("./src/bpf/.output").unwrap(); -- gen_bpf_sched("atropos"); --} -diff --git a/tools/sched_ext/atropos/rustfmt.toml b/tools/sched_ext/atropos/rustfmt.toml -deleted file mode 100644 -index b7258ed0a8d8..000000000000 ---- a/tools/sched_ext/atropos/rustfmt.toml -+++ /dev/null -@@ -1,8 +0,0 @@ --# Get help on options with `rustfmt --help=config` --# Please keep these in alphabetical order. --edition = "2021" --group_imports = "StdExternalCrate" --imports_granularity = "Item" --merge_derives = false --use_field_init_shorthand = true --version = "Two" -diff --git a/tools/sched_ext/atropos/src/atropos_sys.rs b/tools/sched_ext/atropos/src/atropos_sys.rs -deleted file mode 100644 -index bbeaf856d40e..000000000000 ---- a/tools/sched_ext/atropos/src/atropos_sys.rs -+++ /dev/null -@@ -1,10 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#![allow(non_upper_case_globals)] --#![allow(non_camel_case_types)] --#![allow(non_snake_case)] --#![allow(dead_code)] -- --include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs")); -diff --git a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -deleted file mode 100644 -index 3905a403e940..000000000000 ---- a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -+++ /dev/null -@@ -1,743 +0,0 @@ --/* Copyright (c) Meta Platforms, Inc. and affiliates. */ --/* -- * This software may be used and distributed according to the terms of the -- * GNU General Public License version 2. -- * -- * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF -- * part does simple round robin in each domain and the userspace part -- * calculates the load factor of each domain and tells the BPF part how to load -- * balance the domains. -- * -- * Every task has an entry in the task_data map which lists which domain the -- * task belongs to. When a task first enters the system (atropos_prep_enable), -- * they are round-robined to a domain. -- * -- * atropos_select_cpu is the primary scheduling logic, invoked when a task -- * becomes runnable. The lb_data map is populated by userspace to inform the BPF -- * scheduler that a task should be migrated to a new domain. Otherwise, the task -- * is scheduled in priority order as follows: -- * * The current core if the task was woken up synchronously and there are idle -- * cpus in the system -- * * The previous core, if idle -- * * The pinned-to core if the task is pinned to a specific core -- * * Any idle cpu in the domain -- * -- * If none of the above conditions are met, then the task is enqueued to a -- * dispatch queue corresponding to the domain (atropos_enqueue). -- * -- * atropos_dispatch will attempt to consume a task from its domain's -- * corresponding dispatch queue (this occurs after scheduling any tasks directly -- * assigned to it due to the logic in atropos_select_cpu). If no task is found, -- * then greedy load stealing will attempt to find a task on another dispatch -- * queue to run. -- * -- * Load balancing is almost entirely handled by userspace. BPF populates the -- * task weight, dom mask and current dom in the task_data map and executes the -- * load balance based on userspace populating the lb_data map. -- */ --#include "../../../scx_common.bpf.h" --#include "atropos.h" -- --#include --#include --#include --#include --#include --#include -- --char _license[] SEC("license") = "GPL"; -- --/* -- * const volatiles are set during initialization and treated as consts by the -- * jit compiler. -- */ -- --/* -- * Domains and cpus -- */ --const volatile __u32 nr_doms = 32; /* !0 for veristat, set during init */ --const volatile __u32 nr_cpus = 64; /* !0 for veristat, set during init */ --const volatile __u32 cpu_dom_id_map[MAX_CPUS]; --const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; -- --const volatile bool kthreads_local; --const volatile bool fifo_sched; --const volatile bool switch_partial; --const volatile __u32 greedy_threshold; -- --/* base slice duration */ --const volatile __u64 slice_us = 20000; -- --/* -- * Exit info -- */ --int exit_type = SCX_EXIT_NONE; --char exit_msg[SCX_EXIT_MSG_LEN]; -- --struct pcpu_ctx { -- __u32 dom_rr_cur; /* used when scanning other doms */ -- -- /* libbpf-rs does not respect the alignment, so pad out the struct explicitly */ -- __u8 _padding[CACHELINE_SIZE - sizeof(u64)]; --} __attribute__((aligned(CACHELINE_SIZE))); -- --struct pcpu_ctx pcpu_ctx[MAX_CPUS]; -- --/* -- * Domain context -- */ --struct dom_ctx { -- struct bpf_cpumask __kptr *cpumask; -- u64 vtime_now; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __type(key, u32); -- __type(value, struct dom_ctx); -- __uint(max_entries, MAX_DOMS); -- __uint(map_flags, 0); --} dom_ctx SEC(".maps"); -- --/* -- * Statistics -- */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, ATROPOS_NR_STATS); --} stats SEC(".maps"); -- --static inline void stat_add(enum stat_idx idx, u64 addend) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p) += addend; --} -- --/* Map pid -> task_ctx */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, struct task_ctx); -- __uint(max_entries, 1000000); -- __uint(map_flags, 0); --} task_data SEC(".maps"); -- --/* -- * This is populated from userspace to indicate which pids should be reassigned -- * to new doms. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, u32); -- __uint(max_entries, 1000); -- __uint(map_flags, 0); --} lb_data SEC(".maps"); -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p, -- u32 new_dom_id) --{ -- struct dom_ctx *old_domc, *new_domc; -- struct bpf_cpumask *d_cpumask, *t_cpumask; -- u32 old_dom_id = task_ctx->dom_id; -- s64 vtime_delta; -- -- old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id); -- if (!old_domc) { -- scx_bpf_error("No dom%u", old_dom_id); -- return false; -- } -- -- vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now; -- -- new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id); -- if (!new_domc) { -- scx_bpf_error("No dom%u", new_dom_id); -- return false; -- } -- -- d_cpumask = new_domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to get domain %u cpumask kptr", -- new_dom_id); -- return false; -- } -- -- t_cpumask = task_ctx->cpumask; -- if (!t_cpumask) { -- scx_bpf_error("Failed to look up task cpumask"); -- return false; -- } -- -- /* -- * set_cpumask might have happened between userspace requesting LB and -- * here and @p might not be able to run in @dom_id anymore. Verify. -- */ -- if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- p->cpus_ptr)) { -- p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta; -- task_ctx->dom_id = new_dom_id; -- bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask, -- p->cpus_ptr); -- } -- -- return task_ctx->dom_id == new_dom_id; --} -- --s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu, -- u32 wake_flags) --{ -- s32 cpu; -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- struct bpf_cpumask *p_cpumask; -- -- if (!task_ctx) -- return -ENOENT; -- -- if (kthreads_local && -- (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- cpu = prev_cpu; -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local dsq of the waker. -- */ -- if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) { -- struct task_struct *current = (void *)bpf_get_current_task(); -- -- if (!(BPF_CORE_READ(current, flags) & PF_EXITING) && -- task_ctx->dom_id < MAX_DOMS) { -- struct dom_ctx *domc; -- struct bpf_cpumask *d_cpumask; -- const struct cpumask *idle_cpumask; -- bool has_idle; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id); -- if (!domc) { -- scx_bpf_error("Failed to find dom%u", -- task_ctx->dom_id); -- return prev_cpu; -- } -- d_cpumask = domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to acquire domain %u cpumask kptr", -- task_ctx->dom_id); -- return prev_cpu; -- } -- -- idle_cpumask = scx_bpf_get_idle_cpumask(); -- -- has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- idle_cpumask); -- -- scx_bpf_put_idle_cpumask(idle_cpumask); -- -- if (has_idle) { -- cpu = bpf_get_smp_processor_id(); -- if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- stat_add(ATROPOS_STAT_WAKE_SYNC, 1); -- goto local; -- } -- } -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- stat_add(ATROPOS_STAT_PREV_IDLE, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- /* If only one core is allowed, dispatch */ -- if (p->nr_cpus_allowed == 1) { -- stat_add(ATROPOS_STAT_PINNED, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) -- return -ENOENT; -- -- /* If there is an eligible idle CPU, dispatch directly */ -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- if (cpu >= 0) { -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * @prev_cpu may be in a different domain. Returning an out-of-domain -- * CPU can lead to stalls as all in-domain CPUs may be idle by the time -- * @p gets enqueued. -- */ -- if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask)) -- cpu = prev_cpu; -- else -- cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask); -- -- return cpu; -- --local: -- task_ctx->dispatch_local = true; -- return cpu; --} -- --void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- u32 *new_dom; -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- new_dom = bpf_map_lookup_elem(&lb_data, &pid); -- if (new_dom && *new_dom != task_ctx->dom_id && -- task_set_dsq(task_ctx, p, *new_dom)) { -- struct bpf_cpumask *p_cpumask; -- s32 cpu; -- -- stat_add(ATROPOS_STAT_LOAD_BALANCE, 1); -- -- /* -- * If dispatch_local is set, We own @p's idle state but we are -- * not gonna put the task in the associated local dsq which can -- * cause the CPU to stall. Kick it. -- */ -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0); -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) { -- scx_bpf_error("Failed to get task_ctx->cpumask"); -- return; -- } -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- } -- -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags); -- return; -- } -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000, -- enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- u32 dom_id = task_ctx->dom_id; -- struct dom_ctx *domc; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, domc->vtime_now - slice_us * 1000)) -- vtime = domc->vtime_now - slice_us * 1000; -- -- scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --static u32 cpu_to_dom_id(s32 cpu) --{ -- const volatile u32 *dom_idp; -- -- if (nr_doms <= 1) -- return 0; -- -- dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]); -- if (!dom_idp) -- return MAX_DOMS; -- -- return *dom_idp; --} -- --static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) --{ -- s32 cpu; -- -- if (dom_id >= MAX_DOMS) -- return false; -- -- bpf_for(cpu, 0, nr_cpus) { -- if (bpf_cpumask_test_cpu(cpu, cpumask) && -- (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64)))) -- return true; -- } -- return false; --} -- --static u32 dom_rr_next(s32 cpu) --{ -- struct pcpu_ctx *pcpuc; -- u32 dom_id; -- -- pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]); -- if (!pcpuc) -- return 0; -- -- dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms; -- -- if (dom_id == cpu_to_dom_id(cpu)) -- dom_id = (dom_id + 1) % nr_doms; -- -- pcpuc->dom_rr_cur = dom_id; -- return dom_id; --} -- --void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 dom = cpu_to_dom_id(cpu); -- -- if (scx_bpf_consume(dom)) { -- stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1); -- return; -- } -- -- if (!greedy_threshold) -- return; -- -- bpf_repeat(nr_doms - 1) { -- u32 dom_id = dom_rr_next(cpu); -- -- if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold && -- scx_bpf_consume(dom_id)) { -- stat_add(ATROPOS_STAT_GREEDY, 1); -- break; -- } -- } --} -- --void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_at = bpf_ktime_get_ns(); --} -- --void BPF_STRUCT_OPS(atropos_running, struct task_struct *p) --{ -- struct task_ctx *taskc; -- struct dom_ctx *domc; -- pid_t pid = p->pid; -- u32 dom_id; -- -- if (fifo_sched) -- return; -- -- taskc = bpf_map_lookup_elem(&task_data, &pid); -- if (!taskc) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- dom_id = taskc->dom_id; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(domc->vtime_now, p->scx.dsq_vtime)) -- domc->vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at; -- task_ctx->runnable_at = 0; --} -- --void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->weight = weight; --} -- --struct pick_task_domain_loop_ctx { -- struct task_struct *p; -- const struct cpumask *cpumask; -- u64 dom_mask; -- u32 dom_rr_base; -- u32 dom_id; --}; -- --static int pick_task_domain_loopfn(u32 idx, void *data) --{ -- struct pick_task_domain_loop_ctx *lctx = data; -- u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms; -- -- if (dom_id >= MAX_DOMS) -- return 1; -- -- if (cpumask_intersects_domain(lctx->cpumask, dom_id)) { -- lctx->dom_mask |= 1LLU << dom_id; -- if (lctx->dom_id == MAX_DOMS) -- lctx->dom_id = dom_id; -- } -- return 0; --} -- --static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- struct pick_task_domain_loop_ctx lctx = { -- .p = p, -- .cpumask = cpumask, -- .dom_id = MAX_DOMS, -- }; -- s32 cpu = bpf_get_smp_processor_id(); -- -- if (cpu < 0 || cpu >= MAX_CPUS) -- return MAX_DOMS; -- -- lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur); -- -- bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0); -- task_ctx->dom_mask = lctx.dom_mask; -- -- return lctx.dom_id; --} -- --static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- u32 dom_id = 0; -- -- if (nr_doms > 1) -- dom_id = pick_task_domain(task_ctx, p, cpumask); -- -- if (!task_set_dsq(task_ctx, p, dom_id)) -- scx_bpf_error("Failed to set domain %d for %s[%d]", -- dom_id, p->comm, p->pid); --} -- --void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_set_domain(task_ctx, p, cpumask); --} -- --s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct bpf_cpumask *cpumask; -- struct task_ctx task_ctx, *map_value; -- long ret; -- pid_t pid; -- -- memset(&task_ctx, 0, sizeof(task_ctx)); -- -- pid = p->pid; -- ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return ret; -- } -- -- /* -- * Read the entry from the map immediately so we can add the cpumask -- * with bpf_kptr_xchg(). -- */ -- map_value = bpf_map_lookup_elem(&task_data, &pid); -- if (!map_value) -- /* Should never happen -- it was just inserted above. */ -- return -EINVAL; -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- bpf_map_delete_elem(&task_data, &pid); -- return -ENOMEM; -- } -- -- cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask); -- if (cpumask) { -- /* Should never happen as we just inserted it above. */ -- bpf_cpumask_release(cpumask); -- bpf_map_delete_elem(&task_data, &pid); -- return -EINVAL; -- } -- -- task_set_domain(map_value, p, p->cpus_ptr); -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p) --{ -- pid_t pid = p->pid; -- long ret = bpf_map_delete_elem(&task_data, &pid); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return; -- } --} -- --static int create_dom_dsq(u32 idx, void *data) --{ -- struct dom_ctx domc_init = {}, *domc; -- struct bpf_cpumask *cpumask; -- u32 cpu, dom_id = idx; -- s32 ret; -- -- ret = scx_bpf_create_dsq(dom_id, -1); -- if (ret < 0) { -- scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret); -- return 1; -- } -- -- ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0); -- if (ret) { -- scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret); -- return 1; -- } -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- /* Should never happen, we just inserted it above. */ -- scx_bpf_error("No dom%u", dom_id); -- return 1; -- } -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id); -- return 1; -- } -- -- for (cpu = 0; cpu < MAX_CPUS; cpu++) { -- const volatile __u64 *dmask; -- -- dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]); -- if (!dmask) { -- scx_bpf_error("array index error"); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- if (*dmask & (1LLU << (cpu % 64))) -- bpf_cpumask_set_cpu(cpu, cpumask); -- } -- -- cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask); -- if (cpumask) { -- scx_bpf_error("Domain %u was already present", dom_id); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(atropos_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- bpf_loop(nr_doms, create_dom_dsq, NULL, 0); -- -- for (u32 i = 0; i < nr_cpus; i++) -- pcpu_ctx[i].dom_rr_cur = i; -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg); -- exit_type = ei->type; --} -- --SEC(".struct_ops") --struct sched_ext_ops atropos = { -- .select_cpu = (void *)atropos_select_cpu, -- .enqueue = (void *)atropos_enqueue, -- .dispatch = (void *)atropos_dispatch, -- .runnable = (void *)atropos_runnable, -- .running = (void *)atropos_running, -- .stopping = (void *)atropos_stopping, -- .quiescent = (void *)atropos_quiescent, -- .set_weight = (void *)atropos_set_weight, -- .set_cpumask = (void *)atropos_set_cpumask, -- .prep_enable = (void *)atropos_prep_enable, -- .disable = (void *)atropos_disable, -- .init = (void *)atropos_init, -- .exit = (void *)atropos_exit, -- .flags = 0, -- .name = "atropos", --}; -diff --git a/tools/sched_ext/atropos/src/bpf/atropos.h b/tools/sched_ext/atropos/src/bpf/atropos.h -deleted file mode 100644 -index addf29ca104a..000000000000 ---- a/tools/sched_ext/atropos/src/bpf/atropos.h -+++ /dev/null -@@ -1,44 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#ifndef __ATROPOS_H --#define __ATROPOS_H -- --#include --#ifndef __kptr --#ifdef __KERNEL__ --#error "__kptr_ref not defined in the kernel" --#endif --#define __kptr --#endif -- --#define MAX_CPUS 512 --#define MAX_DOMS 64 /* limited to avoid complex bitmask ops */ --#define CACHELINE_SIZE 64 -- --/* Statistics */ --enum stat_idx { -- ATROPOS_STAT_TASK_GET_ERR, -- ATROPOS_STAT_WAKE_SYNC, -- ATROPOS_STAT_PREV_IDLE, -- ATROPOS_STAT_PINNED, -- ATROPOS_STAT_DIRECT_DISPATCH, -- ATROPOS_STAT_DSQ_DISPATCH, -- ATROPOS_STAT_GREEDY, -- ATROPOS_STAT_LOAD_BALANCE, -- ATROPOS_STAT_LAST_TASK, -- ATROPOS_NR_STATS, --}; -- --struct task_ctx { -- unsigned long long dom_mask; /* the domains this task can run on */ -- struct bpf_cpumask __kptr *cpumask; -- unsigned int dom_id; -- unsigned int weight; -- unsigned long long runnable_at; -- unsigned long long runnable_for; -- bool dispatch_local; --}; -- --#endif /* __ATROPOS_H */ -diff --git a/tools/sched_ext/atropos/src/main.rs b/tools/sched_ext/atropos/src/main.rs -deleted file mode 100644 -index 0d313662f713..000000000000 ---- a/tools/sched_ext/atropos/src/main.rs -+++ /dev/null -@@ -1,942 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#[path = "bpf/.output/atropos.skel.rs"] --mod atropos; --pub use atropos::*; --pub mod atropos_sys; -- --use std::cell::Cell; --use std::collections::{BTreeMap, BTreeSet}; --use std::ffi::CStr; --use std::ops::Bound::{Included, Unbounded}; --use std::sync::atomic::{AtomicBool, Ordering}; --use std::sync::Arc; --use std::time::{Duration, SystemTime}; -- --use ::fb_procfs as procfs; --use anyhow::{anyhow, bail, Context, Result}; --use bitvec::prelude::*; --use clap::Parser; --use log::{info, trace, warn}; --use ordered_float::OrderedFloat; -- --/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF --/// part does simple round robin in each domain and the userspace part --/// calculates the load factor of each domain and tells the BPF part how to load --/// balance the domains. -- --/// This scheduler demonstrates dividing scheduling logic between BPF and --/// userspace and using rust to build the userspace part. An earlier variant of --/// this scheduler was used to balance across six domains, each representing a --/// chiplet in a six-chiplet AMD processor, and could match the performance of --/// production setup using CFS. --#[derive(Debug, Parser)] --struct Opts { -- /// Scheduling slice duration in microseconds. -- #[clap(short, long, default_value = "20000")] -- slice_us: u64, -- -- /// Monitoring and load balance interval in seconds. -- #[clap(short, long, default_value = "2.0")] -- interval: f64, -- -- /// Build domains according to how CPUs are grouped at this cache level -- /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id. -- #[clap(short = 'c', long, default_value = "3")] -- cache_level: u32, -- -- /// Instead of using cache locality, set the cpumask for each domain -- /// manually, provide multiple --cpumasks, one for each domain. E.g. -- /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with -- /// the corresponding CPUs belonging to each domain. Each CPU must -- /// belong to precisely one domain. -- #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")] -- cpumasks: Vec, -- -- /// When non-zero, enable greedy task stealing. When a domain is idle, a -- /// cpu will attempt to steal tasks from a domain with at least -- /// greedy_threshold tasks enqueued. These tasks aren't permanently -- /// stolen from the domain. -- #[clap(short, long, default_value = "4")] -- greedy_threshold: u32, -- -- /// The load decay factor. Every interval, the existing load is decayed -- /// by this factor and new load is added. Must be in the range [0.0, -- /// 0.99]. The smaller the value, the more sensitive load calculation -- /// is to recent changes. When 0.0, history is ignored and the load -- /// value from the latest period is used directly. -- #[clap(short, long, default_value = "0.5")] -- load_decay_factor: f64, -- -- /// Disable load balancing. Unless disabled, periodically userspace will -- /// calculate the load factor of each domain and instruct BPF which -- /// processes to move. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- no_load_balance: bool, -- -- /// Put per-cpu kthreads directly into local dsq's. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- kthreads_local: bool, -- -- /// Use FIFO scheduling instead of weighted vtime scheduling. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- fifo_sched: bool, -- -- /// If specified, only tasks which have their scheduling policy set to -- /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all -- /// tasks are switched. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- partial: bool, -- -- /// Enable verbose output including libbpf details. Specify multiple -- /// times to increase verbosity. -- #[clap(short, long, action = clap::ArgAction::Count)] -- verbose: u8, --} -- --fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result { -- Ok(reader -- .read_stat() -- .context("Failed to read procfs")? -- .total_cpu -- .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?) --} -- --fn now_monotonic() -> u64 { -- let mut time = libc::timespec { -- tv_sec: 0, -- tv_nsec: 0, -- }; -- let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) }; -- assert!(ret == 0); -- time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 --} -- --fn clear_map(map: &mut libbpf_rs::Map) { -- // XXX: libbpf_rs has some design flaw that make it impossible to -- // delete while iterating despite it being safe so we alias it here -- let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) }; -- for key in map.keys() { -- let _ = deleter.delete(&key); -- } --} -- --#[derive(Debug)] --struct TaskLoad { -- runnable_for: u64, -- load: f64, --} -- --#[derive(Debug)] --struct TaskInfo { -- pid: i32, -- dom_mask: u64, -- migrated: Cell, --} -- --struct LoadBalancer<'a, 'b, 'c> { -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- -- tasks_by_load: Vec, TaskInfo>>, -- load_avg: f64, -- dom_loads: Vec, -- -- imbal: Vec, -- doms_to_push: BTreeMap, u32>, -- doms_to_pull: BTreeMap, u32>, -- -- nr_lb_data_errors: &'c mut u64, --} -- --impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> { -- const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10; -- const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1; -- const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50; -- -- fn new( -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- nr_lb_data_errors: &'c mut u64, -- ) -> Self { -- Self { -- maps, -- task_loads, -- nr_doms, -- load_decay_factor, -- -- tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(), -- load_avg: 0f64, -- dom_loads: vec![0.0; nr_doms], -- -- imbal: vec![0.0; nr_doms], -- doms_to_pull: BTreeMap::new(), -- doms_to_push: BTreeMap::new(), -- -- nr_lb_data_errors, -- } -- } -- -- fn read_task_loads(&mut self, period: Duration) -> Result<()> { -- let now_mono = now_monotonic(); -- let task_data = self.maps.task_data(); -- let mut this_task_loads = BTreeMap::::new(); -- let mut load_sum = 0.0f64; -- self.dom_loads = vec![0f64; self.nr_doms]; -- -- for key in task_data.keys() { -- if let Some(task_ctx_vec) = task_data -- .lookup(&key, libbpf_rs::MapFlags::ANY) -- .context("Failed to lookup task_data")? -- { -- let task_ctx = -- unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) }; -- let pid = i32::from_ne_bytes( -- key.as_slice() -- .try_into() -- .context("Invalid key length in task_data map")?, -- ); -- -- let (this_at, this_for, weight) = unsafe { -- ( -- std::ptr::read_volatile(&task_ctx.runnable_at as *const u64), -- std::ptr::read_volatile(&task_ctx.runnable_for as *const u64), -- std::ptr::read_volatile(&task_ctx.weight as *const u32), -- ) -- }; -- -- let (mut delta, prev_load) = match self.task_loads.get(&pid) { -- Some(prev) => (this_for - prev.runnable_for, Some(prev.load)), -- None => (this_for, None), -- }; -- -- // Non-zero this_at indicates that the task is currently -- // runnable. Note that we read runnable_at and runnable_for -- // without any synchronization and there is a small window -- // where we end up misaccounting. While this can cause -- // temporary error, it's unlikely to cause any noticeable -- // misbehavior especially given the load value clamping. -- if this_at > 0 && this_at < now_mono { -- delta += now_mono - this_at; -- } -- -- delta = delta.min(period.as_nanos() as u64); -- let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64) -- .clamp(0.0, weight as f64); -- -- let this_load = match prev_load { -- Some(prev_load) => { -- prev_load * self.load_decay_factor -- + this_load * (1.0 - self.load_decay_factor) -- } -- None => this_load, -- }; -- -- this_task_loads.insert( -- pid, -- TaskLoad { -- runnable_for: this_for, -- load: this_load, -- }, -- ); -- -- load_sum += this_load; -- self.dom_loads[task_ctx.dom_id as usize] += this_load; -- // Only record pids that are eligible for load balancing -- if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) { -- continue; -- } -- self.tasks_by_load[task_ctx.dom_id as usize].insert( -- OrderedFloat(this_load), -- TaskInfo { -- pid, -- dom_mask: task_ctx.dom_mask, -- migrated: Cell::new(false), -- }, -- ); -- } -- } -- -- self.load_avg = load_sum / self.nr_doms as f64; -- *self.task_loads = this_task_loads; -- Ok(()) -- } -- -- // To balance dom loads we identify doms with lower and higher load than average -- fn calculate_dom_load_balance(&mut self) -> Result<()> { -- for (dom, dom_load) in self.dom_loads.iter().enumerate() { -- let imbal = dom_load - self.load_avg; -- if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO { -- if imbal > 0f64 { -- self.doms_to_push.insert(OrderedFloat(imbal), dom as u32); -- } else { -- self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32); -- } -- self.imbal[dom] = imbal; -- } -- } -- Ok(()) -- } -- -- // Find the first candidate pid which hasn't already been migrated and -- // can run in @pull_dom. -- fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)> -- where -- I: IntoIterator, &'d TaskInfo)>, -- { -- match tasks_by_load -- .into_iter() -- .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0) -- .next() -- { -- Some((OrderedFloat(load), task)) => Some((*load, task)), -- None => None, -- } -- } -- -- fn pick_victim( -- &self, -- (push_dom, to_push): (u32, f64), -- (pull_dom, to_pull): (u32, f64), -- ) -> Option<(&TaskInfo, f64)> { -- let to_xfer = to_pull.min(to_push); -- -- trace!( -- "considering dom {}@{:.2} -> {}@{:.2}", -- push_dom, -- to_push, -- pull_dom, -- to_pull -- ); -- -- let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs(); -- -- trace!( -- "to_xfer={:.2} tasks_by_load={:?}", -- to_xfer, -- &self.tasks_by_load[push_dom as usize] -- ); -- -- // We want to pick a task to transfer from push_dom to pull_dom to -- // maximize the reduction of load imbalance between the two. IOW, -- // pick a task which has the closest load value to $to_xfer that can -- // be migrated. Find such task by locating the first migratable task -- // while scanning left from $to_xfer and the counterpart while -- // scanning right and picking the better of the two. -- let (load, task, new_imbal) = match ( -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Unbounded, Included(&OrderedFloat(to_xfer)))) -- .rev(), -- pull_dom, -- ), -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Included(&OrderedFloat(to_xfer)), Unbounded)), -- pull_dom, -- ), -- ) { -- (None, None) => return None, -- (Some((load, task)), None) | (None, Some((load, task))) => { -- (load, task, calc_new_imbal(load)) -- } -- (Some((load0, task0)), Some((load1, task1))) => { -- let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1)); -- if new_imbal0 <= new_imbal1 { -- (load0, task0, new_imbal0) -- } else { -- (load1, task1, new_imbal1) -- } -- } -- }; -- -- // If the best candidate can't reduce the imbalance, there's nothing -- // to do for this pair. -- let old_imbal = to_push + to_pull; -- if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal { -- trace!( -- "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal -- ); -- return None; -- } -- -- trace!( -- "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal, -- ); -- -- Some((task, load)) -- } -- -- // Actually execute the load balancing. Concretely this writes pid -> dom -- // entries into the lb_data map for bpf side to consume. -- fn load_balance(&mut self) -> Result<()> { -- clear_map(self.maps.lb_data()); -- -- trace!("imbal={:?}", &self.imbal); -- trace!("doms_to_push={:?}", &self.doms_to_push); -- trace!("doms_to_pull={:?}", &self.doms_to_pull); -- -- // Push from the most imbalanced to least. -- while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() { -- let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO; -- let mut pushed = 0f64; -- -- // Transfer tasks from push_dom to reduce imbalance. -- loop { -- let last_pushed = pushed; -- -- // Pull from the most imbalaned to least. -- let mut doms_to_pull = BTreeMap::<_, _>::new(); -- std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull); -- let mut pull_doms = doms_to_pull.into_iter().rev().collect::>(); -- -- for (to_pull, pull_dom) in pull_doms.iter_mut() { -- if let Some((task, load)) = -- self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull))) -- { -- // Execute migration. -- task.migrated.set(true); -- to_push -= load; -- *to_pull -= load; -- pushed += load; -- -- // Ask BPF code to execute the migration. -- let pid = task.pid; -- let cpid = (pid as libc::pid_t).to_ne_bytes(); -- if let Err(e) = self.maps.lb_data().update( -- &cpid, -- &pull_dom.to_ne_bytes(), -- libbpf_rs::MapFlags::NO_EXIST, -- ) { -- warn!( -- "Failed to update lb_data map for pid={} error={:?}", -- pid, &e -- ); -- *self.nr_lb_data_errors += 1; -- } -- -- // Always break after a successful migration so that -- // the pulling domains are always considered in the -- // descending imbalance order. -- break; -- } -- } -- -- pull_doms -- .into_iter() -- .map(|(k, v)| self.doms_to_pull.insert(k, v)) -- .count(); -- -- // Stop repeating if nothing got transferred or pushed enough. -- if pushed == last_pushed || pushed >= push_max { -- break; -- } -- } -- } -- Ok(()) -- } --} -- --struct Scheduler<'a> { -- skel: AtroposSkel<'a>, -- struct_ops: Option, -- -- nr_cpus: usize, -- nr_doms: usize, -- load_decay_factor: f64, -- balance_load: bool, -- -- proc_reader: procfs::ProcReader, -- -- prev_at: SystemTime, -- prev_total_cpu: procfs::CpuStat, -- task_loads: BTreeMap, -- -- nr_lb_data_errors: u64, --} -- --impl<'a> Scheduler<'a> { -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn parse_cpusets( -- cpumasks: &[String], -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- if cpumasks.len() > atropos_sys::MAX_DOMS as usize { -- bail!( -- "Number of requested DSQs ({}) is greater than MAX_DOMS ({})", -- cpumasks.len(), -- atropos_sys::MAX_DOMS -- ); -- } -- let mut cpus = vec![-1i32; nr_cpus]; -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()]; -- for (dq, cpumask) in cpumasks.iter().enumerate() { -- let hex_str = { -- let mut tmp_str = cpumask -- .strip_prefix("0x") -- .unwrap_or(cpumask) -- .replace('_', ""); -- if tmp_str.len() % 2 != 0 { -- tmp_str = "0".to_string() + &tmp_str; -- } -- tmp_str -- }; -- let byte_vec = hex::decode(&hex_str) -- .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?; -- -- for (index, &val) in byte_vec.iter().rev().enumerate() { -- let mut v = val; -- while v != 0 { -- let lsb = v.trailing_zeros() as usize; -- v &= !(1 << lsb); -- let cpu = index * 8 + lsb; -- if cpu > nr_cpus { -- bail!( -- concat!( -- "Found cpu ({}) in cpumask ({}) which is larger", -- " than the number of cpus on the machine ({})" -- ), -- cpu, -- cpumask, -- nr_cpus -- ); -- } -- if cpus[cpu] != -1 { -- bail!( -- "Found cpu ({}) with dq ({}) but also in cpumask ({})", -- cpu, -- cpus[cpu], -- cpumask -- ); -- } -- cpus[cpu] = dq as i32; -- cpusets[dq].set(cpu, true); -- } -- } -- cpusets[dq].set_uninitialized(false); -- } -- -- for (cpu, &dq) in cpus.iter().enumerate() { -- if dq < 0 { -- bail!( -- "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.", -- cpu -- ); -- } -- } -- -- Ok((cpusets, cpus)) -- } -- -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn cpusets_from_cache( -- level: u32, -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- let mut cpu_to_cache = vec![]; // (cpu_id, cache_id) -- let mut cache_ids = BTreeSet::::new(); -- let mut nr_not_found = 0; -- -- // Build cpu -> cache ID mapping. -- for cpu in 0..nr_cpus { -- let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level); -- let id = match std::fs::read_to_string(&path) { -- Ok(val) => val -- .trim() -- .parse::() -- .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?, -- Err(e) if e.kind() == std::io::ErrorKind::NotFound => { -- nr_not_found += 1; -- 0 -- } -- Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)), -- }; -- -- cpu_to_cache.push(id); -- cache_ids.insert(id); -- } -- -- if nr_not_found > 1 { -- warn!( -- "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0", -- level, nr_not_found, nr_cpus -- ); -- } -- -- // Cache IDs may have holes. Assign consecutive domain IDs to -- // existing cache IDs. -- let mut cache_to_dom = BTreeMap::::new(); -- let mut nr_doms = 0; -- for cache_id in cache_ids.iter() { -- cache_to_dom.insert(*cache_id, nr_doms); -- nr_doms += 1; -- } -- -- if nr_doms > atropos_sys::MAX_DOMS { -- bail!( -- "Total number of doms {} is greater than MAX_DOMS ({})", -- nr_doms, -- atropos_sys::MAX_DOMS -- ); -- } -- -- // Build and return dom -> cpumask and cpu -> dom mappings. -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize]; -- let mut cpu_to_dom = vec![]; -- -- for cpu in 0..nr_cpus { -- let dom_id = cache_to_dom[&cpu_to_cache[cpu]]; -- cpusets[dom_id as usize].set(cpu, true); -- cpu_to_dom.push(dom_id as i32); -- } -- -- Ok((cpusets, cpu_to_dom)) -- } -- -- fn init(opts: &Opts) -> Result { -- // Open the BPF prog first for verification. -- let mut skel_builder = AtroposSkelBuilder::default(); -- skel_builder.obj_builder.debug(opts.verbose > 0); -- let mut skel = skel_builder.open().context("Failed to open BPF program")?; -- -- let nr_cpus = libbpf_rs::num_possible_cpus().unwrap(); -- if nr_cpus > atropos_sys::MAX_CPUS as usize { -- bail!( -- "nr_cpus ({}) is greater than MAX_CPUS ({})", -- nr_cpus, -- atropos_sys::MAX_CPUS -- ); -- } -- -- // Initialize skel according to @opts. -- let (cpusets, cpus) = if opts.cpumasks.len() > 0 { -- Self::parse_cpusets(&opts.cpumasks, nr_cpus)? -- } else { -- Self::cpusets_from_cache(opts.cache_level, nr_cpus)? -- }; -- let nr_doms = cpusets.len(); -- skel.rodata().nr_doms = nr_doms as u32; -- skel.rodata().nr_cpus = nr_cpus as u32; -- -- for (cpu, dom) in cpus.iter().enumerate() { -- skel.rodata().cpu_dom_id_map[cpu] = *dom as u32; -- } -- -- for (dom, cpuset) in cpusets.iter().enumerate() { -- let raw_cpuset_slice = cpuset.as_raw_slice(); -- let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom]; -- let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len()); -- left.clone_from_slice(cpuset.as_raw_slice()); -- let cpumask_str = dom_cpumask_slice -- .iter() -- .take((nr_cpus + 63) / 64) -- .rev() -- .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x)); -- info!( -- "DOM[{:02}] cpumask{} ({} cpus)", -- dom, -- &cpumask_str, -- cpuset.count_ones() -- ); -- } -- -- skel.rodata().slice_us = opts.slice_us; -- skel.rodata().kthreads_local = opts.kthreads_local; -- skel.rodata().fifo_sched = opts.fifo_sched; -- skel.rodata().switch_partial = opts.partial; -- skel.rodata().greedy_threshold = opts.greedy_threshold; -- -- // Attach. -- let mut skel = skel.load().context("Failed to load BPF program")?; -- skel.attach().context("Failed to attach BPF program")?; -- let struct_ops = Some( -- skel.maps_mut() -- .atropos() -- .attach_struct_ops() -- .context("Failed to attach atropos struct ops")?, -- ); -- info!("Atropos Scheduler Attached"); -- -- // Other stuff. -- let mut proc_reader = procfs::ProcReader::new(); -- let prev_total_cpu = read_total_cpu(&mut proc_reader)?; -- -- Ok(Self { -- skel, -- struct_ops, // should be held to keep it attached -- -- nr_cpus, -- nr_doms, -- load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99), -- balance_load: !opts.no_load_balance, -- -- proc_reader, -- -- prev_at: SystemTime::now(), -- prev_total_cpu, -- task_loads: BTreeMap::new(), -- -- nr_lb_data_errors: 0, -- }) -- } -- -- fn get_cpu_busy(&mut self) -> Result { -- let total_cpu = read_total_cpu(&mut self.proc_reader)?; -- let busy = match (&self.prev_total_cpu, &total_cpu) { -- ( -- procfs::CpuStat { -- user_usec: Some(prev_user), -- nice_usec: Some(prev_nice), -- system_usec: Some(prev_system), -- idle_usec: Some(prev_idle), -- iowait_usec: Some(prev_iowait), -- irq_usec: Some(prev_irq), -- softirq_usec: Some(prev_softirq), -- stolen_usec: Some(prev_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- procfs::CpuStat { -- user_usec: Some(curr_user), -- nice_usec: Some(curr_nice), -- system_usec: Some(curr_system), -- idle_usec: Some(curr_idle), -- iowait_usec: Some(curr_iowait), -- irq_usec: Some(curr_irq), -- softirq_usec: Some(curr_softirq), -- stolen_usec: Some(curr_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- ) => { -- let idle_usec = curr_idle - prev_idle; -- let iowait_usec = curr_iowait - prev_iowait; -- let user_usec = curr_user - prev_user; -- let system_usec = curr_system - prev_system; -- let nice_usec = curr_nice - prev_nice; -- let irq_usec = curr_irq - prev_irq; -- let softirq_usec = curr_softirq - prev_softirq; -- let stolen_usec = curr_stolen - prev_stolen; -- -- let busy_usec = -- user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec; -- let total_usec = idle_usec + busy_usec + iowait_usec; -- busy_usec as f64 / total_usec as f64 -- } -- _ => { -- bail!("Some procfs stats are not populated!"); -- } -- }; -- -- self.prev_total_cpu = total_cpu; -- Ok(busy) -- } -- -- fn read_bpf_stats(&mut self) -> Result> { -- let mut maps = self.skel.maps_mut(); -- let stats_map = maps.stats(); -- let mut stats: Vec = Vec::new(); -- let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus]; -- -- for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS { -- let cpu_stat_vec = stats_map -- .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY) -- .with_context(|| format!("Failed to lookup stat {}", stat))? -- .expect("per-cpu stat should exist"); -- let sum = cpu_stat_vec -- .iter() -- .map(|val| { -- u64::from_ne_bytes( -- val.as_slice() -- .try_into() -- .expect("Invalid value length in stat map"), -- ) -- }) -- .sum(); -- stats_map -- .update_percpu( -- &(stat as u32).to_ne_bytes(), -- &zero_vec, -- libbpf_rs::MapFlags::ANY, -- ) -- .context("Failed to zero stat")?; -- stats.push(sum); -- } -- Ok(stats) -- } -- -- fn report( -- &self, -- stats: &Vec, -- cpu_busy: f64, -- processing_dur: Duration, -- load_avg: f64, -- dom_loads: &Vec, -- imbal: &Vec, -- ) { -- let stat = |idx| stats[idx as usize]; -- let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK); -- -- info!( -- "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms", -- cpu_busy * 100.0, -- load_avg, -- stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize], -- stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize], -- self.nr_lb_data_errors, -- processing_dur.as_millis(), -- ); -- -- let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0; -- -- info!( -- "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}", -- total, -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY), -- ); -- -- for i in 0..self.nr_doms { -- info!( -- "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}", -- i, -- dom_loads[i], -- if imbal[i] < 0.0 { -imbal[i] } else { 0.0 }, -- if imbal[i] > 0.0 { imbal[i] } else { 0.0 }, -- ); -- } -- } -- -- fn step(&mut self) -> Result<()> { -- let started_at = std::time::SystemTime::now(); -- let bpf_stats = self.read_bpf_stats()?; -- let cpu_busy = self.get_cpu_busy()?; -- -- let mut lb = LoadBalancer::new( -- self.skel.maps_mut(), -- &mut self.task_loads, -- self.nr_doms, -- self.load_decay_factor, -- &mut self.nr_lb_data_errors, -- ); -- -- lb.read_task_loads(started_at.duration_since(self.prev_at)?)?; -- lb.calculate_dom_load_balance()?; -- -- if self.balance_load { -- lb.load_balance()?; -- } -- -- // Extract fields needed for reporting and drop lb to release -- // mutable borrows. -- let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal); -- -- self.report( -- &bpf_stats, -- cpu_busy, -- std::time::SystemTime::now().duration_since(started_at)?, -- load_avg, -- &dom_loads, -- &imbal, -- ); -- -- self.prev_at = started_at; -- Ok(()) -- } -- -- fn read_bpf_exit_type(&mut self) -> i32 { -- unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) } -- } -- -- fn report_bpf_exit_type(&mut self) -> Result<()> { -- // Report msg if EXT_OPS_EXIT_ERROR. -- match self.read_bpf_exit_type() { -- 0 => Ok(()), -- etype if etype == 2 => { -- let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) }; -- let msg = cstr -- .to_str() -- .context("Failed to convert exit msg to string") -- .unwrap(); -- bail!("BPF exit_type={} msg={}", etype, msg); -- } -- etype => { -- info!("BPF exit_type={}", etype); -- Ok(()) -- } -- } -- } --} -- --impl<'a> Drop for Scheduler<'a> { -- fn drop(&mut self) { -- if let Some(struct_ops) = self.struct_ops.take() { -- drop(struct_ops); -- } -- } --} -- --fn main() -> Result<()> { -- let opts = Opts::parse(); -- -- let llv = match opts.verbose { -- 0 => simplelog::LevelFilter::Info, -- 1 => simplelog::LevelFilter::Debug, -- _ => simplelog::LevelFilter::Trace, -- }; -- let mut lcfg = simplelog::ConfigBuilder::new(); -- lcfg.set_time_level(simplelog::LevelFilter::Error) -- .set_location_level(simplelog::LevelFilter::Off) -- .set_target_level(simplelog::LevelFilter::Off) -- .set_thread_level(simplelog::LevelFilter::Off); -- simplelog::TermLogger::init( -- llv, -- lcfg.build(), -- simplelog::TerminalMode::Stderr, -- simplelog::ColorChoice::Auto, -- )?; -- -- let shutdown = Arc::new(AtomicBool::new(false)); -- let shutdown_clone = shutdown.clone(); -- ctrlc::set_handler(move || { -- shutdown_clone.store(true, Ordering::Relaxed); -- }) -- .context("Error setting Ctrl-C handler")?; -- -- let mut sched = Scheduler::init(&opts)?; -- -- while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 { -- std::thread::sleep(Duration::from_secs_f64(opts.interval)); -- sched.step()?; -- } -- -- sched.report_bpf_exit_type() --} -diff --git a/tools/sched_ext/gnu/stubs.h b/tools/sched_ext/gnu/stubs.h -deleted file mode 100644 -index 719225b16626..000000000000 ---- a/tools/sched_ext/gnu/stubs.h -+++ /dev/null -@@ -1 +0,0 @@ --/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ -diff --git a/tools/sched_ext/scx_common.bpf.h b/tools/sched_ext/scx_common.bpf.h -deleted file mode 100644 -index e56de9dc86f2..000000000000 ---- a/tools/sched_ext/scx_common.bpf.h -+++ /dev/null -@@ -1,288 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __SCHED_EXT_COMMON_BPF_H --#define __SCHED_EXT_COMMON_BPF_H -- --#include "vmlinux.h" --#include --#include --#include --#include "user_exit_info.h" -- --#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ --#define PF_EXITING 0x00000004 --#define CLOCK_MONOTONIC 1 -- --/* -- * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can -- * lead to really confusing misbehaviors. Let's trigger a build failure. -- */ --static inline void ___vmlinux_h_sanity_check___(void) --{ -- _Static_assert(SCX_DSQ_FLAG_BUILTIN, -- "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); --} -- --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; -- --static inline __attribute__((format(printf, 1, 2))) --void ___scx_bpf_error_format_checker(const char *fmt, ...) {} -- --/* -- * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments -- * instead of an array of u64. Note that __param[] must have at least one -- * element to keep the verifier happy. -- */ --#define scx_bpf_error(fmt, args...) \ --({ \ -- static char ___fmt[] = fmt; \ -- unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ -- \ -- _Pragma("GCC diagnostic push") \ -- _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ -- ___bpf_fill(___param, args); \ -- _Pragma("GCC diagnostic pop") \ -- \ -- scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ -- \ -- ___scx_bpf_error_format_checker(fmt, ##args); \ --}) -- --void scx_bpf_switch_all(void) __ksym; --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; --bool scx_bpf_consume(u64 dsq_id) __ksym; --u32 scx_bpf_dispatch_nr_slots(void) __ksym; --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym; --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym; --void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; --s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym; --const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; --const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; --void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; --void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; --bool scx_bpf_task_running(const struct task_struct *p) __ksym; --s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym; --u32 scx_bpf_reenqueue_local(void) __ksym; -- --#define BPF_STRUCT_OPS(name, args...) \ --SEC("struct_ops/"#name) \ --BPF_PROG(name, ##args) -- --#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ --SEC("struct_ops.s/"#name) \ --BPF_PROG(name, ##args) -- --/** -- * MEMBER_VPTR - Obtain the verified pointer to a struct or array member -- * @base: struct or array to index -- * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...) -- * -- * The verifier often gets confused by the instruction sequence the compiler -- * generates for indexing struct fields or arrays. This macro forces the -- * compiler to generate a code sequence which first calculates the byte offset, -- * checks it against the struct or array size and add that byte offset to -- * generate the pointer to the member to help the verifier. -- * -- * Ideally, we want to abort if the calculated offset is out-of-bounds. However, -- * BPF currently doesn't support abort, so evaluate to NULL instead. The caller -- * must check for NULL and take appropriate action to appease the verifier. To -- * avoid confusing the verifier, it's best to check for NULL and dereference -- * immediately. -- * -- * vptr = MEMBER_VPTR(my_array, [i][j]); -- * if (!vptr) -- * return error; -- * *vptr = new_value; -- */ --#define MEMBER_VPTR(base, member) (typeof(base member) *)({ \ -- u64 __base = (u64)base; \ -- u64 __addr = (u64)&(base member) - __base; \ -- asm volatile ( \ -- "if %0 <= %[max] goto +2\n" \ -- "%0 = 0\n" \ -- "goto +1\n" \ -- "%0 += %1\n" \ -- : "+r"(__addr) \ -- : "r"(__base), \ -- [max]"i"(sizeof(base) - sizeof(base member))); \ -- __addr; \ --}) -- --/* -- * BPF core and other generic helpers -- */ -- --/* list and rbtree */ --#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) --#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) -- --void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; --void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; -- --#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) --#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) -- --void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; --struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; --struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, -- struct bpf_rb_node *node) __ksym; --void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, -- bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym; --struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; -- --/* task */ --struct task_struct *bpf_task_from_pid(s32 pid) __ksym; --struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; --void bpf_task_release(struct task_struct *p) __ksym; -- --/* cgroup */ --struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; --void bpf_cgroup_release(struct cgroup *cgrp) __ksym; --struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; -- --/* cpumask */ --struct bpf_cpumask *bpf_cpumask_create(void) __ksym; --struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; --u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; --u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym; -- --/* rcu */ --void bpf_rcu_read_lock(void) __ksym; --void bpf_rcu_read_unlock(void) __ksym; -- --/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */ --struct bpf_iter_num; -- --extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym; --extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym; --extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym; -- --#ifndef bpf_for_each --/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for -- * using BPF open-coded iterators without having to write mundane explicit -- * low-level loop logic. Instead, it provides for()-like generic construct -- * that can be used pretty naturally. E.g., for some hypothetical cgroup -- * iterator, you'd write: -- * -- * struct cgroup *cg, *parent_cg = <...>; -- * -- * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { -- * bpf_printk("Child cgroup id = %d", cg->cgroup_id); -- * if (cg->cgroup_id == 123) -- * break; -- * } -- * -- * I.e., it looks almost like high-level for each loop in other languages, -- * supports continue/break, and is verifiable by BPF verifier. -- * -- * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) -- * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to -- * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int -- * *`, not just `int`. So for integers bpf_for() is more convenient. -- * -- * Note: this macro relies on C99 feature of allowing to declare variables -- * inside for() loop, bound to for() loop lifetime. It also utilizes GCC -- * extension: __attribute__((cleanup())), supported by both GCC and -- * Clang. -- */ --#define bpf_for_each(type, cur, args...) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ -- cleanup(bpf_iter_##type##_destroy))), \ -- /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_##type##_new(&___it, ##args), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_##type##_destroy, (void *)0); \ -- /* iteration and termination check */ \ -- (((cur) = bpf_iter_##type##_next(&___it))); \ --) --#endif /* bpf_for_each */ -- --#ifndef bpf_for --/* bpf_for(i, start, end) implements a for()-like looping construct that sets -- * provided integer variable *i* to values starting from *start* through, -- * but not including, *end*. It also proves to BPF verifier that *i* belongs -- * to range [start, end), so this can be used for accessing arrays without -- * extra checks. -- * -- * Note: *start* and *end* are assumed to be expressions with no side effects -- * and whose values do not change throughout bpf_for() loop execution. They do -- * not have to be statically known or constant, though. -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_for(i, start, end) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, (start), (end)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- ({ \ -- /* iteration step */ \ -- int *___t = bpf_iter_num_next(&___it); \ -- /* termination and bounds check */ \ -- (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ -- }); \ --) --#endif /* bpf_for */ -- --#ifndef bpf_repeat --/* bpf_repeat(N) performs N iterations without exposing iteration number -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_repeat(N) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, 0, (N)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- bpf_iter_num_next(&___it); \ -- /* nothing here */ \ --) --#endif /* bpf_repeat */ -- --#endif /* __SCHED_EXT_COMMON_BPF_H */ -diff --git a/tools/sched_ext/scx_example_central.bpf.c b/tools/sched_ext/scx_example_central.bpf.c -deleted file mode 100644 -index 4cec04b4c2ed..000000000000 ---- a/tools/sched_ext/scx_example_central.bpf.c -+++ /dev/null -@@ -1,334 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A central FIFO sched_ext scheduler which demonstrates the followings: -- * -- * a. Making all scheduling decisions from one CPU: -- * -- * The central CPU is the only one making scheduling decisions. All other -- * CPUs kick the central CPU when they run out of tasks to run. -- * -- * There is one global BPF queue and the central CPU schedules all CPUs by -- * dispatching from the global queue to each CPU's local dsq from dispatch(). -- * This isn't the most straightforward. e.g. It'd be easier to bounce -- * through per-CPU BPF queues. The current design is chosen to maximally -- * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching. -- * -- * b. Tickless operation -- * -- * All tasks are dispatched with the infinite slice which allows stopping the -- * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full -- * parameter. The tickless operation can be observed through -- * /proc/interrupts. -- * -- * Periodic switching is enforced by a periodic timer checking all CPUs and -- * preempting them as necessary. Unfortunately, BPF timer currently doesn't -- * have a way to pin to a specific CPU, so the periodic timer isn't pinned to -- * the central CPU. -- * -- * c. Preemption -- * -- * Kthreads are unconditionally queued to the head of a matching local dsq -- * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always -- * prioritized over user threads, which is required for ensuring forward -- * progress as e.g. the periodic timer may run on a ksoftirqd and if the -- * ksoftirqd gets starved by a user thread, there may not be anything else to -- * vacate that user thread. -- * -- * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the -- * next tasks. -- * -- * This scheduler is designed to maximize usage of various SCX mechanisms. A -- * more practical implementation would likely put the scheduling loop outside -- * the central CPU's dispatch() path and add some form of priority mechanism. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --enum { -- FALLBACK_DSQ_ID = 0, -- MAX_CPUS = 4096, -- MS_TO_NS = 1000LLU * 1000, -- TIMER_INTERVAL_NS = 1 * MS_TO_NS, --}; -- --const volatile bool switch_partial; --const volatile s32 central_cpu; --const volatile u32 nr_cpu_ids = 64; /* !0 for veristat, set during init */ -- --u64 nr_total, nr_locals, nr_queued, nr_lost_pids; --u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries; --u64 nr_overflows; -- --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, s32); --} central_q SEC(".maps"); -- --/* can't use percpu map due to bad lookups */ --static bool cpu_gimme_task[MAX_CPUS]; --static u64 cpu_started_at[MAX_CPUS]; -- --struct central_timer { -- struct bpf_timer timer; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, 1); -- __type(key, u32); -- __type(value, struct central_timer); --} central_timer SEC(".maps"); -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- /* -- * Steer wakeups to the central CPU as much as possible to avoid -- * disturbing other CPUs. It's safe to blindly return the central cpu as -- * select_cpu() is a hint and if @p can't be on it, the kernel will -- * automatically pick a fallback CPU. -- */ -- return central_cpu; --} -- --void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags) --{ -- s32 pid = p->pid; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- /* -- * Push per-cpu kthreads at the head of local dsq's and preempt the -- * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked -- * behind other threads which is necessary for forward progress -- * guarantee as we depend on the BPF timer which may run from ksoftirqd. -- */ -- if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- __sync_fetch_and_add(&nr_locals, 1); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF, -- enq_flags | SCX_ENQ_PREEMPT); -- return; -- } -- -- if (bpf_map_push_elem(¢ral_q, &pid, 0)) { -- __sync_fetch_and_add(&nr_overflows, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_queued, 1); -- -- if (!scx_bpf_task_running(p)) -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); --} -- --static bool dispatch_to_cpu(s32 cpu) --{ -- struct task_struct *p; -- s32 pid; -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (bpf_map_pop_elem(¢ral_q, &pid)) -- break; -- -- __sync_fetch_and_sub(&nr_queued, 1); -- -- p = bpf_task_from_pid(pid); -- if (!p) { -- __sync_fetch_and_add(&nr_lost_pids, 1); -- continue; -- } -- -- /* -- * If we can't run the task at the top, do the dumb thing and -- * bounce it to the fallback dsq. -- */ -- if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- __sync_fetch_and_add(&nr_mismatches, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0); -- bpf_task_release(p); -- continue; -- } -- -- /* dispatch to local and mark that @cpu doesn't need more */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0); -- -- if (cpu != central_cpu) -- scx_bpf_kick_cpu(cpu, 0); -- -- bpf_task_release(p); -- return true; -- } -- -- return false; --} -- --void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (cpu == central_cpu) { -- /* dispatch for all other CPUs first */ -- __sync_fetch_and_add(&nr_dispatches, 1); -- -- bpf_for(cpu, 0, nr_cpu_ids) { -- bool *gimme; -- -- if (!scx_bpf_dispatch_nr_slots()) -- break; -- -- /* central's gimme is never set */ -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme && !*gimme) -- continue; -- -- if (dispatch_to_cpu(cpu)) -- *gimme = false; -- } -- -- /* -- * Retry if we ran out of dispatch buffer slots as we might have -- * skipped some CPUs and also need to dispatch for self. The ext -- * core automatically retries if the local dsq is empty but we -- * can't rely on that as we're dispatching for other CPUs too. -- * Kick self explicitly to retry. -- */ -- if (!scx_bpf_dispatch_nr_slots()) { -- __sync_fetch_and_add(&nr_retries, 1); -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- return; -- } -- -- /* look for a task to run on the central CPU */ -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- dispatch_to_cpu(central_cpu); -- } else { -- bool *gimme; -- -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme) -- *gimme = true; -- -- /* -- * Force dispatch on the scheduling CPU so that it finds a task -- * to run for us. -- */ -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- } --} -- --void BPF_STRUCT_OPS(central_running, struct task_struct *p) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ --} -- --void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = 0; --} -- --static int central_timerfn(void *map, int *key, struct bpf_timer *timer) --{ -- u64 now = bpf_ktime_get_ns(); -- u64 nr_to_kick = nr_queued; -- s32 i; -- -- bpf_for(i, 0, nr_cpu_ids) { -- s32 cpu = (nr_timers + i) % nr_cpu_ids; -- u64 *started_at; -- -- if (cpu == central_cpu) -- continue; -- -- /* kick iff the current one exhausted its slice */ -- started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at && *started_at && -- vtime_before(now, *started_at + SCX_SLICE_DFL)) -- continue; -- -- /* and there's something pending */ -- if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) || -- scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu)) -- ; -- else if (nr_to_kick) -- nr_to_kick--; -- else -- continue; -- -- scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); -- } -- -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- -- bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- __sync_fetch_and_add(&nr_timers, 1); -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(central_init) --{ -- u32 key = 0; -- struct bpf_timer *timer; -- int ret; -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1); -- if (ret) -- return ret; -- -- timer = bpf_map_lookup_elem(¢ral_timer, &key); -- if (!timer) -- return -ESRCH; -- -- bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); -- bpf_timer_set_callback(timer, central_timerfn); -- ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- return ret; --} -- --void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops central_ops = { -- /* -- * We are offloading all scheduling decisions to the central CPU and -- * thus being the last task on a given CPU doesn't mean anything -- * special. Enqueue the last tasks like any other tasks. -- */ -- .flags = SCX_OPS_ENQ_LAST, -- -- .select_cpu = (void *)central_select_cpu, -- .enqueue = (void *)central_enqueue, -- .dispatch = (void *)central_dispatch, -- .running = (void *)central_running, -- .stopping = (void *)central_stopping, -- .init = (void *)central_init, -- .exit = (void *)central_exit, -- .name = "central", --}; -diff --git a/tools/sched_ext/scx_example_central.c b/tools/sched_ext/scx_example_central.c -deleted file mode 100644 -index 7ad591cbdc65..000000000000 ---- a/tools/sched_ext/scx_example_central.c -+++ /dev/null -@@ -1,94 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_central.skel.h" -- --const char help_fmt[] = --"A central FIFO sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-c CPU] [-p]\n" --"\n" --" -c CPU Override the central CPU (default: 0)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_central *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_central__open(); -- assert(skel); -- -- skel->rodata->central_cpu = 0; -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "c:ph")) != -1) { -- switch (opt) { -- case 'c': -- skel->rodata->central_cpu = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_central__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.central_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf("total :%10lu local:%10lu queued:%10lu lost:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_locals, -- skel->bss->nr_queued, -- skel->bss->nr_lost_pids); -- printf("timer :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n", -- skel->bss->nr_timers, -- skel->bss->nr_dispatches, -- skel->bss->nr_mismatches, -- skel->bss->nr_retries); -- printf("overflow:%10lu\n", -- skel->bss->nr_overflows); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_central__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_flatcg.bpf.c b/tools/sched_ext/scx_example_flatcg.bpf.c -deleted file mode 100644 -index f6078b9a681f..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.bpf.c -+++ /dev/null -@@ -1,872 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext flattened cgroup hierarchy scheduler. It implements -- * hierarchical weight-based cgroup CPU control by flattening the cgroup -- * hierarchy into a single layer by compounding the active weight share at each -- * level. Consider the following hierarchy with weights in parentheses: -- * -- * R + A (100) + B (100) -- * | \ C (100) -- * \ D (200) -- * -- * Ignoring the root and threaded cgroups, only B, C and D can contain tasks. -- * Let's say all three have runnable tasks. The total share that each of these -- * three cgroups is entitled to can be calculated by compounding its share at -- * each level. -- * -- * For example, B is competing against C and in that competition its share is -- * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's -- * share in that competition is 200/(200+100) == 1/3. B's eventual share in the -- * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's -- * eventual shaer is the same at 1/6. D is only competing at the top level and -- * its share is 200/(100+200) == 2/3. -- * -- * So, instead of hierarchically scheduling level-by-level, we can consider it -- * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3 -- * and keep updating the eventual shares as the cgroups' runnable states change. -- * -- * This flattening of hierarchy can bring a substantial performance gain when -- * the cgroup hierarchy is nested multiple levels. in a simple benchmark using -- * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it -- * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two -- * apache instances competing with 2:1 weight ratio nested four level deep. -- * -- * However, the gain comes at the cost of not being able to properly handle -- * thundering herd of cgroups. For example, if many cgroups which are nested -- * behind a low priority parent cgroup wake up around the same time, they may be -- * able to consume more CPU cycles than they are entitled to. In many use cases, -- * this isn't a real concern especially given the performance gain. Also, there -- * are ways to mitigate the problem further by e.g. introducing an extra -- * scheduling layer on cgroup delegation boundaries. -- * -- * The scheduler first picks the cgroup to run and then schedule the tasks -- * within by using nested weighted vtime scheduling by default. The -- * cgroup-internal scheduling can be switched to FIFO with the -f option. -- */ --#include "scx_common.bpf.h" --#include "user_exit_info.h" --#include "scx_example_flatcg.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */ --const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL; --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --u64 cvtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, u64); -- __uint(max_entries, FCG_NR_STATS); --} stats SEC(".maps"); -- --static void stat_inc(enum fcg_stat_idx idx) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p)++; --} -- --struct fcg_cpu_ctx { -- u64 cur_cgid; -- u64 cur_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, struct fcg_cpu_ctx); -- __uint(max_entries, 1); --} cpu_ctx SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_cgrp_ctx); --} cgrp_ctx SEC(".maps"); -- --struct cgv_node { -- struct bpf_rb_node rb_node; -- __u64 cvtime; -- __u64 cgid; --}; -- --private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock; --private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node); -- --struct cgv_node_stash { -- struct cgv_node __kptr *node; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, 16384); -- __type(key, __u64); -- __type(value, struct cgv_node_stash); --} cgv_node_stash SEC(".maps"); -- --struct fcg_task_ctx { -- u64 bypassed_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_task_ctx); --} task_ctx SEC(".maps"); -- --/* gets inc'd on weight tree changes to expire the cached hweights */ --unsigned long hweight_gen = 1; -- --static u64 div_round_up(u64 dividend, u64 divisor) --{ -- return (dividend + divisor - 1) / divisor; --} -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) --{ -- struct cgv_node *cgc_a, *cgc_b; -- -- cgc_a = container_of(a, struct cgv_node, rb_node); -- cgc_b = container_of(b, struct cgv_node, rb_node); -- -- return cgc_a->cvtime < cgc_b->cvtime; --} -- --static struct fcg_cpu_ctx *find_cpu_ctx(void) --{ -- struct fcg_cpu_ctx *cpuc; -- u32 idx = 0; -- -- cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx); -- if (!cpuc) { -- scx_bpf_error("cpu_ctx lookup failed"); -- return NULL; -- } -- return cpuc; --} -- --static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id); -- return NULL; -- } -- return cgc; --} -- --static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgrp = bpf_cgroup_ancestor(cgrp, level); -- if (!cgrp) { -- scx_bpf_error("ancestor cgroup lookup failed"); -- return NULL; -- } -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- scx_bpf_error("ancestor cgrp_ctx lookup failed"); -- bpf_cgroup_release(cgrp); -- return cgc; --} -- --static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- int level; -- -- if (!cgc->nr_active) { -- stat_inc(FCG_STAT_HWT_SKIP); -- return; -- } -- -- if (cgc->hweight_gen == hweight_gen) { -- stat_inc(FCG_STAT_HWT_CACHE); -- return; -- } -- -- stat_inc(FCG_STAT_HWT_UPDATES); -- bpf_for(level, 0, cgrp->level + 1) { -- struct fcg_cgrp_ctx *cgc; -- bool is_active; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- -- if (!level) { -- cgc->hweight = FCG_HWEIGHT_ONE; -- cgc->hweight_gen = hweight_gen; -- } else { -- struct fcg_cgrp_ctx *pcgc; -- -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- -- /* -- * We can be oppotunistic here and not grab the -- * cgv_tree_lock and deal with the occasional races. -- * However, hweight updates are already cached and -- * relatively low-frequency. Let's just do the -- * straightforward thing. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- is_active = cgc->nr_active; -- if (is_active) { -- cgc->hweight_gen = pcgc->hweight_gen; -- cgc->hweight = -- div_round_up(pcgc->hweight * cgc->weight, -- pcgc->child_weight_sum); -- } -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!is_active) { -- stat_inc(FCG_STAT_HWT_RACE); -- break; -- } -- } -- } --} -- --static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) --{ -- u64 delta, cvtime, max_budget; -- -- /* -- * A node which is on the rbtree can't be pointed to from elsewhere yet -- * and thus can't be updated and repositioned. Instead, we collect the -- * vtime deltas separately and apply it asynchronously here. -- */ -- delta = cgc->cvtime_delta; -- __sync_fetch_and_sub(&cgc->cvtime_delta, delta); -- cvtime = cgv_node->cvtime + delta; -- -- /* -- * Allow a cgroup to carry the maximum budget proportional to its -- * hweight such that a full-hweight cgroup can immediately take up half -- * of the CPUs at the most while staying at the front of the rbtree. -- */ -- max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / -- (2 * FCG_HWEIGHT_ONE); -- if (vtime_before(cvtime, cvtime_now - max_budget)) -- cvtime = cvtime_now - max_budget; -- -- cgv_node->cvtime = cvtime; --} -- --static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- u64 cgid = cgrp->kn->id; -- -- /* paired with cmpxchg in try_pick_next_cgroup() */ -- if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { -- stat_inc(FCG_STAT_ENQ_SKIP); -- return; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid); -- return; -- } -- -- /* NULL if the node is already on the rbtree */ -- cgv_node = bpf_kptr_xchg(&stash->node, NULL); -- if (!cgv_node) { -- stat_inc(FCG_STAT_ENQ_RACE); -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); --} -- --void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * If select_cpu_dfl() is recommending local enqueue, the target CPU is -- * idle. Follow it and charge the cgroup later in fcg_stopping() after -- * the fact. Use the same mechanism to deal with tasks with custom -- * affinities so that we don't have to worry about per-cgroup dq's -- * containing tasks that can't be executed from some CPUs. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) { -- /* -- * Tell fcg_stopping() that this bypassed the regular scheduling -- * path and should be force charged to the cgroup. 0 is used to -- * indicate that the task isn't bypassing, so if the current -- * runtime is 0, go back by one nanosecond. -- */ -- taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1; -- -- /* -- * The global dq is deprioritized as we don't want to let tasks -- * to boost themselves by constraining its cpumask. The -- * deprioritization is rather severe, so let's not apply that to -- * per-cpu kernel threads. This is ham-fisted. We probably wanna -- * implement per-cgroup fallback dq's instead so that we have -- * more control over when tasks with custom cpumask get issued. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || -- (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) { -- stat_inc(FCG_STAT_LOCAL); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- } else { -- stat_inc(FCG_STAT_GLOBAL); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } -- return; -- } -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- goto out_release; -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 tvtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) -- tvtime = cgc->tvtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, -- tvtime, enq_flags); -- } -- -- cgrp_enqueued(cgrp, cgc); --out_release: -- bpf_cgroup_release(cgrp); --} -- --/* -- * Walk the cgroup tree to update the active weight sums as tasks wake up and -- * sleep. The weight sums are used as the base when calculating the proportion a -- * given cgroup or task is entitled to at each level. -- */ --static void update_active_weight_sums(struct cgroup *cgrp, bool runnable) --{ -- struct fcg_cgrp_ctx *cgc; -- bool updated = false; -- int idx; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- /* -- * In most cases, a hot cgroup would have multiple threads going to -- * sleep and waking up while the whole cgroup stays active. In leaf -- * cgroups, ->nr_runnable which is updated with __sync operations gates -- * ->nr_active updates, so that we don't have to grab the cgv_tree_lock -- * repeatedly for a busy cgroup which is staying active. -- */ -- if (runnable) { -- if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_ACT); -- } else { -- if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_DEACT); -- } -- -- /* -- * If @cgrp is becoming runnable, its hweight should be refreshed after -- * it's added to the weight tree so that enqueue has the up-to-date -- * value. If @cgrp is becoming quiescent, the hweight should be -- * refreshed before it's removed from the weight tree so that the usage -- * charging which happens afterwards has access to the latest value. -- */ -- if (!runnable) -- cgrp_refresh_hweight(cgrp, cgc); -- -- /* propagate upwards */ -- bpf_for(idx, 0, cgrp->level) { -- int level = cgrp->level - idx; -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- bool propagate = false; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- if (level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- } -- -- /* -- * We need the propagation protected by a lock to synchronize -- * against weight changes. There's no reason to drop the lock at -- * each level but bpf_spin_lock() doesn't want any function -- * calls while locked. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- -- if (runnable) { -- if (!cgc->nr_active++) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum += cgc->weight; -- } -- } -- } else { -- if (!--cgc->nr_active) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum -= cgc->weight; -- } -- } -- } -- -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!propagate) -- break; -- } -- -- if (updated) -- __sync_fetch_and_add(&hweight_gen, 1); -- -- if (runnable) -- cgrp_refresh_hweight(cgrp, cgc); --} -- --void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, true); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) --{ -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- if (fifo_sched) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- /* -- * @cgc->tvtime_now always progresses forward as tasks start -- * executing. The test and update can be performed concurrently -- * from multiple CPUs and thus racy. Any error should be -- * contained and temporary. Let's just live with it. -- */ -- if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) -- cgc->tvtime_now = p->scx.dsq_vtime; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- /* scale the execution time by the inverse of the weight and charge */ -- if (!fifo_sched) -- p->scx.dsq_vtime += -- (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- if (!taskc->bypassed_at) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- __sync_fetch_and_add(&cgc->cvtime_delta, -- p->se.sum_exec_runtime - taskc->bypassed_at); -- taskc->bypassed_at = 0; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, false); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) --{ -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- if (cgrp->level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1); -- if (!pcgc) -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- if (pcgc && cgc->nr_active) -- pcgc->child_weight_sum += (s64)weight - cgc->weight; -- cgc->weight = weight; -- bpf_spin_unlock(&cgv_tree_lock); --} -- --static bool try_pick_next_cgroup(u64 *cgidp) --{ -- struct bpf_rb_node *rb_node; -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 cgid; -- -- /* pop the front cgroup and wind cvtime_now accordingly */ -- bpf_spin_lock(&cgv_tree_lock); -- -- rb_node = bpf_rbtree_first(&cgv_tree); -- if (!rb_node) { -- bpf_spin_unlock(&cgv_tree_lock); -- stat_inc(FCG_STAT_PNC_NO_CGRP); -- *cgidp = 0; -- return true; -- } -- -- rb_node = bpf_rbtree_remove(&cgv_tree, rb_node); -- bpf_spin_unlock(&cgv_tree_lock); -- -- cgv_node = container_of(rb_node, struct cgv_node, rb_node); -- cgid = cgv_node->cgid; -- -- if (vtime_before(cvtime_now, cgv_node->cvtime)) -- cvtime_now = cgv_node->cvtime; -- -- /* -- * If lookup fails, the cgroup's gone. Free and move on. See -- * fcg_cgroup_exit(). -- */ -- cgrp = bpf_cgroup_from_id(cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- if (!scx_bpf_consume(cgid)) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_EMPTY); -- goto out_stash; -- } -- -- /* -- * Successfully consumed from the cgroup. This will be our current -- * cgroup for the new slice. Refresh its hweight. -- */ -- cgrp_refresh_hweight(cgrp, cgc); -- -- bpf_cgroup_release(cgrp); -- -- /* -- * As the cgroup may have more tasks, add it back to the rbtree. Note -- * that here we charge the full slice upfront and then exact later -- * according to the actual consumption. This prevents lowpri thundering -- * herd from saturating the machine. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- -- *cgidp = cgid; -- stat_inc(FCG_STAT_PNC_NEXT); -- return true; -- --out_stash: -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- /* -- * Paired with cmpxchg in cgrp_enqueued(). If they see the following -- * transition, they'll enqueue the cgroup. If they are earlier, we'll -- * see their task in the dq below and requeue the cgroup. -- */ -- __sync_val_compare_and_swap(&cgc->queued, 1, 0); -- -- if (scx_bpf_dsq_nr_queued(cgid)) { -- bpf_spin_lock(&cgv_tree_lock); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- goto out_free; -- } -- } -- -- return false; -- --out_free: -- bpf_obj_drop(cgv_node); -- return false; --} -- --void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) --{ -- struct fcg_cpu_ctx *cpuc; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 now = bpf_ktime_get_ns(); -- -- cpuc = find_cpu_ctx(); -- if (!cpuc) -- return; -- -- if (!cpuc->cur_cgid) -- goto pick_next_cgroup; -- -- if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { -- if (scx_bpf_consume(cpuc->cur_cgid)) { -- stat_inc(FCG_STAT_CNS_KEEP); -- return; -- } -- stat_inc(FCG_STAT_CNS_EMPTY); -- } else { -- stat_inc(FCG_STAT_CNS_EXPIRE); -- } -- -- /* -- * The current cgroup is expiring. It was already charged a full slice. -- * Calculate the actual usage and accumulate the delta. -- */ -- cgrp = bpf_cgroup_from_id(cpuc->cur_cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_CNS_GONE); -- goto pick_next_cgroup; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (cgc) { -- /* -- * We want to update the vtime delta and then look for the next -- * cgroup to execute but the latter needs to be done in a loop -- * and we can't keep the lock held. Oh well... -- */ -- bpf_spin_lock(&cgv_tree_lock); -- __sync_fetch_and_add(&cgc->cvtime_delta, -- (cpuc->cur_at + cgrp_slice_ns - now) * -- FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- stat_inc(FCG_STAT_CNS_GONE); -- } -- -- bpf_cgroup_release(cgrp); -- --pick_next_cgroup: -- cpuc->cur_at = now; -- -- if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { -- cpuc->cur_cgid = 0; -- return; -- } -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_pick_next_cgroup(&cpuc->cur_cgid)) -- break; -- } --} -- --s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct fcg_task_ctx *taskc; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- taskc = bpf_task_storage_get(&task_ctx, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!taskc) -- return -ENOMEM; -- -- taskc->bypassed_at = 0; -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp, -- struct scx_cgroup_init_args *args) --{ -- struct fcg_cgrp_ctx *cgc; -- struct cgv_node *cgv_node; -- struct cgv_node_stash empty_stash = {}, *stash; -- u64 cgid = cgrp->kn->id; -- int ret; -- -- /* -- * Technically incorrect as cgroup ID is full 64bit while dq ID is -- * 63bit. Should not be a problem in practice and easy to spot in the -- * unlikely case that it breaks. -- */ -- ret = scx_bpf_create_dsq(cgid, -1); -- if (ret) -- return ret; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!cgc) { -- ret = -ENOMEM; -- goto err_destroy_dsq; -- } -- -- cgc->weight = args->weight; -- cgc->hweight = FCG_HWEIGHT_ONE; -- -- ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash, -- BPF_NOEXIST); -- if (ret) { -- if (ret != -ENOMEM) -- scx_bpf_error("unexpected stash creation error (%d)", -- ret); -- goto err_destroy_dsq; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("unexpected cgv_node stash lookup failure"); -- ret = -ENOENT; -- goto err_destroy_dsq; -- } -- -- cgv_node = bpf_obj_new(struct cgv_node); -- if (!cgv_node) { -- ret = -ENOMEM; -- goto err_del_cgv_node; -- } -- -- cgv_node->cgid = cgid; -- cgv_node->cvtime = cvtime_now; -- -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- ret = -EBUSY; -- goto err_drop; -- } -- -- return 0; -- --err_drop: -- bpf_obj_drop(cgv_node); --err_del_cgv_node: -- bpf_map_delete_elem(&cgv_node_stash, &cgid); --err_destroy_dsq: -- scx_bpf_destroy_dsq(cgid); -- return ret; --} -- --void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- -- /* -- * For now, there's no way find and remove the cgv_node if it's on the -- * cgv_tree. Let's drain them in the dispatch path as they get popped -- * off the front of the tree. -- */ -- bpf_map_delete_elem(&cgv_node_stash, &cgid); -- scx_bpf_destroy_dsq(cgid); --} -- --s32 BPF_STRUCT_OPS(fcg_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops flatcg_ops = { -- .enqueue = (void *)fcg_enqueue, -- .dispatch = (void *)fcg_dispatch, -- .runnable = (void *)fcg_runnable, -- .running = (void *)fcg_running, -- .stopping = (void *)fcg_stopping, -- .quiescent = (void *)fcg_quiescent, -- .prep_enable = (void *)fcg_prep_enable, -- .cgroup_set_weight = (void *)fcg_cgroup_set_weight, -- .cgroup_init = (void *)fcg_cgroup_init, -- .cgroup_exit = (void *)fcg_cgroup_exit, -- .init = (void *)fcg_init, -- .exit = (void *)fcg_exit, -- .flags = SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING, -- .name = "flatcg", --}; -diff --git a/tools/sched_ext/scx_example_flatcg.c b/tools/sched_ext/scx_example_flatcg.c -deleted file mode 100644 -index f9c8a5b84a70..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.c -+++ /dev/null -@@ -1,232 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2023 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2023 Tejun Heo -- * Copyright (c) 2023 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_flatcg.h" --#include "scx_example_flatcg.skel.h" -- --#ifndef FILEID_KERNFS --#define FILEID_KERNFS 0xfe --#endif -- --const char help_fmt[] = --"A flattened cgroup hierarchy sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -i INTERVAL Report interval\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --static float read_cpu_util(__u64 *last_sum, __u64 *last_idle) --{ -- FILE *fp; -- char buf[4096]; -- char *line, *cur = NULL, *tok; -- __u64 sum = 0, idle = 0; -- __u64 delta_sum, delta_idle; -- int idx; -- -- fp = fopen("/proc/stat", "r"); -- if (!fp) { -- perror("fopen(\"/proc/stat\")"); -- return 0.0; -- } -- -- if (!fgets(buf, sizeof(buf), fp)) { -- perror("fgets(\"/proc/stat\")"); -- fclose(fp); -- return 0.0; -- } -- fclose(fp); -- -- line = buf; -- for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) { -- char *endp = NULL; -- __u64 v; -- -- if (idx == 0) { -- line = NULL; -- continue; -- } -- v = strtoull(tok, &endp, 0); -- if (!endp || *endp != '\0') { -- fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n", -- idx, tok); -- continue; -- } -- sum += v; -- if (idx == 4) -- idle = v; -- } -- -- delta_sum = sum - *last_sum; -- delta_idle = idle - *last_idle; -- *last_sum = sum; -- *last_idle = idle; -- -- return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0; --} -- --static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats) --{ -- __u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus]; -- __u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); -- -- for (idx = 0; idx < FCG_NR_STATS; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_flatcg *skel; -- struct bpf_link *link; -- struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 }; -- bool dump_cgrps = false; -- __u64 last_cpu_sum = 0, last_cpu_idle = 0; -- __u64 last_stats[FCG_NR_STATS] = {}; -- unsigned long seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_flatcg__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open: %s\n", strerror(errno)); -- return 1; -- } -- -- skel->rodata->nr_cpus = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) { -- double v; -- -- switch (opt) { -- case 's': -- v = strtod(optarg, NULL); -- skel->rodata->cgrp_slice_ns = v * 1000; -- break; -- case 'i': -- v = strtod(optarg, NULL); -- intv_ts.tv_sec = v; -- intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000; -- break; -- case 'd': -- dump_cgrps = true; -- break; -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- case 'h': -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d", -- (double)skel->rodata->cgrp_slice_ns / 1000000.0, -- (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0, -- dump_cgrps); -- -- if (scx_example_flatcg__load(skel)) { -- fprintf(stderr, "Failed to load: %s\n", strerror(errno)); -- return 1; -- } -- -- link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops); -- if (!link) { -- fprintf(stderr, "Failed to attach_struct_ops: %s\n", -- strerror(errno)); -- return 1; -- } -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- __u64 acc_stats[FCG_NR_STATS]; -- __u64 stats[FCG_NR_STATS]; -- float cpu_util; -- int i; -- -- cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle); -- -- fcg_read_stats(skel, acc_stats); -- for (i = 0; i < FCG_NR_STATS; i++) -- stats[i] = acc_stats[i] - last_stats[i]; -- -- memcpy(last_stats, acc_stats, sizeof(acc_stats)); -- -- printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n", -- seq++, cpu_util * 100.0, skel->data->hweight_gen); -- printf(" act:%6llu deact:%6llu local:%6llu global:%6llu\n", -- stats[FCG_STAT_ACT], -- stats[FCG_STAT_DEACT], -- stats[FCG_STAT_LOCAL], -- stats[FCG_STAT_GLOBAL]); -- printf("HWT skip:%6llu race:%6llu cache:%6llu update:%6llu\n", -- stats[FCG_STAT_HWT_SKIP], -- stats[FCG_STAT_HWT_RACE], -- stats[FCG_STAT_HWT_CACHE], -- stats[FCG_STAT_HWT_UPDATES]); -- printf("ENQ skip:%6llu race:%6llu\n", -- stats[FCG_STAT_ENQ_SKIP], -- stats[FCG_STAT_ENQ_RACE]); -- printf("CNS keep:%6llu expire:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_CNS_KEEP], -- stats[FCG_STAT_CNS_EXPIRE], -- stats[FCG_STAT_CNS_EMPTY], -- stats[FCG_STAT_CNS_GONE]); -- printf("PNC nocgrp:%6llu next:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_PNC_NO_CGRP], -- stats[FCG_STAT_PNC_NEXT], -- stats[FCG_STAT_PNC_EMPTY], -- stats[FCG_STAT_PNC_GONE]); -- printf("BAD remove:%6llu\n", -- acc_stats[FCG_STAT_BAD_REMOVAL]); -- -- nanosleep(&intv_ts, NULL); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_flatcg__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_flatcg.h b/tools/sched_ext/scx_example_flatcg.h -deleted file mode 100644 -index 490758ed41f0..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.h -+++ /dev/null -@@ -1,49 +0,0 @@ --#ifndef __SCX_EXAMPLE_FLATCG_H --#define __SCX_EXAMPLE_FLATCG_H -- --enum { -- FCG_HWEIGHT_ONE = 1LLU << 16, --}; -- --enum fcg_stat_idx { -- FCG_STAT_ACT, -- FCG_STAT_DEACT, -- FCG_STAT_LOCAL, -- FCG_STAT_GLOBAL, -- -- FCG_STAT_HWT_UPDATES, -- FCG_STAT_HWT_CACHE, -- FCG_STAT_HWT_SKIP, -- FCG_STAT_HWT_RACE, -- -- FCG_STAT_ENQ_SKIP, -- FCG_STAT_ENQ_RACE, -- -- FCG_STAT_CNS_KEEP, -- FCG_STAT_CNS_EXPIRE, -- FCG_STAT_CNS_EMPTY, -- FCG_STAT_CNS_GONE, -- -- FCG_STAT_PNC_NO_CGRP, -- FCG_STAT_PNC_NEXT, -- FCG_STAT_PNC_EMPTY, -- FCG_STAT_PNC_GONE, -- -- FCG_STAT_BAD_REMOVAL, -- -- FCG_NR_STATS, --}; -- --struct fcg_cgrp_ctx { -- u32 nr_active; -- u32 nr_runnable; -- u32 queued; -- u32 weight; -- u32 hweight; -- u64 child_weight_sum; -- u64 hweight_gen; -- s64 cvtime_delta; -- u64 tvtime_now; --}; -- --#endif /* __SCX_EXAMPLE_FLATCG_H */ -diff --git a/tools/sched_ext/scx_example_pair.bpf.c b/tools/sched_ext/scx_example_pair.bpf.c -deleted file mode 100644 -index 279efe58b777..000000000000 ---- a/tools/sched_ext/scx_example_pair.bpf.c -+++ /dev/null -@@ -1,627 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext core-scheduler which always makes every sibling CPU pair -- * execute from the same CPU cgroup. -- * -- * This scheduler is a minimal implementation and would need some form of -- * priority handling both inside each cgroup and across the cgroups to be -- * practically useful. -- * -- * Each CPU in the system is paired with exactly one other CPU, according to a -- * "stride" value that can be specified when the BPF scheduler program is first -- * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee -- * that they will only ever schedule tasks that belong to the same CPU cgroup. -- * -- * Scheduler Initialization -- * ------------------------ -- * -- * The scheduler BPF program is first initialized from user space, before it is -- * enabled. During this initialization process, each CPU on the system is -- * assigned several values that are constant throughout its runtime: -- * -- * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling -- * decisions. Paired CPUs always schedule tasks from the same -- * CPU cgroup, and synchronize with each other to guarantee -- * that this constraint is not violated. -- * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access -- * a struct pair_ctx object that is shared between the pair. -- * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the -- * pair. Each struct pair_ctx has an active_mask field, -- * which is a bitmap used to indicate whether each core -- * in the pair currently has an actively running task. -- * This index specifies which entry in the bitmap corresponds -- * to each CPU in the pair. -- * -- * During this initialization, the CPUs are paired according to a "stride" that -- * may be specified when invoking the user space program that initializes and -- * loads the scheduler. By default, the stride is 1/2 the total number of CPUs. -- * -- * Tasks and cgroups -- * ----------------- -- * -- * Every cgroup in the system is registered with the scheduler using the -- * pair_cgroup_init() callback, and every task in the system is associated with -- * exactly one cgroup. At a high level, the idea with the pair scheduler is to -- * always schedule tasks from the same cgroup within a given CPU pair. When a -- * task is enqueued (i.e. passed to the pair_enqueue() callback function), its -- * cgroup ID is read from its task struct, and then a corresponding queue map -- * is used to FIFO-enqueue the task for that cgroup. -- * -- * If you look through the implementation of the scheduler, you'll notice that -- * there is quite a bit of complexity involved with looking up the per-cgroup -- * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash -- * BPF hash map that is used to map a cgroup ID to a globally unique ID that's -- * allocated in the BPF program. This is done because we use separate maps to -- * store the FIFO queue of tasks, and the length of that map, per cgroup. This -- * complexity is only present because of current deficiencies in BPF that will -- * soon be addressed. The main point to keep in mind is that newly enqueued -- * tasks are added to their cgroup's FIFO queue. -- * -- * Dispatching tasks -- * ----------------- -- * -- * This section will describe how enqueued tasks are dispatched and scheduled. -- * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is -- * as follows: -- * -- * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is -- * the structure that's used to synchronize amongst the two pair CPUs in their -- * scheduling decisions. After any of the following events have occurred: -- * -- * - The cgroup's slice run has expired, or -- * - The cgroup becomes empty, or -- * - Either CPU in the pair is preempted by a higher priority scheduling class -- * -- * The cgroup transitions to the draining state and stops executing new tasks -- * from the cgroup. -- * -- * 2. If the pair is still executing a task, mark the pair_ctx as draining, and -- * wait for the pair CPU to be preempted. -- * -- * 3. Otherwise, if the pair CPU is not running a task, we can move onto -- * scheduling new tasks. Pop the next cgroup id from the top_q queue. -- * -- * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it. -- * -- * Note again that this scheduling behavior is simple, but the implementation -- * is complex mostly because this it hits several BPF shortcomings and has to -- * work around in often awkward ways. Most of the shortcomings are expected to -- * be resolved in the near future which should allow greatly simplifying this -- * scheduler. -- * -- * Dealing with preemption -- * ----------------------- -- * -- * SCX is the lowest priority sched_class, and could be preempted by them at -- * any time. To address this, the scheduler implements pair_cpu_release() and -- * pair_cpu_acquire() callbacks which are invoked by the core scheduler when -- * the scheduler loses and gains control of the CPU respectively. -- * -- * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and -- * then invoke: -- * -- * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- * -- * This preempts the pair CPU, and waits until it has re-entered the scheduler -- * before returning. This is necessary to ensure that the higher priority -- * sched_class that preempted our scheduler does not schedule a task -- * concurrently with our pair CPU. -- * -- * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption -- * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable -- * pair scheduling. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include "scx_example_pair.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; -- --/* !0 for veristat, set during init */ --const volatile u32 nr_cpu_ids = 64; -- --/* a pair of CPUs stay on a cgroup for this duration */ --const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL; -- --/* cpu ID -> pair cpu ID */ --const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 }; -- --/* cpu ID -> pair_id */ --const volatile u32 pair_id[MAX_CPUS]; -- --/* CPU ID -> CPU # in the pair (0 or 1) */ --const volatile u32 in_pair_idx[MAX_CPUS]; -- --struct pair_ctx { -- struct bpf_spin_lock lock; -- -- /* the cgroup the pair is currently executing */ -- u64 cgid; -- -- /* the pair started executing the current cgroup at */ -- u64 started_at; -- -- /* whether the current cgroup is draining */ -- bool draining; -- -- /* the CPUs that are currently active on the cgroup */ -- u32 active_mask; -- -- /* -- * the CPUs that are currently preempted and running tasks in a -- * different scheduler. -- */ -- u32 preempted_mask; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, MAX_CPUS / 2); -- __type(key, u32); -- __type(value, struct pair_ctx); --} pair_ctx SEC(".maps"); -- --/* queue of cgrp_q's possibly with tasks on them */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- /* -- * Because it's difficult to build strong synchronization encompassing -- * multiple non-trivial operations in BPF, this queue is managed in an -- * opportunistic way so that we guarantee that a cgroup w/ active tasks -- * is always on it but possibly multiple times. Once we have more robust -- * synchronization constructs and e.g. linked list, we should be able to -- * do this in a prettier way but for now just size it big enough. -- */ -- __uint(max_entries, 4 * MAX_CGRPS); -- __type(value, u64); --} top_q SEC(".maps"); -- --/* per-cgroup q which FIFOs the tasks from the cgroup */ --struct cgrp_q { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, MAX_QUEUED); -- __type(value, u32); --}; -- --/* -- * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local -- * storage; however, a cgroup local storage can only be accessed from the BPF -- * progs attached to the cgroup. For now, work around by allocating array of -- * cgrp_q's and then allocating per-cgroup indices. -- * -- * Another caveat: It's difficult to populate a large array of maps statically -- * or from BPF. Initialize it from userland. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, MAX_CGRPS); -- __type(key, s32); -- __array(values, struct cgrp_q); --} cgrp_q_arr SEC(".maps"); -- --static u64 cgrp_q_len[MAX_CGRPS]; -- --/* -- * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be -- * useful to have as a map type. -- */ --static u32 cgrp_q_idx_cursor; --static u64 cgrp_q_idx_busy[MAX_CGRPS]; -- --/* -- * All added up, the following is what we do: -- * -- * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking -- * for a free ID. If not found, fail cgroup creation with -EBUSY. -- * -- * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following -- * cgrp_q_idx_hash. -- * -- * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from -- * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr. -- * -- * This is sadly complicated for something pretty simple. Hopefully, we should -- * be able to simplify in the future. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, MAX_CGRPS); -- __uint(key_size, sizeof(u64)); /* cgrp ID */ -- __uint(value_size, sizeof(s32)); /* cgrp_q idx */ --} cgrp_q_idx_hash SEC(".maps"); -- --/* statistics */ --u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions; --u64 nr_exps, nr_exp_waits, nr_exp_empty; --u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty; -- --struct user_exit_info uei; -- --static bool time_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- struct cgrp_q *cgq; -- s32 pid = p->pid; -- u64 cgid; -- u32 *q_idx; -- u64 *cgq_len; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- cgrp = scx_bpf_task_cgroup(p); -- cgid = cgrp->kn->id; -- bpf_cgroup_release(cgrp); -- -- /* find the cgroup's q and push @p into it */ -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!q_idx) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return; -- } -- -- cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx); -- if (!cgq) { -- scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]", -- cgid, *q_idx); -- return; -- } -- -- if (bpf_map_push_elem(cgq, &pid, 0)) { -- scx_bpf_error("cgroup[%llu] queue overflow", cgid); -- return; -- } -- -- /* bump q len, if going 0 -> 1, queue cgroup into the top_q */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len) { -- scx_bpf_error("MEMBER_VTPR malfunction"); -- return; -- } -- -- if (!__sync_fetch_and_add(cgq_len, 1) && -- bpf_map_push_elem(&top_q, &cgid, 0)) { -- scx_bpf_error("top_q overflow"); -- return; -- } --} -- --static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) --{ -- u32 *vptr; -- -- vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *pairc = bpf_map_lookup_elem(&pair_ctx, vptr); -- if (!(*pairc)) -- return -EINVAL; -- -- vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *mask = 1U << *vptr; -- -- return 0; --} -- --static int try_dispatch(s32 cpu) --{ -- struct pair_ctx *pairc; -- struct bpf_map *cgq_map; -- struct task_struct *p; -- u64 now = bpf_ktime_get_ns(); -- bool kick_pair = false; -- bool expired, pair_preempted; -- u32 *vptr, in_pair_mask; -- s32 pid, q_idx; -- u64 cgid; -- int ret; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) { -- scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]", -- cpu); -- return -ENOENT; -- } -- -- bpf_spin_lock(&pairc->lock); -- pairc->active_mask &= ~in_pair_mask; -- -- expired = time_before(pairc->started_at + pair_batch_dur_ns, now); -- if (expired || pairc->draining) { -- u64 new_cgid = 0; -- -- __sync_fetch_and_add(&nr_exps, 1); -- -- /* -- * We're done with the current cgid. An obvious optimization -- * would be not draining if the next cgroup is the current one. -- * For now, be dumb and always expire. -- */ -- pairc->draining = true; -- -- pair_preempted = pairc->preempted_mask; -- if (pairc->active_mask || pair_preempted) { -- /* -- * The other CPU is still active, or is no longer under -- * our control due to e.g. being preempted by a higher -- * priority sched_class. We want to wait until this -- * cgroup expires, or until control of our pair CPU has -- * been returned to us. -- * -- * If the pair controls its CPU, and the time already -- * expired, kick. When the other CPU arrives at -- * dispatch and clears its active mask, it'll push the -- * pair to the next cgroup and kick this CPU. -- */ -- __sync_fetch_and_add(&nr_exp_waits, 1); -- bpf_spin_unlock(&pairc->lock); -- if (expired && !pair_preempted) -- kick_pair = true; -- goto out_maybe_kick; -- } -- -- bpf_spin_unlock(&pairc->lock); -- -- /* -- * Pick the next cgroup. It'd be easier / cleaner to not drop -- * pairc->lock and use stronger synchronization here especially -- * given that we'll be switching cgroups significantly less -- * frequently than tasks. Unfortunately, bpf_spin_lock can't -- * really protect anything non-trivial. Let's do opportunistic -- * operations instead. -- */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u32 *q_idx; -- u64 *cgq_len; -- -- if (bpf_map_pop_elem(&top_q, &new_cgid)) { -- /* no active cgroup, go idle */ -- __sync_fetch_and_add(&nr_exp_empty, 1); -- return 0; -- } -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid); -- if (!q_idx) -- continue; -- -- /* -- * This is the only place where empty cgroups are taken -- * off the top_q. -- */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len || !*cgq_len) -- continue; -- -- /* -- * If it has any tasks, requeue as we may race and not -- * execute it. -- */ -- bpf_map_push_elem(&top_q, &new_cgid, 0); -- break; -- } -- -- bpf_spin_lock(&pairc->lock); -- -- /* -- * The other CPU may already have started on a new cgroup while -- * we dropped the lock. Make sure that we're still draining and -- * start on the new cgroup. -- */ -- if (pairc->draining && !pairc->active_mask) { -- __sync_fetch_and_add(&nr_cgrp_next, 1); -- pairc->cgid = new_cgid; -- pairc->started_at = now; -- pairc->draining = false; -- kick_pair = true; -- } else { -- __sync_fetch_and_add(&nr_cgrp_coll, 1); -- } -- } -- -- cgid = pairc->cgid; -- pairc->active_mask |= in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- -- /* again, it'd be better to do all these with the lock held, oh well */ -- vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!vptr) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return -ENOENT; -- } -- q_idx = *vptr; -- -- /* claim one task from cgrp_q w/ q_idx */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u64 *cgq_len, len; -- -- cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]); -- if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) { -- /* the cgroup must be empty, expire and repeat */ -- __sync_fetch_and_add(&nr_cgrp_empty, 1); -- bpf_spin_lock(&pairc->lock); -- pairc->draining = true; -- pairc->active_mask &= ~in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- return -EAGAIN; -- } -- -- if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len) -- continue; -- -- break; -- } -- -- cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx); -- if (!cgq_map) { -- scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- if (bpf_map_pop_elem(cgq_map, &pid)) { -- scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- p = bpf_task_from_pid(pid); -- if (p) { -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } else { -- /* we don't handle dequeues, retry on lost tasks */ -- __sync_fetch_and_add(&nr_missing, 1); -- return -EAGAIN; -- } -- --out_maybe_kick: -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } -- return 0; --} -- --void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) --{ -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_dispatch(cpu) != -EAGAIN) -- break; -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask &= ~in_pair_mask; -- /* Kick the pair CPU, unless it was also preempted. */ -- kick_pair = !pairc->preempted_mask; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask |= in_pair_mask; -- pairc->active_mask &= ~in_pair_mask; -- /* Kick the pair CPU if it's still running. */ -- kick_pair = pairc->active_mask; -- pairc->draining = true; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- } -- } -- __sync_fetch_and_add(&nr_preemptions, 1); --} -- --s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 i, q_idx; -- -- bpf_for(i, 0, MAX_CGRPS) { -- q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS; -- if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1)) -- break; -- } -- if (i == MAX_CGRPS) -- return -EBUSY; -- -- if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]); -- if (busy) -- *busy = 0; -- return -EBUSY; -- } -- -- return 0; --} -- --void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 *q_idx; -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (q_idx) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]); -- if (busy) -- *busy = 0; -- bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid); -- } --} -- --s32 BPF_STRUCT_OPS(pair_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops pair_ops = { -- .enqueue = (void *)pair_enqueue, -- .dispatch = (void *)pair_dispatch, -- .cpu_acquire = (void *)pair_cpu_acquire, -- .cpu_release = (void *)pair_cpu_release, -- .cgroup_init = (void *)pair_cgroup_init, -- .cgroup_exit = (void *)pair_cgroup_exit, -- .init = (void *)pair_init, -- .exit = (void *)pair_exit, -- .name = "pair", --}; -diff --git a/tools/sched_ext/scx_example_pair.c b/tools/sched_ext/scx_example_pair.c -deleted file mode 100644 -index 18e032bbc173..000000000000 ---- a/tools/sched_ext/scx_example_pair.c -+++ /dev/null -@@ -1,143 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_pair.h" --#include "scx_example_pair.skel.h" -- --const char help_fmt[] = --"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n" --"execute from the same CPU cgroup.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-S STRIDE] [-p]\n" --"\n" --" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_pair *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 stride, i, opt, outer_fd; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_pair__open(); -- assert(skel); -- -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- /* pair up the earlier half to the latter by default, override with -s */ -- stride = skel->rodata->nr_cpu_ids / 2; -- -- while ((opt = getopt(argc, argv, "S:ph")) != -1) { -- switch (opt) { -- case 'S': -- stride = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- for (i = 0; i < skel->rodata->nr_cpu_ids; i++) { -- if (skel->rodata->pair_cpu[i] < 0) { -- skel->rodata->pair_cpu[i] = i + stride; -- skel->rodata->pair_cpu[i + stride] = i; -- skel->rodata->pair_id[i] = i; -- skel->rodata->pair_id[i + stride] = i; -- skel->rodata->in_pair_idx[i] = 0; -- skel->rodata->in_pair_idx[i + stride] = 1; -- } -- } -- -- assert(!scx_example_pair__load(skel)); -- -- /* -- * Populate the cgrp_q_arr map which is an array containing per-cgroup -- * queues. It'd probably be better to do this from BPF but there are too -- * many to initialize statically and there's no way to dynamically -- * populate from BPF. -- */ -- outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr); -- assert(outer_fd >= 0); -- -- printf("Initializing"); -- for (i = 0; i < MAX_CGRPS; i++) { -- s32 inner_fd; -- -- if (exit_req) -- break; -- -- inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, -- sizeof(u32), MAX_QUEUED, NULL); -- assert(inner_fd >= 0); -- assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY)); -- close(inner_fd); -- -- if (!(i % 10)) -- printf("."); -- fflush(stdout); -- } -- printf("\n"); -- -- /* -- * Fully initialized, attach and run. -- */ -- link = bpf_map__attach_struct_ops(skel->maps.pair_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf(" total:%10lu dispatch:%10lu missing:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_dispatched, -- skel->bss->nr_missing); -- printf(" kicks:%10lu preemptions:%7lu\n", -- skel->bss->nr_kicks, -- skel->bss->nr_preemptions); -- printf(" exp:%10lu exp_wait:%10lu exp_empty:%10lu\n", -- skel->bss->nr_exps, -- skel->bss->nr_exp_waits, -- skel->bss->nr_exp_empty); -- printf("cgnext:%10lu cgcoll:%10lu cgempty:%10lu\n", -- skel->bss->nr_cgrp_next, -- skel->bss->nr_cgrp_coll, -- skel->bss->nr_cgrp_empty); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_pair__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_pair.h b/tools/sched_ext/scx_example_pair.h -deleted file mode 100644 -index f60b824272f7..000000000000 ---- a/tools/sched_ext/scx_example_pair.h -+++ /dev/null -@@ -1,10 +0,0 @@ --#ifndef __SCX_EXAMPLE_PAIR_H --#define __SCX_EXAMPLE_PAIR_H -- --enum { -- MAX_CPUS = 4096, -- MAX_QUEUED = 4096, -- MAX_CGRPS = 4096, --}; -- --#endif /* __SCX_EXAMPLE_PAIR_H */ -diff --git a/tools/sched_ext/scx_example_qmap.bpf.c b/tools/sched_ext/scx_example_qmap.bpf.c -deleted file mode 100644 -index 579ab21ae403..000000000000 ---- a/tools/sched_ext/scx_example_qmap.bpf.c -+++ /dev/null -@@ -1,401 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple five-level FIFO queue scheduler. -- * -- * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets -- * assigned to one depending on its compound weight. Each CPU round robins -- * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from -- * queue0, 2 from queue1, 4 from queue2 and so on. -- * -- * This scheduler demonstrates: -- * -- * - BPF-side queueing using PIDs. -- * - Sleepable per-task storage allocation using ops.prep_enable(). -- * - Using ops.cpu_release() to handle a higher priority scheduling class taking -- * the CPU away. -- * - Core-sched support. -- * -- * This scheduler is primarily for demonstration and testing of sched_ext -- * features and unlikely to be useful for actual workloads. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include -- --char _license[] SEC("license") = "GPL"; -- --const volatile u64 slice_ns = SCX_SLICE_DFL; --const volatile bool switch_partial; --const volatile u32 stall_user_nth; --const volatile u32 stall_kernel_nth; --const volatile u32 dsp_inf_loop_after; --const volatile s32 disallow_tgid; -- --u32 test_error_cnt; -- --struct user_exit_info uei; -- --struct qmap { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, u32); --} queue0 SEC(".maps"), -- queue1 SEC(".maps"), -- queue2 SEC(".maps"), -- queue3 SEC(".maps"), -- queue4 SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, 5); -- __type(key, int); -- __array(values, struct qmap); --} queue_arr SEC(".maps") = { -- .values = { -- [0] = &queue0, -- [1] = &queue1, -- [2] = &queue2, -- [3] = &queue3, -- [4] = &queue4, -- }, --}; -- --/* -- * Per-queue sequence numbers to implement core-sched ordering. -- * -- * Tail seq is assigned to each queued task and incremented. Head seq tracks the -- * sequence number of the latest dispatched task. The distance between the a -- * task's seq and the associated queue's head seq is called the queue distance -- * and used when comparing two tasks for ordering. See qmap_core_sched_before(). -- */ --static u64 core_sched_head_seqs[5]; --static u64 core_sched_tail_seqs[5]; -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local_dsq */ -- u64 core_sched_seq; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --/* Per-cpu dispatch index and remaining count */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(max_entries, 2); -- __type(key, u32); -- __type(value, u64); --} dispatch_idx_cnt SEC(".maps"); -- --/* Statistics */ --unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued; --unsigned long nr_core_sched_execed; -- --s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- struct task_ctx *tctx; -- s32 cpu; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- return cpu; -- -- return prev_cpu; --} -- --static int weight_to_idx(u32 weight) --{ -- /* Coarsely map the compound weight to a FIFO. */ -- if (weight <= 25) -- return 0; -- else if (weight <= 50) -- return 1; -- else if (weight < 200) -- return 2; -- else if (weight < 400) -- return 3; -- else -- return 4; --} -- --void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) --{ -- static u32 user_cnt, kernel_cnt; -- struct task_ctx *tctx; -- u32 pid = p->pid; -- int idx = weight_to_idx(p->scx.weight); -- void *ring; -- -- if (p->flags & PF_KTHREAD) { -- if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth)) -- return; -- } else { -- if (stall_user_nth && !(++user_cnt % stall_user_nth)) -- return; -- } -- -- if (test_error_cnt && !--test_error_cnt) -- scx_bpf_error("test triggering error"); -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * All enqueued tasks must have their core_sched_seq updated for correct -- * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in -- * qmap_ops.flags. -- */ -- tctx->core_sched_seq = core_sched_tail_seqs[idx]++; -- -- /* -- * If qmap_select_cpu() is telling us to or this is the last runnable -- * task on the CPU, enqueue locally. -- */ -- if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) { -- tctx->force_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); -- return; -- } -- -- /* -- * If the task was re-enqueued due to the CPU being preempted by a -- * higher priority scheduling class, just re-enqueue the task directly -- * on the global DSQ. As we want another CPU to pick it up, find and -- * kick an idle CPU. -- */ -- if (enq_flags & SCX_ENQ_REENQ) { -- s32 cpu; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags); -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- return; -- } -- -- ring = bpf_map_lookup_elem(&queue_arr, &idx); -- if (!ring) { -- scx_bpf_error("failed to find ring %d", idx); -- return; -- } -- -- /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */ -- if (bpf_map_push_elem(ring, &pid, 0)) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_enqueued, 1); --} -- --/* -- * The BPF queue map doesn't support removal and sched_ext can handle spurious -- * dispatches. qmap_dequeue() is only used to collect statistics. -- */ --void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags) --{ -- __sync_fetch_and_add(&nr_dequeued, 1); -- if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) -- __sync_fetch_and_add(&nr_core_sched_execed, 1); --} -- --static void update_core_sched_head_seq(struct task_struct *p) --{ -- struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- int idx = weight_to_idx(p->scx.weight); -- -- if (tctx) -- core_sched_head_seqs[idx] = tctx->core_sched_seq; -- else -- scx_bpf_error("task_ctx lookup failed"); --} -- --void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 zero = 0, one = 1; -- u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero); -- u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one); -- void *fifo; -- s32 pid; -- int i; -- -- if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) { -- struct task_struct *p; -- -- /* -- * PID 2 should be kthreadd which should mostly be idle and off -- * the scheduler. Let's keep dispatching it to force the kernel -- * to call this function over and over again. -- */ -- p = bpf_task_from_pid(2); -- if (p) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- if (!idx || !cnt) { -- scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt); -- return; -- } -- -- for (i = 0; i < 5; i++) { -- /* Advance the dispatch cursor and pick the fifo. */ -- if (!*cnt) { -- *idx = (*idx + 1) % 5; -- *cnt = 1 << *idx; -- } -- (*cnt)--; -- -- fifo = bpf_map_lookup_elem(&queue_arr, idx); -- if (!fifo) { -- scx_bpf_error("failed to find ring %llu", *idx); -- return; -- } -- -- /* Dispatch or advance. */ -- if (!bpf_map_pop_elem(fifo, &pid)) { -- struct task_struct *p; -- -- p = bpf_task_from_pid(pid); -- if (p) { -- update_core_sched_head_seq(p); -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- *cnt = 0; -- } --} -- --/* -- * The distance from the head of the queue scaled by the weight of the queue. -- * The lower the number, the older the task and the higher the priority. -- */ --static s64 task_qdist(struct task_struct *p) --{ -- int idx = weight_to_idx(p->scx.weight); -- struct task_ctx *tctx; -- s64 qdist; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return 0; -- } -- -- qdist = tctx->core_sched_seq - core_sched_head_seqs[idx]; -- -- /* -- * As queue index increments, the priority doubles. The queue w/ index 3 -- * is dispatched twice more frequently than 2. Reflect the difference by -- * scaling qdists accordingly. Note that the shift amount needs to be -- * flipped depending on the sign to avoid flipping priority direction. -- */ -- if (qdist >= 0) -- return qdist << (4 - idx); -- else -- return qdist << idx; --} -- --/* -- * This is called to determine the task ordering when core-sched is picking -- * tasks to execute on SMT siblings and should encode about the same ordering as -- * the regular scheduling path. Use the priority-scaled distances from the head -- * of the queues to compare the two tasks which should be consistent with the -- * dispatch path behavior. -- */ --bool BPF_STRUCT_OPS(qmap_core_sched_before, -- struct task_struct *a, struct task_struct *b) --{ -- return task_qdist(a) > task_qdist(b); --} -- --void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- u32 cnt; -- -- /* -- * Called when @cpu is taken by a higher priority scheduling class. This -- * makes @cpu no longer available for executing sched_ext tasks. As we -- * don't want the tasks in @cpu's local dsq to sit there until @cpu -- * becomes available again, re-enqueue them into the global dsq. See -- * %SCX_ENQ_REENQ handling in qmap_enqueue(). -- */ -- cnt = scx_bpf_reenqueue_local(); -- if (cnt) -- __sync_fetch_and_add(&nr_reenqueued, cnt); --} -- --s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (p->tgid == disallow_tgid) -- p->scx.disallow = true; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(qmap_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops qmap_ops = { -- .select_cpu = (void *)qmap_select_cpu, -- .enqueue = (void *)qmap_enqueue, -- .dequeue = (void *)qmap_dequeue, -- .dispatch = (void *)qmap_dispatch, -- .core_sched_before = (void *)qmap_core_sched_before, -- .cpu_release = (void *)qmap_cpu_release, -- .prep_enable = (void *)qmap_prep_enable, -- .init = (void *)qmap_init, -- .exit = (void *)qmap_exit, -- .flags = SCX_OPS_ENQ_LAST, -- .timeout_ms = 5000U, -- .name = "qmap", --}; -diff --git a/tools/sched_ext/scx_example_qmap.c b/tools/sched_ext/scx_example_qmap.c -deleted file mode 100644 -index ccb4814ee61b..000000000000 ---- a/tools/sched_ext/scx_example_qmap.c -+++ /dev/null -@@ -1,107 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_qmap.skel.h" -- --const char help_fmt[] = --"A simple five-level FIFO queue sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" --" -t COUNT Stall every COUNT'th user thread\n" --" -T COUNT Stall every COUNT'th kernel thread\n" --" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n" --" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_qmap *skel; -- struct bpf_link *link; -- int opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_qmap__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) { -- switch (opt) { -- case 's': -- skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; -- break; -- case 'e': -- skel->bss->test_error_cnt = strtoul(optarg, NULL, 0); -- break; -- case 't': -- skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0); -- break; -- case 'T': -- skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0); -- break; -- case 'l': -- skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0); -- break; -- case 'd': -- skel->rodata->disallow_tgid = strtol(optarg, NULL, 0); -- if (skel->rodata->disallow_tgid < 0) -- skel->rodata->disallow_tgid = getpid(); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_qmap__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.qmap_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- long nr_enqueued = skel->bss->nr_enqueued; -- long nr_dispatched = skel->bss->nr_dispatched; -- -- printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n", -- nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, -- skel->bss->nr_reenqueued, skel->bss->nr_dequeued, -- skel->bss->nr_core_sched_execed); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_qmap__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_simple.bpf.c b/tools/sched_ext/scx_example_simple.bpf.c -deleted file mode 100644 -index 4bccca3e2047..000000000000 ---- a/tools/sched_ext/scx_example_simple.bpf.c -+++ /dev/null -@@ -1,128 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple scheduler. -- * -- * By default, it operates as a simple global weighted vtime scheduler and can -- * be switched to FIFO scheduling. It also demonstrates the following niceties. -- * -- * - Statistics tracking how many tasks are queued to local and global dsq's. -- * - Termination notification for userspace. -- * -- * While very simple, this scheduler should work reasonably well on CPUs with a -- * uniform L3 cache topology. While preemption is not implemented, the fact that -- * the scheduling queue is shared across all CPUs means that whatever is at the -- * front of the queue is likely to be executed fairly quickly given enough -- * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads -- * but comes with the usual problems with FIFO scheduling where saturating -- * threads can easily drown out interactive ones. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --static u64 vtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, 2); /* [local, global] */ --} stats SEC(".maps"); -- --static void stat_inc(u32 idx) --{ -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); -- if (cnt_p) -- (*cnt_p)++; --} -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) --{ -- /* -- * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that -- * running @p on its CPU directly shouldn't affect fairness. Just queue -- * it on the local FIFO. -- */ -- if (enq_flags & SCX_ENQ_LOCAL) { -- stat_inc(0); /* count local queueing */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- return; -- } -- -- stat_inc(1); /* count global queueing */ -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) -- vtime = vtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --void BPF_STRUCT_OPS(simple_running, struct task_struct *p) --{ -- if (fifo_sched) -- return; -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(vtime_now, p->scx.dsq_vtime)) -- vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --s32 BPF_STRUCT_OPS(simple_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .running = (void *)simple_running, -- .stopping = (void *)simple_stopping, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", --}; -diff --git a/tools/sched_ext/scx_example_simple.c b/tools/sched_ext/scx_example_simple.c -deleted file mode 100644 -index 486b401f7c95..000000000000 ---- a/tools/sched_ext/scx_example_simple.c -+++ /dev/null -@@ -1,101 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_simple.skel.h" -- --const char help_fmt[] = --"A simple sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-f] [-p]\n" --"\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int simple) --{ -- exit_req = 1; --} -- --static void read_stats(struct scx_example_simple *skel, u64 *stats) --{ -- int nr_cpus = libbpf_num_possible_cpus(); -- u64 cnts[2][nr_cpus]; -- u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * 2); -- -- for (idx = 0; idx < 2; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_simple *skel; -- struct bpf_link *link; -- u32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_simple__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "fph")) != -1) { -- switch (opt) { -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_simple__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.simple_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- u64 stats[2]; -- -- read_stats(skel, stats); -- printf("local=%lu global=%lu\n", stats[0], stats[1]); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_simple__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_userland.bpf.c b/tools/sched_ext/scx_example_userland.bpf.c -deleted file mode 100644 -index a089bc6bbe86..000000000000 ---- a/tools/sched_ext/scx_example_userland.bpf.c -+++ /dev/null -@@ -1,269 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A minimal userland scheduler. -- * -- * In terms of scheduling, this provides two different types of behaviors: -- * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity. -- * All such tasks are direct-dispatched from the kernel, and are never -- * enqueued in user space. -- * 2. A primitive vruntime scheduler that is implemented in user space, for all -- * other tasks. -- * -- * Some parts of this example user space scheduler could be implemented more -- * efficiently using more complex and sophisticated data structures. For -- * example, rather than using BPF_MAP_TYPE_QUEUE's, -- * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between -- * user space and kernel space. Similarly, we use a simple vruntime-sorted list -- * in user space, but an rbtree could be used instead. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include --#include "scx_common.bpf.h" --#include "scx_example_userland_common.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; --const volatile s32 usersched_pid; -- --/* !0 for veristat, set during init */ --const volatile u32 num_possible_cpus = 64; -- --/* Stats that are printed by user space. */ --u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues; -- --struct user_exit_info uei; -- --/* -- * Whether the user space scheduler needs to be scheduled due to a task being -- * enqueued in user space. -- */ --static bool usersched_needed; -- --/* -- * The map containing tasks that are enqueued in user space from the kernel. -- * -- * This map is drained by the user space scheduler. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, struct scx_userland_enqueued_task); --} enqueued SEC(".maps"); -- --/* -- * The map containing tasks that are dispatched to the kernel from user space. -- * -- * Drained by the kernel in userland_dispatch(). -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, s32); --} dispatched SEC(".maps"); -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local DSQ */ --}; -- --/* Map that contains task-local storage. */ --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --static bool is_usersched_task(const struct task_struct *p) --{ -- return p->pid == usersched_pid; --} -- --static bool keep_in_kernel(const struct task_struct *p) --{ -- return p->nr_cpus_allowed < num_possible_cpus; --} -- --static struct task_struct *usersched_task(void) --{ -- struct task_struct *p; -- -- p = bpf_task_from_pid(usersched_pid); -- /* -- * Should never happen -- the usersched task should always be managed -- * by sched_ext. -- */ -- if (!p) { -- scx_bpf_error("Failed to find usersched task %d", usersched_pid); -- /* -- * We should never hit this path, and we error out of the -- * scheduler above just in case, so the scheduler will soon be -- * be evicted regardless. So as to simplify the logic in the -- * caller to not have to check for NULL, return an acquired -- * reference to the current task here rather than NULL. -- */ -- return bpf_task_acquire(bpf_get_current_task_btf()); -- } -- -- return p; --} -- --s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- if (keep_in_kernel(p)) { -- s32 cpu; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to look up task-local storage for %s", p->comm); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- tctx->force_local = true; -- return cpu; -- } -- } -- -- return prev_cpu; --} -- --static void dispatch_user_scheduler(void) --{ -- struct task_struct *p; -- -- usersched_needed = false; -- p = usersched_task(); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); --} -- --static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags) --{ -- struct scx_userland_enqueued_task task; -- -- memset(&task, 0, sizeof(task)); -- task.pid = p->pid; -- task.sum_exec_runtime = p->se.sum_exec_runtime; -- task.weight = p->scx.weight; -- -- if (bpf_map_push_elem(&enqueued, &task, 0)) { -- /* -- * If we fail to enqueue the task in user space, put it -- * directly on the global DSQ. -- */ -- __sync_fetch_and_add(&nr_failed_enqueues, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- __sync_fetch_and_add(&nr_user_enqueues, 1); -- usersched_needed = true; -- } --} -- --void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags) --{ -- if (keep_in_kernel(p)) { -- u64 dsq_id = SCX_DSQ_GLOBAL; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to lookup task ctx for %s", p->comm); -- return; -- } -- -- if (tctx->force_local) -- dsq_id = SCX_DSQ_LOCAL; -- tctx->force_local = false; -- scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags); -- __sync_fetch_and_add(&nr_kernel_enqueues, 1); -- return; -- } else if (!is_usersched_task(p)) { -- enqueue_task_in_user_space(p, enq_flags); -- } --} -- --void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (usersched_needed) -- dispatch_user_scheduler(); -- -- bpf_repeat(4096) { -- s32 pid; -- struct task_struct *p; -- -- if (bpf_map_pop_elem(&dispatched, &pid)) -- break; -- -- /* -- * The task could have exited by the time we get around to -- * dispatching it. Treat this as a normal occurrence, and simply -- * move onto the next iteration. -- */ -- p = bpf_task_from_pid(pid); -- if (!p) -- continue; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } --} -- --s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(userland_init) --{ -- if (num_possible_cpus == 0) { -- scx_bpf_error("User scheduler # CPUs uninitialized (%d)", -- num_possible_cpus); -- return -EINVAL; -- } -- -- if (usersched_pid <= 0) { -- scx_bpf_error("User scheduler pid uninitialized (%d)", -- usersched_pid); -- return -EINVAL; -- } -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops userland_ops = { -- .select_cpu = (void *)userland_select_cpu, -- .enqueue = (void *)userland_enqueue, -- .dispatch = (void *)userland_dispatch, -- .prep_enable = (void *)userland_prep_enable, -- .init = (void *)userland_init, -- .exit = (void *)userland_exit, -- .timeout_ms = 3000, -- .name = "userland", --}; -diff --git a/tools/sched_ext/scx_example_userland.c b/tools/sched_ext/scx_example_userland.c -deleted file mode 100644 -index 4152b1e65fe1..000000000000 ---- a/tools/sched_ext/scx_example_userland.c -+++ /dev/null -@@ -1,402 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext user space scheduler which provides vruntime semantics -- * using a simple ordered-list implementation. -- * -- * Each CPU in the system resides in a single, global domain. This precludes -- * the need to do any load balancing between domains. The scheduler could -- * easily be extended to support multiple domains, with load balancing -- * happening in user space. -- * -- * Any task which has any CPU affinity is scheduled entirely in BPF. This -- * program only schedules tasks which may run on any CPU. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include "user_exit_info.h" --#include "scx_example_userland_common.h" --#include "scx_example_userland.skel.h" -- --const char help_fmt[] = --"A minimal userland sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-b BATCH] [-p]\n" --"\n" --" -b BATCH The number of tasks to batch when dispatching (default: 8)\n" --" -p Don't switch all, switch only tasks on SCHED_EXT policy\n" --" -h Display this help and exit\n"; -- --/* Defined in UAPI */ --#define SCHED_EXT 7 -- --/* Number of tasks to batch when dispatching to user space. */ --static __u32 batch_size = 8; -- --static volatile int exit_req; --static int enqueued_fd, dispatched_fd; -- --static struct scx_example_userland *skel; --static struct bpf_link *ops_link; -- --/* Stats collected in user space. */ --static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches; -- --/* The data structure containing tasks that are enqueued in user space. */ --struct enqueued_task { -- LIST_ENTRY(enqueued_task) entries; -- __u64 sum_exec_runtime; -- double vruntime; --}; -- --/* -- * Use a vruntime-sorted list to store tasks. This could easily be extended to -- * a more optimal data structure, such as an rbtree as is done in CFS. We -- * currently elect to use a sorted list to simplify the example for -- * illustrative purposes. -- */ --LIST_HEAD(listhead, enqueued_task); -- --/* -- * A vruntime-sorted list of tasks. The head of the list contains the task with -- * the lowest vruntime. That is, the task that has the "highest" claim to be -- * scheduled. -- */ --static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head); -- --/* -- * The statically allocated array of tasks. We use a statically allocated list -- * here to avoid having to allocate on the enqueue path, which could cause a -- * deadlock. A more substantive user space scheduler could e.g. provide a hook -- * for newly enabled tasks that are passed to the scheduler from the -- * .prep_enable() callback to allows the scheduler to allocate on safe paths. -- */ --struct enqueued_task tasks[USERLAND_MAX_TASKS]; -- --static double min_vruntime; -- --static void sigint_handler(int userland) --{ -- exit_req = 1; --} -- --static __u32 task_pid(const struct enqueued_task *task) --{ -- return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task); --} -- --static int dispatch_task(s32 pid) --{ -- int err; -- -- err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d\n", pid); -- exit_req = 1; -- } else { -- nr_vruntime_dispatches++; -- } -- -- return err; --} -- --static struct enqueued_task *get_enqueued_task(__s32 pid) --{ -- if (pid >= USERLAND_MAX_TASKS) -- return NULL; -- -- return &tasks[pid]; --} -- --static double calc_vruntime_delta(__u64 weight, __u64 delta) --{ -- double weight_f = (double)weight / 100.0; -- double delta_f = (double)delta; -- -- return delta_f / weight_f; --} -- --static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task) --{ -- __u64 delta; -- -- delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime; -- -- enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta); -- if (min_vruntime > enqueued->vruntime) -- enqueued->vruntime = min_vruntime; -- enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime; --} -- --static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task) --{ -- struct enqueued_task *curr, *enqueued, *prev; -- -- curr = get_enqueued_task(bpf_task->pid); -- if (!curr) -- return ENOENT; -- -- update_enqueued(curr, bpf_task); -- nr_vruntime_enqueues++; -- -- /* -- * Enqueue the task in a vruntime-sorted list. A more optimal data -- * structure such as an rbtree could easily be used as well. We elect -- * to use a list here simply because it's less code, and thus the -- * example is less convoluted and better serves to illustrate what a -- * user space scheduler could look like. -- */ -- -- if (LIST_EMPTY(&vruntime_head)) { -- LIST_INSERT_HEAD(&vruntime_head, curr, entries); -- return 0; -- } -- -- LIST_FOREACH(enqueued, &vruntime_head, entries) { -- if (curr->vruntime <= enqueued->vruntime) { -- LIST_INSERT_BEFORE(enqueued, curr, entries); -- return 0; -- } -- prev = enqueued; -- } -- -- LIST_INSERT_AFTER(prev, curr, entries); -- -- return 0; --} -- --static void drain_enqueued_map(void) --{ -- while (1) { -- struct scx_userland_enqueued_task task; -- int err; -- -- if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) -- return; -- -- err = vruntime_enqueue(&task); -- if (err) { -- fprintf(stderr, "Failed to enqueue task %d: %s\n", -- task.pid, strerror(err)); -- exit_req = 1; -- return; -- } -- } --} -- --static void dispatch_batch(void) --{ -- __u32 i; -- -- for (i = 0; i < batch_size; i++) { -- struct enqueued_task *task; -- int err; -- __s32 pid; -- -- task = LIST_FIRST(&vruntime_head); -- if (!task) -- return; -- -- min_vruntime = task->vruntime; -- pid = task_pid(task); -- LIST_REMOVE(task, entries); -- err = dispatch_task(pid); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d in %u\n", -- pid, i); -- return; -- } -- } --} -- --static void *run_stats_printer(void *arg) --{ -- while (!exit_req) { -- __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; -- -- nr_failed_enqueues = skel->bss->nr_failed_enqueues; -- nr_kernel_enqueues = skel->bss->nr_kernel_enqueues; -- nr_user_enqueues = skel->bss->nr_user_enqueues; -- total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues; -- -- printf("o-----------------------o\n"); -- printf("| BPF ENQUEUES |\n"); -- printf("|-----------------------|\n"); -- printf("| kern: %10llu |\n", nr_kernel_enqueues); -- printf("| user: %10llu |\n", nr_user_enqueues); -- printf("| failed: %10llu |\n", nr_failed_enqueues); -- printf("| -------------------- |\n"); -- printf("| total: %10llu |\n", total); -- printf("| |\n"); -- printf("|-----------------------|\n"); -- printf("| VRUNTIME / USER |\n"); -- printf("|-----------------------|\n"); -- printf("| enq: %10llu |\n", nr_vruntime_enqueues); -- printf("| disp: %10llu |\n", nr_vruntime_dispatches); -- printf("o-----------------------o\n"); -- printf("\n\n"); -- sleep(1); -- } -- -- return NULL; --} -- --static int spawn_stats_thread(void) --{ -- pthread_t stats_printer; -- -- return pthread_create(&stats_printer, NULL, run_stats_printer, NULL); --} -- --static int bootstrap(int argc, char **argv) --{ -- int err; -- __u32 opt; -- struct sched_param sched_param = { -- .sched_priority = sched_get_priority_max(SCHED_EXT), -- }; -- bool switch_partial = false; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- /* -- * Enforce that the user scheduler task is managed by sched_ext. The -- * task eagerly drains the list of enqueued tasks in its main work -- * loop, and then yields the CPU. The BPF scheduler only schedules the -- * user space scheduler task when at least one other task in the system -- * needs to be scheduled. -- */ -- err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param); -- if (err) { -- fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err)); -- return err; -- } -- -- while ((opt = getopt(argc, argv, "b:ph")) != -1) { -- switch (opt) { -- case 'b': -- batch_size = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- exit(opt != 'h'); -- } -- } -- -- /* -- * It's not always safe to allocate in a user space scheduler, as an -- * enqueued task could hold a lock that we require in order to be able -- * to allocate. -- */ -- err = mlockall(MCL_CURRENT | MCL_FUTURE); -- if (err) { -- fprintf(stderr, "Failed to prefault and lock address space: %s\n", -- strerror(err)); -- return err; -- } -- -- skel = scx_example_userland__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno)); -- return errno; -- } -- skel->rodata->num_possible_cpus = libbpf_num_possible_cpus(); -- assert(skel->rodata->num_possible_cpus > 0); -- skel->rodata->usersched_pid = getpid(); -- assert(skel->rodata->usersched_pid > 0); -- skel->rodata->switch_partial = switch_partial; -- -- err = scx_example_userland__load(skel); -- if (err) { -- fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- enqueued_fd = bpf_map__fd(skel->maps.enqueued); -- dispatched_fd = bpf_map__fd(skel->maps.dispatched); -- assert(enqueued_fd > 0); -- assert(dispatched_fd > 0); -- -- err = spawn_stats_thread(); -- if (err) { -- fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops); -- if (!ops_link) { -- fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno)); -- err = errno; -- goto destroy_skel; -- } -- -- return 0; -- --destroy_skel: -- scx_example_userland__destroy(skel); -- exit_req = 1; -- return err; --} -- --static void sched_main_loop(void) --{ -- while (!exit_req) { -- /* -- * Perform the following work in the main user space scheduler -- * loop: -- * -- * 1. Drain all tasks from the enqueued map, and enqueue them -- * to the vruntime sorted list. -- * -- * 2. Dispatch a batch of tasks from the vruntime sorted list -- * down to the kernel. -- * -- * 3. Yield the CPU back to the system. The BPF scheduler will -- * reschedule the user space scheduler once another task has -- * been enqueued to user space. -- */ -- drain_enqueued_map(); -- dispatch_batch(); -- sched_yield(); -- } --} -- --int main(int argc, char **argv) --{ -- int err; -- -- err = bootstrap(argc, argv); -- if (err) { -- fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err)); -- return err; -- } -- -- sched_main_loop(); -- -- exit_req = 1; -- bpf_link__destroy(ops_link); -- uei_print(&skel->bss->uei); -- scx_example_userland__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_userland_common.h b/tools/sched_ext/scx_example_userland_common.h -deleted file mode 100644 -index 639c6809c5ff..000000000000 ---- a/tools/sched_ext/scx_example_userland_common.h -+++ /dev/null -@@ -1,19 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* Copyright (c) 2022 Meta, Inc */ -- --#ifndef __SCX_USERLAND_COMMON_H --#define __SCX_USERLAND_COMMON_H -- --#define USERLAND_MAX_TASKS 8192 -- --/* -- * An instance of a task that has been enqueued by the kernel for consumption -- * by a user space global scheduler thread. -- */ --struct scx_userland_enqueued_task { -- __s32 pid; -- u64 sum_exec_runtime; -- u64 weight; --}; -- --#endif // __SCX_USERLAND_COMMON_H -diff --git a/tools/sched_ext/user_exit_info.h b/tools/sched_ext/user_exit_info.h -deleted file mode 100644 -index e701ef0e0b86..000000000000 ---- a/tools/sched_ext/user_exit_info.h -+++ /dev/null -@@ -1,50 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Define struct user_exit_info which is shared between BPF and userspace parts -- * to communicate exit status and other information. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __USER_EXIT_INFO_H --#define __USER_EXIT_INFO_H -- --struct user_exit_info { -- int type; -- char reason[128]; -- char msg[1024]; --}; -- --#ifdef __bpf__ -- --#include "vmlinux.h" --#include -- --static inline void uei_record(struct user_exit_info *uei, -- const struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason); -- bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg); -- /* use __sync to force memory barrier */ -- __sync_val_compare_and_swap(&uei->type, uei->type, ei->type); --} -- --#else /* !__bpf__ */ -- --static inline bool uei_exited(struct user_exit_info *uei) --{ -- /* use __sync to force memory barrier */ -- return __sync_val_compare_and_swap(&uei->type, -1, -1); --} -- --static inline void uei_print(const struct user_exit_info *uei) --{ -- fprintf(stderr, "EXIT: %s", uei->reason); -- if (uei->msg[0] != '\0') -- fprintf(stderr, " (%s)", uei->msg); -- fputs("\n", stderr); --} -- --#endif /* __bpf__ */ --#endif /* __USER_EXIT_INFO_H */ -diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c -index 3a7bd62ef6b7..e8c98ab86084 100644 ---- a/drivers/cpufreq/cpufreq.c -+++ b/drivers/cpufreq/cpufreq.c -@@ -810,7 +810,7 @@ static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy, - /* - * show_scaling_governor - show the current policy for the specified CPU - */ --static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) -+ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) - { - if (policy->policy == CPUFREQ_POLICY_POWERSAVE) - return sprintf(buf, "powersave\n"); -@@ -825,7 +825,7 @@ static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) - /* - * store_scaling_governor - store policy for the specified CPU - */ --static ssize_t store_scaling_governor(struct cpufreq_policy *policy, -+ssize_t store_scaling_governor(struct cpufreq_policy *policy, - const char *buf, size_t count) - { - char str_governor[16]; -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -index ce05928392bf..467061a1ffa4 100755 ---- a/kernel/sched/hmbird/hmbird.c -+++ b/kernel/sched/hmbird/hmbird.c -@@ -633,14 +633,14 @@ static void init_isolate_cpus(void) - WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -- cpumask_set_cpu(0, iso_masks.big); -- cpumask_set_cpu(1, iso_masks.big); -- cpumask_set_cpu(2, iso_masks.big); -- cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(0, iso_masks.little); -+ cpumask_set_cpu(1, iso_masks.little); -+ cpumask_set_cpu(2, iso_masks.little); -+ cpumask_set_cpu(3, iso_masks.little); - cpumask_set_cpu(4, iso_masks.big); -- cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(5, iso_masks.big); - cpumask_set_cpu(6, iso_masks.partial); -- cpumask_set_cpu(7, iso_masks.ex_free); -+ cpumask_set_cpu(7, iso_masks.exclusive); - } - - extern spinlock_t css_set_lock; -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index 692c9aa0ffa6..c536364f4736 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -10942,7 +10942,6 @@ void sched_move_task(struct task_struct *tsk) - { - int queued, running, queue_flags = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; -- struct task_group *group; - struct rq_flags rf; - struct rq *rq; - -@@ -10958,7 +10957,7 @@ void sched_move_task(struct task_struct *tsk) - if (running) - put_prev_task(rq, tsk); - -- sched_change_group(tsk, group); -+ sched_change_group(tsk); - - if (queued) - enqueue_task(rq, tsk, queue_flags); diff --git a/oneplus/hmbird/fengchi_OP-PAD-2-PRO_A15.patch b/oneplus/hmbird/fengchi_OP-PAD-2-PRO_A15.patch deleted file mode 100644 index dcf80f96..00000000 --- a/oneplus/hmbird/fengchi_OP-PAD-2-PRO_A15.patch +++ /dev/null @@ -1,9137 +0,0 @@ -diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig -index 60af93c04b45..5ba590235a8c 100644 ---- a/arch/arm64/configs/defconfig -+++ b/arch/arm64/configs/defconfig -@@ -6,6 +6,7 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_BPF_SYSCALL=y - CONFIG_BPF_JIT=y - CONFIG_PREEMPT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_BSD_PROCESS_ACCT=y - CONFIG_BSD_PROCESS_ACCT_V3=y -diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig -index bebb76e87e97..a0c91fd40bf6 100644 ---- a/arch/arm64/configs/gki_defconfig -+++ b/arch/arm64/configs/gki_defconfig -@@ -10,6 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y - # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set - CONFIG_BPF_LSM=y - CONFIG_PREEMPT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_TASKSTATS=y - CONFIG_TASK_DELAY_ACCT=y -diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c -index 4bd028fa7500..5825ce9d6d7b 100755 ---- a/drivers/gpu/drm/msm/msm_drv.c -+++ b/drivers/gpu/drm/msm/msm_drv.c -@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) - } - - sched_set_fifo(ev_thread->worker->task); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3); -+#endif - } - - ret = drm_vblank_init(ddev, priv->num_crtcs); -diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h -index 63029bc7c9dd..c0fee44f5417 100755 ---- a/include/asm-generic/vmlinux.lds.h -+++ b/include/asm-generic/vmlinux.lds.h -@@ -131,6 +131,7 @@ - *(__dl_sched_class) \ - *(__rt_sched_class) \ - *(__fair_sched_class) \ -+ *(__hmbird_sched_class) \ - *(__idle_sched_class) \ - __sched_class_lowest = .; - -diff --git a/include/linux/sched.h b/include/linux/sched.h -index c2fb5603e211..39c797c5cc75 100755 ---- a/include/linux/sched.h -+++ b/include/linux/sched.h -@@ -73,6 +73,9 @@ struct task_delay_info; - struct task_group; - struct user_event_mm; - -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - - /* - * Task state bitmask. NOTE! These bits are also -@@ -1921,6 +1924,91 @@ static inline int task_nice(const struct task_struct *p) - return PRIO_TO_NICE((p)->static_prio); - } - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline bool task_is_top_task(struct task_struct *p) -+{ -+ return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ ->top_task_prop & TOP_TASK_BITS_MASK); -+} -+ -+static inline int get_top_task_prop(struct task_struct *p) -+{ -+ return get_hmbird_ts(p)->top_task_prop; -+} -+ -+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear) -+{ -+ if (set) -+ get_hmbird_ts(p)->top_task_prop |= set; -+ if (clear) -+ get_hmbird_ts(p)->top_task_prop &= ~clear; -+ return 0; -+} -+ -+static inline void reset_top_task_prop(struct task_struct *p) -+{ -+ get_hmbird_ts(p)->top_task_prop = 0; -+} -+ -+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ entity->sched_prop = sp; -+ } -+ return 0; -+} -+ -+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->sched_prop; -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq) -+{ -+ unsigned long new_dsq; -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq; -+ entity->sched_prop = new_dsq; -+ } -+} -+ -+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK); -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ entity->dsq_sync_ux = val; -+} -+ -+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->dsq_sync_ux; -+ else -+ return 0; -+} -+#endif - - extern int can_nice(const struct task_struct *p, const int nice); - extern int task_curr(const struct task_struct *p); -diff --git a/include/linux/sched/hmbird.h b/include/linux/sched/hmbird.h -new file mode 100755 -index 000000000000..3c7861e37ade ---- /dev/null -+++ b/include/linux/sched/hmbird.h -@@ -0,0 +1,381 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class: Documentation/scheduler/hmbird.rst -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef _LINUX_SCHED_HMBIRD_H -+#define _LINUX_SCHED_HMBIRD_H -+ -+#define HMBIRD_TS_IDX 1 -+#define HMBIRD_OPS_IDX 14 -+#define HMBIRD_RQ_IDX 15 -+ -+#define get_hmbird_ts(p) \ -+ ((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ -+#define get_hmbird_rq(rq) \ -+ ((struct hmbird_rq *)(rq->android_oem_data1[HMBIRD_RQ_IDX])) -+ -+#define get_hmbird_ops(rq) \ -+ ((struct hmbird_ops *)(rq->android_oem_data1[HMBIRD_OPS_IDX])) -+ -+#define SCHED_PROP_DEADLINE_MASK (0xFF) /* deadline for ext sched class */ -+/* -+ * Every task has a DEADLINE_LEVEL which stands for -+ * max schedule latency this task can afford. LEVEL1~5 -+ * for user-aware tasks, LEVEL6~9 for other tasks. -+ */ -+#define SCHED_PROP_DEADLINE_LEVEL0 (0) -+#define SCHED_PROP_DEADLINE_LEVEL1 (1) -+#define SCHED_PROP_DEADLINE_LEVEL2 (2) -+#define SCHED_PROP_DEADLINE_LEVEL3 (3) -+#define SCHED_PROP_DEADLINE_LEVEL4 (4) -+#define SCHED_PROP_DEADLINE_LEVEL5 (5) -+#define SCHED_PROP_DEADLINE_LEVEL6 (6) -+#define SCHED_PROP_DEADLINE_LEVEL7 (7) -+#define SCHED_PROP_DEADLINE_LEVEL8 (8) -+#define SCHED_PROP_DEADLINE_LEVEL9 (9) -+/* -+ * Distinguish tasks into periodical tasks which requires -+ * low schedule latency and non-periodical tasks which are -+ * not sensitive to schedule latency. -+ */ -+#define SCHED_HMBIRD_DSQ_TYPE_PERIOD (0) /* period dsq of hmbird */ -+#define SCHED_HMBIRD_DSQ_TYPE_NON_PERIOD (1) /* non period dsq of hmbird */ -+ -+#define TOP_TASK_BITS_MASK (0xFF) -+#define TOP_TASK_BITS (8) -+#include -+ -+extern atomic_t non_hmbird_task; -+extern atomic_t __hmbird_ops_enabled; -+#define hmbird_enabled() atomic_read(&__hmbird_ops_enabled) -+#define MAX_GLOBAL_DSQS (10) -+ -+enum hmbird_consts { -+ HMBIRD_SLICE_DFL = 1 * NSEC_PER_MSEC, -+ HMBIRD_SLICE_ISO = 8 * HMBIRD_SLICE_DFL, -+ HMBIRD_SLICE_INF = U64_MAX, /* infinite, implies nohz */ -+}; -+ -+/* -+ * DSQ (dispatch queue) IDs are 64bit of the format: -+ * -+ * Bits: [63] [62 .. 0] -+ * [ B] [ ID ] -+ * -+ * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -+ * ID: 63 bit ID -+ * -+ * Built-in IDs: -+ * -+ * Bits: [63] [62] [61..32] [31 .. 0] -+ * [ 1] [ L] [ R ] [ V ] -+ * -+ * 1: 1 for built-in DSQs. -+ * L: 1 for LOCAL_ON DSQ IDs, 0 for others -+ * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -+ */ -+enum hmbird_dsq_id_flags { -+ HMBIRD_DSQ_FLAG_BUILTIN = 1LLU << 63, -+ HMBIRD_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -+ -+ HMBIRD_DSQ_INVALID = HMBIRD_DSQ_FLAG_BUILTIN | 0, -+ HMBIRD_DSQ_GLOBAL = HMBIRD_DSQ_FLAG_BUILTIN | 1, -+ HMBIRD_DSQ_LOCAL = HMBIRD_DSQ_FLAG_BUILTIN | 2, -+ HMBIRD_DSQ_LOCAL_ON = HMBIRD_DSQ_FLAG_BUILTIN | HMBIRD_DSQ_FLAG_LOCAL_ON, -+ HMBIRD_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, -+}; -+ -+enum hmbird_switch_type { -+ HMBIRD_SWITCH_PROC, -+ HMBIRD_SWITCH_ERR_WDT, -+ HMBIRD_SWITCH_ERR_HB, -+ HMBIRD_SWITCH_ERR_DSQ, -+ HMBIRD_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ -+ HMBIRD_EXIT_ERROR_HEARTBEAT, /* heart beat has stopped */ -+}; -+ -+/* -+ * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -+ * scheduler core and the BPF scheduler. See the documentation for more details. -+ */ -+struct hmbird_dispatch_q { -+ raw_spinlock_t lock; -+ struct list_head fifo; /* processed in dispatching order */ -+ struct rb_root_cached priq; -+ u32 nr; -+ u64 id; -+ struct llist_node free_node; -+ struct rcu_head rcu; -+ u64 last_consume_at; -+ bool is_timeout; -+}; -+ -+/* hmbird_entity.flags */ -+enum hmbird_ent_flags { -+ HMBIRD_TASK_QUEUED = 1 << 0, /* on hmbird runqueue */ -+ HMBIRD_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -+ HMBIRD_TASK_ENQ_LOCAL = 1 << 2, /* used by hmbird_select_cpu_dfl, set HMBIRD_ENQ_LOCAL */ -+ -+ HMBIRD_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -+ HMBIRD_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -+ -+ HMBIRD_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -+ HMBIRD_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -+ -+ HMBIRD_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ -+}; -+ -+/* hmbird_entity.dsq_flags */ -+enum hmbird_ent_dsq_flags { -+ HMBIRD_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ -+}; -+ -+#define RAVG_HIST_SIZE 5 -+struct hmbird_sched_task_stats { -+ u64 mark_start; -+ u64 window_start; -+ u32 sum; -+ u32 sum_history[RAVG_HIST_SIZE]; -+ int cidx; -+ -+ u32 demand; -+ u16 demand_scaled; -+ void *sdsq; -+}; -+ -+struct hmbird_sched_rq_stats { -+ u64 window_start; -+ u64 latest_clock; -+ u32 prev_window_size; -+ u64 task_exec_scale; -+ u64 prev_runnable_sum; -+ u64 curr_runnable_sum; -+ int *sched_ravg_window_ptr; -+}; -+ -+/* -+ * The following is embedded in task_struct and contains all fields necessary -+ * for a task to be scheduled by HMBIRD. -+ */ -+struct hmbird_entity { -+ struct hmbird_dispatch_q *dsq; -+ struct { -+ struct list_head fifo; /* dispatch order */ -+ struct rb_node priq; -+ } dsq_node; -+ struct list_head watchdog_node; -+ u32 flags; /* protected by rq lock */ -+ u32 dsq_flags; /* protected by dsq lock */ -+ u32 weight; -+ s32 sticky_cpu; -+ s32 holding_cpu; -+ u32 kf_mask; /* see hmbird_kf_mask above */ -+ struct task_struct *kf_tasks[2]; /* see HMBIRD_CALL_OP_TASK() */ -+ atomic64_t ops_state; -+ unsigned long runnable_at; -+ u64 slice; -+ u64 dsq_vtime; -+ bool disallow; /* reject switching into HMBIRD */ -+ u16 demand_scaled; -+ -+ /* cold fields */ -+ struct list_head tasks_node; -+ struct task_struct *task; -+ const struct sched_class *sched_class; -+ unsigned long sched_prop; -+ unsigned long top_task_prop; -+ struct hmbird_sched_task_stats sts; -+ unsigned long running_at; -+ int gdsq_idx; -+ -+ s32 critical_affinity_cpu; -+ int dsq_sync_ux; -+}; -+ -+ -+/* -+ * All variables use 64bits width, -+ * Avoid parsing problems caused by automatic alignment(with padding) of structures. -+ */ -+ -+/* NOTING : Must align to 64bits. */ -+#define DESC_STR_LEN (32) -+/* Supporti up to three-dimensional arrays. */ -+#define PARSE_DIMENS (3) -+struct meta_desc_t { -+ char desc_str[DESC_STR_LEN]; -+ u64 len; -+ u64 parse[PARSE_DIMENS]; -+}; -+ -+#define MAX_SWITCHS (5) -+struct hmbird_switch_t { -+ u64 switch_at; -+ u64 is_success; -+ u64 end_state; -+ u64 switch_reason; -+}; -+#define SWITCH_ITEMS (sizeof(struct hmbird_switch_t) / sizeof(u64)) -+ -+#define MAX_EXCEPS (5) -+enum excep_id { -+ NO_CGROUP_L1, -+ MODULE_UNLOAD, -+ ALREADY_ENABLED, -+ ALREADY_DISABLED, -+ INIT_TASK_FAIL, -+ ALLOC_RQSCX_FAIL, -+ DSQ_ID_ERR, -+ CPU_NO_MASK, -+ SCAN_ENTITY_NULL, -+ ITER_RET_NULL, -+ DEQ_DEQING, -+ ENQ_EXIST1, -+ ENQ_EXIST2, -+ TASK_LINKED1, -+ TASK_UNLINKED, -+ TASK_LINKED2, -+ TASK_UNQUED, -+ TASK_UNWATCHED, -+ HMBIRD_OPN, -+ TASK_WATCHED, -+ RQ_NO_RUNNING, -+ EXTRA_FLAGS, -+ HOLDING_CPU1, -+ HOLDING_CPU2, -+ TASK_OPS_PREPPED, -+ TASK_OPS_UNPREPPED, -+ HMBIRD_OPS_ERR, -+ MAX_EXCEP_ID, -+}; -+ -+struct snap_misc_t { -+ u64 hmbird_enabled; -+ u64 curr_ss; -+ u64 hmbird_ops_enable_state_var; -+ u64 non_ext_task; -+ u64 parctrl_high_ratio; -+ u64 parctrl_low_ratio; -+ u64 parctrl_high_ratio_l; -+ u64 parctrl_low_ratio_l; -+ u64 isoctrl_high_ratio; -+ u64 isoctrl_low_ratio; -+ u64 misfit_ds; -+ u64 partial_enable; -+ u64 iso_free_rescue; -+ u64 isolate_ctrl; -+ u64 snap_jiffies; -+ u64 snap_time; -+}; -+#define SNAP_ITEMS (sizeof(struct snap_misc_t) / sizeof(u64)) -+ -+struct panic_snapshot_t { -+ /* what time dose the first task of dsq turn in runnable, check starvation */ -+ struct meta_desc_t runnable_at_meta; -+ u64 runnable_at[MAX_GLOBAL_DSQS]; -+ -+ struct meta_desc_t rq_nr_meta; -+ u64 rq_nr[NR_CPUS]; -+ -+ struct meta_desc_t scxrq_nr_meta; -+ u64 scxrq_nr[NR_CPUS]; -+ -+ struct meta_desc_t snap_misc_meta; -+ struct snap_misc_t snap_misc; -+}; -+ -+/* -+ * Do not record info in hot paths unless absolutely necessary, -+ * The impact on performance should be minimized. -+ */ -+struct kernel_info_t { -+ struct meta_desc_t sw_rec_meta; -+ struct hmbird_switch_t sw_rec[MAX_SWITCHS]; -+ -+ struct meta_desc_t sw_idx_meta; -+ u64 sw_idx; -+ -+ struct meta_desc_t excep_rec_meta; -+ u64 excep_rec[MAX_EXCEP_ID][MAX_EXCEPS]; -+ -+ struct meta_desc_t excep_idx_meta; -+ u64 excep_idx[MAX_EXCEP_ID]; -+ -+ /* snapshot while panic. */ -+ struct panic_snapshot_t snap; -+}; -+ -+struct ko_info_t {}; -+ -+struct md_meta_t { -+ u64 self_len; -+ u64 unit_size; -+ u64 desc_meta_len; -+ u64 desc_str_len; -+ u64 switches; -+ u64 exceps; -+ u64 global_dsqs; -+ u64 parse_dimens; -+ u64 nr_cpus; -+ u64 real_cpus; -+ u64 nr_meta_desc; -+ u64 dump_real_size; -+}; -+ -+struct md_info_t { -+ struct md_meta_t meta; -+ struct kernel_info_t kern_dump; -+ struct ko_info_t ko_dump; -+}; -+ -+static inline void exceps_update(struct md_info_t *rec, int id, unsigned long jiffies) -+{ -+ u64 *idx; -+ -+ if (!rec) -+ return; -+ -+ idx = &rec->kern_dump.excep_idx[id]; -+ rec->kern_dump.excep_rec[id][*idx] = jiffies; -+ *idx = ++(*idx) % MAX_EXCEPS; -+} -+ -+static inline void sw_update(struct md_info_t *rec, u64 switch_at, -+ u64 is_success, u64 end_state, u64 switch_reason) -+{ -+ u64 *idx; -+ -+ if (!rec) -+ return; -+ -+ idx = &rec->kern_dump.sw_idx; -+ rec->kern_dump.sw_rec[*idx].switch_at = switch_at; -+ rec->kern_dump.sw_rec[*idx].is_success = is_success; -+ rec->kern_dump.sw_rec[*idx].end_state = end_state; -+ rec->kern_dump.sw_rec[*idx].switch_reason = switch_reason; -+ *idx = ++(*idx) % MAX_SWITCHS; -+} -+ -+struct hmbird_ops { -+ bool (*scx_enable)(void); -+ bool (*check_non_task)(void); -+ void (*do_sched_yield_before)(long *skip); -+ void (*window_rollover_run_once)(struct rq *rq); -+ void (*hmbird_get_md_info)(unsigned long *vaddr, unsigned long *size); -+}; -+ -+void hmbird_free(struct task_struct *p); -+ -+enum DSQ_SYNC_UX_FLAG { -+ DSQ_SYNC_UX_NONE = 0, -+ DSQ_SYNC_STATIC_UX = 1, -+ DSQ_SYNC_INHERIT_UX = 1 << 1, -+}; -+ -+#endif /* _LINUX_SCHED_HMBIRD_H */ -diff --git a/include/linux/sched/hmbird_proc_val.h b/include/linux/sched/hmbird_proc_val.h -new file mode 100755 -index 000000000000..e59708b16ea3 ---- /dev/null -+++ b/include/linux/sched/hmbird_proc_val.h -@@ -0,0 +1,22 @@ -+#ifndef __HMBORD_PROC_VAL_H__ -+#define __HMBORD_PROC_VAL_H__ -+ -+extern int scx_enable; -+extern int partial_enable; -+extern int cpuctrl_high_ratio; -+extern int cpuctrl_low_ratio; -+extern int slim_stats; -+extern int hmbirdcore_debug; -+extern int slim_for_app; -+extern int misfit_ds; -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+extern int cpu7_tl; -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int slim_gov_debug; -+extern int scx_gov_ctrl; -+extern int sched_ravg_window_frame_per_sec; -+ -+#endif -\ No newline at end of file -diff --git a/include/linux/sched/hmbird_version.h b/include/linux/sched/hmbird_version.h -new file mode 100755 -index 000000000000..b2843881c26f ---- /dev/null -+++ b/include/linux/sched/hmbird_version.h -@@ -0,0 +1,52 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#ifndef _OPLUS_HMBIRD_VERSION_H_ -+#define _OPLUS_HMBIRD_VERSION_H_ -+#include -+#include -+#include -+ -+enum hmbird_version { -+ HMBIRD_UNINIT, -+ HMBIRD_GKI_VERSION, -+ HMBIRD_OGKI_VERSION, -+ HMBIRD_UNKNOW_VERSION, -+}; -+ -+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT; -+ -+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type" -+ -+static inline enum hmbird_version get_hmbird_version_type(void) -+{ -+ struct device_node *np = NULL; -+ const char *hmbird_version_str = NULL; -+ if (HMBIRD_UNINIT != hmbird_version_type) -+ return hmbird_version_type; -+ np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH); -+ if (np) { -+ of_property_read_string(np, "type", &hmbird_version_str); -+ if (NULL != hmbird_version_str) { -+ if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) { -+ hmbird_version_type = HMBIRD_OGKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi"); -+ } else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) { -+ hmbird_version_type = HMBIRD_GKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi"); -+ } else { -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi"); -+ } -+ return hmbird_version_type; -+ } -+ } -+ -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION"); -+ return hmbird_version_type; -+} -+ -+#endif /*_OPLUS_HMBIRD_VERSION_H_ */ -diff --git a/include/linux/sched/sa_common.h b/include/linux/sched/sa_common.h -new file mode 100755 -index 000000000000..6f76d7cfd7bf ---- /dev/null -+++ b/include/linux/sched/sa_common.h -@@ -0,0 +1,797 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_COMMON_H_ -+#define _OPLUS_SA_COMMON_H_ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+#include -+#endif -+#include -+#include -+#include -+#include -+#include -+ -+#include "sa_oemdata.h" -+#include "sa_common_struct.h" -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) -+#define OPLUS_UX_EEVDF_COMPATIBLE 1 -+#endif -+ -+#define SA_DEBUG_ON 0 -+ -+#if (SA_DEBUG_ON >= 1) -+#define DEBUG_BUG_ON(x) BUG_ON(x) -+#define DEBUG_WARN_ON(x) WARN_ON(x) -+#else -+#define DEBUG_BUG_ON(x) -+#define DEBUG_WARN_ON(x) -+#endif -+ -+#define ux_err(fmt, ...) \ -+ pr_err("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_warn(fmt, ...) \ -+ pr_warn("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_debug(fmt, ...) \ -+ pr_info("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+ -+#define UX_MSG_LEN 64 -+#define UX_DEPTH_MAX 5 -+ -+/* define for debug */ -+#define DEBUG_SYSTRACE (1 << 0) -+#define DEBUG_FTRACE (1 << 1) -+#define DEBUG_KMSG (1 << 2) /* used for frameboost */ -+#define DEBUG_PIPELINE (1 << 3) -+#define DEBUG_DYNAMIC_HZ (1 << 4) -+#define DEBUG_DYNAMIC_PREEMPT (1 << 5) -+#define DEBUG_AMU_INSTRUCTION (1 << 6) -+#define DEBUG_VERBOSE (1 << 10) /* used for frameboost */ -+#define DEBUG_SET_DSQ_ID (1 << 11) /* used for hmbird */ -+ -+/* define for sched assist feature */ -+#define FEATURE_COMMON (1 << 0) -+#define FEATURE_SPREAD (1 << 1) -+ -+#define UX_EXEC_SLICE (4000000U) -+ -+/* define for sched assist thread type, keep same as the define in java file */ -+#define SA_OPT_CLEAR (0) -+#define SA_TYPE_LIGHT (1 << 0) -+#define SA_TYPE_HEAVY (1 << 1) -+#define SA_TYPE_ANIMATOR (1 << 2) -+/* SA_TYPE_LISTPICK for camera */ -+#define SA_TYPE_LISTPICK (1 << 3) -+#define SA_TYPE_MQ_VIP (1 << 4) -+#define SA_OPT_SET (1 << 7) -+#define SA_OPT_RESET (1 << 8) -+#define SA_OPT_SET_PRIORITY (1 << 9) -+ -+/* The following ux value only used in kernel */ -+#define SA_TYPE_SWIFT (1 << 14) -+/* clear ux type when dequeue */ -+#define SA_TYPE_ONCE (1 << 15) -+#define SA_TYPE_INHERIT (1 << 16) -+/* SA_TYPE_COMBINED isn't marked in ux_state, it only exists in return value of function */ -+#define SA_TYPE_COMBINED (1 << 17) -+#define SA_TYPE_URGENT_MASK (SA_TYPE_LIGHT|SA_TYPE_ANIMATOR|SA_TYPE_SWIFT) -+#define SCHED_ASSIST_UX_MASK (SA_TYPE_LIGHT|SA_TYPE_HEAVY|SA_TYPE_ANIMATOR|SA_TYPE_LISTPICK|SA_TYPE_SWIFT) -+ -+/* load balance operation is performed on the following ux types */ -+#define SCHED_ASSIST_LB_UX (SA_TYPE_SWIFT | SA_TYPE_ANIMATOR | SA_TYPE_LIGHT | SA_TYPE_HEAVY) -+#define POSSIBLE_UX_MASK (SA_TYPE_SWIFT | \ -+ SA_TYPE_LIGHT | \ -+ SA_TYPE_HEAVY | \ -+ SA_TYPE_ANIMATOR | \ -+ SA_TYPE_LISTPICK | \ -+ SA_TYPE_ONCE | \ -+ SA_TYPE_INHERIT) -+ -+#define SCHED_ASSIST_UX_PRIORITY_MASK (0xFF000000) -+#define SCHED_ASSIST_UX_PRIORITY_SHIFT 24 -+ -+#define UX_PRIORITY_TOP_APP 0x0A000000 -+#define UX_PRIORITY_AUDIO 0x0A000000 -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+#define UX_PRIORITY_PIPELINE_UI 0x06000000 -+#define UX_PRIORITY_PIPELINE 0x05000000 -+#endif -+ -+/* define for sched assist scene type, keep same as the define in java file */ -+#define SA_SCENE_OPT_CLEAR (0) -+#define SA_LAUNCH (1 << 0) -+#define SA_SLIDE (1 << 1) -+#define SA_CAMERA (1 << 2) -+#define SA_ANIM_START (1 << 3) /* we care about both launcher and top app */ -+#define SA_ANIM (1 << 4) /* we only care about launcher */ -+#define SA_INPUT (1 << 5) -+#define SA_LAUNCHER_SI (1 << 6) -+#define SA_SCENE_OPT_SET (1 << 7) -+ -+#define ROOT_UID 0 -+#define SYSTEM_UID 1000 -+#define FIRST_APPLICATION_UID 10000 -+#define LAST_APPLICATION_UID 19999 -+#define PER_USER_RANGE 100000 -+/* define for clear IM_FLAG -+if im_flag is 70, it should clear im_flag_audio (70 - 64 = 6) -+eg: gerrit patchset "30438485" -+ */ -+#define IM_FLAG_CLEAR 64 -+ -+extern pid_t save_audio_tgid; -+extern pid_t save_top_app_tgid; -+extern unsigned int top_app_type; -+extern int global_lowend_plat_opt; -+ -+#ifdef CONFIG_OPLUS_SCHED_HALT_MASK_PRT -+/* This must be the same as the definition of pause_type in walt_halt.c */ -+enum oplus_pause_type { -+ OPLUS_HALT, -+ OPLUS_PARTIAL_HALT, -+ -+ OPLUS_MAX_PAUSE_TYPE -+}; -+extern cpumask_t cur_cpus_halt_mask; -+extern cpumask_t cur_cpus_phalt_mask; -+DECLARE_PER_CPU(int[OPLUS_MAX_PAUSE_TYPE], oplus_cur_pause_client); -+extern void sa_corectl_systrace_c(void); -+#endif /* CONFIG_OPLUS_SCHED_HALT_MASK_PRT */ -+ -+/* define for boost threshold unit */ -+#define BOOST_THRESHOLD_UNIT (51) -+ -+ -+enum UX_STATE_TYPE { -+ UX_STATE_INVALID = 0, -+ UX_STATE_NONE, -+ UX_STATE_STATIC, -+ UX_STATE_INHERIT, -+ UX_STATE_COMBINED, -+ MAX_UX_STATE_TYPE, -+}; -+ -+enum INHERIT_UX_TYPE { -+ INHERIT_UX_BINDER = 0, -+ INHERIT_UX_RWSEM, -+ INHERIT_UX_MUTEX, -+ INHERIT_UX_FUTEX, -+ INHERIT_UX_PIFUTEX, -+ INHERIT_UX_MAX, -+}; -+ -+/* -+ * WANNING: -+ * new flag should be add before MAX_IM_FLAG_TYPE, never change -+ * the value of those existed flag type. -+ */ -+enum IM_FLAG_TYPE { -+ IM_FLAG_NONE = 0, -+ IM_FLAG_SURFACEFLINGER, -+ IM_FLAG_HWC, /* Discarded */ -+ IM_FLAG_RENDERENGINE, -+ IM_FLAG_WEBVIEW, -+ IM_FLAG_CAMERA_HAL, -+ IM_FLAG_AUDIO, -+ IM_FLAG_HWBINDER, -+ IM_FLAG_LAUNCHER, -+ IM_FLAG_LAUNCHER_NON_UX_RENDER, -+ IM_FLAG_SS_LOCK_OWNER, -+ IM_FLAG_FORBID_SET_CPU_AFFINITY = 11, /* forbid setting cpu affinity from app */ -+ IM_FLAG_SYSTEMSERVER_PID, -+ IM_FLAG_MIDASD, -+ IM_FLAG_AUDIO_CAMERA_HAL, /* audio mode disable camera hal ux */ -+ IM_FLAG_AFFINITY_THREAD, -+ IM_FLAG_TPD_SET_CPU_AFFINITY = 16, -+ IM_FLAG_COMPRESS_THREAD = 17, /* compress thread skips locking protect */ -+ IM_FLAG_RENDER_THREAD = 18, -+ MAX_IM_FLAG_TYPE, -+}; -+ -+#define MAX_IM_FLAG_PRIO MAX_IM_FLAG_TYPE -+ -+enum ots_state { -+ OTS_STATE_SET_AFFINITY, -+ OTS_STATE_DDL_ACTIVE, -+ OTS_STATE_DDL_ACTIVE_PREEMPTED, -+ OTS_STATE_MAX, -+}; -+ -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+DECLARE_PER_CPU(u64, retired_instrs); -+DECLARE_PER_CPU(u64, nvcsw); -+DECLARE_PER_CPU(u64, nivcsw); -+#endif -+ -+struct ux_sched_cluster { -+ struct cpumask cpus; -+ unsigned long capacity; -+}; -+ -+#define OPLUS_NR_CPUS (8) -+#define OPLUS_MAX_CLS (5) -+struct ux_sched_cputopo { -+ int cls_nr; -+ struct ux_sched_cluster sched_cls[OPLUS_NR_CPUS]; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ cpumask_t oplus_cpu_array[2*OPLUS_MAX_CLS][OPLUS_MAX_CLS]; -+#endif -+}; -+ -+struct oplus_rq { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_root_cached ux_list; -+ /* a tree to track minimum exec vruntime */ -+ struct rb_root_cached exec_timeline; -+ /* malloc this spinlock_t instead of built-in to shrank size of oplus_rq */ -+ spinlock_t *ux_list_lock; -+ int nr_running; -+ u64 min_vruntime; -+ u64 load_weight; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) -+ struct rb_root_cached ddl_root; -+ spinlock_t *ddl_lock; -+#endif -+ -+#ifdef CONFIG_LOCKING_PROTECT -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ struct list_head locking_thread_list; -+ spinlock_t *locking_list_lock; -+ int rq_locking_task; -+#else -+ struct sched_entity *last_entity; -+#endif -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ struct oplus_lb lb; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+ struct hrtimer *resched_timer; -+ int cpu; -+#endif -+}; -+ -+extern int global_debug_enabled; -+extern int global_sched_assist_enabled; -+extern int global_sched_assist_scene; -+extern int global_silver_perf_core; -+extern int global_sched_group_enabled; -+ -+struct rq; -+ -+#ifdef CONFIG_LOCKING_PROTECT -+struct sched_assist_locking_ops { -+ void (*check_preempt_tick)(struct task_struct *p, -+ unsigned long *ideal_runtime, bool *skip_preempt, -+ unsigned long delta_exec, struct cfs_rq *cfs_rq, -+ struct sched_entity *curr, unsigned int granularity); -+ void (*check_preempt_wakeup)(struct rq *rq, struct task_struct *p, bool *preempt, bool *nopreempt); -+ void (*state_systrace_c)(unsigned int cpu, struct task_struct *p); -+ void (*locking_tick_hit)(struct task_struct *prev, struct task_struct *next); -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ void (*replace_next_task_fair)(struct rq *rq, -+ struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*enqueue_entity)(struct rq *rq, struct task_struct *p); -+ void (*dequeue_entity)(struct rq *rq, struct task_struct *p); -+#else -+ void (*set_last_entity)(struct task_struct *prev, struct task_struct *next, struct rq *rq); -+ void (*pick_last_entity)(struct rq *rq, struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*clear_last_entity)(struct rq *rq, struct task_struct *p); -+#endif -+ void (*opt_ss_lock_contention)(struct task_struct *p, int old_im, int new_im); -+}; -+ -+extern struct sched_assist_locking_ops *locking_ops; -+ -+ -+#define LOCKING_CALL_OP(op, args...) \ -+do { \ -+ if (locking_ops && locking_ops->op) { \ -+ locking_ops->op(args); \ -+ } \ -+} while (0) -+ -+#define LOCKING_CALL_OP_RET(op, args...) \ -+({ \ -+ __typeof__(locking_ops->op(args)) __ret = 0; \ -+ if (locking_ops && locking_ops->op) \ -+ __ret = locking_ops->op(args); \ -+ __ret; \ -+}) -+ -+void register_sched_assist_locking_ops(struct sched_assist_locking_ops *ops); -+#endif -+ -+/* attention: before insert .ko, task's list->prev/next is init with 0 */ -+static inline bool oplus_list_empty(struct list_head *list) -+{ -+ return list_empty(list) || (list->prev == 0 && list->next == 0); -+} -+ -+static inline bool oplus_rbtree_empty(struct rb_root_cached *list) -+{ -+ return (rb_first_cached(list) == NULL); -+} -+ -+/** -+ * Check if there are ux tasks waiting to run on the specified cpu -+ */ -+static inline bool orq_has_ux_tasks(struct oplus_rq *orq) -+{ -+ return !oplus_rbtree_empty(&orq->ux_list); -+} -+ -+/* attention: before insert .ko, task's node->__rb_parent_color is init with 0 */ -+static inline bool oplus_rbnode_empty(struct rb_node *node) -+{ -+ return RB_EMPTY_NODE(node) || (node->__rb_parent_color == 0); -+} -+ -+static inline struct oplus_task_struct *ux_list_first_entry(struct rb_root_cached *list) -+{ -+ struct rb_node *leftmost = rb_first_cached(list); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, ux_entry); -+} -+ -+static inline struct oplus_task_struct *exec_timeline_first_entry(struct rb_root_cached *tree) -+{ -+ struct rb_node *leftmost = rb_first_cached(tree); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, exec_time_node); -+} -+ -+typedef bool (*migrate_task_callback_t)(struct task_struct *tsk, int src_cpu, int dst_cpu); -+extern migrate_task_callback_t fbg_migrate_task_callback; -+typedef void (*android_rvh_schedule_handler_t)(struct task_struct *prev, -+ struct task_struct *next, struct rq *rq); -+extern android_rvh_schedule_handler_t fbg_android_rvh_schedule_callback; -+ -+extern struct kmem_cache *oplus_task_struct_cachep; -+ -+#define ots_to_ts(ots) (ots->task) -+#define OTS_IDX 0 -+#define ORQ_IDX 0 -+ -+static inline bool test_task_is_fair(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid CFS priority is MAX_RT_PRIO..MAX_PRIO-1 */ -+ if ((task->prio >= MAX_RT_PRIO) && (task->prio <= MAX_PRIO-1)) -+ return true; -+ return false; -+} -+ -+static inline bool test_task_is_rt(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid RT priority is 0..MAX_RT_PRIO-1 */ -+ if (rt_prio(task->prio)) -+ return true; -+ -+ return false; -+} -+ -+static inline struct oplus_task_struct *get_oplus_task_struct(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = NULL; -+ -+ /* not Skip idle thread */ -+ if (!t) -+ return NULL; -+ -+ ots = (struct oplus_task_struct *) READ_ONCE(t->android_oem_data1[OTS_IDX]); -+ if (IS_ERR_OR_NULL(ots)) -+ return NULL; -+ -+ return ots; -+} -+ -+static inline unsigned long oplus_get_im_flag(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return IM_FLAG_NONE; -+ -+ return ots->im_flag; -+} -+ -+static inline bool is_optimized_audio_thread(struct task_struct *t) -+{ -+ unsigned long im_flag; -+ -+ im_flag = oplus_get_im_flag(t); -+ if (test_bit(IM_FLAG_AUDIO, &im_flag)) -+ return true; -+ -+ return false; -+} -+ -+static inline void oplus_set_im_flag(struct task_struct *t, int im_flag) -+{ -+ struct oplus_task_struct *ots = NULL; -+ ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ set_bit(im_flag, &ots->im_flag); -+} -+ -+static inline int get_ots_ux_state(struct oplus_task_struct *ots) -+{ -+ int ux_state; -+ int sub_ux_state; -+ int ux_prio; -+ int ret; -+ -+ ux_prio = ots->ux_state & SCHED_ASSIST_UX_PRIORITY_MASK; -+ ux_state = ots->ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ sub_ux_state = ots->sub_ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ -+ if (sub_ux_state) { -+ ret = ux_state | (sub_ux_state & ~SA_TYPE_INHERIT) | SA_TYPE_COMBINED; -+ } else { -+ ret = ux_state; -+ } -+ -+ if (ret & SCHED_ASSIST_UX_MASK) { -+ return ux_prio | ret; -+ } -+ -+ return 0; -+} -+ -+bool is_multiple_ux(struct oplus_task_struct *ots); -+ -+static inline int oplus_get_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return get_ots_ux_state(ots); -+} -+ -+static inline int oplus_get_static_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return 0; -+ } -+ return ots->ux_state; -+} -+ -+static inline int oplus_get_sub_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->sub_ux_state; -+} -+ -+static inline int oplus_get_inherited_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return ots->ux_state; -+ } -+ return ots->sub_ux_state; -+} -+ -+void oplus_set_ux_state_lock(struct task_struct *t, int ux_state, int inherit_type, bool need_lock_rq); -+ -+static inline s64 oplus_get_inherit_ux(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return atomic64_read(&ots->inherit_ux); -+} -+ -+static inline void oplus_set_inherit_ux(struct task_struct *t, s64 inherit_ux) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ atomic64_set(&ots->inherit_ux, inherit_ux); -+} -+ -+static inline int oplus_get_ux_depth(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->ux_depth; -+} -+ -+static inline void oplus_set_ux_depth(struct task_struct *t, int ux_depth) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->ux_depth = ux_depth; -+} -+ -+static inline u64 oplus_get_enqueue_time(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->enqueue_time; -+} -+ -+static inline void oplus_set_enqueue_time(struct task_struct *t, u64 enqueue_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->enqueue_time = enqueue_time; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* -+ * Record the number of task context switches -+ * and scheduling delays when enqueueing a task. -+ */ -+ ots->snap_pcount = t->sched_info.pcount; -+ ots->snap_run_delay = t->sched_info.run_delay; -+#endif -+} -+ -+static inline void oplus_set_inherit_ux_start(struct task_struct *t, u64 start_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->inherit_ux_start = t->se.sum_exec_runtime; -+} -+ -+static inline void init_task_ux_info(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ RB_CLEAR_NODE(&ots->ux_entry); -+ RB_CLEAR_NODE(&ots->exec_time_node); -+ ots->ux_state = 0; -+ ots->sub_ux_state = 0; -+ atomic64_set(&ots->inherit_ux, 0); -+ ots->ux_depth = 0; -+ ots->enqueue_time = 0; -+ ots->inherit_ux_start = 0; -+ ots->ux_priority = -1; -+ ots->ux_nice = -1; -+ ots->vruntime = 0; -+ ots->preset_vruntime = 0; -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG) -+ ots->abnormal_flag = 0; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_SCHED_SPREAD -+ ots->lb_state = 0; -+ ots->ld_flag = 0; -+#endif -+ ots->exec_calc_runtime = 0; -+ ots->is_update_runtime = 0; -+ ots->target_process = -1; -+ ots->wake_tid = 0; -+ ots->running_start_time = 0; -+ ots->update_running_start_time = false; -+ ots->last_wake_ts = 0; -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ memset(&ots->lkinfo, 0, sizeof(struct locking_info)); -+ INIT_LIST_HEAD(&ots->lkinfo.node); -+/*#endif*/ -+ ots->block_start_time = 0; -+#ifdef CONFIG_LOCKING_PROTECT -+ INIT_LIST_HEAD(&ots->locking_entry); -+ ots->locking_start_time = 0; -+ ots->locking_depth = 0; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ ots->snap_pcount = 0; -+ ots->snap_run_delay = 0; -+ plist_node_init(&ots->rtb, MAX_IM_FLAG_PRIO); -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+ atomic_set(&ots->pipeline_cpu, -1); -+#endif -+ -+#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO) -+ ots->amu_cycle = 0; -+ ots->amu_instruct = 0; -+#endif -+}; -+ -+static inline bool test_ux_type(struct task_struct *task, int ux_type) -+{ -+ return oplus_get_ux_state(task) & ux_type; -+} -+ -+static inline bool is_heavy_ux_task(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return false; -+ -+ return get_ots_ux_state(ots) & SA_TYPE_HEAVY; -+} -+ -+static inline bool sched_assist_scene(unsigned int scene) -+{ -+ if (unlikely(!global_sched_assist_enabled)) -+ return false; -+ -+ return global_sched_assist_scene & scene; -+} -+ -+static inline unsigned long oplus_task_util(struct task_struct *p) -+{ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+ struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; -+ -+ return wts->demand_scaled; -+#else -+ return READ_ONCE(p->se.avg.util_avg); -+#endif -+} -+ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+static inline u32 task_wts_sum(struct task_struct *tsk) -+{ -+ struct walt_task_struct *wts = (struct walt_task_struct *) tsk->android_vendor_data1; -+ -+ return wts->sum; -+} -+#endif -+ -+bool is_min_cluster(int cpu); -+bool is_max_cluster(int cpu); -+bool is_mid_cluster(int cpu); -+bool im_mali(const char *comm); -+bool is_top(struct task_struct *p); -+bool task_is_runnable(struct task_struct *task); -+struct oplus_rq *get_oplus_rq(struct rq *rq); -+ -+typedef unsigned long (*kallsyms_lookup_name_t)(const char *name); -+extern kallsyms_lookup_name_t _kallsyms_lookup_name; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+int ux_mask_to_prio(int ux_mask); -+int ux_prio_to_mask(int prio); -+#endif -+ -+noinline int tracing_mark_write(const char *buf); -+void hwbinder_systrace_c(unsigned int cpu, int flag); -+void sched_assist_init_oplus_rq(void); -+void queue_ux_thread(struct rq *rq, struct task_struct *p, int enqueue); -+ -+void inherit_ux_inc(struct task_struct *task, int type); -+void inherit_ux_sub(struct task_struct *task, int type, int value); -+void set_inherit_ux(struct task_struct *task, int type, int depth, int inherit_val); -+void reset_inherit_ux(struct task_struct *inherit_task, struct task_struct *ux_task, int reset_type); -+void unset_inherit_ux(struct task_struct *task, int type); -+void unset_inherit_ux_value(struct task_struct *task, int type, int value); -+void inc_inherit_ux_refs(struct task_struct *task, int type); -+void clear_all_inherit_type(struct task_struct *p); -+int get_max_inherit_gran(struct task_struct *p); -+ -+bool is_heavy_load_top_task(struct task_struct *p); -+bool test_task_is_fair(struct task_struct *task); -+bool test_task_is_rt(struct task_struct *task); -+ -+bool test_task_ux(struct task_struct *task); -+bool test_task_ux_depth(int ux_depth); -+bool test_inherit_ux(struct task_struct *task, int type); -+bool test_set_inherit_ux(struct task_struct *task); -+int get_ux_state_type(struct task_struct *task); -+void sched_assist_target_comm(struct task_struct *task, const char *comm); -+unsigned int ux_task_exec_limit(struct task_struct *p); -+ -+void update_ux_sched_cputopo(void); -+bool is_task_util_over(struct task_struct *tsk, int threshold); -+bool oplus_task_misfit(struct task_struct *tsk, int cpu); -+ssize_t oplus_show_cpus(const struct cpumask *mask, char *buf); -+void adjust_rt_lowest_mask(struct task_struct *p, struct cpumask *local_cpu_mask, int ret, bool force_adjust); -+bool sa_skip_rt_sync(struct rq *rq, struct task_struct *p, bool *sync); -+void ux_state_systrace_c(unsigned int cpu, struct task_struct *p); -+bool sa_rt_skip_ux_cpu(int cpu); -+int is_vip_mvp(struct task_struct *p); -+ -+/* s64 account_ux_runtime(struct rq *rq, struct task_struct *curr); */ -+void opt_ss_lock_contention(struct task_struct *p, unsigned long old_im, int new_im); -+ -+/* register vender hook in kernel/sched/topology.c */ -+void android_vh_build_sched_domains_handler(void *unused, bool has_asym); -+ -+/* register vender hook in kernel/sched/rt.c */ -+void android_rvh_select_task_rq_rt_handler(void *unused, struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags, int *new_cpu); -+void android_rvh_find_lowest_rq_handler(void *unused, struct task_struct *p, struct cpumask *local_cpu_mask, int ret, int *best_cpu); -+ -+/* register vender hook in kernel/sched/core.c */ -+void android_rvh_sched_fork_handler(void *unused, struct task_struct *p); -+void android_rvh_after_enqueue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_dequeue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_schedule_handler(void *unused, struct task_struct *prev, struct task_struct *next, struct rq *rq); -+void android_vh_scheduler_tick_handler(void *unused, struct rq *rq); -+ -+void android_vh_account_process_tick_gran_handler(void *unused, struct task_struct *p, struct rq *rq, int user_tick, int *ticks); -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+void sa_sched_switch_handler(void *unused, bool preempt, struct task_struct *prev, struct task_struct *next, unsigned int prev_state); -+#endif -+ -+void set_im_flag_with_bit(int im_flag, struct task_struct *task); -+ -+/* register vendor hook in kernel/cgroup/cgroup-v1.c */ -+void android_vh_cgroup_set_task_handler(void *unused, int ret, struct task_struct *task); -+/* register vendor hook in kernel/signal.c */ -+void android_vh_exit_signal_handler(void *unused, struct task_struct *p); -+void android_rvh_set_cpus_allowed_by_task_handler(void *unused, const struct cpumask *cpu_valid_mask, -+ const struct cpumask *new_mask, struct task_struct *task, unsigned int *dest_cpu); -+void android_rvh_setscheduler_handler(void *unused, struct task_struct *p); -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_BAN_APP_SET_AFFINITY) -+void android_vh_sched_setaffinity_early_handler(void *unused, struct task_struct *task, const struct cpumask *new_mask, bool *skip); -+#endif -+ -+#ifdef CONFIG_OPLUS_SCHED_GROUP_OPT -+void android_vh_reweight_entity_handler(void *unused, struct sched_entity *se); -+#endif -+ -+#ifdef CONFIG_BLOCKIO_UX_OPT -+int sa_blockio_init(void); -+void sa_blockio_exit(void); -+#endif -+extern struct notifier_block process_exit_notifier_block; -+#endif /* _OPLUS_SA_COMMON_H_ */ -diff --git a/include/linux/sched/sa_common_struct.h b/include/linux/sched/sa_common_struct.h -new file mode 100755 -index 000000000000..876feff48b36 ---- /dev/null -+++ b/include/linux/sched/sa_common_struct.h -@@ -0,0 +1,277 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+/* -+this file is splited from the sa_common.h to adapt the OKI, -+IS_ENABLED is not allowed in here, because the macro will not work in OKI. -+*/ -+#ifndef _OPLUS_SA_COMMON_STRUCT_H_ -+#define _OPLUS_SA_COMMON_STRUCT_H_ -+ -+#define MAX_CLUSTER (4) -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+/* hot-thread */ -+struct task_record { -+#define RECOED_WINSIZE (1 << 8) -+#define RECOED_WINIDX_MASK (RECOED_WINSIZE - 1) -+ u8 winidx; -+ u8 count; -+}; -+ -+#define MAX_TASK_COMM_LEN 256 -+struct uid_struct { -+ uid_t uid; -+ u64 uid_total_cycle; -+ u64 uid_total_inst; -+ spinlock_t lock; -+ char leader_comm[TASK_COMM_LEN]; -+ char cmdline[MAX_TASK_COMM_LEN]; -+}; -+ -+struct amu_uid_entry { -+ uid_t uid; -+ struct uid_struct *uid_struct; -+ struct hlist_node node; -+}; -+ -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+struct locking_info { -+ u64 waittime_stamp; -+ u64 holdtime_stamp; -+ /* Used in torture acquire latency statistic.*/ -+ u64 acquire_stamp; -+ /* -+ * mutex or rwsem optimistic spin start time. Because a task -+ * can't spin both on mutex and rwsem at one time, use one common -+ * threshold time is OK. -+ */ -+ u64 opt_spin_start_time; -+ struct task_struct *holder; -+ u32 waittype; -+ bool ux_contrib; -+ /* -+ * Whether task is ux when it's going to be added to mutex or -+ * rwsem waiter list. It helps us check whether there is ux -+ * task on mutex or rwsem waiter list. Also, a task can't be -+ * added to both mutex and rwsem at one time, so use one common -+ * field is OK. -+ */ -+ bool is_block_ux; -+ u32 kill_flag; -+ /* for cfs enqueue smoothly.*/ -+ struct list_head node; -+ struct task_struct *owner; -+ struct list_head lock_head; -+ u64 clear_seq; -+ atomic_t lock_depth; -+}; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+#define RAVG_HIST_SIZE 5 -+#define SCX_SLICE_DFL (1 * NSEC_PER_MSEC) -+#define SCX_SLICE_INF U64_MAX -+#define DEFAULT_CGROUP_DL_IDX (8) -+#define EXT_FLAG_RT_CHANGED (1 << 0) -+#define EXT_FLAG_CFS_CHANGED (1 << 1) -+struct scx_task_stats { -+ u64 mark_start; -+ u64 window_start; -+ u32 sum; -+ u32 sum_history[RAVG_HIST_SIZE]; -+ int cidx; -+ u32 demand; -+ u16 demand_scaled; -+ void *sdsq; -+}; -+/* -+ * The following is embedded in task_struct and contains all fields necessary -+ * for a task to be scheduled by SCX. -+ */ -+struct scx_entity { -+ struct scx_dispatch_q *dsq; -+ struct { -+ struct list_head fifo; /* dispatch order */ -+ struct rb_node priq; /* p->scx.dsq_vtime order */ -+ } dsq_node; -+ u32 flags; /* protected by rq lock */ -+ u32 dsq_flags; /* protected by dsq lock */ -+ s32 sticky_cpu; -+ unsigned long runnable_at; -+ u64 slice; -+ u64 dsq_vtime; -+ int gdsq_idx; -+ int ext_flags; -+ int prio_backup; -+ unsigned long sched_prop; -+ struct scx_task_stats sts; -+}; -+/*#endif*/ -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ -+#define MAX_CPU_FREQ_STATE 32 -+#define MAX_CPU_CNT 8 -+ -+#define STATE_IN_POOL 0 -+#define STATE_ACTIVE 1 -+ -+struct powermodel_freq_task_state { -+ u8 powermodel_enqueued:1; -+ u8 powermodel_index:7; -+}; -+ -+struct powermodel_cpu_task_state { -+ struct oplus_task_struct *ots; -+ struct powermodel_freq_task_state powermodel_freq_task_states[MAX_CPU_FREQ_STATE]; -+ u64 powermodel_last_seq; -+ u64 last_seq; -+ struct list_head node; -+ int state; -+ int pool_id; -+}; -+ -+#endif -+ -+ -+/* Please add your own members of task_struct here :) */ -+struct oplus_task_struct { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_node ux_entry; -+ struct rb_node exec_time_node; -+ struct task_struct *task; -+ atomic64_t inherit_ux; -+ u64 enqueue_time; -+ u64 inherit_ux_start; -+ /* u64 sum_exec_baseline; */ -+ u64 total_exec; -+ u64 vruntime; -+ u64 preset_vruntime; -+ /* contains ux state -+ 1. if static and inherited ux both exist, static ux stores in ux_state, inherited ux in sub_ux_state. -+ 2. if only static ux exists, static ux stores in ux_state. -+ 2. if only inherited ux exists, inherited ux stores in ux_state */ -+ int ux_state; -+ int sub_ux_state; -+ u8 ux_depth; -+ s8 ux_priority; -+ s8 ux_nice; -+ pid_t affinity_pid; -+ pid_t affinity_tgid; -+ unsigned long state; -+ unsigned long im_flag; -+ atomic_t is_vip_mvp; -+ -+/* #if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) */ -+ u64 ddl; -+ u64 ddl_active_ts; -+ u64 runnable_ts; -+ struct rb_node ddl_node; -+/* #endif */ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG)*/ -+ int abnormal_flag; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_SCHED_SPREAD */ -+ int lb_state; -+ int ld_flag:1; -+ /* CONFIG_OPLUS_FEATURE_TASK_LOAD */ -+ int is_update_runtime:1; -+ int target_process; -+ u64 wake_tid; -+ u64 running_start_time; -+ bool update_running_start_time; -+ u64 exec_calc_runtime; -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct task_record record[MAX_CLUSTER]; /* 2*u64 */ -+ u64 block_start_time; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_FRAME_BOOST */ -+ struct list_head fbg_list; -+ raw_spinlock_t fbg_list_entry_lock; -+ bool fbg_running; /* task belongs to a group, and in running */ -+ u16 fbg_state; -+ s8 preferred_cluster_id; -+ s8 fbg_depth; -+ u64 last_wake_ts; -+ int fbg_cur_group; -+/*#ifdef CONFIG_LOCKING_PROTECT*/ -+ unsigned long locking_start_time; -+ unsigned long last_jiffies; -+ struct list_head locking_entry; -+ int locking_depth; -+ int lk_tick_hit; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ struct locking_info lkinfo; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+ struct scx_entity scx; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_FDLEAK_CHECK)*/ -+ u8 fdleak_flag; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+ /* for loadbalance */ -+ struct plist_node rtb; /* rt boost task */ -+ -+ /* -+ * The following variables are used to calculate the time -+ * a task spends in the running/runnable state. -+ */ -+ u64 snap_run_delay; -+ unsigned long snap_pcount; -+/*#endif*/ -+#if IS_ENABLED(CONFIG_OPLUS_SCHED_TUNE) -+ int stune_idx; -+#endif -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE)*/ -+ atomic_t pipeline_cpu; -+/*#endif*/ -+ -+ /* for oplus secure guard */ -+ int sg_flag; -+ int sg_scno; -+ uid_t sg_uid; -+ uid_t sg_euid; -+ gid_t sg_gid; -+ gid_t sg_egid; -+/*#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct uid_struct *uid_struct; -+ u64 amu_instruct; -+ u64 amu_cycle; -+/*#endif*/ -+ /* for binder ux */ -+ int binder_async_ux_enable; -+ bool binder_async_ux_sts; -+ int binder_thread_mode; -+ struct binder_node *binder_thread_node; -+ -+/* for powermodel */ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ struct powermodel_cpu_task_state *powermodel_cpu_task_states[MAX_CPU_CNT]; -+ u64 exec_runtime; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_CFBT) -+ int cfbt_cur_group; -+ bool cfbt_running; -+#endif /* CONFIG_OPLUS_FEATURE_SCHED_CFBT */ -+} ____cacheline_aligned; -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+#define INVALID_PID (-1) -+struct oplus_lb { -+ /* used for active_balance to record the running task. */ -+ pid_t pid; -+}; -+/*#endif*/ -+ -+#endif /* _OPLUS_SA_COMMON_STRUCT_H_ */ -+ -diff --git a/include/linux/sched/sa_oemdata.h b/include/linux/sched/sa_oemdata.h -new file mode 100755 -index 000000000000..ebbbc754e391 ---- /dev/null -+++ b/include/linux/sched/sa_oemdata.h -@@ -0,0 +1,13 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_OEMDATA_H_ -+#define _OPLUS_SA_OEMDATA_H_ -+ -+int sa_oemdata_init(void); -+void sa_oemdata_deinit(void); -+ -+#endif /* _OPLUS_SA_OEMDATA_H_ */ -diff --git a/include/linux/sched/sched_ext.h b/include/linux/sched/sched_ext.h -new file mode 100755 -index 000000000000..9f1afdb8a04b ---- /dev/null -+++ b/include/linux/sched/sched_ext.h -@@ -0,0 +1,57 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef _OPLUS_SCHED_EXT_H -+#define _OPLUS_SCHED_EXT_H -+#include "sa_common.h" -+ -+#define SCHED_PROP_TOP_THREAD_SHIFT (8) -+#define SCHED_PROP_TOP_THREAD_MASK (0xf << SCHED_PROP_TOP_THREAD_SHIFT) -+#define SCHED_PROP_DEADLINE_MASK (0xFF) /* deadline for ext sched class */ -+#define SCHED_PROP_DEADLINE_LEVEL1 (1) /* 1ms for user-aware audio tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL2 (2) /* 2ms for user-aware touch tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL3 (3) /* 4ms for user aware dispaly tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL4 (4) /* 6ms */ -+#define SCHED_PROP_DEADLINE_LEVEL5 (5) /* 8ms */ -+#define SCHED_PROP_DEADLINE_LEVEL6 (6) /* 16ms */ -+#define SCHED_PROP_DEADLINE_LEVEL7 (7) /* 32ms */ -+#define SCHED_PROP_DEADLINE_LEVEL8 (8) /* 64ms */ -+#define SCHED_PROP_DEADLINE_LEVEL9 (9) /* 128ms */ -+ -+static inline int sched_prop_get_top_thread_id(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ return -EPERM; -+ } -+ -+ return ((ots->scx.sched_prop & SCHED_PROP_TOP_THREAD_MASK) >> SCHED_PROP_TOP_THREAD_SHIFT); -+} -+ -+static inline int sched_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_set_sched_prop failed! fn=%s\n", __func__); -+ return -EPERM; -+ } -+ -+ ots->scx.sched_prop = sp; -+ return 0; -+} -+ -+static inline unsigned long sched_get_sched_prop(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_get_sched_prop failed! fn=%s\n", __func__); -+ return (unsigned long)-1; -+ } -+ return ots->scx.sched_prop; -+} -+ -+#endif /*_OPLUS_SCHED_EXT_H */ -diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h -index a23af225c898..6a5f0c0d74bd 100644 ---- a/include/linux/sched/task.h -+++ b/include/linux/sched/task.h -@@ -62,6 +62,9 @@ extern void init_idle(struct task_struct *idle, int cpu); - - extern int sched_fork(unsigned long clone_flags, struct task_struct *p); - extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); -+#ifdef CONFIG_HMBIRD_SCHED -+extern void hmbird_cancel_fork(struct task_struct *p); -+#endif - extern void sched_post_fork(struct task_struct *p); - extern void sched_dead(struct task_struct *p); - -diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h -index 3bac0a8ceab2..969716ab4320 100755 ---- a/include/uapi/linux/sched.h -+++ b/include/uapi/linux/sched.h -@@ -118,6 +118,7 @@ struct clone_args { - /* SCHED_ISO: reserved but not implemented yet */ - #define SCHED_IDLE 5 - #define SCHED_DEADLINE 6 -+#define SCHED_HMBIRD 7 - - /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ - #define SCHED_RESET_ON_FORK 0x40000000 -diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt -index 4ba4fd80581b..b131d5662b48 100755 ---- a/kernel/Kconfig.preempt -+++ b/kernel/Kconfig.preempt -@@ -132,3 +132,33 @@ config SCHED_CORE - SCHED_CORE is default disabled. When it is enabled and unused, - which is the likely usage by Linux distributions, there should - be no measurable impact on performance. -+ -+config HMBIRD_SCHED -+ bool "hmbird Scheduling Class(base on ext scheduler)" -+ help -+ This option enables a new scheduler class sched_ext (SCX), which -+ allows scheduling policies to be implemented as BPF programs to -+ achieve the following: -+ -+ - Ease of experimentation and exploration: Enabling rapid -+ iteration of new scheduling policies. -+ - Customization: Building application-specific schedulers which -+ implement policies that are not applicable to general-purpose -+ schedulers. -+ - Rapid scheduler deployments: Non-disruptive swap outs of -+ scheduling policies in production environments. -+ -+ sched_ext leverages BPF’s struct_ops feature to define a structure -+ which exports function callbacks and flags to BPF programs that -+ wish to implement scheduling policies. The struct_ops structure -+ exported by sched_ext is struct sched_ext_ops, and is conceptually -+ similar to struct sched_class. -+ -+ See Documentation/scheduler/sched-ext.rst for more details. -+ -+config DYNAMIC_WALT_SUPPORT -+ bool "Dynamic WALT Support for Extensible Scheduling Class" -+ depends on HMBIRD_SCHED -+ default n -+ help -+ Enable dynamic WALT support for Extensible Scheduling Class. -diff --git a/kernel/fork.c b/kernel/fork.c -index 4bcd5b98eadd..6a393a0d638e 100755 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -23,6 +23,9 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - #include - #include - #include -@@ -995,6 +998,10 @@ void __put_task_struct(struct task_struct *tsk) - WARN_ON(!tsk->exit_state); - WARN_ON(refcount_read(&tsk->usage)); - WARN_ON(tsk == current); -+ -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_free(tsk); -+#endif - io_uring_free(tsk); - cgroup_free(tsk); - task_numa_free(tsk, true); -@@ -2505,7 +2512,11 @@ __latent_entropy struct task_struct *copy_process( - - retval = perf_event_init_task(p, clone_flags); - if (retval) -+#ifdef CONFIG_HMBIRD_SCHED -+ goto bad_fork_hmbird_cancel_fork; -+#else - goto bad_fork_cleanup_policy; -+#endif - retval = audit_alloc(p); - if (retval) - goto bad_fork_cleanup_perf; -@@ -2808,6 +2819,10 @@ __latent_entropy struct task_struct *copy_process( - audit_free(p); - bad_fork_cleanup_perf: - perf_event_free_task(p); -+#ifdef CONFIG_HMBIRD_SCHED -+bad_fork_hmbird_cancel_fork: -+ hmbird_cancel_fork(p); -+#endif - bad_fork_cleanup_policy: - lockdep_free_task(p); - #ifdef CONFIG_NUMA -diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c -index c073bc56a87c..c091539a0f60 100755 ---- a/kernel/sched/build_policy.c -+++ b/kernel/sched/build_policy.c -@@ -28,6 +28,10 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#include -+#endif - - #include - -@@ -51,3 +55,11 @@ - - #include "cputime.c" - #include "deadline.c" -+ -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/hmbird_util_track.c" -+#include "hmbird/hmbird_sched_proc.c" -+#include "hmbird/hmbird_shadow_tick.c" -+#include "hmbird/hmbird.c" -+#include "hmbird/hmbird_misc.c" -+#endif -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index 1924fd95d991..716788a51a5e 100755 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -96,6 +96,12 @@ - #include "../../io_uring/io-wq.h" - #include "../smpboot.h" - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/slim.h" -+#include "hmbird/hmbird_shadow_tick.h" -+#include "hmbird.h" -+#endif -+ - #include - #include - #include -@@ -182,6 +188,11 @@ static inline int __task_prio(const struct task_struct *p) - if (p->sched_class == &idle_sched_class) - return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - -+#ifdef CONFIG_HMBIRD_SCHED -+ if (p->sched_class == &hmbird_sched_class) -+ return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ -+#endif -+ - return MAX_RT_PRIO + MAX_NICE; /* 119, squash fair */ - } - -@@ -211,6 +222,11 @@ static inline bool prio_less(const struct task_struct *a, - if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ - return cfs_prio_less(a, b, in_fi); - -+#ifdef CONFIG_HMBIRD_SCHED -+ if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ -+ return hmbird_prio_less(a, b, in_fi); -+#endif -+ - return false; - } - -@@ -1265,6 +1281,18 @@ bool sched_can_stop_tick(struct rq *rq) - if (fifo_nr_running) - return true; - -+#ifdef CONFIG_HMBIRD_SCHED -+ /* -+ * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks -+ * left. For CFS, if there's more than one we need the tick for -+ * involuntary preemption. For HMBIRD, ask. -+ */ -+ if (!hmbird_enabled() && rq->nr_running > 1) -+ return false; -+ -+ if (hmbird_enabled() && !hmbird_can_stop_tick(rq)) -+ return false; -+#else - /* - * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; - * if there's more than one we need the tick for involuntary -@@ -1272,7 +1300,7 @@ bool sched_can_stop_tick(struct rq *rq) - */ - if (rq->nr_running > 1) - return false; -- -+#endif - /* - * If there is one task and it has CFS runtime bandwidth constraints - * and it's on the cpu now we don't want to stop the tick. -@@ -2197,6 +2225,42 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) - } - EXPORT_SYMBOL_GPL(deactivate_task); - -+#ifdef CONFIG_HMBIRD_SCHED -+struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) -+{ -+ struct hmbird_sched_change_guard cg = { -+ .rq = rq, -+ .p = p, -+ .queued = task_on_rq_queued(p), -+ .running = task_current(rq, p), -+ }; -+ -+ if (cg.queued) { -+ /* -+ * __kthread_bind() may call this on blocked tasks without -+ * holding rq->lock through __do_set_cpus_allowed(). Assert @rq -+ * locked iff @p is queued. -+ */ -+ lockdep_assert_rq_held(rq); -+ dequeue_task(rq, p, flags); -+ } -+ if (cg.running) -+ put_prev_task(rq, p); -+ -+ return cg; -+} -+ -+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags) -+{ -+ if (cg->queued) -+ enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); -+ if (cg->running) -+ set_next_task(cg->rq, cg->p); -+ cg->done = true; -+} -+#endif -+ - static inline int __normal_prio(int policy, int rt_prio, int nice) - { - int prio; -@@ -3731,7 +3795,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class)) -+#else - if (unlikely(!is_cpu_allowed(p, cpu))) -+#endif - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -@@ -4850,6 +4918,9 @@ late_initcall(sched_core_sysctl_init); - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { - -+#ifdef CONFIG_HMBIRD_SCHED -+ int ret; -+#endif - - trace_android_rvh_sched_fork(p); - -@@ -4890,13 +4961,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_reset_on_fork = 0; - } - -+#ifdef CONFIG_HMBIRD_SCHED -+ ret = hmbird_pre_fork(p); -+ if (ret) -+ goto out_cancel; -+ -+ if (dl_prio(p->prio)) { -+ ret = -EAGAIN; -+ goto out_cancel; -+ } else if (task_on_hmbird(p)) { -+ p->sched_class = &hmbird_sched_class; -+ } else if (rt_prio(p->prio)) { -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ } -+#else - if (dl_prio(p->prio)) - return -EAGAIN; - else if (rt_prio(p->prio)) - p->sched_class = &rt_sched_class; - else - p->sched_class = &fair_sched_class; -- -+#endif - init_entity_runnable_average(&p->se); - trace_android_rvh_finish_prio_fork(p); - -@@ -4915,6 +5002,11 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - #endif - return 0; - -+#ifdef CONFIG_HMBIRD_SCHED -+out_cancel: -+ hmbird_cancel_fork(p); -+ return ret; -+#endif - } - - void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) -@@ -4944,12 +5036,17 @@ void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_fork(p); -+#endif - } - - void sched_post_fork(struct task_struct *p) - { - uclamp_post_fork(p); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_post_fork(p); -+#endif - } - - unsigned long to_ratio(u64 period, u64 runtime) -@@ -5814,17 +5911,28 @@ void scheduler_tick(void) - if (sched_feat(LATENCY_WARN) && resched_latency) - resched_latency_warn(cpu, resched_latency); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_notify_sched_tick(); -+#endif - perf_event_task_tick(); - - if (curr->flags & PF_WQ_WORKER) - wq_worker_tick(curr); - - #ifdef CONFIG_SMP -+#ifdef CONFIG_HMBIRD_SCHED -+ if (!hmbird_enabled()) { -+#endif - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); -+#ifdef CONFIG_HMBIRD_SCHED -+ } -+#endif - #endif - trace_android_vh_scheduler_tick(rq); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ scheduler_tick_handler(NULL, NULL); -+#endif - } - - #ifdef CONFIG_NO_HZ_FULL -@@ -6126,8 +6234,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ -- -+#ifdef CONFIG_HMBIRD_SCHED -+ for_balance_class_range(class, prev->sched_class, &idle_sched_class) { -+#else - for_class_range(class, prev->sched_class, &idle_sched_class) { -+#endif - if (class->balance(rq, prev, rf)) - break; - } -@@ -6145,6 +6256,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - const struct sched_class *class; - struct task_struct *p; - -+#ifdef CONFIG_HMBIRD_SCHED -+ if (hmbird_enabled()) -+ goto restart; -+#endif - /* - * Optimization: we know that if all tasks are in the fair class we can - * call that function directly, but only if the @prev task wasn't of a -@@ -6170,11 +6285,20 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - restart: - put_prev_task_balance(rq, prev, rf); - -+#ifdef CONFIG_HMBIRD_SCHED -+ for_each_active_class(class) { -+ p = class->pick_next_task(rq); -+ if (p) { -+ hmbird_notify_pick_next_task(rq, p, class); -+ return p; -+ } -+#else - for_each_class(class) { - p = class->pick_next_task(rq); - if (p) - return p; - } -+#endif - - BUG(); /* The idle class should always have a runnable task. */ - } -@@ -6203,8 +6327,11 @@ static inline struct task_struct *pick_task(struct rq *rq) - const struct sched_class *class; - struct task_struct *p; - -- -+#ifdef CONFIG_HMBIRD_SCHED -+ for_each_active_class(class) { -+#else - for_each_class(class) { -+#endif - p = class->pick_task(rq); - if (p) - return p; -@@ -6848,6 +6975,9 @@ static void __sched notrace __schedule(unsigned int sched_mode) - psi_sched_switch(prev, next, !task_on_rq_queued(prev)); - - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#ifdef CONFIG_HMBIRD_SCHED -+ sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#endif - - /* Also unlocks the rq: */ - rq = context_switch(rq, prev, next, &rf); -@@ -7178,12 +7308,29 @@ EXPORT_SYMBOL(default_wake_function); - - static void __setscheduler_prio(struct task_struct *p, int prio) - { -+#ifdef CONFIG_HMBIRD_SCHED -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) -+ ; -+ else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (rt_prio(prio) && on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#else - if (dl_prio(prio)) - p->sched_class = &dl_sched_class; - else if (rt_prio(prio)) - p->sched_class = &rt_sched_class; - else - p->sched_class = &fair_sched_class; -+#endif - - p->prio = prio; - } -@@ -7808,6 +7955,9 @@ static int __sched_setscheduler(struct task_struct *p, - int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct rq *rq; - bool cpuset_locked = false; -+#ifdef CONFIG_HMBIRD_SCHED -+ unsigned long flags; -+#endif - - - /* The pi code expects interrupts enabled */ -@@ -7874,7 +8024,9 @@ static int __sched_setscheduler(struct task_struct *p, - * To be able to change p->policy safely, the appropriate - * runqueue lock must be held. - */ -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+#endif - rq = task_rq_lock(p, &rf); - update_rq_clock(rq); - -@@ -7886,6 +8038,11 @@ static int __sched_setscheduler(struct task_struct *p, - goto unlock; - } - -+#ifdef CONFIG_HMBIRD_SCHED -+ retval = hmbird_check_setscheduler(p, policy); -+ if (retval) -+ goto unlock; -+#endif - /* - * If not changing anything there's no need to proceed further, - * but store a possible modification of reset_on_fork. -@@ -7942,7 +8099,9 @@ static int __sched_setscheduler(struct task_struct *p, - if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { - policy = oldpolicy = -1; - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - goto recheck; -@@ -8010,7 +8169,9 @@ static int __sched_setscheduler(struct task_struct *p, - preempt_disable(); - head = splice_balance_callbacks(rq); - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (pi) { - if (cpuset_locked) - cpuset_unlock(); -@@ -8025,7 +8186,9 @@ static int __sched_setscheduler(struct task_struct *p, - - unlock: - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - return retval; -@@ -9247,6 +9410,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - break; - } -@@ -9274,6 +9440,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - } - return ret; -@@ -10162,11 +10331,22 @@ void __init sched_init(void) - int i; - - /* Make sure the linker didn't screw up */ -+#ifdef CONFIG_HMBIRD_SCHED -+#ifdef CONFIG_SMP -+ WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+#endif -+ WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class)); -+#else - BUG_ON(&idle_sched_class != &fair_sched_class + 1 || - &fair_sched_class != &rt_sched_class + 1 || - &rt_sched_class != &dl_sched_class + 1); - #ifdef CONFIG_SMP - BUG_ON(&dl_sched_class != &stop_sched_class + 1); -+#endif - #endif - - wait_bit_init(); -@@ -10337,7 +10517,9 @@ void __init sched_init(void) - balance_push_set(smp_processor_id(), false); - #endif - init_sched_fair_class(); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ init_sched_hmbird_class(); -+#endif - psi_init(); - - init_uclamp(); -@@ -10823,6 +11005,10 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) - struct task_group *tg = css_tg(css); - struct task_group *parent = css_tg(css->parent); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_tg_online(tg); -+#endif -+ - if (parent) - sched_online_group(tg, parent); - -@@ -10872,13 +11058,77 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) - } - #endif - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 dl) -+{ -+ int i; -+ -+ for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) { -+ if (dl < HMBIRD_BPF_DSQS_DEADLINE[i]) -+ break; -+ } -+ -+ i = max_t(int, i-1, MIN_CGROUP_DL_IDX); -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return 0; -+ update_cgroup_ids_table(css->cgroup->kn->id, i); -+ -+ return 0; -+} -+ -+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ u8 i; -+ -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX]; -+ i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1); -+ if (i < 0) { -+ pr_err(" <%s> i is %d, less than 0, name is %s\n", -+ __func__, i, css->cgroup->kn->name); -+ i = DEFAULT_CGROUP_DL_IDX; -+ } -+ -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[i]; -+} -+ -+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css, -+ struct task_struct *p, int prio) -+{ -+ if (!css || !rt_prio(prio)) -+ return; -+ -+ if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) { -+ if (!strcmp(css->cgroup->kn->name, "display")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3); -+ else if (!strcmp(css->cgroup->kn->name, "touch")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2); -+ else if (!strcmp(css->cgroup->kn->name, "multimedia")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+ } -+} -+#endif -+ - static void cpu_cgroup_attach(struct cgroup_taskset *tset) - { - struct task_struct *task; - struct cgroup_subsys_state *css; - - cgroup_taskset_for_each(task, css, tset) { -- -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_change_rt_sched_prop(css, task, task->prio); -+#endif - sched_move_task(task); - } - -@@ -11560,7 +11810,13 @@ static struct cftype cpu_legacy_files[] = { - .write_u64 = cpu_uclamp_ls_write_u64, - }, - #endif -- -+#ifdef CONFIG_HMBIRD_SCHED -+ { -+ .name = "hmbird.deadline", -+ .read_u64 = cgroup_read_hmbird_deadline, -+ .write_u64 = cgroup_write_hmbird_deadline, -+ }, -+#endif - { } /* Terminate */ - }; - -diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c -index 0926c682cbae..2fe1a734e640 100755 ---- a/kernel/sched/debug.c -+++ b/kernel/sched/debug.c -@@ -375,6 +375,10 @@ static __init int sched_init_debug(void) - #endif - - debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); -+ -+#ifdef CONFIG_HMBIRD_SCHED -+ debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops); -+#endif - return 0; - } - late_initcall(sched_init_debug); -@@ -1095,6 +1099,11 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - P(dl.runtime); - P(dl.deadline); - } -+#ifdef CONFIG_HMBIRD_SCHED -+ __PS("hmbird.enabled", p->sched_class == &hmbird_sched_class); -+ __PS("sched_prop", get_hmbird_ts(p)->sched_prop); -+ __PS("top_task_prop", get_hmbird_ts(p)->top_task_prop); -+#endif - #undef PN_SCHEDSTAT - #undef P_SCHEDSTAT - -diff --git a/kernel/sched/hmbird.h b/kernel/sched/hmbird.h -new file mode 100755 -index 000000000000..d0d84ddee13d ---- /dev/null -+++ b/kernel/sched/hmbird.h -@@ -0,0 +1,343 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#ifndef _HMBIRD_H_ -+#define _HMBIRD_H_ -+ -+/* -+ * Tag marking a kernel function as a kfunc. This is meant to minimize the -+ * amount of copy-paste that kfunc authors have to include for correctness so -+ * as to avoid issues such as the compiler inlining or eliding either a static -+ * kfunc, or a global kfunc in an LTO build. -+ */ -+ -+#define DEBUG_INTERNAL (1 << 0) -+#define DEBUG_INFO_TRACE (1 << 1) -+#define DEBUG_INFO_SYSTRACE (1 << 2) -+ -+#define NUMS_CGROUP_KINDS (512) -+#define SLIM_FOR_SGAME (1 << 0) -+#define SLIM_FOR_GENSHIN (1 << 1) -+ -+#ifdef MAX_TASK_NR -+#define MAX_KEY_THREAD_RECORD ((MAX_TASK_NR + 1) >> 1) -+#else -+#define MAX_KEY_THREAD_RECORD (1 << 3) -+#endif /* MAX_TASK_NR */ -+ -+#define TOP_TASK_SHIFT (8) -+#define TOP_TASK_MAX (1 << TOP_TASK_SHIFT) -+#ifndef TOP_TASK_BITS_MASK -+#define TOP_TASK_BITS_MASK (TOP_TASK_MAX - 1) -+#endif /* TOP_TASK_BITS_MASK */ -+ -+extern struct md_info_t *md_info; -+ -+#define debug_enabled() \ -+ (unlikely(hmbirdcore_debug)) -+ -+#define hmbird_debug(fmt, ...) \ -+ pr_info(":"fmt, ##__VA_ARGS__) -+ -+#define hmbird_err(id, fmt, ...) \ -+do { \ -+ pr_err(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_deferred_err(id, fmt, ...) \ -+do { \ -+ printk_deferred(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \ -+do { \ -+ if (unlikely(cond)) { \ -+ hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+ } \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_info_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_info_trace(fmt, ...) -+#endif -+ -+#define hmbird_info_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+ } \ -+} while (0) -+ -+#define hmbird_output_systrace(fmt, ...) \ -+do { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_internal_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_internal_trace(fmt, ...) -+#endif -+ -+#define hmbird_internal_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) { \ -+ hmbird_output_systrace(fmt, ##__VA_ARGS__); \ -+ } \ -+} while (0) -+ -+enum hmbird_wake_flags { -+ /* expose select WF_* flags as enums */ -+ HMBIRD_WAKE_EXEC = WF_EXEC, -+ HMBIRD_WAKE_FORK = WF_FORK, -+ HMBIRD_WAKE_TTWU = WF_TTWU, -+ HMBIRD_WAKE_SYNC = WF_SYNC, -+}; -+ -+enum hmbird_enq_flags { -+ /* expose select ENQUEUE_* flags as enums */ -+ HMBIRD_ENQ_WAKEUP = ENQUEUE_WAKEUP, -+ HMBIRD_ENQ_HEAD = ENQUEUE_HEAD, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * Set the following to trigger preemption when calling -+ * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the -+ * current task is cleared to zero and the CPU is kicked into the -+ * scheduling path. Implies %HMBIRD_ENQ_HEAD. -+ */ -+ HMBIRD_ENQ_PREEMPT = 1LLU << 32, -+ HMBIRD_ENQ_REENQ = 1LLU << 40, -+ HMBIRD_ENQ_LAST = 1LLU << 41, -+ -+ /* -+ * A hint indicating that it's advisable to enqueue the task on the -+ * local dsq of the currently selected CPU. Currently used by -+ * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST. -+ */ -+ HMBIRD_ENQ_LOCAL = 1LLU << 42, -+ -+ /* high 8 bits are internal */ -+ __HMBIRD_ENQ_INTERNAL_MASK = 0xffLLU << 56, -+ -+ HMBIRD_ENQ_CLEAR_OPSS = 1LLU << 56, -+ HMBIRD_ENQ_DSQ_PRIQ = 1LLU << 57, -+}; -+ -+enum hmbird_deq_flags { -+ /* expose select DEQUEUE_* flags as enums */ -+ HMBIRD_DEQ_SLEEP = DEQUEUE_SLEEP, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * The generic core-sched layer decided to execute the task even though -+ * it hasn't been dispatched yet. Dequeue from the BPF side. -+ */ -+ HMBIRD_DEQ_CORE_SCHED_EXEC = 1LLU << 32, -+}; -+ -+enum hmbird_kick_flags { -+ HMBIRD_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -+ HMBIRD_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ -+}; -+ -+enum hmbird_task_prop_type { -+ HMBIRD_TASK_PROP_COMMON, -+ HMBIRD_TASK_PROP_PIPELINE, -+ HMBIRD_TASK_PROP_DEBUG_OR_LOG, -+ HMBIRD_TASK_PROP_HIGH_LOAD, -+ HMBIRD_TASK_PROP_IO, -+ HMBIRD_TASK_PROP_NETWORK, -+ HMBIRD_TASK_PROP_PERIODIC, -+ HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL, -+ HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL, -+ HMBIRD_TASK_PROP_ISOLATE, -+ HMBIRD_TASK_PROP_MAX, -+}; -+ -+enum hmbird_preempt_policy_id { -+ HMBIRD_PREEMPT_POLICY_NONE, -+ HMBIRD_PREEMPT_POLICY_PRIO_BASED, -+}; -+ -+extern const struct sched_class hmbird_sched_class; -+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops; -+extern const struct file_operations sched_hmbird_fops; -+extern unsigned long hmbird_watchdog_timeout; -+extern unsigned long hmbird_watchdog_timestamp; -+ -+enum scx_rq_flags { -+ HMBIRD_RQ_CAN_STOP_TICK = 1 << 0, -+}; -+ -+struct hmbird_rq { -+ struct hmbird_dispatch_q local_dsq; -+ struct list_head watchdog_list; -+ u64 ops_qseq; -+ u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -+ u32 nr_running; -+ u32 flags; -+ bool cpu_released; -+ cpumask_var_t cpus_to_kick; -+ cpumask_var_t cpus_to_preempt; -+ cpumask_var_t cpus_to_wait; -+ u64 pnt_seq; -+ u64 *prev_runnable_sum_fixed; -+ u32 *prev_window_size; -+ struct irq_work kick_cpus_irq_work; -+ bool pipeline; -+ bool exclusive; -+ bool period_disallow; -+ bool nonperiod_disallow; -+ struct rq *rq; -+ struct hmbird_sched_rq_stats *srq; -+}; -+ -+struct hmbird_sched_change_guard { -+ struct task_struct *p; -+ struct rq *rq; -+ bool queued; -+ bool running; -+ bool done; -+}; -+ -+extern struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -+ -+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags); -+ -+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -+ for (struct hmbird_sched_change_guard __cg = \ -+ hmbird_sched_change_guard_init(__rq, __p, __flags); \ -+ !__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags)) -+ -+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+bool task_on_hmbird(struct task_struct *p); -+int hmbird_pre_fork(struct task_struct *p); -+int hmbird_fork(struct task_struct *p); -+void hmbird_post_fork(struct task_struct *p); -+void hmbird_cancel_fork(struct task_struct *p); -+int hmbird_check_setscheduler(struct task_struct *p, int policy); -+bool hmbird_can_stop_tick(struct rq *rq); -+void init_sched_hmbird_class(void); -+ -+void hmbird_ops_exit(void); -+#define hmbird_ops_error(fmt, ...) \ -+do { \ -+ hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__); \ -+ hmbird_ops_exit(); \ -+} while (0) -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active); -+ -+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight) -+{ -+ return clamp_t(unsigned long, -+ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -+ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); -+} -+ -+static inline void hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active) -+{ -+ if (!hmbird_enabled()) -+ return; -+#ifdef CONFIG_SMP -+ /* -+ * Pairs with the smp_load_acquire() issued by a CPU in -+ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -+ * resched. -+ */ -+ smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1); -+#endif -+ if (!static_branch_unlikely(&hmbird_ops_cpu_preempt)) -+ return; -+ __hmbird_notify_pick_next_task(rq, p, active); -+} -+ -+extern void hmbird_scheduler_tick(void); -+void scan_timeout(struct rq *rq); -+void hmbird_notify_sched_tick(void); -+ -+static inline const struct sched_class *next_active_class(const struct sched_class *class) -+{ -+ class++; -+ -+ if (hmbird_enabled() && class == &fair_sched_class) -+ class++; -+ if (!hmbird_enabled() && class == &hmbird_sched_class) -+ class++; -+ return class; -+} -+ -+#define for_active_class_range(class, _from, _to) \ -+ for (class = (_from); class != (_to); class = next_active_class(class)) -+ -+#define for_each_active_class(class) \ -+ for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -+ -+/* -+ * HMBIRD requires a balance() call before every pick_next_task() call including -+ * when waking up from idle. -+ */ -+#define for_balance_class_range(class, prev_class, end_class) \ -+ for_active_class_range(class, (prev_class) > &hmbird_sched_class ? \ -+ &hmbird_sched_class : (prev_class), (end_class)) -+ -+#define MIN_CGROUP_DL_IDX (5) /* 8ms */ -+#define DEFAULT_CGROUP_DL_IDX (8) /* 64ms */ -+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS]; -+ -+ -+void __hmbird_update_idle(struct rq *rq, bool idle); -+ -+static inline void hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ if (hmbird_enabled()) -+ __hmbird_update_idle(rq, idle); -+} -+ -+int hmbird_ctrl(bool enable); -+ -+int hmbird_tg_online(struct task_group *tg); -+ -+ -+static inline u16 hmbird_task_util(struct task_struct *p) -+{ -+ return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0; -+} -+u16 hmbird_cpu_util(int cpu); -+ -+bool get_hmbird_ops_enabled(void); -+bool get_non_hmbird_task(void); -+void set_cpu_cluster(u64 cpu_cluster); -+#endif /*_EXT_H_*/ -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -new file mode 100755 -index 000000000000..ce05928392bf ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird.c -@@ -0,0 +1,4313 @@ -+// SPDX-License-Identifier: GPL-2.0 -+ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#include -+#include -+ -+#include "slim.h" -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+#include -+ -+#define CLUSTER_SEPARATE -+ -+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0); -+atomic_t non_hmbird_task; -+atomic_t hmbird_module_loaded = ATOMIC_INIT(0); -+int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0}; -+ -+enum hmbird_internal_consts { -+ HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ, -+}; -+ -+enum hmbird_ops_enable_state { -+ HMBIRD_OPS_PREPPING, -+ HMBIRD_OPS_ENABLING, -+ HMBIRD_OPS_ENABLED, -+ HMBIRD_OPS_DISABLING, -+ HMBIRD_OPS_DISABLED, -+}; -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+/* -+ * hmbird_entity->ops_state -+ * -+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler. -+ * State transitions look as follows: -+ * -+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING -+ * ^ | | -+ * | v v -+ * \-------------------------------/ -+ * -+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -+ * sites for explanations on the conditions being waited upon and why they are -+ * safe. Transitions out of them into NONE or QUEUED must store_release and the -+ * waiters should load_acquire. -+ * -+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether -+ * any given task can be dispatched by the BPF scheduler at all times and thus -+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core -+ * can safely reject invalid dispatches. -+ */ -+enum hmbird_ops_state { -+ HMBIRD_OPSS_NONE, /* owned by the HMBIRD core */ -+ HMBIRD_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -+ HMBIRD_OPSS_QUEUED, /* owned by the BPF scheduler */ -+ HMBIRD_OPSS_DISPATCHING, /* in transit back to the HMBIRD core */ -+ -+ /* -+ * QSEQ brands each QUEUED instance so that, when dispatch races -+ * dequeue/requeue, the dispatcher can tell whether it still has a claim -+ * on the task being dispatched. -+ */ -+ HMBIRD_OPSS_QSEQ_SHIFT = 2, -+ HMBIRD_OPSS_STATE_MASK = (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1, -+ HMBIRD_OPSS_QSEQ_MASK = ~HMBIRD_OPSS_STATE_MASK, -+}; -+ -+enum switch_stat { -+ HMBIRD_DISABLED, -+ HMBIRD_SWITCH_PREP, -+ HMBIRD_RQ_SWITCH_BEGIN, -+ HMBIRD_RQ_SWITCH_DONE, -+ HMBIRD_ENABLED, -+}; -+enum switch_stat curr_ss; -+ -+/* -+ * During exit, a task may schedule after losing its PIDs. When disabling the -+ * BPF scheduler, we need to be able to iterate tasks in every state to -+ * guarantee system safety. Maintain a dedicated task list which contains every -+ * task between its fork and eventual free. -+ */ -+DEFINE_SPINLOCK(hmbird_tasks_lock); -+static LIST_HEAD(hmbird_tasks); -+ -+/* ops enable/disable */ -+static struct kthread_worker *hmbird_ops_helper; -+static DEFINE_MUTEX(hmbird_ops_enable_mutex); -+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem); -+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED); -+ -+static bool hmbird_warned_zero_slice; -+enum hmbird_switch_type sw_type; -+static int skip_num[MAX_GLOBAL_DSQS]; -+static int big_distribute_mask_prev; -+static int little_distribute_mask_prev; -+ -+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0); -+ -+/* -+ * The maximum amount of time in jiffies that a task may be runnable without -+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger -+ * hmbird_ops_error(). -+ */ -+unsigned long hmbird_watchdog_timeout; -+ -+/* -+ * The last time the delayed work was run. This delayed work relies on -+ * ksoftirqd being able to run to service timer interrupts, so it's possible -+ * that this work itself could get wedged. To account for this, we check that -+ * it's not stalled in the timer tick, and trigger an error if it is. -+ */ -+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES; -+ -+static struct delayed_work hmbird_watchdog_work; -+static struct work_struct hmbird_err_exit_work; -+ -+/* idle tracking */ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_CPUMASK_OFFSTACK -+#define CL_ALIGNED_IF_ONSTACK -+#else -+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp -+#endif -+ -+static struct { -+ cpumask_var_t cpu; -+ cpumask_var_t smt; -+} idle_masks CL_ALIGNED_IF_ONSTACK; -+ -+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus; -+#endif /* CONFIG_SMP */ -+ -+/* dispatch queues */ -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global; -+ -+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128}; -+u32 pcp_dsq_deadline = 20; -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS]; -+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq); -+ -+static u64 max_hmbird_dsq_internal_id; -+ -+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/ -+#define CLUSTER_SEPARATE_IDX (8) -+#define GDSQS_ID_BASE (3) -+#define UX_COMPATIBLE_IDX (4) -+#define NON_PERIOD_START (5) -+#define NON_PERIOD_END (MAX_GLOBAL_DSQS) -+#define CREATE_DSQ_LEVEL_WITHIN (1) -+ -+struct hmbird_sched_info { -+ spinlock_t lock; -+ int curr_idx[2]; -+ int rtime[MAX_GLOBAL_DSQS]; -+}; -+ -+struct pcp_sched_info { -+ s64 pcp_seq; -+ int rtime; -+ bool pcp_round; -+}; -+ -+/* -+ * pcp_info may rw by another cpu. -+ * protected by rq->lock. -+ */ -+atomic64_t pcp_dsq_round; -+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info); -+ -+struct md_info_t *md_info; -+ -+static int b_rescue_l, l_rescue_b; -+static struct hmbird_sched_info sinfo; -+ -+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC; -+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = { -+ 0, 0, 0, 0, 0, -+ 32 * NSEC_PER_MSEC, -+ 20 * NSEC_PER_MSEC, -+ 14 * NSEC_PER_MSEC, -+ 8 * NSEC_PER_MSEC, -+ 6 * NSEC_PER_MSEC -+}; -+ -+struct cluster_ctx { -+ /* cpu-dsq map must within [lower, upper) */ -+ int upper; -+ int lower; -+ int tidx; -+}; -+ -+enum stat_items { -+ GLOBAL_STAT, -+ CPU_ALLOW_FAIL, -+ RT_CNT, -+ KEY_TASK_CNT, -+ SWITCH_IDX, -+ TIMEOUT_CNT, -+ -+ TOTAL_DSP_CNT, -+ MOVE_RQ_CNT, -+ SELECT_CPU, -+ -+ DWORD_STAT_END = SELECT_CPU, -+ -+ GDSQ_CNT, -+ ERR_IDX, -+ PCP_TIMEOUT_CNT, -+ PCP_LDSQ_CNT, -+ PCP_ENQL_CNT, -+ -+ MAX_ITEMS, -+}; -+static DEFINE_SPINLOCK(stats_lock); -+static char *stats_str[MAX_ITEMS] = { -+ "global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt", -+ "switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt", -+ "select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt", -+ "pcp_ldsq_cnt", "pcp_enql_cnt" -+}; -+ -+ -+struct stats_struct { -+ u64 global_stat[2]; -+ u64 cpu_allow_fail[2]; -+ u64 rt_cnt[2]; -+ u64 key_task_cnt[2]; -+ u64 switch_idx[2]; -+ u64 timeout_cnt[2]; -+ -+ u64 total_dsp_cnt[2]; -+ u64 move_rq_cnt[2]; -+ u64 select_cpu[2]; -+ -+ u64 gdsq_count[MAX_GLOBAL_DSQS][2]; -+ u64 err_idx[5]; -+ u64 pcp_timeout_cnt[NR_CPUS]; -+ u64 pcp_ldsq_count[NR_CPUS][2]; -+ u64 pcp_enql_cnt[NR_CPUS]; -+} stats_data; -+ -+static struct { -+ cpumask_var_t ex_free; -+ cpumask_var_t exclusive; -+ cpumask_var_t partial; -+ cpumask_var_t big; -+ cpumask_var_t little; -+} iso_masks __read_mostly; -+ -+/* -+ * Need more synchronization for these two variables? -+ * I choose not to. -+ */ -+static int l_need_rescue, b_need_rescue; -+ -+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...) \ -+{ \ -+ char buf[MAX_FATAL_INFO]; \ -+ \ -+ scnprintf(buf, MAX_FATAL_INFO, fmt, ##args); \ -+ hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf); \ -+ trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable), \ -+ READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf); \ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); \ -+} \ -+ -+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2) -+{ -+ int c1, c2; -+ struct cpumask mask = {.bits = {0}}; -+ struct cpumask tmp = {.bits = {0}}; -+ -+ if (!slim_stats) -+ return false; -+ -+ if (!rq1 || !rq2) -+ return false; -+ -+ c1 = cpu_of(rq1); -+ c2 = cpu_of(rq2); -+ cpumask_set_cpu(c1, &mask); -+ cpumask_set_cpu(c2, &mask); -+ -+ if (cpumask_and(&tmp, iso_masks.little, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.big, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.partial, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ return false; -+} -+ -+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu) -+{ -+ unsigned long flags; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ if (!slim_stats) -+ return; -+ -+ switch (item) { -+ case GLOBAL_STAT: -+ fallthrough; -+ case CPU_ALLOW_FAIL: -+ fallthrough; -+ case RT_CNT: -+ fallthrough; -+ case KEY_TASK_CNT: -+ fallthrough; -+ case SWITCH_IDX: -+ fallthrough; -+ case TIMEOUT_CNT: -+ fallthrough; -+ case TOTAL_DSP_CNT: -+ fallthrough; -+ case MOVE_RQ_CNT: -+ fallthrough; -+ case SELECT_CPU: -+ pval = pbase + item * 2 + idx; -+ break; -+ case GDSQ_CNT: -+ pval = &stats_data.gdsq_count[dsq_id][idx]; -+ break; -+ case ERR_IDX: -+ pval = &stats_data.err_idx[idx]; -+ break; -+ case PCP_TIMEOUT_CNT: -+ pval = &stats_data.pcp_timeout_cnt[cpu]; -+ break; -+ case PCP_LDSQ_CNT: -+ pval = &stats_data.pcp_ldsq_count[cpu][idx]; -+ break; -+ case PCP_ENQL_CNT: -+ pval = &stats_data.pcp_enql_cnt[cpu]; -+ break; -+ default: -+ return; -+ } -+ -+ spin_lock_irqsave(&stats_lock, flags); -+ *pval += 1; -+ spin_unlock_irqrestore(&stats_lock, flags); -+} -+ -+static inline bool handle_ret(int ret, int *idx, int len) -+{ -+ if (ret < 0 || ret >= len - *idx) -+ return true; -+ *idx += ret; -+ return false; -+} -+ -+#define PRINT_INTV (5 * HZ) -+void stats_print(char *buf, int len) -+{ -+ int idx = 0, i, j, ret; -+ int item = 0; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ for (item = 0; item < MAX_ITEMS; item++) { -+ if (item <= DWORD_STAT_END) { -+ pval = pbase + item * 2; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n", -+ stats_str[item], pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == GDSQ_CNT) { -+ for (j = 0; j < MAX_GLOBAL_DSQS; j++) { -+ pval = (u64 *)&stats_data.gdsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == ERR_IDX) { -+ pval = (u64 *)&stats_data.err_idx; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n", -+ stats_str[item], pval[0], -+ pval[1], pval[2], pval[3], pval[4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == PCP_TIMEOUT_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_timeout_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_LDSQ_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_ldsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_ENQL_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_enql_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } -+ } -+ -+ if (!md_info) { -+ buf[idx] = '\0'; -+ return; -+ } -+ -+ ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_SWITCHS; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "sw_rec[%d] = %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.sw_rec[i].switch_at, -+ md_info->kern_dump.sw_rec[i].is_success, -+ md_info->kern_dump.sw_rec[i].end_state, -+ md_info->kern_dump.sw_rec[i].switch_reason); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_EXCEP_ID; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "excep[%d] = %llu %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.excep_rec[i][0], -+ md_info->kern_dump.excep_rec[i][1], -+ md_info->kern_dump.excep_rec[i][2], -+ md_info->kern_dump.excep_rec[i][3], -+ md_info->kern_dump.excep_rec[i][4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n", -+ i, md_info->kern_dump.excep_idx[i]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ -+ buf[idx] = '\0'; -+} -+ -+enum cpu_type { -+ LITTLE, -+ BIG, -+ PARTIAL, -+ EXCLUSIVE, -+ EX_FREE, -+ INVALID -+}; -+ -+enum dsq_type { -+ GLOBAL_DSQ, -+ PCP_DSQ, -+ OTHER, -+ MAX_DSQ_TYPE, -+}; -+ -+static enum cpu_type cpu_cluster(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (get_hmbird_rq(rq)->exclusive) { -+ return EXCLUSIVE; -+ } else { -+ if (cpumask_test_cpu(cpu, iso_masks.little)) -+ return LITTLE; -+ else if (cpumask_test_cpu(cpu, iso_masks.big)) -+ return BIG; -+ else if (cpumask_test_cpu(cpu, iso_masks.partial)) -+ return PARTIAL; -+ else if (cpumask_test_cpu(cpu, iso_masks.exclusive)) -+ return EXCLUSIVE; -+ else if (cpumask_test_cpu(cpu, iso_masks.ex_free)) -+ return EX_FREE; -+ } -+ return INVALID; -+} -+ -+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq) -+{ -+ if (!dsq) -+ return OTHER; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= GDSQS_ID_BASE) && -+ ((dsq->id & 0xff) < MAX_GLOBAL_DSQS)) -+ return GLOBAL_DSQ; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) && -+ ((dsq->id & 0xff) < max_hmbird_dsq_internal_id)) -+ return PCP_DSQ; -+ -+ return OTHER; -+} -+ -+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq) -+{ -+ enum dsq_type type; -+ -+ type = get_dsq_type(dsq); -+ switch (type) { -+ case GLOBAL_DSQ: -+ case PCP_DSQ: -+ return (dsq->id & 0xff) - GDSQS_ID_BASE; -+ default: -+ return -1; -+ } -+ return -1; -+} -+ -+static void update_cpus_idle(bool set, struct cpumask *mask) -+{ -+ int cpu; -+ struct rq *rq; -+ -+ if (set) { -+ for_each_cpu(cpu, mask) { -+ rq = cpu_rq(cpu); -+ if (is_idle_task(rq->curr)) -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ } -+ } else -+ cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask); -+} -+ -+static void set_partial_status(bool enable, bool little, bool big) -+{ -+ WRITE_ONCE(partial_enable, enable); -+ WRITE_ONCE(l_need_rescue, little); -+ WRITE_ONCE(b_need_rescue, big); -+} -+ -+static bool is_little_need_rescue(void) -+{ -+ return READ_ONCE(l_need_rescue); -+} -+ -+static bool is_big_need_rescue(void) -+{ -+ return READ_ONCE(b_need_rescue); -+} -+ -+static bool is_partial_enabled(void) -+{ -+ return READ_ONCE(partial_enable); -+} -+ -+static bool is_partial_cpu(int cpu) -+{ -+ return cpumask_test_cpu(cpu, iso_masks.partial); -+} -+ -+static void set_iso_par_free(bool enable) -+{ -+ WRITE_ONCE(isolate_ctrl, enable); -+} -+ -+static bool is_iso_par_free(void) -+{ -+ return READ_ONCE(isolate_ctrl); -+} -+ -+static bool skip_update_idle(void) -+{ -+ int cpu = smp_processor_id(); -+ enum cpu_type type = cpu_cluster(cpu); -+ -+ if ((type == EXCLUSIVE && !is_iso_par_free()) || -+ /* partial enable may changed during idle, it doesn't matter. */ -+ (!is_partial_enabled() && type == PARTIAL)) -+ return true; -+ -+ return false; -+} -+ -+static void init_isolate_cpus(void) -+{ -+ WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -+ cpumask_set_cpu(0, iso_masks.big); -+ cpumask_set_cpu(1, iso_masks.big); -+ cpumask_set_cpu(2, iso_masks.big); -+ cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(4, iso_masks.big); -+ cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(6, iso_masks.partial); -+ cpumask_set_cpu(7, iso_masks.ex_free); -+} -+ -+extern spinlock_t css_set_lock; -+ -+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp) -+{ -+ int i; -+ struct cgroup *anc; -+ -+ spin_lock_irq(&css_set_lock); -+ for (i = 0; i < cgrp->level; i++) { -+ anc = cgrp->ancestors[i]; -+ if (anc->level != CREATE_DSQ_LEVEL_WITHIN) -+ continue; -+ cgroup_get(anc); -+ spin_unlock_irq(&css_set_lock); -+ return anc; -+ } -+ spin_unlock_irq(&css_set_lock); -+ hmbird_err(NO_CGROUP_L1, ":error cgroup = %s\n", cgrp->kn->name); -+ return NULL; -+} -+ -+#define PCP_IDX_BIT (1 << 31) -+ -+static bool is_pcp_rt(struct task_struct *p) -+{ -+ return rt_prio(p->prio) && (p->nr_cpus_allowed == 1); -+} -+ -+static bool is_pcp_idx(int idx) -+{ -+ return idx >= MAX_GLOBAL_DSQS; -+} -+ -+static bool is_critical_system_task(struct task_struct *p) -+{ -+ int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK; -+ -+ return (p->prio < (MAX_RT_PRIO >> 1) && -+ (sp_dl < SCHED_PROP_DEADLINE_LEVEL3)); -+} -+ -+#define ISOLATE_TASK_TOP_BIT (1 << 17) -+#define PIPELINE_TASK_TOP_BIT (1 << 9) -+static bool is_pipeline_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT); -+} -+ -+static bool is_isolate_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT); -+} -+ -+static bool is_critical_app_task_without_isolate(struct task_struct *p) -+{ -+ return task_is_top_task(p) && !is_isolate_task(p); -+} -+ -+static int find_idx_from_task(struct task_struct *p) -+{ -+ int idx, cpu; -+ int sp_dl; -+ struct task_group *tg = p->sched_task_group; -+ -+ if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) { -+ cpu = cpumask_any(p->cpus_ptr); -+ idx = cpu + MAX_GLOBAL_DSQS; -+ return idx; -+ } -+ -+ if (is_critical_system_task(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL0; -+ goto done; -+ } -+ -+ if (is_critical_app_task_without_isolate(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL1; -+ goto done; -+ } -+ -+ sp_dl = hmbird_get_dsq_id(p); -+ if (sp_dl) { -+ idx = sp_dl; -+ goto done; -+ } -+ -+ if (rt_prio(p->prio)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL3; -+ goto done; -+ } -+ -+ if (tg && tg->css.cgroup && tg->css.cgroup->kn) { -+ if (likely(tg->css.cgroup->kn->id >= 0 && -+ tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS)) -+ idx = cgroup_ids_table[tg->css.cgroup->kn->id]; -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ -+done: -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) { -+ hmbird_err(DSQ_ID_ERR, " : idx error, idx = %d-----\n", idx); -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } -+ return idx; -+} -+ -+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p) -+{ -+ int idx; -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq; -+ -+ if (!p) -+ return NULL; -+ -+ idx = find_idx_from_task(p); -+ if (is_pcp_idx(idx)) { -+ idx -= MAX_GLOBAL_DSQS; -+ dsq = &per_cpu(pcp_ldsq, idx); -+ get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq); -+ slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx); -+ } else { -+ dsq = &gdsqs[idx]; -+ get_hmbird_ts(p)->gdsq_idx = idx; -+ slim_stats_record(GDSQ_CNT, 0, idx, 0); -+ } -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ dsq->last_consume_at = jiffies; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ return dsq; -+} -+ -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq); -+ -+static void set_partial_rescue(bool p_state, bool l_over, bool b_over) -+{ -+ set_partial_status(p_state, l_over, b_over); -+ update_cpus_idle(p_state, iso_masks.partial); -+ hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled()); -+ hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue()); -+ hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue()); -+} -+ -+static void free_isocpu(bool enable) -+{ -+ set_iso_par_free(enable); -+ update_cpus_idle(enable, iso_masks.ex_free); -+ hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free()); -+} -+ -+inline u64 get_hmbird_cpu_util(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed) -+ return 0; -+ u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed); -+ u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size); -+ -+ do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT); -+ -+ return prev_runnable_sum_fixed; -+} -+ -+static inline unsigned int get_scaling_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->max; -+} -+ -+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static u64 get_cpus_max_util(struct cpumask *mask) -+{ -+ int cpu; -+ u64 max = 0; -+ u64 util, ratio; -+ unsigned long effective_cap = 0; -+ for_each_cpu(cpu, mask) { -+ if (slim_walt_ctrl) -+ slim_get_cpu_util(cpu, &util); -+ else -+ util = get_hmbird_cpu_util(cpu); -+ -+ /* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/ -+ if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu))) -+ effective_cap = arch_scale_cpu_capacity(cpu); -+ else -+ effective_cap = arch_scale_cpu_capacity(cpu) * -+ get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu); -+ -+ ratio = util * 100 / effective_cap; -+ hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util); -+ hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n", -+ cpu, (u64)arch_scale_cpu_capacity(cpu)); -+ hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n", -+ cpu, (u64)effective_cap); -+ -+ if (ratio > 100) -+ ratio = 100; -+ -+ if (ratio > max) -+ max = ratio; -+ } -+ return max; -+} -+ -+static bool cluster_need_rescue(struct cpumask *mask, int hres) -+{ -+ return get_cpus_max_util(mask) > hres; -+} -+ -+void partial_dynamic_ctrl(void) -+{ -+ u64 lmax = 0, bmax = 0; -+ bool l_over, l_under, b_over, b_under; -+ static bool last_l_over, last_b_over; -+ static unsigned long last_check; -+ -+ /* Check partial every jiffies. need lock? */ -+ if (time_before_eq(jiffies, READ_ONCE(last_check))) -+ return; -+ -+ WRITE_ONCE(last_check, jiffies); -+ -+ lmax = get_cpus_max_util(iso_masks.little); -+ l_over = lmax > parctrl_high_ratio_l; -+ l_under = lmax < parctrl_low_ratio_l; -+ -+ bmax = get_cpus_max_util(iso_masks.big); -+ b_over = bmax > parctrl_high_ratio; -+ b_under = bmax < parctrl_low_ratio; -+ -+ if (is_partial_enabled() && (l_over || b_over)) { -+ if (last_l_over != l_over || last_b_over != b_over) -+ set_partial_rescue(true, l_over, b_over); -+ } else if (!is_partial_enabled() && (l_over || b_over)) { -+ set_partial_rescue(true, l_over, b_over); -+ } else if (is_partial_enabled() && l_under && b_under) { -+ set_partial_rescue(false, false, false); -+ } -+ last_l_over = l_over; -+ last_b_over = b_over; -+ -+ if (is_partial_enabled()) { -+ bmax = get_cpus_max_util(iso_masks.partial); -+ if (!is_iso_par_free() && bmax > isoctrl_high_ratio) { -+ hmbird_info_trace("partial max = %llu\n", bmax); -+ free_isocpu(true); -+ } else if (is_iso_par_free() && bmax < isoctrl_low_ratio) -+ free_isocpu(false); -+ } else if (is_iso_par_free()) { -+ free_isocpu(false); -+ } -+} -+ -+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx) -+{ -+ hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx); -+} -+ -+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx) -+{ -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[idx])) { -+ slim_stats_record(GDSQ_CNT, 1, idx, 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ int i; -+ -+ for (i = 0; i < UX_COMPATIBLE_IDX; i++) { -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ slim_stats_record(GDSQ_CNT, 1, i, 0); -+ return 1; -+ } -+ } -+ return 0; -+} -+ -+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) { -+ slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ goto clear_timeout; -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) -+ goto clear_timeout; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ hmbird_info_trace( -+ "dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n", -+ dsq_id_to_internal(dsq), entity->task->comm, -+ jiffies, msecs_to_jiffies(deadline), entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1); -+ return; -+ -+clear_timeout: -+ hmbird_info_trace("dsq[%d] clear timeout\n", -+ dsq_id_to_internal(dsq)); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0); -+ dsq->is_timeout = false; -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq)); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime) -+{ -+ hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime); -+} -+ -+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any) -+{ -+ bool is_timeout; -+ int cpu = cpu_of(rq); -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu); -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ is_timeout = dsq->is_timeout; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ /* -+ * dsq->is_timeout may change here, let it be. -+ * it won't cause serious problems. -+ * the same for consume_dispatch_q later. -+ */ -+ if (!is_timeout && !any) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, dsq)) { -+ if (is_timeout) { -+ hmbird_info_trace("dsq[%d] consume a pcp timeout task\n", -+ dsq_id_to_internal(dsq)); -+ update_timeout_stats(rq, dsq, pcp_dsq_deadline); -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu); -+ } -+ slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu); -+ return 1; -+ } -+ /* -+ * No pcp task, clear quota. -+ */ -+ if (any) { -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+ } -+ return 0; -+} -+ -+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf) -+{ -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ } -+ return 0; -+} -+ -+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf, -+ int tmp, int cidx, int tidx) -+{ -+ unsigned long flags; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) { -+ if (tmp != cidx) { -+ spin_lock(&sinfo.lock); -+ sinfo.curr_idx[tidx] = tmp; -+ spin_unlock(&sinfo.lock); -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]); -+ slim_stats_record(SWITCH_IDX, 0, 0, 0); -+ } -+ slim_stats_record(GDSQ_CNT, 1, tmp, 0); -+ -+ raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags); -+ gdsqs[tmp].last_consume_at = jiffies; -+ raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags); -+ return 1; -+ } -+ return 0; -+ -+} -+ -+static int get_cidx(struct cluster_ctx *ctx) -+{ -+ int cidx; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ if (cidx < ctx->lower || cidx >= ctx->upper) { -+ sinfo.curr_idx[ctx->tidx] = ctx->lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]); -+ slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ } -+ spin_unlock(&sinfo.lock); -+ -+ return cidx; -+} -+ -+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = CLUSTER_SEPARATE_IDX; -+ if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) { -+ pr_debug(" %s iso_masks.little first is %d\n", -+ __func__, cpumask_first(iso_masks.little)); -+ ctx->upper = NON_PERIOD_END; -+ } -+ ctx->tidx = 0; -+ break; -+ case LITTLE: -+ ctx->lower = CLUSTER_SEPARATE_IDX; -+ ctx->upper = NON_PERIOD_END; -+ if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) { -+ pr_debug(" %s iso_masks.big first is %d", -+ __func__, cpumask_first(iso_masks.big)); -+ ctx->lower = NON_PERIOD_START; -+ } -+ ctx->tidx = 1; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ case LITTLE: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = NON_PERIOD_END; -+ ctx->tidx = 0; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ if (cluster_separate) { -+ return gen_cluster_ctx_separate(ctx, type); -+ } else { -+ return gen_cluster_ctx_common(ctx, type); -+ } -+} -+ -+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ int i; -+ bool is_timeout; -+ unsigned long flags; -+ struct cluster_ctx ctx; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ /* Third param means only consume timeout pcp dsq. */ -+ if (consume_pcp_dsq(rq, rf, false)) -+ return 1; -+ -+ for (i = ctx.lower; i < ctx.upper; i++) { -+ raw_spin_lock_irqsave(&gdsqs[i].lock, flags); -+ is_timeout = gdsqs[i].is_timeout; -+ raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags); -+ /* gdsqs[i].is_timeout may change here, let it be... */ -+ if (is_timeout) { -+ /* -+ * consume_dispatch_q will acquire dsq-lock, -+ * So cannot keep lock here, annoy enough. -+ * may rewrite a consume_dispatch_q_locked, TODO. -+ */ -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ hmbird_info_trace("dsq[%d] consume a timeout task\n", i); -+ slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0); -+ update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]); -+ return 1; -+ } -+ } -+ } -+ return 0; -+} -+ -+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ struct cluster_ctx ctx; -+ int cidx; -+ int tmp; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ cidx = get_cidx(&ctx); -+ tmp = cidx; -+ do { -+ if (check_pcp_dsq_round(rq, rf)) -+ return 1; -+ if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx)) -+ return 1; -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[tmp] = 0; -+ systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]); -+ tmp++; -+ if (tmp >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ tmp = ctx.lower; -+ } -+ spin_unlock(&sinfo.lock); -+ } while (tmp != cidx); -+ -+ return consume_pcp_dsq(rq, rf, true); -+} -+ -+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ enum cpu_type type = cpu_cluster(cpu_of(rq)); -+ bool period_allowed = !get_hmbird_rq(rq)->period_disallow; -+ bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow; -+ -+ switch (type) { -+ case EX_FREE: -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ break; -+ case EXCLUSIVE: -+ if (!is_iso_par_free()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ } -+ /* Only non-period task can run on isolate cpus */ -+ if (!READ_ONCE(iso_free_rescue)) { -+ if (is_big_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ } else { -+ /* Free run, can run on any task. */ -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ case PARTIAL: -+ if (!is_partial_enabled()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ return 0; -+ } -+ if (is_big_need_rescue()) { -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ -+ case BIG: -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ if (is_iso_par_free() || (is_little_need_rescue() && -+ !cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) { -+ hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++); -+ return 1; -+ } -+ } -+ break; -+ -+ case LITTLE: -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ -+ if (is_iso_par_free() || (is_big_need_rescue() && -+ !cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) { -+ hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++); -+ return 1; -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return 0; -+} -+ -+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf) -+{ -+ return consume_hmbird_global_dsq(rq, rf); -+} -+ -+ -+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time) -+{ -+ int idx; -+ -+ /* which dsq belongs to while task enqueue, task will consume its running time. */ -+ idx = get_hmbird_ts(p)->gdsq_idx; -+ /* Only non-period dsq share running time between each other. */ -+ if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id) -+ return; -+ -+ if (idx >= MAX_GLOBAL_DSQS) { -+ per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } else { -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[idx] += exec_time; -+ spin_unlock(&sinfo.lock); -+ systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]); -+ } -+} -+ -+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type) -+{ -+ int cidx; -+ struct cluster_ctx ctx; -+ int cpu = cpu_of(rq); -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ if (cidx < ctx.lower || cidx >= ctx.upper) { -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ slim_stats_record(ERR_IDX, ctx.tidx, 0, 0); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ } -+ -+ while (1) { -+ if (per_cpu(pcp_info, cpu).pcp_round) { -+ if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) { -+ hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n", -+ cpu, per_cpu(pcp_info, cpu).rtime); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } -+ } -+ if (sinfo.rtime[cidx] < dsq_quota[cidx]) -+ break; -+ -+ /* clear current dsq rtime */ -+ sinfo.rtime[cidx] = 0; -+ systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]); -+ -+ sinfo.curr_idx[ctx.tidx]++; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", -+ ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ } -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ slim_stats_record(SWITCH_IDX, 1, 0, 0); -+ } -+ spin_unlock(&sinfo.lock); -+} -+ -+ -+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p) -+{ -+ enum cpu_type type; -+ -+ if (!rq || !p) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ switch (type) { -+ case PARTIAL: -+ return; -+ case EXCLUSIVE: -+ return; -+ case EX_FREE: -+ return; -+ default: -+ break; -+ } -+ update_dsq_idx(rq, p, type); -+} -+ -+ -+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ int dsq_id; -+ -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo) || dsq->is_timeout) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ hmbird_deferred_err(SCAN_ENTITY_NULL, -+ " : entity is NULL, dsq->id = %llu\n", dsq->id); -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ dsq->is_timeout = true; -+ dsq_id = dsq_id_to_internal(dsq); -+ hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n", -+ dsq_id, entity->task->comm, jiffies, entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1); -+ raw_spin_unlock(&dsq->lock); -+ -+ return true; -+} -+ -+void scan_timeout(struct rq *rq) -+{ -+ int i; -+ int cpu = cpu_of(rq); -+ struct hmbird_dispatch_q *dsq; -+ static u64 last_scan_at; -+ static DEFINE_PER_CPU(u64, pcp_last_scan_at); -+ -+ if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu))) -+ return; -+ per_cpu(pcp_last_scan_at, cpu) = jiffies; -+ -+ dsq = &per_cpu(pcp_ldsq, cpu); -+ scan_dsq_timeout(rq, dsq, pcp_dsq_deadline); -+ -+ if (time_before_eq(jiffies, (unsigned long)last_scan_at)) -+ return; -+ last_scan_at = jiffies; -+ -+ for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) { -+ dsq = &gdsqs[i]; -+ scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]); -+ } -+} -+ -+/*******************************Initialize***********************************/ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id); -+static void init_dsq_at_boot(void) -+{ -+ int i, cpu; -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i)); -+ } -+ for_each_possible_cpu(cpu) -+ init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i + cpu)); -+ -+ max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu; -+ spin_lock_init(&sinfo.lock); -+} -+ -+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgrp_name_to_idx(struct cgroup *cgrp) -+{ -+ int idx; -+ -+ if (!cgrp) -+ return -1; -+ -+ if (!strcmp(cgrp->kn->name, "display") -+ || !strcmp(cgrp->kn->name, "multimedia")) -+ idx = 5; /* 8ms */ -+ else if (!strcmp(cgrp->kn->name, "top-app") -+ || !strcmp(cgrp->kn->name, "ss-top")) -+ idx = 6; /* 16ms */ -+ else if (!strcmp(cgrp->kn->name, "ssfg") -+ || !strcmp(cgrp->kn->name, "foreground")) -+ idx = 7; /* 32ms */ -+ else if (!strcmp(cgrp->kn->name, "bg") -+ || !strcmp(cgrp->kn->name, "log") -+ || !strcmp(cgrp->kn->name, "dex2oat") -+ || !strcmp(cgrp->kn->name, "background")) -+ idx = 9; /* 128ms */ -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */ -+ -+ return idx; -+} -+ -+static void init_root_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX); -+} -+ -+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) { -+ pr_err("%s idx err!\n", __func__); -+ return; -+ } -+ -+ if (cgroup_ids_table[cgrp->kn->id] == -1) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp)); -+} -+ -+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ struct cgroup *l1cgrp; -+ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ l1cgrp = cgroup_ancestor_l1(cgrp); -+ if (l1cgrp) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp)); -+ cgroup_put(l1cgrp); -+} -+ -+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg) -+{ -+ switch (cgrp->level) { -+ case 0: -+ init_root_tg(cgrp, tg); -+ break; -+ case 1: -+ init_level1_tg(cgrp, tg); -+ break; -+ default: -+ init_child_tg(cgrp, tg); -+ break; -+ } -+} -+ -+/**************************************************************************/ -+ -+struct hmbird_task_iter { -+ struct hmbird_entity cursor; -+ struct task_struct *locked; -+ struct rq *rq; -+ struct rq_flags rf; -+}; -+ -+/** -+ * hmbird_task_iter_init - Initialize a task iterator -+ * @iter: iterator to init -+ * -+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized, -+ * @iter must eventually be exited with hmbird_task_iter_exit(). -+ * -+ * hmbird_tasks_lock may be released between this and the first next() call or -+ * between any two next() calls. If hmbird_tasks_lock is released between two -+ * next() calls, the caller is responsible for ensuring that the task being -+ * iterated remains accessible either through RCU read lock or obtaining a -+ * reference count. -+ * -+ * All tasks which existed when the iteration started are guaranteed to be -+ * visited as long as they still exist. -+ */ -+static void hmbird_task_iter_init(struct hmbird_task_iter *iter) -+{ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR }; -+ list_add(&iter->cursor.tasks_node, &hmbird_tasks); -+ iter->locked = NULL; -+} -+ -+/** -+ * hmbird_task_iter_exit - Exit a task iterator -+ * @iter: iterator to exit -+ * -+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held. -+ * If the iterator holds a task's rq lock, that rq lock is released. See -+ * hmbird_task_iter_init() for details. -+ */ -+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ if (list_empty(cursor)) -+ return; -+ -+ list_del_init(cursor); -+} -+ -+/** -+ * hmbird_task_iter_next - Next task -+ * @iter: iterator to walk -+ * -+ * Visit the next task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ struct hmbird_entity *pos; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ list_for_each_entry(pos, cursor, tasks_node) { -+ if (&pos->tasks_node == &hmbird_tasks) -+ return NULL; -+ if (!(pos->flags & HMBIRD_TASK_CURSOR)) { -+ list_move(cursor, &pos->tasks_node); -+ return pos->task; -+ } -+ } -+ -+ /* can't happen, should always terminate at hmbird_tasks above */ -+ hmbird_deferred_err(ITER_RET_NULL, " : unreachable path in scx_task_iter_next\n"); -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered - Next non-idle task -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ while ((p = hmbird_task_iter_next(iter))) { -+ if (!is_idle_task(p)) -+ return p; -+ } -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init() -+ * for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ p = hmbird_task_iter_next_filtered(iter); -+ if (!p) -+ return NULL; -+ -+ iter->rq = task_rq_lock(p, &iter->rf); -+ iter->locked = p; -+ return p; -+} -+ -+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void) -+{ -+ return atomic_read(&hmbird_ops_enable_state_var); -+} -+ -+static enum hmbird_ops_enable_state -+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to) -+{ -+ return atomic_xchg(&hmbird_ops_enable_state_var, to); -+} -+ -+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to, -+ enum hmbird_ops_enable_state from) -+{ -+ int from_v = from; -+ -+ return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to); -+} -+ -+static bool hmbird_ops_disabling(void) -+{ -+ return false; -+} -+ -+/** -+ * wait_ops_state - Busy-wait the specified ops state to end -+ * @p: target task -+ * @opss: state to wait the end of -+ * -+ * Busy-wait for @p to transition out of @opss. This can only be used when the -+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also -+ * has load_acquire semantics to ensure that the caller can see the updates made -+ * in the enqueueing and dispatching paths. -+ */ -+static void wait_ops_state(struct task_struct *p, u64 opss) -+{ -+ do { -+ cpu_relax(); -+ } while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss); -+} -+ -+ -+static void update_curr_hmbird(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ u64 now = rq_clock_task(rq); -+ u64 delta_exec; -+ -+ if (time_before_eq64(now, curr->se.exec_start)) -+ return; -+ -+ delta_exec = now - curr->se.exec_start; -+ curr->se.exec_start = now; -+ update_runningtime(rq, curr, delta_exec); -+ curr->se.sum_exec_runtime += delta_exec; -+ account_group_exec_runtime(curr, delta_exec); -+ cgroup_account_cputime(curr, delta_exec); -+ -+ if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF) -+ get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec); -+ -+ trace_sched_stat_runtime(curr, delta_exec, 0); -+} -+ -+static bool hmbird_dsq_priq_less(struct rb_node *node_a, -+ const struct rb_node *node_b) -+{ -+ const struct hmbird_entity *a = -+ container_of(node_a, struct hmbird_entity, dsq_node.priq); -+ const struct hmbird_entity *b = -+ container_of(node_b, struct hmbird_entity, dsq_node.priq); -+ -+ return time_before64(a->dsq_vtime, b->dsq_vtime); -+} -+ -+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ bool is_local = dsq->id == HMBIRD_DSQ_LOCAL; -+ unsigned long flags; -+ -+ hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq || -+ !list_empty(&get_hmbird_ts(p)->dsq_node.fifo), -+ "task = %s, dsq->id = %llu\n", p->comm, dsq->id); -+ hmbird_cond_deferred_err(ENQ_EXIST2, -+ (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq), -+ "task = %s\n", p->comm); -+ -+ if (!is_local) { -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ); -+ hmbird_ops_error(": %s\n", -+ "attempting to dispatch to a destroyed dsq"); -+ /* fall back to the global dsq */ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ dsq = &hmbird_dsq_global; -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ } -+ } -+ -+ if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) { -+ get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ; -+ rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq, -+ hmbird_dsq_priq_less); -+ } else { -+ if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT)) -+ list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ else -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ } -+ dsq->nr++; -+ get_hmbird_ts(p)->dsq = dsq; -+ -+ /* -+ * We're transitioning out of QUEUEING or DISPATCHING. store_release to -+ * match waiters' load_acquire. -+ */ -+ if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS) -+ atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE); -+ -+ if (is_local) { -+ struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq); -+ struct rq *rq = hmbird->rq; -+ bool preempt = false; -+ -+ if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr && -+ rq->curr->sched_class == &hmbird_sched_class) { -+ get_hmbird_ts(rq->curr)->slice = 0; -+ preempt = true; -+ } -+ -+ if (preempt || sched_class_above(&hmbird_sched_class, -+ rq->curr->sched_class)) -+ resched_curr(rq); -+ } else { -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ } -+} -+ -+static void task_unlink_from_dsq(struct task_struct *p, -+ struct hmbird_dispatch_q *dsq) -+{ -+ if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) { -+ rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ; -+ } else { -+ list_del_init(&get_hmbird_ts(p)->dsq_node.fifo); -+ } -+} -+ -+static bool task_linked_on_dsq(struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+} -+ -+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq; -+ bool is_local = dsq == &hmbird_rq->local_dsq; -+ -+ if (!dsq) { -+ hmbird_cond_deferred_err(TASK_LINKED1, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ /* -+ * When dispatching directly from the BPF scheduler to a local -+ * DSQ, the task isn't associated with any DSQ but -+ * @get_hmbird_ts(p)->holding_cpu may be set under the protection of -+ * %HMBIRD_OPSS_DISPATCHING. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu >= 0) -+ get_hmbird_ts(p)->holding_cpu = -1; -+ return; -+ } -+ -+ if (!is_local) -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ /* -+ * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node -+ * can't change underneath us. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu < 0) { -+ /* @p must still be on @dsq, dequeue */ -+ hmbird_cond_deferred_err(TASK_UNLINKED, -+ !task_linked_on_dsq(p), "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ } else { -+ /* -+ * We're racing against dispatch_to_local_dsq() which already -+ * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the -+ * holding_cpu which tells dispatch_to_local_dsq() that it lost -+ * the race. -+ */ -+ hmbird_cond_deferred_err(TASK_LINKED2, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ get_hmbird_ts(p)->holding_cpu = -1; -+ } -+ get_hmbird_ts(p)->dsq = NULL; -+ -+ if (!is_local) -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+ -+static bool test_rq_online(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ return rq->online; -+#else -+ return true; -+#endif -+} -+ -+static void refill_task_slice(struct task_struct *p) -+{ -+ if (is_isolate_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO; -+ else if (is_pipeline_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2; -+ else -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+} -+ -+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -+ int sticky_cpu) -+{ -+ struct hmbird_dispatch_q *d; -+ s32 cpu = -1; -+ -+ hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), -+ "task = %s\n", p->comm); -+ -+ if (is_pcp_rt(p)) { -+ /* Enqueue percpu rt task to local directly. */ -+ /* Or cause a bug when disable dispatch. */ -+ if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ } -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (cpu >= 0) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ -+ if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ /* rq migration */ -+ if (sticky_cpu == cpu_of(rq)) -+ goto local_norefill; -+ /* -+ * If !rq->online, we already told the BPF scheduler that the CPU is -+ * offline. We're just trying to on/offline the CPU. Don't bother the -+ * BPF scheduler. -+ */ -+ if (unlikely(!test_rq_online(rq))) -+ goto local; -+ -+ /* see %HMBIRD_OPS_ENQ_LAST */ -+ if (enq_flags & HMBIRD_ENQ_LAST) -+ goto local; -+ -+ if (enq_flags & HMBIRD_ENQ_LOCAL) -+ goto local; -+ else -+ goto global; -+local: -+ /* -+ * For task-ordering, slice refill must be treated as implying the end -+ * of the current slice. Otherwise, the longer @p stays on the CPU, the -+ * higher priority it becomes from hmbird_prio_less()'s POV. -+ */ -+ refill_task_slice(p); -+local_norefill: -+ dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags); -+ slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq)); -+ return; -+ -+global: -+ d = find_dsq_from_task(p); -+ if (d) { -+ refill_task_slice(p); -+ dispatch_enqueue(d, p, enq_flags); -+ return; -+ } -+ slim_stats_record(GLOBAL_STAT, 0, 0, 0); -+ refill_task_slice(p); -+ dispatch_enqueue(&hmbird_dsq_global, p, enq_flags); -+} -+ -+static bool watchdog_task_watched(const struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->watchdog_node); -+} -+ -+static void watchdog_watch_task(struct rq *rq, struct task_struct *p) -+{ -+ lockdep_assert_rq_held(rq); -+ if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ get_hmbird_ts(p)->runnable_at = jiffies; -+ clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list); -+} -+ -+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) -+{ -+ list_del_init(&get_hmbird_ts(p)->watchdog_node); -+ if (reset_timeout) -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags) -+{ -+ int sticky_cpu = get_hmbird_ts(p)->sticky_cpu; -+ -+ enq_flags |= get_hmbird_rq(rq)->extra_enq_flags; -+ -+ if (sticky_cpu >= 0) -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ -+ /* -+ * Restoring a running task will be immediately followed by -+ * set_next_task_hmbird() which expects the task to not be on the BPF -+ * scheduler as tasks can only start running through local DSQs. Force -+ * direct-dispatch into the local DSQ by setting the sticky_cpu. -+ */ -+ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -+ sticky_cpu = cpu_of(rq); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_UNWATCHED, -+ !watchdog_task_watched(p), "task = %s\n", p->comm); -+ return; -+ } -+ -+ watchdog_watch_task(rq, p); -+ set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ get_hmbird_rq(rq)->nr_running++; -+ add_nr_running(rq, 1); -+ -+ do_enqueue_task(rq, p, enq_flags, sticky_cpu); -+} -+ -+static void ops_dequeue(struct task_struct *p, u64 deq_flags) -+{ -+ u64 opss; -+ -+ watchdog_unwatch_task(p, false); -+ -+ /* acquire ensures that we see the preceding updates on QUEUED */ -+ opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state); -+ -+ switch (opss & HMBIRD_OPSS_STATE_MASK) { -+ case HMBIRD_OPSS_NONE: -+ break; -+ case HMBIRD_OPSS_QUEUEING: -+ /* -+ * QUEUEING is started and finished while holding @p's rq lock. -+ * As we're holding the rq lock now, we shouldn't see QUEUEING. -+ */ -+ hmbird_deferred_err(DEQ_DEQING, " : unreachable path in %s\n", __func__); -+ break; -+ case HMBIRD_OPSS_QUEUED: -+ if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss, -+ HMBIRD_OPSS_NONE)) -+ break; -+ fallthrough; -+ case HMBIRD_OPSS_DISPATCHING: -+ /* -+ * If @p is being dispatched from the BPF scheduler to a DSQ, -+ * wait for the transfer to complete so that @p doesn't get -+ * added to its DSQ after dequeueing is complete. -+ * -+ * As we're waiting on DISPATCHING with the rq locked, the -+ * dispatching side shouldn't try to lock the rq while -+ * DISPATCHING is set. See dispatch_to_local_dsq(). -+ * -+ * DISPATCHING shouldn't have qseq set and control can reach -+ * here with NONE @opss from the above QUEUED case block. -+ * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss. -+ */ -+ wait_ops_state(p, HMBIRD_OPSS_DISPATCHING); -+ hmbird_cond_deferred_err(HMBIRD_OPN, -+ atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE, -+ "task = %s\n", p->comm); -+ break; -+ } -+} -+ -+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ -+ if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p), -+ "task = %s\n", p->comm); -+ return; -+ } -+ -+ ops_dequeue(p, deq_flags); -+ -+ if (slim_walt_ctrl) { -+ if (task_current(rq, p)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ if (deq_flags & HMBIRD_DEQ_SLEEP) -+ set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else -+ clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), -+ (unsigned long *)&get_hmbird_ts(p)->flags); -+ -+ clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm); -+ hmbird_rq->nr_running--; -+ sub_nr_running(rq, 1); -+ dispatch_dequeue(hmbird_rq, p); -+} -+ -+static void yield_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ get_hmbird_ts(p)->slice = 0; -+} -+ -+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to) -+{ -+ return false; -+} -+ -+#ifdef CONFIG_SMP -+/** -+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -+ * @rq: rq to move the task into, currently locked -+ * @p: task to move -+ * @enq_flags: %HMBIRD_ENQ_* -+ * -+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller -+ * must: -+ * -+ * 1. Start with exclusive access to @p either through its DSQ lock or -+ * %HMBIRD_OPSS_DISPATCHING flag. -+ * -+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id(). -+ * -+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't -+ * deadlock with dequeue. -+ * -+ * 4. Lock @rq and the task_rq from #3. -+ * -+ * 5. Call this function. -+ * -+ * Returns %true if @p was successfully moved. %false after racing dequeue and -+ * losing. -+ */ -+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ struct rq *task_rq; -+ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * If dequeue got to @p while we were trying to lock both rq's, it'd -+ * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have -+ * updated it to different values afterwards, as this operation can't be -+ * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become -+ * raw_smp_processor_id() again before we're done. Thus, we can tell -+ * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is -+ * still raw_smp_processor_id(). -+ * -+ * See dispatch_dequeue() for the counterpart. -+ */ -+ if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id())) -+ return false; -+ -+ /* @p->rq couldn't have changed if we're still the holding cpu */ -+ task_rq = task_rq(p); -+ lockdep_assert_rq_held(task_rq); -+ deactivate_task(task_rq, p, 0); -+ set_task_cpu(p, cpu_of(rq)); -+ get_hmbird_ts(p)->sticky_cpu = cpu_of(rq); -+ -+ /* -+ * We want to pass hmbird-specific enq_flags but activate_task() will -+ * truncate the upper 32 bit. As we own @rq, we can pass them through -+ * @get_hmbird_rq(rq)->extra_enq_flags instead. -+ */ -+ hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags, -+ "task = %s\n", p->comm); -+ get_hmbird_rq(rq)->extra_enq_flags = enq_flags; -+ activate_task(rq, p, 0); -+ get_hmbird_rq(rq)->extra_enq_flags = 0; -+ -+ return true; -+} -+ -+#endif /* CONFIG_SMP */ -+ -+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu) -+{ -+ int fitable = 1; -+ -+ return fitable; -+} -+ -+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq, -+ struct hmbird_dispatch_q *dsq) -+{ -+ bool dsq_misfit; -+ int cpu = cpu_of(rq); -+ u64 task_util = 0; -+ struct cluster_ctx ctx; -+ int dsq_int = dsq_id_to_internal(dsq); -+ -+ if (!cpumask_test_cpu(cpu, iso_masks.little)) -+ return false; -+ -+ gen_cluster_ctx(&ctx, BIG); -+ dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 && -+ dsq_int <= SCHED_PROP_DEADLINE_LEVEL4); -+#ifdef CLUSTER_SEPARATE -+ /* In rescue mode, little will consume big cluster's dsq.*/ -+ dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper); -+#endif -+ if (!dsq_misfit) -+ return false; -+ -+ if (p) { -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &task_util); -+ else -+ task_util = get_hmbird_ts(p)->demand_scaled; -+ } -+ -+ if (task_util <= misfit_ds) -+ return false; -+ -+ hmbird_info_trace(":task %s can't run on cpu%d, util = %llu\n", -+ p->comm, cpu, task_util); -+ return true; -+} -+ -+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq) -+{ -+ if (!task_fits_cpu_hmbird(p, cpu_of(rq))) -+ return false; -+ -+ if (check_misfit_task_on_little(p, rq, dsq)) -+ return false; -+ -+ return likely(test_rq_online(rq)); -+} -+ -+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return; -+ -+ if (add) -+ skipn[idx]++; -+ else -+ skipn[idx] = 0; -+} -+ -+static bool skip_too_much(struct hmbird_dispatch_q *dsq) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return false; -+ -+ if (skip_num[idx] > 3) { -+ skip_num[idx] = 0; -+ return true; -+ } -+ -+ return false; -+} -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rb_node *rb_node; -+ struct rq *task_rq; -+ unsigned long flags; -+ bool moved = false; -+ struct task_struct *may_fit = NULL; -+ int skip = 0; -+ -+retry: -+ if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -+ return false; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) { -+ set_skip_num(dsq, skip_num, (bool)may_fit); -+ goto this_rq; -+ } -+ if (skip_too_much(dsq)) -+ goto remote_rq; -+ if (!may_fit) -+ may_fit = p; -+ if (++skip <= 3) -+ continue; -+ /* -+ * If the recent 3 tasks not fit, use the first one. -+ * and clear the skip, because the first one is consumed. -+ */ -+ set_skip_num(dsq, skip_num, false); -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ /* No more task, use the first may fit task.*/ -+ if (may_fit) { -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ -+ for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) -+ goto this_rq; -+ goto remote_rq; -+ } -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ return false; -+ -+this_rq: -+ /* @dsq is locked and @p is on this rq */ -+ hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo); -+ dsq->nr--; -+ hmbird_rq->local_dsq.nr++; -+ get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ -+remote_rq: -+#ifdef CONFIG_SMP -+ if (cpu_same_cluster_stat(p, rq, task_rq)) -+ slim_stats_record(MOVE_RQ_CNT, 0, 0, 0); -+ else -+ slim_stats_record(MOVE_RQ_CNT, 1, 0, 0); -+ /* -+ * @dsq is locked and @p is on a remote rq. @p is currently protected by -+ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -+ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -+ * rq lock or fail, do a little dancing from our side. See -+ * move_task_to_local_dsq(). -+ */ -+ hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id(); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ rq_unpin_lock(rq, rf); -+ double_lock_balance(rq, task_rq); -+ rq_repin_lock(rq, rf); -+ -+ moved = move_task_to_local_dsq(rq, p, 0); -+ -+ double_unlock_balance(rq, task_rq); -+#endif /* CONFIG_SMP */ -+ if (likely(moved)) { -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ } -+ may_fit = NULL; -+ goto retry; -+} -+ -+ -+static int balance_one(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf, bool local) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class; -+ -+ if (!hmbird_rq) -+ return 1; -+ -+ lockdep_assert_rq_held(rq); -+ -+ if (static_branch_unlikely(&hmbird_ops_cpu_preempt) && -+ unlikely(get_hmbird_rq(rq)->cpu_released)) { -+ /* -+ * If the previous sched_class for the current CPU was not HMBIRD, -+ * notify the BPF scheduler that it again has control of the -+ * core. This callback complements ->cpu_release(), which is -+ * emitted in hmbird_notify_pick_next_task(). -+ */ -+ get_hmbird_rq(rq)->cpu_released = false; -+ } -+ -+ if (prev_on_hmbird) -+ update_curr_hmbird(rq); -+ /* if there already are tasks to run, nothing to do */ -+ if (hmbird_rq->local_dsq.nr) -+ return 1; -+ -+ if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) { -+ slim_stats_record(GLOBAL_STAT, 1, 0, 0); -+ return 1; -+ } -+ -+ if (consume_dispatch_global(rq, rf)) -+ return 1; -+ -+ return 0; -+} -+ -+static int balance_hmbird(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf) -+{ -+ return balance_one(rq, prev, rf, true); -+} -+ -+/* -+ * output task util to systrace, only for debug mode. -+ * we can not output too many logs to systrace buffer even in debug mode -+ * only output debug-info while it exceed misfit_ds. -+ */ -+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p) -+{ -+ static DEFINE_PER_CPU(int, is_last_exceed); -+ int cpu = cpu_of(rq); -+ u64 util = 0; -+ -+ if (likely(!debug_enabled())) -+ return; -+ -+ if (!p) -+ return; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ util = uclamp_rq_util_with(rq, util, p); -+ -+ if (util >= misfit_ds) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util); -+ per_cpu(is_last_exceed, cpu) = true; -+ } else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0); -+ per_cpu(is_last_exceed, cpu) = false; -+ } else { -+ } -+} -+ -+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first) -+{ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ /* -+ * Core-sched might decide to execute @p before it is -+ * dispatched. Call ops_dequeue() to notify the BPF scheduler. -+ */ -+ ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC); -+ dispatch_dequeue(get_hmbird_rq(rq), p); -+ } -+ -+ p->se.exec_start = rq_clock_task(rq); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK); -+ } -+ -+ watchdog_unwatch_task(p, true); -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx); -+ systrace_output_cpu_ds(rq, p); -+ /* -+ * @p is getting newly scheduled or got kicked after someone updated its -+ * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird(). -+ */ -+ if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) != -+ (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) { -+ if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) -+ get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK; -+ else -+ get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK; -+ -+ sched_update_tick_dependency(rq); -+ } -+ -+ p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime; -+} -+ -+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p) -+{ -+ update_curr_hmbird(rq); -+ -+ update_dispatch_dsq_info(rq, p); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ watchdog_watch_task(rq, p); -+ -+ if (is_pipeline_task(p)) { -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1); -+ return; -+ } -+ -+ /* -+ * If we're in the pick_next_task path, balance_hmbird() should -+ * have already populated the local DSQ if there are any other -+ * available tasks. If empty, tell ops.enqueue() that @p is the -+ * only one available for this cpu. ops.enqueue() should put it -+ * on the local DSQ so that the subsequent pick_next_task_hmbird() -+ * can find the task unless it wants to trigger a separate -+ * follow-up scheduling event. -+ */ -+ if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1); -+ else -+ do_enqueue_task(rq, p, 0, -1); -+ } -+} -+ -+static struct task_struct *first_local_task(struct rq *rq) -+{ -+ struct rb_node *rb_node; -+ struct hmbird_entity *entity; -+ -+ if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) { -+ entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo, -+ struct hmbird_entity, dsq_node.fifo); -+ return entity->task; -+ } -+ -+ rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq); -+ if (rb_node) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ return entity->task; -+ } -+ return NULL; -+} -+ -+static struct task_struct *pick_next_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p; -+ -+ p = first_local_task(rq); -+ if (!p) -+ return NULL; -+ -+ if (unlikely(!get_hmbird_ts(p)->slice)) { -+ if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice) -+ hmbird_warned_zero_slice = true; -+ -+ refill_task_slice(p); -+ } -+ -+ set_next_task_hmbird(rq, p, true); -+ -+ return p; -+} -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task, -+ const struct sched_class *active) -+{ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * The callback is conceptually meant to convey that the CPU is no -+ * longer under the control of HMBIRD. Therefore, don't invoke the -+ * callback if the CPU is staying on HMBIRD, or going idle (in which -+ * case the HMBIRD scheduler has actively decided not to schedule any -+ * tasks on the CPU). -+ */ -+ if (likely(active >= &hmbird_sched_class)) -+ return; -+ -+ /* -+ * At this point we know that HMBIRD was preempted by a higher priority -+ * sched_class, so invoke the ->cpu_release() callback if we have not -+ * done so already. We only send the callback once between HMBIRD being -+ * preempted, and it regaining control of the CPU. -+ * -+ * ->cpu_release() complements ->cpu_acquire(), which is emitted the -+ * next time that balance_hmbird() is invoked. -+ */ -+ if (!get_hmbird_rq(rq)->cpu_released) -+ get_hmbird_rq(rq)->cpu_released = true; -+} -+ -+#ifdef CONFIG_SMP -+ -+static bool test_and_clear_cpu_idle(int cpu) -+{ -+ if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -+ if (cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ return true; -+ } else { -+ return false; -+ } -+} -+ -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) -+{ -+ int cpu; -+ -+ do { -+ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -+ if (cpu >= nr_cpu_ids) -+ return -EBUSY; -+ } while (!test_and_clear_cpu_idle(cpu)); -+ -+ return cpu; -+} -+ -+ -+static bool prev_cpu_misfit(int prev) -+{ -+ if (!is_partial_enabled() && is_partial_cpu(prev)) -+ return true; -+ -+ return false; -+} -+ -+static int heavy_rt_placement(struct task_struct *p, int prev) -+{ -+ struct cpumask tmp = {.bits = {0}}; -+ int cpu; -+ u64 util = 0; -+ -+ if (!rt_prio(p->prio)) -+ return -EFAULT; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ -+ if (util < misfit_ds) -+ return -EFAULT; -+ -+ if (is_partial_enabled()) -+ cpumask_or(&tmp, iso_masks.big, iso_masks.partial); -+ else -+ cpumask_copy(&tmp, iso_masks.big); -+ -+ cpu = hmbird_pick_idle_cpu(&tmp); -+ if (cpu >= 0) -+ return cpu; -+ -+ if (cpumask_test_cpu(prev, iso_masks.big) || -+ (is_partial_enabled() && is_partial_cpu(prev))) -+ return prev; -+ -+ return cpumask_first(&tmp); -+} -+ -+static int spec_task_before_pick_idle(struct task_struct *p, int prev) -+{ -+ int cpu; -+ -+ cpu = heavy_rt_placement(p, prev); -+ if (cpu >= 0) -+ return cpu; -+ return -EFAULT; -+} -+ -+static int cpumask_distribute_next(struct cpumask *mask, int *prev) -+{ -+ int p = *prev, n; -+ -+ n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1); -+ if (n < nr_cpu_ids) -+ WRITE_ONCE(*prev, n); -+ return n; -+} -+ -+static int repick_fallback_cpu(void) -+{ -+ /* -+ * partial cpu follow big cluster's Scheduling policy, -+ * simply return first bit cpu. -+ */ -+ return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev); -+} -+ -+/* -+ * Must return a valid cpu num, as this task's cpu. -+ */ -+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu, -+ struct cpumask *mask, int *prev_mask) -+{ -+ int cpu; -+ -+ cpu = hmbird_pick_idle_cpu(mask); -+ if (cpu >= 0) -+ return cpu; -+ return cpumask_distribute_next(mask, prev_mask); -+} -+ -+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type) -+{ -+ int idx; -+ struct cluster_ctx ctx; -+ -+ idx = find_idx_from_task(p); -+ if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS) -+ return false; -+ -+ gen_cluster_ctx(&ctx, type); -+ if (idx >= ctx.lower && idx < ctx.upper) -+ return true; -+ return false; -+} -+ -+static bool is_valid_cpu(int cpu) -+{ -+ return (cpu >= 0) && (cpu < nr_cpu_ids); -+} -+ -+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) -+{ -+ s32 cpu = -1; -+ struct cpumask mask = {.bits = {0}}; -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (is_valid_cpu(cpu)) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ partial_dynamic_ctrl(); -+ -+ if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ iso_masks.big, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ if (p->nr_cpus_allowed == 1) { -+ cpu = cpumask_any(p->cpus_ptr); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* For non-period global dsq, not contain pcp task. */ -+ if (task_only_blongs_to_cluster(p, LITTLE)) { -+ cpumask_copy(&mask, iso_masks.little); -+ if (unlikely(l_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &little_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ if (task_only_blongs_to_cluster(p, BIG)) { -+ cpumask_copy(&mask, iso_masks.big); -+ if (unlikely(b_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ cpu = spec_task_before_pick_idle(p, prev_cpu); -+ if (is_valid_cpu(cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* if the previous CPU is idle, dispatch directly to it */ -+ if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+ } -+ -+ cpu = hmbird_pick_idle_cpu(cpu_possible_mask); -+ if (is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ if (prev_cpu_misfit(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ cpu = repick_fallback_cpu(); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+} -+ -+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags) -+{ -+ return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags); -+} -+ -+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx) -+{ -+ set_cpus_allowed_common(p, ctx); -+} -+ -+static void reset_idle_masks(void) -+{ -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little); -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big); -+ if (is_partial_enabled()) -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial); -+ hmbird_has_idle_cpus = true; -+} -+ -+void __hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ int cpu = cpu_of(rq); -+ struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -+ -+ if (skip_update_idle()) -+ return; -+ -+ if (idle) { -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ if (!hmbird_has_idle_cpus) -+ hmbird_has_idle_cpus = true; -+ -+ /* -+ * idle_masks.smt handling is racy but that's fine as it's only -+ * for optimization and self-correcting. -+ */ -+ for_each_cpu(cpu, sib_mask) { -+ if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -+ return; -+ } -+ cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -+ } else { -+ cpumask_clear_cpu(cpu, idle_masks.cpu); -+ if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ -+ cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -+ } -+} -+ -+#else /* !CONFIG_SMP */ -+ -+static bool test_and_clear_cpu_idle(int cpu) { return false; } -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } -+static void reset_idle_masks(void) {} -+ -+#endif /* CONFIG_SMP */ -+ -+static bool check_rq_for_timeouts(struct rq *rq) -+{ -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rq_flags rf; -+ -+ rq_lock_irqsave(rq, &rf); -+ list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) { -+ unsigned long last_runnable; -+ -+ p = entity->task; -+ last_runnable = get_hmbird_ts(p)->runnable_at; -+ -+ if (unlikely(time_after(jiffies, -+ last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -+ -+ rq_unlock_irqrestore(rq, &rf); -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb", -+ p->comm, p->pid, -+ dur_ms / 1000, dur_ms % 1000, -+ get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0, -+ cpumask_pr_args(&p->cpus_mask)); -+ return 1; -+ } -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ return 0; -+} -+ -+static void hmbird_watchdog_workfn(struct work_struct *work) -+{ -+ int cpu; -+ bool timeout; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ -+ for_each_online_cpu(cpu) { -+ timeout = check_rq_for_timeouts(cpu_rq(cpu)); -+ if (unlikely(timeout)) -+ break; -+ -+ cond_resched(); -+ } -+ if (!timeout) -+ queue_delayed_work(system_unbound_wq, to_delayed_work(work), -+ hmbird_watchdog_timeout / 2); -+} -+ -+static void set_pcp_round(struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) { -+ per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round); -+ per_cpu(pcp_info, cpu).pcp_round = true; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+} -+ -+ -+/* -+ * Just for debug: output hmbird on/off state per 10s. -+ */ -+#define OUTPUT_INTVAL (msecs_to_jiffies(10 * 1000)) -+static void inform_hmbird_onoff_from_systrace(void) -+{ -+ static unsigned long __read_mostly next_print; -+ -+ if (time_before(jiffies, READ_ONCE(next_print))) -+ return; -+ -+ WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL); -+ hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio); -+} -+ -+void hmbird_notify_sched_tick(void) -+{ -+ unsigned long last_check; -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ hmbird_scheduler_tick(); -+ -+ if (!hmbird_enabled()) -+ return; -+ -+ last_check = hmbird_watchdog_timestamp; -+ if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -+ -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "watchdog failed to check in for %u.%03us", -+ dur_ms / 1000, dur_ms % 1000); -+ } -+ scan_timeout(rq); -+} -+ -+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued) -+{ -+ update_curr_hmbird(rq); -+ -+ set_pcp_round(rq); -+ -+ if (slim_walt_ctrl) -+ hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE); -+ /* -+ * While disabling, always resched and refresh core-sched timestamp as -+ * we can't trust the slice management or ops.core_sched_before(). -+ */ -+ if (hmbird_ops_disabling()) -+ get_hmbird_ts(curr)->slice = 0; -+ -+ if (!get_hmbird_ts(curr)->slice) -+ resched_curr(rq); -+ -+ inform_hmbird_onoff_from_systrace(); -+} -+ -+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg) -+{ -+ hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ get_hmbird_ts(p)->disallow = false; -+ -+ hmbird_sched_init_task(p); -+ -+ set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ return 0; -+} -+ -+static void hmbird_ops_enable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void hmbird_ops_disable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void set_task_hmbird_weight(struct task_struct *p) -+{ -+ u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -+ -+ get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight); -+} -+ -+/** -+ * refresh_hmbird_weight - Refresh a task's hmbird weight -+ * @p: task to refresh hmbird weight for -+ * -+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to -+ * enable easy access from the BPF scheduler. To keep it synchronized with the -+ * current task priority, this function should be called when a new task is -+ * created, priority is changed for a task on hmbird, and a task is switched -+ * to hmbird from other classes. -+ */ -+static void refresh_hmbird_weight(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ set_task_hmbird_weight(p); -+} -+ -+int hmbird_pre_fork(struct task_struct *p) -+{ -+ int ret = 0; -+ -+ p->android_oem_data1[HMBIRD_TS_IDX] = -+ (u64)(kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL)); -+ if (!get_hmbird_ts(p)) { -+ ret = -1; -+ goto lock; -+ } -+ -+ get_hmbird_ts(p)->dsq = NULL; -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node); -+ get_hmbird_ts(p)->flags = 0; -+ get_hmbird_ts(p)->weight = 0; -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ get_hmbird_ts(p)->holding_cpu = -1; -+ get_hmbird_ts(p)->kf_mask = 0; -+ atomic64_set(&get_hmbird_ts(p)->ops_state, 0); -+ get_hmbird_ts(p)->runnable_at = INITIAL_JIFFIES; -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+ get_hmbird_ts(p)->task = p; -+ hmbird_set_sched_prop(p, 0); -+ -+ get_hmbird_ts(p)->critical_affinity_cpu = -1; -+ get_hmbird_ts(p)->sched_class = &hmbird_sched_class; -+ -+ /* -+ * BPF scheduler enable/disable paths want to be able to iterate and -+ * update all tasks which can become complex when racing forks. As -+ * enable/disable are very cold paths, let's use a percpu_rwsem to -+ * exclude forks. -+ */ -+lock: -+ percpu_down_read(&hmbird_fork_rwsem); -+ -+ return ret; -+} -+ -+int hmbird_fork(struct task_struct *p) -+{ -+ percpu_rwsem_assert_held(&hmbird_fork_rwsem); -+ -+ if (hmbird_enabled()) -+ return hmbird_ops_prepare_task(p, task_group(p)); -+ else -+ return 0; -+} -+ -+void hmbird_post_fork(struct task_struct *p) -+{ -+ if (hmbird_enabled()) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ /* -+ * Set the weight manually before calling ops.enable() so that -+ * the scheduler doesn't see a stale value if they inspect the -+ * task struct. We'll invoke ops.set_weight() afterwards, as it -+ * would be odd to receive a callback on the task before we -+ * tell the scheduler that it's been fully enabled. -+ */ -+ set_task_hmbird_weight(p); -+ hmbird_ops_enable_task(p); -+ refresh_hmbird_weight(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ -+ spin_lock_irq(&hmbird_tasks_lock); -+ list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_cancel_fork(struct task_struct *p) -+{ -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ if (hmbird_enabled()) -+ hmbird_ops_disable_task(p); -+ -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_free(struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+ list_del_init(&get_hmbird_ts(p)->tasks_node); -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+ -+ /* -+ * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED -> -+ * ENABLED transitions can't race us. Disable ops for @p. -+ */ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) || -+ test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ hmbird_ops_disable_task(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+} -+ -+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio) -+{ -+} -+ -+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type) -+{ -+ return (prop >> TOP_TASK_SHIFT) & (1 << type); -+} -+ -+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p) -+{ -+ uint32_t prop = get_top_task_prop(p); -+ -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE)) -+ return HMBIRD_TASK_PROP_PIPELINE; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) || -+ !task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) { -+ return HMBIRD_TASK_PROP_COMMON; -+ } -+ return HMBIRD_TASK_PROP_DEBUG_OR_LOG; -+} -+ -+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b) -+{ -+ int type_a = hmbird_get_task_type(a); -+ int type_b = hmbird_get_task_type(b); -+ -+ return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b]; -+} -+ -+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ enum cpu_type type; -+ int sp_dl; -+ struct task_struct *curr = NULL; -+ -+ switch (hmbird_preempt_policy) { -+ case HMBIRD_PREEMPT_POLICY_PRIO_BASED: -+ curr = rq->curr; -+ if (curr && hmbird_prio_higher(p, curr)) -+ goto preempt; -+ break; -+ default: -+ break; -+ } -+ if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) || -+ (is_critical_system_task(p) && !is_critical_system_task(rq->curr))) -+ goto preempt; -+ -+ sp_dl = find_idx_from_task(p); -+ if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled())) -+ return; -+ -+ if (rq->curr->prio > p->prio) -+ goto preempt; -+ -+ return; -+ -+preempt: -+ resched_curr(rq); -+} -+ -+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {} -+ -+int hmbird_check_setscheduler(struct task_struct *p, int policy) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ /* if disallow, reject transitioning into HMBIRD */ -+ if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) && -+ p->policy != policy && policy == SCHED_HMBIRD) -+ return -EACCES; -+ -+ return 0; -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool hmbird_can_stop_tick(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ if (hmbird_ops_disabling()) -+ return false; -+ -+ if (p->sched_class != &hmbird_sched_class) -+ return true; -+ -+ return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK; -+} -+#endif -+ -+int hmbird_tg_online(struct task_group *tg) -+{ -+ struct cgroup *cgrp; -+ -+ if (!tg) -+ return 0; -+ cgrp = tg->css.cgroup; -+ if (!cgrp || !(cgrp->kn)) -+ return 0; -+ update_cgroup_ids_table(cgrp->kn->id, -1); -+ return 0; -+} -+ -+/* -+ * Omitted operations: -+ * -+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -+ * task isn't tied to the CPU at that point. Preemption is implemented by -+ * resetting the victim task's slice to 0 and triggering reschedule on the -+ * target CPU. -+ * -+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -+ * -+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -+ * their current sched_class. Call them directly from sched core instead. -+ * -+ * - task_woken, switched_from: Unnecessary. -+ */ -+DEFINE_SCHED_CLASS(hmbird) = { -+ .enqueue_task = enqueue_task_hmbird, -+ .dequeue_task = dequeue_task_hmbird, -+ .yield_task = yield_task_hmbird, -+ .yield_to_task = yield_to_task_hmbird, -+ -+ .check_preempt_curr = check_preempt_curr_hmbird, -+ -+ .pick_next_task = pick_next_task_hmbird, -+ -+ .put_prev_task = put_prev_task_hmbird, -+ .set_next_task = set_next_task_hmbird, -+ -+#ifdef CONFIG_SMP -+ .balance = balance_hmbird, -+ .select_task_rq = select_task_rq_hmbird, -+ .set_cpus_allowed = set_cpus_allowed_hmbird, -+#endif -+ .task_tick = task_tick_hmbird, -+ -+ .switched_to = switched_to_hmbird, -+ .prio_changed = prio_changed_hmbird, -+ -+ .update_curr = update_curr_hmbird, -+ -+#ifdef CONFIG_UCLAMP_TASK -+ .uclamp_enabled = 0, -+#endif -+}; -+ -+/* -+ * Must with rq lock held. -+ */ -+bool task_is_hmbird(struct task_struct *p) -+{ -+ return p->sched_class == &hmbird_sched_class; -+} -+ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id) -+{ -+ memset(dsq, 0, sizeof(*dsq)); -+ -+ raw_spin_lock_init(&dsq->lock); -+ INIT_LIST_HEAD(&dsq->fifo); -+ dsq->id = dsq_id; -+} -+ -+static int hmbird_cgroup_init(void) -+{ -+ struct cgroup_subsys_state *css; -+ -+ css_for_each_descendant_pre(css, &root_task_group.css) { -+ struct task_group *tg = css_tg(css); -+ -+ cgrp_dsq_idx_init(css->cgroup, tg); -+ } -+ return 0; -+} -+ -+/* -+ * Used by sched_fork() and __setscheduler_prio() to pick the matching -+ * sched_class. dl/rt are already handled. -+ */ -+bool task_on_hmbird(struct task_struct *p) -+{ -+ return hmbird_enabled(); -+} -+ -+static void __setscheduler_prio(struct task_struct *p, int prio) -+{ -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) { -+ p->prio = prio; -+ return; -+ } else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (on_hmbird && rt_prio(prio)) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ -+ p->prio = prio; -+} -+ -+/* -+ * Heartbeat, avoid humbird keep running while APP already exit. -+ * Check whether APP send alive-signal periodly. -+ */ -+#define HEARTBEAT_TIMEOUT (msecs_to_jiffies(2500)) -+#define HEARTBEAT_CHECK_INTERVAL (msecs_to_jiffies(1000)) -+static struct timer_list hb_timer; -+static unsigned long next_hb; -+ -+void hb_timer_handler(struct timer_list *timer) -+{ -+ if (!heartbeat_enable) -+ goto refill; -+ -+ pr_err(": enter timer.\n"); -+ if (heartbeat) { -+ heartbeat = 0; -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ } -+ -+ /* can't detect heartbeat, disable ext. */ -+ if (time_after(jiffies, READ_ONCE(next_hb))) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT, -+ "can't detect heartbeat, disable ext\n"); -+ } -+refill: -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_start(void) -+{ -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_init(void) -+{ -+ timer_setup(&hb_timer, hb_timer_handler, 0); -+} -+ -+static void hb_timer_exit(void) -+{ -+ del_timer(&hb_timer); -+} -+ -+static bool check_and_disable_cpuhp(void) -+{ -+ struct rq *rq; -+ struct rq_flags rf; -+ int cpu; -+ -+ cpu_hotplug_disable(); -+ cpus_read_lock(); -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ rq_lock_irqsave(rq, &rf); -+ if (!rq->online) { -+ rq_unlock_irqrestore(rq, &rf); -+ goto err_offline; -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ cpus_read_unlock(); -+ return true; -+ -+err_offline: -+ cpus_read_unlock(); -+ cpu_hotplug_enable(); -+ return false; -+} -+ -+static void reenable_cpuhp(void) -+{ -+ cpu_hotplug_enable(); -+} -+ -+static void scheduler_switch_done(bool final_state) -+{ -+ /* set scx_enable again, switch may fail. */ -+ scx_enable = final_state; -+ /* reset heartbeat*/ -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ -+ if (final_state) -+ hb_timer_start(); -+ else -+ hb_timer_exit(); -+} -+ -+/** -+ * ss : curr switch state -+ * finish : success or fail -+ * enable : curr operation is diabling or enabling? -+ * fail_reason : string output when failed, empty string when success. -+ */ -+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason) -+{ -+ char *s1 = finish ? "finished" : "failed"; -+ char *s2 = enable ? "enabled" : "disabled"; -+ -+ hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss); -+ if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) { -+ sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type)); -+ hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n", -+ s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason); -+ } -+ curr_ss = ss; -+} -+ -+bool get_hmbird_ops_enabled(void) -+{ -+ return atomic_read(&__hmbird_ops_enabled); -+} -+ -+bool get_non_hmbird_task(void) -+{ -+ return atomic_read(&non_hmbird_task); -+} -+ -+static void hmbird_ops_disable_workfn(struct kthread_work *work) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int cpu; -+ -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, ""); -+ cancel_delayed_work_sync(&hmbird_watchdog_work); -+ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) { -+ case HMBIRD_OPS_DISABLED: -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled"); -+ scheduler_switch_done(false); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return; -+ case HMBIRD_OPS_PREPPING: -+ fallthrough; -+ case HMBIRD_OPS_DISABLING: -+ /* shouldn't happen but handle it like ENABLING if it does */ -+ WARN_ONCE(true, "hmbird: duplicate disabling instance?"); -+ fallthrough; -+ case HMBIRD_OPS_ENABLING: -+ case HMBIRD_OPS_ENABLED: -+ break; -+ } -+ -+ /* kick all CPUs to restore ticks */ -+ for_each_possible_cpu(cpu) -+ resched_cpu(cpu); -+ -+ /* avoid racing against fork and cgroup changes */ -+ cpus_read_lock(); -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, ""); -+ spin_lock_irq(&hmbird_tasks_lock); -+ atomic_set(&__hmbird_ops_enabled, false); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ bool alive = READ_ONCE(p->__state) != TASK_DEAD; -+ -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ get_hmbird_ts(p)->slice = min_t(u64, -+ get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL); -+ -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ if (alive) -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ -+ hmbird_ops_disable_task(p); -+ } -+ hmbird_task_iter_exit(&sti); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, ""); -+ -+ atomic_set(&non_hmbird_task, true); -+ /* no task is on hmbird, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt); -+ synchronize_rcu(); -+ -+ percpu_up_write(&hmbird_fork_rwsem); -+ cpus_read_unlock(); -+ -+ if (slim_walt_ctrl) -+ slim_walt_enable(false); -+ -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 1, 0, ""); -+ scheduler_switch_done(false); -+ reenable_cpuhp(); -+} -+ -+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn); -+ -+static void schedule_hmbird_ops_disable_work(void) -+{ -+ struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper); -+ -+ /* -+ * We may be called spuriously before the first bpf_hmbird_reg(). If -+ * hmbird_ops_helper isn't set up yet, there's nothing to do. -+ */ -+ if (helper) -+ kthread_queue_work(helper, &hmbird_ops_disable_work); -+} -+ -+static void hmbird_ops_disable(void) -+{ -+ schedule_hmbird_ops_disable_work(); -+} -+ -+static void hmbird_err_exit_workfn(struct work_struct *work) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ hmbird_ctrl(false); -+ -+ for_each_present_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy) -+ continue; -+ if (cpu != policy->cpu) -+ goto put; -+ down_write(&policy->rwsem); -+ WARN_ON(store_scaling_governor(policy, -+ saved_gov[cpu], strlen(saved_gov[cpu])) <= 0); -+ up_write(&policy->rwsem); -+ hmbird_info_trace(":restore origin gov : %s\n", saved_gov[cpu]); -+put: -+ cpufreq_cpu_put(policy); -+ } -+ memset((char *)saved_gov, 0, sizeof(saved_gov)); -+} -+ -+void hmbird_ops_exit(void) -+{ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); -+} -+ -+static struct kthread_worker *hmbird_create_rt_helper(const char *name) -+{ -+ struct kthread_worker *helper; -+ -+ helper = kthread_create_worker(0, name); -+ if (helper) -+ sched_set_fifo(helper->task); -+ return helper; -+} -+ -+static inline void set_audio_thread_sched_prop(struct task_struct *p) -+{ -+ struct cgroup_subsys_state *css; -+ -+ if (likely(p->prio >= MAX_RT_PRIO)) -+ return; -+ -+ rcu_read_lock(); -+ css = task_css(p, cpuset_cgrp_id); -+ if (!css) { -+ rcu_read_unlock(); -+ return; -+ } -+ rcu_read_unlock(); -+ -+ if (!strcmp(css->cgroup->kn->name, "audio-app")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+} -+ -+static int hmbird_ops_enable(void *unused) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int ret; -+ int tcnt = 0; -+ unsigned long long start = 0; -+ -+ if (!check_and_disable_cpuhp()) { -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline"); -+ return -EBUSY; -+ } -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, ""); -+ mutex_lock(&hmbird_ops_enable_mutex); -+ -+ if (!hmbird_ops_helper) { -+ WRITE_ONCE(hmbird_ops_helper, -+ hmbird_create_rt_helper("hmbird_ops_helper")); -+ if (!hmbird_ops_helper) { -+ ret = -ENOMEM; -+ goto err_unlock; -+ } -+ } -+ -+ if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) { -+ ret = -EBUSY; -+ goto err_unlock; -+ } -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) != -+ HMBIRD_OPS_DISABLED); -+ -+ hmbird_warned_zero_slice = false; -+ -+ atomic64_set(&hmbird_nr_rejected, 0); -+ -+ /* -+ * Keep CPUs stable during enable so that the BPF scheduler can track -+ * online CPUs by watching ->on/offline_cpu() after ->init(). -+ */ -+ cpus_read_lock(); -+ -+ hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work, -+ hmbird_watchdog_timeout / 2); -+ -+ /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ */ -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ reset_idle_masks(); -+ -+ /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. -+ */ -+ ret = hmbird_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock; -+ -+ /* -+ * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem -+ * preventing new tasks from being added. No need to exclude tasks -+ * leaving as hmbird_free() can handle both prepped and enabled -+ * tasks. Prep all tasks first and then enable them with preemption -+ * disabled. -+ */ -+ spin_lock_irq(&hmbird_tasks_lock); -+ -+ atomic_set(&non_hmbird_task, false); -+ atomic_set(&__hmbird_ops_enabled, true); -+ -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered(&sti))) -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ hmbird_task_iter_exit(&sti); -+ -+ /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. -+ */ -+ preempt_disable(); -+ -+ /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ */ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) { -+ atomic_set(&non_hmbird_task, true); -+ atomic_set(&__hmbird_ops_enabled, false); -+ preempt_enable(); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ ret = -EBUSY; -+ goto err_disable_unlock; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED -+ * transitions here are synchronized against hmbird_free() through -+ * hmbird_tasks_lock. -+ */ -+ start = sched_clock(); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, ""); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ tcnt++; -+ if (READ_ONCE(p->__state) != TASK_DEAD) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ -+ set_audio_thread_sched_prop(p); -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ hmbird_ops_enable_task(p); -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ } else { -+ hmbird_ops_disable_task(p); -+ } -+ } -+ hmbird_task_iter_exit(&sti); -+ -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, ""); -+ preempt_enable(); -+ percpu_up_write(&hmbird_fork_rwsem); -+ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) { -+ ret = -EBUSY; -+ goto err_disable; -+ } -+ -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_ENABLED, 1, 1, ""); -+ scheduler_switch_done(true); -+ -+ return 0; -+ -+err_unlock: -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock"); -+ scheduler_switch_done(false); -+ return ret; -+ -+err_disable_unlock: -+ percpu_up_write(&hmbird_fork_rwsem); -+err_disable: -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ /* must be fully disabled before returning */ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable"); -+ scheduler_switch_done(false); -+ return ret; -+} -+ -+#ifdef CONFIG_SCHED_DEBUG -+static const char *hmbird_ops_enable_state_str[] = { -+ [HMBIRD_OPS_PREPPING] = "prepping", -+ [HMBIRD_OPS_ENABLING] = "enabling", -+ [HMBIRD_OPS_ENABLED] = "enabled", -+ [HMBIRD_OPS_DISABLING] = "disabling", -+ [HMBIRD_OPS_DISABLED] = "disabled", -+}; -+ -+static int hmbird_debug_show(struct seq_file *m, void *v) -+{ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled()); -+ seq_printf(m, "%-30s: %s\n", "enable_state", -+ hmbird_ops_enable_state_str[hmbird_ops_enable_state()]); -+ seq_printf(m, "%-30s: %llu\n", "nr_rejected", -+ atomic64_read(&hmbird_nr_rejected)); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return 0; -+} -+ -+static int hmbird_debug_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_debug_show, NULL); -+} -+ -+const struct file_operations sched_hmbird_fops = { -+ .open = hmbird_debug_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif -+ -+ -+static int bpf_hmbird_reg(void *kdata) -+{ -+ return hmbird_ops_enable(kdata); -+} -+ -+static int bpf_hmbird_unreg(void *kdata) -+{ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ return 0; -+} -+ -+void set_hmbird_module_loaded(int is_loaded) -+{ -+ atomic_set(&hmbird_module_loaded, is_loaded); -+} -+ -+/* -+ * MUST load hmbird module before enable hmbird scheduler -+ * load track & hmbird gover implemented in hmbird module -+ */ -+int hmbird_ctrl(bool enable) -+{ -+ if (!atomic_read(&hmbird_module_loaded)) { -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "ext module unloaded\n"); -+ return -EINVAL; -+ } -+ -+ if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED -+ || hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING -+ || hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already enabled(ing)\n"); -+ hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING || -+ hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already disabled(ing)\n"); -+ hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (enable) -+ return bpf_hmbird_reg(NULL); -+ else -+ return bpf_hmbird_unreg(NULL); -+} -+ -+void set_cpu_isomask(int cpu, cpumask_var_t *mask) -+{ -+ cpumask_clear_cpu(cpu, iso_masks.ex_free); -+ cpumask_clear_cpu(cpu, iso_masks.exclusive); -+ cpumask_clear_cpu(cpu, iso_masks.partial); -+ cpumask_clear_cpu(cpu, iso_masks.big); -+ cpumask_clear_cpu(cpu, iso_masks.little); -+ cpumask_set_cpu(cpu, *mask); -+} -+ -+void set_cpu_cluster(u64 cpu_cluster) -+{ -+ int cpu; -+ -+ for_each_present_cpu(cpu) { -+ u64 pos = 1 << cpu; -+ -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.ex_free)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.exclusive)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.partial)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.big)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) -+ set_cpu_isomask(cpu, &(iso_masks.little)); -+ } -+} -+ -+static void get_hmbird_snapshot(struct panic_snapshot_t *p) -+{ -+ struct hmbird_dispatch_q *dsq; -+ struct rq *rq; -+ int cpu, i; -+ -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ p->rq_nr[cpu] = rq->nr_running; -+ p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running; -+ } -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ struct hmbird_entity *entity; -+ -+ dsq = &gdsqs[i]; -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo)) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ p->runnable_at[i] = entity->runnable_at; -+ raw_spin_unlock(&dsq->lock); -+ -+ } -+ -+ p->snap_misc.hmbird_enabled = hmbird_enabled(); -+ p->snap_misc.curr_ss = curr_ss; -+ p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state(); -+ p->snap_misc.parctrl_high_ratio = parctrl_high_ratio; -+ p->snap_misc.parctrl_low_ratio = parctrl_low_ratio; -+ p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l; -+ p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l; -+ p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio; -+ p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio; -+ p->snap_misc.misfit_ds = misfit_ds; -+ p->snap_misc.partial_enable = partial_enable; -+ p->snap_misc.iso_free_rescue = iso_free_rescue; -+ p->snap_misc.isolate_ctrl = isolate_ctrl; -+ p->snap_misc.snap_jiffies = jiffies; -+ p->snap_misc.snap_time = local_clock(); -+} -+ -+// MTK minidump begin -+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3) -+{ -+ strscpy(m->desc_str, str, DESC_STR_LEN); -+ m->len = d1 * d2 * d3; -+ m->parse[0] = d1; -+ m->parse[1] = d2; -+ m->parse[2] = d3; -+} -+ -+static void init_desc_metas(struct md_info_t *m) -+{ -+ init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :", -+ 1, MAX_SWITCHS, SWITCH_ITEMS); -+ -+ init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1); -+ -+ init_desc_meta(&m->kern_dump.excep_rec_meta, -+ "excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS); -+ -+ init_desc_meta(&m->kern_dump.excep_idx_meta, -+ "excep idx :", 1, 1, MAX_EXCEP_ID); -+ -+ init_desc_meta(&m->kern_dump.snap.runnable_at_meta, -+ "each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS); -+ -+ init_desc_meta(&m->kern_dump.snap.rq_nr_meta, -+ "rq runnable task nr:", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta, -+ "scxrq runnable task nr :", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.snap_misc_meta, -+ "misc snap params :", 1, 1, SNAP_ITEMS); -+} -+ -+static void init_md_meta(struct md_info_t *m) -+{ -+ m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64); -+ m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64); -+ m->meta.unit_size = sizeof(u64); -+ m->meta.switches = MAX_SWITCHS; -+ m->meta.exceps = MAX_EXCEPS; -+ m->meta.global_dsqs = MAX_GLOBAL_DSQS; -+ m->meta.parse_dimens = PARSE_DIMENS; -+ m->meta.nr_cpus = num_possible_cpus(); -+ m->meta.real_cpus = nr_cpu_ids; -+ m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64); -+ m->meta.nr_meta_desc = 8; -+ m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64); -+ -+ init_desc_metas(m); -+} -+ -+#define MINIDUMP_DFL_SIZE (4 * 1024) -+ -+struct notifier_block hmbird_panic_blk; -+static int hmbird_panic_handler(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ if (!md_info) -+ return NOTIFY_DONE; -+ -+ get_hmbird_snapshot(&md_info->kern_dump.snap); -+ -+ return NOTIFY_DONE; -+} -+ -+static void panic_blk_init(void) -+{ -+ int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE); -+ -+ md_info = kzalloc(dump_size, GFP_KERNEL); -+ if (!md_info) -+ return; -+ init_md_meta(md_info); -+ -+ hmbird_panic_blk.notifier_call = hmbird_panic_handler; -+ /* make sure to execute before minidump. */ -+ hmbird_panic_blk.priority = INT_MAX; -+ atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk); -+ -+ hmbird_debug("register minidump.\n"); -+} -+ -+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size) -+{ -+ *vaddr = (unsigned long)md_info; -+ *size = sizeof(struct md_info_t); -+} -+// MTK minidump end -+ -+static inline void init_sched_prop_to_preempt_prio(void) -+{ -+ for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) { -+ switch (i) { -+ case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 5; -+ break; -+ -+ case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 4; -+ break; -+ -+ case HMBIRD_TASK_PROP_PIPELINE: -+ case HMBIRD_TASK_PROP_ISOLATE: -+ sched_prop_to_preempt_prio[i] = 3; -+ break; -+ -+ case HMBIRD_TASK_PROP_COMMON: -+ sched_prop_to_preempt_prio[i] = 1; -+ break; -+ -+ case HMBIRD_TASK_PROP_DEBUG_OR_LOG: -+ sched_prop_to_preempt_prio[i] = 0; -+ break; -+ -+ default: -+ sched_prop_to_preempt_prio[i] = 2; -+ break; -+ } -+ } -+} -+ -+void __init init_sched_hmbird_class(void) -+{ -+ int cpu; -+ u32 v; -+ struct hmbird_entity *init_hmbird; -+ -+ /* -+ * The following is to prevent the compiler from optimizing out the enum -+ * definitions so that BPF scheduler implementations can use them -+ * through the generated vmlinux.h. -+ */ -+ WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP | -+ HMBIRD_KICK_PREEMPT); -+ -+ init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL); -+ init_dsq_at_boot(); -+ init_isolate_cpus(); -+ hb_timer_init(); -+ init_sched_prop_to_preempt_prio(); -+#ifdef CONFIG_SMP -+ WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); -+#endif -+ -+ /* -+ * we can't static init init_task's hmbird struct, init here. -+ * init_task->hmbird would not use during boot. -+ */ -+ init_hmbird = kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL); -+ init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird; -+ if (init_hmbird) { -+ INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo); -+ INIT_LIST_HEAD(&init_hmbird->watchdog_node); -+ init_hmbird->sticky_cpu = -1; -+ init_hmbird->holding_cpu = -1; -+ atomic64_set(&init_hmbird->ops_state, 0); -+ init_hmbird->runnable_at = jiffies; -+ init_hmbird->slice = HMBIRD_SLICE_DFL; -+ init_hmbird->task = &init_task; -+ hmbird_set_sched_prop(&init_task, 0); -+ } else { -+ hmbird_err(INIT_TASK_FAIL, ":alloc init_task.scx failed!!!\n"); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ /* -+ * exec during boot phase, no need to care about alloc failed. -+ * lifecycle same to rq, no need to free. -+ */ -+ rq->android_oem_data1[HMBIRD_RQ_IDX] = -+ (u64)kmalloc(sizeof(struct hmbird_rq), GFP_KERNEL); -+ if (get_hmbird_rq(rq)) { -+ get_hmbird_rq(rq)->rq = rq; -+ get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window; -+ } else { -+ hmbird_err(ALLOC_RQSCX_FAIL, ":alloc rq->scx failed!!!\n"); -+ } -+ -+ rq->android_oem_data1[HMBIRD_OPS_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL); -+ if (!get_hmbird_ops(rq)) -+ pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n"); -+ init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL); -+ INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list); -+ -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL)); -+ -+ hmbird_ops_init(get_hmbird_ops(rq)); -+ } -+ -+ INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn); -+ INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn); -+ hmbird_misc_init(); -+ -+ panic_blk_init(); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_misc.c b/kernel/sched/hmbird/hmbird_misc.c -new file mode 100755 -index 000000000000..52582c22052c ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_misc.c -@@ -0,0 +1,124 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * @Description: -+ * @Version: -+ * @Author: wangrui8 -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include "hmbird_sched.h" -+#include -+#include -+#include "slim.h" -+ -+noinline int tracing_mark_write(const char *buf) -+{ -+ trace_printk(buf); -+ return 0; -+} -+ -+struct yield_opt_params yield_opt_params = { -+ .enable = 0, -+ .frame_per_sec = 120, -+ .frame_time_ns = NSEC_PER_SEC / 120, -+ .yield_headroom = 10, -+}; -+ -+DEFINE_PER_CPU(struct sched_yield_state, ystate); -+ -+static inline void hmbird_yield_state_update(struct sched_yield_state *ys) -+{ -+ if (!raw_spin_is_locked(&ys->lock)) -+ return; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ -+ if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1 -+ || ys->yield_cnt_after_sleep > yield_headroom) { -+ ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP); -+ } else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) { -+ ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP); -+ } -+ ys->yield_cnt = 0; -+ ys->sleep_times = 0; -+ ys->yield_cnt_after_sleep = 0; -+} -+ -+void hmbird_skip_yield(long *skip) -+{ -+ if (!get_hmbird_ops_enabled() || !yield_opt_params.enable) -+ return; -+ unsigned long flags, sleep_now = 0; -+ struct sched_yield_state *ys; -+ int cpu = raw_smp_processor_id(), cont_yield, new_frame; -+ int frame_time_ns = yield_opt_params.frame_time_ns; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ u64 wc; -+ -+ if (!(*skip)) { -+ wc = sched_clock(); -+ ys = &per_cpu(ystate, cpu); -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ -+ cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP; -+ new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1); -+ -+ if (!cont_yield && new_frame) { -+ hmbird_yield_state_update(ys); -+ ys->last_update_time = wc; -+ ys->sleep_end = ys->last_yield_time + frame_time_ns -+ - yield_headroom * YIELD_DURATION; -+ } -+ -+ if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) { -+ *skip = true; -+ -+ sleep_now = ys->sleep_times ? -+ max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep; -+ if (wc + sleep_now > ys->sleep_end) { -+ u64 delta = ys->sleep_end - wc; -+ -+ if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION) -+ sleep_now = delta; -+ else -+ sleep_now = 0; -+ } -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ if (sleep_now) { -+ sleep_now = div64_u64(sleep_now, 1000); -+ usleep_range_state(sleep_now, sleep_now, TASK_IDLE); -+ } -+ ys->sleep_times++; -+ ys->last_yield_time = sched_clock(); -+ return; -+ } -+ if (ys->sleep_times) -+ ys->yield_cnt_after_sleep++; -+ else -+ (ys->yield_cnt)++; -+ ys->last_yield_time = wc; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+} -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops) -+{ -+ hmbird_ops->scx_enable = get_hmbird_ops_enabled; -+ hmbird_ops->check_non_task = get_non_hmbird_task; -+ hmbird_ops->hmbird_get_md_info = hmbird_get_md_info; -+} -+ -+void hmbird_misc_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_init(&ys->lock); -+ } -+ -+ set_hmbird_module_loaded(1); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_sched.h b/kernel/sched/hmbird/hmbird_sched.h -new file mode 100755 -index 000000000000..6b5ef43cb827 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef __HMBIRD_SCHED__ -+#define __HMBIRD_SCHED__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include "../../time/tick-sched.h" -+#include "../../sched/sched.h" -+#include -+#include -+#include -+ -+#define REGISTER_TRACE_VH(vender_hook, handler) \ -+{ \ -+ ret = register_trace_##vender_hook(handler, NULL); \ -+ if (ret) { \ -+ pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \ -+ } \ -+} -+#define REGISTER_TRACE(vendor_hook, handler, data, err) \ -+do { \ -+ ret = register_trace_##vendor_hook(handler, data); \ -+ if (ret) { \ -+ pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret); \ -+ goto err; \ -+ } \ -+} while (0) -+ -+#define UNREGISTER_TRACE(vendor_hook, handler, data) \ -+ unregister_trace_##vendor_hook(handler, data) \ -+ -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+ -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int sched_ravg_window_frame_per_sec; -+extern int slim_gov_debug; -+extern int cpu7_tl; -+extern int scx_gov_ctrl; -+extern spinlock_t new_sched_ravg_window_lock; -+extern int cluster_separate; -+ -+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER BIT(31) -+#define MAX_YIELD_SLEEP (2000000ULL) -+#define MIN_YIELD_SLEEP (200000ULL) -+#define YIELD_DURATION (5000ULL) -+#define DEFAULT_YIELD_SLEEP_TH (10) -+ -+struct sched_yield_state { -+ raw_spinlock_t lock; -+ u64 last_yield_time; -+ u64 last_update_time; -+ u64 sleep_end; -+ unsigned long yield_cnt; -+ unsigned long yield_cnt_after_sleep; -+ unsigned long sleep; -+ int sleep_times; -+}; -+ -+DECLARE_PER_CPU(struct sched_yield_state, ystate); -+ -+void hmbird_window_rollover_run_once(struct rq *rq); -+void hmbird_yield_state_update_per_frame(void); -+void hmbird_misc_init(void); -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops); -+#endif /*__HMBIRD_SCHED__*/ -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.c b/kernel/sched/hmbird/hmbird_sched_proc.c -new file mode 100755 -index 000000000000..234768fc767a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.c -@@ -0,0 +1,527 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "hmbird_sched_proc.h" -+#include -+ -+#include "hmbird_util_track.h" -+#include "slim.h" -+ -+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched" -+#define SLIM_FREQ_GOV_DIR "slim_freq_gov" -+#define LOAD_TRACK_DIR "slim_walt" -+#define HMBIRD_PROC_PERMISSION 0666 -+ -+int scx_enable; -+int partial_enable; -+int cpuctrl_high_ratio = 55; -+int cpuctrl_low_ratio = 40; -+int slim_stats; -+int hmbirdcore_debug; -+int slim_for_app; -+int misfit_ds = 90; -+unsigned int highres_tick_ctrl; -+unsigned int highres_tick_ctrl_dbg; -+int cpu7_tl = 70; -+int slim_walt_ctrl; -+int slim_walt_dump; -+int slim_walt_policy; -+int slim_gov_debug; -+int scx_gov_ctrl = 1; -+int sched_ravg_window_frame_per_sec = 125; -+int parctrl_high_ratio = 55; -+int parctrl_low_ratio = 40; -+int parctrl_high_ratio_l = 65; -+int parctrl_low_ratio_l = 50; -+int isoctrl_high_ratio = 75; -+int isoctrl_low_ratio = 60; -+int isolate_ctrl; -+int iso_free_rescue; -+int heartbeat; -+int heartbeat_enable = 1; -+int watchdog_enable; -+int save_gov; -+u64 cpu_cluster_masks; -+int hmbird_preempt_policy; -+int cluster_separate; -+ -+char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= 32) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtoint(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoint\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+static int set_proc_buf_val_u64(struct file *file, const char __user *buf, -+ size_t count, u64 *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= sizeof(kbuf)) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtou64(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoul\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/* common ops begin */ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ return count; -+} -+ -+static int hmbird_common_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%d\n", *(int *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_show, pde_data(inode)); -+} -+ -+static int hmbird_common_ul_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%lu\n", *(unsigned long *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_ul_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_ul_show, pde_data(inode)); -+} -+ -+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write); -+/* common ops end */ -+ -+/* scx_enable ops begin */ -+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC); -+ if (hmbird_ctrl(*pval)) -+ return -EFAULT; -+ -+ return count; -+} -+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write); -+/* scx_enable ops end */ -+ -+/* hmbird_stats ops begin */ -+#define MAX_STATS_BUF (4096) -+static int hmbird_stats_proc_show(struct seq_file *m, void *v) -+{ -+ char *buf; -+ -+ buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ stats_print(buf, MAX_STATS_BUF); -+ -+ seq_printf(m, "%s\n", buf); -+ -+ kfree(buf); -+ return 0; -+} -+ -+static int hmbird_stats_proc_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_stats_proc_show, inode); -+} -+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL); -+/* hmbird_stats ops end */ -+ -+/* sched_ravg_window_frame_per_sec ops begin */ -+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file, -+ const char __user *buf, size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ sched_ravg_window_change(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open, -+ sched_ravg_window_frame_per_sec_proc_write); -+/* sched_ravg_window_frame_per_sec ops end */ -+ -+static ssize_t save_gov_str(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ for_each_possible_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy || (cpu != policy->cpu)) -+ continue; -+ WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0); -+ hmbird_info_systrace(":save origin gov : %s\n", saved_gov[cpu]); -+ } -+ return count; -+} -+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str); -+ -+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ u64 *pval = (u64 *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val_u64(file, buf, count, pval)) -+ return -EFAULT; -+ -+ if (scx_enable == 0) -+ set_cpu_cluster(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write); -+ -+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ int tmp_val; -+ -+ if (set_proc_buf_val(file, buf, count, &tmp_val)) -+ return -EFAULT; -+ -+ slim_walt_enable(tmp_val); -+ *pval = tmp_val; -+ -+ return count; -+} -+ -+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write); -+ -+/* yield_opt ops begin */ -+static int yield_opt_show(struct seq_file *m, void *v) -+{ -+ struct yield_opt_params *data = m->private; -+ -+ seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n", -+ data->enable, data->frame_per_sec, data->yield_headroom); -+ return 0; -+} -+ -+static int yield_opt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, yield_opt_show, pde_data(inode)); -+} -+ -+static ssize_t yield_opt_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu; -+ unsigned long flags; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if (copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp -+ != 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) || -+ (yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp; -+ yield_opt_params.frame_per_sec = frame_per_sec_tmp; -+ yield_opt_params.yield_headroom = yield_headroom_tmp; -+ yield_opt_params.enable = enable_tmp; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ ys->last_yield_time = 0; -+ ys->last_update_time = 0; -+ ys->sleep_end = 0; -+ ys->yield_cnt = 0; -+ ys->yield_cnt_after_sleep = 0; -+ ys->sleep = 0; -+ ys->sleep_times = 0; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write); -+ -+ -+static int __init hmbird_proc_init(void) -+{ -+ struct proc_dir_entry *hmbird_dir; -+ struct proc_dir_entry *load_track_dir; -+ struct proc_dir_entry *freq_gov_dir; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ -+ /* mkdir /proc/hmbird_sched */ -+ hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL); -+ if (!hmbird_dir) { -+ pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &scx_enable_proc_ops, -+ &scx_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &partial_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_stats); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbirdcore_debug); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_for_app); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &misfit_ds); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl_dbg); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpu7_tl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &cpu_cluster_masks_proc_ops, -+ &cpu_cluster_masks); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &save_gov_proc_ops, -+ &save_gov); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &watchdog_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isolate_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &iso_free_rescue); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_stats_proc_ops); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &yield_opt_proc_ops, -+ &yield_opt_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbird_preempt_policy); -+ /* /proc/hmbird_sched--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_walt */ -+ load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir); -+ if (!load_track_dir) { -+ pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_walt--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &slim_walt_ctrl_proc_ops, -+ &slim_walt_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_dump); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_policy); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &sched_ravg_window_frame_per_sec_proc_ops, -+ &sched_ravg_window_frame_per_sec); -+ /* /proc/hmbird_sched/slim_walt--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_freq_gov */ -+ freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir); -+ if (!freq_gov_dir) { -+ pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_freq_gov--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &slim_gov_debug); -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &scx_gov_ctrl); -+ /* /proc/hmbird_sched/slim_freq_gov--end */ -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cluster_separate); -+ -+ return 0; -+} -+ -+device_initcall(hmbird_proc_init); -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.h b/kernel/sched/hmbird/hmbird_sched_proc.h -new file mode 100755 -index 000000000000..e22bc75f1dd7 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SCHED_PROC_H__ -+#define __HMBIRD_SCHED_PROC_H__ -+ -+#include -+#include -+ -+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \ -+ do { \ -+ if (!proc_create(name, mode, parent, proc_ops)) { \ -+ pr_err("Error creating proc entry %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \ -+ do { \ -+ if (!proc_create_data(name, mode, parent, proc_ops, data)) { \ -+ pr_err("Error creating proc entry with data %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_PROC_OPS(name, open_func, write_func) \ -+ static const struct proc_ops name##_proc_ops = { \ -+ .proc_open = open_func, \ -+ .proc_write = write_func, \ -+ .proc_read = seq_read, \ -+ .proc_lseek = seq_lseek, \ -+ .proc_release = single_release, \ -+ } -+ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos); -+static int hmbird_common_show(struct seq_file *m, void *v); -+static int hmbird_common_open(struct inode *inode, struct file *file); -+ -+#endif -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.c b/kernel/sched/hmbird/hmbird_shadow_tick.c -new file mode 100755 -index 000000000000..21de551c5132 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.c -@@ -0,0 +1,159 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include -+#include "../../time/tick-sched.h" -+#include -+ -+#include "hmbird_shadow_tick.h" -+ -+#define HIGHRES_WATCH_CPU 0 -+ -+#include -+static bool shadow_tick_enable(void) {return highres_tick_ctrl; } -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; } -+#endif -+static bool shadow_tick_timer_init_flag; -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define shadow_tick_printk(fmt, args...) \ -+do { \ -+ int cpu = smp_processor_id(); \ -+ if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU) \ -+ trace_printk("hmbird shadow tick :"fmt, args); \ -+} while (0) -+#else -+#define shadow_tick_printk(fmt, args...) -+#endif -+ -+DEFINE_PER_CPU(struct hrtimer, stt); -+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu))) -+#define STOP_IDLE_TRIGGER (1) -+#define PERIODIC_TICK_TRIGGER (2) -+#define TICK_INTVAL (1000000ULL) -+/* -+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms, -+ * so we can't restart hrtimer in scheduler tick. -+ */ -+static DEFINE_PER_CPU(u8, trigger_event); -+ -+/* -+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick. -+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler. -+ */ -+static DEFINE_PER_CPU(u8, tick_phase); -+ -+static inline void highres_timer_ctrl(bool enable, int cpu) -+{ -+ if (enable && hmbird_enabled()) { -+ if (!hrtimer_active(shadow_tick_timer(cpu))) -+ hrtimer_start(shadow_tick_timer(cpu), -+ ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED); -+ } else { -+ if (!enable) -+ hrtimer_cancel(shadow_tick_timer(cpu)); -+ } -+} -+ -+static inline void high_res_clear_phase(int cpu) -+{ -+ per_cpu(tick_phase, cpu) = 0; -+} -+ -+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer) -+{ -+ per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3; -+ if (per_cpu(tick_phase, cpu)) { -+ hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) { -+ per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ struct rq_flags rf; -+ -+ rq_lock(rq, &rf); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ rq_unlock(rq, &rf); -+ -+ return highres_next_phase(cpu, timer); -+} -+ -+void shadow_tick_timer_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance; -+ } -+} -+ -+void start_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable()) { -+ if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER) -+ highres_timer_ctrl(false, cpu); -+ per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static void stop_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ per_cpu(trigger_event, cpu) = 0; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(false, cpu); -+} -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ stop_shadow_tick_timer(); -+} -+ -+void scheduler_tick_handler(void *unused, struct rq *rq) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ start_shadow_tick_timer(); -+} -+ -+static int __init hmbird_shadow_tick_init(void) -+{ -+ int ret = 0; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ shadow_tick_timer_init(); -+ shadow_tick_timer_init_flag = true; -+ return ret; -+} -+ -+device_initcall(hmbird_shadow_tick_init); -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.h b/kernel/sched/hmbird/hmbird_shadow_tick.h -new file mode 100755 -index 000000000000..0c26fd984f0a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.h -@@ -0,0 +1,11 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SHADOW_TICK_H__ -+#define __HMBIRD_SHADOW_TICK_H__ -+ -+#include -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+void scheduler_tick_handler(void *unused, struct rq *rq); -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state); -+#endif -diff --git a/kernel/sched/hmbird/hmbird_trace.h b/kernel/sched/hmbird/hmbird_trace.h -new file mode 100755 -index 000000000000..8468b406f347 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_trace.h -@@ -0,0 +1,92 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hmbird -+ -+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HMBIRD_H -+ -+#include -+#include -+#include -+ -+#define MAX_FATAL_INFO (64) -+ -+TRACE_EVENT(hmbird_fatal_info, -+ -+ TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info), -+ -+ TP_ARGS(type, pe, lnr, bnr, info), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, type) -+ __field(int, pe) -+ __field(int, lnr) -+ __field(int, bnr) -+ __array(char, info, MAX_FATAL_INFO)), -+ -+ TP_fast_assign( -+ __entry->type = type; -+ __entry->pe = pe; -+ __entry->lnr = lnr; -+ __entry->bnr = bnr; -+ memcpy(__entry->info, info, MAX_FATAL_INFO);), -+ -+ TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s", -+ __entry->type, __entry->pe, __entry->lnr, __entry->bnr, -+ __entry->info) -+); -+ -+TRACE_EVENT(hmbird_update_history, -+ -+ TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq, -+ struct task_struct *p, u32 runtime, int samples, int event), -+ -+ TP_ARGS(hmbird, rq, p, runtime, samples, event), -+ -+ TP_STRUCT__entry( -+ __array(char, comm, TASK_COMM_LEN) -+ __field(pid_t, pid) -+ __field(unsigned int, runtime) -+ __field(int, samples) -+ __field(int, event) -+ __field(unsigned int, demand) -+ __array(u32, hist, RAVG_HIST_SIZE) -+ __field(u16, task_util) -+ __field(int, cpu)), -+ -+ TP_fast_assign( -+ memcpy(__entry->comm, p->comm, TASK_COMM_LEN); -+ __entry->pid = p->pid; -+ __entry->runtime = runtime; -+ __entry->samples = samples; -+ __entry->event = event; -+ __entry->demand = hmbird->sts.demand; -+ memcpy(__entry->hist, hmbird->sts.sum_history, -+ RAVG_HIST_SIZE * sizeof(u32)); -+ __entry->task_util = hmbird->sts.demand_scaled; -+ __entry->cpu = rq->cpu;), -+ -+ TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d", -+ __entry->comm, __entry->pid, -+ __entry->runtime, __entry->samples, -+ __entry->event, -+ __entry->demand, -+ __entry->hist[0], __entry->hist[1], -+ __entry->hist[2], __entry->hist[3], -+ __entry->hist[4], -+ __entry->task_util, -+ __entry->cpu) -+); -+ -+#endif /*_TRACE_HMBIRD_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird -+ -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE hmbird_trace -+/* This part must be outside protection */ -+#include -diff --git a/kernel/sched/hmbird/hmbird_util_track.c b/kernel/sched/hmbird/hmbird_util_track.c -new file mode 100755 -index 000000000000..d520a8403401 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.c -@@ -0,0 +1,626 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+ -+#define CREATE_TRACE_POINTS -+#include "hmbird_trace.h" -+#undef CREATE_TRACE_POINTS -+ -+#define HMBIRD_DEBUG_PANIC (1 << 3) -+static bool init_irq_work_inited; -+ -+extern noinline int tracing_mark_write(const char *buf); -+ -+int hmbird_sched_ravg_window = 8000000; -+int new_hmbird_sched_ravg_window = 8000000; -+DEFINE_SPINLOCK(new_sched_ravg_window_lock); -+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+ -+static struct irq_work hmbird_slim_walt_irq_work; -+ -+/*Sysctl related interface*/ -+#define WINDOW_STATS_RECENT 0 -+#define WINDOW_STATS_MAX 1 -+#define WINDOW_STATS_MAX_RECENT_AVG 2 -+#define WINDOW_STATS_AVG 3 -+#define WINDOW_STATS_INVALID_POLICY 4 -+ -+ -+#define SCX_SCHED_CAPACITY_SHIFT 10 -+#define SCHED_ACCOUNT_WAIT_TIME 0 -+ -+atomic64_t hmbird_irq_work_lastq_ws; -+static u64 tick_sched_clock; -+ -+static inline u64 scale_exec_time(u64 delta, struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT; -+} -+ -+static inline u64 scale_time_to_util(u64 d) -+{ -+ do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT); -+ return d; -+} -+ -+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ delta = scale_exec_time(delta, rq); -+ sts->sum += delta; -+ if (unlikely(sts->sum > hmbird_sched_ravg_window)) -+ sts->sum = hmbird_sched_ravg_window; -+ -+ return delta; -+} -+ -+ -+static int -+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) -+{ -+ /* -+ * No need to bother updating task demand for the idle task. -+ */ -+ if (is_idle_task(p)) -+ return 0; -+ -+ /* -+ * When a task is waking up it is completing a segment of non-busy -+ * time. Likewise, if wait time is not treated as busy time, then -+ * when a task begins to run or is migrated, it is not running and -+ * is completing a segment of non-busy time. -+ */ -+ if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && -+ (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) -+ return 0; -+ -+ /* -+ * The idle exit time is not accounted for the first task _picked_ up to -+ * run on the idle CPU. -+ */ -+ if (event == PICK_NEXT_TASK && rq->curr == rq->idle) -+ return 0; -+ -+ /* -+ * TASK_UPDATE can be called on sleeping task, when its moved between -+ * related groups -+ */ -+ if (event == TASK_UPDATE) { -+ if (rq->curr == p) -+ return 1; -+ -+ return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; -+ } -+ -+ return 1; -+} -+ -+static void rollover_cpu_window(struct rq *rq, bool full_window) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 curr_sum = srq->curr_runnable_sum; -+ -+ if (unlikely(full_window)) -+ curr_sum = 0; -+ -+ srq->prev_runnable_sum = curr_sum; -+ srq->curr_runnable_sum = 0; -+} -+ -+static u64 -+update_window_start(struct rq *rq, u64 wallclock, int event) -+{ -+ s64 delta; -+ int nr_windows; -+ bool full_window; -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start = srq->window_start; -+ -+ if (wallclock < srq->latest_clock) -+ wallclock = srq->latest_clock; -+ delta = wallclock - srq->window_start; -+ if (delta < 0) { -+ delta = 0; -+ wallclock = srq->window_start; -+ } -+ srq->latest_clock = wallclock; -+ if (delta < hmbird_sched_ravg_window) -+ return old_window_start; -+ -+ nr_windows = div64_u64(delta, hmbird_sched_ravg_window); -+ srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window; -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ full_window = nr_windows > 1; -+ rollover_cpu_window(rq, full_window); -+ -+ return old_window_start; -+} -+ -+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -+ -+static inline unsigned int get_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static inline unsigned int cpu_cur_freq(int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cur; -+} -+ -+static void -+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, -+ u64 wallclock) -+{ -+ int cpu = cpu_of(rq); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * -+ arch_scale_cpu_capacity(cpu), get_max_freq(cpu)); -+} -+ -+/* -+ * Called when new window is starting for a task, to record cpu usage over -+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 -+ * when, say, a real-time task runs without preemption for several windows at a -+ * stretch. -+ */ -+static void update_history(struct rq *rq, struct task_struct *p, -+ u32 runtime, int samples, int event) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u32 *hist = &sts->sum_history[0]; -+ int i; -+ u32 max = 0, avg, demand; -+ u64 sum = 0; -+ u16 demand_scaled; -+ -+ /* Ignore windows where task had no activity */ -+ if (!runtime || is_idle_task(p) || !samples) -+ goto done; -+ -+ /* Push new 'runtime' value onto stack */ -+ for (; samples > 0; samples--) { -+ hist[sts->cidx] = runtime; -+ sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE; -+ } -+ -+ for (i = 0; i < RAVG_HIST_SIZE; i++) { -+ sum += hist[i]; -+ if (hist[i] > max) -+ max = hist[i]; -+ } -+ -+ sts->sum = 0; -+ avg = div64_u64(sum, RAVG_HIST_SIZE); -+ -+ switch (slim_walt_policy) { -+ case WINDOW_STATS_RECENT: -+ demand = runtime; -+ break; -+ case WINDOW_STATS_MAX: -+ demand = max; -+ break; -+ case WINDOW_STATS_AVG: -+ demand = avg; -+ break; -+ default: -+ demand = max(avg, runtime); -+ } -+ -+ demand_scaled = scale_time_to_util(demand); -+ -+ sts->demand = demand; -+ sts->demand_scaled = demand_scaled; -+ -+done: -+ return; -+} -+ -+ -+static u64 update_task_demand(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ -+ u64 delta, window_start = srq->window_start; -+ int new_window, nr_full_windows; -+ u32 window_size = hmbird_sched_ravg_window; -+ u64 runtime; -+ -+ new_window = mark_start < window_start; -+ if (!account_busy_for_task_demand(rq, p, event)) { -+ if (new_window) -+ /* -+ * If the time accounted isn't being accounted as -+ * busy time, and a new window started, only the -+ * previous window need be closed out with the -+ * pre-existing demand. Multiple windows may have -+ * elapsed, but since empty windows are dropped, -+ * it is not necessary to account those. -+ */ -+ update_history(rq, p, sts->sum, 1, event); -+ return 0; -+ } -+ -+ if (!new_window) { -+ /* -+ * The simple case - busy time contained within the existing -+ * window. -+ */ -+ return add_to_task_demand(rq, p, wallclock - mark_start); -+ } -+ -+ /* -+ * Busy time spans at least two windows. Temporarily rewind -+ * window_start to first window boundary after mark_start. -+ */ -+ delta = window_start - mark_start; -+ nr_full_windows = div64_u64(delta, window_size); -+ window_start -= (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (window_start - mark_start) first */ -+ runtime = add_to_task_demand(rq, p, window_start - mark_start); -+ -+ /* Push new sample(s) into task's demand history */ -+ update_history(rq, p, sts->sum, 1, event); -+ if (nr_full_windows) { -+ u64 scaled_window = scale_exec_time(window_size, rq); -+ -+ update_history(rq, p, scaled_window, nr_full_windows, event); -+ runtime += nr_full_windows * scaled_window; -+ } -+ -+ /* -+ * Roll window_start back to current to process any remainder -+ * in current window. -+ */ -+ window_start += (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (wallclock - window_start) next */ -+ mark_start = window_start; -+ runtime += add_to_task_demand(rq, p, wallclock - mark_start); -+ -+ return runtime; -+} -+ -+u16 slim_walt_cpu_util(int cpu) -+{ -+ u64 prev_runnable_sum; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ prev_runnable_sum = srq->prev_runnable_sum; -+ do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT); -+ -+ return (u16)prev_runnable_sum; -+} -+ -+static DEFINE_PER_CPU(u16, prev_cpu_util); -+static inline void cpu_util_update_systrace_c(int cpu) -+{ -+ char buf[256]; -+ u16 cpu_util = slim_walt_cpu_util(cpu); -+ -+ if (cpu_util != per_cpu(prev_cpu_util, cpu)) { -+ snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n", -+ cpu, cpu_util); -+ tracing_mark_write(buf); -+ per_cpu(prev_cpu_util, cpu) = cpu_util; -+ } -+} -+ -+ -+static inline int account_busy_for_cpu_time(struct rq *rq, -+ struct task_struct *p, -+ int event) -+{ -+ return !is_idle_task(p) && (event == PUT_PREV_TASK -+ || event == TASK_UPDATE); -+} -+ -+ -+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ int new_window, full_window = 0; -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 window_start = srq->window_start; -+ u32 window_size = srq->prev_window_size; -+ u64 delta; -+ u64 *curr_runnable_sum = &srq->curr_runnable_sum; -+ u64 *prev_runnable_sum = &srq->prev_runnable_sum; -+ -+ new_window = mark_start < window_start; -+ if (new_window) -+ full_window = (window_start - mark_start) >= window_size; -+ -+ -+ if (!account_busy_for_cpu_time(rq, p, event)) -+ goto done; -+ -+ -+ if (!new_window) { -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. No rollover -+ * since we didn't start a new window. An example of this is -+ * when a task starts execution and then sleeps within the -+ * same window. -+ */ -+ delta = wallclock - mark_start; -+ -+ delta = scale_exec_time(delta, rq); -+ *curr_runnable_sum += delta; -+ -+ goto done; -+ } -+ -+ /* -+ * situations below this need window rollover, -+ * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done -+ * in update_window_start() -+ * -+ * For task counters curr/prev_window[_cpu] are rolled over in the early part of -+ * this function. If full_window(s) have expired and time since last update needs -+ * to be accounted as busy time, set the prev to a complete window size time, else -+ * add the prev window portion. -+ * -+ * For task curr counters a new window has begun, always assign -+ */ -+ -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. A new window -+ * must have been started in udpate_window_start() -+ * If any of these three above conditions are true -+ * then this busy time can't be accounted as irqtime. -+ * -+ * Busy time for the idle task need not be accounted. -+ * -+ * An example of this would be a task that starts execution -+ * and then sleeps once a new window has begun. -+ */ -+ -+ /* -+ * A full window hasn't elapsed, account partial -+ * contribution to previous completed window. -+ */ -+ -+ -+ delta = full_window ? scale_exec_time(window_size, rq) : -+ scale_exec_time(window_start - mark_start, rq); -+ -+ *prev_runnable_sum += delta; -+ -+ /* Account piece of busy time in the current window. */ -+ delta = scale_exec_time(wallclock - window_start, rq); -+ *curr_runnable_sum += delta; -+ -+done: -+ if (slim_walt_dump && new_window) -+ cpu_util_update_systrace_c(rq->cpu); -+} -+ -+ -+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq) -+{ -+ u64 result; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 new_window_start = srq->window_start; -+ -+ if (old_window_start == new_window_start) -+ return; -+ -+ result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws, -+ old_window_start, new_window_start); -+ if (result != old_window_start) -+ return; -+ -+ if (likely(cpu_online(raw_smp_processor_id()))) -+ irq_work_queue(&hmbird_slim_walt_irq_work); -+ else -+ irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask)); -+} -+ -+/* -+ * In the core scheduler, most of the load update points update the rq_clock after -+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of -+ * obtaining the time, but to prevent the subsequent migration of the load update -+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock -+ * update here. -+ */ -+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p, -+ struct rq *rq, int event) -+{ -+ if (!(rq->clock_update_flags & RQCF_UPDATED)) -+ update_rq_clock(rq); -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock)); -+} -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start; -+ -+ if (!slim_walt_ctrl) -+ return; -+ -+ if (!srq->window_start || sts->mark_start == wallclock) -+ return; -+ -+ old_window_start = update_window_start(rq, wallclock, event); -+ -+ if (!sts->window_start) -+ sts->window_start = srq->window_start; -+ -+ if (!sts->mark_start) -+ goto done; -+ -+ update_task_rq_cpu_cycles(p, rq, event, wallclock); -+ update_task_demand(p, rq, event, wallclock); -+ update_cpu_busy_time(p, rq, event, wallclock); -+ -+ sts->window_start = srq->window_start; -+ -+done: -+ sts->mark_start = wallclock; -+ slim_walt_window_rollover_run_once(old_window_start, rq); -+} -+ -+static void slim_walt_irq_work(struct irq_work *irq_work) -+{ -+ cpumask_t lock_cpus; -+ struct hmbird_sched_rq_stats *srq; -+ struct rq *rq; -+ int cpu; -+ int level = 0; -+ u64 wc; -+ unsigned long flags; -+ -+ cpumask_copy(&lock_cpus, cpu_possible_mask); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ if (level == 0) -+ raw_spin_lock(&cpu_rq(cpu)->__lock); -+ else -+ raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); -+ level++; -+ } -+ -+ wc = sched_clock(); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ rq = cpu_rq(cpu); -+ hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc); -+ } -+ -+ cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER); -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) { -+ srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id()); -+ if (wc < srq->window_start + new_hmbird_sched_ravg_window) -+ hmbird_sched_ravg_window = new_hmbird_sched_ravg_window; -+ } -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ raw_spin_unlock(&cpu_rq(cpu)->__lock); -+ } -+} -+static void hmbird_sched_init_rq(struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ srq->task_exec_scale = 1024; -+ srq->window_start = 0; -+} -+ -+void hmbird_sched_init_task(struct task_struct *p) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ memset(sts, 0, sizeof(struct hmbird_sched_task_stats)); -+} -+ -+static void hmbird_sched_stats_init(void) -+{ -+ unsigned long flags; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ raw_spin_lock_irqsave(&rq->__lock, flags); -+ hmbird_sched_init_rq(rq); -+ raw_spin_unlock_irqrestore(&rq->__lock, flags); -+ } -+ slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG; -+ -+ if (false == init_irq_work_inited) { -+ init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work); -+ init_irq_work_inited = true; -+ } -+} -+ -+void hmbird_scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (unlikely(!tick_sched_clock)) { -+ /* -+ * Let the window begin 20us prior to the tick, -+ * that way we are guaranteed a rollover when the tick occurs. -+ * Use rq->clock directly instead of rq_clock() since -+ * we do not have the rq lock and -+ * rq->clock was updated in the tick callpath. -+ */ -+ if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000)) -+ return; -+ for_each_possible_cpu(cpu) { -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ srq->window_start = tick_sched_clock; -+ } -+ atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock); -+ } -+} -+ -+void slim_walt_enable(int enable) -+{ -+ if (1 == !!enable) { -+ hmbird_sched_stats_init(); -+ WRITE_ONCE(tick_sched_clock, 0); -+ } else -+ slim_walt_ctrl = 0; -+} -+ -+void slim_get_cpu_util(int cpu, u64 *util) -+{ -+ if (cpu < 0) -+ return; -+ -+ *util = slim_walt_cpu_util(cpu); -+} -+ -+void slim_get_task_util(struct task_struct *p, u64 *util) -+{ -+ if (p == NULL) -+ return; -+ -+ *util = get_hmbird_ts(p)->sts.demand_scaled; -+} -+ -+void sched_ravg_window_change(int frame_per_sec) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec; -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+} -diff --git a/kernel/sched/hmbird/hmbird_util_track.h b/kernel/sched/hmbird/hmbird_util_track.h -new file mode 100755 -index 000000000000..17b85df7f83b ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.h -@@ -0,0 +1,33 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef __HMBIRD_UTIL_TRACK_H__ -+#define __HMBIRD_UTIL_TRACK_H__ -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock); -+void hmbird_sched_init_task(struct task_struct *p); -+void slim_walt_enable(int enable); -+void slim_get_cpu_util(int cpu, u64 *util); -+void slim_get_task_util(struct task_struct *p, u64 *util); -+ -+extern atomic64_t hmbird_irq_work_lastq_ws; -+ -+enum task_event { -+ PUT_PREV_TASK = 0, -+ PICK_NEXT_TASK = 1, -+ TASK_WAKE = 2, -+ TASK_MIGRATE = 3, -+ TASK_UPDATE = 4, -+ IRQ_UPDATE = 5, -+}; -+ -+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+#endif /* __HMBIRD_UTIL_TRACK_H__ */ -diff --git a/kernel/sched/hmbird/slim.h b/kernel/sched/hmbird/slim.h -new file mode 100755 -index 000000000000..eb386260e950 ---- /dev/null -+++ b/kernel/sched/hmbird/slim.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+ -+#ifndef __SLIM_H -+#define __SLIM_H -+ -+extern atomic_t __hmbird_ops_enabled; -+extern atomic_t non_hmbird_task; -+extern int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+extern int heartbeat; -+extern int heartbeat_enable; -+extern int watchdog_enable; -+extern int isolate_ctrl; -+extern int parctrl_high_ratio; -+extern int parctrl_low_ratio; -+extern int isoctrl_high_ratio; -+extern int isoctrl_low_ratio; -+extern int iso_free_rescue; -+extern int yield_opt; -+ -+extern enum hmbird_switch_type sw_type; -+extern noinline int tracing_mark_write(const char *buf); -+int task_top_id(struct task_struct *p); -+void stats_print(char *buf, int len); -+void hmbird_skip_yield(long *skip); -+extern spinlock_t hmbird_tasks_lock; -+ -+struct yield_opt_params { -+ int enable; -+ int frame_per_sec; -+ u64 frame_time_ns; -+ int yield_headroom; -+}; -+ -+extern struct yield_opt_params yield_opt_params; -+ -+#define MAX_GOV_LEN (16) -+extern char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+#endif -diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c -index 5007b25c5bc6..487255ac6832 100755 ---- a/kernel/sched/idle.c -+++ b/kernel/sched/idle.c -@@ -408,11 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl - - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) - { -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, false); -+#endif - } - - static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) - { - update_idle_core(rq); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, true); -+#endif - schedstat_inc(rq->sched_goidle); - } - -diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h -index 2b838a3a200a..8e438bfcec5a 100755 ---- a/kernel/sched/sched.h -+++ b/kernel/sched/sched.h -@@ -190,8 +190,11 @@ static inline int idle_policy(int policy) - - static inline int fair_policy(int policy) - { -+#ifdef CONFIG_HMBIRD_SCHED -+ return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#else - return policy == SCHED_NORMAL || policy == SCHED_BATCH; -- -+#endif - } - - static inline int rt_policy(int policy) -@@ -239,7 +242,8 @@ static inline void update_avg(u64 *avg, u64 sample) - #define shr_bound(val, shift) \ - (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) - --/* -+ -+/* - * !! For sched_setattr_nocheck() (kernel) only !! - * - * This is actually gross. :( -@@ -506,6 +510,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) { } - #endif /* CONFIG_SMP */ - #else /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_HMBIRD_SCHED -+static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) -+{ -+ return 0; -+} -+#endif - #endif /* CONFIG_FAIR_GROUP_SCHED */ - - #else /* CONFIG_CGROUP_SCHED */ -@@ -3600,5 +3610,8 @@ static inline bool cpu_busy_with_softirqs(int cpu) - } - #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird.h" -+#endif - - #endif /* _KERNEL_SCHED_SCHED_H */ -diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c -index 998c2eefe173..2a3f819b9e88 100755 ---- a/kernel/time/tick-sched.c -+++ b/kernel/time/tick-sched.c -@@ -34,6 +34,10 @@ - - #include - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "../sched/hmbird/hmbird_shadow_tick.h" -+#endif -+ - /* - * Per-CPU nohz control structure - */ -@@ -1112,6 +1116,9 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) - return true; - } - -+#ifdef CONFIG_HMBIRD_SCHED -+extern void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+#endif - /** - * tick_nohz_idle_stop_tick - stop the idle tick from the idle task - * -@@ -1124,6 +1131,9 @@ void tick_nohz_idle_stop_tick(void) - ktime_t expires; - - trace_android_vh_tick_nohz_idle_stop_tick(NULL); -+#ifdef CONFIG_HMBIRD_SCHED -+ android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL); -+#endif - /* - * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the - * tick timer expiration time is known already. -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -index ce05928392bf..f47f79079297 100755 ---- a/kernel/sched/hmbird/hmbird.c -+++ b/kernel/sched/hmbird/hmbird.c -@@ -633,14 +633,14 @@ static void init_isolate_cpus(void) - WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -- cpumask_set_cpu(0, iso_masks.big); -- cpumask_set_cpu(1, iso_masks.big); -- cpumask_set_cpu(2, iso_masks.big); -- cpumask_set_cpu(3, iso_masks.big); -- cpumask_set_cpu(4, iso_masks.big); -- cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(0, iso_masks.little); -+ cpumask_set_cpu(1, iso_masks.little); -+ cpumask_set_cpu(2, iso_masks.little); -+ cpumask_set_cpu(3, iso_masks.little); -+ cpumask_set_cpu(4, iso_masks.little); -+ cpumask_set_cpu(5, iso_masks.little); - cpumask_set_cpu(6, iso_masks.partial); -- cpumask_set_cpu(7, iso_masks.ex_free); -+ cpumask_set_cpu(7, iso_masks.big); - } - - extern spinlock_t css_set_lock; -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index 716788a51a5e..a258adcea40c 100755 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -4917,7 +4917,6 @@ late_initcall(sched_core_sysctl_init); - */ - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { -- - #ifdef CONFIG_HMBIRD_SCHED - int ret; - #endif -@@ -4973,7 +4972,7 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_class = &hmbird_sched_class; - } else if (rt_prio(p->prio)) { - p->sched_class = &rt_sched_class; -- else -+ } else { - p->sched_class = &fair_sched_class; - } - #else -@@ -6291,6 +6290,7 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - if (p) { - hmbird_notify_pick_next_task(rq, p, class); - return p; -+ } - } - #else - for_each_class(class) { diff --git a/oneplus/hmbird/fengchi_OP-PAD-2-PRO_A16.patch b/oneplus/hmbird/fengchi_OP-PAD-2-PRO_A16.patch deleted file mode 100644 index e6008b42..00000000 --- a/oneplus/hmbird/fengchi_OP-PAD-2-PRO_A16.patch +++ /dev/null @@ -1,19909 +0,0 @@ -diff --git b/Documentation/scheduler/index.rst a/Documentation/scheduler/index.rst -index 0b650bb550e6..3170747226f6 100644 ---- b/Documentation/scheduler/index.rst -+++ a/Documentation/scheduler/index.rst -@@ -19,7 +19,6 @@ Scheduler - sched-nice-design - sched-rt-group - sched-stats -- sched-ext - sched-debug - - text_files -diff --git b/Documentation/scheduler/sched-ext.rst a/Documentation/scheduler/sched-ext.rst -deleted file mode 100644 -index 84c30b44f104..000000000000 ---- b/Documentation/scheduler/sched-ext.rst -+++ /dev/null -@@ -1,230 +0,0 @@ --========================== --Extensible Scheduler Class --========================== -- --sched_ext is a scheduler class whose behavior can be defined by a set of BPF --programs - the BPF scheduler. -- --* sched_ext exports a full scheduling interface so that any scheduling -- algorithm can be implemented on top. -- --* The BPF scheduler can group CPUs however it sees fit and schedule them -- together, as tasks aren't tied to specific CPUs at the time of wakeup. -- --* The BPF scheduler can be turned on and off dynamically anytime. -- --* The system integrity is maintained no matter what the BPF scheduler does. -- The default scheduling behavior is restored anytime an error is detected, -- a runnable task stalls, or on invoking the SysRq key sequence -- :kbd:`SysRq-S`. -- --Switching to and from sched_ext --=============================== -- --``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and --``tools/sched_ext`` contains the example schedulers. -- --sched_ext is used only when the BPF scheduler is loaded and running. -- --If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be --treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is --loaded. On load, such tasks will be switched to and scheduled by sched_ext. -- --The BPF scheduler can choose to schedule all normal and lower class tasks by --calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this --case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and --``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers, --this mode can be selected with the ``-a`` option. -- --Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or --detection of any internal error including stalled runnable tasks aborts the --BPF scheduler and reverts all tasks back to CFS. -- --.. code-block:: none -- -- # make -j16 -C tools/sched_ext -- # tools/sched_ext/scx_example_simple -- local=0 global=3 -- local=5 global=24 -- local=9 global=44 -- local=13 global=56 -- local=17 global=72 -- ^CEXIT: BPF scheduler unregistered -- --If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler --and whether a given task is on sched_ext can be determined as follows: -- --.. code-block:: none -- -- # cat /sys/kernel/debug/sched/ext -- ops : simple -- enabled : 1 -- switching_all : 1 -- switched_all : 1 -- enable_state : enabled -- -- # grep ext /proc/self/sched -- ext.enabled : 1 -- --The Basics --========== -- --Userspace can implement an arbitrary BPF scheduler by loading a set of BPF --programs that implement ``struct sched_ext_ops``. The only mandatory field --is ``ops.name`` which must be a valid BPF object name. All operations are --optional. The following modified excerpt is from --``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO --scheduler. -- --.. code-block:: c -- -- s32 BPF_STRUCT_OPS(simple_init) -- { -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; -- } -- -- void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) -- { -- if (enq_flags & SCX_ENQ_LOCAL) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags); -- } -- -- void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) -- { -- exit_type = ei->type; -- } -- -- SEC(".struct_ops") -- struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", -- }; -- --Dispatch Queues ----------------- -- --To match the impedance between the scheduler core and the BPF scheduler, --sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a --priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``), --and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage --an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and --``scx_bpf_destroy_dsq()``. -- --A CPU always executes a task from its local DSQ. A task is "dispatched" to a --DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's --local DSQ. -- --When a CPU is looking for the next task to run, if the local DSQ is not --empty, the first task is picked. Otherwise, the CPU tries to consume the --global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()`` --is invoked. -- --Scheduling Cycle ------------------ -- --The following briefly shows how a waking task is scheduled and executed. -- --1. When a task is waking up, ``ops.select_cpu()`` is the first operation -- invoked. This serves two purposes. First, CPU selection optimization -- hint. Second, waking up the selected CPU if idle. -- -- The CPU selected by ``ops.select_cpu()`` is an optimization hint and not -- binding. The actual decision is made at the last step of scheduling. -- However, there is a small performance gain if the CPU -- ``ops.select_cpu()`` returns matches the CPU the task eventually runs on. -- -- A side-effect of selecting a CPU is waking it up from idle. While a BPF -- scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper, -- using ``ops.select_cpu()`` judiciously can be simpler and more efficient. -- -- Note that the scheduler core will ignore an invalid CPU selection, for -- example, if it's outside the allowed cpumask of the task. -- --2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can -- make one of the following decisions: -- -- * Immediately dispatch the task to either the global or local DSQ by -- calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or -- ``SCX_DSQ_LOCAL``, respectively. -- -- * Immediately dispatch the task to a custom DSQ by calling -- ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63. -- -- * Queue the task on the BPF side. -- --3. When a CPU is ready to schedule, it first looks at its local DSQ. If -- empty, it then looks at the global DSQ. If there still isn't a task to -- run, ``ops.dispatch()`` is invoked which can use the following two -- functions to populate the local DSQ. -- -- * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can -- be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``, -- ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()`` -- currently can't be called with BPF locks held, this is being worked on -- and will be supported. ``scx_bpf_dispatch()`` schedules dispatching -- rather than performing them immediately. There can be up to -- ``ops.dispatch_max_batch`` pending tasks. -- -- * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ -- to the dispatching DSQ. This function cannot be called with any BPF -- locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks -- before trying to consume the specified DSQ. -- --4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ, -- the CPU runs the first one. If empty, the following steps are taken: -- -- * Try to consume the global DSQ. If successful, run the task. -- -- * If ``ops.dispatch()`` has dispatched any tasks, retry #3. -- -- * If the previous task is an SCX task and still runnable, keep executing -- it (see ``SCX_OPS_ENQ_LAST``). -- -- * Go idle. -- --Note that the BPF scheduler can always choose to dispatch tasks immediately --in ``ops.enqueue()`` as illustrated in the above simple example. If only the --built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as --a task is never queued on the BPF scheduler and both the local and global --DSQs are consumed automatically. -- --``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use --``scx_bpf_dispatch_vtime()`` for the priority queue. See the function --documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for --more information. -- --Where to Look --============= -- --* ``include/linux/sched/ext.h`` defines the core data structures, ops table -- and constants. -- --* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. -- The functions prefixed with ``scx_bpf_`` can be called from the BPF -- scheduler. -- --* ``tools/sched_ext/`` hosts example BPF scheduler implementations. -- -- * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example -- using a custom DSQ. -- -- * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting -- five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. -- --ABI Instability --=============== -- --The APIs provided by sched_ext to BPF schedulers programs have no stability --guarantees. This includes the ops table callbacks and constants defined in --``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in --``kernel/sched/ext.c``. -- --While we will attempt to provide a relatively stable API surface when --possible, they are subject to change without warning between kernel --versions. -diff --git b/MAINTAINERS a/MAINTAINERS -index 9fc6108503c3..2384b55682ee 100644 ---- b/MAINTAINERS -+++ a/MAINTAINERS -@@ -19132,8 +19132,6 @@ R: Ben Segall (CONFIG_CFS_BANDWIDTH) - R: Mel Gorman (CONFIG_NUMA_BALANCING) - R: Daniel Bristot de Oliveira (SCHED_DEADLINE) - R: Valentin Schneider (TOPOLOGY) --R: Tejun Heo (SCHED_EXT) --R: David Vernet (SCHED_EXT) - L: linux-kernel@vger.kernel.org - S: Maintained - T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core -@@ -19142,7 +19140,6 @@ F: include/linux/sched.h - F: include/linux/wait.h - F: include/uapi/linux/sched.h - F: kernel/sched/ --F: tools/sched_ext/ - - SCSI LIBSAS SUBSYSTEM - R: John Garry -diff --git b/arch/arm64/configs/defconfig a/arch/arm64/configs/defconfig -index f93980c09d7f..c2d530535980 100644 ---- b/arch/arm64/configs/defconfig -+++ a/arch/arm64/configs/defconfig -@@ -6,8 +6,6 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_BPF_SYSCALL=y - CONFIG_BPF_JIT=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_BSD_PROCESS_ACCT=y - CONFIG_BSD_PROCESS_ACCT_V3=y -@@ -1587,3 +1585,4 @@ CONFIG_CORESIGHT_STM=m - CONFIG_CORESIGHT_CPU_DEBUG=m - CONFIG_CORESIGHT_CTI=m - CONFIG_MEMTEST=y -+CONFIG_HMBIRD_SCHED=y -diff --git b/arch/arm64/configs/gki_defconfig a/arch/arm64/configs/gki_defconfig -index 4f756dca5e05..6c1bb94f875d 100644 ---- b/arch/arm64/configs/gki_defconfig -+++ a/arch/arm64/configs/gki_defconfig -@@ -10,8 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y - # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set - CONFIG_BPF_LSM=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_TASKSTATS=y - CONFIG_TASK_DELAY_ACCT=y -diff --git b/drivers/gpu/drm/msm/msm_drv.c a/drivers/gpu/drm/msm/msm_drv.c -index 4bd028fa7500..5825ce9d6d7b 100755 ---- b/drivers/gpu/drm/msm/msm_drv.c -+++ a/drivers/gpu/drm/msm/msm_drv.c -@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) - } - - sched_set_fifo(ev_thread->worker->task); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3); -+#endif - } - - ret = drm_vblank_init(ddev, priv->num_crtcs); -diff --git b/drivers/tty/sysrq.c a/drivers/tty/sysrq.c -index bd001445815e..dcf2e1ebf9c5 100644 ---- b/drivers/tty/sysrq.c -+++ a/drivers/tty/sysrq.c -@@ -524,7 +524,6 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { - NULL, /* P */ - NULL, /* Q */ - NULL, /* R */ -- /* S: May be registered by sched_ext for resetting */ - NULL, /* S */ - NULL, /* T */ - NULL, /* U */ -diff --git b/include/asm-generic/vmlinux.lds.h a/include/asm-generic/vmlinux.lds.h -index c45078a445c8..6c15659f874e 100755 ---- b/include/asm-generic/vmlinux.lds.h -+++ a/include/asm-generic/vmlinux.lds.h -@@ -131,7 +131,7 @@ - *(__dl_sched_class) \ - *(__rt_sched_class) \ - *(__fair_sched_class) \ -- *(__ext_sched_class) \ -+ *(__hmbird_sched_class) \ - *(__idle_sched_class) \ - __sched_class_lowest = .; - -diff --git b/include/linux/sched.h a/include/linux/sched.h -index ba49d7bd2b67..dd6b1d58af01 100755 ---- b/include/linux/sched.h -+++ a/include/linux/sched.h -@@ -73,7 +73,9 @@ struct task_delay_info; - struct task_group; - struct user_event_mm; - --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - - /* - * Task state bitmask. NOTE! These bits are also -@@ -298,11 +300,6 @@ enum { - - extern void scheduler_tick(void); - --#ifdef CONFIG_SLIM_SCHED --extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer); --extern void stop_shadow_tick_timer(void); --#endif -- - #define MAX_SCHEDULE_TIMEOUT LONG_MAX - - extern long schedule_timeout(long timeout); -@@ -1523,13 +1520,8 @@ struct task_struct { - */ - struct callback_head l1d_flush_kill; - #endif --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, unsigned long sched_prop); -- ANDROID_KABI_USE(2, struct sched_ext_entity *scx); --#else - ANDROID_KABI_RESERVE(1); - ANDROID_KABI_RESERVE(2); --#endif - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); - ANDROID_KABI_RESERVE(5); -@@ -1933,6 +1925,91 @@ static inline int task_nice(const struct task_struct *p) - return PRIO_TO_NICE((p)->static_prio); - } - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline bool task_is_top_task(struct task_struct *p) -+{ -+ return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ ->top_task_prop & TOP_TASK_BITS_MASK); -+} -+ -+static inline int get_top_task_prop(struct task_struct *p) -+{ -+ return get_hmbird_ts(p)->top_task_prop; -+} -+ -+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear) -+{ -+ if (set) -+ get_hmbird_ts(p)->top_task_prop |= set; -+ if (clear) -+ get_hmbird_ts(p)->top_task_prop &= ~clear; -+ return 0; -+} -+ -+static inline void reset_top_task_prop(struct task_struct *p) -+{ -+ get_hmbird_ts(p)->top_task_prop = 0; -+} -+ -+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ entity->sched_prop = sp; -+ } -+ return 0; -+} -+ -+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->sched_prop; -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq) -+{ -+ unsigned long new_dsq; -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq; -+ entity->sched_prop = new_dsq; -+ } -+} -+ -+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK); -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ entity->dsq_sync_ux = val; -+} -+ -+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->dsq_sync_ux; -+ else -+ return 0; -+} -+#endif - - extern int can_nice(const struct task_struct *p, const int nice); - extern int task_curr(const struct task_struct *p); -diff --git b/include/linux/sched/ext.h a/include/linux/sched/ext.h -deleted file mode 100755 -index b737a00c1373..000000000000 ---- b/include/linux/sched/ext.h -+++ /dev/null -@@ -1,647 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef _LINUX_SCHED_EXT_H --#define _LINUX_SCHED_EXT_H -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --#include -- --enum scx_consts { -- SCX_OPS_NAME_LEN = 128, -- SCX_EXIT_REASON_LEN = 128, -- SCX_EXIT_BT_LEN = 64, -- SCX_EXIT_MSG_LEN = 1024, -- -- SCX_SLICE_DFL = 20 * NSEC_PER_MSEC, -- SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */ --}; -- --/* -- * DSQ (dispatch queue) IDs are 64bit of the format: -- * -- * Bits: [63] [62 .. 0] -- * [ B] [ ID ] -- * -- * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -- * ID: 63 bit ID -- * -- * Built-in IDs: -- * -- * Bits: [63] [62] [61..32] [31 .. 0] -- * [ 1] [ L] [ R ] [ V ] -- * -- * 1: 1 for built-in DSQs. -- * L: 1 for LOCAL_ON DSQ IDs, 0 for others -- * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -- */ --enum scx_dsq_id_flags { -- SCX_DSQ_FLAG_BUILTIN = 1LLU << 63, -- SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -- -- SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0, -- SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, -- SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, -- SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, -- SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, --}; -- --enum scx_exit_type { -- SCX_EXIT_NONE, -- SCX_EXIT_DONE, -- -- SCX_EXIT_UNREG = 64, /* BPF unregistration */ -- SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -- SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ -- SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ --}; -- --/* -- * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is -- * being disabled. -- */ --struct scx_exit_info { -- /* %SCX_EXIT_* - broad category of the exit reason */ -- enum scx_exit_type type; -- /* textual representation of the above */ -- char reason[SCX_EXIT_REASON_LEN]; -- /* number of entries in the backtrace */ -- u32 bt_len; -- /* backtrace if exiting due to an error */ -- unsigned long bt[SCX_EXIT_BT_LEN]; -- /* extra message */ -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --/* sched_ext_ops.flags */ --enum scx_ops_flags { -- /* -- * Keep built-in idle tracking even if ops.update_idle() is implemented. -- */ -- SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, -- -- /* -- * By default, if there are no other task to run on the CPU, ext core -- * keeps running the current task even after its slice expires. If this -- * flag is specified, such tasks are passed to ops.enqueue() with -- * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. -- */ -- SCX_OPS_ENQ_LAST = 1LLU << 1, -- -- /* -- * An exiting task may schedule after PF_EXITING is set. In such cases, -- * bpf_task_from_pid() may not be able to find the task and if the BPF -- * scheduler depends on pid lookup for dispatching, the task will be -- * lost leading to various issues including RCU grace period stalls. -- * -- * To mask this problem, by default, unhashed tasks are automatically -- * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't -- * depend on pid lookups and wants to handle these tasks directly, the -- * following flag can be used. -- */ -- SCX_OPS_ENQ_EXITING = 1LLU << 2, -- -- /* -- * CPU cgroup knob enable flags -- */ -- SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16, /* cpu.weight */ -- -- SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | -- SCX_OPS_ENQ_LAST | -- SCX_OPS_ENQ_EXITING | -- SCX_OPS_CGROUP_KNOB_WEIGHT, --}; -- --/* argument container for ops.enable() and friends */ --struct scx_enable_args { --}; -- --/* argument container for ops->cgroup_init() */ --struct scx_cgroup_init_args { -- /* the weight of the cgroup [1..10000] */ -- u32 weight; --}; -- --enum scx_cpu_preempt_reason { -- /* next task is being scheduled by &sched_class_rt */ -- SCX_CPU_PREEMPT_RT, -- /* next task is being scheduled by &sched_class_dl */ -- SCX_CPU_PREEMPT_DL, -- /* next task is being scheduled by &sched_class_stop */ -- SCX_CPU_PREEMPT_STOP, -- /* unknown reason for SCX being preempted */ -- SCX_CPU_PREEMPT_UNKNOWN, --}; -- --/* -- * Argument container for ops->cpu_acquire(). Currently empty, but may be -- * expanded in the future. -- */ --struct scx_cpu_acquire_args {}; -- --/* argument container for ops->cpu_release() */ --struct scx_cpu_release_args { -- /* the reason the CPU was preempted */ -- enum scx_cpu_preempt_reason reason; -- -- /* the task that's going to be scheduled on the CPU */ -- struct task_struct *task; --}; -- --/** -- * struct sched_ext_ops - Operation table for BPF scheduler implementation -- * -- * Userland can implement an arbitrary scheduling policy by implementing and -- * loading operations in this table. -- */ --struct sched_ext_ops { -- /** -- * select_cpu - Pick the target CPU for a task which is being woken up -- * @p: task being woken up -- * @prev_cpu: the cpu @p was on before sleeping -- * @wake_flags: SCX_WAKE_* -- * -- * Decision made here isn't final. @p may be moved to any CPU while it -- * is getting dispatched for execution later. However, as @p is not on -- * the rq at this point, getting the eventual execution CPU right here -- * saves a small bit of overhead down the line. -- * -- * If an idle CPU is returned, the CPU is kicked and will try to -- * dispatch. While an explicit custom mechanism can be added, -- * select_cpu() serves as the default way to wake up idle CPUs. -- */ -- s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); -- -- /** -- * enqueue - Enqueue a task on the BPF scheduler -- * @p: task being enqueued -- * @enq_flags: %SCX_ENQ_* -- * -- * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() -- * or enqueue on the BPF scheduler. If not directly dispatched, the bpf -- * scheduler owns @p and if it fails to dispatch @p, the task will -- * stall. -- */ -- void (*enqueue)(struct task_struct *p, u64 enq_flags); -- -- /** -- * dequeue - Remove a task from the BPF scheduler -- * @p: task being dequeued -- * @deq_flags: %SCX_DEQ_* -- * -- * Remove @p from the BPF scheduler. This is usually called to isolate -- * the task while updating its scheduling properties (e.g. priority). -- * -- * The ext core keeps track of whether the BPF side owns a given task or -- * not and can gracefully ignore spurious dispatches from BPF side, -- * which makes it safe to not implement this method. However, depending -- * on the scheduling logic, this can lead to confusing behaviors - e.g. -- * scheduling position not being updated across a priority change. -- */ -- void (*dequeue)(struct task_struct *p, u64 deq_flags); -- -- /** -- * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs -- * @cpu: CPU to dispatch tasks for -- * @prev: previous task being switched out -- * -- * Called when a CPU's local dsq is empty. The operation should dispatch -- * one or more tasks from the BPF scheduler into the DSQs using -- * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using -- * scx_bpf_consume(). -- * -- * The maximum number of times scx_bpf_dispatch() can be called without -- * an intervening scx_bpf_consume() is specified by -- * ops.dispatch_max_batch. See the comments on top of the two functions -- * for more details. -- * -- * When not %NULL, @prev is an SCX task with its slice depleted. If -- * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in -- * @prev->scx.flags, it is not enqueued yet and will be enqueued after -- * ops.dispatch() returns. To keep executing @prev, return without -- * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. -- */ -- void (*dispatch)(s32 cpu, struct task_struct *prev); -- -- /** -- * runnable - A task is becoming runnable on its associated CPU -- * @p: task becoming runnable -- * @enq_flags: %SCX_ENQ_* -- * -- * This and the following three functions can be used to track a task's -- * execution state transitions. A task becomes ->runnable() on a CPU, -- * and then goes through one or more ->running() and ->stopping() pairs -- * as it runs on the CPU, and eventually becomes ->quiescent() when it's -- * done running on the CPU. -- * -- * @p is becoming runnable on the CPU because it's -- * -- * - waking up (%SCX_ENQ_WAKEUP) -- * - being moved from another CPU -- * - being restored after temporarily taken off the queue for an -- * attribute change. -- * -- * This and ->enqueue() are related but not coupled. This operation -- * notifies @p's state transition and may not be followed by ->enqueue() -- * e.g. when @p is being dispatched to a remote CPU. Likewise, a task -- * may be ->enqueue()'d without being preceded by this operation e.g. -- * after exhausting its slice. -- */ -- void (*runnable)(struct task_struct *p, u64 enq_flags); -- -- /** -- * running - A task is starting to run on its associated CPU -- * @p: task starting to run -- * -- * See ->runnable() for explanation on the task state notifiers. -- */ -- void (*running)(struct task_struct *p); -- -- /** -- * stopping - A task is stopping execution -- * @p: task stopping to run -- * @runnable: is task @p still runnable? -- * -- * See ->runnable() for explanation on the task state notifiers. If -- * !@runnable, ->quiescent() will be invoked after this operation -- * returns. -- */ -- void (*stopping)(struct task_struct *p, bool runnable); -- -- /** -- * quiescent - A task is becoming not runnable on its associated CPU -- * @p: task becoming not runnable -- * @deq_flags: %SCX_DEQ_* -- * -- * See ->runnable() for explanation on the task state notifiers. -- * -- * @p is becoming quiescent on the CPU because it's -- * -- * - sleeping (%SCX_DEQ_SLEEP) -- * - being moved to another CPU -- * - being temporarily taken off the queue for an attribute change -- * (%SCX_DEQ_SAVE) -- * -- * This and ->dequeue() are related but not coupled. This operation -- * notifies @p's state transition and may not be preceded by ->dequeue() -- * e.g. when @p is being dispatched to a remote CPU. -- */ -- void (*quiescent)(struct task_struct *p, u64 deq_flags); -- -- /** -- * yield - Yield CPU -- * @from: yielding task -- * @to: optional yield target task -- * -- * If @to is NULL, @from is yielding the CPU to other runnable tasks. -- * The BPF scheduler should ensure that other available tasks are -- * dispatched before the yielding task. Return value is ignored in this -- * case. -- * -- * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf -- * scheduler can implement the request, return %true; otherwise, %false. -- */ -- bool (*yield)(struct task_struct *from, struct task_struct *to); -- -- /** -- * core_sched_before - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Used by core-sched to determine the ordering between two tasks. See -- * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on -- * core-sched. -- * -- * Both @a and @b are runnable and may or may not currently be queued on -- * the BPF scheduler. Should return %true if @a should run before @b. -- * %false if there's no required ordering or @b should run before @a. -- * -- * If not specified, the default is ordering them according to when they -- * became runnable. -- */ -- bool (*core_sched_before)(struct task_struct *a,struct task_struct *b); -- -- /** -- * set_weight - Set task weight -- * @p: task to set weight for -- * @weight: new eight [1..10000] -- * -- * Update @p's weight to @weight. -- */ -- void (*set_weight)(struct task_struct *p, u32 weight); -- -- /** -- * set_cpumask - Set CPU affinity -- * @p: task to set CPU affinity for -- * @cpumask: cpumask of cpus that @p can run on -- * -- * Update @p's CPU affinity to @cpumask. -- */ -- void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask); -- -- /** -- * update_idle - Update the idle state of a CPU -- * @cpu: CPU to udpate the idle state for -- * @idle: whether entering or exiting the idle state -- * -- * This operation is called when @rq's CPU goes or leaves the idle -- * state. By default, implementing this operation disables the built-in -- * idle CPU tracking and the following helpers become unavailable: -- * -- * - scx_bpf_select_cpu_dfl() -- * - scx_bpf_test_and_clear_cpu_idle() -- * - scx_bpf_pick_idle_cpu() -- * - scx_bpf_any_idle_cpu() -- * -- * The user also must implement ops.select_cpu() as the default -- * implementation relies on scx_bpf_select_cpu_dfl(). -- * -- * If you keep the built-in idle tracking, specify the -- * %SCX_OPS_KEEP_BUILTIN_IDLE flag. -- */ -- void (*update_idle)(s32 cpu, bool idle); -- -- /** -- * cpu_acquire - A CPU is becoming available to the BPF scheduler -- * @cpu: The CPU being acquired by the BPF scheduler. -- * @args: Acquire arguments, see the struct definition. -- * -- * A CPU that was previously released from the BPF scheduler is now once -- * again under its control. -- */ -- void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); -- -- /** -- * cpu_release - A CPU is taken away from the BPF scheduler -- * @cpu: The CPU being released by the BPF scheduler. -- * @args: Release arguments, see the struct definition. -- * -- * The specified CPU is no longer under the control of the BPF -- * scheduler. This could be because it was preempted by a higher -- * priority sched_class, though there may be other reasons as well. The -- * caller should consult @args->reason to determine the cause. -- */ -- void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -- -- /** -- * cpu_online - A CPU became online -- * @cpu: CPU which just came up -- * -- * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs beforehand. -- */ -- void (*cpu_online)(s32 cpu); -- -- /** -- * cpu_offline - A CPU is going offline -- * @cpu: CPU which is going offline -- * -- * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs afterwards. -- */ -- void (*cpu_offline)(s32 cpu); -- -- /** -- * prep_enable - Prepare to enable BPF scheduling for a task -- * @p: task to prepare BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Either we're loading a BPF scheduler or a new task is being forked. -- * Prepare BPF scheduling for @p. This operation may block and can be -- * used for allocations. -- * -- * Return 0 for success, -errno for failure. An error return while -- * loading will abort loading of the BPF scheduler. During a fork, will -- * abort the specific fork. -- */ -- s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * enable - Enable BPF scheduling for a task -- * @p: task to enable BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Enable @p for BPF scheduling. @p is now in the cgroup specified for -- * the preceding prep_enable() and will start running soon. -- */ -- void (*enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * cancel_enable - Cancel prep_enable() -- * @p: task being canceled -- * @args: enable arguments, see the struct definition -- * -- * @p was prep_enable()'d but failed before reaching enable(). Undo the -- * preparation. -- */ -- void (*cancel_enable)(struct task_struct *p, -- struct scx_enable_args *args); -- -- /** -- * disable - Disable BPF scheduling for a task -- * @p: task to disable BPF scheduling for -- * -- * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. -- * Disable BPF scheduling for @p. -- */ -- void (*disable)(struct task_struct *p); -- -- /* -- * All online ops must come before ops.init(). -- */ -- -- /** -- * init - Initialize the BPF scheduler -- */ -- s32 (*init)(void); -- -- /** -- * exit - Clean up after the BPF scheduler -- * @info: Exit info -- */ -- void (*exit)(struct scx_exit_info *info); -- -- /** -- * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch -- */ -- u32 dispatch_max_batch; -- -- /** -- * flags - %SCX_OPS_* flags -- */ -- u64 flags; -- -- /** -- * timeout_ms - The maximum amount of time, in milliseconds, that a -- * runnable task should be able to wait before being scheduled. The -- * maximum timeout may not exceed the default timeout of 30 seconds. -- * -- * Defaults to the maximum allowed timeout value of 30 seconds. -- */ -- u32 timeout_ms; -- -- /** -- * name - BPF scheduler's name -- * -- * Must be a non-zero valid BPF object name including only isalnum(), -- * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the -- * BPF scheduler is enabled. -- */ -- char name[SCX_OPS_NAME_LEN]; --}; -- --/* -- * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -- * scheduler core and the BPF scheduler. See the documentation for more details. -- */ --struct scx_dispatch_q { -- raw_spinlock_t lock; -- struct list_head fifo; /* processed in dispatching order */ -- struct rb_root_cached priq; /* processed in p->scx.dsq_vtime order */ -- u32 nr; -- u64 id; -- struct llist_node free_node; -- struct rcu_head rcu; --}; -- --/* scx_entity.flags */ --enum scx_ent_flags { -- SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ -- SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -- SCX_TASK_ENQ_LOCAL = 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */ -- -- SCX_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -- SCX_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -- -- SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -- SCX_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -- -- SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ --}; -- --/* scx_entity.dsq_flags */ --enum scx_ent_dsq_flags { -- SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ --}; -- --/* -- * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from -- * everywhere and the following bits track which kfunc sets are currently -- * allowed for %current. This simple per-task tracking works because SCX ops -- * nest in a limited way. BPF will likely implement a way to allow and disallow -- * kfuncs depending on the calling context which will replace this manual -- * mechanism. See scx_kf_allow(). -- */ --enum scx_kf_mask { -- SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ -- /* all non-sleepables may be nested inside INIT and SLEEPABLE */ -- SCX_KF_INIT = 1 << 0, /* running ops.init() */ -- SCX_KF_SLEEPABLE = 1 << 1, /* other sleepable init operations */ -- /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ -- SCX_KF_CPU_RELEASE = 1 << 2, /* ops.cpu_release() */ -- /* ops.dequeue (in REST) may be nested inside DISPATCH */ -- SCX_KF_DISPATCH = 1 << 3, /* ops.dispatch() */ -- SCX_KF_ENQUEUE = 1 << 4, /* ops.enqueue() */ -- SCX_KF_REST = 1 << 5, /* other rq-locked operations */ -- -- __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | -- SCX_KF_ENQUEUE | SCX_KF_REST, -- __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_REST, --}; -- --#define RAVG_HIST_SIZE 5 --struct scx_sched_task_stats { -- u64 mark_start; -- u64 window_start; -- u32 sum; -- u32 sum_history[RAVG_HIST_SIZE]; -- int cidx; -- -- u32 demand; -- u16 demand_scaled; -- void *sdsq; --}; -- -- -- --/* -- * The following is embedded in task_struct and contains all fields necessary -- * for a task to be scheduled by SCX. -- */ --struct sched_ext_entity { -- struct scx_dispatch_q *dsq; -- struct { -- struct list_head fifo; /* dispatch order */ -- struct rb_node priq; /* p->scx.dsq_vtime order */ -- } dsq_node; -- struct list_head watchdog_node; -- u32 flags; /* protected by rq lock */ -- u32 dsq_flags; /* protected by dsq lock */ -- u32 weight; -- s32 sticky_cpu; -- s32 holding_cpu; -- u32 kf_mask; /* see scx_kf_mask above */ -- struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ -- atomic64_t ops_state; -- unsigned long runnable_at; --#ifdef CONFIG_SCHED_CORE -- u64 core_sched_at; /* see scx_prio_less() */ --#endif -- -- /* BPF scheduler modifiable fields */ -- -- /* -- * Runtime budget in nsecs. This is usually set through -- * scx_bpf_dispatch() but can also be modified directly by the BPF -- * scheduler. Automatically decreased by SCX as the task executes. On -- * depletion, a scheduling event is triggered. -- * -- * This value is cleared to zero if the task is preempted by -- * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the -- * task ran. Use p->se.sum_exec_runtime instead. -- */ -- u64 slice; -- -- /* -- * Used to order tasks when dispatching to the vtime-ordered priority -- * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime() -- * but can also be modified directly by the BPF scheduler. Modifying it -- * while a task is queued on a dsq may mangle the ordering and is not -- * recommended. -- */ -- u64 dsq_vtime; -- -- /* -- * If set, reject future sched_setscheduler(2) calls updating the policy -- * to %SCHED_EXT with -%EACCES. -- * -- * If set from ops.prep_enable() and the task's policy is already -- * %SCHED_EXT, which can happen while the BPF scheduler is being loaded -- * or by inhering the parent's policy during fork, the task's policy is -- * rejected and forcefully reverted to %SCHED_NORMAL. The number of such -- * events are reported through /sys/kernel/debug/sched_ext::nr_rejected. -- */ -- bool disallow; /* reject switching into SCX */ -- -- /* cold fields */ -- struct list_head tasks_node; -- struct task_struct *task; -- struct scx_sched_task_stats sts; --}; -- --void sched_ext_free(struct task_struct *p); -- --#else /* !CONFIG_SCHED_CLASS_EXT */ -- --static inline void sched_ext_free(struct task_struct *p) {} -- --#endif /* CONFIG_SCHED_CLASS_EXT */ --#endif /* _LINUX_SCHED_EXT_H */ -diff --git b/include/linux/sched/hmbird_proc_val.h a/include/linux/sched/hmbird_proc_val.h -new file mode 100755 -index 000000000000..e59708b16ea3 ---- /dev/null -+++ a/include/linux/sched/hmbird_proc_val.h -@@ -0,0 +1,22 @@ -+#ifndef __HMBORD_PROC_VAL_H__ -+#define __HMBORD_PROC_VAL_H__ -+ -+extern int scx_enable; -+extern int partial_enable; -+extern int cpuctrl_high_ratio; -+extern int cpuctrl_low_ratio; -+extern int slim_stats; -+extern int hmbirdcore_debug; -+extern int slim_for_app; -+extern int misfit_ds; -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+extern int cpu7_tl; -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int slim_gov_debug; -+extern int scx_gov_ctrl; -+extern int sched_ravg_window_frame_per_sec; -+ -+#endif -\ No newline at end of file -diff --git b/include/linux/sched/hmbird_version.h a/include/linux/sched/hmbird_version.h -new file mode 100755 -index 000000000000..b2843881c26f ---- /dev/null -+++ a/include/linux/sched/hmbird_version.h -@@ -0,0 +1,52 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#ifndef _OPLUS_HMBIRD_VERSION_H_ -+#define _OPLUS_HMBIRD_VERSION_H_ -+#include -+#include -+#include -+ -+enum hmbird_version { -+ HMBIRD_UNINIT, -+ HMBIRD_GKI_VERSION, -+ HMBIRD_OGKI_VERSION, -+ HMBIRD_UNKNOW_VERSION, -+}; -+ -+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT; -+ -+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type" -+ -+static inline enum hmbird_version get_hmbird_version_type(void) -+{ -+ struct device_node *np = NULL; -+ const char *hmbird_version_str = NULL; -+ if (HMBIRD_UNINIT != hmbird_version_type) -+ return hmbird_version_type; -+ np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH); -+ if (np) { -+ of_property_read_string(np, "type", &hmbird_version_str); -+ if (NULL != hmbird_version_str) { -+ if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) { -+ hmbird_version_type = HMBIRD_OGKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi"); -+ } else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) { -+ hmbird_version_type = HMBIRD_GKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi"); -+ } else { -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi"); -+ } -+ return hmbird_version_type; -+ } -+ } -+ -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION"); -+ return hmbird_version_type; -+} -+ -+#endif /*_OPLUS_HMBIRD_VERSION_H_ */ -diff --git b/include/linux/sched/task.h a/include/linux/sched/task.h -index 03d35e3eda3c..6a5f0c0d74bd 100644 ---- b/include/linux/sched/task.h -+++ a/include/linux/sched/task.h -@@ -61,8 +61,10 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); - extern void init_idle(struct task_struct *idle, int cpu); - - extern int sched_fork(unsigned long clone_flags, struct task_struct *p); --extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); --extern void sched_cancel_fork(struct task_struct *p); -+extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); -+#ifdef CONFIG_HMBIRD_SCHED -+extern void hmbird_cancel_fork(struct task_struct *p); -+#endif - extern void sched_post_fork(struct task_struct *p); - extern void sched_dead(struct task_struct *p); - -diff --git b/include/uapi/linux/sched.h a/include/uapi/linux/sched.h -index 359a14cc76a4..969716ab4320 100755 ---- b/include/uapi/linux/sched.h -+++ a/include/uapi/linux/sched.h -@@ -118,7 +118,7 @@ struct clone_args { - /* SCHED_ISO: reserved but not implemented yet */ - #define SCHED_IDLE 5 - #define SCHED_DEADLINE 6 --#define SCHED_EXT 7 -+#define SCHED_HMBIRD 7 - - /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ - #define SCHED_RESET_ON_FORK 0x40000000 -diff --git b/init/Kconfig a/init/Kconfig -index 337276cbc7f8..bf052ca532b4 100644 ---- b/init/Kconfig -+++ a/init/Kconfig -@@ -1030,10 +1030,6 @@ config RT_GROUP_SCHED - realtime bandwidth for them. - See Documentation/scheduler/sched-rt-group.rst for more information. - --config EXT_GROUP_SCHED -- bool -- depends on SCHED_CLASS_EXT && CGROUP_SCHED -- default y - endif #CGROUP_SCHED - - config SCHED_MM_CID -diff --git b/init/init_task.c a/init/init_task.c -index 69a046388995..31ceb0e469f7 100755 ---- b/init/init_task.c -+++ a/init/init_task.c -@@ -6,7 +6,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -102,9 +101,6 @@ struct task_struct init_task - #endif - #ifdef CONFIG_CGROUP_SCHED - .sched_task_group = &root_task_group, --#endif --#ifdef CONFIG_SLIM_SCHED -- .scx = NULL, - #endif - .ptraced = LIST_HEAD_INIT(init_task.ptraced), - .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), -diff --git b/kernel/Kconfig.preempt a/kernel/Kconfig.preempt -index cf22ef6107b8..ca8de6eb1e16 100755 ---- b/kernel/Kconfig.preempt -+++ a/kernel/Kconfig.preempt -@@ -133,12 +133,8 @@ config SCHED_CORE - which is the likely usage by Linux distributions, there should - be no measurable impact on performance. - --config SLIM_SCHED -- bool "slim sched" -- default n --config SCHED_CLASS_EXT -- bool "Extensible Scheduling Class" -- depends on BPF_SYSCALL && BPF_JIT -+config HMBIRD_SCHED -+ bool "hmbird Scheduling Class(base on ext scheduler)" - help - This option enables a new scheduler class sched_ext (SCX), which - allows scheduling policies to be implemented as BPF programs to -diff --git b/kernel/bpf/bpf_struct_ops_types.h a/kernel/bpf/bpf_struct_ops_types.h -index 3618769d853d..5678a9ddf817 100644 ---- b/kernel/bpf/bpf_struct_ops_types.h -+++ a/kernel/bpf/bpf_struct_ops_types.h -@@ -9,8 +9,4 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) - #include - BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) - #endif --#ifdef CONFIG_SCHED_CLASS_EXT --#include --BPF_STRUCT_OPS_TYPE(sched_ext_ops) --#endif - #endif -diff --git b/kernel/fork.c a/kernel/fork.c -index a43f97302332..7a91505b9378 100755 ---- b/kernel/fork.c -+++ a/kernel/fork.c -@@ -23,7 +23,9 @@ - #include - #include - #include --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - #include - #include - #include -@@ -999,8 +1001,9 @@ void __put_task_struct(struct task_struct *tsk) - WARN_ON(!tsk->exit_state); - WARN_ON(refcount_read(&tsk->usage)); - WARN_ON(tsk == current); -- -- sched_ext_free(tsk); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_free(tsk); -+#endif - io_uring_free(tsk); - cgroup_free(tsk); - task_numa_free(tsk, true); -@@ -2517,7 +2520,11 @@ __latent_entropy struct task_struct *copy_process( - - retval = perf_event_init_task(p, clone_flags); - if (retval) -- goto bad_fork_sched_cancel_fork; -+#ifdef CONFIG_HMBIRD_SCHED -+ goto bad_fork_hmbird_cancel_fork; -+#else -+ goto bad_fork_cleanup_policy; -+#endif - retval = audit_alloc(p); - if (retval) - goto bad_fork_cleanup_perf; -@@ -2649,9 +2656,7 @@ __latent_entropy struct task_struct *copy_process( - * cgroup specific, it unconditionally needs to place the task on a - * runqueue. - */ -- retval = sched_cgroup_fork(p, args); -- if (retval) -- goto bad_fork_cancel_cgroup; -+ sched_cgroup_fork(p, args); - - /* - * From this point on we must avoid any synchronous user-space -@@ -2697,13 +2702,13 @@ __latent_entropy struct task_struct *copy_process( - /* Don't start children in a dying pid namespace */ - if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { - retval = -ENOMEM; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* Let kill terminate clone/fork in the middle */ - if (fatal_signal_pending(current)) { - retval = -EINTR; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* No more failure paths after this point. */ -@@ -2780,11 +2785,10 @@ __latent_entropy struct task_struct *copy_process( - - return p; - --bad_fork_core_free: -+bad_fork_cancel_cgroup: - sched_core_free(p); - spin_unlock(¤t->sighand->siglock); - write_unlock_irq(&tasklist_lock); --bad_fork_cancel_cgroup: - cgroup_cancel_fork(p, args); - bad_fork_put_pidfd: - if (clone_flags & CLONE_PIDFD) { -@@ -2823,8 +2827,10 @@ __latent_entropy struct task_struct *copy_process( - audit_free(p); - bad_fork_cleanup_perf: - perf_event_free_task(p); --bad_fork_sched_cancel_fork: -- sched_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+bad_fork_hmbird_cancel_fork: -+ hmbird_cancel_fork(p); -+#endif - bad_fork_cleanup_policy: - lockdep_free_task(p); - #ifdef CONFIG_NUMA -diff --git b/kernel/sched/build_policy.c a/kernel/sched/build_policy.c -index 005025f55bea..c091539a0f60 100755 ---- b/kernel/sched/build_policy.c -+++ a/kernel/sched/build_policy.c -@@ -28,8 +28,10 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED - #include - #include -+#endif - - #include - -@@ -54,6 +56,10 @@ - #include "cputime.c" - #include "deadline.c" - --#ifdef CONFIG_SCHED_CLASS_EXT --# include "ext.c" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/hmbird_util_track.c" -+#include "hmbird/hmbird_sched_proc.c" -+#include "hmbird/hmbird_shadow_tick.c" -+#include "hmbird/hmbird.c" -+#include "hmbird/hmbird_misc.c" - #endif -diff --git b/kernel/sched/core.c a/kernel/sched/core.c -index 25d5703df992..b894417ce7df 100755 ---- b/kernel/sched/core.c -+++ a/kernel/sched/core.c -@@ -96,6 +96,12 @@ - #include "../../io_uring/io-wq.h" - #include "../smpboot.h" - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/slim.h" -+#include "hmbird/hmbird_shadow_tick.h" -+#include "hmbird.h" -+#endif -+ - #include - #include - #include -@@ -170,7 +176,6 @@ __read_mostly int scheduler_running; - - DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); - -- - /* kernel prio, less is more */ - static inline int __task_prio(const struct task_struct *p) - { -@@ -183,11 +188,10 @@ static inline int __task_prio(const struct task_struct *p) - if (p->sched_class == &idle_sched_class) - return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - --#ifdef CONFIG_SCHED_CLASS_EXT -- if (p->sched_class == &ext_sched_class) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (p->sched_class == &hmbird_sched_class) - return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ - #endif -- - return MAX_RT_PRIO + MAX_NICE; /* 119, squash fair */ - } - -@@ -217,9 +221,9 @@ static inline bool prio_less(const struct task_struct *a, - if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ - return cfs_prio_less(a, b, in_fi); - --#ifdef CONFIG_SCHED_CLASS_EXT -+#ifdef CONFIG_HMBIRD_SCHED - if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ -- return scx_prio_less(a, b, in_fi); -+ return hmbird_prio_less(a, b, in_fi); - #endif - - return false; -@@ -1279,17 +1283,26 @@ bool sched_can_stop_tick(struct rq *rq) - if (fifo_nr_running) - return true; - -+#ifdef CONFIG_HMBIRD_SCHED - /* -- * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks -+ * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks - * left. For CFS, if there's more than one we need the tick for -- * involuntary preemption. For SCX, ask. -+ * involuntary preemption. For HMBIRD, ask. - */ -- if (!scx_switched_all() && rq->nr_running > 1) -+ if (!hmbird_enabled() && rq->nr_running > 1) - return false; - -- if (scx_enabled() && !scx_can_stop_tick(rq)) -+ if (hmbird_enabled() && !hmbird_can_stop_tick(rq)) - return false; -- -+#else -+ /* -+ * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; -+ * if there's more than one we need the tick for involuntary -+ * preemption. -+ */ -+ if (rq->nr_running > 1) -+ return false; -+#endif - /* - * If there is one task and it has CFS runtime bandwidth constraints - * and it's on the cpu now we don't want to stop the tick. -@@ -2222,10 +2235,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) - } - EXPORT_SYMBOL_GPL(deactivate_task); - --struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) -+#ifdef CONFIG_HMBIRD_SCHED -+struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - { -- struct sched_change_guard cg = { -+ struct hmbird_sched_change_guard cg = { - .rq = rq, - .p = p, - .queued = task_on_rq_queued(p), -@@ -2247,7 +2261,7 @@ sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - return cg; - } - --void sched_change_guard_fini(struct sched_change_guard *cg, int flags) -+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags) - { - if (cg->queued) - enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); -@@ -2255,6 +2269,7 @@ void sched_change_guard_fini(struct sched_change_guard *cg, int flags) - set_next_task(cg->rq, cg->p); - cg->done = true; - } -+#endif - - static inline int __normal_prio(int policy, int rt_prio, int nice) - { -@@ -2320,9 +2335,9 @@ inline int task_curr(const struct task_struct *p) - * this means any call to check_class_changed() must be followed by a call to - * balance_callback(). - */ --void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio) -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) - { - if (prev_class != p->sched_class) { - if (prev_class->switched_from) -@@ -2885,6 +2900,7 @@ static void - __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - { - struct rq *rq = task_rq(p); -+ bool queued, running; - - /* - * This here violates the locking rules for affinity, since we're only -@@ -2903,9 +2919,26 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - else - lockdep_assert_held(&p->pi_lock); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->sched_class->set_cpus_allowed(p, ctx); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ -+ if (queued) { -+ /* -+ * Because __kthread_bind() calls this on blocked tasks without -+ * holding rq->lock. -+ */ -+ lockdep_assert_rq_held(rq); -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); - } -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->sched_class->set_cpus_allowed(p, ctx); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - } - - /* -@@ -3772,7 +3805,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class)) -+#else - if (unlikely(!is_cpu_allowed(p, cpu))) -+#endif - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -@@ -4891,8 +4928,9 @@ late_initcall(sched_core_sysctl_init); - */ - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { -- -+#ifdef CONFIG_HMBIRD_SCHED - int ret; -+#endif - - trace_android_rvh_sched_fork(p); - -@@ -4933,21 +4971,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_reset_on_fork = 0; - } - -- scx_pre_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ ret = hmbird_pre_fork(p); -+ if (ret) -+ goto out_cancel; - - if (dl_prio(p->prio)) { - ret = -EAGAIN; - goto out_cancel; --#ifdef CONFIG_SCHED_CLASS_EXT -- } else if (task_on_scx(p)) { -- p->sched_class = &ext_sched_class; --#endif -+ } else if (task_on_hmbird(p)) { -+ p->sched_class = &hmbird_sched_class; - } else if (rt_prio(p->prio)) { - p->sched_class = &rt_sched_class; - } else { - p->sched_class = &fair_sched_class; - } -- -+#else -+ if (dl_prio(p->prio)) -+ return -EAGAIN; -+ else if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - init_entity_runnable_average(&p->se); - trace_android_rvh_finish_prio_fork(p); - -@@ -4966,12 +5012,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - #endif - return 0; - -+#ifdef CONFIG_HMBIRD_SCHED - out_cancel: -- scx_cancel_fork(p); -+ hmbird_cancel_fork(p); - return ret; -+#endif - } - --int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) -+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - { - unsigned long flags; - -@@ -4998,20 +5046,14 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -- -- return scx_fork(p); --} -- --void sched_cancel_fork(struct task_struct *p) --{ -- scx_cancel_fork(p); - } - - void sched_post_fork(struct task_struct *p) - { - uclamp_post_fork(p); -- -- scx_post_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_post_fork(p); -+#endif - } - - unsigned long to_ratio(u64 period, u64 runtime) -@@ -5875,21 +5917,28 @@ void scheduler_tick(void) - - if (sched_feat(LATENCY_WARN) && resched_latency) - resched_latency_warn(cpu, resched_latency); -- -- scx_notify_sched_tick(); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_notify_sched_tick(); -+#endif - perf_event_task_tick(); - - if (curr->flags & PF_WQ_WORKER) - wq_worker_tick(curr); - - #ifdef CONFIG_SMP -- if (!scx_switched_all()) { -+#ifdef CONFIG_HMBIRD_SCHED -+ if (!hmbird_enabled()) { -+#endif - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); -+#ifdef CONFIG_HMBIRD_SCHED - } -+#endif - #endif - trace_android_vh_scheduler_tick(rq); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ scheduler_tick_handler(NULL, NULL); -+#endif - } - - #ifdef CONFIG_NO_HZ_FULL -@@ -6191,7 +6240,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ -+#ifdef CONFIG_HMBIRD_SCHED - for_balance_class_range(class, prev->sched_class, &idle_sched_class) { -+#else -+ for_class_range(class, prev->sched_class, &idle_sched_class) { -+#endif - if (class->balance(rq, prev, rf)) - break; - } -@@ -6209,9 +6262,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - const struct sched_class *class; - struct task_struct *p; - -- if (scx_enabled()) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (hmbird_enabled()) - goto restart; -- -+#endif - /* - * Optimization: we know that if all tasks are in the fair class we can - * call that function directly, but only if the @prev task wasn't of a -@@ -6237,13 +6291,21 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - restart: - put_prev_task_balance(rq, prev, rf); - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { - p = class->pick_next_task(rq); - if (p) { -- scx_notify_pick_next_task(rq, p, class); -+ hmbird_notify_pick_next_task(rq, p, class); - return p; - } - } -+#else -+ for_each_class(class) { -+ p = class->pick_next_task(rq); -+ if (p) -+ return p; -+ } -+#endif - - BUG(); /* The idle class should always have a runnable task. */ - } -@@ -6272,7 +6334,11 @@ static inline struct task_struct *pick_task(struct rq *rq) - const struct sched_class *class; - struct task_struct *p; - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { -+#else -+ for_each_class(class) { -+#endif - p = class->pick_task(rq); - if (p) - return p; -@@ -6916,7 +6982,9 @@ static void __sched notrace __schedule(unsigned int sched_mode) - psi_sched_switch(prev, next, !task_on_rq_queued(prev)); - - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#endif - /* Also unlocks the rq: */ - rq = context_switch(rq, prev, next, &rf); - } else { -@@ -7244,24 +7312,31 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag - } - EXPORT_SYMBOL(default_wake_function); - --void __setscheduler_prio(struct task_struct *p, int prio) -+static void __setscheduler_prio(struct task_struct *p, int prio) - { -- /* -- * After switching all rt and fair class to ext, -- * stop class is switched to ext class too. Just -- * skip it. */ -+#ifdef CONFIG_HMBIRD_SCHED -+ bool on_hmbird = task_on_hmbird(p); -+ - if (p->sched_class == &stop_sched_class) -- ;/* do nothing */ -+ ; - else if (dl_prio(prio)) - p->sched_class = &dl_sched_class; --#ifdef CONFIG_SCHED_CLASS_EXT -- else if (task_on_scx(p)) -- p->sched_class = &ext_sched_class; --#endif -+ else if (rt_prio(prio) && on_hmbird) -+ p->sched_class = &hmbird_sched_class; - else if (rt_prio(prio)) - p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; - else - p->sched_class = &fair_sched_class; -+#else -+ if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - - p->prio = prio; - trace_android_rvh_setscheduler_prio(p); -@@ -7297,7 +7372,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - */ - void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - { -- int prio, oldprio, queue_flag = -+ int prio, oldprio, queued, running, queue_flag = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *prev_class; - struct rq_flags rf; -@@ -7360,42 +7435,52 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - queue_flag &= ~DEQUEUE_MOVE; - - prev_class = p->sched_class; -- SCHED_CHANGE_BLOCK(rq, p, queue_flag) { -- /* -- * Boosting condition are: -- * 1. -rt task is running and holds mutex A -- * --> -dl task blocks on mutex A -- * -- * 2. -dl task is running and holds mutex A -- * --> -dl task blocks on mutex A and could preempt the -- * running task -- */ -- if (dl_prio(prio)) { -- if (!dl_prio(p->normal_prio) || -- (pi_task && dl_prio(pi_task->prio) && -- dl_entity_preempt(&pi_task->dl, &p->dl))) { -- p->dl.pi_se = pi_task->dl.pi_se; -- queue_flag |= ENQUEUE_REPLENISH; -- } else { -- p->dl.pi_se = &p->dl; -- } -- } else if (rt_prio(prio)) { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- if (oldprio < prio) -- queue_flag |= ENQUEUE_HEAD; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flag); -+ if (running) -+ put_prev_task(rq, p); -+ -+ /* -+ * Boosting condition are: -+ * 1. -rt task is running and holds mutex A -+ * --> -dl task blocks on mutex A -+ * -+ * 2. -dl task is running and holds mutex A -+ * --> -dl task blocks on mutex A and could preempt the -+ * running task -+ */ -+ if (dl_prio(prio)) { -+ if (!dl_prio(p->normal_prio) || -+ (pi_task && dl_prio(pi_task->prio) && -+ dl_entity_preempt(&pi_task->dl, &p->dl))) { -+ p->dl.pi_se = pi_task->dl.pi_se; -+ queue_flag |= ENQUEUE_REPLENISH; - } else { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- else if (rt_prio(oldprio)) -- p->rt.timeout = 0; -- else if (!task_has_idle_policy(p)) -- reweight_task(p, prio - MAX_RT_PRIO); -+ p->dl.pi_se = &p->dl; - } -- -- __setscheduler_prio(p, prio); -+ } else if (rt_prio(prio)) { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ if (oldprio < prio) -+ queue_flag |= ENQUEUE_HEAD; -+ } else { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ else if (rt_prio(oldprio)) -+ p->rt.timeout = 0; -+ else if (!task_has_idle_policy(p)) -+ reweight_task(p, prio - MAX_RT_PRIO); - } - -+ __setscheduler_prio(p, prio); -+ -+ if (queued) -+ enqueue_task(rq, p, queue_flag); -+ if (running) -+ set_next_task(rq, p); -+ - check_class_changed(rq, p, prev_class, oldprio); - out_unlock: - /* Avoid rq from going away on us: */ -@@ -7416,7 +7501,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - - void set_user_nice(struct task_struct *p, long nice) - { -- bool allowed = false; -+ bool queued, running, allowed = false; - int old_prio; - struct rq_flags rf; - struct rq *rq; -@@ -7445,13 +7530,22 @@ void set_user_nice(struct task_struct *p, long nice) - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); -+ if (running) -+ put_prev_task(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->static_prio = NICE_TO_PRIO(nice); -- set_load_weight(p, true); -- old_prio = p->prio; -- p->prio = effective_prio(p); -- } -+ p->static_prio = NICE_TO_PRIO(nice); -+ set_load_weight(p, true); -+ old_prio = p->prio; -+ p->prio = effective_prio(p); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - - /* - * If the task increased its priority or is running and -@@ -7868,7 +7962,7 @@ static int __sched_setscheduler(struct task_struct *p, - bool user, bool pi) - { - int oldpolicy = -1, policy = attr->sched_policy; -- int retval, oldprio, newprio; -+ int retval, oldprio, newprio, queued, running; - const struct sched_class *prev_class; - struct balance_callback *head; - struct rq_flags rf; -@@ -7876,7 +7970,9 @@ static int __sched_setscheduler(struct task_struct *p, - int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct rq *rq; - bool cpuset_locked = false; -- -+#ifdef CONFIG_HMBIRD_SCHED -+ unsigned long flags; -+#endif - - /* The pi code expects interrupts enabled */ - BUG_ON(pi && in_interrupt()); -@@ -7942,7 +8038,9 @@ static int __sched_setscheduler(struct task_struct *p, - * To be able to change p->policy safely, the appropriate - * runqueue lock must be held. - */ -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+#endif - rq = task_rq_lock(p, &rf); - update_rq_clock(rq); - -@@ -7954,10 +8052,11 @@ static int __sched_setscheduler(struct task_struct *p, - goto unlock; - } - -- retval = scx_check_setscheduler(p, policy); -+#ifdef CONFIG_HMBIRD_SCHED -+ retval = hmbird_check_setscheduler(p, policy); - if (retval) - goto unlock; -- -+#endif - /* - * If not changing anything there's no need to proceed further, - * but store a possible modification of reset_on_fork. -@@ -8014,7 +8113,9 @@ static int __sched_setscheduler(struct task_struct *p, - if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { - policy = oldpolicy = -1; - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - goto recheck; -@@ -8047,16 +8148,23 @@ static int __sched_setscheduler(struct task_struct *p, - queue_flags &= ~DEQUEUE_MOVE; - } - -- SCHED_CHANGE_BLOCK(rq, p, queue_flags) { -- prev_class = p->sched_class; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flags); -+ if (running) -+ put_prev_task(rq, p); -+ -+ prev_class = p->sched_class; - -- if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -- __setscheduler_params(p, attr); -- __setscheduler_prio(p, newprio); -- trace_android_rvh_setscheduler(p); -- } -- __setscheduler_uclamp(p, attr); -+ if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -+ __setscheduler_params(p, attr); -+ __setscheduler_prio(p, newprio); -+ trace_android_rvh_setscheduler(p); -+ } -+ __setscheduler_uclamp(p, attr); - -+ if (queued) { - /* - * We enqueue to tail when the priority of a task is - * increased (user space view). -@@ -8064,7 +8172,10 @@ static int __sched_setscheduler(struct task_struct *p, - if (oldprio < p->prio) - queue_flags |= ENQUEUE_HEAD; - -+ enqueue_task(rq, p, queue_flags); - } -+ if (running) -+ set_next_task(rq, p); - - check_class_changed(rq, p, prev_class, oldprio); - -@@ -8072,7 +8183,9 @@ static int __sched_setscheduler(struct task_struct *p, - preempt_disable(); - head = splice_balance_callbacks(rq); - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (pi) { - if (cpuset_locked) - cpuset_unlock(); -@@ -8087,7 +8200,9 @@ static int __sched_setscheduler(struct task_struct *p, - - unlock: - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - return retval; -@@ -8785,6 +8900,9 @@ static void do_sched_yield(void) - long skip = 0; - - trace_android_rvh_before_do_sched_yield(&skip); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_skip_yield(&skip); -+#endif - if (skip) - return; - -@@ -9309,7 +9427,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - break; - } -@@ -9337,7 +9457,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - } - return ret; -@@ -9638,15 +9760,25 @@ int migrate_task_to(struct task_struct *p, int target_cpu) - */ - void sched_setnuma(struct task_struct *p, int nid) - { -+ bool queued, running; - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) { -- p->numa_preferred_nid = nid; -- } -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE); -+ if (running) -+ put_prev_task(rq, p); - -+ p->numa_preferred_nid = nid; -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - task_rq_unlock(rq, p, &rf); - } - #endif /* CONFIG_NUMA_BALANCING */ -@@ -10212,17 +10344,23 @@ void __init sched_init(void) - int i; - - /* Make sure the linker didn't screw up */ -+#ifdef CONFIG_HMBIRD_SCHED - #ifdef CONFIG_SMP -- BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+#endif -+ WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class)); -+#else -+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 || -+ &fair_sched_class != &rt_sched_class + 1 || -+ &rt_sched_class != &dl_sched_class + 1); -+#ifdef CONFIG_SMP -+ BUG_ON(&dl_sched_class != &stop_sched_class + 1); - #endif -- BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); -- BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); -- BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); --#ifdef CONFIG_SCHED_CLASS_EXT -- BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); -- BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); - #endif -- - wait_bit_init(); - - #ifdef CONFIG_FAIR_GROUP_SCHED -@@ -10392,8 +10530,9 @@ void __init sched_init(void) - balance_push_set(smp_processor_id(), false); - #endif - init_sched_fair_class(); -- init_sched_ext_class(); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ init_sched_hmbird_class(); -+#endif - psi_init(); - - init_uclamp(); -@@ -10863,6 +11002,11 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) - struct task_group *tg = css_tg(css); - struct task_group *parent = css_tg(css->parent); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_tg_online(tg); -+#endif -+ -+ - if (parent) - sched_online_group(tg, parent); - -@@ -10912,13 +11056,77 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) - } - #endif - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 dl) -+{ -+ int i; -+ -+ for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) { -+ if (dl < HMBIRD_BPF_DSQS_DEADLINE[i]) -+ break; -+ } -+ -+ i = max_t(int, i-1, MIN_CGROUP_DL_IDX); -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return 0; -+ update_cgroup_ids_table(css->cgroup->kn->id, i); -+ -+ return 0; -+} -+ -+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ u8 i; -+ -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX]; -+ i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1); -+ if (i < 0) { -+ pr_err(" <%s> i is %d, less than 0, name is %s\n", -+ __func__, i, css->cgroup->kn->name); -+ i = DEFAULT_CGROUP_DL_IDX; -+ } -+ -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[i]; -+} -+ -+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css, -+ struct task_struct *p, int prio) -+{ -+ if (!css || !rt_prio(prio)) -+ return; -+ -+ if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) { -+ if (!strcmp(css->cgroup->kn->name, "display")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3); -+ else if (!strcmp(css->cgroup->kn->name, "touch")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2); -+ else if (!strcmp(css->cgroup->kn->name, "multimedia")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+ } -+} -+#endif -+ - static void cpu_cgroup_attach(struct cgroup_taskset *tset) - { - struct task_struct *task; - struct cgroup_subsys_state *css; - - cgroup_taskset_for_each(task, css, tset) { -- -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_change_rt_sched_prop(css, task, task->prio); -+#endif - sched_move_task(task); - } - -@@ -11600,7 +11808,13 @@ static struct cftype cpu_legacy_files[] = { - .write_u64 = cpu_uclamp_ls_write_u64, - }, - #endif -- -+#ifdef CONFIG_HMBIRD_SCHED -+ { -+ .name = "hmbird.deadline", -+ .read_u64 = cgroup_read_hmbird_deadline, -+ .write_u64 = cgroup_write_hmbird_deadline, -+ }, -+#endif - { } /* Terminate */ - }; - -@@ -11649,7 +11863,6 @@ static int cpu_local_stat_show(struct seq_file *sf, - } - - #ifdef CONFIG_FAIR_GROUP_SCHED -- - static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, - struct cftype *cft) - { -diff --git b/kernel/sched/debug.c a/kernel/sched/debug.c -index a6c99df9edf9..e09904f4d6e5 100755 ---- b/kernel/sched/debug.c -+++ a/kernel/sched/debug.c -@@ -375,9 +375,8 @@ static __init int sched_init_debug(void) - #endif - - debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); -- --#ifdef CONFIG_SCHED_CLASS_EXT -- debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops); -+#ifdef CONFIG_HMBIRD_SCHED -+ debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops); - #endif - return 0; - } -@@ -1006,9 +1005,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - SEQ_printf(m, - "---------------------------------------------------------" - "----------\n"); --#ifdef CONFIG_SLIM_SCHED -- SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop); --#endif - - #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) - #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) -@@ -1104,8 +1100,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - } else if (fair_policy(p->policy)) { - P(se.slice); - } --#ifdef CONFIG_SCHED_CLASS_EXT -- __PS("ext.enabled", p->sched_class == &ext_sched_class); -+#ifdef CONFIG_HMBIRD_SCHED -+ __PS("hmbird.enabled", p->sched_class == &hmbird_sched_class); -+ __PS("sched_prop", get_hmbird_ts(p)->sched_prop); -+ __PS("top_task_prop", get_hmbird_ts(p)->top_task_prop); - #endif - #undef PN_SCHEDSTAT - #undef P_SCHEDSTAT -diff --git b/kernel/sched/ext.c a/kernel/sched/ext.c -deleted file mode 100755 -index 4845d441df2b..000000000000 ---- b/kernel/sched/ext.c -+++ /dev/null -@@ -1,3978 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -- --enum scx_internal_consts { -- SCX_NR_ONLINE_OPS = SCX_OP_IDX(init), -- SCX_DSP_DFL_MAX_BATCH = 32, -- SCX_DSP_MAX_LOOPS = 32, -- SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, --}; -- --enum scx_ops_enable_state { -- SCX_OPS_PREPPING, -- SCX_OPS_ENABLING, -- SCX_OPS_ENABLED, -- SCX_OPS_DISABLING, -- SCX_OPS_DISABLED, --}; -- --static inline struct task_group *css_tg(struct cgroup_subsys_state *css) --{ -- return css ? container_of(css, struct task_group, css) : NULL; --} -- --/* -- * sched_ext_entity->ops_state -- * -- * Used to track the task ownership between the SCX core and the BPF scheduler. -- * State transitions look as follows: -- * -- * NONE -> QUEUEING -> QUEUED -> DISPATCHING -- * ^ | | -- * | v v -- * \-------------------------------/ -- * -- * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -- * sites for explanations on the conditions being waited upon and why they are -- * safe. Transitions out of them into NONE or QUEUED must store_release and the -- * waiters should load_acquire. -- * -- * Tracking scx_ops_state enables sched_ext core to reliably determine whether -- * any given task can be dispatched by the BPF scheduler at all times and thus -- * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -- * to try to dispatch any task anytime regardless of its state as the SCX core -- * can safely reject invalid dispatches. -- */ --enum scx_ops_state { -- SCX_OPSS_NONE, /* owned by the SCX core */ -- SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -- SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ -- SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ -- -- /* -- * QSEQ brands each QUEUED instance so that, when dispatch races -- * dequeue/requeue, the dispatcher can tell whether it still has a claim -- * on the task being dispatched. -- */ -- SCX_OPSS_QSEQ_SHIFT = 2, -- SCX_OPSS_STATE_MASK = (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1, -- SCX_OPSS_QSEQ_MASK = ~SCX_OPSS_STATE_MASK, --}; -- --/* -- * During exit, a task may schedule after losing its PIDs. When disabling the -- * BPF scheduler, we need to be able to iterate tasks in every state to -- * guarantee system safety. Maintain a dedicated task list which contains every -- * task between its fork and eventual free. -- */ --static DEFINE_SPINLOCK(scx_tasks_lock); --static LIST_HEAD(scx_tasks); -- --/* ops enable/disable */ --static struct kthread_worker *scx_ops_helper; --static DEFINE_MUTEX(scx_ops_enable_mutex); --DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); --DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); --static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); --static bool scx_switch_all_req; --static bool scx_switching_all; --DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -- --static struct sched_ext_ops scx_ops; --static bool scx_warned_zero_slice; -- --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); --DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); --static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); -- --struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] = -- { [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT }; -- --static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE); --static struct scx_exit_info scx_exit_info; -- --static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0); -- --/* -- * The maximum amount of time in jiffies that a task may be runnable without -- * being scheduled on a CPU. If this timeout is exceeded, it will trigger -- * scx_ops_error(). -- */ --unsigned long scx_watchdog_timeout; -- --/* -- * The last time the delayed work was run. This delayed work relies on -- * ksoftirqd being able to run to service timer interrupts, so it's possible -- * that this work itself could get wedged. To account for this, we check that -- * it's not stalled in the timer tick, and trigger an error if it is. -- */ --unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; -- --static struct delayed_work scx_watchdog_work; -- --/* idle tracking */ --#ifdef CONFIG_SMP --#ifdef CONFIG_CPUMASK_OFFSTACK --#define CL_ALIGNED_IF_ONSTACK --#else --#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp --#endif -- --static struct { -- cpumask_var_t cpu; -- cpumask_var_t smt; --} idle_masks CL_ALIGNED_IF_ONSTACK; -- --static bool __cacheline_aligned_in_smp scx_has_idle_cpus; --#endif /* CONFIG_SMP */ -- --/* for %SCX_KICK_WAIT */ --static u64 __percpu *scx_kick_cpus_pnt_seqs; -- --/* -- * Direct dispatch marker. -- * -- * Non-NULL values are used for direct dispatch from enqueue path. A valid -- * pointer points to the task currently being enqueued. An ERR_PTR value is used -- * to indicate that direct dispatch has already happened. -- */ --static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); -- --/* dispatch queues */ --static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; -- --static LLIST_HEAD(dsqs_to_free); -- --/* dispatch buf */ --struct scx_dsp_buf_ent { -- struct task_struct *task; -- u64 qseq; -- u64 dsq_id; -- u64 enq_flags; --}; -- --static u32 scx_dsp_max_batch; --static struct scx_dsp_buf_ent __percpu *scx_dsp_buf; -- --struct scx_dsp_ctx { -- struct rq *rq; -- struct rq_flags *rf; -- u32 buf_cursor; -- u32 nr_tasks; --}; -- --static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx); -- --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags); --void scx_bpf_kick_cpu(s32 cpu, u64 flags); -- --struct scx_task_iter { -- struct sched_ext_entity cursor; -- struct task_struct *locked; -- struct rq *rq; -- struct rq_flags rf; --}; -- --#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) -- --/* if the highest set bit is N, return a mask with bits [N+1, 31] set */ --static u32 higher_bits(u32 flags) --{ -- return ~((1 << fls(flags)) - 1); --} -- --/* return the mask with only the highest bit set */ --static u32 highest_bit(u32 flags) --{ -- int bit = fls(flags); -- return bit ? 1 << (bit - 1) : 0; --} -- --/* -- * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX -- * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate -- * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check -- * whether it's running from an allowed context. -- * -- * @mask is constant, always inline to cull the mask calculations. -- */ --static __always_inline void scx_kf_allow(u32 mask) --{ -- /* nesting is allowed only in increasing scx_kf_mask order */ -- WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask, -- "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n", -- current->scx->kf_mask, mask); -- current->scx->kf_mask |= mask; --} -- --static void scx_kf_disallow(u32 mask) --{ -- current->scx->kf_mask &= ~mask; --} -- --#define SCX_CALL_OP(mask, op, args...) \ --do { \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- scx_ops.op(args); \ -- } \ --} while (0) -- --#define SCX_CALL_OP_RET(mask, op, args...) \ --({ \ -- __typeof__(scx_ops.op(args)) __ret; \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- __ret = scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- __ret = scx_ops.op(args); \ -- } \ -- __ret; \ --}) -- --/* -- * Some kfuncs are allowed only on the tasks that are subjects of the -- * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such -- * restrictions, the following SCX_CALL_OP_*() variants should be used when -- * invoking scx_ops operations that take task arguments. These can only be used -- * for non-nesting operations due to the way the tasks are tracked. -- * -- * kfuncs which can only operate on such tasks can in turn use -- * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on -- * the specific task. -- */ --#define SCX_CALL_OP_TASK(mask, op, task, args...) \ --do { \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- SCX_CALL_OP(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ --} while (0) -- --#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ --({ \ -- __typeof__(scx_ops.op(task, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- __ret; \ --}) -- --#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ --({ \ -- __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task0; \ -- current->scx->kf_tasks[1] = task1; \ -- __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- current->scx->kf_tasks[1] = NULL; \ -- __ret; \ --}) -- --//#include "./slim_walt.c" -- --/* @mask is constant, always inline to cull unnecessary branches */ --static __always_inline bool scx_kf_allowed(u32 mask) --{ -- if (unlikely(!(current->scx->kf_mask & mask))) { -- scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", -- mask, current->scx->kf_mask); -- return false; -- } -- -- if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) && -- in_interrupt())) { -- scx_ops_error("sleepable kfunc called from non-sleepable context"); -- return false; -- } -- -- /* -- * Enforce nesting boundaries. e.g. A kfunc which can be called from -- * DISPATCH must not be called if we're running DEQUEUE which is nested -- * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE -- * boundary thanks to the above in_interrupt() check. -- */ -- if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && -- (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { -- scx_ops_error("cpu_release kfunc called from a nested operation"); -- return false; -- } -- -- if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && -- (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) { -- scx_ops_error("dispatch kfunc called from a nested operation"); -- return false; -- } -- -- return true; --} -- --/* see SCX_CALL_OP_TASK() */ --static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, -- struct task_struct *p) --{ -- if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED)) -- return false; -- -- if (unlikely((p != current->scx->kf_tasks[0] && -- p != current->scx->kf_tasks[1]))) { -- scx_ops_error("called on a task not being operated on"); -- return false; -- } -- -- return true; --} -- --/** -- * scx_task_iter_init - Initialize a task iterator -- * @iter: iterator to init -- * -- * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized, -- * @iter must eventually be exited with scx_task_iter_exit(). -- * -- * scx_tasks_lock may be released between this and the first next() call or -- * between any two next() calls. If scx_tasks_lock is released between two -- * next() calls, the caller is responsible for ensuring that the task being -- * iterated remains accessible either through RCU read lock or obtaining a -- * reference count. -- * -- * All tasks which existed when the iteration started are guaranteed to be -- * visited as long as they still exist. -- */ --static void scx_task_iter_init(struct scx_task_iter *iter) --{ -- lockdep_assert_held(&scx_tasks_lock); -- -- iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; -- list_add(&iter->cursor.tasks_node, &scx_tasks); -- iter->locked = NULL; --} -- --/** -- * scx_task_iter_exit - Exit a task iterator -- * @iter: iterator to exit -- * -- * Exit a previously initialized @iter. Must be called with scx_tasks_lock held. -- * If the iterator holds a task's rq lock, that rq lock is released. See -- * scx_task_iter_init() for details. -- */ --static void scx_task_iter_exit(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- if (list_empty(cursor)) -- return; -- -- list_del_init(cursor); --} -- --/** -- * scx_task_iter_next - Next task -- * @iter: iterator to walk -- * -- * Visit the next task. See scx_task_iter_init() for details. -- */ --static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- struct sched_ext_entity *pos; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- list_for_each_entry(pos, cursor, tasks_node) { -- if (&pos->tasks_node == &scx_tasks) -- return NULL; -- if (!(pos->flags & SCX_TASK_CURSOR)) { -- list_move(cursor, &pos->tasks_node); -- return pos->task; -- } -- } -- -- /* can't happen, should always terminate at scx_tasks above */ -- BUG(); --} -- --/** -- * scx_task_iter_next_filtered - Next non-idle task -- * @iter: iterator to walk -- * -- * Visit the next non-idle task. See scx_task_iter_init() for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- while ((p = scx_task_iter_next(iter))) { -- if (!is_idle_task(p)) -- return p; -- } -- return NULL; --} -- --/** -- * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked -- * @iter: iterator to walk -- * -- * Visit the next non-idle task with its rq lock held. See scx_task_iter_init() -- * for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered_locked(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- p = scx_task_iter_next_filtered(iter); -- if (!p) -- return NULL; -- -- iter->rq = task_rq_lock(p, &iter->rf); -- iter->locked = p; -- return p; --} -- --static enum scx_ops_enable_state scx_ops_enable_state(void) --{ -- return atomic_read(&scx_ops_enable_state_var); --} -- --static enum scx_ops_enable_state --scx_ops_set_enable_state(enum scx_ops_enable_state to) --{ -- return atomic_xchg(&scx_ops_enable_state_var, to); --} -- --static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, -- enum scx_ops_enable_state from) --{ -- int from_v = from; -- -- return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); --} -- --static bool scx_ops_disabling(void) --{ -- return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING); --} -- --/** -- * wait_ops_state - Busy-wait the specified ops state to end -- * @p: target task -- * @opss: state to wait the end of -- * -- * Busy-wait for @p to transition out of @opss. This can only be used when the -- * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also -- * has load_acquire semantics to ensure that the caller can see the updates made -- * in the enqueueing and dispatching paths. -- */ --static void wait_ops_state(struct task_struct *p, u64 opss) --{ -- do { -- cpu_relax(); -- } while (atomic64_read_acquire(&p->scx->ops_state) == opss); --} -- --/** -- * ops_cpu_valid - Verify a cpu number -- * @cpu: cpu number which came from a BPF ops -- * -- * @cpu is a cpu number which came from the BPF scheduler and can be any value. -- * Verify that it is in range and one of the possible cpus. -- */ --static bool ops_cpu_valid(s32 cpu) --{ -- return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); --} -- --/** -- * ops_sanitize_err - Sanitize a -errno value -- * @ops_name: operation to blame on failure -- * @err: -errno value to sanitize -- * -- * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return -- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can -- * cause misbehaviors. For an example, a large negative return from -- * ops.prep_enable() triggers an oops when passed up the call chain because the -- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is -- * handled as a pointer. -- */ --static int ops_sanitize_err(const char *ops_name, s32 err) --{ -- if (err < 0 && err >= -MAX_ERRNO) -- return err; -- -- scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); -- return -EPROTO; --} -- --/** -- * touch_core_sched - Update timestamp used for core-sched task ordering -- * @rq: rq to read clock from, must be locked -- * @p: task to update the timestamp for -- * -- * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to -- * implement global or local-DSQ FIFO ordering for core-sched. Should be called -- * when a task becomes runnable and its turn on the CPU ends (e.g. slice -- * exhaustion). -- */ --static void touch_core_sched(struct rq *rq, struct task_struct *p) --{ --#ifdef CONFIG_SCHED_CORE -- /* -- * It's okay to update the timestamp spuriously. Use -- * sched_core_disabled() which is cheaper than enabled(). -- */ -- if (!sched_core_disabled()) -- p->scx->core_sched_at = rq_clock_task(rq); --#endif --} -- --/** -- * touch_core_sched_dispatch - Update core-sched timestamp on dispatch -- * @rq: rq to read clock from, must be locked -- * @p: task being dispatched -- * -- * If the BPF scheduler implements custom core-sched ordering via -- * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO -- * ordering within each local DSQ. This function is called from dispatch paths -- * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect. -- */ --static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- assert_clock_updated(rq); -- --#ifdef CONFIG_SCHED_CORE -- if (SCX_HAS_OP(core_sched_before)) -- touch_core_sched(rq, p); --#endif --} -- --static void update_curr_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- u64 now = rq_clock_task(rq); -- u64 delta_exec; -- -- if (time_before_eq64(now, curr->se.exec_start)) -- return; -- -- delta_exec = now - curr->se.exec_start; -- curr->se.exec_start = now; -- curr->se.sum_exec_runtime += delta_exec; -- account_group_exec_runtime(curr, delta_exec); -- cgroup_account_cputime(curr, delta_exec); -- -- if (curr->scx->slice != SCX_SLICE_INF) { -- curr->scx->slice -= min(curr->scx->slice, delta_exec); -- if (!curr->scx->slice) -- touch_core_sched(rq, curr); -- } --} -- --static bool scx_dsq_priq_less(struct rb_node *node_a, -- const struct rb_node *node_b) --{ -- const struct sched_ext_entity *a = -- container_of(node_a, struct sched_ext_entity, dsq_node.priq); -- const struct sched_ext_entity *b = -- container_of(node_b, struct sched_ext_entity, dsq_node.priq); -- -- return time_before64(a->dsq_vtime, b->dsq_vtime); --} -- --static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, -- u64 enq_flags) --{ -- bool is_local = dsq->id == SCX_DSQ_LOCAL; -- -- WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo)); -- WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq)); -- -- if (!is_local) { -- raw_spin_lock(&dsq->lock); -- if (unlikely(dsq->id == SCX_DSQ_INVALID)) { -- scx_ops_error("attempting to dispatch to a destroyed dsq"); -- /* fall back to the global dsq */ -- raw_spin_unlock(&dsq->lock); -- dsq = &scx_dsq_global; -- raw_spin_lock(&dsq->lock); -- } -- } -- -- if (enq_flags & SCX_ENQ_DSQ_PRIQ) { -- p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; -- rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq, -- scx_dsq_priq_less); -- } else { -- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) -- list_add(&p->scx->dsq_node.fifo, &dsq->fifo); -- else -- list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo); -- } -- dsq->nr++; -- p->scx->dsq = dsq; -- -- /* -- * We're transitioning out of QUEUEING or DISPATCHING. store_release to -- * match waiters' load_acquire. -- */ -- if (enq_flags & SCX_ENQ_CLEAR_OPSS) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- if (is_local) { -- struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq); -- struct rq *rq = scx->rq; -- bool preempt = false; -- -- if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && -- rq->curr->sched_class == &ext_sched_class) { -- rq->curr->scx->slice = 0; -- preempt = true; -- } -- -- if (preempt || sched_class_above(&ext_sched_class, -- rq->curr->sched_class)) -- resched_curr(rq); -- } else { -- raw_spin_unlock(&dsq->lock); -- } --} -- --static void task_unlink_from_dsq(struct task_struct *p, -- struct scx_dispatch_q *dsq) --{ -- if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { -- rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; -- } else { -- list_del_init(&p->scx->dsq_node.fifo); -- } -- --} -- --static bool task_linked_on_dsq(struct task_struct *p) --{ -- return !list_empty(&p->scx->dsq_node.fifo) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq); --} -- --static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p) --{ -- struct scx_dispatch_q *dsq = p->scx->dsq; -- bool is_local = dsq == &scx_rq->local_dsq; -- -- if (!dsq) { -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- /* -- * When dispatching directly from the BPF scheduler to a local -- * DSQ, the task isn't associated with any DSQ but -- * @p->scx->holding_cpu may be set under the protection of -- * %SCX_OPSS_DISPATCHING. -- */ -- if (p->scx->holding_cpu >= 0) -- p->scx->holding_cpu = -1; -- return; -- } -- -- if (!is_local) -- raw_spin_lock(&dsq->lock); -- -- /* -- * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node -- * can't change underneath us. -- */ -- if (p->scx->holding_cpu < 0) { -- /* @p must still be on @dsq, dequeue */ -- WARN_ON_ONCE(!task_linked_on_dsq(p)); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- } else { -- /* -- * We're racing against dispatch_to_local_dsq() which already -- * removed @p from @dsq and set @p->scx->holding_cpu. Clear the -- * holding_cpu which tells dispatch_to_local_dsq() that it lost -- * the race. -- */ -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- p->scx->holding_cpu = -1; -- } -- p->scx->dsq = NULL; -- -- if (!is_local) -- raw_spin_unlock(&dsq->lock); --} -- --static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) --{ -- lockdep_assert(rcu_read_lock_any_held()); -- -- return &scx_dsq_global; --} -- --static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, -- struct task_struct *p) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id == SCX_DSQ_LOCAL) -- return &rq->scx->local_dsq; -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", -- dsq_id, p->comm, p->pid); -- return &scx_dsq_global; -- } -- -- return dsq; --} -- --static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- -- /* @p must match the task which is being enqueued */ -- if (unlikely(p != ddsp_task)) { -- if (IS_ERR(ddsp_task)) -- scx_ops_error("%s[%d] already direct-dispatched", -- p->comm, p->pid); -- else -- scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]", -- ddsp_task->comm, ddsp_task->pid, -- p->comm, p->pid); -- return; -- } -- -- /* -- * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because -- * dispatching to the local DSQ of a different CPU requires unlocking -- * the current rq which isn't allowed in the enqueue path. Use -- * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL. -- */ -- if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) { -- scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch"); -- return; -- } -- -- touch_core_sched_dispatch(task_rq(p), p); -- -- dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- -- /* -- * Mark that dispatch already happened by spoiling direct_dispatch_task -- * with a non-NULL value which can never match a valid task pointer. -- */ -- __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); --} -- --static bool test_rq_online(struct rq *rq) --{ --#ifdef CONFIG_SMP -- return rq->online; --#else -- return true; --#endif --} -- --static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- int sticky_cpu) --{ -- struct task_struct **ddsp_taskp; -- u64 qseq; -- -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- if (p->scx->flags & SCX_TASK_ENQ_LOCAL) { -- enq_flags |= SCX_ENQ_LOCAL; -- p->scx->flags &= ~SCX_TASK_ENQ_LOCAL; -- } -- -- /* rq migration */ -- if (sticky_cpu == cpu_of(rq)) -- goto local_norefill; -- -- /* -- * If !rq->online, we already told the BPF scheduler that the CPU is -- * offline. We're just trying to on/offline the CPU. Don't bother the -- * BPF scheduler. -- */ -- if (unlikely(!test_rq_online(rq))) -- goto local; -- -- /* see %SCX_OPS_ENQ_EXITING */ -- if (!static_branch_unlikely(&scx_ops_enq_exiting) && -- unlikely(p->flags & PF_EXITING)) -- goto local; -- -- /* see %SCX_OPS_ENQ_LAST */ -- if (!static_branch_unlikely(&scx_ops_enq_last) && -- (enq_flags & SCX_ENQ_LAST)) -- goto local; -- -- if (!SCX_HAS_OP(enqueue)) { -- if (enq_flags & SCX_ENQ_LOCAL) -- goto local; -- else -- goto global; -- } -- -- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ -- qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; -- -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq); -- -- ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); -- WARN_ON_ONCE(*ddsp_taskp); -- *ddsp_taskp = p; -- -- SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); -- -- /* -- * If not directly dispatched, QUEUEING isn't clear yet and dispatch or -- * dequeue may be waiting. The store_release matches their load_acquire. -- */ -- if (*ddsp_taskp == p) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq); -- *ddsp_taskp = NULL; -- return; -- --local: -- /* -- * For task-ordering, slice refill must be treated as implying the end -- * of the current slice. Otherwise, the longer @p stays on the CPU, the -- * higher priority it becomes from scx_prio_less()'s POV. -- */ -- touch_core_sched(rq, p); -- p->scx->slice = SCX_SLICE_DFL; --local_norefill: -- dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags); -- return; -- --global: -- touch_core_sched(rq, p); /* see the comment in local: */ -- p->scx->slice = SCX_SLICE_DFL; -- dispatch_enqueue(&scx_dsq_global, p, enq_flags); --} -- --static bool watchdog_task_watched(const struct task_struct *p) --{ -- return !list_empty(&p->scx->watchdog_node); --} -- --static void watchdog_watch_task(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- if (p->scx->flags & SCX_TASK_WATCHDOG_RESET) -- p->scx->runnable_at = jiffies; -- p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET; -- list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list); --} -- --static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) --{ -- list_del_init(&p->scx->watchdog_node); -- if (reset_timeout) -- p->scx->flags |= SCX_TASK_WATCHDOG_RESET; --} -- --static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) --{ -- int sticky_cpu = p->scx->sticky_cpu; -- -- enq_flags |= rq->scx->extra_enq_flags; -- -- if (sticky_cpu >= 0) -- p->scx->sticky_cpu = -1; -- -- /* -- * Restoring a running task will be immediately followed by -- * set_next_task_scx() which expects the task to not be on the BPF -- * scheduler as tasks can only start running through local DSQs. Force -- * direct-dispatch into the local DSQ by setting the sticky_cpu. -- */ -- if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -- sticky_cpu = cpu_of(rq); -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- WARN_ON_ONCE(!watchdog_task_watched(p)); -- return; -- } -- -- watchdog_watch_task(rq, p); -- p->scx->flags |= SCX_TASK_QUEUED; -- rq->scx->nr_running++; -- add_nr_running(rq, 1); -- -- if (SCX_HAS_OP(runnable)) -- SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); -- -- if (enq_flags & SCX_ENQ_WAKEUP) -- touch_core_sched(rq, p); -- -- do_enqueue_task(rq, p, enq_flags, sticky_cpu); --} -- --static void ops_dequeue(struct task_struct *p, u64 deq_flags) --{ -- u64 opss; -- -- watchdog_unwatch_task(p, false); -- -- /* acquire ensures that we see the preceding updates on QUEUED */ -- opss = atomic64_read_acquire(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_NONE: -- break; -- case SCX_OPSS_QUEUEING: -- /* -- * QUEUEING is started and finished while holding @p's rq lock. -- * As we're holding the rq lock now, we shouldn't see QUEUEING. -- */ -- BUG(); -- case SCX_OPSS_QUEUED: -- if (SCX_HAS_OP(dequeue)) -- SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); -- -- if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_NONE)) -- break; -- fallthrough; -- case SCX_OPSS_DISPATCHING: -- /* -- * If @p is being dispatched from the BPF scheduler to a DSQ, -- * wait for the transfer to complete so that @p doesn't get -- * added to its DSQ after dequeueing is complete. -- * -- * As we're waiting on DISPATCHING with the rq locked, the -- * dispatching side shouldn't try to lock the rq while -- * DISPATCHING is set. See dispatch_to_local_dsq(). -- * -- * DISPATCHING shouldn't have qseq set and control can reach -- * here with NONE @opss from the above QUEUED case block. -- * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. -- */ -- wait_ops_state(p, SCX_OPSS_DISPATCHING); -- BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- break; -- } --} -- --static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) --{ -- struct scx_rq *scx_rq = rq->scx; -- -- if (!(p->scx->flags & SCX_TASK_QUEUED)) { -- WARN_ON_ONCE(watchdog_task_watched(p)); -- return; -- } -- -- ops_dequeue(p, deq_flags); -- -- /* -- * A currently running task which is going off @rq first gets dequeued -- * and then stops running. As we want running <-> stopping transitions -- * to be contained within runnable <-> quiescent transitions, trigger -- * ->stopping() early here instead of in put_prev_task_scx(). -- * -- * @p may go through multiple stopping <-> running transitions between -- * here and put_prev_task_scx() if task attribute changes occur while -- * balance_scx() leaves @rq unlocked. However, they don't contain any -- * information meaningful to the BPF scheduler and can be suppressed by -- * skipping the callbacks if the task is !QUEUED. -- */ -- if (SCX_HAS_OP(stopping) && task_current(rq, p)) { -- update_curr_scx(rq); -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); -- } -- -- //if(task_current(rq, p)) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- if (SCX_HAS_OP(quiescent)) -- SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); -- -- if (deq_flags & SCX_DEQ_SLEEP) -- p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP; -- else -- p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP; -- -- p->scx->flags &= ~SCX_TASK_QUEUED; -- BUG_ON(!scx_rq->nr_running); -- scx_rq->nr_running--; -- sub_nr_running(rq, 1); -- -- dispatch_dequeue(scx_rq, p); --} -- --static void yield_task_scx(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); -- else -- p->scx->slice = 0; --} -- --static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) --{ -- struct task_struct *from = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); -- else -- return false; --} -- --#ifdef CONFIG_SMP --/** -- * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -- * @rq: rq to move the task into, currently locked -- * @p: task to move -- * @enq_flags: %SCX_ENQ_* -- * -- * Move @p which is currently on a different rq to @rq's local DSQ. The caller -- * must: -- * -- * 1. Start with exclusive access to @p either through its DSQ lock or -- * %SCX_OPSS_DISPATCHING flag. -- * -- * 2. Set @p->scx->holding_cpu to raw_smp_processor_id(). -- * -- * 3. Remember task_rq(@p). Release the exclusive access so that we don't -- * deadlock with dequeue. -- * -- * 4. Lock @rq and the task_rq from #3. -- * -- * 5. Call this function. -- * -- * Returns %true if @p was successfully moved. %false after racing dequeue and -- * losing. -- */ --static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -- u64 enq_flags) --{ -- struct rq *task_rq; -- -- lockdep_assert_rq_held(rq); -- -- /* -- * If dequeue got to @p while we were trying to lock both rq's, it'd -- * have cleared @p->scx->holding_cpu to -1. While other cpus may have -- * updated it to different values afterwards, as this operation can't be -- * preempted or recurse, @p->scx->holding_cpu can never become -- * raw_smp_processor_id() again before we're done. Thus, we can tell -- * whether we lost to dequeue by testing whether @p->scx->holding_cpu is -- * still raw_smp_processor_id(). -- * -- * See dispatch_dequeue() for the counterpart. -- */ -- if (unlikely(p->scx->holding_cpu != raw_smp_processor_id())) -- return false; -- -- /* @p->rq couldn't have changed if we're still the holding cpu */ -- task_rq = task_rq(p); -- lockdep_assert_rq_held(task_rq); -- -- WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)); -- deactivate_task(task_rq, p, 0); -- set_task_cpu(p, cpu_of(rq)); -- p->scx->sticky_cpu = cpu_of(rq); -- -- /* -- * We want to pass scx-specific enq_flags but activate_task() will -- * truncate the upper 32 bit. As we own @rq, we can pass them through -- * @rq->scx->extra_enq_flags instead. -- */ -- WARN_ON_ONCE(rq->scx->extra_enq_flags); -- rq->scx->extra_enq_flags = enq_flags; -- activate_task(rq, p, 0); -- rq->scx->extra_enq_flags = 0; -- -- return true; --} -- --/** -- * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * We're holding @rq lock and trying to dispatch a task from @src_rq to -- * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether -- * @rq stays locked isn't important as long as the state is restored after -- * dispatch_to_local_dsq_unlock(). -- */ --static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- rq_unpin_lock(rq, rf); -- -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(rq); -- raw_spin_rq_lock(dst_rq); -- } else if (rq == src_rq) { -- double_lock_balance(rq, dst_rq); -- rq_repin_lock(rq, rf); -- } else if (rq == dst_rq) { -- double_lock_balance(rq, src_rq); -- rq_repin_lock(rq, rf); -- } else { -- raw_spin_rq_unlock(rq); -- double_rq_lock(src_rq, dst_rq); -- } --} -- --/** -- * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock() -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return. -- */ --static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } else if (rq == src_rq) { -- double_unlock_balance(rq, dst_rq); -- } else if (rq == dst_rq) { -- double_unlock_balance(rq, src_rq); -- } else { -- double_rq_unlock(src_rq, dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } --} --#endif /* CONFIG_SMP */ -- -- --static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq) --{ -- return likely(test_rq_online(rq)) && !is_migration_disabled(p) && -- cpumask_test_cpu(cpu_of(rq), p->cpus_ptr); --} -- --static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -- struct scx_dispatch_q *dsq) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rb_node *rb_node; -- struct rq *task_rq; -- bool moved = false; --retry: -- if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -- return false; -- -- raw_spin_lock(&dsq->lock); -- -- list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- for (rb_node = rb_first_cached(&dsq->priq); rb_node; -- rb_node = rb_next(rb_node)) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- raw_spin_unlock(&dsq->lock); -- return false; -- --this_rq: -- /* @dsq is locked and @p is on this rq */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo); -- dsq->nr--; -- scx_rq->local_dsq.nr++; -- p->scx->dsq = &scx_rq->local_dsq; -- raw_spin_unlock(&dsq->lock); -- return true; -- --remote_rq: --#ifdef CONFIG_SMP -- /* -- * @dsq is locked and @p is on a remote rq. @p is currently protected by -- * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -- * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -- * rq lock or fail, do a little dancing from our side. See -- * move_task_to_local_dsq(). -- */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- p->scx->holding_cpu = raw_smp_processor_id(); -- raw_spin_unlock(&dsq->lock); -- -- rq_unpin_lock(rq, rf); -- double_lock_balance(rq, task_rq); -- rq_repin_lock(rq, rf); -- -- moved = move_task_to_local_dsq(rq, p, 0); -- -- double_unlock_balance(rq, task_rq); --#endif /* CONFIG_SMP */ -- if (likely(moved)) -- return true; -- goto retry; --} -- --enum dispatch_to_local_dsq_ret { -- DTL_DISPATCHED, /* successfully dispatched */ -- DTL_LOST, /* lost race to dequeue */ -- DTL_NOT_LOCAL, /* destination is not a local DSQ */ -- DTL_INVALID, /* invalid local dsq_id */ --}; -- --/** -- * dispatch_to_local_dsq - Dispatch a task to a local dsq -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @dsq_id: destination dsq ID -- * @p: task to dispatch -- * @enq_flags: %SCX_ENQ_* -- * -- * We're holding @rq lock and want to dispatch @p to the local DSQ identified by -- * @dsq_id. This function performs all the synchronization dancing needed -- * because local DSQs are protected with rq locks. -- * -- * The caller must have exclusive ownership of @p (e.g. through -- * %SCX_OPSS_DISPATCHING). -- */ --static enum dispatch_to_local_dsq_ret --dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id, -- struct task_struct *p, u64 enq_flags) --{ -- struct rq *src_rq = task_rq(p); -- struct rq *dst_rq; -- -- /* -- * We're synchronized against dequeue through DISPATCHING. As @p can't -- * be dequeued, its task_rq and cpus_allowed are stable too. -- */ -- if (dsq_id == SCX_DSQ_LOCAL) { -- dst_rq = rq; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]", -- cpu, p->comm, p->pid); -- return DTL_INVALID; -- } -- dst_rq = cpu_rq(cpu); -- } else { -- return DTL_NOT_LOCAL; -- } -- -- /* if dispatching to @rq that @p is already on, no lock dancing needed */ -- if (rq == src_rq && rq == dst_rq) { -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags | SCX_ENQ_CLEAR_OPSS); -- return DTL_DISPATCHED; -- } -- --#ifdef CONFIG_SMP -- if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { -- struct rq *locked_dst_rq = dst_rq; -- bool dsp; -- -- /* -- * @p is on a possibly remote @src_rq which we need to lock to -- * move the task. If dequeue is in progress, it'd be locking -- * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq -- * lock while holding DISPATCHING. -- * -- * As DISPATCHING guarantees that @p is wholly ours, we can -- * pretend that we're moving from a DSQ and use the same -- * mechanism - mark the task under transfer with holding_cpu, -- * release DISPATCHING and then follow the same protocol. -- */ -- p->scx->holding_cpu = raw_smp_processor_id(); -- -- /* store_release ensures that dequeue sees the above */ -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq); -- -- /* -- * We don't require the BPF scheduler to avoid dispatching to -- * offline CPUs mostly for convenience but also because CPUs can -- * go offline between scx_bpf_dispatch() calls and here. If @p -- * is destined to an offline CPU, queue it on its current CPU -- * instead, which should always be safe. As this is an allowed -- * behavior, don't trigger an ops error. -- */ -- if (unlikely(!test_rq_online(dst_rq))) -- dst_rq = src_rq; -- -- if (src_rq == dst_rq) { -- /* -- * As @p is staying on the same rq, there's no need to -- * go through the full deactivate/activate cycle. -- * Optimize by abbreviating the operations in -- * move_task_to_local_dsq(). -- */ -- dsp = p->scx->holding_cpu == raw_smp_processor_id(); -- if (likely(dsp)) { -- p->scx->holding_cpu = -1; -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags); -- } -- } else { -- dsp = move_task_to_local_dsq(dst_rq, p, enq_flags); -- } -- -- /* if the destination CPU is idle, wake it up */ -- if (dsp && p->sched_class > dst_rq->curr->sched_class) -- resched_curr(dst_rq); -- -- dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq); -- -- return dsp ? DTL_DISPATCHED : DTL_LOST; -- } --#endif /* CONFIG_SMP */ -- -- scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", -- cpu_of(dst_rq), p->comm, p->pid); -- return DTL_INVALID; --} -- --/** -- * finish_dispatch - Asynchronously finish dispatching a task -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @p: task to finish dispatching -- * @qseq_at_dispatch: qseq when @p started getting dispatched -- * @dsq_id: destination DSQ ID -- * @enq_flags: %SCX_ENQ_* -- * -- * Dispatching to local DSQs may need to wait for queueing to complete or -- * require rq lock dancing. As we don't wanna do either while inside -- * ops.dispatch() to avoid locking order inversion, we split dispatching into -- * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the -- * task and its qseq. Once ops.dispatch() returns, this function is called to -- * finish up. -- * -- * There is no guarantee that @p is still valid for dispatching or even that it -- * was valid in the first place. Make sure that the task is still owned by the -- * BPF scheduler and claim the ownership before dispatching. -- */ --static void finish_dispatch(struct rq *rq, struct rq_flags *rf, -- struct task_struct *p, u64 qseq_at_dispatch, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- u64 opss; -- -- touch_core_sched_dispatch(rq, p); --retry: -- /* -- * No need for _acquire here. @p is accessed only after a successful -- * try_cmpxchg to DISPATCHING. -- */ -- opss = atomic64_read(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_DISPATCHING: -- case SCX_OPSS_NONE: -- /* someone else already got to it */ -- return; -- case SCX_OPSS_QUEUED: -- /* -- * If qseq doesn't match, @p has gone through at least one -- * dispatch/dequeue and re-enqueue cycle between -- * scx_bpf_dispatch() and here and we have no claim on it. -- */ -- if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) -- return; -- -- /* -- * While we know @p is accessible, we don't yet have a claim on -- * it - the BPF scheduler is allowed to dispatch tasks -- * spuriously and there can be a racing dequeue attempt. Let's -- * claim @p by atomically transitioning it from QUEUED to -- * DISPATCHING. -- */ -- if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_DISPATCHING))) -- break; -- goto retry; -- case SCX_OPSS_QUEUEING: -- /* -- * do_enqueue_task() is in the process of transferring the task -- * to the BPF scheduler while holding @p's rq lock. As we aren't -- * holding any kernel or BPF resource that the enqueue path may -- * depend upon, it's safe to wait. -- */ -- wait_ops_state(p, opss); -- goto retry; -- } -- -- BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) { -- case DTL_DISPATCHED: -- break; -- case DTL_LOST: -- break; -- case DTL_INVALID: -- dsq_id = SCX_DSQ_GLOBAL; -- fallthrough; -- case DTL_NOT_LOCAL: -- dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()), -- dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- break; -- } --} -- --static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- u32 u; -- -- for (u = 0; u < dspc->buf_cursor; u++) { -- struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u]; -- -- finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id, -- ent->enq_flags); -- } -- -- dspc->nr_tasks += dspc->buf_cursor; -- dspc->buf_cursor = 0; --} -- --static int balance_one(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf, bool local) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- bool prev_on_scx = prev->sched_class == &ext_sched_class; -- int nr_loops = SCX_DSP_MAX_LOOPS; -- -- lockdep_assert_rq_held(rq); -- -- if (static_branch_unlikely(&scx_ops_cpu_preempt) && -- unlikely(rq->scx->cpu_released)) { -- /* -- * If the previous sched_class for the current CPU was not SCX, -- * notify the BPF scheduler that it again has control of the -- * core. This callback complements ->cpu_release(), which is -- * emitted in scx_notify_pick_next_task(). -- */ -- if (SCX_HAS_OP(cpu_acquire)) -- SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq), -- NULL); -- rq->scx->cpu_released = false; -- } -- -- if (prev_on_scx) { -- WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP)); -- update_curr_scx(rq); -- -- /* -- * If @prev is runnable & has slice left, it has priority and -- * fetching more just increases latency for the fetched tasks. -- * Tell put_prev_task_scx() to put @prev on local_dsq. If the -- * BPF scheduler wants to handle this explicitly, it should -- * implement ->cpu_released(). -- * -- * See scx_ops_disable_workfn() for the explanation on the -- * disabling() test. -- * -- * When balancing a remote CPU for core-sched, there won't be a -- * following put_prev_task_scx() call and we don't own -- * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the -- * same conditions later and pick @rq->curr accordingly. -- */ -- if ((prev->scx->flags & SCX_TASK_QUEUED) && -- prev->scx->slice && !scx_ops_disabling()) { -- if (local) -- prev->scx->flags |= SCX_TASK_BAL_KEEP; -- return 1; -- } -- } -- -- /* if there already are tasks to run, nothing to do */ -- if (scx_rq->local_dsq.nr) -- return 1; -- -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- if (!SCX_HAS_OP(dispatch)) -- return 0; -- -- dspc->rq = rq; -- dspc->rf = rf; -- -- /* -- * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, -- * the local DSQ might still end up empty after a successful -- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() -- * produced some tasks, retry. The BPF scheduler may depend on this -- * looping behavior to simplify its implementation. -- */ -- do { -- dspc->nr_tasks = 0; -- -- SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), -- prev_on_scx ? prev : NULL); -- -- flush_dispatch_buf(rq, rf); -- -- if (scx_rq->local_dsq.nr) -- return 1; -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- /* -- * ops.dispatch() can trap us in this loop by repeatedly -- * dispatching ineligible tasks. Break out once in a while to -- * allow the watchdog to run. As IRQ can't be enabled in -- * balance(), we want to complete this scheduling cycle and then -- * start a new one. IOW, we want to call resched_curr() on the -- * next, most likely idle, task, not the current one. Use -- * scx_bpf_kick_cpu() for deferred kicking. -- */ -- if (unlikely(!--nr_loops)) { -- scx_bpf_kick_cpu(cpu_of(rq), 0); -- break; -- } -- } while (dspc->nr_tasks); -- -- return 0; --} -- --static int balance_scx(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf) --{ -- int ret; -- -- ret = balance_one(rq, prev, rf, true); --#ifdef CONFIG_SCHED_SMT -- /* -- * When core-sched is enabled, this ops.balance() call will be followed -- * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx() -- * on the SMT siblings. Balance the siblings too. -- */ -- if (sched_core_enabled(rq)) { -- const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); -- int scpu; -- -- for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { -- struct rq *srq = cpu_rq(scpu); -- struct rq_flags srf; -- struct task_struct *sprev = srq->curr; -- -- /* -- * While core-scheduling, rq lock is shared among -- * siblings but the debug annotations and rq clock -- * aren't. Do pinning dance to transfer the ownership. -- */ -- WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); -- rq_unpin_lock(rq, rf); -- rq_pin_lock(srq, &srf); -- -- update_rq_clock(srq); -- balance_one(srq, sprev, &srf, false); -- -- rq_unpin_lock(srq, &srf); -- rq_repin_lock(rq, rf); -- } -- } --#endif -- return ret; --} -- --static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) --{ -- if (p->scx->flags & SCX_TASK_QUEUED) { -- /* -- * Core-sched might decide to execute @p before it is -- * dispatched. Call ops_dequeue() to notify the BPF scheduler. -- */ -- ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); -- dispatch_dequeue(rq->scx, p); -- } -- -- p->se.exec_start = rq_clock_task(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, running, p); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock); -- -- watchdog_unwatch_task(p, true); -- -- /* -- * @p is getting newly scheduled or got kicked after someone updated its -- * slice. Refresh whether tick can be stopped. See can_stop_tick_scx(). -- */ -- if ((p->scx->slice == SCX_SLICE_INF) != -- (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) { -- if (p->scx->slice == SCX_SLICE_INF) -- rq->scx->flags |= SCX_RQ_CAN_STOP_TICK; -- else -- rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK; -- -- sched_update_tick_dependency(rq); -- } --} -- --static void put_prev_task_scx(struct rq *rq, struct task_struct *p) --{ --#ifndef CONFIG_SMP -- /* -- * UP workaround. -- * -- * Because SCX may transfer tasks across CPUs during dispatch, dispatch -- * is performed from its balance operation which isn't called in UP. -- * Let's work around by calling it from the operations which come right -- * after. -- * -- * 1. If the prev task is on SCX, pick_next_task() calls -- * .put_prev_task() right after. As .put_prev_task() is also called -- * from other places, we need to distinguish the calls which can be -- * done by looking at the previous task's state - if still queued or -- * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task(). -- * This case is handled here. -- * -- * 2. If the prev task is not on SCX, the first following call into SCX -- * will be .pick_next_task(), which is covered by calling -- * balance_scx() from pick_next_task_scx(). -- * -- * Note that we can't merge the first case into the second as -- * balance_scx() must be called before the previous SCX task goes -- * through put_prev_task_scx(). -- * -- * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf. -- * Pass in %NULL. -- */ -- if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP)) -- balance_scx(rq, p, NULL); --#endif -- -- update_curr_scx(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- -- /* -- * If we're being called from put_prev_task_balance(), balance_scx() may -- * have decided that @p should keep running. -- */ -- if (p->scx->flags & SCX_TASK_BAL_KEEP) { -- p->scx->flags &= ~SCX_TASK_BAL_KEEP; -- watchdog_watch_task(rq, p); -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- watchdog_watch_task(rq, p); -- -- /* -- * If @p has slice left and balance_scx() didn't tag it for -- * keeping, @p is getting preempted by a higher priority -- * scheduler class or core-sched forcing a different task. Leave -- * it at the head of the local DSQ. -- */ -- if (p->scx->slice && !scx_ops_disabling()) { -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- /* -- * If we're in the pick_next_task path, balance_scx() should -- * have already populated the local DSQ if there are any other -- * available tasks. If empty, tell ops.enqueue() that @p is the -- * only one available for this cpu. ops.enqueue() should put it -- * on the local DSQ so that the subsequent pick_next_task_scx() -- * can find the task unless it wants to trigger a separate -- * follow-up scheduling event. -- */ -- if (list_empty(&rq->scx->local_dsq.fifo)) -- do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1); -- else -- do_enqueue_task(rq, p, 0, -1); -- } --} -- --static struct task_struct *first_local_task(struct rq *rq) --{ -- struct rb_node *rb_node; -- struct sched_ext_entity *entity; -- -- if (!list_empty(&rq->scx->local_dsq.fifo)) { -- entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo); -- return entity->task; -- } -- -- rb_node = rb_first_cached(&rq->scx->local_dsq.priq); -- if (rb_node) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- return entity->task; -- } -- -- return NULL; --} -- --static struct task_struct *pick_next_task_scx(struct rq *rq) --{ -- struct task_struct *p; -- --#ifndef CONFIG_SMP -- /* UP workaround - see the comment at the head of put_prev_task_scx() */ -- if (unlikely(rq->curr->sched_class != &ext_sched_class)) -- balance_scx(rq, rq->curr, NULL); --#endif -- -- p = first_local_task(rq); -- if (!p) -- return NULL; -- -- if (unlikely(!p->scx->slice)) { -- if (!scx_ops_disabling() && !scx_warned_zero_slice) { -- printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n", -- p->comm, p->pid); -- scx_warned_zero_slice = true; -- } -- p->scx->slice = SCX_SLICE_DFL; -- } -- -- set_next_task_scx(rq, p, true); -- -- return p; --} -- --#ifdef CONFIG_SCHED_CORE --/** -- * scx_prio_less - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Core-sched is implemented as an additional scheduling layer on top of the -- * usual sched_class'es and needs to find out the expected task ordering. For -- * SCX, core-sched calls this function to interrogate the task ordering. -- * -- * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used -- * to implement the default task ordering. The older the timestamp, the higher -- * prority the task - the global FIFO ordering matching the default scheduling -- * behavior. -- * -- * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to -- * implement FIFO ordering within each local DSQ. See pick_task_scx(). -- */ --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi) --{ -- /* -- * The const qualifiers are dropped from task_struct pointers when -- * calling ops.core_sched_before(). Accesses are controlled by the -- * verifier. -- */ -- if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling()) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, -- (struct task_struct *)a, -- (struct task_struct *)b); -- else -- return time_after64(a->scx->core_sched_at, b->scx->core_sched_at); --} -- --/** -- * pick_task_scx - Pick a candidate task for core-sched -- * @rq: rq to pick the candidate task from -- * -- * Core-sched calls this function on each SMT sibling to determine the next -- * tasks to run on the SMT siblings. balance_one() has been called on all -- * siblings and put_prev_task_scx() has been called only for the current CPU. -- * -- * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look -- * at the first task in the local dsq. @rq->curr has to be considered explicitly -- * to mimic %SCX_TASK_BAL_KEEP. -- */ --static struct task_struct *pick_task_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- struct task_struct *first = first_local_task(rq); -- -- if (curr->scx->flags & SCX_TASK_QUEUED) { -- /* is curr the only runnable task? */ -- if (!first) -- return curr; -- -- /* -- * Does curr trump first? We can always go by core_sched_at for -- * this comparison as it represents global FIFO ordering when -- * the default core-sched ordering is used and local-DSQ FIFO -- * ordering otherwise. -- * -- * We can have a task with an earlier timestamp on the DSQ. For -- * example, when a current task is preempted by a sibling -- * picking a different cookie, the task would be requeued at the -- * head of the local DSQ with an earlier timestamp than the -- * core-sched picked next task. Besides, the BPF scheduler may -- * dispatch any tasks to the local DSQ anytime. -- */ -- if (curr->scx->slice && time_before64(curr->scx->core_sched_at, -- first->scx->core_sched_at)) -- return curr; -- } -- -- return first; /* this may be %NULL */ --} --#endif /* CONFIG_SCHED_CORE */ -- --static enum scx_cpu_preempt_reason --preempt_reason_from_class(const struct sched_class *class) --{ --#ifdef CONFIG_SMP -- if (class == &stop_sched_class) -- return SCX_CPU_PREEMPT_STOP; --#endif -- if (class == &dl_sched_class) -- return SCX_CPU_PREEMPT_DL; -- if (class == &rt_sched_class) -- return SCX_CPU_PREEMPT_RT; -- return SCX_CPU_PREEMPT_UNKNOWN; --} -- --void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task, -- const struct sched_class *active) --{ -- lockdep_assert_rq_held(rq); -- -- /* -- * The callback is conceptually meant to convey that the CPU is no -- * longer under the control of SCX. Therefore, don't invoke the -- * callback if the CPU is is staying on SCX, or going idle (in which -- * case the SCX scheduler has actively decided not to schedule any -- * tasks on the CPU). -- */ -- if (likely(active >= &ext_sched_class)) -- return; -- -- /* -- * At this point we know that SCX was preempted by a higher priority -- * sched_class, so invoke the ->cpu_release() callback if we have not -- * done so already. We only send the callback once between SCX being -- * preempted, and it regaining control of the CPU. -- * -- * ->cpu_release() complements ->cpu_acquire(), which is emitted the -- * next time that balance_scx() is invoked. -- */ -- if (!rq->scx->cpu_released) { -- if (SCX_HAS_OP(cpu_release)) { -- struct scx_cpu_release_args args = { -- .reason = preempt_reason_from_class(active), -- .task = task, -- }; -- -- SCX_CALL_OP(SCX_KF_CPU_RELEASE, -- cpu_release, cpu_of(rq), &args); -- } -- rq->scx->cpu_released = true; -- } --} -- --#ifdef CONFIG_SMP -- --static bool test_and_clear_cpu_idle(int cpu) --{ -- if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -- if (cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- return true; -- } else { -- return false; -- } --} -- --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- int cpu; -- -- do { -- cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); -- if (cpu < nr_cpu_ids) { -- const struct cpumask *sbm = topology_sibling_cpumask(cpu); -- -- /* -- * If offline, @cpu is not its own sibling and we can -- * get caught in an infinite loop as @cpu is never -- * cleared from idle_masks.smt. Clear @cpu directly in -- * such cases. -- */ -- if (likely(cpumask_test_cpu(cpu, sbm))) -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm); -- else -- cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu)); -- } else { -- cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -- if (cpu >= nr_cpu_ids) -- return -EBUSY; -- } -- } while (!test_and_clear_cpu_idle(cpu)); -- -- return cpu; --} -- --static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) --{ -- s32 cpu; -- -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return prev_cpu; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local DSQ of the waker. -- */ -- if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 && -- scx_has_idle_cpus && !(current->flags & PF_EXITING)) { -- cpu = smp_processor_id(); -- if (cpumask_test_cpu(cpu, p->cpus_ptr)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (test_and_clear_cpu_idle(prev_cpu)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return prev_cpu; -- } -- -- if (p->nr_cpus_allowed == 1) -- return prev_cpu; -- -- cpu = scx_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- -- return prev_cpu; --} -- --static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) --{ -- if (SCX_HAS_OP(select_cpu)) { -- s32 cpu; -- -- cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu, -- wake_flags); -- if (ops_cpu_valid(cpu)) { -- return cpu; -- } else { -- scx_ops_error("select_cpu returned invalid cpu %d", cpu); -- return prev_cpu; -- } -- } else { -- return scx_select_cpu_dfl(p, prev_cpu, wake_flags); -- } --} -- --static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx) --{ -- set_cpus_allowed_common(p, ctx); -- -- /* -- * The effective cpumask is stored in @p->cpus_ptr which may temporarily -- * differ from the configured one in @p->cpus_mask. Always tell the bpf -- * scheduler the effective one. -- * -- * Fine-grained memory write control is enforced by BPF making the const -- * designation pointless. Cast it away when calling the operation. -- */ -- if (SCX_HAS_OP(set_cpumask)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, -- (struct cpumask *)p->cpus_ptr); --} -- --static void reset_idle_masks(void) --{ -- /* consider all cpus idle, should converge to the actual state quickly */ -- cpumask_setall(idle_masks.cpu); -- cpumask_setall(idle_masks.smt); -- scx_has_idle_cpus = true; --} -- --void __scx_update_idle(struct rq *rq, bool idle) --{ -- int cpu = cpu_of(rq); -- struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -- -- if (SCX_HAS_OP(update_idle)) { -- SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); -- if (!static_branch_unlikely(&scx_builtin_idle_enabled)) -- return; -- } -- -- if (idle) { -- cpumask_set_cpu(cpu, idle_masks.cpu); -- if (!scx_has_idle_cpus) -- scx_has_idle_cpus = true; -- -- /* -- * idle_masks.smt handling is racy but that's fine as it's only -- * for optimization and self-correcting. -- */ -- for_each_cpu(cpu, sib_mask) { -- if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -- return; -- } -- cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -- } else { -- cpumask_clear_cpu(cpu, idle_masks.cpu); -- if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -- } --} -- --#else /* !CONFIG_SMP */ -- --static bool test_and_clear_cpu_idle(int cpu) { return false; } --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } --static void reset_idle_masks(void) {} -- --#endif /* CONFIG_SMP */ -- --static bool check_rq_for_timeouts(struct rq *rq) --{ -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rq_flags rf; -- bool timed_out = false; -- -- rq_lock_irqsave(rq, &rf); -- list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) { -- unsigned long last_runnable; -- -- p = entity->task; -- last_runnable = p->scx->runnable_at; -- -- if (unlikely(time_after(jiffies, -- last_runnable + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "%s[%d] failed to run for %u.%03us", -- p->comm, p->pid, -- dur_ms / 1000, dur_ms % 1000); -- timed_out = true; -- break; -- } -- } -- rq_unlock_irqrestore(rq, &rf); -- -- return timed_out; --} -- --static void scx_watchdog_workfn(struct work_struct *work) --{ -- int cpu; -- -- scx_watchdog_timestamp = jiffies; -- -- for_each_online_cpu(cpu) { -- if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) -- break; -- -- cond_resched(); -- } -- queue_delayed_work(system_unbound_wq, to_delayed_work(work), -- scx_watchdog_timeout / 2); --} -- --static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) --{ -- update_curr_scx(rq); -- //scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock); -- /* -- * While disabling, always resched and refresh core-sched timestamp as -- * we can't trust the slice management or ops.core_sched_before(). -- */ -- if (scx_ops_disabling()) { -- curr->scx->slice = 0; -- touch_core_sched(rq, curr); -- } -- -- if (!curr->scx->slice) -- resched_curr(rq); --} -- --#define SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- --static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg) --{ -- int ret; -- -- WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED); -- -- p->scx->disallow = false; -- -- if (SCX_HAS_OP(prep_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- }; -- -- ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args); -- if (unlikely(ret)) { -- ret = ops_sanitize_err("prep_enable", ret); -- return ret; -- } -- } -- //scx_sched_init_task(p); -- -- if (p->scx->disallow) { -- struct rq *rq; -- struct rq_flags rf; -- -- rq = task_rq_lock(p, &rf); -- -- /* -- * We're either in fork or load path and @p->policy will be -- * applied right after. Reverting @p->policy here and rejecting -- * %SCHED_EXT transitions from scx_check_setscheduler() -- * guarantees that if ops.prep_enable() sets @p->disallow, @p -- * can never be in SCX. -- */ -- if (p->policy == SCHED_EXT) { -- p->policy = SCHED_NORMAL; -- atomic64_inc(&scx_nr_rejected); -- } -- -- task_rq_unlock(rq, p, &rf); -- } -- -- p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET); -- return 0; --} -- --static void scx_ops_enable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED)); -- -- if (SCX_HAS_OP(enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- p->scx->flags |= SCX_TASK_OPS_ENABLED; --} -- --static void scx_ops_disable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- if (p->scx->flags & SCX_TASK_OPS_PREPPED) { -- if (SCX_HAS_OP(cancel_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- } else if (p->scx->flags & SCX_TASK_OPS_ENABLED) { -- if (SCX_HAS_OP(disable)) -- SCX_CALL_OP(SCX_KF_REST, disable, p); -- p->scx->flags &= ~SCX_TASK_OPS_ENABLED; -- } --} -- --static void set_task_scx_weight(struct task_struct *p) --{ -- u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -- -- p->scx->weight = sched_weight_to_cgroup(weight); --} -- --/** -- * refresh_scx_weight - Refresh a task's ext weight -- * @p: task to refresh ext weight for -- * -- * @p->scx->weight carries the task's static priority in cgroup weight scale to -- * enable easy access from the BPF scheduler. To keep it synchronized with the -- * current task priority, this function should be called when a new task is -- * created, priority is changed for a task on sched_ext, and a task is switched -- * to sched_ext from other classes. -- */ --static void refresh_scx_weight(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- set_task_scx_weight(p); -- if (SCX_HAS_OP(set_weight)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight); --} -- --void scx_pre_fork(struct task_struct *p) --{ -- p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL); -- if (!p->scx) { -- goto lock; -- } -- -- p->scx->dsq = NULL; -- INIT_LIST_HEAD(&p->scx->dsq_node.fifo); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- INIT_LIST_HEAD(&p->scx->watchdog_node); -- p->scx->flags = 0; -- p->scx->weight = 0; -- p->scx->sticky_cpu = -1; -- p->scx->holding_cpu = -1; -- p->scx->kf_mask = 0; -- atomic64_set(&p->scx->ops_state, 0); -- p->scx->runnable_at = INITIAL_JIFFIES; -- p->scx->slice = SCX_SLICE_DFL; -- p->scx->task = p; -- p->sched_prop = 0; -- -- /* -- * BPF scheduler enable/disable paths want to be able to iterate and -- * update all tasks which can become complex when racing forks. As -- * enable/disable are very cold paths, let's use a percpu_rwsem to -- * exclude forks. -- */ --lock: -- percpu_down_read(&scx_fork_rwsem); --} -- --int scx_fork(struct task_struct *p) --{ -- percpu_rwsem_assert_held(&scx_fork_rwsem); -- -- if (scx_enabled()) -- return scx_ops_prepare_task(p, task_group(p)); -- else -- return 0; --} -- --void scx_post_fork(struct task_struct *p) --{ -- if (scx_enabled()) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- /* -- * Set the weight manually before calling ops.enable() so that -- * the scheduler doesn't see a stale value if they inspect the -- * task struct. We'll invoke ops.set_weight() afterwards, as it -- * would be odd to receive a callback on the task before we -- * tell the scheduler that it's been fully enabled. -- */ -- set_task_scx_weight(p); -- scx_ops_enable_task(p); -- refresh_scx_weight(p); -- task_rq_unlock(rq, p, &rf); -- } -- -- spin_lock_irq(&scx_tasks_lock); -- list_add_tail(&p->scx->tasks_node, &scx_tasks); -- spin_unlock_irq(&scx_tasks_lock); -- -- percpu_up_read(&scx_fork_rwsem); --} -- --void scx_cancel_fork(struct task_struct *p) --{ -- if (scx_enabled()) -- scx_ops_disable_task(p); -- percpu_up_read(&scx_fork_rwsem); --} -- --void sched_ext_free(struct task_struct *p) --{ -- unsigned long flags; -- -- spin_lock_irqsave(&scx_tasks_lock, flags); -- list_del_init(&p->scx->tasks_node); -- spin_unlock_irqrestore(&scx_tasks_lock, flags); -- -- /* -- * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED -> -- * ENABLED transitions can't race us. Disable ops for @p. -- */ -- if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- scx_ops_disable_task(p); -- task_rq_unlock(rq, p, &rf); -- } --} -- --static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) --{ --} -- --static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} --static void switched_to_scx(struct rq *rq, struct task_struct *p) {} -- --int scx_check_setscheduler(struct task_struct *p, int policy) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- /* if disallow, reject transitioning into SCX */ -- if (scx_enabled() && READ_ONCE(p->scx->disallow) && -- p->policy != policy && policy == SCHED_EXT) -- return -EACCES; -- -- return 0; --} -- --#ifdef CONFIG_NO_HZ_FULL --bool scx_can_stop_tick(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (scx_ops_disabling()) -- return false; -- -- if (p->sched_class != &ext_sched_class) -- return true; -- -- /* -- * @rq can dispatch from different DSQs, so we can't tell whether it -- * needs the tick or not by looking at nr_running. Allow stopping ticks -- * iff the BPF scheduler indicated so. See set_next_task_scx(). -- */ -- return rq->scx->flags & SCX_RQ_CAN_STOP_TICK; --} --#endif -- --static inline void scx_cgroup_lock(void) {} --static inline void scx_cgroup_unlock(void) {} -- --/* -- * Omitted operations: -- * -- * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -- * task isn't tied to the CPU at that point. Preemption is implemented by -- * resetting the victim task's slice to 0 and triggering reschedule on the -- * target CPU. -- * -- * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -- * -- * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -- * their current sched_class. Call them directly from sched core instead. -- * -- * - task_woken, switched_from: Unnecessary. -- */ --DEFINE_SCHED_CLASS(ext) = { -- .enqueue_task = enqueue_task_scx, -- .dequeue_task = dequeue_task_scx, -- .yield_task = yield_task_scx, -- .yield_to_task = yield_to_task_scx, -- -- .check_preempt_curr = check_preempt_curr_scx, -- -- .pick_next_task = pick_next_task_scx, -- -- .put_prev_task = put_prev_task_scx, -- .set_next_task = set_next_task_scx, -- --#ifdef CONFIG_SMP -- .balance = balance_scx, -- .select_task_rq = select_task_rq_scx, -- .set_cpus_allowed = set_cpus_allowed_scx, --#endif -- --#ifdef CONFIG_SCHED_CORE -- .pick_task = pick_task_scx, --#endif -- -- .task_tick = task_tick_scx, -- -- .switched_to = switched_to_scx, -- .prio_changed = prio_changed_scx, -- -- .update_curr = update_curr_scx, -- --#ifdef CONFIG_UCLAMP_TASK -- .uclamp_enabled = 0, --#endif --}; -- --static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) --{ -- memset(dsq, 0, sizeof(*dsq)); -- -- raw_spin_lock_init(&dsq->lock); -- INIT_LIST_HEAD(&dsq->fifo); -- dsq->id = dsq_id; --} -- --static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id & SCX_DSQ_FLAG_BUILTIN) -- return ERR_PTR(-EINVAL); -- -- dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node); -- if (!dsq) -- return ERR_PTR(-ENOMEM); -- -- init_dsq(dsq, dsq_id); -- -- return dsq; --} -- --static void free_dsq_irq_workfn(struct irq_work *irq_work) --{ -- struct llist_node *to_free = llist_del_all(&dsqs_to_free); -- struct scx_dispatch_q *dsq, *tmp_dsq; -- -- llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) -- kfree_rcu(dsq, rcu); --} -- --static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); -- --static void destroy_dsq(u64 dsq_id) --{ --} -- --static void scx_cgroup_exit(void) {} --static int scx_cgroup_init(void) { return 0; } --static void scx_cgroup_config_knobs(void) {} -- --/* -- * Used by sched_fork() and __setscheduler_prio() to pick the matching -- * sched_class. dl/rt are already handled. -- */ --bool task_on_scx(struct task_struct *p) --{ -- if (!scx_enabled() || scx_ops_disabling()) -- return false; -- if (READ_ONCE(scx_switching_all)) -- return true; -- return p->policy == SCHED_EXT; --} -- --static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags) --{ -- if (enq_flags & SCX_ENQ_LAST) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); --} -- --static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {} -- --static void scx_ops_disable_workfn(struct kthread_work *work) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- struct scx_task_iter sti; -- struct task_struct *p; -- const char *reason; -- int i, cpu, type; -- -- type = atomic_read(&scx_exit_type); -- while (true) { -- /* -- * NONE indicates that a new scx_ops has been registered since -- * disable was scheduled - don't kill the new ops. DONE -- * indicates that the ops has already been disabled. -- */ -- if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE) -- return; -- if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE)) -- break; -- } -- -- cancel_delayed_work_sync(&scx_watchdog_work); -- -- switch (type) { -- case SCX_EXIT_UNREG: -- reason = "BPF scheduler unregistered"; -- break; -- case SCX_EXIT_SYSRQ: -- reason = "disabled by sysrq-S"; -- break; -- case SCX_EXIT_ERROR: -- reason = "runtime error"; -- break; -- case SCX_EXIT_ERROR_BPF: -- reason = "scx_bpf_error"; -- break; -- case SCX_EXIT_ERROR_STALL: -- reason = "runnable task stall"; -- break; -- default: -- reason = ""; -- } -- -- ei->type = type; -- strlcpy(ei->reason, reason, sizeof(ei->reason)); -- -- switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { -- case SCX_OPS_DISABLED: -- pr_warn("sched_ext: ops error detected without ops (%s)\n", -- scx_exit_info.msg); -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- return; -- case SCX_OPS_PREPPING: -- goto forward_progress_guaranteed; -- case SCX_OPS_DISABLING: -- /* shouldn't happen but handle it like ENABLING if it does */ -- WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); -- fallthrough; -- case SCX_OPS_ENABLING: -- case SCX_OPS_ENABLED: -- break; -- } -- -- /* -- * DISABLING is set and ops was either ENABLING or ENABLED indicating -- * that the ops and static branches are set. -- * -- * We must guarantee that all runnable tasks make forward progress -- * without trusting the BPF scheduler. We can't grab any mutexes or -- * rwsems as they might be held by tasks that the BPF scheduler is -- * forgetting to run, which unfortunately also excludes toggling the -- * static branches. -- * -- * Let's work around by overriding a couple ops and modifying behaviors -- * based on the DISABLING state and then cycling the tasks through -- * dequeue/enqueue to force global FIFO scheduling. -- * -- * a. ops.enqueue() and .dispatch() are overridden for simple global -- * FIFO scheduling. -- * -- * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value -- * can't be trusted. Whenever a tick triggers, the running task is -- * rotated to the tail of the queue with core_sched_at touched. -- * -- * c. pick_next_task() suppresses zero slice warning. -- * -- * d. scx_prio_less() reverts to the default core_sched_at order. -- */ -- scx_ops.enqueue = scx_ops_fallback_enqueue; -- scx_ops.dispatch = scx_ops_fallback_dispatch; -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- SCHED_CHANGE_BLOCK(task_rq(p), p, -- DEQUEUE_SAVE | DEQUEUE_MOVE) { -- /* cycling deq/enq is enough, see above */ -- } -- } -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* kick all CPUs to restore ticks */ -- for_each_possible_cpu(cpu) -- resched_cpu(cpu); -- --forward_progress_guaranteed: -- /* -- * Here, every runnable task is guaranteed to make forward progress and -- * we can safely use blocking synchronization constructs. Actually -- * disable ops. -- */ -- mutex_lock(&scx_ops_enable_mutex); -- -- static_branch_disable(&__scx_switched_all); -- WRITE_ONCE(scx_switching_all, false); -- -- /* avoid racing against fork and cgroup changes */ -- cpus_read_lock(); -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- bool alive = READ_ONCE(p->__state) != TASK_DEAD; -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL); -- -- __setscheduler_prio(p, p->prio); -- } -- -- if (alive) -- check_class_changed(task_rq(p), p, old_class, p->prio); -- -- scx_ops_disable_task(p); -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* no task is on scx, turn off all the switches and flush in-progress calls */ -- static_branch_disable_cpuslocked(&__scx_ops_enabled); -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- static_branch_disable_cpuslocked(&scx_has_op[i]); -- static_branch_disable_cpuslocked(&scx_ops_enq_last); -- static_branch_disable_cpuslocked(&scx_ops_enq_exiting); -- static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- synchronize_rcu(); -- -- scx_cgroup_exit(); -- -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- cpus_read_unlock(); -- -- if (ei->type >= SCX_EXIT_ERROR) { -- printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name); -- -- if (ei->msg[0] == '\0') -- printk(KERN_ERR "sched_ext: %s\n", ei->reason); -- else -- printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg); -- -- stack_trace_print(ei->bt, ei->bt_len, 2); -- } -- -- if (scx_ops.exit) -- SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); -- -- memset(&scx_ops, 0, sizeof(scx_ops)); -- -- free_percpu(scx_dsp_buf); -- scx_dsp_buf = NULL; -- scx_dsp_max_batch = 0; -- -- mutex_unlock(&scx_ops_enable_mutex); -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- -- scx_cgroup_config_knobs(); --} -- --static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); -- --static void schedule_scx_ops_disable_work(void) --{ -- struct kthread_worker *helper = READ_ONCE(scx_ops_helper); -- -- /* -- * We may be called spuriously before the first bpf_sched_ext_reg(). If -- * scx_ops_helper isn't set up yet, there's nothing to do. -- */ -- if (helper) -- kthread_queue_work(helper, &scx_ops_disable_work); --} -- --static void scx_ops_disable(enum scx_exit_type type) --{ -- int none = SCX_EXIT_NONE; -- -- if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)) -- type = SCX_EXIT_ERROR; -- -- atomic_try_cmpxchg(&scx_exit_type, &none, type); -- -- schedule_scx_ops_disable_work(); --} -- --static void scx_ops_error_irq_workfn(struct irq_work *irq_work) --{ -- schedule_scx_ops_disable_work(); --} -- --static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- int none = SCX_EXIT_NONE; -- va_list args; -- -- if (!atomic_try_cmpxchg(&scx_exit_type, &none, type)) -- return; -- -- ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1); -- -- va_start(args, fmt); -- vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args); -- va_end(args); -- -- irq_work_queue(&scx_ops_error_irq_work); --} -- --static struct kthread_worker *scx_create_rt_helper(const char *name) --{ -- struct kthread_worker *helper; -- -- helper = kthread_create_worker(0, name); -- if (helper) -- sched_set_fifo(helper->task); -- return helper; --} -- --static int scx_ops_enable(struct sched_ext_ops *ops) --{ -- struct scx_task_iter sti; -- struct task_struct *p; -- int i, ret; -- int tcnt = 0; -- unsigned long long start = 0; -- unsigned long long total_start = sched_clock(); -- -- mutex_lock(&scx_ops_enable_mutex); -- -- if (!scx_ops_helper) { -- WRITE_ONCE(scx_ops_helper, -- scx_create_rt_helper("sched_ext_ops_helper")); -- if (!scx_ops_helper) { -- ret = -ENOMEM; -- goto err_unlock; -- } -- } -- -- if (scx_ops_enable_state() != SCX_OPS_DISABLED) { -- ret = -EBUSY; -- goto err_unlock; -- } -- -- //slim_walt_enable(true); -- -- /* -- * Set scx_ops, transition to PREPPING and clear exit info to arm the -- * disable path. Failure triggers full disabling from here on. -- */ -- scx_ops = *ops; -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != -- SCX_OPS_DISABLED); -- -- memset(&scx_exit_info, 0, sizeof(scx_exit_info)); -- atomic_set(&scx_exit_type, SCX_EXIT_NONE); -- scx_warned_zero_slice = false; -- -- atomic64_set(&scx_nr_rejected, 0); -- -- /* -- * Keep CPUs stable during enable so that the BPF scheduler can track -- * online CPUs by watching ->on/offline_cpu() after ->init(). -- */ -- cpus_read_lock(); -- -- scx_switch_all_req = true; -- if (scx_ops.init) { -- ret = SCX_CALL_OP_RET(SCX_KF_INIT, init); -- if (ret) { -- ret = ops_sanitize_err("init", ret); -- goto err_disable; -- } -- -- /* -- * Exit early if ops.init() triggered scx_bpf_error(). Not -- * strictly necessary as we'll fail transitioning into ENABLING -- * later but that'd be after calling ops.prep_enable() on all -- * tasks and with -EBUSY which isn't very intuitive. Let's exit -- * early with success so that the condition is notified through -- * ops.exit() like other scx_bpf_error() invocations. -- */ -- if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE) -- goto err_disable; -- } -- -- WARN_ON_ONCE(scx_dsp_buf); -- scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; -- scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch, -- __alignof__(scx_dsp_buf[0])); -- if (!scx_dsp_buf) { -- ret = -ENOMEM; -- goto err_disable; -- } -- -- scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; -- if (ops->timeout_ms) -- scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); -- -- scx_watchdog_timestamp = jiffies; -- queue_delayed_work(system_unbound_wq, &scx_watchdog_work, -- scx_watchdog_timeout / 2); -- -- /* -- * Lock out forks, cgroup on/offlining and moves before opening the -- * floodgate so that they don't wander into the operations prematurely. -- */ -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- if (((void (**)(void))ops)[i]) -- static_branch_enable_cpuslocked(&scx_has_op[i]); -- -- if (ops->flags & SCX_OPS_ENQ_LAST) -- static_branch_enable_cpuslocked(&scx_ops_enq_last); -- -- if (ops->flags & SCX_OPS_ENQ_EXITING) -- static_branch_enable_cpuslocked(&scx_ops_enq_exiting); -- if (scx_ops.cpu_acquire || scx_ops.cpu_release) -- static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); -- -- if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { -- reset_idle_masks(); -- static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); -- } else { -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- } -- -- /* -- * All cgroups should be initialized before letting in tasks. cgroup -- * on/offlining and task migrations are already locked out. -- */ -- ret = scx_cgroup_init(); -- if (ret) -- goto err_disable_unlock; -- -- static_branch_enable_cpuslocked(&__scx_ops_enabled); -- -- /* -- * Enable ops for every task. Fork is excluded by scx_fork_rwsem -- * preventing new tasks from being added. No need to exclude tasks -- * leaving as sched_ext_free() can handle both prepped and enabled -- * tasks. Prep all tasks first and then enable them with preemption -- * disabled. -- */ -- spin_lock_irq(&scx_tasks_lock); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered(&sti))) { -- get_task_struct(p); -- spin_unlock_irq(&scx_tasks_lock); -- -- ret = scx_ops_prepare_task(p, task_group(p)); -- if (ret) { -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n", -- ret, p->comm, p->pid); -- goto err_disable_unlock; -- } -- -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- } -- scx_task_iter_exit(&sti); -- -- /* -- * All tasks are prepped but are still ops-disabled. Ensure that -- * %current can't be scheduled out and switch everyone. -- * preempt_disable() is necessary because we can't guarantee that -- * %current won't be starved if scheduled out while switching. -- */ -- preempt_disable(); -- -- /* -- * From here on, the disable path must assume that tasks have ops -- * enabled and need to be recovered. -- */ -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { -- preempt_enable(); -- spin_unlock_irq(&scx_tasks_lock); -- ret = -EBUSY; -- goto err_disable_unlock; -- } -- -- /* -- * We're fully committed and can't fail. The PREPPED -> ENABLED -- * transitions here are synchronized against sched_ext_free() through -- * scx_tasks_lock. -- */ -- WRITE_ONCE(scx_switching_all, scx_switch_all_req); -- start = sched_clock(); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- tcnt++; -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- scx_ops_enable_task(p); -- __setscheduler_prio(p, p->prio); -- } -- -- check_class_changed(task_rq(p), p, old_class, p->prio); -- } else { -- scx_ops_disable_task(p); -- } -- } -- printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id()); -- scx_task_iter_exit(&sti); -- -- spin_unlock_irq(&scx_tasks_lock); -- preempt_enable(); -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { -- ret = -EBUSY; -- goto err_disable; -- } -- -- if (scx_switch_all_req) -- static_branch_enable_cpuslocked(&__scx_switched_all); -- -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- -- scx_cgroup_config_knobs(); -- printk("\n total duration = %llu , current cpu = %d\n", sched_clock() - total_start, smp_processor_id()); -- -- return 0; -- --err_unlock: -- mutex_unlock(&scx_ops_enable_mutex); -- return ret; -- --err_disable_unlock: -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); --err_disable: -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- /* must be fully disabled before returning */ -- scx_ops_disable(SCX_EXIT_ERROR); -- kthread_flush_work(&scx_ops_disable_work); -- return ret; --} -- --#ifdef CONFIG_SCHED_DEBUG --static const char *scx_ops_enable_state_str[] = { -- [SCX_OPS_PREPPING] = "prepping", -- [SCX_OPS_ENABLING] = "enabling", -- [SCX_OPS_ENABLED] = "enabled", -- [SCX_OPS_DISABLING] = "disabling", -- [SCX_OPS_DISABLED] = "disabled", --}; -- --static int scx_debug_show(struct seq_file *m, void *v) --{ -- mutex_lock(&scx_ops_enable_mutex); -- seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name); -- seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled()); -- seq_printf(m, "%-30s: %d\n", "switching_all", -- READ_ONCE(scx_switching_all)); -- seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all()); -- seq_printf(m, "%-30s: %s\n", "enable_state", -- scx_ops_enable_state_str[scx_ops_enable_state()]); -- seq_printf(m, "%-30s: %llu\n", "nr_rejected", -- atomic64_read(&scx_nr_rejected)); -- mutex_unlock(&scx_ops_enable_mutex); -- return 0; --} -- --static int scx_debug_open(struct inode *inode, struct file *file) --{ -- return single_open(file, scx_debug_show, NULL); --} -- --const struct file_operations sched_ext_fops = { -- .open = scx_debug_open, -- .read = seq_read, -- .llseek = seq_lseek, -- .release = single_release, --}; --#endif -- --/******************************************************************************** -- * bpf_struct_ops plumbing. -- */ --#include --#include --#include -- --extern struct btf *btf_vmlinux; --static const struct btf_type *task_struct_type; -- --static bool bpf_scx_is_valid_access(int off, int size, -- enum bpf_access_type type, -- const struct bpf_prog *prog, -- struct bpf_insn_access_aux *info) --{ -- if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) -- return false; -- if (type != BPF_READ) -- return false; -- if (off % size != 0) -- return false; -- -- return btf_ctx_access(off, size, type, prog, info); --} -- --static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, -- const struct bpf_reg_state *reg, -- int off, int size) --{ -- return 0; --} -- --static const struct bpf_func_proto * --bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) --{ -- switch (func_id) { -- case BPF_FUNC_task_storage_get: -- return &bpf_task_storage_get_proto; -- case BPF_FUNC_task_storage_delete: -- return &bpf_task_storage_delete_proto; -- default: -- return bpf_base_func_proto(func_id); -- } --} -- --const struct bpf_verifier_ops bpf_scx_verifier_ops = { -- .get_func_proto = bpf_scx_get_func_proto, -- .is_valid_access = bpf_scx_is_valid_access, -- .btf_struct_access = bpf_scx_btf_struct_access, --}; -- --static int bpf_scx_init_member(const struct btf_type *t, -- const struct btf_member *member, -- void *kdata, const void *udata) --{ -- const struct sched_ext_ops *uops = udata; -- struct sched_ext_ops *ops = kdata; -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- int ret; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, dispatch_max_batch): -- if (*(u32 *)(udata + moff) > INT_MAX) -- return -E2BIG; -- ops->dispatch_max_batch = *(u32 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, flags): -- if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) -- return -EINVAL; -- ops->flags = *(u64 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, name): -- ret = bpf_obj_name_cpy(ops->name, uops->name, -- sizeof(ops->name)); -- if (ret < 0) -- return ret; -- if (ret == 0) -- return -EINVAL; -- return 1; -- case offsetof(struct sched_ext_ops, timeout_ms): -- if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT) -- return -E2BIG; -- ops->timeout_ms = *(u32 *)(udata + moff); -- return 1; -- } -- -- return 0; --} -- --static int bpf_scx_check_member(const struct btf_type *t, -- const struct btf_member *member, -- const struct bpf_prog *prog) --{ -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, prep_enable): -- case offsetof(struct sched_ext_ops, init): -- case offsetof(struct sched_ext_ops, exit): -- break; -- default: -- /* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */ -- /* if (prog->aux->sleepable) -- return -EINVAL; */ -- break; -- } -- -- return 0; --} -- --static int bpf_scx_reg(void *kdata) --{ -- return scx_ops_enable(kdata); --} -- --static void bpf_scx_unreg(void *kdata) --{ -- scx_ops_disable(SCX_EXIT_UNREG); -- kthread_flush_work(&scx_ops_disable_work); --} -- --static int bpf_scx_init(struct btf *btf) --{ -- u32 type_id; -- -- type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); -- if (type_id < 0) -- return -EINVAL; -- task_struct_type = btf_type_by_id(btf, type_id); -- -- return 0; --} -- --/* "extern" to avoid sparse warning, only used in this file */ --extern struct bpf_struct_ops bpf_sched_ext_ops; -- --struct bpf_struct_ops bpf_sched_ext_ops = { -- .verifier_ops = &bpf_scx_verifier_ops, -- .reg = bpf_scx_reg, -- .unreg = bpf_scx_unreg, -- .check_member = bpf_scx_check_member, -- .init_member = bpf_scx_init_member, -- .init = bpf_scx_init, -- .name = "sched_ext_ops", --}; -- --static void sysrq_handle_sched_ext_reset(u8 key) --{ -- if (scx_ops_helper) -- scx_ops_disable(SCX_EXIT_SYSRQ); -- else -- pr_info("sched_ext: BPF scheduler not yet used\n"); --} -- --static const struct sysrq_key_op sysrq_sched_ext_reset_op = { -- .handler = sysrq_handle_sched_ext_reset, -- .help_msg = "reset-sched-ext(S)", -- .action_msg = "Disable sched_ext and revert all tasks to CFS", -- .enable_mask = SYSRQ_ENABLE_RTNICE, --}; -- --static void kick_cpus_irq_workfn(struct irq_work *irq_work) --{ -- struct rq *this_rq = this_rq(); -- u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); -- int this_cpu = cpu_of(this_rq); -- int cpu; -- -- for_each_cpu(cpu, this_rq->scx->cpus_to_kick) { -- struct rq *rq = cpu_rq(cpu); -- unsigned long flags; -- -- raw_spin_rq_lock_irqsave(rq, flags); -- -- if (cpu_online(cpu) || cpu == this_cpu) { -- if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) && -- rq->curr->sched_class == &ext_sched_class) -- rq->curr->scx->slice = 0; -- pseqs[cpu] = rq->scx->pnt_seq; -- resched_curr(rq); -- } else { -- cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait); -- } -- -- raw_spin_rq_unlock_irqrestore(rq, flags); -- } -- -- for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait, -- cpumask_of(this_cpu)) { -- /* -- * Pairs with smp_store_release() issued by this CPU in -- * scx_notify_pick_next_task() on the resched path. -- * -- * We busy-wait here to guarantee that no other task can be -- * scheduled on our core before the target CPU has entered the -- * resched path. -- */ -- while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu]) -- cpu_relax(); -- } -- -- cpumask_clear(this_rq->scx->cpus_to_kick); -- cpumask_clear(this_rq->scx->cpus_to_preempt); -- cpumask_clear(this_rq->scx->cpus_to_wait); --} -- --void __init init_sched_ext_class(void) --{ -- int cpu; -- u32 v; -- -- /* -- * The following is to prevent the compiler from optimizing out the enum -- * definitions so that BPF scheduler implementations can use them -- * through the generated vmlinux.h. -- */ -- WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | -- SCX_TG_ONLINE | SCX_KICK_PREEMPT); -- -- init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); --#ifdef CONFIG_SMP -- BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -- BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); --#endif -- scx_kick_cpus_pnt_seqs = -- __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * -- num_possible_cpus(), -- __alignof__(scx_kick_cpus_pnt_seqs[0])); -- BUG_ON(!scx_kick_cpus_pnt_seqs); -- -- for_each_possible_cpu(cpu) { -- struct rq *rq = cpu_rq(cpu); -- -- rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL); -- if (rq->scx) -- rq->scx->rq = rq; -- else -- pr_err("fatal error : alloc rq->scx failed!!!\n"); -- -- init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL); -- INIT_LIST_HEAD(&rq->scx->watchdog_list); -- -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL)); -- init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn); -- } -- -- register_sysrq_key('S', &sysrq_sched_ext_reset_op); -- INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); -- scx_cgroup_config_knobs(); --} -- -- --/******************************************************************************** -- * Helpers that can be called from the BPF scheduler. -- */ --#include -- --/* Disables missing prototype warnings for kfuncs */ --__diag_push(); --__diag_ignore_all("-Wmissing-prototypes", -- "Global functions as their definitions will be in vmlinux BTF"); -- --/** -- * scx_bpf_switch_all - Switch all tasks into SCX -- * @into_scx: switch direction -- * -- * If @into_scx is %true, all existing and future non-dl/rt tasks are switched -- * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on -- * SCX. The actual switching is asynchronous. Can be called from ops.init(). -- */ --void scx_bpf_switch_all(void) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return; -- -- scx_switch_all_req = true; --} -- --BTF_SET8_START(scx_kfunc_ids_init) --BTF_ID_FLAGS(func, scx_bpf_switch_all) --BTF_SET8_END(scx_kfunc_ids_init) -- --static const struct btf_kfunc_id_set scx_kfunc_set_init = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_init, --}; -- --/** -- * scx_bpf_create_dsq - Create a custom DSQ -- * @dsq_id: DSQ to create -- * @node: NUMA node to allocate from -- * -- * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(), -- * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move(). -- */ --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return -EINVAL; -- -- if (unlikely(node >= (int)nr_node_ids || -- (node < 0 && node != NUMA_NO_NODE))) -- return -EINVAL; -- return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); --} -- --BTF_SET8_START(scx_kfunc_ids_sleepable) --BTF_ID_FLAGS(func, scx_bpf_create_dsq) --BTF_SET8_END(scx_kfunc_ids_sleepable) -- --static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_sleepable, --}; -- --static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) --{ -- if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) -- return false; -- -- lockdep_assert_irqs_disabled(); -- -- if (unlikely(!p)) { -- scx_ops_error("called with NULL task"); -- return false; -- } -- -- if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { -- scx_ops_error("invalid enq_flags 0x%llx", enq_flags); -- return false; -- } -- -- return true; --} -- --static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) --{ -- struct task_struct *ddsp_task; -- int idx; -- -- ddsp_task = __this_cpu_read(direct_dispatch_task); -- if (ddsp_task) { -- direct_dispatch(ddsp_task, p, dsq_id, enq_flags); -- return; -- } -- -- idx = __this_cpu_read(scx_dsp_ctx.buf_cursor); -- if (unlikely(idx >= scx_dsp_max_batch)) { -- scx_ops_error("dispatch buffer overflow"); -- return; -- } -- -- this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){ -- .task = p, -- .qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK, -- .dsq_id = dsq_id, -- .enq_flags = enq_flags, -- }; -- __this_cpu_inc(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe -- * to call this function spuriously. Can be called from ops.enqueue() and -- * ops.dispatch(). -- * -- * When called from ops.enqueue(), it's for direct dispatch and @p must match -- * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the -- * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be -- * on the target CPU in the first place. -- * -- * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id -- * and this function can be called upto ops.dispatch_max_batch times to dispatch -- * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the -- * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. -- * -- * This function doesn't have any locking restrictions and may be called under -- * BPF locks (in the future when BPF introduces more flexible locking). -- * -- * @p is allowed to run for @slice. The scheduling path is triggered on slice -- * exhaustion. If zero, the current residual slice is maintained. If -- * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with -- * scx_bpf_kick_cpu() to trigger scheduling. -- */ --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- scx_dispatch_commit(p, dsq_id, enq_flags); --} -- --/** -- * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. -- * Tasks queued into the priority queue are ordered by @vtime and always -- * consumed after the tasks in the FIFO queue. All other aspects are identical -- * to scx_bpf_dispatch(). -- * -- * @vtime ordering is according to time_before64() which considers wrapping. A -- * numerically larger vtime may indicate an earlier position in the ordering and -- * vice-versa. -- */ --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 vtime, u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- p->scx->dsq_vtime = vtime; -- -- scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); --} -- --BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime) --BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_enqueue_dispatch, --}; -- --/** -- * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots -- * -- * Can only be called from ops.dispatch(). -- */ --u32 scx_bpf_dispatch_nr_slots(void) --{ -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return 0; -- -- return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ -- * @dsq_id: DSQ to consume -- * -- * Consume a task from the non-local DSQ identified by @dsq_id and transfer it -- * to the current CPU's local DSQ for execution. Can only be called from -- * ops.dispatch(). -- * -- * This function flushes the in-flight dispatches from scx_bpf_dispatch() before -- * trying to consume the specified DSQ. It may also grab rq locks and thus can't -- * be called under any BPF locks. -- * -- * Returns %true if a task has been consumed, %false if there isn't any task to -- * consume. -- */ --bool scx_bpf_consume(u64 dsq_id) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- struct scx_dispatch_q *dsq; -- -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return false; -- -- flush_dispatch_buf(dspc->rq, dspc->rf); -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); -- return false; -- } -- -- if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) { -- /* -- * A successfully consumed task can be dequeued before it starts -- * running while the CPU is trying to migrate other dispatched -- * tasks. Bump nr_tasks to tell balance_scx() to retry on empty -- * local DSQ. -- */ -- dspc->nr_tasks++; -- return true; -- } else { -- return false; -- } --} -- --BTF_SET8_START(scx_kfunc_ids_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) --BTF_ID_FLAGS(func, scx_bpf_consume) --BTF_SET8_END(scx_kfunc_ids_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_dispatch, --}; -- --/** -- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ -- * -- * Iterate over all of the tasks currently enqueued on the local DSQ of the -- * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of -- * processed tasks. Can only be called from ops.cpu_release(). -- */ --u32 scx_bpf_reenqueue_local(void) --{ -- u32 nr_enqueued, i; -- struct rq *rq; -- struct scx_rq *scx_rq; -- -- if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) -- return 0; -- -- rq = cpu_rq(smp_processor_id()); -- lockdep_assert_rq_held(rq); -- scx_rq = rq->scx; -- -- /* -- * Get the number of tasks on the local DSQ before iterating over it to -- * pull off tasks. The enqueue callback below can signal that it wants -- * the task to stay on the local DSQ, and we want to prevent the BPF -- * scheduler from causing us to loop indefinitely. -- */ -- nr_enqueued = scx_rq->local_dsq.nr; -- for (i = 0; i < nr_enqueued; i++) { -- struct task_struct *p; -- -- p = first_local_task(rq); -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- WARN_ON_ONCE(p->scx->holding_cpu != -1); -- dispatch_dequeue(scx_rq, p); -- do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); -- } -- -- return nr_enqueued; --} -- --BTF_SET8_START(scx_kfunc_ids_cpu_release) --BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) --BTF_SET8_END(scx_kfunc_ids_cpu_release) -- --static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_cpu_release, --}; -- --/** -- * scx_bpf_kick_cpu - Trigger reschedule on a CPU -- * @cpu: cpu to kick -- * @flags: SCX_KICK_* flags -- * -- * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or -- * trigger rescheduling on a busy CPU. This can be called from any online -- * scx_ops operation and the actual kicking is performed asynchronously through -- * an irq work. -- */ --void scx_bpf_kick_cpu(s32 cpu, u64 flags) --{ -- struct rq *rq; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d", cpu); -- return; -- } -- -- preempt_disable(); -- rq = this_rq(); -- -- /* -- * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting -- * rq locks. We can probably be smarter and avoid bouncing if called -- * from ops which don't hold a rq lock. -- */ -- cpumask_set_cpu(cpu, rq->scx->cpus_to_kick); -- if (flags & SCX_KICK_PREEMPT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt); -- if (flags & SCX_KICK_WAIT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_wait); -- -- irq_work_queue(&rq->scx->kick_cpus_irq_work); -- preempt_enable(); --} -- --/** -- * scx_bpf_dsq_nr_queued - Return the number of queued tasks -- * @dsq_id: id of the DSQ -- * -- * Return the number of tasks in the DSQ matching @dsq_id. If not found, -- * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops -- * operations. -- */ --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) --{ -- struct scx_dispatch_q *dsq; -- -- lockdep_assert(rcu_read_lock_any_held()); -- -- if (dsq_id == SCX_DSQ_LOCAL) { -- return this_rq()->scx->local_dsq.nr; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (ops_cpu_valid(cpu)) -- return cpu_rq(cpu)->scx->local_dsq.nr; -- } else { -- dsq = find_non_local_dsq(dsq_id); -- if (dsq) -- return dsq->nr; -- } -- return -ENOENT; --} -- --/** -- * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state -- * @cpu: cpu to test and clear idle for -- * -- * Returns %true if @cpu was idle and its idle state was successfully cleared. -- * %false otherwise. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return false; -- } -- -- if (ops_cpu_valid(cpu)) -- return test_and_clear_cpu_idle(cpu); -- else -- return false; --} -- --/** -- * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu -- * @cpus_allowed: Allowed cpumask -- * -- * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked -- * idle cpu number on success. -%EBUSY if no matching cpu was found. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return -EBUSY; -- } -- -- return scx_pick_idle_cpu(cpus_allowed); --} -- --/** -- * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking -- * per-CPU cpumask. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_cpumask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.cpu; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, -- * per-physical-core cpumask. Can be used to determine if an entire physical -- * core is free. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_smtmask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.smt; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to -- * either the percpu, or SMT idle-tracking cpumask. -- */ --void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) --{ -- /* -- * Empty function body because we aren't actually acquiring or -- * releasing a reference to a global idle cpumask, which is read-only -- * in the caller and is never released. The acquire / release semantics -- * here are just used to make the cpumask is a trusted pointer in the -- * caller. -- */ --} -- --struct scx_bpf_error_bstr_bufs { -- u64 data[MAX_BPRINTF_VARARGS]; -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs); -- --/** -- * scx_bpf_error_bstr - Indicate fatal error -- * @fmt: error message format string -- * @data: format string parameters packaged using ___bpf_fill() macro -- * @data__sz: @data len, must end in '__sz' for the verifier -- * -- * Indicate that the BPF scheduler encountered a fatal error and initiate ops -- * disabling. -- */ --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) --{ -- struct scx_bpf_error_bstr_bufs *bufs; -- unsigned long flags; -- struct bpf_bprintf_data args = {}; -- int ret; -- -- local_irq_save(flags); -- bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs); -- -- if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || -- (data__sz && !data)) { -- scx_ops_error("invalid data=%p and data__sz=%u", -- (void *)data, data__sz); -- goto out_restore; -- } -- -- ret = copy_from_kernel_nofault(bufs->data, data, data__sz); -- if (ret) { -- scx_ops_error("failed to read data fields (%d)", ret); -- goto out_restore; -- } -- -- ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args); -- if (ret < 0) { -- scx_ops_error("failed to format prepration (%d)", ret); -- goto out_restore; -- } -- -- ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt, -- args.bin_args); -- bpf_bprintf_cleanup(&args); -- if (ret < 0) { -- scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format", -- fmt, data, data__sz); -- goto out_restore; -- } -- -- scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg); --out_restore: -- local_irq_restore(flags); --} -- --/** -- * scx_bpf_destroy_dsq - Destroy a custom DSQ -- * @dsq_id: DSQ to destroy -- * -- * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with -- * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is -- * empty and no further tasks are dispatched to it. Ignored if called on a DSQ -- * which doesn't exist. Can be called from any online scx_ops operations. -- */ --void scx_bpf_destroy_dsq(u64 dsq_id) --{ -- destroy_dsq(dsq_id); --} -- --/** -- * scx_bpf_task_running - Is task currently running? -- * @p: task of interest -- */ --bool scx_bpf_task_running(const struct task_struct *p) --{ -- return task_rq(p)->curr == p; --} -- --/** -- * scx_bpf_task_cpu - CPU a task is currently associated with -- * @p: task of interest -- */ --s32 scx_bpf_task_cpu(const struct task_struct *p) --{ -- return task_cpu(p); --} -- --/** -- * scx_bpf_task_cgroup - Return the sched cgroup of a task -- * @p: task of interest -- * -- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with -- * from the scheduler's POV. SCX operations should use this function to -- * determine @p's current cgroup as, unlike following @p->cgroups, -- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all -- * rq-locked operations. Can be called on the parameter tasks of rq-locked -- * operations. The restriction guarantees that @p's rq is locked by the caller. -- */ --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) --{ -- struct task_group *tg = p->sched_task_group; -- struct cgroup *cgrp = &cgrp_dfl_root.cgrp; -- -- if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) -- goto out; -- -- /* -- * A task_group may either be a cgroup or an autogroup. In the latter -- * case, @tg->css.cgroup is %NULL. A task_group can't become the other -- * kind once created. -- */ -- if (tg && tg->css.cgroup) -- cgrp = tg->css.cgroup; -- else -- cgrp = &cgrp_dfl_root.cgrp; --out: -- cgroup_get(cgrp); -- return cgrp; --} -- --BTF_SET8_START(scx_kfunc_ids_any) --BTF_ID_FLAGS(func, scx_bpf_kick_cpu) --BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) --BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) --BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu) --BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) --BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) --BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) --BTF_ID_FLAGS(func, scx_bpf_task_running) --BTF_ID_FLAGS(func, scx_bpf_task_cpu) --BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE) --BTF_SET8_END(scx_kfunc_ids_any) -- --static const struct btf_kfunc_id_set scx_kfunc_set_any = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_any, --}; -- --__diag_pop(); -- --/* -- * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set() -- * needs most of the system to be up. -- */ --static int __init register_ext_kfuncs(void) --{ -- int ret; -- -- /* -- * Some kfuncs are context-sensitive and can only be called from -- * specific SCX ops. They are grouped into BTF sets accordingly. -- * Unfortunately, BPF currently doesn't have a way of enforcing such -- * restrictions. Eventually, the verifier should be able to enforce -- * them. For now, register them the same and make each kfunc explicitly -- * check using scx_kf_allowed(). -- */ -- if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_init)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_sleepable)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_enqueue_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_cpu_release)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_any))) { -- pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret); -- return ret; -- } -- -- return 0; --} --__initcall(register_ext_kfuncs); -diff --git b/kernel/sched/ext.h a/kernel/sched/ext.h -deleted file mode 100755 -index fba8ed845f99..000000000000 ---- b/kernel/sched/ext.h -+++ /dev/null -@@ -1,257 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ -- -- --/* -- * Tag marking a kernel function as a kfunc. This is meant to minimize the -- * amount of copy-paste that kfunc authors have to include for correctness so -- * as to avoid issues such as the compiler inlining or eliding either a static -- * kfunc, or a global kfunc in an LTO build. -- */ --#define __bpf_kfunc __used noinline -- --enum scx_wake_flags { -- /* expose select WF_* flags as enums */ -- SCX_WAKE_EXEC = WF_EXEC, -- SCX_WAKE_FORK = WF_FORK, -- SCX_WAKE_TTWU = WF_TTWU, -- SCX_WAKE_SYNC = WF_SYNC, --}; -- --enum scx_enq_flags { -- /* expose select ENQUEUE_* flags as enums */ -- SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, -- SCX_ENQ_HEAD = ENQUEUE_HEAD, -- -- /* high 32bits are SCX specific */ -- -- /* -- * Set the following to trigger preemption when calling -- * scx_bpf_dispatch() with a local dsq as the target. The slice of the -- * current task is cleared to zero and the CPU is kicked into the -- * scheduling path. Implies %SCX_ENQ_HEAD. -- */ -- SCX_ENQ_PREEMPT = 1LLU << 32, -- -- /* -- * The task being enqueued was previously enqueued on the current CPU's -- * %SCX_DSQ_LOCAL, but was removed from it in a call to the -- * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was -- * invoked in a ->cpu_release() callback, and the task is again -- * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the -- * task will not be scheduled on the CPU until at least the next invocation -- * of the ->cpu_acquire() callback. -- */ -- SCX_ENQ_REENQ = 1LLU << 40, -- -- /* -- * The task being enqueued is the only task available for the cpu. By -- * default, ext core keeps executing such tasks but when -- * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with -- * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set. -- * -- * If the BPF scheduler wants to continue executing the task, -- * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately. -- * If the task gets queued on a different dsq or the BPF side, the BPF -- * scheduler is responsible for triggering a follow-up scheduling event. -- * Otherwise, Execution may stall. -- */ -- SCX_ENQ_LAST = 1LLU << 41, -- -- /* -- * A hint indicating that it's advisable to enqueue the task on the -- * local dsq of the currently selected CPU. Currently used by -- * select_cpu_dfl() and together with %SCX_ENQ_LAST. -- */ -- SCX_ENQ_LOCAL = 1LLU << 42, -- -- /* high 8 bits are internal */ -- __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, -- -- SCX_ENQ_CLEAR_OPSS = 1LLU << 56, -- SCX_ENQ_DSQ_PRIQ = 1LLU << 57, --}; -- --enum scx_deq_flags { -- /* expose select DEQUEUE_* flags as enums */ -- SCX_DEQ_SLEEP = DEQUEUE_SLEEP, -- -- /* high 32bits are SCX specific */ -- -- /* -- * The generic core-sched layer decided to execute the task even though -- * it hasn't been dispatched yet. Dequeue from the BPF side. -- */ -- SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, --}; -- --enum scx_tg_flags { -- SCX_TG_ONLINE = 1U << 0, -- SCX_TG_INITED = 1U << 1, --}; -- --enum scx_kick_flags { -- SCX_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -- SCX_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ --}; -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --extern const struct sched_class ext_sched_class; --extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops; --extern const struct file_operations sched_ext_fops; --extern unsigned long scx_watchdog_timeout; --extern unsigned long scx_watchdog_timestamp; -- --DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); --DECLARE_STATIC_KEY_FALSE(__scx_switched_all); --#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) --#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) -- --DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); -- --bool task_on_scx(struct task_struct *p); --void scx_pre_fork(struct task_struct *p); --int scx_fork(struct task_struct *p); --void scx_post_fork(struct task_struct *p); --void scx_cancel_fork(struct task_struct *p); --int scx_check_setscheduler(struct task_struct *p, int policy); --bool scx_can_stop_tick(struct rq *rq); --void init_sched_ext_class(void); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...); --#define scx_ops_error(fmt, args...) \ -- scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args) -- --void __scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active); -- --static inline void scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active) --{ -- if (!scx_enabled()) -- return; --#ifdef CONFIG_SMP -- /* -- * Pairs with the smp_load_acquire() issued by a CPU in -- * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -- * resched. -- */ -- smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1); --#endif -- if (!static_branch_unlikely(&scx_ops_cpu_preempt)) -- return; -- __scx_notify_pick_next_task(rq, p, active); --} -- --//extern void scx_scheduler_tick(void); --static inline void scx_notify_sched_tick(void) --{ -- unsigned long last_check; -- -- if (!scx_enabled()) -- return; -- //scx_scheduler_tick(); -- -- last_check = scx_watchdog_timestamp; -- if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "watchdog failed to check in for %u.%03us", -- dur_ms / 1000, dur_ms % 1000); -- } --} -- --static inline const struct sched_class *next_active_class(const struct sched_class *class) --{ -- class++; -- if (scx_switched_all() && class == &fair_sched_class) -- class++; -- if (!scx_enabled() && class == &ext_sched_class) -- class++; -- return class; --} -- --#define for_active_class_range(class, _from, _to) \ -- for (class = (_from); class != (_to); class = next_active_class(class)) -- --#define for_each_active_class(class) \ -- for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -- --/* -- * SCX requires a balance() call before every pick_next_task() call including -- * when waking up from idle. -- */ --#define for_balance_class_range(class, prev_class, end_class) \ -- for_active_class_range(class, (prev_class) > &ext_sched_class ? \ -- &ext_sched_class : (prev_class), (end_class)) -- --#ifdef CONFIG_SCHED_CORE --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi); --#endif -- --#else /* CONFIG_SCHED_CLASS_EXT */ -- --#define scx_enabled() false --#define scx_switched_all() false -- --static inline void scx_pre_fork(struct task_struct *p) {} --static inline int scx_fork(struct task_struct *p) { return 0; } --static inline void scx_post_fork(struct task_struct *p) {} --static inline void scx_cancel_fork(struct task_struct *p) {} --static inline int scx_check_setscheduler(struct task_struct *p, -- int policy) { return 0; } --static inline bool scx_can_stop_tick(struct rq *rq) { return true; } --static inline void init_sched_ext_class(void) {} --static inline void scx_notify_pick_next_task(struct rq *rq, -- const struct task_struct *p, -- const struct sched_class *active) {} --static inline void scx_notify_sched_tick(void) {} -- --#define for_each_active_class for_each_class --#define for_balance_class_range for_class_range -- --#endif /* CONFIG_SCHED_CLASS_EXT */ -- --#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) --void __scx_update_idle(struct rq *rq, bool idle); -- --static inline void scx_update_idle(struct rq *rq, bool idle) --{ -- if (scx_enabled()) -- __scx_update_idle(rq, idle); --} --#else --static inline void scx_update_idle(struct rq *rq, bool idle) {} --#endif -- --#ifdef CONFIG_CGROUP_SCHED --#ifdef CONFIG_EXT_GROUP_SCHED --int scx_tg_online(struct task_group *tg); --void scx_tg_offline(struct task_group *tg); --int scx_cgroup_can_attach(struct cgroup_taskset *tset); --void scx_move_task(struct task_struct *p); --void scx_cgroup_finish_attach(void); --void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); --void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); --#else /* CONFIG_EXT_GROUP_SCHED */ --static inline int scx_tg_online(struct task_group *tg) { return 0; } --static inline void scx_tg_offline(struct task_group *tg) {} --static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } --static inline void scx_move_task(struct task_struct *p) {} --static inline void scx_cgroup_finish_attach(void) {} --static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} --static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} --#endif /* CONFIG_EXT_GROUP_SCHED */ --#endif /* CONFIG_CGROUP_SCHED */ -diff --git b/kernel/sched/hmbird.h a/kernel/sched/hmbird.h -new file mode 100755 -index 000000000000..fa76c7f09977 ---- /dev/null -+++ a/kernel/sched/hmbird.h -@@ -0,0 +1,342 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#ifndef _HMBIRD_H_ -+#define _HMBIRD_H_ -+ -+/* -+ * Tag marking a kernel function as a kfunc. This is meant to minimize the -+ * amount of copy-paste that kfunc authors have to include for correctness so -+ * as to avoid issues such as the compiler inlining or eliding either a static -+ * kfunc, or a global kfunc in an LTO build. -+ */ -+ -+#define DEBUG_INTERNAL (1 << 0) -+#define DEBUG_INFO_TRACE (1 << 1) -+#define DEBUG_INFO_SYSTRACE (1 << 2) -+ -+#define NUMS_CGROUP_KINDS (512) -+#define SLIM_FOR_SGAME (1 << 0) -+#define SLIM_FOR_GENSHIN (1 << 1) -+ -+#ifdef MAX_TASK_NR -+#define MAX_KEY_THREAD_RECORD ((MAX_TASK_NR + 1) >> 1) -+#else -+#define MAX_KEY_THREAD_RECORD (1 << 3) -+#endif /* MAX_TASK_NR */ -+ -+#define TOP_TASK_SHIFT (8) -+#define TOP_TASK_MAX (1 << TOP_TASK_SHIFT) -+#ifndef TOP_TASK_BITS_MASK -+#define TOP_TASK_BITS_MASK (TOP_TASK_MAX - 1) -+#endif /* TOP_TASK_BITS_MASK */ -+ -+extern struct md_info_t *md_info; -+ -+#define debug_enabled() \ -+ (unlikely(hmbirdcore_debug)) -+ -+#define hmbird_debug(fmt, ...) \ -+ pr_info(":"fmt, ##__VA_ARGS__) -+ -+#define hmbird_err(id, fmt, ...) \ -+do { \ -+ pr_err(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_deferred_err(id, fmt, ...) \ -+do { \ -+ printk_deferred(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \ -+do { \ -+ if (unlikely(cond)) { \ -+ hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+ } \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_info_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_info_trace(fmt, ...) -+#endif -+ -+#define hmbird_info_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+ } \ -+} while (0) -+ -+#define hmbird_output_systrace(fmt, ...) \ -+do { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_internal_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_internal_trace(fmt, ...) -+#endif -+ -+#define hmbird_internal_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) { \ -+ hmbird_output_systrace(fmt, ##__VA_ARGS__); \ -+ } \ -+} while (0) -+ -+enum hmbird_wake_flags { -+ /* expose select WF_* flags as enums */ -+ HMBIRD_WAKE_EXEC = WF_EXEC, -+ HMBIRD_WAKE_FORK = WF_FORK, -+ HMBIRD_WAKE_TTWU = WF_TTWU, -+ HMBIRD_WAKE_SYNC = WF_SYNC, -+}; -+ -+enum hmbird_enq_flags { -+ /* expose select ENQUEUE_* flags as enums */ -+ HMBIRD_ENQ_WAKEUP = ENQUEUE_WAKEUP, -+ HMBIRD_ENQ_HEAD = ENQUEUE_HEAD, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * Set the following to trigger preemption when calling -+ * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the -+ * current task is cleared to zero and the CPU is kicked into the -+ * scheduling path. Implies %HMBIRD_ENQ_HEAD. -+ */ -+ HMBIRD_ENQ_PREEMPT = 1LLU << 32, -+ HMBIRD_ENQ_REENQ = 1LLU << 40, -+ HMBIRD_ENQ_LAST = 1LLU << 41, -+ -+ /* -+ * A hint indicating that it's advisable to enqueue the task on the -+ * local dsq of the currently selected CPU. Currently used by -+ * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST. -+ */ -+ HMBIRD_ENQ_LOCAL = 1LLU << 42, -+ -+ /* high 8 bits are internal */ -+ __HMBIRD_ENQ_INTERNAL_MASK = 0xffLLU << 56, -+ -+ HMBIRD_ENQ_CLEAR_OPSS = 1LLU << 56, -+ HMBIRD_ENQ_DSQ_PRIQ = 1LLU << 57, -+}; -+ -+enum hmbird_deq_flags { -+ /* expose select DEQUEUE_* flags as enums */ -+ HMBIRD_DEQ_SLEEP = DEQUEUE_SLEEP, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * The generic core-sched layer decided to execute the task even though -+ * it hasn't been dispatched yet. Dequeue from the BPF side. -+ */ -+ HMBIRD_DEQ_CORE_SCHED_EXEC = 1LLU << 32, -+}; -+ -+enum hmbird_kick_flags { -+ HMBIRD_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -+ HMBIRD_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ -+}; -+ -+enum hmbird_task_prop_type { -+ HMBIRD_TASK_PROP_COMMON, -+ HMBIRD_TASK_PROP_PIPELINE, -+ HMBIRD_TASK_PROP_DEBUG_OR_LOG, -+ HMBIRD_TASK_PROP_HIGH_LOAD, -+ HMBIRD_TASK_PROP_IO, -+ HMBIRD_TASK_PROP_NETWORK, -+ HMBIRD_TASK_PROP_PERIODIC, -+ HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL, -+ HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL, -+ HMBIRD_TASK_PROP_ISOLATE, -+ HMBIRD_TASK_PROP_MAX, -+}; -+ -+enum hmbird_preempt_policy_id { -+ HMBIRD_PREEMPT_POLICY_NONE, -+ HMBIRD_PREEMPT_POLICY_PRIO_BASED, -+}; -+ -+extern const struct sched_class hmbird_sched_class; -+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops; -+extern const struct file_operations sched_hmbird_fops; -+extern unsigned long hmbird_watchdog_timeout; -+extern unsigned long hmbird_watchdog_timestamp; -+ -+enum scx_rq_flags { -+ HMBIRD_RQ_CAN_STOP_TICK = 1 << 0, -+}; -+ -+struct hmbird_rq { -+ struct hmbird_dispatch_q local_dsq; -+ struct list_head watchdog_list; -+ u64 ops_qseq; -+ u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -+ u32 nr_running; -+ u32 flags; -+ bool cpu_released; -+ cpumask_var_t cpus_to_kick; -+ cpumask_var_t cpus_to_preempt; -+ cpumask_var_t cpus_to_wait; -+ u64 pnt_seq; -+ u64 *prev_runnable_sum_fixed; -+ u32 *prev_window_size; -+ struct irq_work kick_cpus_irq_work; -+ bool pipeline; -+ bool exclusive; -+ bool period_disallow; -+ bool nonperiod_disallow; -+ struct rq *rq; -+ struct hmbird_sched_rq_stats *srq; -+}; -+ -+struct hmbird_sched_change_guard { -+ struct task_struct *p; -+ struct rq *rq; -+ bool queued; -+ bool running; -+ bool done; -+}; -+ -+extern struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -+ -+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags); -+ -+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -+ for (struct hmbird_sched_change_guard __cg = \ -+ hmbird_sched_change_guard_init(__rq, __p, __flags); \ -+ !__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags)) -+ -+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+bool task_on_hmbird(struct task_struct *p); -+int hmbird_pre_fork(struct task_struct *p); -+void hmbird_post_fork(struct task_struct *p); -+void hmbird_cancel_fork(struct task_struct *p); -+int hmbird_check_setscheduler(struct task_struct *p, int policy); -+bool hmbird_can_stop_tick(struct rq *rq); -+void init_sched_hmbird_class(void); -+ -+void hmbird_ops_exit(void); -+#define hmbird_ops_error(fmt, ...) \ -+do { \ -+ hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__); \ -+ hmbird_ops_exit(); \ -+} while (0) -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active); -+ -+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight) -+{ -+ return clamp_t(unsigned long, -+ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -+ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); -+} -+ -+static inline void hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active) -+{ -+ if (!hmbird_enabled()) -+ return; -+#ifdef CONFIG_SMP -+ /* -+ * Pairs with the smp_load_acquire() issued by a CPU in -+ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -+ * resched. -+ */ -+ smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1); -+#endif -+ if (!static_branch_unlikely(&hmbird_ops_cpu_preempt)) -+ return; -+ __hmbird_notify_pick_next_task(rq, p, active); -+} -+ -+extern void hmbird_scheduler_tick(void); -+void scan_timeout(struct rq *rq); -+void hmbird_notify_sched_tick(void); -+ -+static inline const struct sched_class *next_active_class(const struct sched_class *class) -+{ -+ class++; -+ -+ if (hmbird_enabled() && class == &fair_sched_class) -+ class++; -+ if (!hmbird_enabled() && class == &hmbird_sched_class) -+ class++; -+ return class; -+} -+ -+#define for_active_class_range(class, _from, _to) \ -+ for (class = (_from); class != (_to); class = next_active_class(class)) -+ -+#define for_each_active_class(class) \ -+ for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -+ -+/* -+ * HMBIRD requires a balance() call before every pick_next_task() call including -+ * when waking up from idle. -+ */ -+#define for_balance_class_range(class, prev_class, end_class) \ -+ for_active_class_range(class, (prev_class) > &hmbird_sched_class ? \ -+ &hmbird_sched_class : (prev_class), (end_class)) -+ -+#define MIN_CGROUP_DL_IDX (5) /* 8ms */ -+#define DEFAULT_CGROUP_DL_IDX (8) /* 64ms */ -+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS]; -+ -+ -+void __hmbird_update_idle(struct rq *rq, bool idle); -+ -+static inline void hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ if (hmbird_enabled()) -+ __hmbird_update_idle(rq, idle); -+} -+ -+int hmbird_ctrl(bool enable); -+ -+int hmbird_tg_online(struct task_group *tg); -+ -+ -+static inline u16 hmbird_task_util(struct task_struct *p) -+{ -+ return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0; -+} -+u16 hmbird_cpu_util(int cpu); -+ -+bool get_hmbird_ops_enabled(void); -+bool get_non_hmbird_task(void); -+void set_cpu_cluster(u64 cpu_cluster); -+#endif /*_EXT_H_*/ -diff --git b/kernel/sched/hmbird/hmbird.c a/kernel/sched/hmbird/hmbird.c -new file mode 100755 -index 000000000000..86f9fd092424 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird.c -@@ -0,0 +1,4354 @@ -+// SPDX-License-Identifier: GPL-2.0 -+ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#include -+#include -+ -+#include "slim.h" -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+#include -+ -+#define CLUSTER_SEPARATE -+ -+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0); -+atomic_t non_hmbird_task; -+atomic_t hmbird_module_loaded = ATOMIC_INIT(0); -+int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0}; -+ -+enum hmbird_internal_consts { -+ HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ, -+}; -+ -+enum hmbird_ops_enable_state { -+ HMBIRD_OPS_PREPPING, -+ HMBIRD_OPS_ENABLING, -+ HMBIRD_OPS_ENABLED, -+ HMBIRD_OPS_DISABLING, -+ HMBIRD_OPS_DISABLED, -+}; -+ -+static inline void put_hmbird_ts(struct task_struct *p) -+{ -+ kfree((void *)p->android_oem_data1[HMBIRD_TS_IDX]); -+ p->android_oem_data1[HMBIRD_TS_IDX] = 0; -+} -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+/* -+ * hmbird_entity->ops_state -+ * -+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler. -+ * State transitions look as follows: -+ * -+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING -+ * ^ | | -+ * | v v -+ * \-------------------------------/ -+ * -+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -+ * sites for explanations on the conditions being waited upon and why they are -+ * safe. Transitions out of them into NONE or QUEUED must store_release and the -+ * waiters should load_acquire. -+ * -+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether -+ * any given task can be dispatched by the BPF scheduler at all times and thus -+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core -+ * can safely reject invalid dispatches. -+ */ -+enum hmbird_ops_state { -+ HMBIRD_OPSS_NONE, /* owned by the HMBIRD core */ -+ HMBIRD_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -+ HMBIRD_OPSS_QUEUED, /* owned by the BPF scheduler */ -+ HMBIRD_OPSS_DISPATCHING, /* in transit back to the HMBIRD core */ -+ -+ /* -+ * QSEQ brands each QUEUED instance so that, when dispatch races -+ * dequeue/requeue, the dispatcher can tell whether it still has a claim -+ * on the task being dispatched. -+ */ -+ HMBIRD_OPSS_QSEQ_SHIFT = 2, -+ HMBIRD_OPSS_STATE_MASK = (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1, -+ HMBIRD_OPSS_QSEQ_MASK = ~HMBIRD_OPSS_STATE_MASK, -+}; -+ -+enum switch_stat { -+ HMBIRD_DISABLED, -+ HMBIRD_SWITCH_PREP, -+ HMBIRD_RQ_SWITCH_BEGIN, -+ HMBIRD_RQ_SWITCH_DONE, -+ HMBIRD_ENABLED, -+}; -+enum switch_stat curr_ss; -+ -+/* -+ * During exit, a task may schedule after losing its PIDs. When disabling the -+ * BPF scheduler, we need to be able to iterate tasks in every state to -+ * guarantee system safety. Maintain a dedicated task list which contains every -+ * task between its fork and eventual free. -+ */ -+DEFINE_SPINLOCK(hmbird_tasks_lock); -+static LIST_HEAD(hmbird_tasks); -+ -+/* ops enable/disable */ -+static struct kthread_worker *hmbird_ops_helper; -+static DEFINE_MUTEX(hmbird_ops_enable_mutex); -+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem); -+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED); -+ -+static bool hmbird_warned_zero_slice; -+enum hmbird_switch_type sw_type; -+static int skip_num[MAX_GLOBAL_DSQS]; -+static int big_distribute_mask_prev; -+static int little_distribute_mask_prev; -+ -+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0); -+ -+/* -+ * The maximum amount of time in jiffies that a task may be runnable without -+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger -+ * hmbird_ops_error(). -+ */ -+unsigned long hmbird_watchdog_timeout; -+ -+/* -+ * The last time the delayed work was run. This delayed work relies on -+ * ksoftirqd being able to run to service timer interrupts, so it's possible -+ * that this work itself could get wedged. To account for this, we check that -+ * it's not stalled in the timer tick, and trigger an error if it is. -+ */ -+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES; -+ -+static struct delayed_work hmbird_watchdog_work; -+static struct work_struct hmbird_err_exit_work; -+ -+/* idle tracking */ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_CPUMASK_OFFSTACK -+#define CL_ALIGNED_IF_ONSTACK -+#else -+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp -+#endif -+ -+static struct { -+ cpumask_var_t cpu; -+ cpumask_var_t smt; -+} idle_masks CL_ALIGNED_IF_ONSTACK; -+ -+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus; -+#endif /* CONFIG_SMP */ -+ -+/* dispatch queues */ -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global; -+ -+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128}; -+u32 pcp_dsq_deadline = 20; -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS]; -+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq); -+ -+static u64 max_hmbird_dsq_internal_id; -+ -+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/ -+#define CLUSTER_SEPARATE_IDX (8) -+#define GDSQS_ID_BASE (3) -+#define UX_COMPATIBLE_IDX (4) -+#define NON_PERIOD_START (5) -+#define NON_PERIOD_END (MAX_GLOBAL_DSQS) -+#define CREATE_DSQ_LEVEL_WITHIN (1) -+ -+struct hmbird_sched_info { -+ spinlock_t lock; -+ int curr_idx[2]; -+ int rtime[MAX_GLOBAL_DSQS]; -+}; -+ -+struct pcp_sched_info { -+ s64 pcp_seq; -+ int rtime; -+ bool pcp_round; -+}; -+ -+/* -+ * pcp_info may rw by another cpu. -+ * protected by rq->lock. -+ */ -+atomic64_t pcp_dsq_round; -+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info); -+ -+struct md_info_t *md_info; -+ -+static int b_rescue_l, l_rescue_b; -+static struct hmbird_sched_info sinfo; -+ -+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC; -+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = { -+ 0, 0, 0, 0, 0, -+ 32 * NSEC_PER_MSEC, -+ 20 * NSEC_PER_MSEC, -+ 14 * NSEC_PER_MSEC, -+ 8 * NSEC_PER_MSEC, -+ 6 * NSEC_PER_MSEC -+}; -+ -+struct cluster_ctx { -+ /* cpu-dsq map must within [lower, upper) */ -+ int upper; -+ int lower; -+ int tidx; -+}; -+ -+enum stat_items { -+ GLOBAL_STAT, -+ CPU_ALLOW_FAIL, -+ RT_CNT, -+ KEY_TASK_CNT, -+ SWITCH_IDX, -+ TIMEOUT_CNT, -+ -+ TOTAL_DSP_CNT, -+ MOVE_RQ_CNT, -+ SELECT_CPU, -+ -+ DWORD_STAT_END = SELECT_CPU, -+ -+ GDSQ_CNT, -+ ERR_IDX, -+ PCP_TIMEOUT_CNT, -+ PCP_LDSQ_CNT, -+ PCP_ENQL_CNT, -+ -+ MAX_ITEMS, -+}; -+static DEFINE_SPINLOCK(stats_lock); -+static char *stats_str[MAX_ITEMS] = { -+ "global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt", -+ "switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt", -+ "select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt", -+ "pcp_ldsq_cnt", "pcp_enql_cnt" -+}; -+ -+ -+struct stats_struct { -+ u64 global_stat[2]; -+ u64 cpu_allow_fail[2]; -+ u64 rt_cnt[2]; -+ u64 key_task_cnt[2]; -+ u64 switch_idx[2]; -+ u64 timeout_cnt[2]; -+ -+ u64 total_dsp_cnt[2]; -+ u64 move_rq_cnt[2]; -+ u64 select_cpu[2]; -+ -+ u64 gdsq_count[MAX_GLOBAL_DSQS][2]; -+ u64 err_idx[5]; -+ u64 pcp_timeout_cnt[NR_CPUS]; -+ u64 pcp_ldsq_count[NR_CPUS][2]; -+ u64 pcp_enql_cnt[NR_CPUS]; -+} stats_data; -+ -+static struct { -+ cpumask_var_t ex_free; -+ cpumask_var_t exclusive; -+ cpumask_var_t partial; -+ cpumask_var_t big; -+ cpumask_var_t little; -+} iso_masks __read_mostly; -+ -+/* -+ * Need more synchronization for these two variables? -+ * I choose not to. -+ */ -+static int l_need_rescue, b_need_rescue; -+ -+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...) \ -+{ \ -+ char buf[MAX_FATAL_INFO]; \ -+ \ -+ scnprintf(buf, MAX_FATAL_INFO, fmt, ##args); \ -+ hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf); \ -+ trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable), \ -+ READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf); \ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); \ -+} \ -+ -+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2) -+{ -+ int c1, c2; -+ struct cpumask mask = {.bits = {0}}; -+ struct cpumask tmp = {.bits = {0}}; -+ -+ if (!slim_stats) -+ return false; -+ -+ if (!rq1 || !rq2) -+ return false; -+ -+ c1 = cpu_of(rq1); -+ c2 = cpu_of(rq2); -+ cpumask_set_cpu(c1, &mask); -+ cpumask_set_cpu(c2, &mask); -+ -+ if (cpumask_and(&tmp, iso_masks.little, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.big, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.partial, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ return false; -+} -+ -+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu) -+{ -+ unsigned long flags; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ if (!slim_stats) -+ return; -+ -+ switch (item) { -+ case GLOBAL_STAT: -+ fallthrough; -+ case CPU_ALLOW_FAIL: -+ fallthrough; -+ case RT_CNT: -+ fallthrough; -+ case KEY_TASK_CNT: -+ fallthrough; -+ case SWITCH_IDX: -+ fallthrough; -+ case TIMEOUT_CNT: -+ fallthrough; -+ case TOTAL_DSP_CNT: -+ fallthrough; -+ case MOVE_RQ_CNT: -+ fallthrough; -+ case SELECT_CPU: -+ pval = pbase + item * 2 + idx; -+ break; -+ case GDSQ_CNT: -+ pval = &stats_data.gdsq_count[dsq_id][idx]; -+ break; -+ case ERR_IDX: -+ pval = &stats_data.err_idx[idx]; -+ break; -+ case PCP_TIMEOUT_CNT: -+ pval = &stats_data.pcp_timeout_cnt[cpu]; -+ break; -+ case PCP_LDSQ_CNT: -+ pval = &stats_data.pcp_ldsq_count[cpu][idx]; -+ break; -+ case PCP_ENQL_CNT: -+ pval = &stats_data.pcp_enql_cnt[cpu]; -+ break; -+ default: -+ return; -+ } -+ -+ spin_lock_irqsave(&stats_lock, flags); -+ *pval += 1; -+ spin_unlock_irqrestore(&stats_lock, flags); -+} -+ -+static inline bool handle_ret(int ret, int *idx, int len) -+{ -+ if (ret < 0 || ret >= len - *idx) -+ return true; -+ *idx += ret; -+ return false; -+} -+ -+#define PRINT_INTV (5 * HZ) -+void stats_print(char *buf, int len) -+{ -+ int idx = 0, i, j, ret; -+ int item = 0; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ for (item = 0; item < MAX_ITEMS; item++) { -+ if (item <= DWORD_STAT_END) { -+ pval = pbase + item * 2; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n", -+ stats_str[item], pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == GDSQ_CNT) { -+ for (j = 0; j < MAX_GLOBAL_DSQS; j++) { -+ pval = (u64 *)&stats_data.gdsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == ERR_IDX) { -+ pval = (u64 *)&stats_data.err_idx; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n", -+ stats_str[item], pval[0], -+ pval[1], pval[2], pval[3], pval[4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == PCP_TIMEOUT_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_timeout_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_LDSQ_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_ldsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_ENQL_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_enql_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } -+ } -+ -+ if (!md_info) { -+ buf[idx] = '\0'; -+ return; -+ } -+ -+ ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_SWITCHS; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "sw_rec[%d] = %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.sw_rec[i].switch_at, -+ md_info->kern_dump.sw_rec[i].is_success, -+ md_info->kern_dump.sw_rec[i].end_state, -+ md_info->kern_dump.sw_rec[i].switch_reason); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_EXCEP_ID; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "excep[%d] = %llu %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.excep_rec[i][0], -+ md_info->kern_dump.excep_rec[i][1], -+ md_info->kern_dump.excep_rec[i][2], -+ md_info->kern_dump.excep_rec[i][3], -+ md_info->kern_dump.excep_rec[i][4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n", -+ i, md_info->kern_dump.excep_idx[i]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ -+ buf[idx] = '\0'; -+} -+ -+enum cpu_type { -+ LITTLE, -+ BIG, -+ PARTIAL, -+ EXCLUSIVE, -+ EX_FREE, -+ INVALID -+}; -+ -+enum dsq_type { -+ GLOBAL_DSQ, -+ PCP_DSQ, -+ OTHER, -+ MAX_DSQ_TYPE, -+}; -+ -+static enum cpu_type cpu_cluster(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (get_hmbird_rq(rq)->exclusive) { -+ return EXCLUSIVE; -+ } else { -+ if (cpumask_test_cpu(cpu, iso_masks.little)) -+ return LITTLE; -+ else if (cpumask_test_cpu(cpu, iso_masks.big)) -+ return BIG; -+ else if (cpumask_test_cpu(cpu, iso_masks.partial)) -+ return PARTIAL; -+ else if (cpumask_test_cpu(cpu, iso_masks.exclusive)) -+ return EXCLUSIVE; -+ else if (cpumask_test_cpu(cpu, iso_masks.ex_free)) -+ return EX_FREE; -+ } -+ return INVALID; -+} -+ -+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq) -+{ -+ if (!dsq) -+ return OTHER; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= GDSQS_ID_BASE) && -+ ((dsq->id & 0xff) < MAX_GLOBAL_DSQS)) -+ return GLOBAL_DSQ; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) && -+ ((dsq->id & 0xff) < max_hmbird_dsq_internal_id)) -+ return PCP_DSQ; -+ -+ return OTHER; -+} -+ -+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq) -+{ -+ enum dsq_type type; -+ -+ type = get_dsq_type(dsq); -+ switch (type) { -+ case GLOBAL_DSQ: -+ case PCP_DSQ: -+ return (dsq->id & 0xff) - GDSQS_ID_BASE; -+ default: -+ return -1; -+ } -+ return -1; -+} -+ -+static void update_cpus_idle(bool set, struct cpumask *mask) -+{ -+ int cpu; -+ struct rq *rq; -+ -+ if (set) { -+ for_each_cpu(cpu, mask) { -+ rq = cpu_rq(cpu); -+ if (is_idle_task(rq->curr)) -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ } -+ } else -+ cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask); -+} -+ -+static void set_partial_status(bool enable, bool little, bool big) -+{ -+ WRITE_ONCE(partial_enable, enable); -+ WRITE_ONCE(l_need_rescue, little); -+ WRITE_ONCE(b_need_rescue, big); -+} -+ -+static bool is_little_need_rescue(void) -+{ -+ return READ_ONCE(l_need_rescue); -+} -+ -+static bool is_big_need_rescue(void) -+{ -+ return READ_ONCE(b_need_rescue); -+} -+ -+static bool is_partial_enabled(void) -+{ -+ return READ_ONCE(partial_enable); -+} -+ -+static bool is_partial_cpu(int cpu) -+{ -+ return cpumask_test_cpu(cpu, iso_masks.partial); -+} -+ -+static void set_iso_par_free(bool enable) -+{ -+ WRITE_ONCE(isolate_ctrl, enable); -+} -+ -+static bool is_iso_par_free(void) -+{ -+ return READ_ONCE(isolate_ctrl); -+} -+ -+static bool skip_update_idle(void) -+{ -+ int cpu = smp_processor_id(); -+ enum cpu_type type = cpu_cluster(cpu); -+ -+ if ((type == EXCLUSIVE && !is_iso_par_free()) || -+ /* partial enable may changed during idle, it doesn't matter. */ -+ (!is_partial_enabled() && type == PARTIAL)) -+ return true; -+ -+ return false; -+} -+ -+static void init_isolate_cpus(void) -+{ -+ WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -+ cpumask_set_cpu(0, iso_masks.big); -+ cpumask_set_cpu(1, iso_masks.big); -+ cpumask_set_cpu(2, iso_masks.big); -+ cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(4, iso_masks.big); -+ cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(6, iso_masks.partial); -+ cpumask_set_cpu(7, iso_masks.ex_free); -+} -+ -+extern spinlock_t css_set_lock; -+ -+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp) -+{ -+ int i; -+ struct cgroup *anc; -+ -+ spin_lock_irq(&css_set_lock); -+ for (i = 0; i < cgrp->level; i++) { -+ anc = cgrp->ancestors[i]; -+ if (anc->level != CREATE_DSQ_LEVEL_WITHIN) -+ continue; -+ cgroup_get(anc); -+ spin_unlock_irq(&css_set_lock); -+ return anc; -+ } -+ spin_unlock_irq(&css_set_lock); -+ hmbird_err(NO_CGROUP_L1, ":error cgroup = %s\n", cgrp->kn->name); -+ return NULL; -+} -+ -+#define PCP_IDX_BIT (1 << 31) -+ -+static bool is_pcp_rt(struct task_struct *p) -+{ -+ return rt_prio(p->prio) && (p->nr_cpus_allowed == 1); -+} -+ -+static bool is_pcp_idx(int idx) -+{ -+ return idx >= MAX_GLOBAL_DSQS; -+} -+ -+static bool is_critical_system_task(struct task_struct *p) -+{ -+ int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK; -+ -+ return (p->prio < (MAX_RT_PRIO >> 1) && -+ (sp_dl < SCHED_PROP_DEADLINE_LEVEL3)); -+} -+ -+#define ISOLATE_TASK_TOP_BIT (1 << 17) -+#define PIPELINE_TASK_TOP_BIT (1 << 9) -+static bool is_pipeline_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT); -+} -+ -+static bool is_isolate_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT); -+} -+ -+static bool is_critical_app_task_without_isolate(struct task_struct *p) -+{ -+ return task_is_top_task(p) && !is_isolate_task(p); -+} -+ -+static int find_idx_from_task(struct task_struct *p) -+{ -+ int idx, cpu; -+ int sp_dl; -+ struct task_group *tg = p->sched_task_group; -+ -+ if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) { -+ cpu = cpumask_any(p->cpus_ptr); -+ idx = cpu + MAX_GLOBAL_DSQS; -+ return idx; -+ } -+ -+ if (is_critical_system_task(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL0; -+ goto done; -+ } -+ -+ if (is_critical_app_task_without_isolate(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL1; -+ goto done; -+ } -+ -+ if (p->pid == scx_systemui_pid) { -+ idx = SCHED_PROP_DEADLINE_LEVEL4; -+ goto done; -+ } -+ -+ sp_dl = hmbird_get_dsq_id(p); -+ if (sp_dl) { -+ idx = sp_dl; -+ goto done; -+ } -+ -+ if (rt_prio(p->prio)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL3; -+ goto done; -+ } -+ -+ if (tg && tg->css.cgroup && tg->css.cgroup->kn) { -+ if (likely(tg->css.cgroup->kn->id >= 0 && -+ tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS)) -+ idx = cgroup_ids_table[tg->css.cgroup->kn->id]; -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ -+done: -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) { -+ hmbird_err(DSQ_ID_ERR, " : idx error, idx = %d-----\n", idx); -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } -+ return idx; -+} -+ -+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p) -+{ -+ int idx; -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq; -+ -+ if (!p) -+ return NULL; -+ -+ idx = find_idx_from_task(p); -+ if (is_pcp_idx(idx)) { -+ idx -= MAX_GLOBAL_DSQS; -+ dsq = &per_cpu(pcp_ldsq, idx); -+ get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq); -+ slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx); -+ } else { -+ dsq = &gdsqs[idx]; -+ get_hmbird_ts(p)->gdsq_idx = idx; -+ slim_stats_record(GDSQ_CNT, 0, idx, 0); -+ } -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ dsq->last_consume_at = jiffies; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ return dsq; -+} -+ -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq); -+ -+static void set_partial_rescue(bool p_state, bool l_over, bool b_over) -+{ -+ set_partial_status(p_state, l_over, b_over); -+ update_cpus_idle(p_state, iso_masks.partial); -+ hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled()); -+ hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue()); -+ hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue()); -+} -+ -+static void free_isocpu(bool enable) -+{ -+ set_iso_par_free(enable); -+ update_cpus_idle(enable, iso_masks.ex_free); -+ hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free()); -+} -+ -+inline u64 get_hmbird_cpu_util(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed) -+ return 0; -+ u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed); -+ u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size); -+ -+ do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT); -+ -+ return prev_runnable_sum_fixed; -+} -+ -+static inline unsigned int get_scaling_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->max; -+} -+ -+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static u64 get_cpus_max_util(struct cpumask *mask) -+{ -+ int cpu; -+ u64 max = 0; -+ u64 util, ratio; -+ unsigned long effective_cap = 0; -+ -+ for_each_cpu(cpu, mask) { -+ if (slim_walt_ctrl) -+ slim_get_cpu_util(cpu, &util); -+ else -+ util = get_hmbird_cpu_util(cpu); -+ -+ /* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/ -+ if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu))) -+ effective_cap = arch_scale_cpu_capacity(cpu); -+ else -+ effective_cap = arch_scale_cpu_capacity(cpu) * -+ get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu); -+ -+ ratio = util * 100 / effective_cap; -+ hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util); -+ hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n", -+ cpu, (u64)arch_scale_cpu_capacity(cpu)); -+ hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n", -+ cpu, (u64)effective_cap); -+ -+ if (ratio > 100) -+ ratio = 100; -+ -+ if (ratio > max) -+ max = ratio; -+ } -+ return max; -+} -+ -+static bool cluster_need_rescue(struct cpumask *mask, int hres) -+{ -+ return get_cpus_max_util(mask) > hres; -+} -+ -+void partial_dynamic_ctrl(void) -+{ -+ u64 lmax = 0, bmax = 0; -+ bool l_over, l_under, b_over, b_under; -+ static bool last_l_over, last_b_over; -+ static unsigned long last_check; -+ -+ /* Check partial every jiffies. need lock? */ -+ if (time_before_eq(jiffies, READ_ONCE(last_check))) -+ return; -+ -+ WRITE_ONCE(last_check, jiffies); -+ -+ lmax = get_cpus_max_util(iso_masks.little); -+ l_over = lmax > parctrl_high_ratio_l; -+ l_under = lmax < parctrl_low_ratio_l; -+ -+ bmax = get_cpus_max_util(iso_masks.big); -+ b_over = bmax > parctrl_high_ratio; -+ b_under = bmax < parctrl_low_ratio; -+ -+ if (is_partial_enabled() && (l_over || b_over)) { -+ if (last_l_over != l_over || last_b_over != b_over) -+ set_partial_rescue(true, l_over, b_over); -+ } else if (!is_partial_enabled() && (l_over || b_over)) { -+ set_partial_rescue(true, l_over, b_over); -+ } else if (is_partial_enabled() && l_under && b_under) { -+ set_partial_rescue(false, false, false); -+ } -+ last_l_over = l_over; -+ last_b_over = b_over; -+ -+ if (is_partial_enabled()) { -+ if (cpumask_empty(iso_masks.partial)) { -+ bmax = bmax > lmax ? bmax : lmax; -+ } else { -+ bmax = get_cpus_max_util(iso_masks.partial); -+ } -+ if (!is_iso_par_free() && bmax > isoctrl_high_ratio) { -+ hmbird_info_trace("partial max = %llu\n", bmax); -+ free_isocpu(true); -+ } else if (is_iso_par_free() && bmax < isoctrl_low_ratio) -+ free_isocpu(false); -+ } else if (is_iso_par_free()) { -+ free_isocpu(false); -+ } -+} -+ -+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx) -+{ -+ hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx); -+} -+ -+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx) -+{ -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[idx])) { -+ slim_stats_record(GDSQ_CNT, 1, idx, 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ int i; -+ -+ for (i = 0; i < UX_COMPATIBLE_IDX; i++) { -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ slim_stats_record(GDSQ_CNT, 1, i, 0); -+ return 1; -+ } -+ } -+ return 0; -+} -+ -+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) { -+ slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ goto clear_timeout; -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) -+ goto clear_timeout; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ hmbird_info_trace( -+ "dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n", -+ dsq_id_to_internal(dsq), entity->task->comm, -+ jiffies, msecs_to_jiffies(deadline), entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1); -+ return; -+ -+clear_timeout: -+ hmbird_info_trace("dsq[%d] clear timeout\n", -+ dsq_id_to_internal(dsq)); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0); -+ dsq->is_timeout = false; -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq)); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime) -+{ -+ hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime); -+} -+ -+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any) -+{ -+ bool is_timeout; -+ int cpu = cpu_of(rq); -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu); -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ is_timeout = dsq->is_timeout; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ /* -+ * dsq->is_timeout may change here, let it be. -+ * it won't cause serious problems. -+ * the same for consume_dispatch_q later. -+ */ -+ if (!is_timeout && !any) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, dsq)) { -+ if (is_timeout) { -+ hmbird_info_trace("dsq[%d] consume a pcp timeout task\n", -+ dsq_id_to_internal(dsq)); -+ update_timeout_stats(rq, dsq, pcp_dsq_deadline); -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu); -+ } -+ slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu); -+ return 1; -+ } -+ /* -+ * No pcp task, clear quota. -+ */ -+ if (any) { -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+ } -+ return 0; -+} -+ -+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf) -+{ -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ } -+ return 0; -+} -+ -+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf, -+ int tmp, int cidx, int tidx) -+{ -+ unsigned long flags; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) { -+ if (tmp != cidx) { -+ spin_lock(&sinfo.lock); -+ sinfo.curr_idx[tidx] = tmp; -+ spin_unlock(&sinfo.lock); -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]); -+ slim_stats_record(SWITCH_IDX, 0, 0, 0); -+ } -+ slim_stats_record(GDSQ_CNT, 1, tmp, 0); -+ -+ raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags); -+ gdsqs[tmp].last_consume_at = jiffies; -+ raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags); -+ return 1; -+ } -+ return 0; -+ -+} -+ -+static int get_cidx(struct cluster_ctx *ctx) -+{ -+ int cidx; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ if (cidx < ctx->lower || cidx >= ctx->upper) { -+ sinfo.curr_idx[ctx->tidx] = ctx->lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]); -+ slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ } -+ spin_unlock(&sinfo.lock); -+ -+ return cidx; -+} -+ -+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = CLUSTER_SEPARATE_IDX; -+ if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) { -+ pr_debug(" %s iso_masks.little first is %d\n", -+ __func__, cpumask_first(iso_masks.little)); -+ ctx->upper = NON_PERIOD_END; -+ } -+ ctx->tidx = 0; -+ break; -+ case LITTLE: -+ ctx->lower = CLUSTER_SEPARATE_IDX; -+ ctx->upper = NON_PERIOD_END; -+ if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) { -+ pr_debug(" %s iso_masks.big first is %d", -+ __func__, cpumask_first(iso_masks.big)); -+ ctx->lower = NON_PERIOD_START; -+ } -+ ctx->tidx = 1; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ case LITTLE: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = NON_PERIOD_END; -+ ctx->tidx = 0; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ if (cluster_separate) { -+ return gen_cluster_ctx_separate(ctx, type); -+ } else { -+ return gen_cluster_ctx_common(ctx, type); -+ } -+} -+ -+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ int i; -+ bool is_timeout; -+ unsigned long flags; -+ struct cluster_ctx ctx; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ /* Third param means only consume timeout pcp dsq. */ -+ if (consume_pcp_dsq(rq, rf, false)) -+ return 1; -+ -+ for (i = ctx.lower; i < ctx.upper; i++) { -+ raw_spin_lock_irqsave(&gdsqs[i].lock, flags); -+ is_timeout = gdsqs[i].is_timeout; -+ raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags); -+ /* gdsqs[i].is_timeout may change here, let it be... */ -+ if (is_timeout) { -+ /* -+ * consume_dispatch_q will acquire dsq-lock, -+ * So cannot keep lock here, annoy enough. -+ * may rewrite a consume_dispatch_q_locked, TODO. -+ */ -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ hmbird_info_trace("dsq[%d] consume a timeout task\n", i); -+ slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0); -+ update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]); -+ return 1; -+ } -+ } -+ } -+ return 0; -+} -+ -+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ struct cluster_ctx ctx; -+ int cidx; -+ int tmp; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ cidx = get_cidx(&ctx); -+ tmp = cidx; -+ do { -+ if (check_pcp_dsq_round(rq, rf)) -+ return 1; -+ if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx)) -+ return 1; -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[tmp] = 0; -+ systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]); -+ tmp++; -+ if (tmp >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ tmp = ctx.lower; -+ } -+ spin_unlock(&sinfo.lock); -+ } while (tmp != cidx); -+ -+ return consume_pcp_dsq(rq, rf, true); -+} -+ -+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ enum cpu_type type = cpu_cluster(cpu_of(rq)); -+ bool period_allowed = !get_hmbird_rq(rq)->period_disallow; -+ bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow; -+ -+ switch (type) { -+ case EX_FREE: -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ break; -+ case EXCLUSIVE: -+ if (!is_iso_par_free()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ } -+ /* Only non-period task can run on isolate cpus */ -+ if (!READ_ONCE(iso_free_rescue)) { -+ if (is_big_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ } else { -+ /* Free run, can run on any task. */ -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ case PARTIAL: -+ if (!is_partial_enabled()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ return 0; -+ } -+ if (is_big_need_rescue()) { -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ -+ case BIG: -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ if (is_iso_par_free() || (is_little_need_rescue() && -+ !cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) { -+ hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++); -+ return 1; -+ } -+ } -+ break; -+ -+ case LITTLE: -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ -+ if (is_iso_par_free() || (is_big_need_rescue() && -+ !cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) { -+ hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++); -+ return 1; -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return 0; -+} -+ -+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf) -+{ -+ return consume_hmbird_global_dsq(rq, rf); -+} -+ -+ -+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time) -+{ -+ int idx; -+ -+ /* which dsq belongs to while task enqueue, task will consume its running time. */ -+ idx = get_hmbird_ts(p)->gdsq_idx; -+ /* Only non-period dsq share running time between each other. */ -+ if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id) -+ return; -+ -+ if (idx >= MAX_GLOBAL_DSQS) { -+ per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } else { -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[idx] += exec_time; -+ spin_unlock(&sinfo.lock); -+ systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]); -+ } -+} -+ -+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type) -+{ -+ int cidx; -+ struct cluster_ctx ctx; -+ int cpu = cpu_of(rq); -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ if (cidx < ctx.lower || cidx >= ctx.upper) { -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ slim_stats_record(ERR_IDX, ctx.tidx, 0, 0); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ } -+ -+ while (1) { -+ if (per_cpu(pcp_info, cpu).pcp_round) { -+ if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) { -+ hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n", -+ cpu, per_cpu(pcp_info, cpu).rtime); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } -+ } -+ if (sinfo.rtime[cidx] < dsq_quota[cidx]) -+ break; -+ -+ /* clear current dsq rtime */ -+ sinfo.rtime[cidx] = 0; -+ systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]); -+ -+ sinfo.curr_idx[ctx.tidx]++; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", -+ ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ } -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ slim_stats_record(SWITCH_IDX, 1, 0, 0); -+ } -+ spin_unlock(&sinfo.lock); -+} -+ -+ -+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p) -+{ -+ enum cpu_type type; -+ -+ if (!rq || !p) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ switch (type) { -+ case PARTIAL: -+ return; -+ case EXCLUSIVE: -+ return; -+ case EX_FREE: -+ return; -+ default: -+ break; -+ } -+ update_dsq_idx(rq, p, type); -+} -+ -+ -+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ int dsq_id; -+ -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo) || dsq->is_timeout) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ hmbird_deferred_err(SCAN_ENTITY_NULL, -+ " : entity is NULL, dsq->id = %llu\n", dsq->id); -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ dsq->is_timeout = true; -+ dsq_id = dsq_id_to_internal(dsq); -+ hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n", -+ dsq_id, entity->task->comm, jiffies, entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1); -+ raw_spin_unlock(&dsq->lock); -+ -+ return true; -+} -+ -+void scan_timeout(struct rq *rq) -+{ -+ int i; -+ int cpu = cpu_of(rq); -+ struct hmbird_dispatch_q *dsq; -+ static u64 last_scan_at; -+ static DEFINE_PER_CPU(u64, pcp_last_scan_at); -+ -+ if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu))) -+ return; -+ per_cpu(pcp_last_scan_at, cpu) = jiffies; -+ -+ dsq = &per_cpu(pcp_ldsq, cpu); -+ scan_dsq_timeout(rq, dsq, pcp_dsq_deadline); -+ -+ if (time_before_eq(jiffies, (unsigned long)last_scan_at)) -+ return; -+ last_scan_at = jiffies; -+ -+ for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) { -+ dsq = &gdsqs[i]; -+ scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]); -+ } -+} -+ -+/*******************************Initialize***********************************/ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id); -+static void init_dsq_at_boot(void) -+{ -+ int i, cpu; -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i)); -+ } -+ for_each_possible_cpu(cpu) -+ init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i + cpu)); -+ -+ max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu; -+ spin_lock_init(&sinfo.lock); -+} -+ -+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgrp_name_to_idx(struct cgroup *cgrp) -+{ -+ int idx; -+ -+ if (!cgrp) -+ return -1; -+ -+ if (!strcmp(cgrp->kn->name, "display") -+ || !strcmp(cgrp->kn->name, "multimedia")) -+ idx = 5; /* 8ms */ -+ else if (!strcmp(cgrp->kn->name, "top-app") -+ || !strcmp(cgrp->kn->name, "ss-top")) -+ idx = 6; /* 16ms */ -+ else if (!strcmp(cgrp->kn->name, "ssfg") -+ || !strcmp(cgrp->kn->name, "foreground")) -+ idx = 7; /* 32ms */ -+ else if (!strcmp(cgrp->kn->name, "bg") -+ || !strcmp(cgrp->kn->name, "log") -+ || !strcmp(cgrp->kn->name, "dex2oat") -+ || !strcmp(cgrp->kn->name, "background")) -+ idx = 9; /* 128ms */ -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */ -+ -+ return idx; -+} -+ -+static void init_root_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX); -+} -+ -+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) { -+ pr_err("%s idx err!\n", __func__); -+ return; -+ } -+ -+ if (cgroup_ids_table[cgrp->kn->id] == -1) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp)); -+} -+ -+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ struct cgroup *l1cgrp; -+ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ l1cgrp = cgroup_ancestor_l1(cgrp); -+ if (l1cgrp) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp)); -+ cgroup_put(l1cgrp); -+} -+ -+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg) -+{ -+ switch (cgrp->level) { -+ case 0: -+ init_root_tg(cgrp, tg); -+ break; -+ case 1: -+ init_level1_tg(cgrp, tg); -+ break; -+ default: -+ init_child_tg(cgrp, tg); -+ break; -+ } -+} -+ -+/**************************************************************************/ -+ -+struct hmbird_task_iter { -+ struct hmbird_entity cursor; -+ struct task_struct *locked; -+ struct rq *rq; -+ struct rq_flags rf; -+}; -+ -+/** -+ * hmbird_task_iter_init - Initialize a task iterator -+ * @iter: iterator to init -+ * -+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized, -+ * @iter must eventually be exited with hmbird_task_iter_exit(). -+ * -+ * hmbird_tasks_lock may be released between this and the first next() call or -+ * between any two next() calls. If hmbird_tasks_lock is released between two -+ * next() calls, the caller is responsible for ensuring that the task being -+ * iterated remains accessible either through RCU read lock or obtaining a -+ * reference count. -+ * -+ * All tasks which existed when the iteration started are guaranteed to be -+ * visited as long as they still exist. -+ */ -+static void hmbird_task_iter_init(struct hmbird_task_iter *iter) -+{ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR }; -+ list_add(&iter->cursor.tasks_node, &hmbird_tasks); -+ iter->locked = NULL; -+} -+ -+/** -+ * hmbird_task_iter_exit - Exit a task iterator -+ * @iter: iterator to exit -+ * -+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held. -+ * If the iterator holds a task's rq lock, that rq lock is released. See -+ * hmbird_task_iter_init() for details. -+ */ -+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ if (list_empty(cursor)) -+ return; -+ -+ list_del_init(cursor); -+} -+ -+/** -+ * hmbird_task_iter_next - Next task -+ * @iter: iterator to walk -+ * -+ * Visit the next task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ struct hmbird_entity *pos; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ list_for_each_entry(pos, cursor, tasks_node) { -+ if (&pos->tasks_node == &hmbird_tasks) -+ return NULL; -+ if (!(pos->flags & HMBIRD_TASK_CURSOR)) { -+ list_move(cursor, &pos->tasks_node); -+ return pos->task; -+ } -+ } -+ -+ /* can't happen, should always terminate at hmbird_tasks above */ -+ hmbird_deferred_err(ITER_RET_NULL, " : unreachable path in scx_task_iter_next\n"); -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered - Next non-idle task -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ while ((p = hmbird_task_iter_next(iter))) { -+ if (!is_idle_task(p)) -+ return p; -+ } -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init() -+ * for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ p = hmbird_task_iter_next_filtered(iter); -+ if (!p) -+ return NULL; -+ -+ iter->rq = task_rq_lock(p, &iter->rf); -+ iter->locked = p; -+ return p; -+} -+ -+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void) -+{ -+ return atomic_read(&hmbird_ops_enable_state_var); -+} -+ -+static enum hmbird_ops_enable_state -+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to) -+{ -+ return atomic_xchg(&hmbird_ops_enable_state_var, to); -+} -+ -+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to, -+ enum hmbird_ops_enable_state from) -+{ -+ int from_v = from; -+ -+ return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to); -+} -+ -+static bool hmbird_ops_disabling(void) -+{ -+ return false; -+} -+ -+/** -+ * wait_ops_state - Busy-wait the specified ops state to end -+ * @p: target task -+ * @opss: state to wait the end of -+ * -+ * Busy-wait for @p to transition out of @opss. This can only be used when the -+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also -+ * has load_acquire semantics to ensure that the caller can see the updates made -+ * in the enqueueing and dispatching paths. -+ */ -+static void wait_ops_state(struct task_struct *p, u64 opss) -+{ -+ do { -+ cpu_relax(); -+ } while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss); -+} -+ -+ -+static void update_curr_hmbird(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ u64 now = rq_clock_task(rq); -+ u64 delta_exec; -+ -+ if (time_before_eq64(now, curr->se.exec_start)) -+ return; -+ -+ delta_exec = now - curr->se.exec_start; -+ curr->se.exec_start = now; -+ update_runningtime(rq, curr, delta_exec); -+ curr->se.sum_exec_runtime += delta_exec; -+ account_group_exec_runtime(curr, delta_exec); -+ cgroup_account_cputime(curr, delta_exec); -+ -+ if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF) -+ get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec); -+ -+ trace_sched_stat_runtime(curr, delta_exec, 0); -+} -+ -+static bool hmbird_dsq_priq_less(struct rb_node *node_a, -+ const struct rb_node *node_b) -+{ -+ const struct hmbird_entity *a = -+ container_of(node_a, struct hmbird_entity, dsq_node.priq); -+ const struct hmbird_entity *b = -+ container_of(node_b, struct hmbird_entity, dsq_node.priq); -+ -+ return time_before64(a->dsq_vtime, b->dsq_vtime); -+} -+ -+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ bool is_local = dsq->id == HMBIRD_DSQ_LOCAL; -+ unsigned long flags; -+ -+ hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq || -+ !list_empty(&get_hmbird_ts(p)->dsq_node.fifo), -+ "task = %s, dsq->id = %llu\n", p->comm, dsq->id); -+ hmbird_cond_deferred_err(ENQ_EXIST2, -+ (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq), -+ "task = %s\n", p->comm); -+ -+ if (!is_local) { -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ); -+ hmbird_ops_error(": %s\n", -+ "attempting to dispatch to a destroyed dsq"); -+ /* fall back to the global dsq */ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ dsq = &hmbird_dsq_global; -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ } -+ } -+ -+ if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) { -+ get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ; -+ rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq, -+ hmbird_dsq_priq_less); -+ } else { -+ if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT)) -+ list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ else -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ } -+ dsq->nr++; -+ get_hmbird_ts(p)->dsq = dsq; -+ -+ /* -+ * We're transitioning out of QUEUEING or DISPATCHING. store_release to -+ * match waiters' load_acquire. -+ */ -+ if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS) -+ atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE); -+ -+ if (is_local) { -+ struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq); -+ struct rq *rq = hmbird->rq; -+ bool preempt = false; -+ -+ if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr && -+ rq->curr->sched_class == &hmbird_sched_class) { -+ get_hmbird_ts(rq->curr)->slice = 0; -+ preempt = true; -+ } -+ -+ if (preempt || sched_class_above(&hmbird_sched_class, -+ rq->curr->sched_class)) -+ resched_curr(rq); -+ } else { -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ } -+} -+ -+static void task_unlink_from_dsq(struct task_struct *p, -+ struct hmbird_dispatch_q *dsq) -+{ -+ if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) { -+ rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ; -+ } else { -+ list_del_init(&get_hmbird_ts(p)->dsq_node.fifo); -+ } -+} -+ -+static bool task_linked_on_dsq(struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+} -+ -+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq; -+ bool is_local = dsq == &hmbird_rq->local_dsq; -+ -+ if (!dsq) { -+ hmbird_cond_deferred_err(TASK_LINKED1, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ /* -+ * When dispatching directly from the BPF scheduler to a local -+ * DSQ, the task isn't associated with any DSQ but -+ * @get_hmbird_ts(p)->holding_cpu may be set under the protection of -+ * %HMBIRD_OPSS_DISPATCHING. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu >= 0) -+ get_hmbird_ts(p)->holding_cpu = -1; -+ return; -+ } -+ -+ if (!is_local) -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ /* -+ * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node -+ * can't change underneath us. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu < 0) { -+ /* @p must still be on @dsq, dequeue */ -+ hmbird_cond_deferred_err(TASK_UNLINKED, -+ !task_linked_on_dsq(p), "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ } else { -+ /* -+ * We're racing against dispatch_to_local_dsq() which already -+ * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the -+ * holding_cpu which tells dispatch_to_local_dsq() that it lost -+ * the race. -+ */ -+ hmbird_cond_deferred_err(TASK_LINKED2, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ get_hmbird_ts(p)->holding_cpu = -1; -+ } -+ get_hmbird_ts(p)->dsq = NULL; -+ -+ if (!is_local) -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+ -+static bool test_rq_online(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ return rq->online; -+#else -+ return true; -+#endif -+} -+ -+static void refill_task_slice(struct task_struct *p) -+{ -+ if (is_isolate_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO; -+ else if (is_pipeline_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2; -+ else -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+} -+ -+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -+ int sticky_cpu) -+{ -+ struct hmbird_dispatch_q *d; -+ s32 cpu = -1; -+ -+ hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), -+ "task = %s\n", p->comm); -+ -+ if (is_pcp_rt(p)) { -+ /* Enqueue percpu rt task to local directly. */ -+ /* Or cause a bug when disable dispatch. */ -+ if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ } -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (cpu >= 0) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ -+ if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ /* rq migration */ -+ if (sticky_cpu == cpu_of(rq)) -+ goto local_norefill; -+ /* -+ * If !rq->online, we already told the BPF scheduler that the CPU is -+ * offline. We're just trying to on/offline the CPU. Don't bother the -+ * BPF scheduler. -+ */ -+ if (unlikely(!test_rq_online(rq))) -+ goto local; -+ -+ /* see %HMBIRD_OPS_ENQ_LAST */ -+ if (enq_flags & HMBIRD_ENQ_LAST) -+ goto local; -+ -+ if (enq_flags & HMBIRD_ENQ_LOCAL) -+ goto local; -+ else -+ goto global; -+local: -+ /* -+ * For task-ordering, slice refill must be treated as implying the end -+ * of the current slice. Otherwise, the longer @p stays on the CPU, the -+ * higher priority it becomes from hmbird_prio_less()'s POV. -+ */ -+ refill_task_slice(p); -+local_norefill: -+ dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags); -+ slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq)); -+ return; -+ -+global: -+ d = find_dsq_from_task(p); -+ if (d) { -+ refill_task_slice(p); -+ dispatch_enqueue(d, p, enq_flags); -+ return; -+ } -+ slim_stats_record(GLOBAL_STAT, 0, 0, 0); -+ refill_task_slice(p); -+ dispatch_enqueue(&hmbird_dsq_global, p, enq_flags); -+} -+ -+static bool watchdog_task_watched(const struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->watchdog_node); -+} -+ -+static void watchdog_watch_task(struct rq *rq, struct task_struct *p) -+{ -+ lockdep_assert_rq_held(rq); -+ if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ get_hmbird_ts(p)->runnable_at = jiffies; -+ clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list); -+} -+ -+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) -+{ -+ list_del_init(&get_hmbird_ts(p)->watchdog_node); -+ if (reset_timeout) -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags) -+{ -+ int sticky_cpu = get_hmbird_ts(p)->sticky_cpu; -+ -+ enq_flags |= get_hmbird_rq(rq)->extra_enq_flags; -+ -+ if (sticky_cpu >= 0) -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ -+ /* -+ * Restoring a running task will be immediately followed by -+ * set_next_task_hmbird() which expects the task to not be on the BPF -+ * scheduler as tasks can only start running through local DSQs. Force -+ * direct-dispatch into the local DSQ by setting the sticky_cpu. -+ */ -+ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -+ sticky_cpu = cpu_of(rq); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_UNWATCHED, -+ !watchdog_task_watched(p), "task = %s\n", p->comm); -+ return; -+ } -+ -+ watchdog_watch_task(rq, p); -+ set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ get_hmbird_rq(rq)->nr_running++; -+ add_nr_running(rq, 1); -+ -+ do_enqueue_task(rq, p, enq_flags, sticky_cpu); -+} -+ -+static void ops_dequeue(struct task_struct *p, u64 deq_flags) -+{ -+ u64 opss; -+ -+ watchdog_unwatch_task(p, false); -+ -+ /* acquire ensures that we see the preceding updates on QUEUED */ -+ opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state); -+ -+ switch (opss & HMBIRD_OPSS_STATE_MASK) { -+ case HMBIRD_OPSS_NONE: -+ break; -+ case HMBIRD_OPSS_QUEUEING: -+ /* -+ * QUEUEING is started and finished while holding @p's rq lock. -+ * As we're holding the rq lock now, we shouldn't see QUEUEING. -+ */ -+ hmbird_deferred_err(DEQ_DEQING, " : unreachable path in %s\n", __func__); -+ break; -+ case HMBIRD_OPSS_QUEUED: -+ if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss, -+ HMBIRD_OPSS_NONE)) -+ break; -+ fallthrough; -+ case HMBIRD_OPSS_DISPATCHING: -+ /* -+ * If @p is being dispatched from the BPF scheduler to a DSQ, -+ * wait for the transfer to complete so that @p doesn't get -+ * added to its DSQ after dequeueing is complete. -+ * -+ * As we're waiting on DISPATCHING with the rq locked, the -+ * dispatching side shouldn't try to lock the rq while -+ * DISPATCHING is set. See dispatch_to_local_dsq(). -+ * -+ * DISPATCHING shouldn't have qseq set and control can reach -+ * here with NONE @opss from the above QUEUED case block. -+ * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss. -+ */ -+ wait_ops_state(p, HMBIRD_OPSS_DISPATCHING); -+ hmbird_cond_deferred_err(HMBIRD_OPN, -+ atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE, -+ "task = %s\n", p->comm); -+ break; -+ } -+} -+ -+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ -+ if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p), -+ "task = %s\n", p->comm); -+ return; -+ } -+ -+ ops_dequeue(p, deq_flags); -+ -+ if (slim_walt_ctrl) { -+ if (task_current(rq, p)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ if (deq_flags & HMBIRD_DEQ_SLEEP) -+ set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else -+ clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), -+ (unsigned long *)&get_hmbird_ts(p)->flags); -+ -+ clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm); -+ hmbird_rq->nr_running--; -+ sub_nr_running(rq, 1); -+ dispatch_dequeue(hmbird_rq, p); -+} -+ -+static void yield_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ get_hmbird_ts(p)->slice = 0; -+} -+ -+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to) -+{ -+ return false; -+} -+ -+#ifdef CONFIG_SMP -+/** -+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -+ * @rq: rq to move the task into, currently locked -+ * @p: task to move -+ * @enq_flags: %HMBIRD_ENQ_* -+ * -+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller -+ * must: -+ * -+ * 1. Start with exclusive access to @p either through its DSQ lock or -+ * %HMBIRD_OPSS_DISPATCHING flag. -+ * -+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id(). -+ * -+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't -+ * deadlock with dequeue. -+ * -+ * 4. Lock @rq and the task_rq from #3. -+ * -+ * 5. Call this function. -+ * -+ * Returns %true if @p was successfully moved. %false after racing dequeue and -+ * losing. -+ */ -+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ struct rq *task_rq; -+ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * If dequeue got to @p while we were trying to lock both rq's, it'd -+ * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have -+ * updated it to different values afterwards, as this operation can't be -+ * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become -+ * raw_smp_processor_id() again before we're done. Thus, we can tell -+ * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is -+ * still raw_smp_processor_id(). -+ * -+ * See dispatch_dequeue() for the counterpart. -+ */ -+ if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id())) -+ return false; -+ -+ /* @p->rq couldn't have changed if we're still the holding cpu */ -+ task_rq = task_rq(p); -+ lockdep_assert_rq_held(task_rq); -+ deactivate_task(task_rq, p, 0); -+ set_task_cpu(p, cpu_of(rq)); -+ get_hmbird_ts(p)->sticky_cpu = cpu_of(rq); -+ -+ /* -+ * We want to pass hmbird-specific enq_flags but activate_task() will -+ * truncate the upper 32 bit. As we own @rq, we can pass them through -+ * @get_hmbird_rq(rq)->extra_enq_flags instead. -+ */ -+ hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags, -+ "task = %s\n", p->comm); -+ get_hmbird_rq(rq)->extra_enq_flags = enq_flags; -+ activate_task(rq, p, 0); -+ get_hmbird_rq(rq)->extra_enq_flags = 0; -+ -+ return true; -+} -+ -+#endif /* CONFIG_SMP */ -+ -+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu) -+{ -+ int fitable = 1; -+ -+ return fitable; -+} -+ -+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq, -+ struct hmbird_dispatch_q *dsq) -+{ -+ bool dsq_misfit; -+ int cpu = cpu_of(rq); -+ u64 task_util = 0; -+ struct cluster_ctx ctx; -+ int dsq_int = dsq_id_to_internal(dsq); -+ -+ if (!cpumask_test_cpu(cpu, iso_masks.little)) -+ return false; -+ if (p->pid == scx_systemui_pid) -+ return true; -+ -+ gen_cluster_ctx(&ctx, BIG); -+ dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 && -+ dsq_int <= SCHED_PROP_DEADLINE_LEVEL4); -+#ifdef CLUSTER_SEPARATE -+ /* In rescue mode, little will consume big cluster's dsq.*/ -+ dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper); -+#endif -+ if (!dsq_misfit) -+ return false; -+ -+ if (p) { -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &task_util); -+ else -+ task_util = get_hmbird_ts(p)->demand_scaled; -+ } -+ -+ if (task_util <= misfit_ds) -+ return false; -+ -+ hmbird_info_trace(":task %s can't run on cpu%d, util = %llu\n", -+ p->comm, cpu, task_util); -+ return true; -+} -+ -+static int check_misfit_task_on_fake_big(struct task_struct *p, struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (likely(p->pid != scx_systemui_pid)) -+ return false; -+ -+ if (topology_cluster_id(num_possible_cpus() - 1) > 1 && -+ arch_scale_cpu_capacity(cpu) == arch_scale_cpu_capacity(0) && -+ (parctrl_high_ratio <= 0 || parctrl_high_ratio_l <= 0)) -+ return true; -+ -+ return false; -+} -+ -+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq) -+{ -+ if (!cpumask_test_cpu(cpu_of(rq), task_cpu_possible_mask(p))) -+ return false; -+ -+ if (!task_fits_cpu_hmbird(p, cpu_of(rq))) -+ return false; -+ -+ if (check_misfit_task_on_little(p, rq, dsq)) -+ return false; -+ -+ if (check_misfit_task_on_fake_big(p, rq)) -+ return false; -+ -+ return likely(test_rq_online(rq)); -+} -+ -+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return; -+ -+ if (add) -+ skipn[idx]++; -+ else -+ skipn[idx] = 0; -+} -+ -+static bool skip_too_much(struct hmbird_dispatch_q *dsq) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return false; -+ -+ if (skip_num[idx] > 3) { -+ skip_num[idx] = 0; -+ return true; -+ } -+ -+ return false; -+} -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rb_node *rb_node; -+ struct rq *task_rq; -+ unsigned long flags; -+ bool moved = false; -+ struct task_struct *may_fit = NULL; -+ int skip = 0; -+ -+retry: -+ if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -+ return false; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) { -+ set_skip_num(dsq, skip_num, (bool)may_fit); -+ goto this_rq; -+ } -+ if (skip_too_much(dsq)) -+ goto remote_rq; -+ if (!may_fit) -+ may_fit = p; -+ if (++skip <= 3) -+ continue; -+ /* -+ * If the recent 3 tasks not fit, use the first one. -+ * and clear the skip, because the first one is consumed. -+ */ -+ set_skip_num(dsq, skip_num, false); -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ /* No more task, use the first may fit task.*/ -+ if (may_fit) { -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ -+ for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) -+ goto this_rq; -+ goto remote_rq; -+ } -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ return false; -+ -+this_rq: -+ /* @dsq is locked and @p is on this rq */ -+ hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo); -+ dsq->nr--; -+ hmbird_rq->local_dsq.nr++; -+ get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ -+remote_rq: -+#ifdef CONFIG_SMP -+ if (cpu_same_cluster_stat(p, rq, task_rq)) -+ slim_stats_record(MOVE_RQ_CNT, 0, 0, 0); -+ else -+ slim_stats_record(MOVE_RQ_CNT, 1, 0, 0); -+ /* -+ * @dsq is locked and @p is on a remote rq. @p is currently protected by -+ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -+ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -+ * rq lock or fail, do a little dancing from our side. See -+ * move_task_to_local_dsq(). -+ */ -+ hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id(); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ rq_unpin_lock(rq, rf); -+ double_lock_balance(rq, task_rq); -+ rq_repin_lock(rq, rf); -+ -+ moved = move_task_to_local_dsq(rq, p, 0); -+ -+ double_unlock_balance(rq, task_rq); -+#endif /* CONFIG_SMP */ -+ if (likely(moved)) { -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ } -+ may_fit = NULL; -+ goto retry; -+} -+ -+ -+static int balance_one(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf, bool local) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class; -+ -+ if (!hmbird_rq) -+ return 1; -+ -+ lockdep_assert_rq_held(rq); -+ -+ if (static_branch_unlikely(&hmbird_ops_cpu_preempt) && -+ unlikely(get_hmbird_rq(rq)->cpu_released)) { -+ /* -+ * If the previous sched_class for the current CPU was not HMBIRD, -+ * notify the BPF scheduler that it again has control of the -+ * core. This callback complements ->cpu_release(), which is -+ * emitted in hmbird_notify_pick_next_task(). -+ */ -+ get_hmbird_rq(rq)->cpu_released = false; -+ } -+ -+ if (prev_on_hmbird) -+ update_curr_hmbird(rq); -+ /* if there already are tasks to run, nothing to do */ -+ if (hmbird_rq->local_dsq.nr) -+ return 1; -+ -+ if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) { -+ slim_stats_record(GLOBAL_STAT, 1, 0, 0); -+ return 1; -+ } -+ -+ if (consume_dispatch_global(rq, rf)) -+ return 1; -+ -+ return 0; -+} -+ -+static int balance_hmbird(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf) -+{ -+ return balance_one(rq, prev, rf, true); -+} -+ -+/* -+ * output task util to systrace, only for debug mode. -+ * we can not output too many logs to systrace buffer even in debug mode -+ * only output debug-info while it exceed misfit_ds. -+ */ -+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p) -+{ -+ static DEFINE_PER_CPU(int, is_last_exceed); -+ int cpu = cpu_of(rq); -+ u64 util = 0; -+ -+ if (likely(!debug_enabled())) -+ return; -+ -+ if (!p) -+ return; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ util = uclamp_rq_util_with(rq, util, p); -+ -+ if (util >= misfit_ds) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util); -+ per_cpu(is_last_exceed, cpu) = true; -+ } else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0); -+ per_cpu(is_last_exceed, cpu) = false; -+ } else { -+ } -+} -+ -+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first) -+{ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ /* -+ * Core-sched might decide to execute @p before it is -+ * dispatched. Call ops_dequeue() to notify the BPF scheduler. -+ */ -+ ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC); -+ dispatch_dequeue(get_hmbird_rq(rq), p); -+ } -+ -+ p->se.exec_start = rq_clock_task(rq); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK); -+ } -+ -+ watchdog_unwatch_task(p, true); -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx); -+ systrace_output_cpu_ds(rq, p); -+ /* -+ * @p is getting newly scheduled or got kicked after someone updated its -+ * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird(). -+ */ -+ if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) != -+ (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) { -+ if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) -+ get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK; -+ else -+ get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK; -+ -+ sched_update_tick_dependency(rq); -+ } -+ -+ p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime; -+} -+ -+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p) -+{ -+ update_curr_hmbird(rq); -+ -+ update_dispatch_dsq_info(rq, p); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ watchdog_watch_task(rq, p); -+ -+ if (is_pipeline_task(p)) { -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1); -+ return; -+ } -+ -+ /* -+ * If we're in the pick_next_task path, balance_hmbird() should -+ * have already populated the local DSQ if there are any other -+ * available tasks. If empty, tell ops.enqueue() that @p is the -+ * only one available for this cpu. ops.enqueue() should put it -+ * on the local DSQ so that the subsequent pick_next_task_hmbird() -+ * can find the task unless it wants to trigger a separate -+ * follow-up scheduling event. -+ */ -+ if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1); -+ else -+ do_enqueue_task(rq, p, 0, -1); -+ } -+} -+ -+static struct task_struct *first_local_task(struct rq *rq) -+{ -+ struct rb_node *rb_node; -+ struct hmbird_entity *entity; -+ -+ if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) { -+ entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo, -+ struct hmbird_entity, dsq_node.fifo); -+ return entity->task; -+ } -+ -+ rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq); -+ if (rb_node) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ return entity->task; -+ } -+ return NULL; -+} -+ -+static struct task_struct *pick_next_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p; -+ -+ p = first_local_task(rq); -+ if (!p) -+ return NULL; -+ -+ if (unlikely(!get_hmbird_ts(p)->slice)) { -+ if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice) -+ hmbird_warned_zero_slice = true; -+ -+ refill_task_slice(p); -+ } -+ -+ set_next_task_hmbird(rq, p, true); -+ -+ return p; -+} -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task, -+ const struct sched_class *active) -+{ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * The callback is conceptually meant to convey that the CPU is no -+ * longer under the control of HMBIRD. Therefore, don't invoke the -+ * callback if the CPU is staying on HMBIRD, or going idle (in which -+ * case the HMBIRD scheduler has actively decided not to schedule any -+ * tasks on the CPU). -+ */ -+ if (likely(active >= &hmbird_sched_class)) -+ return; -+ -+ /* -+ * At this point we know that HMBIRD was preempted by a higher priority -+ * sched_class, so invoke the ->cpu_release() callback if we have not -+ * done so already. We only send the callback once between HMBIRD being -+ * preempted, and it regaining control of the CPU. -+ * -+ * ->cpu_release() complements ->cpu_acquire(), which is emitted the -+ * next time that balance_hmbird() is invoked. -+ */ -+ if (!get_hmbird_rq(rq)->cpu_released) -+ get_hmbird_rq(rq)->cpu_released = true; -+} -+ -+#ifdef CONFIG_SMP -+ -+static bool test_and_clear_cpu_idle(int cpu) -+{ -+ if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -+ if (cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ return true; -+ } else { -+ return false; -+ } -+} -+ -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) -+{ -+ int cpu; -+ -+ do { -+ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -+ if (cpu >= nr_cpu_ids) -+ return -EBUSY; -+ } while (!test_and_clear_cpu_idle(cpu)); -+ -+ return cpu; -+} -+ -+ -+static bool prev_cpu_misfit(int prev) -+{ -+ if (!is_partial_enabled() && is_partial_cpu(prev)) -+ return true; -+ -+ return false; -+} -+ -+static int heavy_rt_placement(struct task_struct *p, int prev) -+{ -+ struct cpumask tmp = {.bits = {0}}; -+ int cpu; -+ u64 util = 0; -+ -+ if (!rt_prio(p->prio)) -+ return -EFAULT; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ -+ if (util < misfit_ds) -+ return -EFAULT; -+ -+ if (is_partial_enabled()) -+ cpumask_or(&tmp, iso_masks.big, iso_masks.partial); -+ else -+ cpumask_copy(&tmp, iso_masks.big); -+ -+ cpu = hmbird_pick_idle_cpu(&tmp); -+ if (cpu >= 0) -+ return cpu; -+ -+ if (cpumask_test_cpu(prev, iso_masks.big) || -+ (is_partial_enabled() && is_partial_cpu(prev))) -+ return prev; -+ -+ return cpumask_first(&tmp); -+} -+ -+static int spec_task_before_pick_idle(struct task_struct *p, int prev) -+{ -+ int cpu; -+ -+ cpu = heavy_rt_placement(p, prev); -+ if (cpu >= 0) -+ return cpu; -+ return -EFAULT; -+} -+ -+static int cpumask_distribute_next(struct cpumask *mask, int *prev) -+{ -+ int p = *prev, n; -+ -+ n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1); -+ if (n < nr_cpu_ids) -+ WRITE_ONCE(*prev, n); -+ return n; -+} -+ -+static int repick_fallback_cpu(void) -+{ -+ /* -+ * partial cpu follow big cluster's Scheduling policy, -+ * simply return first bit cpu. -+ */ -+ return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev); -+} -+ -+/* -+ * Must return a valid cpu num, as this task's cpu. -+ */ -+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu, -+ struct cpumask *mask, int *prev_mask) -+{ -+ int cpu; -+ -+ cpu = hmbird_pick_idle_cpu(mask); -+ if (cpu >= 0) -+ return cpu; -+ return cpumask_distribute_next(mask, prev_mask); -+} -+ -+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type) -+{ -+ int idx; -+ struct cluster_ctx ctx; -+ -+ idx = find_idx_from_task(p); -+ if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS) -+ return false; -+ -+ gen_cluster_ctx(&ctx, type); -+ if (idx >= ctx.lower && idx < ctx.upper) -+ return true; -+ return false; -+} -+ -+static bool is_valid_cpu(int cpu) -+{ -+ return (cpu >= 0) && (cpu < nr_cpu_ids); -+} -+ -+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) -+{ -+ s32 cpu = -1; -+ struct cpumask mask = {.bits = {0}}; -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (is_valid_cpu(cpu)) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ partial_dynamic_ctrl(); -+ -+ if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ iso_masks.big, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ if (p->nr_cpus_allowed == 1) { -+ cpu = cpumask_any(p->cpus_ptr); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* For non-period global dsq, not contain pcp task. */ -+ if (task_only_blongs_to_cluster(p, LITTLE)) { -+ cpumask_copy(&mask, iso_masks.little); -+ if (unlikely(l_need_rescue)) { -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ cpumask_or(&mask, iso_masks.ex_free, &mask); -+ } -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &little_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ if (task_only_blongs_to_cluster(p, BIG)) { -+ cpumask_copy(&mask, iso_masks.big); -+ if (unlikely(b_need_rescue)) { -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ cpumask_or(&mask, iso_masks.ex_free, &mask); -+ } -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ cpu = spec_task_before_pick_idle(p, prev_cpu); -+ if (is_valid_cpu(cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* if the previous CPU is idle, dispatch directly to it */ -+ if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+ } -+ -+ cpu = hmbird_pick_idle_cpu(cpu_possible_mask); -+ if (is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ if (prev_cpu_misfit(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ cpu = repick_fallback_cpu(); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+} -+ -+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags) -+{ -+ return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags); -+} -+ -+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx) -+{ -+ set_cpus_allowed_common(p, ctx); -+} -+ -+static void reset_idle_masks(void) -+{ -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little); -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big); -+ if (is_partial_enabled()) -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial); -+ hmbird_has_idle_cpus = true; -+} -+ -+void __hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ int cpu = cpu_of(rq); -+ struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -+ -+ if (skip_update_idle()) -+ return; -+ -+ if (idle) { -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ if (!hmbird_has_idle_cpus) -+ hmbird_has_idle_cpus = true; -+ -+ /* -+ * idle_masks.smt handling is racy but that's fine as it's only -+ * for optimization and self-correcting. -+ */ -+ for_each_cpu(cpu, sib_mask) { -+ if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -+ return; -+ } -+ cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -+ } else { -+ cpumask_clear_cpu(cpu, idle_masks.cpu); -+ if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ -+ cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -+ } -+} -+ -+#else /* !CONFIG_SMP */ -+ -+static bool test_and_clear_cpu_idle(int cpu) { return false; } -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } -+static void reset_idle_masks(void) {} -+ -+#endif /* CONFIG_SMP */ -+ -+static bool check_rq_for_timeouts(struct rq *rq) -+{ -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rq_flags rf; -+ -+ rq_lock_irqsave(rq, &rf); -+ list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) { -+ unsigned long last_runnable; -+ -+ p = entity->task; -+ last_runnable = get_hmbird_ts(p)->runnable_at; -+ -+ if (unlikely(time_after(jiffies, -+ last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -+ -+ rq_unlock_irqrestore(rq, &rf); -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb", -+ p->comm, p->pid, -+ dur_ms / 1000, dur_ms % 1000, -+ get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0, -+ cpumask_pr_args(&p->cpus_mask)); -+ return 1; -+ } -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ return 0; -+} -+ -+static void hmbird_watchdog_workfn(struct work_struct *work) -+{ -+ int cpu; -+ bool timeout; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ -+ for_each_online_cpu(cpu) { -+ timeout = check_rq_for_timeouts(cpu_rq(cpu)); -+ if (unlikely(timeout)) -+ break; -+ -+ cond_resched(); -+ } -+ if (!timeout) -+ queue_delayed_work(system_unbound_wq, to_delayed_work(work), -+ hmbird_watchdog_timeout / 2); -+} -+ -+static void set_pcp_round(struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) { -+ per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round); -+ per_cpu(pcp_info, cpu).pcp_round = true; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+} -+ -+ -+/* -+ * Just for debug: output hmbird on/off state per 10s. -+ */ -+#define OUTPUT_INTVAL (msecs_to_jiffies(10 * 1000)) -+static void inform_hmbird_onoff_from_systrace(void) -+{ -+ static unsigned long __read_mostly next_print; -+ -+ if (time_before(jiffies, READ_ONCE(next_print))) -+ return; -+ -+ WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL); -+ hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio); -+} -+ -+void hmbird_notify_sched_tick(void) -+{ -+ unsigned long last_check; -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ hmbird_scheduler_tick(); -+ -+ if (!hmbird_enabled()) -+ return; -+ -+ last_check = hmbird_watchdog_timestamp; -+ if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -+ -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "watchdog failed to check in for %u.%03us", -+ dur_ms / 1000, dur_ms % 1000); -+ } -+ scan_timeout(rq); -+} -+ -+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued) -+{ -+ update_curr_hmbird(rq); -+ -+ set_pcp_round(rq); -+ -+ if (slim_walt_ctrl) -+ hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE); -+ /* -+ * While disabling, always resched and refresh core-sched timestamp as -+ * we can't trust the slice management or ops.core_sched_before(). -+ */ -+ if (hmbird_ops_disabling()) -+ get_hmbird_ts(curr)->slice = 0; -+ -+ if (!get_hmbird_ts(curr)->slice) -+ resched_curr(rq); -+ -+ inform_hmbird_onoff_from_systrace(); -+} -+ -+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg) -+{ -+ hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ get_hmbird_ts(p)->disallow = false; -+ -+ hmbird_sched_init_task(p); -+ -+ set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ return 0; -+} -+ -+static void hmbird_ops_enable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void hmbird_ops_disable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void set_task_hmbird_weight(struct task_struct *p) -+{ -+ u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -+ -+ get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight); -+} -+ -+/** -+ * refresh_hmbird_weight - Refresh a task's hmbird weight -+ * @p: task to refresh hmbird weight for -+ * -+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to -+ * enable easy access from the BPF scheduler. To keep it synchronized with the -+ * current task priority, this function should be called when a new task is -+ * created, priority is changed for a task on hmbird, and a task is switched -+ * to hmbird from other classes. -+ */ -+static void refresh_hmbird_weight(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ set_task_hmbird_weight(p); -+} -+ -+int hmbird_pre_fork(struct task_struct *p) -+{ -+ int ret = 0; -+ -+ p->android_oem_data1[HMBIRD_TS_IDX] = -+ (u64)(kzalloc(sizeof(struct hmbird_entity), GFP_KERNEL)); -+ if (!get_hmbird_ts(p)) -+ return -1; -+ -+ get_hmbird_ts(p)->dsq = NULL; -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node); -+ get_hmbird_ts(p)->flags = 0; -+ get_hmbird_ts(p)->weight = 0; -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ get_hmbird_ts(p)->holding_cpu = -1; -+ get_hmbird_ts(p)->kf_mask = 0; -+ atomic64_set(&get_hmbird_ts(p)->ops_state, 0); -+ get_hmbird_ts(p)->runnable_at = INITIAL_JIFFIES; -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+ get_hmbird_ts(p)->task = p; -+ hmbird_set_sched_prop(p, 0); -+ -+ get_hmbird_ts(p)->critical_affinity_cpu = -1; -+ get_hmbird_ts(p)->sched_class = &hmbird_sched_class; -+ get_hmbird_ts(p)->tick_hit_count = 0; -+ get_hmbird_ts(p)->start_jiffies = 0; -+ -+ /* -+ * BPF scheduler enable/disable paths want to be able to iterate and -+ * update all tasks which can become complex when racing forks. As -+ * enable/disable are very cold paths, let's use a percpu_rwsem to -+ * exclude forks. -+ */ -+ -+ return ret; -+} -+ -+void hmbird_post_fork(struct task_struct *p) -+{ -+ percpu_down_read(&hmbird_fork_rwsem); -+ -+ if (hmbird_enabled()) { -+ struct rq_flags rf; -+ struct rq *rq; -+ p->sched_class = &hmbird_sched_class; -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ rq = task_rq_lock(p, &rf); -+ /* -+ * Set the weight manually before calling ops.enable() so that -+ * the scheduler doesn't see a stale value if they inspect the -+ * task struct. We'll invoke ops.set_weight() afterwards, as it -+ * would be odd to receive a callback on the task before we -+ * tell the scheduler that it's been fully enabled. -+ */ -+ set_task_hmbird_weight(p); -+ hmbird_ops_enable_task(p); -+ refresh_hmbird_weight(p); -+ task_rq_unlock(rq, p, &rf); -+ } else { -+ if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ } -+ -+ spin_lock_irq(&hmbird_tasks_lock); -+ list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_cancel_fork(struct task_struct *p) -+{ -+ if (hmbird_enabled()) -+ hmbird_ops_disable_task(p); -+ put_hmbird_ts(p); -+} -+ -+void hmbird_free(struct task_struct *p) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+ list_del_init(&get_hmbird_ts(p)->tasks_node); -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+ -+ /* -+ * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED -> -+ * ENABLED transitions can't race us. Disable ops for @p. -+ */ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) || -+ test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ hmbird_ops_disable_task(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ put_hmbird_ts(p); -+} -+ -+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio) -+{ -+} -+ -+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type) -+{ -+ return (prop >> TOP_TASK_SHIFT) & (1 << type); -+} -+ -+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p) -+{ -+ uint32_t prop = get_top_task_prop(p); -+ -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE)) -+ return HMBIRD_TASK_PROP_PIPELINE; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) || -+ !task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) { -+ return HMBIRD_TASK_PROP_COMMON; -+ } -+ return HMBIRD_TASK_PROP_DEBUG_OR_LOG; -+} -+ -+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b) -+{ -+ int type_a = hmbird_get_task_type(a); -+ int type_b = hmbird_get_task_type(b); -+ -+ return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b]; -+} -+ -+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ enum cpu_type type; -+ int sp_dl; -+ struct task_struct *curr = NULL; -+ -+ switch (hmbird_preempt_policy) { -+ case HMBIRD_PREEMPT_POLICY_PRIO_BASED: -+ curr = rq->curr; -+ if (curr && hmbird_prio_higher(p, curr)) -+ goto preempt; -+ break; -+ default: -+ break; -+ } -+ if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) || -+ (is_critical_system_task(p) && !is_critical_system_task(rq->curr))) -+ goto preempt; -+ -+ sp_dl = find_idx_from_task(p); -+ if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled())) -+ return; -+ -+ if (rq->curr->prio > p->prio) -+ goto preempt; -+ -+ return; -+ -+preempt: -+ resched_curr(rq); -+} -+ -+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {} -+ -+int hmbird_check_setscheduler(struct task_struct *p, int policy) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ /* if disallow, reject transitioning into HMBIRD */ -+ if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) && -+ p->policy != policy && policy == SCHED_HMBIRD) -+ return -EACCES; -+ -+ return 0; -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool hmbird_can_stop_tick(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ if (hmbird_ops_disabling()) -+ return false; -+ -+ if (p->sched_class != &hmbird_sched_class) -+ return true; -+ -+ return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK; -+} -+#endif -+ -+int hmbird_tg_online(struct task_group *tg) -+{ -+ struct cgroup *cgrp; -+ -+ if (!tg) -+ return 0; -+ cgrp = tg->css.cgroup; -+ if (!cgrp || !(cgrp->kn)) -+ return 0; -+ update_cgroup_ids_table(cgrp->kn->id, -1); -+ return 0; -+} -+ -+/* -+ * Omitted operations: -+ * -+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -+ * task isn't tied to the CPU at that point. Preemption is implemented by -+ * resetting the victim task's slice to 0 and triggering reschedule on the -+ * target CPU. -+ * -+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -+ * -+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -+ * their current sched_class. Call them directly from sched core instead. -+ * -+ * - task_woken, switched_from: Unnecessary. -+ */ -+DEFINE_SCHED_CLASS(hmbird) = { -+ .enqueue_task = enqueue_task_hmbird, -+ .dequeue_task = dequeue_task_hmbird, -+ .yield_task = yield_task_hmbird, -+ .yield_to_task = yield_to_task_hmbird, -+ -+ .check_preempt_curr = check_preempt_curr_hmbird, -+ -+ .pick_next_task = pick_next_task_hmbird, -+ -+ .put_prev_task = put_prev_task_hmbird, -+ .set_next_task = set_next_task_hmbird, -+ -+#ifdef CONFIG_SMP -+ .balance = balance_hmbird, -+ .select_task_rq = select_task_rq_hmbird, -+ .set_cpus_allowed = set_cpus_allowed_hmbird, -+#endif -+ .task_tick = task_tick_hmbird, -+ -+ .switched_to = switched_to_hmbird, -+ .prio_changed = prio_changed_hmbird, -+ -+ .update_curr = update_curr_hmbird, -+ -+#ifdef CONFIG_UCLAMP_TASK -+ .uclamp_enabled = 0, -+#endif -+}; -+ -+/* -+ * Must with rq lock held. -+ */ -+bool task_is_hmbird(struct task_struct *p) -+{ -+ return p->sched_class == &hmbird_sched_class; -+} -+ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id) -+{ -+ memset(dsq, 0, sizeof(*dsq)); -+ -+ raw_spin_lock_init(&dsq->lock); -+ INIT_LIST_HEAD(&dsq->fifo); -+ dsq->id = dsq_id; -+} -+ -+static int hmbird_cgroup_init(void) -+{ -+ struct cgroup_subsys_state *css; -+ -+ css_for_each_descendant_pre(css, &root_task_group.css) { -+ struct task_group *tg = css_tg(css); -+ -+ cgrp_dsq_idx_init(css->cgroup, tg); -+ } -+ return 0; -+} -+ -+/* -+ * Used by sched_fork() and __setscheduler_prio() to pick the matching -+ * sched_class. dl/rt are already handled. -+ */ -+bool task_on_hmbird(struct task_struct *p) -+{ -+ return hmbird_enabled(); -+} -+ -+static void __setscheduler_prio(struct task_struct *p, int prio) -+{ -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) { -+ p->prio = prio; -+ return; -+ } else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (on_hmbird && rt_prio(prio)) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ -+ p->prio = prio; -+} -+ -+/* -+ * Heartbeat, avoid humbird keep running while APP already exit. -+ * Check whether APP send alive-signal periodly. -+ */ -+#define HEARTBEAT_TIMEOUT (msecs_to_jiffies(2500)) -+#define HEARTBEAT_CHECK_INTERVAL (msecs_to_jiffies(1000)) -+static struct timer_list hb_timer; -+static unsigned long next_hb; -+ -+void hb_timer_handler(struct timer_list *timer) -+{ -+ if (!heartbeat_enable) -+ goto refill; -+ -+ pr_info(": enter timer.\n"); -+ if (heartbeat) { -+ heartbeat = 0; -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ } -+ -+ /* can't detect heartbeat, disable ext. */ -+ if (time_after(jiffies, READ_ONCE(next_hb))) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT, -+ "can't detect heartbeat, disable ext\n"); -+ } -+refill: -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_start(void) -+{ -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_init(void) -+{ -+ timer_setup(&hb_timer, hb_timer_handler, 0); -+} -+ -+static void hb_timer_exit(void) -+{ -+ del_timer(&hb_timer); -+} -+ -+static bool check_and_disable_cpuhp(void) -+{ -+ struct rq *rq; -+ struct rq_flags rf; -+ int cpu; -+ -+ cpu_hotplug_disable(); -+ cpus_read_lock(); -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ rq_lock_irqsave(rq, &rf); -+ if (!rq->online) { -+ rq_unlock_irqrestore(rq, &rf); -+ goto err_offline; -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ cpus_read_unlock(); -+ return true; -+ -+err_offline: -+ cpus_read_unlock(); -+ cpu_hotplug_enable(); -+ return false; -+} -+ -+static void reenable_cpuhp(void) -+{ -+ cpu_hotplug_enable(); -+} -+ -+static void scheduler_switch_done(bool final_state) -+{ -+ /* set scx_enable again, switch may fail. */ -+ scx_enable = final_state; -+ /* reset heartbeat*/ -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ -+ if (final_state) -+ hb_timer_start(); -+ else -+ hb_timer_exit(); -+} -+ -+/** -+ * ss : curr switch state -+ * finish : success or fail -+ * enable : curr operation is diabling or enabling? -+ * fail_reason : string output when failed, empty string when success. -+ */ -+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason) -+{ -+ char *s1 = finish ? "finished" : "failed"; -+ char *s2 = enable ? "enabled" : "disabled"; -+ -+ hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss); -+ if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) { -+ sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type)); -+ hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n", -+ s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason); -+ } -+ curr_ss = ss; -+} -+ -+bool get_hmbird_ops_enabled(void) -+{ -+ return atomic_read(&__hmbird_ops_enabled); -+} -+ -+bool get_non_hmbird_task(void) -+{ -+ return atomic_read(&non_hmbird_task); -+} -+ -+static void hmbird_ops_disable_workfn(struct kthread_work *work) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int cpu; -+ -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, ""); -+ cancel_delayed_work_sync(&hmbird_watchdog_work); -+ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) { -+ case HMBIRD_OPS_DISABLED: -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled"); -+ scheduler_switch_done(false); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return; -+ case HMBIRD_OPS_PREPPING: -+ fallthrough; -+ case HMBIRD_OPS_DISABLING: -+ /* shouldn't happen but handle it like ENABLING if it does */ -+ WARN_ONCE(true, "hmbird: duplicate disabling instance?"); -+ fallthrough; -+ case HMBIRD_OPS_ENABLING: -+ case HMBIRD_OPS_ENABLED: -+ break; -+ } -+ -+ /* kick all CPUs to restore ticks */ -+ for_each_possible_cpu(cpu) -+ resched_cpu(cpu); -+ -+ /* avoid racing against fork and cgroup changes */ -+ cpus_read_lock(); -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, ""); -+ spin_lock_irq(&hmbird_tasks_lock); -+ atomic_set(&__hmbird_ops_enabled, false); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ bool alive = READ_ONCE(p->__state) != TASK_DEAD; -+ -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ get_hmbird_ts(p)->slice = min_t(u64, -+ get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL); -+ -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ if (alive) -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ -+ hmbird_ops_disable_task(p); -+ } -+ hmbird_task_iter_exit(&sti); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, ""); -+ -+ atomic_set(&non_hmbird_task, true); -+ /* no task is on hmbird, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt); -+ synchronize_rcu(); -+ -+ percpu_up_write(&hmbird_fork_rwsem); -+ cpus_read_unlock(); -+ -+ if (slim_walt_ctrl) -+ slim_walt_enable(false); -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ -+ hmbird_switch_log(HMBIRD_DISABLED, 1, 0, ""); -+ scheduler_switch_done(false); -+ reenable_cpuhp(); -+} -+ -+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn); -+ -+static void schedule_hmbird_ops_disable_work(void) -+{ -+ struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper); -+ -+ /* -+ * We may be called spuriously before the first bpf_hmbird_reg(). If -+ * hmbird_ops_helper isn't set up yet, there's nothing to do. -+ */ -+ if (helper) -+ kthread_queue_work(helper, &hmbird_ops_disable_work); -+} -+ -+static void hmbird_ops_disable(void) -+{ -+ schedule_hmbird_ops_disable_work(); -+} -+ -+static void hmbird_err_exit_workfn(struct work_struct *work) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ hmbird_ctrl(false); -+ -+ for_each_present_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy) -+ continue; -+ if (cpu != policy->cpu) -+ goto put; -+ down_write(&policy->rwsem); -+ WARN_ON(store_scaling_governor(policy, -+ saved_gov[cpu], strlen(saved_gov[cpu])) <= 0); -+ up_write(&policy->rwsem); -+ hmbird_info_trace(":restore origin gov : %s\n", saved_gov[cpu]); -+put: -+ cpufreq_cpu_put(policy); -+ } -+ memset((char *)saved_gov, 0, sizeof(saved_gov)); -+} -+ -+void hmbird_ops_exit(void) -+{ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); -+} -+ -+static struct kthread_worker *hmbird_create_rt_helper(const char *name) -+{ -+ struct kthread_worker *helper; -+ -+ helper = kthread_create_worker(KTW_FREEZABLE, name); -+ if (helper) -+ sched_set_fifo(helper->task); -+ return helper; -+} -+ -+static inline void set_audio_thread_sched_prop(struct task_struct *p) -+{ -+ struct cgroup_subsys_state *css; -+ -+ if (likely(p->prio >= MAX_RT_PRIO)) -+ return; -+ -+ rcu_read_lock(); -+ css = task_css(p, cpuset_cgrp_id); -+ if (!css) { -+ rcu_read_unlock(); -+ return; -+ } -+ rcu_read_unlock(); -+ -+ if (!strcmp(css->cgroup->kn->name, "audio-app")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+} -+ -+int scx_systemui_pid = -1; -+void set_systemui_thread_pid(struct task_struct *p) -+{ -+ if ((strcmp(p->comm, "ndroid.systemui") == 0) && (p->pid == p->tgid)) -+ scx_systemui_pid = p->pid; -+} -+ -+static int hmbird_ops_enable(void *unused) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int ret; -+ int tcnt = 0; -+ unsigned long long start = 0; -+ -+ if (!check_and_disable_cpuhp()) { -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline"); -+ return -EBUSY; -+ } -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, ""); -+ mutex_lock(&hmbird_ops_enable_mutex); -+ -+ if (!hmbird_ops_helper) { -+ WRITE_ONCE(hmbird_ops_helper, -+ hmbird_create_rt_helper("hmbird_ops_helper")); -+ if (!hmbird_ops_helper) { -+ ret = -ENOMEM; -+ goto err_unlock; -+ } -+ } -+ -+ if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) { -+ ret = -EBUSY; -+ goto err_unlock; -+ } -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) != -+ HMBIRD_OPS_DISABLED); -+ -+ hmbird_warned_zero_slice = false; -+ -+ atomic64_set(&hmbird_nr_rejected, 0); -+ -+ /* -+ * Keep CPUs stable during enable so that the BPF scheduler can track -+ * online CPUs by watching ->on/offline_cpu() after ->init(). -+ */ -+ cpus_read_lock(); -+ -+ hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work, -+ hmbird_watchdog_timeout / 2); -+ -+ /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ */ -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ reset_idle_masks(); -+ -+ /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. -+ */ -+ ret = hmbird_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock; -+ -+ /* -+ * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem -+ * preventing new tasks from being added. No need to exclude tasks -+ * leaving as hmbird_free() can handle both prepped and enabled -+ * tasks. Prep all tasks first and then enable them with preemption -+ * disabled. -+ */ -+ spin_lock_irq(&hmbird_tasks_lock); -+ -+ atomic_set(&non_hmbird_task, false); -+ atomic_set(&__hmbird_ops_enabled, true); -+ -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered(&sti))) -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ hmbird_task_iter_exit(&sti); -+ -+ /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. -+ */ -+ preempt_disable(); -+ -+ /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ */ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) { -+ atomic_set(&non_hmbird_task, true); -+ atomic_set(&__hmbird_ops_enabled, false); -+ preempt_enable(); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ ret = -EBUSY; -+ goto err_disable_unlock; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED -+ * transitions here are synchronized against hmbird_free() through -+ * hmbird_tasks_lock. -+ */ -+ start = sched_clock(); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, ""); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ tcnt++; -+ if (READ_ONCE(p->__state) != TASK_DEAD) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ -+ set_audio_thread_sched_prop(p); -+ set_systemui_thread_pid(p); -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ hmbird_ops_enable_task(p); -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ } else { -+ hmbird_ops_disable_task(p); -+ } -+ } -+ hmbird_task_iter_exit(&sti); -+ -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, ""); -+ preempt_enable(); -+ percpu_up_write(&hmbird_fork_rwsem); -+ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) { -+ ret = -EBUSY; -+ goto err_disable; -+ } -+ -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_ENABLED, 1, 1, ""); -+ scheduler_switch_done(true); -+ -+ return 0; -+ -+err_unlock: -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock"); -+ scheduler_switch_done(false); -+ return ret; -+ -+err_disable_unlock: -+ percpu_up_write(&hmbird_fork_rwsem); -+err_disable: -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ /* must be fully disabled before returning */ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable"); -+ scheduler_switch_done(false); -+ return ret; -+} -+ -+#ifdef CONFIG_SCHED_DEBUG -+static const char *hmbird_ops_enable_state_str[] = { -+ [HMBIRD_OPS_PREPPING] = "prepping", -+ [HMBIRD_OPS_ENABLING] = "enabling", -+ [HMBIRD_OPS_ENABLED] = "enabled", -+ [HMBIRD_OPS_DISABLING] = "disabling", -+ [HMBIRD_OPS_DISABLED] = "disabled", -+}; -+ -+static int hmbird_debug_show(struct seq_file *m, void *v) -+{ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled()); -+ seq_printf(m, "%-30s: %s\n", "enable_state", -+ hmbird_ops_enable_state_str[hmbird_ops_enable_state()]); -+ seq_printf(m, "%-30s: %llu\n", "nr_rejected", -+ atomic64_read(&hmbird_nr_rejected)); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return 0; -+} -+ -+static int hmbird_debug_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_debug_show, NULL); -+} -+ -+const struct file_operations sched_hmbird_fops = { -+ .open = hmbird_debug_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif -+ -+ -+static int bpf_hmbird_reg(void *kdata) -+{ -+ return hmbird_ops_enable(kdata); -+} -+ -+static int bpf_hmbird_unreg(void *kdata) -+{ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ return 0; -+} -+ -+void set_hmbird_module_loaded(int is_loaded) -+{ -+ atomic_set(&hmbird_module_loaded, is_loaded); -+} -+ -+/* -+ * MUST load hmbird module before enable hmbird scheduler -+ * load track & hmbird gover implemented in hmbird module -+ */ -+int hmbird_ctrl(bool enable) -+{ -+ if (!atomic_read(&hmbird_module_loaded)) { -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "ext module unloaded\n"); -+ return -EINVAL; -+ } -+ -+ if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED -+ || hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING -+ || hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already enabled(ing)\n"); -+ hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING || -+ hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already disabled(ing)\n"); -+ hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (enable) -+ return bpf_hmbird_reg(NULL); -+ else -+ return bpf_hmbird_unreg(NULL); -+} -+ -+void set_cpu_isomask(int cpu, cpumask_var_t *mask) -+{ -+ cpumask_clear_cpu(cpu, iso_masks.ex_free); -+ cpumask_clear_cpu(cpu, iso_masks.exclusive); -+ cpumask_clear_cpu(cpu, iso_masks.partial); -+ cpumask_clear_cpu(cpu, iso_masks.big); -+ cpumask_clear_cpu(cpu, iso_masks.little); -+ cpumask_set_cpu(cpu, *mask); -+} -+ -+void set_cpu_cluster(u64 cpu_cluster) -+{ -+ int cpu; -+ -+ for_each_present_cpu(cpu) { -+ u64 pos = 1 << cpu; -+ -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.ex_free)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.exclusive)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.partial)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.big)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) -+ set_cpu_isomask(cpu, &(iso_masks.little)); -+ } -+} -+ -+static void get_hmbird_snapshot(struct panic_snapshot_t *p) -+{ -+ struct hmbird_dispatch_q *dsq; -+ struct rq *rq; -+ int cpu, i; -+ -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ p->rq_nr[cpu] = rq->nr_running; -+ p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running; -+ } -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ struct hmbird_entity *entity; -+ -+ dsq = &gdsqs[i]; -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo)) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ p->runnable_at[i] = entity->runnable_at; -+ raw_spin_unlock(&dsq->lock); -+ -+ } -+ -+ p->snap_misc.hmbird_enabled = hmbird_enabled(); -+ p->snap_misc.curr_ss = curr_ss; -+ p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state(); -+ p->snap_misc.parctrl_high_ratio = parctrl_high_ratio; -+ p->snap_misc.parctrl_low_ratio = parctrl_low_ratio; -+ p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l; -+ p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l; -+ p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio; -+ p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio; -+ p->snap_misc.misfit_ds = misfit_ds; -+ p->snap_misc.partial_enable = partial_enable; -+ p->snap_misc.iso_free_rescue = iso_free_rescue; -+ p->snap_misc.isolate_ctrl = isolate_ctrl; -+ p->snap_misc.snap_jiffies = jiffies; -+ p->snap_misc.snap_time = local_clock(); -+} -+ -+// MTK minidump begin -+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3) -+{ -+ strscpy(m->desc_str, str, DESC_STR_LEN); -+ m->len = d1 * d2 * d3; -+ m->parse[0] = d1; -+ m->parse[1] = d2; -+ m->parse[2] = d3; -+} -+ -+static void init_desc_metas(struct md_info_t *m) -+{ -+ init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :", -+ 1, MAX_SWITCHS, SWITCH_ITEMS); -+ -+ init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1); -+ -+ init_desc_meta(&m->kern_dump.excep_rec_meta, -+ "excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS); -+ -+ init_desc_meta(&m->kern_dump.excep_idx_meta, -+ "excep idx :", 1, 1, MAX_EXCEP_ID); -+ -+ init_desc_meta(&m->kern_dump.snap.runnable_at_meta, -+ "each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS); -+ -+ init_desc_meta(&m->kern_dump.snap.rq_nr_meta, -+ "rq runnable task nr:", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta, -+ "scxrq runnable task nr :", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.snap_misc_meta, -+ "misc snap params :", 1, 1, SNAP_ITEMS); -+} -+ -+static void init_md_meta(struct md_info_t *m) -+{ -+ m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64); -+ m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64); -+ m->meta.unit_size = sizeof(u64); -+ m->meta.switches = MAX_SWITCHS; -+ m->meta.exceps = MAX_EXCEPS; -+ m->meta.global_dsqs = MAX_GLOBAL_DSQS; -+ m->meta.parse_dimens = PARSE_DIMENS; -+ m->meta.nr_cpus = num_possible_cpus(); -+ m->meta.real_cpus = nr_cpu_ids; -+ m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64); -+ m->meta.nr_meta_desc = 8; -+ m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64); -+ -+ init_desc_metas(m); -+} -+ -+#define MINIDUMP_DFL_SIZE (4 * 1024) -+ -+struct notifier_block hmbird_panic_blk; -+static int hmbird_panic_handler(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ if (!md_info) -+ return NOTIFY_DONE; -+ -+ get_hmbird_snapshot(&md_info->kern_dump.snap); -+ -+ return NOTIFY_DONE; -+} -+ -+static void panic_blk_init(void) -+{ -+ int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE); -+ -+ md_info = kzalloc(dump_size, GFP_KERNEL); -+ if (!md_info) -+ return; -+ init_md_meta(md_info); -+ -+ hmbird_panic_blk.notifier_call = hmbird_panic_handler; -+ /* make sure to execute before minidump. */ -+ hmbird_panic_blk.priority = INT_MAX; -+ atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk); -+ -+ hmbird_debug("register minidump.\n"); -+} -+ -+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size) -+{ -+ *vaddr = (unsigned long)md_info; -+ *size = sizeof(struct md_info_t); -+} -+// MTK minidump end -+ -+static inline void init_sched_prop_to_preempt_prio(void) -+{ -+ for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) { -+ switch (i) { -+ case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 5; -+ break; -+ -+ case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 4; -+ break; -+ -+ case HMBIRD_TASK_PROP_PIPELINE: -+ case HMBIRD_TASK_PROP_ISOLATE: -+ sched_prop_to_preempt_prio[i] = 3; -+ break; -+ -+ case HMBIRD_TASK_PROP_COMMON: -+ sched_prop_to_preempt_prio[i] = 1; -+ break; -+ -+ case HMBIRD_TASK_PROP_DEBUG_OR_LOG: -+ sched_prop_to_preempt_prio[i] = 0; -+ break; -+ -+ default: -+ sched_prop_to_preempt_prio[i] = 2; -+ break; -+ } -+ } -+} -+ -+void __init init_sched_hmbird_class(void) -+{ -+ int cpu; -+ u32 v; -+ struct hmbird_entity *init_hmbird; -+ -+ /* -+ * The following is to prevent the compiler from optimizing out the enum -+ * definitions so that BPF scheduler implementations can use them -+ * through the generated vmlinux.h. -+ */ -+ WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP | -+ HMBIRD_KICK_PREEMPT); -+ -+ init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL); -+ init_dsq_at_boot(); -+ init_isolate_cpus(); -+ hb_timer_init(); -+ init_sched_prop_to_preempt_prio(); -+#ifdef CONFIG_SMP -+ WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); -+#endif -+ -+ /* -+ * we can't static init init_task's hmbird struct, init here. -+ * init_task->hmbird would not use during boot. -+ */ -+ init_hmbird = kzalloc(sizeof(struct hmbird_entity), GFP_KERNEL); -+ init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird; -+ if (init_hmbird) { -+ INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo); -+ INIT_LIST_HEAD(&init_hmbird->watchdog_node); -+ init_hmbird->sticky_cpu = -1; -+ init_hmbird->holding_cpu = -1; -+ atomic64_set(&init_hmbird->ops_state, 0); -+ init_hmbird->runnable_at = jiffies; -+ init_hmbird->slice = HMBIRD_SLICE_DFL; -+ init_hmbird->task = &init_task; -+ hmbird_set_sched_prop(&init_task, 0); -+ } else { -+ hmbird_err(INIT_TASK_FAIL, ":alloc init_task.scx failed!!!\n"); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ /* -+ * exec during boot phase, no need to care about alloc failed. -+ * lifecycle same to rq, no need to free. -+ */ -+ rq->android_oem_data1[HMBIRD_RQ_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_rq), GFP_KERNEL); -+ if (get_hmbird_rq(rq)) { -+ get_hmbird_rq(rq)->rq = rq; -+ get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window; -+ } else { -+ hmbird_err(ALLOC_RQSCX_FAIL, ":alloc rq->scx failed!!!\n"); -+ } -+ -+ rq->android_oem_data1[HMBIRD_OPS_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL); -+ if (!get_hmbird_ops(rq)) -+ pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n"); -+ init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL); -+ INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list); -+ -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL)); -+ -+ hmbird_ops_init(get_hmbird_ops(rq)); -+ } -+ -+ INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn); -+ INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn); -+ hmbird_misc_init(); -+ -+ panic_blk_init(); -+} -+ -diff --git b/kernel/sched/hmbird/hmbird_misc.c a/kernel/sched/hmbird/hmbird_misc.c -new file mode 100755 -index 000000000000..895e8a29df33 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_misc.c -@@ -0,0 +1,148 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * @Description: -+ * @Version: -+ * @Author: wangrui8 -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include "hmbird_sched.h" -+#include -+#include -+#include "slim.h" -+ -+noinline int tracing_mark_write(const char *buf) -+{ -+ trace_printk(buf); -+ return 0; -+} -+ -+struct yield_opt_params yield_opt_params = { -+ .enable = 0, -+ .frame_per_sec = 120, -+ .frame_time_ns = NSEC_PER_SEC / 120, -+ .yield_headroom = 10, -+}; -+ -+DEFINE_PER_CPU(struct sched_yield_state, ystate); -+ -+static inline void hmbird_yield_state_update(struct sched_yield_state *ys) -+{ -+ if (!raw_spin_is_locked(&ys->lock)) -+ return; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ -+ if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1 -+ || ys->yield_cnt_after_sleep > yield_headroom) { -+ ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP); -+ } else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) { -+ ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP); -+ } -+ ys->yield_cnt = 0; -+ ys->sleep_times = 0; -+ ys->yield_cnt_after_sleep = 0; -+} -+ -+void hmbird_skip_yield(long *skip) -+{ -+ if (!get_hmbird_ops_enabled() || !yield_opt_params.enable) -+ return; -+ unsigned long flags, sleep_now = 0; -+ struct sched_yield_state *ys; -+ int cpu = raw_smp_processor_id(), cont_yield, new_frame; -+ int frame_time_ns = yield_opt_params.frame_time_ns; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ u64 wc; -+ -+ if (!(*skip)) { -+ wc = sched_clock(); -+ ys = &per_cpu(ystate, cpu); -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ -+ cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP; -+ new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1); -+ -+ if (!cont_yield && new_frame) { -+ hmbird_yield_state_update(ys); -+ ys->last_update_time = wc; -+ ys->sleep_end = ys->last_yield_time + frame_time_ns -+ - yield_headroom * YIELD_DURATION; -+ } -+ -+ if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) { -+ *skip = true; -+ -+ sleep_now = ys->sleep_times ? -+ max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep; -+ if (wc + sleep_now > ys->sleep_end) { -+ u64 delta = ys->sleep_end - wc; -+ -+ if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION) -+ sleep_now = delta; -+ else -+ sleep_now = 0; -+ } -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ if (sleep_now) { -+ sleep_now = div64_u64(sleep_now, 1000); -+ usleep_range_state(sleep_now, sleep_now, TASK_IDLE); -+ } -+ ys->sleep_times++; -+ ys->last_yield_time = sched_clock(); -+ return; -+ } -+ if (ys->sleep_times) -+ ys->yield_cnt_after_sleep++; -+ else -+ (ys->yield_cnt)++; -+ ys->last_yield_time = wc; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+} -+ -+struct boost_policy_params boost_policy_params = { -+ .enable = 0, -+ .bottom_freq = 1200000, -+ .boost_weight = 120, -+}; -+ -+int hmbird_get_boost_enable(void) -+{ -+ return boost_policy_params.enable; -+} -+ -+unsigned int hmbird_get_boost_bottom_freq(void) -+{ -+ return boost_policy_params.bottom_freq; -+} -+ -+int hmbird_get_boost_weight(void) -+{ -+ return boost_policy_params.boost_weight; -+} -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops) -+{ -+ hmbird_ops->scx_enable = get_hmbird_ops_enabled; -+ hmbird_ops->check_non_task = get_non_hmbird_task; -+ hmbird_ops->hmbird_get_md_info = hmbird_get_md_info; -+ hmbird_ops->hmbird_get_boost_enable = hmbird_get_boost_enable; -+ hmbird_ops->hmbird_get_boost_bottom_freq = hmbird_get_boost_bottom_freq; -+ hmbird_ops->hmbird_get_boost_weight = hmbird_get_boost_weight; -+} -+ -+void hmbird_misc_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_init(&ys->lock); -+ } -+ -+ set_hmbird_module_loaded(1); -+} -+ -diff --git b/kernel/sched/hmbird/hmbird_sched.h a/kernel/sched/hmbird/hmbird_sched.h -new file mode 100755 -index 000000000000..76acbe9685e4 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef __HMBIRD_SCHED__ -+#define __HMBIRD_SCHED__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include <../../kernel/time/tick-sched.h> -+#include <../../kernel/sched/sched.h> -+#include -+#include -+#include -+ -+#define REGISTER_TRACE_VH(vender_hook, handler) \ -+{ \ -+ ret = register_trace_##vender_hook(handler, NULL); \ -+ if (ret) { \ -+ pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \ -+ } \ -+} -+#define REGISTER_TRACE(vendor_hook, handler, data, err) \ -+do { \ -+ ret = register_trace_##vendor_hook(handler, data); \ -+ if (ret) { \ -+ pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret); \ -+ goto err; \ -+ } \ -+} while (0) -+ -+#define UNREGISTER_TRACE(vendor_hook, handler, data) \ -+ unregister_trace_##vendor_hook(handler, data) \ -+ -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+ -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int sched_ravg_window_frame_per_sec; -+extern int slim_gov_debug; -+extern int cpu7_tl; -+extern int scx_gov_ctrl; -+extern spinlock_t new_sched_ravg_window_lock; -+extern int cluster_separate; -+ -+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER BIT(31) -+#define MAX_YIELD_SLEEP (2000000ULL) -+#define MIN_YIELD_SLEEP (200000ULL) -+#define YIELD_DURATION (5000ULL) -+#define DEFAULT_YIELD_SLEEP_TH (10) -+ -+struct sched_yield_state { -+ raw_spinlock_t lock; -+ u64 last_yield_time; -+ u64 last_update_time; -+ u64 sleep_end; -+ unsigned long yield_cnt; -+ unsigned long yield_cnt_after_sleep; -+ unsigned long sleep; -+ int sleep_times; -+}; -+ -+DECLARE_PER_CPU(struct sched_yield_state, ystate); -+ -+void hmbird_window_rollover_run_once(struct rq *rq); -+void hmbird_yield_state_update_per_frame(void); -+void hmbird_misc_init(void); -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops); -+#endif /*__HMBIRD_SCHED__*/ -diff --git b/kernel/sched/hmbird/hmbird_sched_proc.c a/kernel/sched/hmbird/hmbird_sched_proc.c -new file mode 100755 -index 000000000000..295d45404608 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched_proc.c -@@ -0,0 +1,640 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "hmbird_sched_proc.h" -+#include -+ -+#include "hmbird_util_track.h" -+#include "slim.h" -+ -+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched" -+#define SLIM_FREQ_GOV_DIR "slim_freq_gov" -+#define LOAD_TRACK_DIR "slim_walt" -+#define HMBIRD_PROC_PERMISSION 0666 -+ -+int scx_enable; -+int partial_enable; -+int cpuctrl_high_ratio = 55; -+int cpuctrl_low_ratio = 40; -+int slim_stats; -+int hmbirdcore_debug; -+int slim_for_app; -+int misfit_ds = 90; -+unsigned int highres_tick_ctrl; -+unsigned int highres_tick_ctrl_dbg; -+int cpu7_tl = 70; -+int slim_walt_ctrl; -+int slim_walt_dump; -+int slim_walt_policy; -+int slim_gov_debug; -+int scx_gov_ctrl = 1; -+int sched_ravg_window_frame_per_sec = 125; -+int parctrl_high_ratio = 55; -+int parctrl_low_ratio = 40; -+int parctrl_high_ratio_l = 65; -+int parctrl_low_ratio_l = 50; -+int isoctrl_high_ratio = 75; -+int isoctrl_low_ratio = 60; -+int isolate_ctrl; -+int iso_free_rescue; -+int heartbeat; -+int heartbeat_enable = 1; -+int watchdog_enable; -+int save_gov; -+u64 cpu_cluster_masks; -+int hmbird_preempt_policy; -+int cluster_separate; -+ -+char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= 32) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtoint(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoint\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+static int set_proc_buf_val_u64(struct file *file, const char __user *buf, -+ size_t count, u64 *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= sizeof(kbuf)) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtou64(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoul\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/* common ops begin */ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ return count; -+} -+ -+static int hmbird_common_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%d\n", *(int *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_show, pde_data(inode)); -+} -+ -+static int hmbird_common_ul_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%lu\n", *(unsigned long *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_ul_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_ul_show, pde_data(inode)); -+} -+ -+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write); -+/* common ops end */ -+ -+/* scx_enable ops begin */ -+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC); -+ if (hmbird_ctrl(*pval)) -+ return -EFAULT; -+ -+ return count; -+} -+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write); -+/* scx_enable ops end */ -+ -+/* hmbird_stats ops begin */ -+#define MAX_STATS_BUF (4096) -+static int hmbird_stats_proc_show(struct seq_file *m, void *v) -+{ -+ char *buf; -+ -+ buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ stats_print(buf, MAX_STATS_BUF); -+ -+ seq_printf(m, "%s\n", buf); -+ -+ kfree(buf); -+ return 0; -+} -+ -+static int hmbird_stats_proc_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_stats_proc_show, inode); -+} -+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL); -+/* hmbird_stats ops end */ -+ -+/* sched_ravg_window_frame_per_sec ops begin */ -+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file, -+ const char __user *buf, size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ sched_ravg_window_change(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open, -+ sched_ravg_window_frame_per_sec_proc_write); -+/* sched_ravg_window_frame_per_sec ops end */ -+ -+static ssize_t save_gov_str(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ for_each_possible_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy || (cpu != policy->cpu)) -+ continue; -+ WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0); -+ hmbird_info_systrace(":save origin gov : %s\n", saved_gov[cpu]); -+ } -+ return count; -+} -+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str); -+ -+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ u64 *pval = (u64 *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val_u64(file, buf, count, pval)) -+ return -EFAULT; -+ -+ if (scx_enable == 0) -+ set_cpu_cluster(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write); -+ -+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ int tmp_val; -+ -+ if (set_proc_buf_val(file, buf, count, &tmp_val)) -+ return -EFAULT; -+ -+ slim_walt_enable(tmp_val); -+ *pval = tmp_val; -+ -+ return count; -+} -+ -+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write); -+ -+/* yield_opt ops begin */ -+static int yield_opt_show(struct seq_file *m, void *v) -+{ -+ struct yield_opt_params *data = m->private; -+ -+ seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n", -+ data->enable, data->frame_per_sec, data->yield_headroom); -+ return 0; -+} -+ -+static int yield_opt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, yield_opt_show, pde_data(inode)); -+} -+ -+static ssize_t yield_opt_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu; -+ unsigned long flags; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if (copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp -+ != 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) || -+ (yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp; -+ yield_opt_params.frame_per_sec = frame_per_sec_tmp; -+ yield_opt_params.yield_headroom = yield_headroom_tmp; -+ yield_opt_params.enable = enable_tmp; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ ys->last_yield_time = 0; -+ ys->last_update_time = 0; -+ ys->sleep_end = 0; -+ ys->yield_cnt = 0; -+ ys->yield_cnt_after_sleep = 0; -+ ys->sleep = 0; -+ ys->sleep_times = 0; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write); -+ -+/* boost_policy ops begin */ -+static int boost_policy_show(struct seq_file *m, void *v) -+{ -+ struct boost_policy_params *data = m->private; -+ -+ seq_printf(m, "boost_policy:{\"enable\":%d; \"bottom_freq\":%u; \"boost_weight\":%d}\n", -+ data->enable, data->bottom_freq, data->boost_weight); -+ return 0; -+} -+ -+static int boost_policy_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, boost_policy_show, pde_data(inode)); -+} -+ -+static ssize_t boost_policy_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, bottom_freq_tmp, boost_weight_tmp; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if(copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &bottom_freq_tmp, &boost_weight_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || -+ (boost_weight_tmp < 50 || boost_weight_tmp > 300) || -+ (bottom_freq_tmp < 400000 || bottom_freq_tmp > 2200000)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ boost_policy_params.bottom_freq = bottom_freq_tmp; -+ boost_policy_params.boost_weight = boost_weight_tmp; -+ boost_policy_params.enable = enable_tmp; -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(boost_policy, boost_policy_open, boost_policy_write); -+ -+/* tick_hit ops begin */ -+static int tick_hit_show(struct seq_file *m, void *v) -+{ -+ struct tick_hit_params *data = m->private; -+ -+ seq_printf(m, "tick_hit:{\"enable\":%d; \"hit_count_thres\":%d; \"jiffies_num\":%lu}\n", -+ data->enable, data->hit_count_thres, data->jiffies_num); -+ return 0; -+} -+ -+static int tick_hit_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, tick_hit_show, pde_data(inode)); -+} -+ -+static ssize_t tick_hit_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, hit_count_thres_tmp, jiffies_num_tmp; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if(copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &hit_count_thres_tmp, &jiffies_num_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ tick_hit_params.hit_count_thres = hit_count_thres_tmp; -+ tick_hit_params.jiffies_num = jiffies_num_tmp; -+ tick_hit_params.enable = enable_tmp; -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(tick_hit, tick_hit_open, tick_hit_write); -+ -+static int __init hmbird_proc_init(void) -+{ -+ struct proc_dir_entry *hmbird_dir; -+ struct proc_dir_entry *load_track_dir; -+ struct proc_dir_entry *freq_gov_dir; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ -+ /* mkdir /proc/hmbird_sched */ -+ hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL); -+ if (!hmbird_dir) { -+ pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &scx_enable_proc_ops, -+ &scx_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &partial_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_stats); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbirdcore_debug); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_for_app); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &misfit_ds); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl_dbg); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpu7_tl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &cpu_cluster_masks_proc_ops, -+ &cpu_cluster_masks); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &save_gov_proc_ops, -+ &save_gov); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &watchdog_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isolate_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &iso_free_rescue); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_stats_proc_ops); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &yield_opt_proc_ops, -+ &yield_opt_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("boost_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &boost_policy_proc_ops, -+ &boost_policy_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("tick_hit", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &tick_hit_proc_ops, -+ &tick_hit_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbird_preempt_policy); -+ /* /proc/hmbird_sched--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_walt */ -+ load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir); -+ if (!load_track_dir) { -+ pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_walt--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &slim_walt_ctrl_proc_ops, -+ &slim_walt_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_dump); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_policy); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &sched_ravg_window_frame_per_sec_proc_ops, -+ &sched_ravg_window_frame_per_sec); -+ /* /proc/hmbird_sched/slim_walt--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_freq_gov */ -+ freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir); -+ if (!freq_gov_dir) { -+ pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_freq_gov--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &slim_gov_debug); -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &scx_gov_ctrl); -+ /* /proc/hmbird_sched/slim_freq_gov--end */ -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cluster_separate); -+ -+ return 0; -+} -+ -+device_initcall(hmbird_proc_init); -diff --git b/kernel/sched/hmbird/hmbird_sched_proc.h a/kernel/sched/hmbird/hmbird_sched_proc.h -new file mode 100755 -index 000000000000..e22bc75f1dd7 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched_proc.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SCHED_PROC_H__ -+#define __HMBIRD_SCHED_PROC_H__ -+ -+#include -+#include -+ -+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \ -+ do { \ -+ if (!proc_create(name, mode, parent, proc_ops)) { \ -+ pr_err("Error creating proc entry %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \ -+ do { \ -+ if (!proc_create_data(name, mode, parent, proc_ops, data)) { \ -+ pr_err("Error creating proc entry with data %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_PROC_OPS(name, open_func, write_func) \ -+ static const struct proc_ops name##_proc_ops = { \ -+ .proc_open = open_func, \ -+ .proc_write = write_func, \ -+ .proc_read = seq_read, \ -+ .proc_lseek = seq_lseek, \ -+ .proc_release = single_release, \ -+ } -+ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos); -+static int hmbird_common_show(struct seq_file *m, void *v); -+static int hmbird_common_open(struct inode *inode, struct file *file); -+ -+#endif -diff --git b/kernel/sched/hmbird/hmbird_shadow_tick.c a/kernel/sched/hmbird/hmbird_shadow_tick.c -new file mode 100755 -index 000000000000..2744cd42bbfd ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_shadow_tick.c -@@ -0,0 +1,198 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include -+#include <../../kernel/time/tick-sched.h> -+#include -+ -+#include "hmbird_shadow_tick.h" -+#include "slim.h" -+ -+#define HIGHRES_WATCH_CPU 0 -+ -+#include -+static bool shadow_tick_enable(void) {return highres_tick_ctrl; } -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; } -+#endif -+static bool shadow_tick_timer_init_flag; -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define shadow_tick_printk(fmt, args...) \ -+do { \ -+ int cpu = smp_processor_id(); \ -+ if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU) \ -+ trace_printk("hmbird shadow tick :"fmt, args); \ -+} while (0) -+#else -+#define shadow_tick_printk(fmt, args...) -+#endif -+ -+DEFINE_PER_CPU(struct hrtimer, stt); -+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu))) -+#define STOP_IDLE_TRIGGER (1) -+#define PERIODIC_TICK_TRIGGER (2) -+#define TICK_INTVAL (1000000ULL) -+#define HMBIRD_TICK_HIT_BOOST BIT(29) -+/* -+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms, -+ * so we can't restart hrtimer in scheduler tick. -+ */ -+static DEFINE_PER_CPU(u8, trigger_event); -+ -+/* -+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick. -+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler. -+ */ -+static DEFINE_PER_CPU(u8, tick_phase); -+ -+static inline void highres_timer_ctrl(bool enable, int cpu) -+{ -+ if (enable && hmbird_enabled()) { -+ if (!hrtimer_active(shadow_tick_timer(cpu))) -+ hrtimer_start(shadow_tick_timer(cpu), -+ ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED); -+ } else { -+ if (!enable) -+ hrtimer_cancel(shadow_tick_timer(cpu)); -+ } -+} -+ -+static inline void high_res_clear_phase(int cpu) -+{ -+ per_cpu(tick_phase, cpu) = 0; -+} -+ -+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer) -+{ -+ per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3; -+ if (per_cpu(tick_phase, cpu)) { -+ hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) { -+ per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+struct tick_hit_params tick_hit_params = { -+ .enable = 0, -+ .jiffies_num = 2, -+ .hit_count_thres = 6, -+}; -+ -+static void tick_hit_critical_task(struct task_struct *curr, struct rq *rq) -+{ -+ struct hmbird_entity *see = get_hmbird_ts(curr); -+ if (!tick_hit_params.enable || !see) -+ return; -+ -+ if ((!task_is_top_task(curr) || curr->pid != curr->tgid) && (curr->pid != scx_systemui_pid)) -+ return; -+ -+ if (see->tick_hit_count == 0) { -+ if (see->start_jiffies == 0) { -+ see->start_jiffies = jiffies; -+ see->tick_hit_count = 1; -+ } else { -+ see->start_jiffies = 0; /* status reset */ -+ see->tick_hit_count = 0; -+ } -+ } else { -+ if (time_before_eq(jiffies, see->start_jiffies + tick_hit_params.jiffies_num)) { -+ see->tick_hit_count++; -+ if (see->tick_hit_count >= tick_hit_params.hit_count_thres) { -+ cpufreq_update_util(rq, HMBIRD_TICK_HIT_BOOST); -+ } -+ } else { -+ see->tick_hit_count = 1; -+ see->start_jiffies = jiffies; -+ } -+ } -+} -+ -+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ struct rq_flags rf; -+ -+ rq_lock(rq, &rf); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ tick_hit_critical_task(curr, rq); -+ rq_unlock(rq, &rf); -+ -+ return highres_next_phase(cpu, timer); -+} -+ -+void shadow_tick_timer_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance; -+ } -+} -+ -+void start_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable()) { -+ if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER) -+ highres_timer_ctrl(false, cpu); -+ per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static void stop_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ per_cpu(trigger_event, cpu) = 0; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(false, cpu); -+} -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ stop_shadow_tick_timer(); -+} -+ -+void scheduler_tick_handler(void *unused, struct rq *rq) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ start_shadow_tick_timer(); -+} -+ -+static int __init hmbird_shadow_tick_init(void) -+{ -+ int ret = 0; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ shadow_tick_timer_init(); -+ shadow_tick_timer_init_flag = true; -+ return ret; -+} -+ -+device_initcall(hmbird_shadow_tick_init); -diff --git b/kernel/sched/hmbird/hmbird_shadow_tick.h a/kernel/sched/hmbird/hmbird_shadow_tick.h -new file mode 100755 -index 000000000000..0c26fd984f0a ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_shadow_tick.h -@@ -0,0 +1,11 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SHADOW_TICK_H__ -+#define __HMBIRD_SHADOW_TICK_H__ -+ -+#include -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+void scheduler_tick_handler(void *unused, struct rq *rq); -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state); -+#endif -diff --git b/kernel/sched/hmbird/hmbird_trace.h a/kernel/sched/hmbird/hmbird_trace.h -new file mode 100755 -index 000000000000..8468b406f347 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_trace.h -@@ -0,0 +1,92 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hmbird -+ -+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HMBIRD_H -+ -+#include -+#include -+#include -+ -+#define MAX_FATAL_INFO (64) -+ -+TRACE_EVENT(hmbird_fatal_info, -+ -+ TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info), -+ -+ TP_ARGS(type, pe, lnr, bnr, info), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, type) -+ __field(int, pe) -+ __field(int, lnr) -+ __field(int, bnr) -+ __array(char, info, MAX_FATAL_INFO)), -+ -+ TP_fast_assign( -+ __entry->type = type; -+ __entry->pe = pe; -+ __entry->lnr = lnr; -+ __entry->bnr = bnr; -+ memcpy(__entry->info, info, MAX_FATAL_INFO);), -+ -+ TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s", -+ __entry->type, __entry->pe, __entry->lnr, __entry->bnr, -+ __entry->info) -+); -+ -+TRACE_EVENT(hmbird_update_history, -+ -+ TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq, -+ struct task_struct *p, u32 runtime, int samples, int event), -+ -+ TP_ARGS(hmbird, rq, p, runtime, samples, event), -+ -+ TP_STRUCT__entry( -+ __array(char, comm, TASK_COMM_LEN) -+ __field(pid_t, pid) -+ __field(unsigned int, runtime) -+ __field(int, samples) -+ __field(int, event) -+ __field(unsigned int, demand) -+ __array(u32, hist, RAVG_HIST_SIZE) -+ __field(u16, task_util) -+ __field(int, cpu)), -+ -+ TP_fast_assign( -+ memcpy(__entry->comm, p->comm, TASK_COMM_LEN); -+ __entry->pid = p->pid; -+ __entry->runtime = runtime; -+ __entry->samples = samples; -+ __entry->event = event; -+ __entry->demand = hmbird->sts.demand; -+ memcpy(__entry->hist, hmbird->sts.sum_history, -+ RAVG_HIST_SIZE * sizeof(u32)); -+ __entry->task_util = hmbird->sts.demand_scaled; -+ __entry->cpu = rq->cpu;), -+ -+ TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d", -+ __entry->comm, __entry->pid, -+ __entry->runtime, __entry->samples, -+ __entry->event, -+ __entry->demand, -+ __entry->hist[0], __entry->hist[1], -+ __entry->hist[2], __entry->hist[3], -+ __entry->hist[4], -+ __entry->task_util, -+ __entry->cpu) -+); -+ -+#endif /*_TRACE_HMBIRD_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird -+ -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE hmbird_trace -+/* This part must be outside protection */ -+#include -diff --git b/kernel/sched/hmbird/hmbird_util_track.c a/kernel/sched/hmbird/hmbird_util_track.c -new file mode 100755 -index 000000000000..d520a8403401 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_util_track.c -@@ -0,0 +1,626 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+ -+#define CREATE_TRACE_POINTS -+#include "hmbird_trace.h" -+#undef CREATE_TRACE_POINTS -+ -+#define HMBIRD_DEBUG_PANIC (1 << 3) -+static bool init_irq_work_inited; -+ -+extern noinline int tracing_mark_write(const char *buf); -+ -+int hmbird_sched_ravg_window = 8000000; -+int new_hmbird_sched_ravg_window = 8000000; -+DEFINE_SPINLOCK(new_sched_ravg_window_lock); -+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+ -+static struct irq_work hmbird_slim_walt_irq_work; -+ -+/*Sysctl related interface*/ -+#define WINDOW_STATS_RECENT 0 -+#define WINDOW_STATS_MAX 1 -+#define WINDOW_STATS_MAX_RECENT_AVG 2 -+#define WINDOW_STATS_AVG 3 -+#define WINDOW_STATS_INVALID_POLICY 4 -+ -+ -+#define SCX_SCHED_CAPACITY_SHIFT 10 -+#define SCHED_ACCOUNT_WAIT_TIME 0 -+ -+atomic64_t hmbird_irq_work_lastq_ws; -+static u64 tick_sched_clock; -+ -+static inline u64 scale_exec_time(u64 delta, struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT; -+} -+ -+static inline u64 scale_time_to_util(u64 d) -+{ -+ do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT); -+ return d; -+} -+ -+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ delta = scale_exec_time(delta, rq); -+ sts->sum += delta; -+ if (unlikely(sts->sum > hmbird_sched_ravg_window)) -+ sts->sum = hmbird_sched_ravg_window; -+ -+ return delta; -+} -+ -+ -+static int -+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) -+{ -+ /* -+ * No need to bother updating task demand for the idle task. -+ */ -+ if (is_idle_task(p)) -+ return 0; -+ -+ /* -+ * When a task is waking up it is completing a segment of non-busy -+ * time. Likewise, if wait time is not treated as busy time, then -+ * when a task begins to run or is migrated, it is not running and -+ * is completing a segment of non-busy time. -+ */ -+ if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && -+ (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) -+ return 0; -+ -+ /* -+ * The idle exit time is not accounted for the first task _picked_ up to -+ * run on the idle CPU. -+ */ -+ if (event == PICK_NEXT_TASK && rq->curr == rq->idle) -+ return 0; -+ -+ /* -+ * TASK_UPDATE can be called on sleeping task, when its moved between -+ * related groups -+ */ -+ if (event == TASK_UPDATE) { -+ if (rq->curr == p) -+ return 1; -+ -+ return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; -+ } -+ -+ return 1; -+} -+ -+static void rollover_cpu_window(struct rq *rq, bool full_window) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 curr_sum = srq->curr_runnable_sum; -+ -+ if (unlikely(full_window)) -+ curr_sum = 0; -+ -+ srq->prev_runnable_sum = curr_sum; -+ srq->curr_runnable_sum = 0; -+} -+ -+static u64 -+update_window_start(struct rq *rq, u64 wallclock, int event) -+{ -+ s64 delta; -+ int nr_windows; -+ bool full_window; -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start = srq->window_start; -+ -+ if (wallclock < srq->latest_clock) -+ wallclock = srq->latest_clock; -+ delta = wallclock - srq->window_start; -+ if (delta < 0) { -+ delta = 0; -+ wallclock = srq->window_start; -+ } -+ srq->latest_clock = wallclock; -+ if (delta < hmbird_sched_ravg_window) -+ return old_window_start; -+ -+ nr_windows = div64_u64(delta, hmbird_sched_ravg_window); -+ srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window; -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ full_window = nr_windows > 1; -+ rollover_cpu_window(rq, full_window); -+ -+ return old_window_start; -+} -+ -+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -+ -+static inline unsigned int get_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static inline unsigned int cpu_cur_freq(int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cur; -+} -+ -+static void -+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, -+ u64 wallclock) -+{ -+ int cpu = cpu_of(rq); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * -+ arch_scale_cpu_capacity(cpu), get_max_freq(cpu)); -+} -+ -+/* -+ * Called when new window is starting for a task, to record cpu usage over -+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 -+ * when, say, a real-time task runs without preemption for several windows at a -+ * stretch. -+ */ -+static void update_history(struct rq *rq, struct task_struct *p, -+ u32 runtime, int samples, int event) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u32 *hist = &sts->sum_history[0]; -+ int i; -+ u32 max = 0, avg, demand; -+ u64 sum = 0; -+ u16 demand_scaled; -+ -+ /* Ignore windows where task had no activity */ -+ if (!runtime || is_idle_task(p) || !samples) -+ goto done; -+ -+ /* Push new 'runtime' value onto stack */ -+ for (; samples > 0; samples--) { -+ hist[sts->cidx] = runtime; -+ sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE; -+ } -+ -+ for (i = 0; i < RAVG_HIST_SIZE; i++) { -+ sum += hist[i]; -+ if (hist[i] > max) -+ max = hist[i]; -+ } -+ -+ sts->sum = 0; -+ avg = div64_u64(sum, RAVG_HIST_SIZE); -+ -+ switch (slim_walt_policy) { -+ case WINDOW_STATS_RECENT: -+ demand = runtime; -+ break; -+ case WINDOW_STATS_MAX: -+ demand = max; -+ break; -+ case WINDOW_STATS_AVG: -+ demand = avg; -+ break; -+ default: -+ demand = max(avg, runtime); -+ } -+ -+ demand_scaled = scale_time_to_util(demand); -+ -+ sts->demand = demand; -+ sts->demand_scaled = demand_scaled; -+ -+done: -+ return; -+} -+ -+ -+static u64 update_task_demand(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ -+ u64 delta, window_start = srq->window_start; -+ int new_window, nr_full_windows; -+ u32 window_size = hmbird_sched_ravg_window; -+ u64 runtime; -+ -+ new_window = mark_start < window_start; -+ if (!account_busy_for_task_demand(rq, p, event)) { -+ if (new_window) -+ /* -+ * If the time accounted isn't being accounted as -+ * busy time, and a new window started, only the -+ * previous window need be closed out with the -+ * pre-existing demand. Multiple windows may have -+ * elapsed, but since empty windows are dropped, -+ * it is not necessary to account those. -+ */ -+ update_history(rq, p, sts->sum, 1, event); -+ return 0; -+ } -+ -+ if (!new_window) { -+ /* -+ * The simple case - busy time contained within the existing -+ * window. -+ */ -+ return add_to_task_demand(rq, p, wallclock - mark_start); -+ } -+ -+ /* -+ * Busy time spans at least two windows. Temporarily rewind -+ * window_start to first window boundary after mark_start. -+ */ -+ delta = window_start - mark_start; -+ nr_full_windows = div64_u64(delta, window_size); -+ window_start -= (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (window_start - mark_start) first */ -+ runtime = add_to_task_demand(rq, p, window_start - mark_start); -+ -+ /* Push new sample(s) into task's demand history */ -+ update_history(rq, p, sts->sum, 1, event); -+ if (nr_full_windows) { -+ u64 scaled_window = scale_exec_time(window_size, rq); -+ -+ update_history(rq, p, scaled_window, nr_full_windows, event); -+ runtime += nr_full_windows * scaled_window; -+ } -+ -+ /* -+ * Roll window_start back to current to process any remainder -+ * in current window. -+ */ -+ window_start += (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (wallclock - window_start) next */ -+ mark_start = window_start; -+ runtime += add_to_task_demand(rq, p, wallclock - mark_start); -+ -+ return runtime; -+} -+ -+u16 slim_walt_cpu_util(int cpu) -+{ -+ u64 prev_runnable_sum; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ prev_runnable_sum = srq->prev_runnable_sum; -+ do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT); -+ -+ return (u16)prev_runnable_sum; -+} -+ -+static DEFINE_PER_CPU(u16, prev_cpu_util); -+static inline void cpu_util_update_systrace_c(int cpu) -+{ -+ char buf[256]; -+ u16 cpu_util = slim_walt_cpu_util(cpu); -+ -+ if (cpu_util != per_cpu(prev_cpu_util, cpu)) { -+ snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n", -+ cpu, cpu_util); -+ tracing_mark_write(buf); -+ per_cpu(prev_cpu_util, cpu) = cpu_util; -+ } -+} -+ -+ -+static inline int account_busy_for_cpu_time(struct rq *rq, -+ struct task_struct *p, -+ int event) -+{ -+ return !is_idle_task(p) && (event == PUT_PREV_TASK -+ || event == TASK_UPDATE); -+} -+ -+ -+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ int new_window, full_window = 0; -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 window_start = srq->window_start; -+ u32 window_size = srq->prev_window_size; -+ u64 delta; -+ u64 *curr_runnable_sum = &srq->curr_runnable_sum; -+ u64 *prev_runnable_sum = &srq->prev_runnable_sum; -+ -+ new_window = mark_start < window_start; -+ if (new_window) -+ full_window = (window_start - mark_start) >= window_size; -+ -+ -+ if (!account_busy_for_cpu_time(rq, p, event)) -+ goto done; -+ -+ -+ if (!new_window) { -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. No rollover -+ * since we didn't start a new window. An example of this is -+ * when a task starts execution and then sleeps within the -+ * same window. -+ */ -+ delta = wallclock - mark_start; -+ -+ delta = scale_exec_time(delta, rq); -+ *curr_runnable_sum += delta; -+ -+ goto done; -+ } -+ -+ /* -+ * situations below this need window rollover, -+ * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done -+ * in update_window_start() -+ * -+ * For task counters curr/prev_window[_cpu] are rolled over in the early part of -+ * this function. If full_window(s) have expired and time since last update needs -+ * to be accounted as busy time, set the prev to a complete window size time, else -+ * add the prev window portion. -+ * -+ * For task curr counters a new window has begun, always assign -+ */ -+ -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. A new window -+ * must have been started in udpate_window_start() -+ * If any of these three above conditions are true -+ * then this busy time can't be accounted as irqtime. -+ * -+ * Busy time for the idle task need not be accounted. -+ * -+ * An example of this would be a task that starts execution -+ * and then sleeps once a new window has begun. -+ */ -+ -+ /* -+ * A full window hasn't elapsed, account partial -+ * contribution to previous completed window. -+ */ -+ -+ -+ delta = full_window ? scale_exec_time(window_size, rq) : -+ scale_exec_time(window_start - mark_start, rq); -+ -+ *prev_runnable_sum += delta; -+ -+ /* Account piece of busy time in the current window. */ -+ delta = scale_exec_time(wallclock - window_start, rq); -+ *curr_runnable_sum += delta; -+ -+done: -+ if (slim_walt_dump && new_window) -+ cpu_util_update_systrace_c(rq->cpu); -+} -+ -+ -+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq) -+{ -+ u64 result; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 new_window_start = srq->window_start; -+ -+ if (old_window_start == new_window_start) -+ return; -+ -+ result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws, -+ old_window_start, new_window_start); -+ if (result != old_window_start) -+ return; -+ -+ if (likely(cpu_online(raw_smp_processor_id()))) -+ irq_work_queue(&hmbird_slim_walt_irq_work); -+ else -+ irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask)); -+} -+ -+/* -+ * In the core scheduler, most of the load update points update the rq_clock after -+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of -+ * obtaining the time, but to prevent the subsequent migration of the load update -+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock -+ * update here. -+ */ -+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p, -+ struct rq *rq, int event) -+{ -+ if (!(rq->clock_update_flags & RQCF_UPDATED)) -+ update_rq_clock(rq); -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock)); -+} -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start; -+ -+ if (!slim_walt_ctrl) -+ return; -+ -+ if (!srq->window_start || sts->mark_start == wallclock) -+ return; -+ -+ old_window_start = update_window_start(rq, wallclock, event); -+ -+ if (!sts->window_start) -+ sts->window_start = srq->window_start; -+ -+ if (!sts->mark_start) -+ goto done; -+ -+ update_task_rq_cpu_cycles(p, rq, event, wallclock); -+ update_task_demand(p, rq, event, wallclock); -+ update_cpu_busy_time(p, rq, event, wallclock); -+ -+ sts->window_start = srq->window_start; -+ -+done: -+ sts->mark_start = wallclock; -+ slim_walt_window_rollover_run_once(old_window_start, rq); -+} -+ -+static void slim_walt_irq_work(struct irq_work *irq_work) -+{ -+ cpumask_t lock_cpus; -+ struct hmbird_sched_rq_stats *srq; -+ struct rq *rq; -+ int cpu; -+ int level = 0; -+ u64 wc; -+ unsigned long flags; -+ -+ cpumask_copy(&lock_cpus, cpu_possible_mask); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ if (level == 0) -+ raw_spin_lock(&cpu_rq(cpu)->__lock); -+ else -+ raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); -+ level++; -+ } -+ -+ wc = sched_clock(); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ rq = cpu_rq(cpu); -+ hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc); -+ } -+ -+ cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER); -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) { -+ srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id()); -+ if (wc < srq->window_start + new_hmbird_sched_ravg_window) -+ hmbird_sched_ravg_window = new_hmbird_sched_ravg_window; -+ } -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ raw_spin_unlock(&cpu_rq(cpu)->__lock); -+ } -+} -+static void hmbird_sched_init_rq(struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ srq->task_exec_scale = 1024; -+ srq->window_start = 0; -+} -+ -+void hmbird_sched_init_task(struct task_struct *p) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ memset(sts, 0, sizeof(struct hmbird_sched_task_stats)); -+} -+ -+static void hmbird_sched_stats_init(void) -+{ -+ unsigned long flags; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ raw_spin_lock_irqsave(&rq->__lock, flags); -+ hmbird_sched_init_rq(rq); -+ raw_spin_unlock_irqrestore(&rq->__lock, flags); -+ } -+ slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG; -+ -+ if (false == init_irq_work_inited) { -+ init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work); -+ init_irq_work_inited = true; -+ } -+} -+ -+void hmbird_scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (unlikely(!tick_sched_clock)) { -+ /* -+ * Let the window begin 20us prior to the tick, -+ * that way we are guaranteed a rollover when the tick occurs. -+ * Use rq->clock directly instead of rq_clock() since -+ * we do not have the rq lock and -+ * rq->clock was updated in the tick callpath. -+ */ -+ if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000)) -+ return; -+ for_each_possible_cpu(cpu) { -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ srq->window_start = tick_sched_clock; -+ } -+ atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock); -+ } -+} -+ -+void slim_walt_enable(int enable) -+{ -+ if (1 == !!enable) { -+ hmbird_sched_stats_init(); -+ WRITE_ONCE(tick_sched_clock, 0); -+ } else -+ slim_walt_ctrl = 0; -+} -+ -+void slim_get_cpu_util(int cpu, u64 *util) -+{ -+ if (cpu < 0) -+ return; -+ -+ *util = slim_walt_cpu_util(cpu); -+} -+ -+void slim_get_task_util(struct task_struct *p, u64 *util) -+{ -+ if (p == NULL) -+ return; -+ -+ *util = get_hmbird_ts(p)->sts.demand_scaled; -+} -+ -+void sched_ravg_window_change(int frame_per_sec) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec; -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+} -diff --git b/kernel/sched/hmbird/hmbird_util_track.h a/kernel/sched/hmbird/hmbird_util_track.h -new file mode 100644 -index 000000000000..17b85df7f83b ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_util_track.h -@@ -0,0 +1,33 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef __HMBIRD_UTIL_TRACK_H__ -+#define __HMBIRD_UTIL_TRACK_H__ -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock); -+void hmbird_sched_init_task(struct task_struct *p); -+void slim_walt_enable(int enable); -+void slim_get_cpu_util(int cpu, u64 *util); -+void slim_get_task_util(struct task_struct *p, u64 *util); -+ -+extern atomic64_t hmbird_irq_work_lastq_ws; -+ -+enum task_event { -+ PUT_PREV_TASK = 0, -+ PICK_NEXT_TASK = 1, -+ TASK_WAKE = 2, -+ TASK_MIGRATE = 3, -+ TASK_UPDATE = 4, -+ IRQ_UPDATE = 5, -+}; -+ -+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+#endif /* __HMBIRD_UTIL_TRACK_H__ */ -diff --git b/kernel/sched/hmbird/slim.h a/kernel/sched/hmbird/slim.h -new file mode 100755 -index 000000000000..dffe6506658e ---- /dev/null -+++ a/kernel/sched/hmbird/slim.h -@@ -0,0 +1,56 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+ -+#ifndef __SLIM_H -+#define __SLIM_H -+ -+extern atomic_t __hmbird_ops_enabled; -+extern atomic_t non_hmbird_task; -+extern int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+extern int heartbeat; -+extern int heartbeat_enable; -+extern int watchdog_enable; -+extern int isolate_ctrl; -+extern int parctrl_high_ratio; -+extern int parctrl_low_ratio; -+extern int isoctrl_high_ratio; -+extern int isoctrl_low_ratio; -+extern int iso_free_rescue; -+extern int yield_opt; -+ -+extern enum hmbird_switch_type sw_type; -+extern noinline int tracing_mark_write(const char *buf); -+int task_top_id(struct task_struct *p); -+void stats_print(char *buf, int len); -+void hmbird_skip_yield(long *skip); -+extern spinlock_t hmbird_tasks_lock; -+extern int scx_systemui_pid; -+ -+struct yield_opt_params { -+ int enable; -+ int frame_per_sec; -+ u64 frame_time_ns; -+ int yield_headroom; -+}; -+ -+extern struct yield_opt_params yield_opt_params; -+ -+struct tick_hit_params { -+ int enable; -+ unsigned long jiffies_num; -+ int hit_count_thres; -+}; -+ -+extern struct tick_hit_params tick_hit_params; -+ -+struct boost_policy_params { -+ int enable; -+ unsigned int bottom_freq; -+ int boost_weight; -+}; -+ -+extern struct boost_policy_params boost_policy_params; -+ -+#define MAX_GOV_LEN (16) -+extern char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+#endif -diff --git b/kernel/sched/idle.c a/kernel/sched/idle.c -index b33cefeb4188..487255ac6832 100755 ---- b/kernel/sched/idle.c -+++ a/kernel/sched/idle.c -@@ -408,13 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl - - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) - { -- scx_update_idle(rq, false); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, false); -+#endif - } - - static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) - { - update_idle_core(rq); -- scx_update_idle(rq, true); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, true); -+#endif - schedstat_inc(rq->sched_goidle); - } - -diff --git b/kernel/sched/sched.h a/kernel/sched/sched.h -index 509e8dc616ab..a7e9caea1efe 100755 ---- b/kernel/sched/sched.h -+++ a/kernel/sched/sched.h -@@ -188,18 +188,13 @@ static inline int idle_policy(int policy) - return policy == SCHED_IDLE; - } - --static inline int normal_policy(int policy) --{ --#ifdef CONFIG_SCHED_CLASS_EXT -- if (policy == SCHED_EXT) -- return true; --#endif -- return policy == SCHED_NORMAL; --} -- - static inline int fair_policy(int policy) - { -- return normal_policy(policy) || policy == SCHED_BATCH; -+#ifdef CONFIG_HMBIRD_SCHED -+ return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#else -+ return policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#endif - } - - static inline int rt_policy(int policy) -@@ -247,25 +242,7 @@ static inline void update_avg(u64 *avg, u64 sample) - #define shr_bound(val, shift) \ - (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) - --/* -- * cgroup weight knobs should use the common MIN, DFL and MAX values which are -- * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it -- * maps pretty well onto the shares value used by scheduler and the round-trip -- * conversions preserve the original value over the entire range. -- */ --static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) --{ -- return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); --} -- --static inline unsigned long sched_weight_to_cgroup(unsigned long weight) --{ -- return clamp_t(unsigned long, -- DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -- CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); --} -- --/* -+/* - * !! For sched_setattr_nocheck() (kernel) only !! - * - * This is actually gross. :( -@@ -532,10 +509,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) { } - #endif /* CONFIG_SMP */ - #else /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_HMBIRD_SCHED - static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) - { - return 0; - } -+#endif - #endif /* CONFIG_FAIR_GROUP_SCHED */ - - #else /* CONFIG_CGROUP_SCHED */ -@@ -700,29 +679,6 @@ struct cfs_rq { - #endif /* CONFIG_FAIR_GROUP_SCHED */ - }; - --#ifdef CONFIG_SCHED_CLASS_EXT --/* scx_rq->flags, protected by the rq lock */ --enum scx_rq_flags { -- SCX_RQ_CAN_STOP_TICK = 1 << 0, --}; -- --struct scx_rq { -- struct scx_dispatch_q local_dsq; -- struct list_head watchdog_list; -- u64 ops_qseq; -- u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -- u32 nr_running; -- u32 flags; -- bool cpu_released; -- cpumask_var_t cpus_to_kick; -- cpumask_var_t cpus_to_preempt; -- cpumask_var_t cpus_to_wait; -- u64 pnt_seq; -- struct irq_work kick_cpus_irq_work; -- struct rq *rq; --}; --#endif /* CONFIG_SCHED_CLASS_EXT */ -- - static inline int rt_bandwidth_enabled(void) - { - return sysctl_sched_rt_runtime >= 0; -@@ -1258,11 +1214,7 @@ struct rq { - - ANDROID_OEM_DATA_ARRAY(1, 16); - --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, struct scx_rq *scx); --#else - ANDROID_KABI_RESERVE(1); --#endif - ANDROID_KABI_RESERVE(2); - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); -@@ -2340,11 +2292,6 @@ struct affinity_context { - unsigned int flags; - }; - --enum rq_onoff_reason { -- RQ_ONOFF_HOTPLUG, /* CPU is going on/offline */ -- RQ_ONOFF_TOPOLOGY, /* sched domain topology update */ --}; -- - struct sched_class { - - #ifdef CONFIG_UCLAMP_TASK -@@ -2551,7 +2498,6 @@ extern void init_sched_rt_class(void); - extern void init_sched_fair_class(void); - - extern void reweight_task(struct task_struct *p, int prio); --extern void __setscheduler_prio(struct task_struct *p, int prio); - - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); -@@ -2632,53 +2578,6 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) - extern void activate_task(struct rq *rq, struct task_struct *p, int flags); - extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); - --struct sched_change_guard { -- struct task_struct *p; -- struct rq *rq; -- bool queued; -- bool running; -- bool done; --}; -- --extern struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -- --extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags); -- --/** -- * SCHED_CHANGE_BLOCK - Nested block for task attribute updates -- * @__rq: Runqueue the target task belongs to -- * @__p: Target task -- * @__flags: DEQUEUE/ENQUEUE_* flags -- * -- * A task may need to be dequeued and put_prev_task'd for attribute updates and -- * set_next_task'd and re-enqueued afterwards. This helper defines a nested -- * block which automatically handles these preparation and cleanup operations. -- * -- * SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- * update_attribute(p); -- * ... -- * } -- * -- * If @__flags is a variable, the variable may be updated in the block body and -- * the updated value will be used when re-enqueueing @p. -- * -- * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling -- * update_rq_clock() beforehand. Otherwise, the rq clock is automatically -- * updated iff the task needs to be dequeued and re-enqueued. Only the former -- * case guarantees that the rq clock is up-to-date inside and after the block. -- */ --#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -- for (struct sched_change_guard __cg = \ -- sched_change_guard_init(__rq, __p, __flags); \ -- !__cg.done; sched_change_guard_fini(&__cg, __flags)) -- --extern void check_class_changing(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class); --extern void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio); -- - extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); - - #ifdef CONFIG_PREEMPT_RT -@@ -3710,6 +3609,8 @@ static inline bool cpu_busy_with_softirqs(int cpu) - } - #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ - --#include "ext.h" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird.h" -+#endif - - #endif /* _KERNEL_SCHED_SCHED_H */ -diff --git b/kernel/time/tick-sched.c a/kernel/time/tick-sched.c -index 998c2eefe173..c13772ee823c 100755 ---- b/kernel/time/tick-sched.c -+++ a/kernel/time/tick-sched.c -@@ -34,6 +34,10 @@ - - #include - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "../sched/hmbird/hmbird_shadow_tick.h" -+#endif -+ - /* - * Per-CPU nohz control structure - */ -@@ -1124,6 +1128,9 @@ void tick_nohz_idle_stop_tick(void) - ktime_t expires; - - trace_android_vh_tick_nohz_idle_stop_tick(NULL); -+#ifdef CONFIG_HMBIRD_SCHED -+ android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL); -+#endif - /* - * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the - * tick timer expiration time is known already. -diff --git b/tools/sched_ext/.gitignore a/tools/sched_ext/.gitignore -deleted file mode 100644 -index a3240f9f7eba..000000000000 ---- b/tools/sched_ext/.gitignore -+++ /dev/null -@@ -1,9 +0,0 @@ --scx_example_simple --scx_example_qmap --scx_example_central --scx_example_pair --scx_example_flatcg --scx_example_userland --*.skel.h --*.subskel.h --/tools/ -diff --git b/tools/sched_ext/Makefile a/tools/sched_ext/Makefile -deleted file mode 100644 -index 73c43782837d..000000000000 ---- b/tools/sched_ext/Makefile -+++ /dev/null -@@ -1,216 +0,0 @@ --# SPDX-License-Identifier: GPL-2.0 --# Copyright (c) 2022 Meta Platforms, Inc. and affiliates. --include ../build/Build.include --include ../scripts/Makefile.arch --include ../scripts/Makefile.include -- --ifneq ($(LLVM),) --ifneq ($(filter %/,$(LLVM)),) --LLVM_PREFIX := $(LLVM) --else ifneq ($(filter -%,$(LLVM)),) --LLVM_SUFFIX := $(LLVM) --endif -- --CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi --CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu --CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl --CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu --CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu --CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu --CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu --CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu --CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu --CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) -- --ifeq ($(CROSS_COMPILE),) --ifeq ($(CLANG_TARGET_FLAGS),) --$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk --else --CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) --endif # CLANG_TARGET_FLAGS --else --CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) --endif # CROSS_COMPILE -- --CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as --else --CC := $(CROSS_COMPILE)gcc --endif # LLVM -- --CURDIR := $(abspath .) --TOOLSDIR := $(abspath ..) --LIBDIR := $(TOOLSDIR)/lib --BPFDIR := $(LIBDIR)/bpf --TOOLSINCDIR := $(TOOLSDIR)/include --BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool --APIDIR := $(TOOLSINCDIR)/uapi --GENDIR := $(abspath ../../include/generated) --GENHDR := $(GENDIR)/autoconf.h -- --SCRATCH_DIR := $(CURDIR)/tools --BUILD_DIR := $(SCRATCH_DIR)/build --INCLUDE_DIR := $(SCRATCH_DIR)/include --BPFOBJ_DIR := $(BUILD_DIR)/libbpf --BPFOBJ := $(BPFOBJ_DIR)/libbpf.a --ifneq ($(CROSS_COMPILE),) --HOST_BUILD_DIR := $(BUILD_DIR)/host --HOST_SCRATCH_DIR := host-tools --HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include --else --HOST_BUILD_DIR := $(BUILD_DIR) --HOST_SCRATCH_DIR := $(SCRATCH_DIR) --HOST_INCLUDE_DIR := $(INCLUDE_DIR) --endif --HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a --RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids --DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -- --VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ -- $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ -- ../../vmlinux \ -- /sys/kernel/btf/vmlinux \ -- /boot/vmlinux-$(shell uname -r) --VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) --ifeq ($(VMLINUX_BTF),) --$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") --endif -- --BPFTOOL ?= $(DEFAULT_BPFTOOL) -- --ifneq ($(wildcard $(GENHDR)),) -- GENFLAGS := -DHAVE_GENHDR --endif -- --CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ -- -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -- -I$(TOOLSINCDIR) -I$(APIDIR) -- --CARGOFLAGS := --release -- --# Silence some warnings when compiled with clang --ifneq ($(LLVM),) --CFLAGS += -Wno-unused-command-line-argument --endif -- --LDFLAGS = -lelf -lz -lpthread -- --IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - &1 \ -- | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ --$(shell $(1) -dM -E - $@ --else -- $(call msg,CP,,$@) -- $(Q)cp "$(VMLINUX_H)" $@ --endif -- --%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \ -- | $(BPFOBJ) -- $(call msg,CLNG-BPF,,$@) -- $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ -- --%.skel.h: %.bpf.o $(BPFTOOL) -- $(call msg,GEN-SKEL,,$@) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) -- $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) -- $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@ -- $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h) -- --scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_userland: scx_example_userland.c scx_example_userland.skel.h \ -- scx_example_userland_common.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR) --atropos: export ATROPOS_CLANG = $(CLANG) --atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS) --atropos: $(INCLUDE_DIR)/vmlinux.h -- cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS) -- --clean: -- cargo clean --manifest-path=atropos/Cargo.toml -- rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) -- rm -f *.o *.bpf.o *.skel.h *.subskel.h -- rm -f scx_example_simple scx_example_qmap scx_example_central \ -- scx_example_pair scx_example_flatcg scx_example_userland -- --.PHONY: all atropos clean -- --# delete failed targets --.DELETE_ON_ERROR: -- --# keep intermediate (.skel.h, .bpf.o, etc) targets --.SECONDARY: -diff --git b/tools/sched_ext/atropos/.gitignore a/tools/sched_ext/atropos/.gitignore -deleted file mode 100644 -index 186dba259ec2..000000000000 ---- b/tools/sched_ext/atropos/.gitignore -+++ /dev/null -@@ -1,3 +0,0 @@ --src/bpf/.output --Cargo.lock --target -diff --git b/tools/sched_ext/atropos/Cargo.toml a/tools/sched_ext/atropos/Cargo.toml -deleted file mode 100644 -index 7462a836d53d..000000000000 ---- b/tools/sched_ext/atropos/Cargo.toml -+++ /dev/null -@@ -1,28 +0,0 @@ --[package] --name = "scx_atropos" --version = "0.5.0" --authors = ["Dan Schatzberg ", "Meta"] --edition = "2021" --description = "Userspace scheduling with BPF" --license = "GPL-2.0-only" -- --[dependencies] --anyhow = "1.0.65" --bitvec = { version = "1.0", features = ["serde"] } --clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } --ctrlc = { version = "3.1", features = ["termination"] } --fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"} --hex = "0.4.3" --libbpf-rs = "0.19.1" --libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] } --libc = "0.2.137" --log = "0.4.17" --ordered-float = "3.4.0" --simplelog = "0.12.0" -- --[build-dependencies] --bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false } --libbpf-cargo = "0.13.0" -- --[features] --enable_backtrace = [] -diff --git b/tools/sched_ext/atropos/build.rs a/tools/sched_ext/atropos/build.rs -deleted file mode 100644 -index 26e792c5e17e..000000000000 ---- b/tools/sched_ext/atropos/build.rs -+++ /dev/null -@@ -1,70 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --extern crate bindgen; -- --use std::env; --use std::fs::create_dir_all; --use std::path::Path; --use std::path::PathBuf; -- --use libbpf_cargo::SkeletonBuilder; -- --const HEADER_PATH: &str = "src/bpf/atropos.h"; -- --fn bindgen_atropos() { -- // Tell cargo to invalidate the built crate whenever the wrapper changes -- println!("cargo:rerun-if-changed={}", HEADER_PATH); -- -- // The bindgen::Builder is the main entry point -- // to bindgen, and lets you build up options for -- // the resulting bindings. -- let bindings = bindgen::Builder::default() -- // The input header we would like to generate -- // bindings for. -- .header(HEADER_PATH) -- // Tell cargo to invalidate the built crate whenever any of the -- // included header files changed. -- .parse_callbacks(Box::new(bindgen::CargoCallbacks)) -- // Finish the builder and generate the bindings. -- .generate() -- // Unwrap the Result and panic on failure. -- .expect("Unable to generate bindings"); -- -- // Write the bindings to the $OUT_DIR/bindings.rs file. -- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); -- bindings -- .write_to_file(out_path.join("atropos-sys.rs")) -- .expect("Couldn't write bindings!"); --} -- --fn gen_bpf_sched(name: &str) { -- let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap(); -- let clang = env::var("ATROPOS_CLANG").unwrap(); -- eprintln!("{}", clang); -- let outpath = format!("./src/bpf/.output/{}.skel.rs", name); -- let skel = Path::new(&outpath); -- let src = format!("./src/bpf/{}.bpf.c", name); -- SkeletonBuilder::new() -- .source(src.clone()) -- .clang(clang) -- .clang_args(bpf_cflags) -- .build_and_generate(&skel) -- .unwrap(); -- println!("cargo:rerun-if-changed={}", src); --} -- --fn main() { -- bindgen_atropos(); -- // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton. -- // Reasons are because the generated skeleton contains compiler attributes -- // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]` -- // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside -- // the path attribute either (see https://github.com/rust-lang/rust/pull/83366). -- // -- // However, there is hope! When the above feature stabilizes we can clean this -- // all up. -- create_dir_all("./src/bpf/.output").unwrap(); -- gen_bpf_sched("atropos"); --} -diff --git b/tools/sched_ext/atropos/rustfmt.toml a/tools/sched_ext/atropos/rustfmt.toml -deleted file mode 100644 -index b7258ed0a8d8..000000000000 ---- b/tools/sched_ext/atropos/rustfmt.toml -+++ /dev/null -@@ -1,8 +0,0 @@ --# Get help on options with `rustfmt --help=config` --# Please keep these in alphabetical order. --edition = "2021" --group_imports = "StdExternalCrate" --imports_granularity = "Item" --merge_derives = false --use_field_init_shorthand = true --version = "Two" -diff --git b/tools/sched_ext/atropos/src/atropos_sys.rs a/tools/sched_ext/atropos/src/atropos_sys.rs -deleted file mode 100644 -index bbeaf856d40e..000000000000 ---- b/tools/sched_ext/atropos/src/atropos_sys.rs -+++ /dev/null -@@ -1,10 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#![allow(non_upper_case_globals)] --#![allow(non_camel_case_types)] --#![allow(non_snake_case)] --#![allow(dead_code)] -- --include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs")); -diff --git b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -deleted file mode 100644 -index 3905a403e940..000000000000 ---- b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -+++ /dev/null -@@ -1,743 +0,0 @@ --/* Copyright (c) Meta Platforms, Inc. and affiliates. */ --/* -- * This software may be used and distributed according to the terms of the -- * GNU General Public License version 2. -- * -- * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF -- * part does simple round robin in each domain and the userspace part -- * calculates the load factor of each domain and tells the BPF part how to load -- * balance the domains. -- * -- * Every task has an entry in the task_data map which lists which domain the -- * task belongs to. When a task first enters the system (atropos_prep_enable), -- * they are round-robined to a domain. -- * -- * atropos_select_cpu is the primary scheduling logic, invoked when a task -- * becomes runnable. The lb_data map is populated by userspace to inform the BPF -- * scheduler that a task should be migrated to a new domain. Otherwise, the task -- * is scheduled in priority order as follows: -- * * The current core if the task was woken up synchronously and there are idle -- * cpus in the system -- * * The previous core, if idle -- * * The pinned-to core if the task is pinned to a specific core -- * * Any idle cpu in the domain -- * -- * If none of the above conditions are met, then the task is enqueued to a -- * dispatch queue corresponding to the domain (atropos_enqueue). -- * -- * atropos_dispatch will attempt to consume a task from its domain's -- * corresponding dispatch queue (this occurs after scheduling any tasks directly -- * assigned to it due to the logic in atropos_select_cpu). If no task is found, -- * then greedy load stealing will attempt to find a task on another dispatch -- * queue to run. -- * -- * Load balancing is almost entirely handled by userspace. BPF populates the -- * task weight, dom mask and current dom in the task_data map and executes the -- * load balance based on userspace populating the lb_data map. -- */ --#include "../../../scx_common.bpf.h" --#include "atropos.h" -- --#include --#include --#include --#include --#include --#include -- --char _license[] SEC("license") = "GPL"; -- --/* -- * const volatiles are set during initialization and treated as consts by the -- * jit compiler. -- */ -- --/* -- * Domains and cpus -- */ --const volatile __u32 nr_doms = 32; /* !0 for veristat, set during init */ --const volatile __u32 nr_cpus = 64; /* !0 for veristat, set during init */ --const volatile __u32 cpu_dom_id_map[MAX_CPUS]; --const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; -- --const volatile bool kthreads_local; --const volatile bool fifo_sched; --const volatile bool switch_partial; --const volatile __u32 greedy_threshold; -- --/* base slice duration */ --const volatile __u64 slice_us = 20000; -- --/* -- * Exit info -- */ --int exit_type = SCX_EXIT_NONE; --char exit_msg[SCX_EXIT_MSG_LEN]; -- --struct pcpu_ctx { -- __u32 dom_rr_cur; /* used when scanning other doms */ -- -- /* libbpf-rs does not respect the alignment, so pad out the struct explicitly */ -- __u8 _padding[CACHELINE_SIZE - sizeof(u64)]; --} __attribute__((aligned(CACHELINE_SIZE))); -- --struct pcpu_ctx pcpu_ctx[MAX_CPUS]; -- --/* -- * Domain context -- */ --struct dom_ctx { -- struct bpf_cpumask __kptr *cpumask; -- u64 vtime_now; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __type(key, u32); -- __type(value, struct dom_ctx); -- __uint(max_entries, MAX_DOMS); -- __uint(map_flags, 0); --} dom_ctx SEC(".maps"); -- --/* -- * Statistics -- */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, ATROPOS_NR_STATS); --} stats SEC(".maps"); -- --static inline void stat_add(enum stat_idx idx, u64 addend) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p) += addend; --} -- --/* Map pid -> task_ctx */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, struct task_ctx); -- __uint(max_entries, 1000000); -- __uint(map_flags, 0); --} task_data SEC(".maps"); -- --/* -- * This is populated from userspace to indicate which pids should be reassigned -- * to new doms. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, u32); -- __uint(max_entries, 1000); -- __uint(map_flags, 0); --} lb_data SEC(".maps"); -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p, -- u32 new_dom_id) --{ -- struct dom_ctx *old_domc, *new_domc; -- struct bpf_cpumask *d_cpumask, *t_cpumask; -- u32 old_dom_id = task_ctx->dom_id; -- s64 vtime_delta; -- -- old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id); -- if (!old_domc) { -- scx_bpf_error("No dom%u", old_dom_id); -- return false; -- } -- -- vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now; -- -- new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id); -- if (!new_domc) { -- scx_bpf_error("No dom%u", new_dom_id); -- return false; -- } -- -- d_cpumask = new_domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to get domain %u cpumask kptr", -- new_dom_id); -- return false; -- } -- -- t_cpumask = task_ctx->cpumask; -- if (!t_cpumask) { -- scx_bpf_error("Failed to look up task cpumask"); -- return false; -- } -- -- /* -- * set_cpumask might have happened between userspace requesting LB and -- * here and @p might not be able to run in @dom_id anymore. Verify. -- */ -- if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- p->cpus_ptr)) { -- p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta; -- task_ctx->dom_id = new_dom_id; -- bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask, -- p->cpus_ptr); -- } -- -- return task_ctx->dom_id == new_dom_id; --} -- --s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu, -- u32 wake_flags) --{ -- s32 cpu; -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- struct bpf_cpumask *p_cpumask; -- -- if (!task_ctx) -- return -ENOENT; -- -- if (kthreads_local && -- (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- cpu = prev_cpu; -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local dsq of the waker. -- */ -- if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) { -- struct task_struct *current = (void *)bpf_get_current_task(); -- -- if (!(BPF_CORE_READ(current, flags) & PF_EXITING) && -- task_ctx->dom_id < MAX_DOMS) { -- struct dom_ctx *domc; -- struct bpf_cpumask *d_cpumask; -- const struct cpumask *idle_cpumask; -- bool has_idle; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id); -- if (!domc) { -- scx_bpf_error("Failed to find dom%u", -- task_ctx->dom_id); -- return prev_cpu; -- } -- d_cpumask = domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to acquire domain %u cpumask kptr", -- task_ctx->dom_id); -- return prev_cpu; -- } -- -- idle_cpumask = scx_bpf_get_idle_cpumask(); -- -- has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- idle_cpumask); -- -- scx_bpf_put_idle_cpumask(idle_cpumask); -- -- if (has_idle) { -- cpu = bpf_get_smp_processor_id(); -- if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- stat_add(ATROPOS_STAT_WAKE_SYNC, 1); -- goto local; -- } -- } -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- stat_add(ATROPOS_STAT_PREV_IDLE, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- /* If only one core is allowed, dispatch */ -- if (p->nr_cpus_allowed == 1) { -- stat_add(ATROPOS_STAT_PINNED, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) -- return -ENOENT; -- -- /* If there is an eligible idle CPU, dispatch directly */ -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- if (cpu >= 0) { -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * @prev_cpu may be in a different domain. Returning an out-of-domain -- * CPU can lead to stalls as all in-domain CPUs may be idle by the time -- * @p gets enqueued. -- */ -- if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask)) -- cpu = prev_cpu; -- else -- cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask); -- -- return cpu; -- --local: -- task_ctx->dispatch_local = true; -- return cpu; --} -- --void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- u32 *new_dom; -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- new_dom = bpf_map_lookup_elem(&lb_data, &pid); -- if (new_dom && *new_dom != task_ctx->dom_id && -- task_set_dsq(task_ctx, p, *new_dom)) { -- struct bpf_cpumask *p_cpumask; -- s32 cpu; -- -- stat_add(ATROPOS_STAT_LOAD_BALANCE, 1); -- -- /* -- * If dispatch_local is set, We own @p's idle state but we are -- * not gonna put the task in the associated local dsq which can -- * cause the CPU to stall. Kick it. -- */ -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0); -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) { -- scx_bpf_error("Failed to get task_ctx->cpumask"); -- return; -- } -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- } -- -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags); -- return; -- } -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000, -- enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- u32 dom_id = task_ctx->dom_id; -- struct dom_ctx *domc; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, domc->vtime_now - slice_us * 1000)) -- vtime = domc->vtime_now - slice_us * 1000; -- -- scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --static u32 cpu_to_dom_id(s32 cpu) --{ -- const volatile u32 *dom_idp; -- -- if (nr_doms <= 1) -- return 0; -- -- dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]); -- if (!dom_idp) -- return MAX_DOMS; -- -- return *dom_idp; --} -- --static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) --{ -- s32 cpu; -- -- if (dom_id >= MAX_DOMS) -- return false; -- -- bpf_for(cpu, 0, nr_cpus) { -- if (bpf_cpumask_test_cpu(cpu, cpumask) && -- (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64)))) -- return true; -- } -- return false; --} -- --static u32 dom_rr_next(s32 cpu) --{ -- struct pcpu_ctx *pcpuc; -- u32 dom_id; -- -- pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]); -- if (!pcpuc) -- return 0; -- -- dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms; -- -- if (dom_id == cpu_to_dom_id(cpu)) -- dom_id = (dom_id + 1) % nr_doms; -- -- pcpuc->dom_rr_cur = dom_id; -- return dom_id; --} -- --void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 dom = cpu_to_dom_id(cpu); -- -- if (scx_bpf_consume(dom)) { -- stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1); -- return; -- } -- -- if (!greedy_threshold) -- return; -- -- bpf_repeat(nr_doms - 1) { -- u32 dom_id = dom_rr_next(cpu); -- -- if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold && -- scx_bpf_consume(dom_id)) { -- stat_add(ATROPOS_STAT_GREEDY, 1); -- break; -- } -- } --} -- --void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_at = bpf_ktime_get_ns(); --} -- --void BPF_STRUCT_OPS(atropos_running, struct task_struct *p) --{ -- struct task_ctx *taskc; -- struct dom_ctx *domc; -- pid_t pid = p->pid; -- u32 dom_id; -- -- if (fifo_sched) -- return; -- -- taskc = bpf_map_lookup_elem(&task_data, &pid); -- if (!taskc) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- dom_id = taskc->dom_id; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(domc->vtime_now, p->scx.dsq_vtime)) -- domc->vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at; -- task_ctx->runnable_at = 0; --} -- --void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->weight = weight; --} -- --struct pick_task_domain_loop_ctx { -- struct task_struct *p; -- const struct cpumask *cpumask; -- u64 dom_mask; -- u32 dom_rr_base; -- u32 dom_id; --}; -- --static int pick_task_domain_loopfn(u32 idx, void *data) --{ -- struct pick_task_domain_loop_ctx *lctx = data; -- u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms; -- -- if (dom_id >= MAX_DOMS) -- return 1; -- -- if (cpumask_intersects_domain(lctx->cpumask, dom_id)) { -- lctx->dom_mask |= 1LLU << dom_id; -- if (lctx->dom_id == MAX_DOMS) -- lctx->dom_id = dom_id; -- } -- return 0; --} -- --static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- struct pick_task_domain_loop_ctx lctx = { -- .p = p, -- .cpumask = cpumask, -- .dom_id = MAX_DOMS, -- }; -- s32 cpu = bpf_get_smp_processor_id(); -- -- if (cpu < 0 || cpu >= MAX_CPUS) -- return MAX_DOMS; -- -- lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur); -- -- bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0); -- task_ctx->dom_mask = lctx.dom_mask; -- -- return lctx.dom_id; --} -- --static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- u32 dom_id = 0; -- -- if (nr_doms > 1) -- dom_id = pick_task_domain(task_ctx, p, cpumask); -- -- if (!task_set_dsq(task_ctx, p, dom_id)) -- scx_bpf_error("Failed to set domain %d for %s[%d]", -- dom_id, p->comm, p->pid); --} -- --void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_set_domain(task_ctx, p, cpumask); --} -- --s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct bpf_cpumask *cpumask; -- struct task_ctx task_ctx, *map_value; -- long ret; -- pid_t pid; -- -- memset(&task_ctx, 0, sizeof(task_ctx)); -- -- pid = p->pid; -- ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return ret; -- } -- -- /* -- * Read the entry from the map immediately so we can add the cpumask -- * with bpf_kptr_xchg(). -- */ -- map_value = bpf_map_lookup_elem(&task_data, &pid); -- if (!map_value) -- /* Should never happen -- it was just inserted above. */ -- return -EINVAL; -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- bpf_map_delete_elem(&task_data, &pid); -- return -ENOMEM; -- } -- -- cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask); -- if (cpumask) { -- /* Should never happen as we just inserted it above. */ -- bpf_cpumask_release(cpumask); -- bpf_map_delete_elem(&task_data, &pid); -- return -EINVAL; -- } -- -- task_set_domain(map_value, p, p->cpus_ptr); -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p) --{ -- pid_t pid = p->pid; -- long ret = bpf_map_delete_elem(&task_data, &pid); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return; -- } --} -- --static int create_dom_dsq(u32 idx, void *data) --{ -- struct dom_ctx domc_init = {}, *domc; -- struct bpf_cpumask *cpumask; -- u32 cpu, dom_id = idx; -- s32 ret; -- -- ret = scx_bpf_create_dsq(dom_id, -1); -- if (ret < 0) { -- scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret); -- return 1; -- } -- -- ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0); -- if (ret) { -- scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret); -- return 1; -- } -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- /* Should never happen, we just inserted it above. */ -- scx_bpf_error("No dom%u", dom_id); -- return 1; -- } -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id); -- return 1; -- } -- -- for (cpu = 0; cpu < MAX_CPUS; cpu++) { -- const volatile __u64 *dmask; -- -- dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]); -- if (!dmask) { -- scx_bpf_error("array index error"); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- if (*dmask & (1LLU << (cpu % 64))) -- bpf_cpumask_set_cpu(cpu, cpumask); -- } -- -- cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask); -- if (cpumask) { -- scx_bpf_error("Domain %u was already present", dom_id); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(atropos_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- bpf_loop(nr_doms, create_dom_dsq, NULL, 0); -- -- for (u32 i = 0; i < nr_cpus; i++) -- pcpu_ctx[i].dom_rr_cur = i; -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg); -- exit_type = ei->type; --} -- --SEC(".struct_ops") --struct sched_ext_ops atropos = { -- .select_cpu = (void *)atropos_select_cpu, -- .enqueue = (void *)atropos_enqueue, -- .dispatch = (void *)atropos_dispatch, -- .runnable = (void *)atropos_runnable, -- .running = (void *)atropos_running, -- .stopping = (void *)atropos_stopping, -- .quiescent = (void *)atropos_quiescent, -- .set_weight = (void *)atropos_set_weight, -- .set_cpumask = (void *)atropos_set_cpumask, -- .prep_enable = (void *)atropos_prep_enable, -- .disable = (void *)atropos_disable, -- .init = (void *)atropos_init, -- .exit = (void *)atropos_exit, -- .flags = 0, -- .name = "atropos", --}; -diff --git b/tools/sched_ext/atropos/src/bpf/atropos.h a/tools/sched_ext/atropos/src/bpf/atropos.h -deleted file mode 100644 -index addf29ca104a..000000000000 ---- b/tools/sched_ext/atropos/src/bpf/atropos.h -+++ /dev/null -@@ -1,44 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#ifndef __ATROPOS_H --#define __ATROPOS_H -- --#include --#ifndef __kptr --#ifdef __KERNEL__ --#error "__kptr_ref not defined in the kernel" --#endif --#define __kptr --#endif -- --#define MAX_CPUS 512 --#define MAX_DOMS 64 /* limited to avoid complex bitmask ops */ --#define CACHELINE_SIZE 64 -- --/* Statistics */ --enum stat_idx { -- ATROPOS_STAT_TASK_GET_ERR, -- ATROPOS_STAT_WAKE_SYNC, -- ATROPOS_STAT_PREV_IDLE, -- ATROPOS_STAT_PINNED, -- ATROPOS_STAT_DIRECT_DISPATCH, -- ATROPOS_STAT_DSQ_DISPATCH, -- ATROPOS_STAT_GREEDY, -- ATROPOS_STAT_LOAD_BALANCE, -- ATROPOS_STAT_LAST_TASK, -- ATROPOS_NR_STATS, --}; -- --struct task_ctx { -- unsigned long long dom_mask; /* the domains this task can run on */ -- struct bpf_cpumask __kptr *cpumask; -- unsigned int dom_id; -- unsigned int weight; -- unsigned long long runnable_at; -- unsigned long long runnable_for; -- bool dispatch_local; --}; -- --#endif /* __ATROPOS_H */ -diff --git b/tools/sched_ext/atropos/src/main.rs a/tools/sched_ext/atropos/src/main.rs -deleted file mode 100644 -index 0d313662f713..000000000000 ---- b/tools/sched_ext/atropos/src/main.rs -+++ /dev/null -@@ -1,942 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#[path = "bpf/.output/atropos.skel.rs"] --mod atropos; --pub use atropos::*; --pub mod atropos_sys; -- --use std::cell::Cell; --use std::collections::{BTreeMap, BTreeSet}; --use std::ffi::CStr; --use std::ops::Bound::{Included, Unbounded}; --use std::sync::atomic::{AtomicBool, Ordering}; --use std::sync::Arc; --use std::time::{Duration, SystemTime}; -- --use ::fb_procfs as procfs; --use anyhow::{anyhow, bail, Context, Result}; --use bitvec::prelude::*; --use clap::Parser; --use log::{info, trace, warn}; --use ordered_float::OrderedFloat; -- --/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF --/// part does simple round robin in each domain and the userspace part --/// calculates the load factor of each domain and tells the BPF part how to load --/// balance the domains. -- --/// This scheduler demonstrates dividing scheduling logic between BPF and --/// userspace and using rust to build the userspace part. An earlier variant of --/// this scheduler was used to balance across six domains, each representing a --/// chiplet in a six-chiplet AMD processor, and could match the performance of --/// production setup using CFS. --#[derive(Debug, Parser)] --struct Opts { -- /// Scheduling slice duration in microseconds. -- #[clap(short, long, default_value = "20000")] -- slice_us: u64, -- -- /// Monitoring and load balance interval in seconds. -- #[clap(short, long, default_value = "2.0")] -- interval: f64, -- -- /// Build domains according to how CPUs are grouped at this cache level -- /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id. -- #[clap(short = 'c', long, default_value = "3")] -- cache_level: u32, -- -- /// Instead of using cache locality, set the cpumask for each domain -- /// manually, provide multiple --cpumasks, one for each domain. E.g. -- /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with -- /// the corresponding CPUs belonging to each domain. Each CPU must -- /// belong to precisely one domain. -- #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")] -- cpumasks: Vec, -- -- /// When non-zero, enable greedy task stealing. When a domain is idle, a -- /// cpu will attempt to steal tasks from a domain with at least -- /// greedy_threshold tasks enqueued. These tasks aren't permanently -- /// stolen from the domain. -- #[clap(short, long, default_value = "4")] -- greedy_threshold: u32, -- -- /// The load decay factor. Every interval, the existing load is decayed -- /// by this factor and new load is added. Must be in the range [0.0, -- /// 0.99]. The smaller the value, the more sensitive load calculation -- /// is to recent changes. When 0.0, history is ignored and the load -- /// value from the latest period is used directly. -- #[clap(short, long, default_value = "0.5")] -- load_decay_factor: f64, -- -- /// Disable load balancing. Unless disabled, periodically userspace will -- /// calculate the load factor of each domain and instruct BPF which -- /// processes to move. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- no_load_balance: bool, -- -- /// Put per-cpu kthreads directly into local dsq's. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- kthreads_local: bool, -- -- /// Use FIFO scheduling instead of weighted vtime scheduling. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- fifo_sched: bool, -- -- /// If specified, only tasks which have their scheduling policy set to -- /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all -- /// tasks are switched. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- partial: bool, -- -- /// Enable verbose output including libbpf details. Specify multiple -- /// times to increase verbosity. -- #[clap(short, long, action = clap::ArgAction::Count)] -- verbose: u8, --} -- --fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result { -- Ok(reader -- .read_stat() -- .context("Failed to read procfs")? -- .total_cpu -- .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?) --} -- --fn now_monotonic() -> u64 { -- let mut time = libc::timespec { -- tv_sec: 0, -- tv_nsec: 0, -- }; -- let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) }; -- assert!(ret == 0); -- time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 --} -- --fn clear_map(map: &mut libbpf_rs::Map) { -- // XXX: libbpf_rs has some design flaw that make it impossible to -- // delete while iterating despite it being safe so we alias it here -- let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) }; -- for key in map.keys() { -- let _ = deleter.delete(&key); -- } --} -- --#[derive(Debug)] --struct TaskLoad { -- runnable_for: u64, -- load: f64, --} -- --#[derive(Debug)] --struct TaskInfo { -- pid: i32, -- dom_mask: u64, -- migrated: Cell, --} -- --struct LoadBalancer<'a, 'b, 'c> { -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- -- tasks_by_load: Vec, TaskInfo>>, -- load_avg: f64, -- dom_loads: Vec, -- -- imbal: Vec, -- doms_to_push: BTreeMap, u32>, -- doms_to_pull: BTreeMap, u32>, -- -- nr_lb_data_errors: &'c mut u64, --} -- --impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> { -- const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10; -- const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1; -- const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50; -- -- fn new( -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- nr_lb_data_errors: &'c mut u64, -- ) -> Self { -- Self { -- maps, -- task_loads, -- nr_doms, -- load_decay_factor, -- -- tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(), -- load_avg: 0f64, -- dom_loads: vec![0.0; nr_doms], -- -- imbal: vec![0.0; nr_doms], -- doms_to_pull: BTreeMap::new(), -- doms_to_push: BTreeMap::new(), -- -- nr_lb_data_errors, -- } -- } -- -- fn read_task_loads(&mut self, period: Duration) -> Result<()> { -- let now_mono = now_monotonic(); -- let task_data = self.maps.task_data(); -- let mut this_task_loads = BTreeMap::::new(); -- let mut load_sum = 0.0f64; -- self.dom_loads = vec![0f64; self.nr_doms]; -- -- for key in task_data.keys() { -- if let Some(task_ctx_vec) = task_data -- .lookup(&key, libbpf_rs::MapFlags::ANY) -- .context("Failed to lookup task_data")? -- { -- let task_ctx = -- unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) }; -- let pid = i32::from_ne_bytes( -- key.as_slice() -- .try_into() -- .context("Invalid key length in task_data map")?, -- ); -- -- let (this_at, this_for, weight) = unsafe { -- ( -- std::ptr::read_volatile(&task_ctx.runnable_at as *const u64), -- std::ptr::read_volatile(&task_ctx.runnable_for as *const u64), -- std::ptr::read_volatile(&task_ctx.weight as *const u32), -- ) -- }; -- -- let (mut delta, prev_load) = match self.task_loads.get(&pid) { -- Some(prev) => (this_for - prev.runnable_for, Some(prev.load)), -- None => (this_for, None), -- }; -- -- // Non-zero this_at indicates that the task is currently -- // runnable. Note that we read runnable_at and runnable_for -- // without any synchronization and there is a small window -- // where we end up misaccounting. While this can cause -- // temporary error, it's unlikely to cause any noticeable -- // misbehavior especially given the load value clamping. -- if this_at > 0 && this_at < now_mono { -- delta += now_mono - this_at; -- } -- -- delta = delta.min(period.as_nanos() as u64); -- let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64) -- .clamp(0.0, weight as f64); -- -- let this_load = match prev_load { -- Some(prev_load) => { -- prev_load * self.load_decay_factor -- + this_load * (1.0 - self.load_decay_factor) -- } -- None => this_load, -- }; -- -- this_task_loads.insert( -- pid, -- TaskLoad { -- runnable_for: this_for, -- load: this_load, -- }, -- ); -- -- load_sum += this_load; -- self.dom_loads[task_ctx.dom_id as usize] += this_load; -- // Only record pids that are eligible for load balancing -- if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) { -- continue; -- } -- self.tasks_by_load[task_ctx.dom_id as usize].insert( -- OrderedFloat(this_load), -- TaskInfo { -- pid, -- dom_mask: task_ctx.dom_mask, -- migrated: Cell::new(false), -- }, -- ); -- } -- } -- -- self.load_avg = load_sum / self.nr_doms as f64; -- *self.task_loads = this_task_loads; -- Ok(()) -- } -- -- // To balance dom loads we identify doms with lower and higher load than average -- fn calculate_dom_load_balance(&mut self) -> Result<()> { -- for (dom, dom_load) in self.dom_loads.iter().enumerate() { -- let imbal = dom_load - self.load_avg; -- if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO { -- if imbal > 0f64 { -- self.doms_to_push.insert(OrderedFloat(imbal), dom as u32); -- } else { -- self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32); -- } -- self.imbal[dom] = imbal; -- } -- } -- Ok(()) -- } -- -- // Find the first candidate pid which hasn't already been migrated and -- // can run in @pull_dom. -- fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)> -- where -- I: IntoIterator, &'d TaskInfo)>, -- { -- match tasks_by_load -- .into_iter() -- .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0) -- .next() -- { -- Some((OrderedFloat(load), task)) => Some((*load, task)), -- None => None, -- } -- } -- -- fn pick_victim( -- &self, -- (push_dom, to_push): (u32, f64), -- (pull_dom, to_pull): (u32, f64), -- ) -> Option<(&TaskInfo, f64)> { -- let to_xfer = to_pull.min(to_push); -- -- trace!( -- "considering dom {}@{:.2} -> {}@{:.2}", -- push_dom, -- to_push, -- pull_dom, -- to_pull -- ); -- -- let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs(); -- -- trace!( -- "to_xfer={:.2} tasks_by_load={:?}", -- to_xfer, -- &self.tasks_by_load[push_dom as usize] -- ); -- -- // We want to pick a task to transfer from push_dom to pull_dom to -- // maximize the reduction of load imbalance between the two. IOW, -- // pick a task which has the closest load value to $to_xfer that can -- // be migrated. Find such task by locating the first migratable task -- // while scanning left from $to_xfer and the counterpart while -- // scanning right and picking the better of the two. -- let (load, task, new_imbal) = match ( -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Unbounded, Included(&OrderedFloat(to_xfer)))) -- .rev(), -- pull_dom, -- ), -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Included(&OrderedFloat(to_xfer)), Unbounded)), -- pull_dom, -- ), -- ) { -- (None, None) => return None, -- (Some((load, task)), None) | (None, Some((load, task))) => { -- (load, task, calc_new_imbal(load)) -- } -- (Some((load0, task0)), Some((load1, task1))) => { -- let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1)); -- if new_imbal0 <= new_imbal1 { -- (load0, task0, new_imbal0) -- } else { -- (load1, task1, new_imbal1) -- } -- } -- }; -- -- // If the best candidate can't reduce the imbalance, there's nothing -- // to do for this pair. -- let old_imbal = to_push + to_pull; -- if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal { -- trace!( -- "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal -- ); -- return None; -- } -- -- trace!( -- "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal, -- ); -- -- Some((task, load)) -- } -- -- // Actually execute the load balancing. Concretely this writes pid -> dom -- // entries into the lb_data map for bpf side to consume. -- fn load_balance(&mut self) -> Result<()> { -- clear_map(self.maps.lb_data()); -- -- trace!("imbal={:?}", &self.imbal); -- trace!("doms_to_push={:?}", &self.doms_to_push); -- trace!("doms_to_pull={:?}", &self.doms_to_pull); -- -- // Push from the most imbalanced to least. -- while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() { -- let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO; -- let mut pushed = 0f64; -- -- // Transfer tasks from push_dom to reduce imbalance. -- loop { -- let last_pushed = pushed; -- -- // Pull from the most imbalaned to least. -- let mut doms_to_pull = BTreeMap::<_, _>::new(); -- std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull); -- let mut pull_doms = doms_to_pull.into_iter().rev().collect::>(); -- -- for (to_pull, pull_dom) in pull_doms.iter_mut() { -- if let Some((task, load)) = -- self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull))) -- { -- // Execute migration. -- task.migrated.set(true); -- to_push -= load; -- *to_pull -= load; -- pushed += load; -- -- // Ask BPF code to execute the migration. -- let pid = task.pid; -- let cpid = (pid as libc::pid_t).to_ne_bytes(); -- if let Err(e) = self.maps.lb_data().update( -- &cpid, -- &pull_dom.to_ne_bytes(), -- libbpf_rs::MapFlags::NO_EXIST, -- ) { -- warn!( -- "Failed to update lb_data map for pid={} error={:?}", -- pid, &e -- ); -- *self.nr_lb_data_errors += 1; -- } -- -- // Always break after a successful migration so that -- // the pulling domains are always considered in the -- // descending imbalance order. -- break; -- } -- } -- -- pull_doms -- .into_iter() -- .map(|(k, v)| self.doms_to_pull.insert(k, v)) -- .count(); -- -- // Stop repeating if nothing got transferred or pushed enough. -- if pushed == last_pushed || pushed >= push_max { -- break; -- } -- } -- } -- Ok(()) -- } --} -- --struct Scheduler<'a> { -- skel: AtroposSkel<'a>, -- struct_ops: Option, -- -- nr_cpus: usize, -- nr_doms: usize, -- load_decay_factor: f64, -- balance_load: bool, -- -- proc_reader: procfs::ProcReader, -- -- prev_at: SystemTime, -- prev_total_cpu: procfs::CpuStat, -- task_loads: BTreeMap, -- -- nr_lb_data_errors: u64, --} -- --impl<'a> Scheduler<'a> { -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn parse_cpusets( -- cpumasks: &[String], -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- if cpumasks.len() > atropos_sys::MAX_DOMS as usize { -- bail!( -- "Number of requested DSQs ({}) is greater than MAX_DOMS ({})", -- cpumasks.len(), -- atropos_sys::MAX_DOMS -- ); -- } -- let mut cpus = vec![-1i32; nr_cpus]; -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()]; -- for (dq, cpumask) in cpumasks.iter().enumerate() { -- let hex_str = { -- let mut tmp_str = cpumask -- .strip_prefix("0x") -- .unwrap_or(cpumask) -- .replace('_', ""); -- if tmp_str.len() % 2 != 0 { -- tmp_str = "0".to_string() + &tmp_str; -- } -- tmp_str -- }; -- let byte_vec = hex::decode(&hex_str) -- .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?; -- -- for (index, &val) in byte_vec.iter().rev().enumerate() { -- let mut v = val; -- while v != 0 { -- let lsb = v.trailing_zeros() as usize; -- v &= !(1 << lsb); -- let cpu = index * 8 + lsb; -- if cpu > nr_cpus { -- bail!( -- concat!( -- "Found cpu ({}) in cpumask ({}) which is larger", -- " than the number of cpus on the machine ({})" -- ), -- cpu, -- cpumask, -- nr_cpus -- ); -- } -- if cpus[cpu] != -1 { -- bail!( -- "Found cpu ({}) with dq ({}) but also in cpumask ({})", -- cpu, -- cpus[cpu], -- cpumask -- ); -- } -- cpus[cpu] = dq as i32; -- cpusets[dq].set(cpu, true); -- } -- } -- cpusets[dq].set_uninitialized(false); -- } -- -- for (cpu, &dq) in cpus.iter().enumerate() { -- if dq < 0 { -- bail!( -- "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.", -- cpu -- ); -- } -- } -- -- Ok((cpusets, cpus)) -- } -- -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn cpusets_from_cache( -- level: u32, -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- let mut cpu_to_cache = vec![]; // (cpu_id, cache_id) -- let mut cache_ids = BTreeSet::::new(); -- let mut nr_not_found = 0; -- -- // Build cpu -> cache ID mapping. -- for cpu in 0..nr_cpus { -- let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level); -- let id = match std::fs::read_to_string(&path) { -- Ok(val) => val -- .trim() -- .parse::() -- .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?, -- Err(e) if e.kind() == std::io::ErrorKind::NotFound => { -- nr_not_found += 1; -- 0 -- } -- Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)), -- }; -- -- cpu_to_cache.push(id); -- cache_ids.insert(id); -- } -- -- if nr_not_found > 1 { -- warn!( -- "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0", -- level, nr_not_found, nr_cpus -- ); -- } -- -- // Cache IDs may have holes. Assign consecutive domain IDs to -- // existing cache IDs. -- let mut cache_to_dom = BTreeMap::::new(); -- let mut nr_doms = 0; -- for cache_id in cache_ids.iter() { -- cache_to_dom.insert(*cache_id, nr_doms); -- nr_doms += 1; -- } -- -- if nr_doms > atropos_sys::MAX_DOMS { -- bail!( -- "Total number of doms {} is greater than MAX_DOMS ({})", -- nr_doms, -- atropos_sys::MAX_DOMS -- ); -- } -- -- // Build and return dom -> cpumask and cpu -> dom mappings. -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize]; -- let mut cpu_to_dom = vec![]; -- -- for cpu in 0..nr_cpus { -- let dom_id = cache_to_dom[&cpu_to_cache[cpu]]; -- cpusets[dom_id as usize].set(cpu, true); -- cpu_to_dom.push(dom_id as i32); -- } -- -- Ok((cpusets, cpu_to_dom)) -- } -- -- fn init(opts: &Opts) -> Result { -- // Open the BPF prog first for verification. -- let mut skel_builder = AtroposSkelBuilder::default(); -- skel_builder.obj_builder.debug(opts.verbose > 0); -- let mut skel = skel_builder.open().context("Failed to open BPF program")?; -- -- let nr_cpus = libbpf_rs::num_possible_cpus().unwrap(); -- if nr_cpus > atropos_sys::MAX_CPUS as usize { -- bail!( -- "nr_cpus ({}) is greater than MAX_CPUS ({})", -- nr_cpus, -- atropos_sys::MAX_CPUS -- ); -- } -- -- // Initialize skel according to @opts. -- let (cpusets, cpus) = if opts.cpumasks.len() > 0 { -- Self::parse_cpusets(&opts.cpumasks, nr_cpus)? -- } else { -- Self::cpusets_from_cache(opts.cache_level, nr_cpus)? -- }; -- let nr_doms = cpusets.len(); -- skel.rodata().nr_doms = nr_doms as u32; -- skel.rodata().nr_cpus = nr_cpus as u32; -- -- for (cpu, dom) in cpus.iter().enumerate() { -- skel.rodata().cpu_dom_id_map[cpu] = *dom as u32; -- } -- -- for (dom, cpuset) in cpusets.iter().enumerate() { -- let raw_cpuset_slice = cpuset.as_raw_slice(); -- let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom]; -- let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len()); -- left.clone_from_slice(cpuset.as_raw_slice()); -- let cpumask_str = dom_cpumask_slice -- .iter() -- .take((nr_cpus + 63) / 64) -- .rev() -- .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x)); -- info!( -- "DOM[{:02}] cpumask{} ({} cpus)", -- dom, -- &cpumask_str, -- cpuset.count_ones() -- ); -- } -- -- skel.rodata().slice_us = opts.slice_us; -- skel.rodata().kthreads_local = opts.kthreads_local; -- skel.rodata().fifo_sched = opts.fifo_sched; -- skel.rodata().switch_partial = opts.partial; -- skel.rodata().greedy_threshold = opts.greedy_threshold; -- -- // Attach. -- let mut skel = skel.load().context("Failed to load BPF program")?; -- skel.attach().context("Failed to attach BPF program")?; -- let struct_ops = Some( -- skel.maps_mut() -- .atropos() -- .attach_struct_ops() -- .context("Failed to attach atropos struct ops")?, -- ); -- info!("Atropos Scheduler Attached"); -- -- // Other stuff. -- let mut proc_reader = procfs::ProcReader::new(); -- let prev_total_cpu = read_total_cpu(&mut proc_reader)?; -- -- Ok(Self { -- skel, -- struct_ops, // should be held to keep it attached -- -- nr_cpus, -- nr_doms, -- load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99), -- balance_load: !opts.no_load_balance, -- -- proc_reader, -- -- prev_at: SystemTime::now(), -- prev_total_cpu, -- task_loads: BTreeMap::new(), -- -- nr_lb_data_errors: 0, -- }) -- } -- -- fn get_cpu_busy(&mut self) -> Result { -- let total_cpu = read_total_cpu(&mut self.proc_reader)?; -- let busy = match (&self.prev_total_cpu, &total_cpu) { -- ( -- procfs::CpuStat { -- user_usec: Some(prev_user), -- nice_usec: Some(prev_nice), -- system_usec: Some(prev_system), -- idle_usec: Some(prev_idle), -- iowait_usec: Some(prev_iowait), -- irq_usec: Some(prev_irq), -- softirq_usec: Some(prev_softirq), -- stolen_usec: Some(prev_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- procfs::CpuStat { -- user_usec: Some(curr_user), -- nice_usec: Some(curr_nice), -- system_usec: Some(curr_system), -- idle_usec: Some(curr_idle), -- iowait_usec: Some(curr_iowait), -- irq_usec: Some(curr_irq), -- softirq_usec: Some(curr_softirq), -- stolen_usec: Some(curr_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- ) => { -- let idle_usec = curr_idle - prev_idle; -- let iowait_usec = curr_iowait - prev_iowait; -- let user_usec = curr_user - prev_user; -- let system_usec = curr_system - prev_system; -- let nice_usec = curr_nice - prev_nice; -- let irq_usec = curr_irq - prev_irq; -- let softirq_usec = curr_softirq - prev_softirq; -- let stolen_usec = curr_stolen - prev_stolen; -- -- let busy_usec = -- user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec; -- let total_usec = idle_usec + busy_usec + iowait_usec; -- busy_usec as f64 / total_usec as f64 -- } -- _ => { -- bail!("Some procfs stats are not populated!"); -- } -- }; -- -- self.prev_total_cpu = total_cpu; -- Ok(busy) -- } -- -- fn read_bpf_stats(&mut self) -> Result> { -- let mut maps = self.skel.maps_mut(); -- let stats_map = maps.stats(); -- let mut stats: Vec = Vec::new(); -- let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus]; -- -- for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS { -- let cpu_stat_vec = stats_map -- .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY) -- .with_context(|| format!("Failed to lookup stat {}", stat))? -- .expect("per-cpu stat should exist"); -- let sum = cpu_stat_vec -- .iter() -- .map(|val| { -- u64::from_ne_bytes( -- val.as_slice() -- .try_into() -- .expect("Invalid value length in stat map"), -- ) -- }) -- .sum(); -- stats_map -- .update_percpu( -- &(stat as u32).to_ne_bytes(), -- &zero_vec, -- libbpf_rs::MapFlags::ANY, -- ) -- .context("Failed to zero stat")?; -- stats.push(sum); -- } -- Ok(stats) -- } -- -- fn report( -- &self, -- stats: &Vec, -- cpu_busy: f64, -- processing_dur: Duration, -- load_avg: f64, -- dom_loads: &Vec, -- imbal: &Vec, -- ) { -- let stat = |idx| stats[idx as usize]; -- let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK); -- -- info!( -- "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms", -- cpu_busy * 100.0, -- load_avg, -- stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize], -- stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize], -- self.nr_lb_data_errors, -- processing_dur.as_millis(), -- ); -- -- let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0; -- -- info!( -- "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}", -- total, -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY), -- ); -- -- for i in 0..self.nr_doms { -- info!( -- "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}", -- i, -- dom_loads[i], -- if imbal[i] < 0.0 { -imbal[i] } else { 0.0 }, -- if imbal[i] > 0.0 { imbal[i] } else { 0.0 }, -- ); -- } -- } -- -- fn step(&mut self) -> Result<()> { -- let started_at = std::time::SystemTime::now(); -- let bpf_stats = self.read_bpf_stats()?; -- let cpu_busy = self.get_cpu_busy()?; -- -- let mut lb = LoadBalancer::new( -- self.skel.maps_mut(), -- &mut self.task_loads, -- self.nr_doms, -- self.load_decay_factor, -- &mut self.nr_lb_data_errors, -- ); -- -- lb.read_task_loads(started_at.duration_since(self.prev_at)?)?; -- lb.calculate_dom_load_balance()?; -- -- if self.balance_load { -- lb.load_balance()?; -- } -- -- // Extract fields needed for reporting and drop lb to release -- // mutable borrows. -- let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal); -- -- self.report( -- &bpf_stats, -- cpu_busy, -- std::time::SystemTime::now().duration_since(started_at)?, -- load_avg, -- &dom_loads, -- &imbal, -- ); -- -- self.prev_at = started_at; -- Ok(()) -- } -- -- fn read_bpf_exit_type(&mut self) -> i32 { -- unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) } -- } -- -- fn report_bpf_exit_type(&mut self) -> Result<()> { -- // Report msg if EXT_OPS_EXIT_ERROR. -- match self.read_bpf_exit_type() { -- 0 => Ok(()), -- etype if etype == 2 => { -- let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) }; -- let msg = cstr -- .to_str() -- .context("Failed to convert exit msg to string") -- .unwrap(); -- bail!("BPF exit_type={} msg={}", etype, msg); -- } -- etype => { -- info!("BPF exit_type={}", etype); -- Ok(()) -- } -- } -- } --} -- --impl<'a> Drop for Scheduler<'a> { -- fn drop(&mut self) { -- if let Some(struct_ops) = self.struct_ops.take() { -- drop(struct_ops); -- } -- } --} -- --fn main() -> Result<()> { -- let opts = Opts::parse(); -- -- let llv = match opts.verbose { -- 0 => simplelog::LevelFilter::Info, -- 1 => simplelog::LevelFilter::Debug, -- _ => simplelog::LevelFilter::Trace, -- }; -- let mut lcfg = simplelog::ConfigBuilder::new(); -- lcfg.set_time_level(simplelog::LevelFilter::Error) -- .set_location_level(simplelog::LevelFilter::Off) -- .set_target_level(simplelog::LevelFilter::Off) -- .set_thread_level(simplelog::LevelFilter::Off); -- simplelog::TermLogger::init( -- llv, -- lcfg.build(), -- simplelog::TerminalMode::Stderr, -- simplelog::ColorChoice::Auto, -- )?; -- -- let shutdown = Arc::new(AtomicBool::new(false)); -- let shutdown_clone = shutdown.clone(); -- ctrlc::set_handler(move || { -- shutdown_clone.store(true, Ordering::Relaxed); -- }) -- .context("Error setting Ctrl-C handler")?; -- -- let mut sched = Scheduler::init(&opts)?; -- -- while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 { -- std::thread::sleep(Duration::from_secs_f64(opts.interval)); -- sched.step()?; -- } -- -- sched.report_bpf_exit_type() --} -diff --git b/tools/sched_ext/gnu/stubs.h a/tools/sched_ext/gnu/stubs.h -deleted file mode 100644 -index 719225b16626..000000000000 ---- b/tools/sched_ext/gnu/stubs.h -+++ /dev/null -@@ -1 +0,0 @@ --/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ -diff --git b/tools/sched_ext/scx_common.bpf.h a/tools/sched_ext/scx_common.bpf.h -deleted file mode 100644 -index e56de9dc86f2..000000000000 ---- b/tools/sched_ext/scx_common.bpf.h -+++ /dev/null -@@ -1,288 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __SCHED_EXT_COMMON_BPF_H --#define __SCHED_EXT_COMMON_BPF_H -- --#include "vmlinux.h" --#include --#include --#include --#include "user_exit_info.h" -- --#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ --#define PF_EXITING 0x00000004 --#define CLOCK_MONOTONIC 1 -- --/* -- * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can -- * lead to really confusing misbehaviors. Let's trigger a build failure. -- */ --static inline void ___vmlinux_h_sanity_check___(void) --{ -- _Static_assert(SCX_DSQ_FLAG_BUILTIN, -- "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); --} -- --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; -- --static inline __attribute__((format(printf, 1, 2))) --void ___scx_bpf_error_format_checker(const char *fmt, ...) {} -- --/* -- * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments -- * instead of an array of u64. Note that __param[] must have at least one -- * element to keep the verifier happy. -- */ --#define scx_bpf_error(fmt, args...) \ --({ \ -- static char ___fmt[] = fmt; \ -- unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ -- \ -- _Pragma("GCC diagnostic push") \ -- _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ -- ___bpf_fill(___param, args); \ -- _Pragma("GCC diagnostic pop") \ -- \ -- scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ -- \ -- ___scx_bpf_error_format_checker(fmt, ##args); \ --}) -- --void scx_bpf_switch_all(void) __ksym; --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; --bool scx_bpf_consume(u64 dsq_id) __ksym; --u32 scx_bpf_dispatch_nr_slots(void) __ksym; --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym; --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym; --void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; --s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym; --const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; --const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; --void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; --void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; --bool scx_bpf_task_running(const struct task_struct *p) __ksym; --s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym; --u32 scx_bpf_reenqueue_local(void) __ksym; -- --#define BPF_STRUCT_OPS(name, args...) \ --SEC("struct_ops/"#name) \ --BPF_PROG(name, ##args) -- --#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ --SEC("struct_ops.s/"#name) \ --BPF_PROG(name, ##args) -- --/** -- * MEMBER_VPTR - Obtain the verified pointer to a struct or array member -- * @base: struct or array to index -- * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...) -- * -- * The verifier often gets confused by the instruction sequence the compiler -- * generates for indexing struct fields or arrays. This macro forces the -- * compiler to generate a code sequence which first calculates the byte offset, -- * checks it against the struct or array size and add that byte offset to -- * generate the pointer to the member to help the verifier. -- * -- * Ideally, we want to abort if the calculated offset is out-of-bounds. However, -- * BPF currently doesn't support abort, so evaluate to NULL instead. The caller -- * must check for NULL and take appropriate action to appease the verifier. To -- * avoid confusing the verifier, it's best to check for NULL and dereference -- * immediately. -- * -- * vptr = MEMBER_VPTR(my_array, [i][j]); -- * if (!vptr) -- * return error; -- * *vptr = new_value; -- */ --#define MEMBER_VPTR(base, member) (typeof(base member) *)({ \ -- u64 __base = (u64)base; \ -- u64 __addr = (u64)&(base member) - __base; \ -- asm volatile ( \ -- "if %0 <= %[max] goto +2\n" \ -- "%0 = 0\n" \ -- "goto +1\n" \ -- "%0 += %1\n" \ -- : "+r"(__addr) \ -- : "r"(__base), \ -- [max]"i"(sizeof(base) - sizeof(base member))); \ -- __addr; \ --}) -- --/* -- * BPF core and other generic helpers -- */ -- --/* list and rbtree */ --#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) --#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) -- --void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; --void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; -- --#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) --#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) -- --void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; --struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; --struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, -- struct bpf_rb_node *node) __ksym; --void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, -- bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym; --struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; -- --/* task */ --struct task_struct *bpf_task_from_pid(s32 pid) __ksym; --struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; --void bpf_task_release(struct task_struct *p) __ksym; -- --/* cgroup */ --struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; --void bpf_cgroup_release(struct cgroup *cgrp) __ksym; --struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; -- --/* cpumask */ --struct bpf_cpumask *bpf_cpumask_create(void) __ksym; --struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; --u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; --u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym; -- --/* rcu */ --void bpf_rcu_read_lock(void) __ksym; --void bpf_rcu_read_unlock(void) __ksym; -- --/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */ --struct bpf_iter_num; -- --extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym; --extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym; --extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym; -- --#ifndef bpf_for_each --/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for -- * using BPF open-coded iterators without having to write mundane explicit -- * low-level loop logic. Instead, it provides for()-like generic construct -- * that can be used pretty naturally. E.g., for some hypothetical cgroup -- * iterator, you'd write: -- * -- * struct cgroup *cg, *parent_cg = <...>; -- * -- * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { -- * bpf_printk("Child cgroup id = %d", cg->cgroup_id); -- * if (cg->cgroup_id == 123) -- * break; -- * } -- * -- * I.e., it looks almost like high-level for each loop in other languages, -- * supports continue/break, and is verifiable by BPF verifier. -- * -- * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) -- * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to -- * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int -- * *`, not just `int`. So for integers bpf_for() is more convenient. -- * -- * Note: this macro relies on C99 feature of allowing to declare variables -- * inside for() loop, bound to for() loop lifetime. It also utilizes GCC -- * extension: __attribute__((cleanup())), supported by both GCC and -- * Clang. -- */ --#define bpf_for_each(type, cur, args...) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ -- cleanup(bpf_iter_##type##_destroy))), \ -- /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_##type##_new(&___it, ##args), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_##type##_destroy, (void *)0); \ -- /* iteration and termination check */ \ -- (((cur) = bpf_iter_##type##_next(&___it))); \ --) --#endif /* bpf_for_each */ -- --#ifndef bpf_for --/* bpf_for(i, start, end) implements a for()-like looping construct that sets -- * provided integer variable *i* to values starting from *start* through, -- * but not including, *end*. It also proves to BPF verifier that *i* belongs -- * to range [start, end), so this can be used for accessing arrays without -- * extra checks. -- * -- * Note: *start* and *end* are assumed to be expressions with no side effects -- * and whose values do not change throughout bpf_for() loop execution. They do -- * not have to be statically known or constant, though. -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_for(i, start, end) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, (start), (end)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- ({ \ -- /* iteration step */ \ -- int *___t = bpf_iter_num_next(&___it); \ -- /* termination and bounds check */ \ -- (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ -- }); \ --) --#endif /* bpf_for */ -- --#ifndef bpf_repeat --/* bpf_repeat(N) performs N iterations without exposing iteration number -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_repeat(N) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, 0, (N)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- bpf_iter_num_next(&___it); \ -- /* nothing here */ \ --) --#endif /* bpf_repeat */ -- --#endif /* __SCHED_EXT_COMMON_BPF_H */ -diff --git b/tools/sched_ext/scx_example_central.bpf.c a/tools/sched_ext/scx_example_central.bpf.c -deleted file mode 100644 -index 4cec04b4c2ed..000000000000 ---- b/tools/sched_ext/scx_example_central.bpf.c -+++ /dev/null -@@ -1,334 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A central FIFO sched_ext scheduler which demonstrates the followings: -- * -- * a. Making all scheduling decisions from one CPU: -- * -- * The central CPU is the only one making scheduling decisions. All other -- * CPUs kick the central CPU when they run out of tasks to run. -- * -- * There is one global BPF queue and the central CPU schedules all CPUs by -- * dispatching from the global queue to each CPU's local dsq from dispatch(). -- * This isn't the most straightforward. e.g. It'd be easier to bounce -- * through per-CPU BPF queues. The current design is chosen to maximally -- * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching. -- * -- * b. Tickless operation -- * -- * All tasks are dispatched with the infinite slice which allows stopping the -- * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full -- * parameter. The tickless operation can be observed through -- * /proc/interrupts. -- * -- * Periodic switching is enforced by a periodic timer checking all CPUs and -- * preempting them as necessary. Unfortunately, BPF timer currently doesn't -- * have a way to pin to a specific CPU, so the periodic timer isn't pinned to -- * the central CPU. -- * -- * c. Preemption -- * -- * Kthreads are unconditionally queued to the head of a matching local dsq -- * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always -- * prioritized over user threads, which is required for ensuring forward -- * progress as e.g. the periodic timer may run on a ksoftirqd and if the -- * ksoftirqd gets starved by a user thread, there may not be anything else to -- * vacate that user thread. -- * -- * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the -- * next tasks. -- * -- * This scheduler is designed to maximize usage of various SCX mechanisms. A -- * more practical implementation would likely put the scheduling loop outside -- * the central CPU's dispatch() path and add some form of priority mechanism. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --enum { -- FALLBACK_DSQ_ID = 0, -- MAX_CPUS = 4096, -- MS_TO_NS = 1000LLU * 1000, -- TIMER_INTERVAL_NS = 1 * MS_TO_NS, --}; -- --const volatile bool switch_partial; --const volatile s32 central_cpu; --const volatile u32 nr_cpu_ids = 64; /* !0 for veristat, set during init */ -- --u64 nr_total, nr_locals, nr_queued, nr_lost_pids; --u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries; --u64 nr_overflows; -- --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, s32); --} central_q SEC(".maps"); -- --/* can't use percpu map due to bad lookups */ --static bool cpu_gimme_task[MAX_CPUS]; --static u64 cpu_started_at[MAX_CPUS]; -- --struct central_timer { -- struct bpf_timer timer; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, 1); -- __type(key, u32); -- __type(value, struct central_timer); --} central_timer SEC(".maps"); -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- /* -- * Steer wakeups to the central CPU as much as possible to avoid -- * disturbing other CPUs. It's safe to blindly return the central cpu as -- * select_cpu() is a hint and if @p can't be on it, the kernel will -- * automatically pick a fallback CPU. -- */ -- return central_cpu; --} -- --void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags) --{ -- s32 pid = p->pid; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- /* -- * Push per-cpu kthreads at the head of local dsq's and preempt the -- * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked -- * behind other threads which is necessary for forward progress -- * guarantee as we depend on the BPF timer which may run from ksoftirqd. -- */ -- if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- __sync_fetch_and_add(&nr_locals, 1); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF, -- enq_flags | SCX_ENQ_PREEMPT); -- return; -- } -- -- if (bpf_map_push_elem(¢ral_q, &pid, 0)) { -- __sync_fetch_and_add(&nr_overflows, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_queued, 1); -- -- if (!scx_bpf_task_running(p)) -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); --} -- --static bool dispatch_to_cpu(s32 cpu) --{ -- struct task_struct *p; -- s32 pid; -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (bpf_map_pop_elem(¢ral_q, &pid)) -- break; -- -- __sync_fetch_and_sub(&nr_queued, 1); -- -- p = bpf_task_from_pid(pid); -- if (!p) { -- __sync_fetch_and_add(&nr_lost_pids, 1); -- continue; -- } -- -- /* -- * If we can't run the task at the top, do the dumb thing and -- * bounce it to the fallback dsq. -- */ -- if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- __sync_fetch_and_add(&nr_mismatches, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0); -- bpf_task_release(p); -- continue; -- } -- -- /* dispatch to local and mark that @cpu doesn't need more */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0); -- -- if (cpu != central_cpu) -- scx_bpf_kick_cpu(cpu, 0); -- -- bpf_task_release(p); -- return true; -- } -- -- return false; --} -- --void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (cpu == central_cpu) { -- /* dispatch for all other CPUs first */ -- __sync_fetch_and_add(&nr_dispatches, 1); -- -- bpf_for(cpu, 0, nr_cpu_ids) { -- bool *gimme; -- -- if (!scx_bpf_dispatch_nr_slots()) -- break; -- -- /* central's gimme is never set */ -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme && !*gimme) -- continue; -- -- if (dispatch_to_cpu(cpu)) -- *gimme = false; -- } -- -- /* -- * Retry if we ran out of dispatch buffer slots as we might have -- * skipped some CPUs and also need to dispatch for self. The ext -- * core automatically retries if the local dsq is empty but we -- * can't rely on that as we're dispatching for other CPUs too. -- * Kick self explicitly to retry. -- */ -- if (!scx_bpf_dispatch_nr_slots()) { -- __sync_fetch_and_add(&nr_retries, 1); -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- return; -- } -- -- /* look for a task to run on the central CPU */ -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- dispatch_to_cpu(central_cpu); -- } else { -- bool *gimme; -- -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme) -- *gimme = true; -- -- /* -- * Force dispatch on the scheduling CPU so that it finds a task -- * to run for us. -- */ -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- } --} -- --void BPF_STRUCT_OPS(central_running, struct task_struct *p) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ --} -- --void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = 0; --} -- --static int central_timerfn(void *map, int *key, struct bpf_timer *timer) --{ -- u64 now = bpf_ktime_get_ns(); -- u64 nr_to_kick = nr_queued; -- s32 i; -- -- bpf_for(i, 0, nr_cpu_ids) { -- s32 cpu = (nr_timers + i) % nr_cpu_ids; -- u64 *started_at; -- -- if (cpu == central_cpu) -- continue; -- -- /* kick iff the current one exhausted its slice */ -- started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at && *started_at && -- vtime_before(now, *started_at + SCX_SLICE_DFL)) -- continue; -- -- /* and there's something pending */ -- if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) || -- scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu)) -- ; -- else if (nr_to_kick) -- nr_to_kick--; -- else -- continue; -- -- scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); -- } -- -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- -- bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- __sync_fetch_and_add(&nr_timers, 1); -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(central_init) --{ -- u32 key = 0; -- struct bpf_timer *timer; -- int ret; -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1); -- if (ret) -- return ret; -- -- timer = bpf_map_lookup_elem(¢ral_timer, &key); -- if (!timer) -- return -ESRCH; -- -- bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); -- bpf_timer_set_callback(timer, central_timerfn); -- ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- return ret; --} -- --void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops central_ops = { -- /* -- * We are offloading all scheduling decisions to the central CPU and -- * thus being the last task on a given CPU doesn't mean anything -- * special. Enqueue the last tasks like any other tasks. -- */ -- .flags = SCX_OPS_ENQ_LAST, -- -- .select_cpu = (void *)central_select_cpu, -- .enqueue = (void *)central_enqueue, -- .dispatch = (void *)central_dispatch, -- .running = (void *)central_running, -- .stopping = (void *)central_stopping, -- .init = (void *)central_init, -- .exit = (void *)central_exit, -- .name = "central", --}; -diff --git b/tools/sched_ext/scx_example_central.c a/tools/sched_ext/scx_example_central.c -deleted file mode 100644 -index 7ad591cbdc65..000000000000 ---- b/tools/sched_ext/scx_example_central.c -+++ /dev/null -@@ -1,94 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_central.skel.h" -- --const char help_fmt[] = --"A central FIFO sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-c CPU] [-p]\n" --"\n" --" -c CPU Override the central CPU (default: 0)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_central *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_central__open(); -- assert(skel); -- -- skel->rodata->central_cpu = 0; -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "c:ph")) != -1) { -- switch (opt) { -- case 'c': -- skel->rodata->central_cpu = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_central__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.central_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf("total :%10lu local:%10lu queued:%10lu lost:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_locals, -- skel->bss->nr_queued, -- skel->bss->nr_lost_pids); -- printf("timer :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n", -- skel->bss->nr_timers, -- skel->bss->nr_dispatches, -- skel->bss->nr_mismatches, -- skel->bss->nr_retries); -- printf("overflow:%10lu\n", -- skel->bss->nr_overflows); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_central__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_flatcg.bpf.c a/tools/sched_ext/scx_example_flatcg.bpf.c -deleted file mode 100644 -index f6078b9a681f..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.bpf.c -+++ /dev/null -@@ -1,872 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext flattened cgroup hierarchy scheduler. It implements -- * hierarchical weight-based cgroup CPU control by flattening the cgroup -- * hierarchy into a single layer by compounding the active weight share at each -- * level. Consider the following hierarchy with weights in parentheses: -- * -- * R + A (100) + B (100) -- * | \ C (100) -- * \ D (200) -- * -- * Ignoring the root and threaded cgroups, only B, C and D can contain tasks. -- * Let's say all three have runnable tasks. The total share that each of these -- * three cgroups is entitled to can be calculated by compounding its share at -- * each level. -- * -- * For example, B is competing against C and in that competition its share is -- * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's -- * share in that competition is 200/(200+100) == 1/3. B's eventual share in the -- * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's -- * eventual shaer is the same at 1/6. D is only competing at the top level and -- * its share is 200/(100+200) == 2/3. -- * -- * So, instead of hierarchically scheduling level-by-level, we can consider it -- * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3 -- * and keep updating the eventual shares as the cgroups' runnable states change. -- * -- * This flattening of hierarchy can bring a substantial performance gain when -- * the cgroup hierarchy is nested multiple levels. in a simple benchmark using -- * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it -- * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two -- * apache instances competing with 2:1 weight ratio nested four level deep. -- * -- * However, the gain comes at the cost of not being able to properly handle -- * thundering herd of cgroups. For example, if many cgroups which are nested -- * behind a low priority parent cgroup wake up around the same time, they may be -- * able to consume more CPU cycles than they are entitled to. In many use cases, -- * this isn't a real concern especially given the performance gain. Also, there -- * are ways to mitigate the problem further by e.g. introducing an extra -- * scheduling layer on cgroup delegation boundaries. -- * -- * The scheduler first picks the cgroup to run and then schedule the tasks -- * within by using nested weighted vtime scheduling by default. The -- * cgroup-internal scheduling can be switched to FIFO with the -f option. -- */ --#include "scx_common.bpf.h" --#include "user_exit_info.h" --#include "scx_example_flatcg.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */ --const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL; --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --u64 cvtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, u64); -- __uint(max_entries, FCG_NR_STATS); --} stats SEC(".maps"); -- --static void stat_inc(enum fcg_stat_idx idx) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p)++; --} -- --struct fcg_cpu_ctx { -- u64 cur_cgid; -- u64 cur_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, struct fcg_cpu_ctx); -- __uint(max_entries, 1); --} cpu_ctx SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_cgrp_ctx); --} cgrp_ctx SEC(".maps"); -- --struct cgv_node { -- struct bpf_rb_node rb_node; -- __u64 cvtime; -- __u64 cgid; --}; -- --private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock; --private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node); -- --struct cgv_node_stash { -- struct cgv_node __kptr *node; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, 16384); -- __type(key, __u64); -- __type(value, struct cgv_node_stash); --} cgv_node_stash SEC(".maps"); -- --struct fcg_task_ctx { -- u64 bypassed_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_task_ctx); --} task_ctx SEC(".maps"); -- --/* gets inc'd on weight tree changes to expire the cached hweights */ --unsigned long hweight_gen = 1; -- --static u64 div_round_up(u64 dividend, u64 divisor) --{ -- return (dividend + divisor - 1) / divisor; --} -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) --{ -- struct cgv_node *cgc_a, *cgc_b; -- -- cgc_a = container_of(a, struct cgv_node, rb_node); -- cgc_b = container_of(b, struct cgv_node, rb_node); -- -- return cgc_a->cvtime < cgc_b->cvtime; --} -- --static struct fcg_cpu_ctx *find_cpu_ctx(void) --{ -- struct fcg_cpu_ctx *cpuc; -- u32 idx = 0; -- -- cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx); -- if (!cpuc) { -- scx_bpf_error("cpu_ctx lookup failed"); -- return NULL; -- } -- return cpuc; --} -- --static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id); -- return NULL; -- } -- return cgc; --} -- --static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgrp = bpf_cgroup_ancestor(cgrp, level); -- if (!cgrp) { -- scx_bpf_error("ancestor cgroup lookup failed"); -- return NULL; -- } -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- scx_bpf_error("ancestor cgrp_ctx lookup failed"); -- bpf_cgroup_release(cgrp); -- return cgc; --} -- --static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- int level; -- -- if (!cgc->nr_active) { -- stat_inc(FCG_STAT_HWT_SKIP); -- return; -- } -- -- if (cgc->hweight_gen == hweight_gen) { -- stat_inc(FCG_STAT_HWT_CACHE); -- return; -- } -- -- stat_inc(FCG_STAT_HWT_UPDATES); -- bpf_for(level, 0, cgrp->level + 1) { -- struct fcg_cgrp_ctx *cgc; -- bool is_active; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- -- if (!level) { -- cgc->hweight = FCG_HWEIGHT_ONE; -- cgc->hweight_gen = hweight_gen; -- } else { -- struct fcg_cgrp_ctx *pcgc; -- -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- -- /* -- * We can be oppotunistic here and not grab the -- * cgv_tree_lock and deal with the occasional races. -- * However, hweight updates are already cached and -- * relatively low-frequency. Let's just do the -- * straightforward thing. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- is_active = cgc->nr_active; -- if (is_active) { -- cgc->hweight_gen = pcgc->hweight_gen; -- cgc->hweight = -- div_round_up(pcgc->hweight * cgc->weight, -- pcgc->child_weight_sum); -- } -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!is_active) { -- stat_inc(FCG_STAT_HWT_RACE); -- break; -- } -- } -- } --} -- --static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) --{ -- u64 delta, cvtime, max_budget; -- -- /* -- * A node which is on the rbtree can't be pointed to from elsewhere yet -- * and thus can't be updated and repositioned. Instead, we collect the -- * vtime deltas separately and apply it asynchronously here. -- */ -- delta = cgc->cvtime_delta; -- __sync_fetch_and_sub(&cgc->cvtime_delta, delta); -- cvtime = cgv_node->cvtime + delta; -- -- /* -- * Allow a cgroup to carry the maximum budget proportional to its -- * hweight such that a full-hweight cgroup can immediately take up half -- * of the CPUs at the most while staying at the front of the rbtree. -- */ -- max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / -- (2 * FCG_HWEIGHT_ONE); -- if (vtime_before(cvtime, cvtime_now - max_budget)) -- cvtime = cvtime_now - max_budget; -- -- cgv_node->cvtime = cvtime; --} -- --static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- u64 cgid = cgrp->kn->id; -- -- /* paired with cmpxchg in try_pick_next_cgroup() */ -- if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { -- stat_inc(FCG_STAT_ENQ_SKIP); -- return; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid); -- return; -- } -- -- /* NULL if the node is already on the rbtree */ -- cgv_node = bpf_kptr_xchg(&stash->node, NULL); -- if (!cgv_node) { -- stat_inc(FCG_STAT_ENQ_RACE); -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); --} -- --void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * If select_cpu_dfl() is recommending local enqueue, the target CPU is -- * idle. Follow it and charge the cgroup later in fcg_stopping() after -- * the fact. Use the same mechanism to deal with tasks with custom -- * affinities so that we don't have to worry about per-cgroup dq's -- * containing tasks that can't be executed from some CPUs. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) { -- /* -- * Tell fcg_stopping() that this bypassed the regular scheduling -- * path and should be force charged to the cgroup. 0 is used to -- * indicate that the task isn't bypassing, so if the current -- * runtime is 0, go back by one nanosecond. -- */ -- taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1; -- -- /* -- * The global dq is deprioritized as we don't want to let tasks -- * to boost themselves by constraining its cpumask. The -- * deprioritization is rather severe, so let's not apply that to -- * per-cpu kernel threads. This is ham-fisted. We probably wanna -- * implement per-cgroup fallback dq's instead so that we have -- * more control over when tasks with custom cpumask get issued. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || -- (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) { -- stat_inc(FCG_STAT_LOCAL); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- } else { -- stat_inc(FCG_STAT_GLOBAL); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } -- return; -- } -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- goto out_release; -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 tvtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) -- tvtime = cgc->tvtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, -- tvtime, enq_flags); -- } -- -- cgrp_enqueued(cgrp, cgc); --out_release: -- bpf_cgroup_release(cgrp); --} -- --/* -- * Walk the cgroup tree to update the active weight sums as tasks wake up and -- * sleep. The weight sums are used as the base when calculating the proportion a -- * given cgroup or task is entitled to at each level. -- */ --static void update_active_weight_sums(struct cgroup *cgrp, bool runnable) --{ -- struct fcg_cgrp_ctx *cgc; -- bool updated = false; -- int idx; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- /* -- * In most cases, a hot cgroup would have multiple threads going to -- * sleep and waking up while the whole cgroup stays active. In leaf -- * cgroups, ->nr_runnable which is updated with __sync operations gates -- * ->nr_active updates, so that we don't have to grab the cgv_tree_lock -- * repeatedly for a busy cgroup which is staying active. -- */ -- if (runnable) { -- if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_ACT); -- } else { -- if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_DEACT); -- } -- -- /* -- * If @cgrp is becoming runnable, its hweight should be refreshed after -- * it's added to the weight tree so that enqueue has the up-to-date -- * value. If @cgrp is becoming quiescent, the hweight should be -- * refreshed before it's removed from the weight tree so that the usage -- * charging which happens afterwards has access to the latest value. -- */ -- if (!runnable) -- cgrp_refresh_hweight(cgrp, cgc); -- -- /* propagate upwards */ -- bpf_for(idx, 0, cgrp->level) { -- int level = cgrp->level - idx; -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- bool propagate = false; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- if (level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- } -- -- /* -- * We need the propagation protected by a lock to synchronize -- * against weight changes. There's no reason to drop the lock at -- * each level but bpf_spin_lock() doesn't want any function -- * calls while locked. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- -- if (runnable) { -- if (!cgc->nr_active++) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum += cgc->weight; -- } -- } -- } else { -- if (!--cgc->nr_active) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum -= cgc->weight; -- } -- } -- } -- -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!propagate) -- break; -- } -- -- if (updated) -- __sync_fetch_and_add(&hweight_gen, 1); -- -- if (runnable) -- cgrp_refresh_hweight(cgrp, cgc); --} -- --void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, true); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) --{ -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- if (fifo_sched) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- /* -- * @cgc->tvtime_now always progresses forward as tasks start -- * executing. The test and update can be performed concurrently -- * from multiple CPUs and thus racy. Any error should be -- * contained and temporary. Let's just live with it. -- */ -- if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) -- cgc->tvtime_now = p->scx.dsq_vtime; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- /* scale the execution time by the inverse of the weight and charge */ -- if (!fifo_sched) -- p->scx.dsq_vtime += -- (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- if (!taskc->bypassed_at) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- __sync_fetch_and_add(&cgc->cvtime_delta, -- p->se.sum_exec_runtime - taskc->bypassed_at); -- taskc->bypassed_at = 0; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, false); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) --{ -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- if (cgrp->level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1); -- if (!pcgc) -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- if (pcgc && cgc->nr_active) -- pcgc->child_weight_sum += (s64)weight - cgc->weight; -- cgc->weight = weight; -- bpf_spin_unlock(&cgv_tree_lock); --} -- --static bool try_pick_next_cgroup(u64 *cgidp) --{ -- struct bpf_rb_node *rb_node; -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 cgid; -- -- /* pop the front cgroup and wind cvtime_now accordingly */ -- bpf_spin_lock(&cgv_tree_lock); -- -- rb_node = bpf_rbtree_first(&cgv_tree); -- if (!rb_node) { -- bpf_spin_unlock(&cgv_tree_lock); -- stat_inc(FCG_STAT_PNC_NO_CGRP); -- *cgidp = 0; -- return true; -- } -- -- rb_node = bpf_rbtree_remove(&cgv_tree, rb_node); -- bpf_spin_unlock(&cgv_tree_lock); -- -- cgv_node = container_of(rb_node, struct cgv_node, rb_node); -- cgid = cgv_node->cgid; -- -- if (vtime_before(cvtime_now, cgv_node->cvtime)) -- cvtime_now = cgv_node->cvtime; -- -- /* -- * If lookup fails, the cgroup's gone. Free and move on. See -- * fcg_cgroup_exit(). -- */ -- cgrp = bpf_cgroup_from_id(cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- if (!scx_bpf_consume(cgid)) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_EMPTY); -- goto out_stash; -- } -- -- /* -- * Successfully consumed from the cgroup. This will be our current -- * cgroup for the new slice. Refresh its hweight. -- */ -- cgrp_refresh_hweight(cgrp, cgc); -- -- bpf_cgroup_release(cgrp); -- -- /* -- * As the cgroup may have more tasks, add it back to the rbtree. Note -- * that here we charge the full slice upfront and then exact later -- * according to the actual consumption. This prevents lowpri thundering -- * herd from saturating the machine. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- -- *cgidp = cgid; -- stat_inc(FCG_STAT_PNC_NEXT); -- return true; -- --out_stash: -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- /* -- * Paired with cmpxchg in cgrp_enqueued(). If they see the following -- * transition, they'll enqueue the cgroup. If they are earlier, we'll -- * see their task in the dq below and requeue the cgroup. -- */ -- __sync_val_compare_and_swap(&cgc->queued, 1, 0); -- -- if (scx_bpf_dsq_nr_queued(cgid)) { -- bpf_spin_lock(&cgv_tree_lock); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- goto out_free; -- } -- } -- -- return false; -- --out_free: -- bpf_obj_drop(cgv_node); -- return false; --} -- --void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) --{ -- struct fcg_cpu_ctx *cpuc; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 now = bpf_ktime_get_ns(); -- -- cpuc = find_cpu_ctx(); -- if (!cpuc) -- return; -- -- if (!cpuc->cur_cgid) -- goto pick_next_cgroup; -- -- if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { -- if (scx_bpf_consume(cpuc->cur_cgid)) { -- stat_inc(FCG_STAT_CNS_KEEP); -- return; -- } -- stat_inc(FCG_STAT_CNS_EMPTY); -- } else { -- stat_inc(FCG_STAT_CNS_EXPIRE); -- } -- -- /* -- * The current cgroup is expiring. It was already charged a full slice. -- * Calculate the actual usage and accumulate the delta. -- */ -- cgrp = bpf_cgroup_from_id(cpuc->cur_cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_CNS_GONE); -- goto pick_next_cgroup; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (cgc) { -- /* -- * We want to update the vtime delta and then look for the next -- * cgroup to execute but the latter needs to be done in a loop -- * and we can't keep the lock held. Oh well... -- */ -- bpf_spin_lock(&cgv_tree_lock); -- __sync_fetch_and_add(&cgc->cvtime_delta, -- (cpuc->cur_at + cgrp_slice_ns - now) * -- FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- stat_inc(FCG_STAT_CNS_GONE); -- } -- -- bpf_cgroup_release(cgrp); -- --pick_next_cgroup: -- cpuc->cur_at = now; -- -- if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { -- cpuc->cur_cgid = 0; -- return; -- } -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_pick_next_cgroup(&cpuc->cur_cgid)) -- break; -- } --} -- --s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct fcg_task_ctx *taskc; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- taskc = bpf_task_storage_get(&task_ctx, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!taskc) -- return -ENOMEM; -- -- taskc->bypassed_at = 0; -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp, -- struct scx_cgroup_init_args *args) --{ -- struct fcg_cgrp_ctx *cgc; -- struct cgv_node *cgv_node; -- struct cgv_node_stash empty_stash = {}, *stash; -- u64 cgid = cgrp->kn->id; -- int ret; -- -- /* -- * Technically incorrect as cgroup ID is full 64bit while dq ID is -- * 63bit. Should not be a problem in practice and easy to spot in the -- * unlikely case that it breaks. -- */ -- ret = scx_bpf_create_dsq(cgid, -1); -- if (ret) -- return ret; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!cgc) { -- ret = -ENOMEM; -- goto err_destroy_dsq; -- } -- -- cgc->weight = args->weight; -- cgc->hweight = FCG_HWEIGHT_ONE; -- -- ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash, -- BPF_NOEXIST); -- if (ret) { -- if (ret != -ENOMEM) -- scx_bpf_error("unexpected stash creation error (%d)", -- ret); -- goto err_destroy_dsq; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("unexpected cgv_node stash lookup failure"); -- ret = -ENOENT; -- goto err_destroy_dsq; -- } -- -- cgv_node = bpf_obj_new(struct cgv_node); -- if (!cgv_node) { -- ret = -ENOMEM; -- goto err_del_cgv_node; -- } -- -- cgv_node->cgid = cgid; -- cgv_node->cvtime = cvtime_now; -- -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- ret = -EBUSY; -- goto err_drop; -- } -- -- return 0; -- --err_drop: -- bpf_obj_drop(cgv_node); --err_del_cgv_node: -- bpf_map_delete_elem(&cgv_node_stash, &cgid); --err_destroy_dsq: -- scx_bpf_destroy_dsq(cgid); -- return ret; --} -- --void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- -- /* -- * For now, there's no way find and remove the cgv_node if it's on the -- * cgv_tree. Let's drain them in the dispatch path as they get popped -- * off the front of the tree. -- */ -- bpf_map_delete_elem(&cgv_node_stash, &cgid); -- scx_bpf_destroy_dsq(cgid); --} -- --s32 BPF_STRUCT_OPS(fcg_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops flatcg_ops = { -- .enqueue = (void *)fcg_enqueue, -- .dispatch = (void *)fcg_dispatch, -- .runnable = (void *)fcg_runnable, -- .running = (void *)fcg_running, -- .stopping = (void *)fcg_stopping, -- .quiescent = (void *)fcg_quiescent, -- .prep_enable = (void *)fcg_prep_enable, -- .cgroup_set_weight = (void *)fcg_cgroup_set_weight, -- .cgroup_init = (void *)fcg_cgroup_init, -- .cgroup_exit = (void *)fcg_cgroup_exit, -- .init = (void *)fcg_init, -- .exit = (void *)fcg_exit, -- .flags = SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING, -- .name = "flatcg", --}; -diff --git b/tools/sched_ext/scx_example_flatcg.c a/tools/sched_ext/scx_example_flatcg.c -deleted file mode 100644 -index f9c8a5b84a70..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.c -+++ /dev/null -@@ -1,232 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2023 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2023 Tejun Heo -- * Copyright (c) 2023 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_flatcg.h" --#include "scx_example_flatcg.skel.h" -- --#ifndef FILEID_KERNFS --#define FILEID_KERNFS 0xfe --#endif -- --const char help_fmt[] = --"A flattened cgroup hierarchy sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -i INTERVAL Report interval\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --static float read_cpu_util(__u64 *last_sum, __u64 *last_idle) --{ -- FILE *fp; -- char buf[4096]; -- char *line, *cur = NULL, *tok; -- __u64 sum = 0, idle = 0; -- __u64 delta_sum, delta_idle; -- int idx; -- -- fp = fopen("/proc/stat", "r"); -- if (!fp) { -- perror("fopen(\"/proc/stat\")"); -- return 0.0; -- } -- -- if (!fgets(buf, sizeof(buf), fp)) { -- perror("fgets(\"/proc/stat\")"); -- fclose(fp); -- return 0.0; -- } -- fclose(fp); -- -- line = buf; -- for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) { -- char *endp = NULL; -- __u64 v; -- -- if (idx == 0) { -- line = NULL; -- continue; -- } -- v = strtoull(tok, &endp, 0); -- if (!endp || *endp != '\0') { -- fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n", -- idx, tok); -- continue; -- } -- sum += v; -- if (idx == 4) -- idle = v; -- } -- -- delta_sum = sum - *last_sum; -- delta_idle = idle - *last_idle; -- *last_sum = sum; -- *last_idle = idle; -- -- return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0; --} -- --static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats) --{ -- __u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus]; -- __u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); -- -- for (idx = 0; idx < FCG_NR_STATS; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_flatcg *skel; -- struct bpf_link *link; -- struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 }; -- bool dump_cgrps = false; -- __u64 last_cpu_sum = 0, last_cpu_idle = 0; -- __u64 last_stats[FCG_NR_STATS] = {}; -- unsigned long seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_flatcg__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open: %s\n", strerror(errno)); -- return 1; -- } -- -- skel->rodata->nr_cpus = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) { -- double v; -- -- switch (opt) { -- case 's': -- v = strtod(optarg, NULL); -- skel->rodata->cgrp_slice_ns = v * 1000; -- break; -- case 'i': -- v = strtod(optarg, NULL); -- intv_ts.tv_sec = v; -- intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000; -- break; -- case 'd': -- dump_cgrps = true; -- break; -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- case 'h': -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d", -- (double)skel->rodata->cgrp_slice_ns / 1000000.0, -- (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0, -- dump_cgrps); -- -- if (scx_example_flatcg__load(skel)) { -- fprintf(stderr, "Failed to load: %s\n", strerror(errno)); -- return 1; -- } -- -- link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops); -- if (!link) { -- fprintf(stderr, "Failed to attach_struct_ops: %s\n", -- strerror(errno)); -- return 1; -- } -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- __u64 acc_stats[FCG_NR_STATS]; -- __u64 stats[FCG_NR_STATS]; -- float cpu_util; -- int i; -- -- cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle); -- -- fcg_read_stats(skel, acc_stats); -- for (i = 0; i < FCG_NR_STATS; i++) -- stats[i] = acc_stats[i] - last_stats[i]; -- -- memcpy(last_stats, acc_stats, sizeof(acc_stats)); -- -- printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n", -- seq++, cpu_util * 100.0, skel->data->hweight_gen); -- printf(" act:%6llu deact:%6llu local:%6llu global:%6llu\n", -- stats[FCG_STAT_ACT], -- stats[FCG_STAT_DEACT], -- stats[FCG_STAT_LOCAL], -- stats[FCG_STAT_GLOBAL]); -- printf("HWT skip:%6llu race:%6llu cache:%6llu update:%6llu\n", -- stats[FCG_STAT_HWT_SKIP], -- stats[FCG_STAT_HWT_RACE], -- stats[FCG_STAT_HWT_CACHE], -- stats[FCG_STAT_HWT_UPDATES]); -- printf("ENQ skip:%6llu race:%6llu\n", -- stats[FCG_STAT_ENQ_SKIP], -- stats[FCG_STAT_ENQ_RACE]); -- printf("CNS keep:%6llu expire:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_CNS_KEEP], -- stats[FCG_STAT_CNS_EXPIRE], -- stats[FCG_STAT_CNS_EMPTY], -- stats[FCG_STAT_CNS_GONE]); -- printf("PNC nocgrp:%6llu next:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_PNC_NO_CGRP], -- stats[FCG_STAT_PNC_NEXT], -- stats[FCG_STAT_PNC_EMPTY], -- stats[FCG_STAT_PNC_GONE]); -- printf("BAD remove:%6llu\n", -- acc_stats[FCG_STAT_BAD_REMOVAL]); -- -- nanosleep(&intv_ts, NULL); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_flatcg__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_flatcg.h a/tools/sched_ext/scx_example_flatcg.h -deleted file mode 100644 -index 490758ed41f0..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.h -+++ /dev/null -@@ -1,49 +0,0 @@ --#ifndef __SCX_EXAMPLE_FLATCG_H --#define __SCX_EXAMPLE_FLATCG_H -- --enum { -- FCG_HWEIGHT_ONE = 1LLU << 16, --}; -- --enum fcg_stat_idx { -- FCG_STAT_ACT, -- FCG_STAT_DEACT, -- FCG_STAT_LOCAL, -- FCG_STAT_GLOBAL, -- -- FCG_STAT_HWT_UPDATES, -- FCG_STAT_HWT_CACHE, -- FCG_STAT_HWT_SKIP, -- FCG_STAT_HWT_RACE, -- -- FCG_STAT_ENQ_SKIP, -- FCG_STAT_ENQ_RACE, -- -- FCG_STAT_CNS_KEEP, -- FCG_STAT_CNS_EXPIRE, -- FCG_STAT_CNS_EMPTY, -- FCG_STAT_CNS_GONE, -- -- FCG_STAT_PNC_NO_CGRP, -- FCG_STAT_PNC_NEXT, -- FCG_STAT_PNC_EMPTY, -- FCG_STAT_PNC_GONE, -- -- FCG_STAT_BAD_REMOVAL, -- -- FCG_NR_STATS, --}; -- --struct fcg_cgrp_ctx { -- u32 nr_active; -- u32 nr_runnable; -- u32 queued; -- u32 weight; -- u32 hweight; -- u64 child_weight_sum; -- u64 hweight_gen; -- s64 cvtime_delta; -- u64 tvtime_now; --}; -- --#endif /* __SCX_EXAMPLE_FLATCG_H */ -diff --git b/tools/sched_ext/scx_example_pair.bpf.c a/tools/sched_ext/scx_example_pair.bpf.c -deleted file mode 100644 -index 279efe58b777..000000000000 ---- b/tools/sched_ext/scx_example_pair.bpf.c -+++ /dev/null -@@ -1,627 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext core-scheduler which always makes every sibling CPU pair -- * execute from the same CPU cgroup. -- * -- * This scheduler is a minimal implementation and would need some form of -- * priority handling both inside each cgroup and across the cgroups to be -- * practically useful. -- * -- * Each CPU in the system is paired with exactly one other CPU, according to a -- * "stride" value that can be specified when the BPF scheduler program is first -- * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee -- * that they will only ever schedule tasks that belong to the same CPU cgroup. -- * -- * Scheduler Initialization -- * ------------------------ -- * -- * The scheduler BPF program is first initialized from user space, before it is -- * enabled. During this initialization process, each CPU on the system is -- * assigned several values that are constant throughout its runtime: -- * -- * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling -- * decisions. Paired CPUs always schedule tasks from the same -- * CPU cgroup, and synchronize with each other to guarantee -- * that this constraint is not violated. -- * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access -- * a struct pair_ctx object that is shared between the pair. -- * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the -- * pair. Each struct pair_ctx has an active_mask field, -- * which is a bitmap used to indicate whether each core -- * in the pair currently has an actively running task. -- * This index specifies which entry in the bitmap corresponds -- * to each CPU in the pair. -- * -- * During this initialization, the CPUs are paired according to a "stride" that -- * may be specified when invoking the user space program that initializes and -- * loads the scheduler. By default, the stride is 1/2 the total number of CPUs. -- * -- * Tasks and cgroups -- * ----------------- -- * -- * Every cgroup in the system is registered with the scheduler using the -- * pair_cgroup_init() callback, and every task in the system is associated with -- * exactly one cgroup. At a high level, the idea with the pair scheduler is to -- * always schedule tasks from the same cgroup within a given CPU pair. When a -- * task is enqueued (i.e. passed to the pair_enqueue() callback function), its -- * cgroup ID is read from its task struct, and then a corresponding queue map -- * is used to FIFO-enqueue the task for that cgroup. -- * -- * If you look through the implementation of the scheduler, you'll notice that -- * there is quite a bit of complexity involved with looking up the per-cgroup -- * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash -- * BPF hash map that is used to map a cgroup ID to a globally unique ID that's -- * allocated in the BPF program. This is done because we use separate maps to -- * store the FIFO queue of tasks, and the length of that map, per cgroup. This -- * complexity is only present because of current deficiencies in BPF that will -- * soon be addressed. The main point to keep in mind is that newly enqueued -- * tasks are added to their cgroup's FIFO queue. -- * -- * Dispatching tasks -- * ----------------- -- * -- * This section will describe how enqueued tasks are dispatched and scheduled. -- * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is -- * as follows: -- * -- * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is -- * the structure that's used to synchronize amongst the two pair CPUs in their -- * scheduling decisions. After any of the following events have occurred: -- * -- * - The cgroup's slice run has expired, or -- * - The cgroup becomes empty, or -- * - Either CPU in the pair is preempted by a higher priority scheduling class -- * -- * The cgroup transitions to the draining state and stops executing new tasks -- * from the cgroup. -- * -- * 2. If the pair is still executing a task, mark the pair_ctx as draining, and -- * wait for the pair CPU to be preempted. -- * -- * 3. Otherwise, if the pair CPU is not running a task, we can move onto -- * scheduling new tasks. Pop the next cgroup id from the top_q queue. -- * -- * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it. -- * -- * Note again that this scheduling behavior is simple, but the implementation -- * is complex mostly because this it hits several BPF shortcomings and has to -- * work around in often awkward ways. Most of the shortcomings are expected to -- * be resolved in the near future which should allow greatly simplifying this -- * scheduler. -- * -- * Dealing with preemption -- * ----------------------- -- * -- * SCX is the lowest priority sched_class, and could be preempted by them at -- * any time. To address this, the scheduler implements pair_cpu_release() and -- * pair_cpu_acquire() callbacks which are invoked by the core scheduler when -- * the scheduler loses and gains control of the CPU respectively. -- * -- * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and -- * then invoke: -- * -- * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- * -- * This preempts the pair CPU, and waits until it has re-entered the scheduler -- * before returning. This is necessary to ensure that the higher priority -- * sched_class that preempted our scheduler does not schedule a task -- * concurrently with our pair CPU. -- * -- * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption -- * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable -- * pair scheduling. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include "scx_example_pair.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; -- --/* !0 for veristat, set during init */ --const volatile u32 nr_cpu_ids = 64; -- --/* a pair of CPUs stay on a cgroup for this duration */ --const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL; -- --/* cpu ID -> pair cpu ID */ --const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 }; -- --/* cpu ID -> pair_id */ --const volatile u32 pair_id[MAX_CPUS]; -- --/* CPU ID -> CPU # in the pair (0 or 1) */ --const volatile u32 in_pair_idx[MAX_CPUS]; -- --struct pair_ctx { -- struct bpf_spin_lock lock; -- -- /* the cgroup the pair is currently executing */ -- u64 cgid; -- -- /* the pair started executing the current cgroup at */ -- u64 started_at; -- -- /* whether the current cgroup is draining */ -- bool draining; -- -- /* the CPUs that are currently active on the cgroup */ -- u32 active_mask; -- -- /* -- * the CPUs that are currently preempted and running tasks in a -- * different scheduler. -- */ -- u32 preempted_mask; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, MAX_CPUS / 2); -- __type(key, u32); -- __type(value, struct pair_ctx); --} pair_ctx SEC(".maps"); -- --/* queue of cgrp_q's possibly with tasks on them */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- /* -- * Because it's difficult to build strong synchronization encompassing -- * multiple non-trivial operations in BPF, this queue is managed in an -- * opportunistic way so that we guarantee that a cgroup w/ active tasks -- * is always on it but possibly multiple times. Once we have more robust -- * synchronization constructs and e.g. linked list, we should be able to -- * do this in a prettier way but for now just size it big enough. -- */ -- __uint(max_entries, 4 * MAX_CGRPS); -- __type(value, u64); --} top_q SEC(".maps"); -- --/* per-cgroup q which FIFOs the tasks from the cgroup */ --struct cgrp_q { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, MAX_QUEUED); -- __type(value, u32); --}; -- --/* -- * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local -- * storage; however, a cgroup local storage can only be accessed from the BPF -- * progs attached to the cgroup. For now, work around by allocating array of -- * cgrp_q's and then allocating per-cgroup indices. -- * -- * Another caveat: It's difficult to populate a large array of maps statically -- * or from BPF. Initialize it from userland. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, MAX_CGRPS); -- __type(key, s32); -- __array(values, struct cgrp_q); --} cgrp_q_arr SEC(".maps"); -- --static u64 cgrp_q_len[MAX_CGRPS]; -- --/* -- * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be -- * useful to have as a map type. -- */ --static u32 cgrp_q_idx_cursor; --static u64 cgrp_q_idx_busy[MAX_CGRPS]; -- --/* -- * All added up, the following is what we do: -- * -- * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking -- * for a free ID. If not found, fail cgroup creation with -EBUSY. -- * -- * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following -- * cgrp_q_idx_hash. -- * -- * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from -- * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr. -- * -- * This is sadly complicated for something pretty simple. Hopefully, we should -- * be able to simplify in the future. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, MAX_CGRPS); -- __uint(key_size, sizeof(u64)); /* cgrp ID */ -- __uint(value_size, sizeof(s32)); /* cgrp_q idx */ --} cgrp_q_idx_hash SEC(".maps"); -- --/* statistics */ --u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions; --u64 nr_exps, nr_exp_waits, nr_exp_empty; --u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty; -- --struct user_exit_info uei; -- --static bool time_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- struct cgrp_q *cgq; -- s32 pid = p->pid; -- u64 cgid; -- u32 *q_idx; -- u64 *cgq_len; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- cgrp = scx_bpf_task_cgroup(p); -- cgid = cgrp->kn->id; -- bpf_cgroup_release(cgrp); -- -- /* find the cgroup's q and push @p into it */ -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!q_idx) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return; -- } -- -- cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx); -- if (!cgq) { -- scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]", -- cgid, *q_idx); -- return; -- } -- -- if (bpf_map_push_elem(cgq, &pid, 0)) { -- scx_bpf_error("cgroup[%llu] queue overflow", cgid); -- return; -- } -- -- /* bump q len, if going 0 -> 1, queue cgroup into the top_q */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len) { -- scx_bpf_error("MEMBER_VTPR malfunction"); -- return; -- } -- -- if (!__sync_fetch_and_add(cgq_len, 1) && -- bpf_map_push_elem(&top_q, &cgid, 0)) { -- scx_bpf_error("top_q overflow"); -- return; -- } --} -- --static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) --{ -- u32 *vptr; -- -- vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *pairc = bpf_map_lookup_elem(&pair_ctx, vptr); -- if (!(*pairc)) -- return -EINVAL; -- -- vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *mask = 1U << *vptr; -- -- return 0; --} -- --static int try_dispatch(s32 cpu) --{ -- struct pair_ctx *pairc; -- struct bpf_map *cgq_map; -- struct task_struct *p; -- u64 now = bpf_ktime_get_ns(); -- bool kick_pair = false; -- bool expired, pair_preempted; -- u32 *vptr, in_pair_mask; -- s32 pid, q_idx; -- u64 cgid; -- int ret; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) { -- scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]", -- cpu); -- return -ENOENT; -- } -- -- bpf_spin_lock(&pairc->lock); -- pairc->active_mask &= ~in_pair_mask; -- -- expired = time_before(pairc->started_at + pair_batch_dur_ns, now); -- if (expired || pairc->draining) { -- u64 new_cgid = 0; -- -- __sync_fetch_and_add(&nr_exps, 1); -- -- /* -- * We're done with the current cgid. An obvious optimization -- * would be not draining if the next cgroup is the current one. -- * For now, be dumb and always expire. -- */ -- pairc->draining = true; -- -- pair_preempted = pairc->preempted_mask; -- if (pairc->active_mask || pair_preempted) { -- /* -- * The other CPU is still active, or is no longer under -- * our control due to e.g. being preempted by a higher -- * priority sched_class. We want to wait until this -- * cgroup expires, or until control of our pair CPU has -- * been returned to us. -- * -- * If the pair controls its CPU, and the time already -- * expired, kick. When the other CPU arrives at -- * dispatch and clears its active mask, it'll push the -- * pair to the next cgroup and kick this CPU. -- */ -- __sync_fetch_and_add(&nr_exp_waits, 1); -- bpf_spin_unlock(&pairc->lock); -- if (expired && !pair_preempted) -- kick_pair = true; -- goto out_maybe_kick; -- } -- -- bpf_spin_unlock(&pairc->lock); -- -- /* -- * Pick the next cgroup. It'd be easier / cleaner to not drop -- * pairc->lock and use stronger synchronization here especially -- * given that we'll be switching cgroups significantly less -- * frequently than tasks. Unfortunately, bpf_spin_lock can't -- * really protect anything non-trivial. Let's do opportunistic -- * operations instead. -- */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u32 *q_idx; -- u64 *cgq_len; -- -- if (bpf_map_pop_elem(&top_q, &new_cgid)) { -- /* no active cgroup, go idle */ -- __sync_fetch_and_add(&nr_exp_empty, 1); -- return 0; -- } -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid); -- if (!q_idx) -- continue; -- -- /* -- * This is the only place where empty cgroups are taken -- * off the top_q. -- */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len || !*cgq_len) -- continue; -- -- /* -- * If it has any tasks, requeue as we may race and not -- * execute it. -- */ -- bpf_map_push_elem(&top_q, &new_cgid, 0); -- break; -- } -- -- bpf_spin_lock(&pairc->lock); -- -- /* -- * The other CPU may already have started on a new cgroup while -- * we dropped the lock. Make sure that we're still draining and -- * start on the new cgroup. -- */ -- if (pairc->draining && !pairc->active_mask) { -- __sync_fetch_and_add(&nr_cgrp_next, 1); -- pairc->cgid = new_cgid; -- pairc->started_at = now; -- pairc->draining = false; -- kick_pair = true; -- } else { -- __sync_fetch_and_add(&nr_cgrp_coll, 1); -- } -- } -- -- cgid = pairc->cgid; -- pairc->active_mask |= in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- -- /* again, it'd be better to do all these with the lock held, oh well */ -- vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!vptr) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return -ENOENT; -- } -- q_idx = *vptr; -- -- /* claim one task from cgrp_q w/ q_idx */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u64 *cgq_len, len; -- -- cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]); -- if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) { -- /* the cgroup must be empty, expire and repeat */ -- __sync_fetch_and_add(&nr_cgrp_empty, 1); -- bpf_spin_lock(&pairc->lock); -- pairc->draining = true; -- pairc->active_mask &= ~in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- return -EAGAIN; -- } -- -- if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len) -- continue; -- -- break; -- } -- -- cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx); -- if (!cgq_map) { -- scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- if (bpf_map_pop_elem(cgq_map, &pid)) { -- scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- p = bpf_task_from_pid(pid); -- if (p) { -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } else { -- /* we don't handle dequeues, retry on lost tasks */ -- __sync_fetch_and_add(&nr_missing, 1); -- return -EAGAIN; -- } -- --out_maybe_kick: -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } -- return 0; --} -- --void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) --{ -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_dispatch(cpu) != -EAGAIN) -- break; -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask &= ~in_pair_mask; -- /* Kick the pair CPU, unless it was also preempted. */ -- kick_pair = !pairc->preempted_mask; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask |= in_pair_mask; -- pairc->active_mask &= ~in_pair_mask; -- /* Kick the pair CPU if it's still running. */ -- kick_pair = pairc->active_mask; -- pairc->draining = true; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- } -- } -- __sync_fetch_and_add(&nr_preemptions, 1); --} -- --s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 i, q_idx; -- -- bpf_for(i, 0, MAX_CGRPS) { -- q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS; -- if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1)) -- break; -- } -- if (i == MAX_CGRPS) -- return -EBUSY; -- -- if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]); -- if (busy) -- *busy = 0; -- return -EBUSY; -- } -- -- return 0; --} -- --void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 *q_idx; -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (q_idx) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]); -- if (busy) -- *busy = 0; -- bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid); -- } --} -- --s32 BPF_STRUCT_OPS(pair_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops pair_ops = { -- .enqueue = (void *)pair_enqueue, -- .dispatch = (void *)pair_dispatch, -- .cpu_acquire = (void *)pair_cpu_acquire, -- .cpu_release = (void *)pair_cpu_release, -- .cgroup_init = (void *)pair_cgroup_init, -- .cgroup_exit = (void *)pair_cgroup_exit, -- .init = (void *)pair_init, -- .exit = (void *)pair_exit, -- .name = "pair", --}; -diff --git b/tools/sched_ext/scx_example_pair.c a/tools/sched_ext/scx_example_pair.c -deleted file mode 100644 -index 18e032bbc173..000000000000 ---- b/tools/sched_ext/scx_example_pair.c -+++ /dev/null -@@ -1,143 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_pair.h" --#include "scx_example_pair.skel.h" -- --const char help_fmt[] = --"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n" --"execute from the same CPU cgroup.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-S STRIDE] [-p]\n" --"\n" --" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_pair *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 stride, i, opt, outer_fd; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_pair__open(); -- assert(skel); -- -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- /* pair up the earlier half to the latter by default, override with -s */ -- stride = skel->rodata->nr_cpu_ids / 2; -- -- while ((opt = getopt(argc, argv, "S:ph")) != -1) { -- switch (opt) { -- case 'S': -- stride = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- for (i = 0; i < skel->rodata->nr_cpu_ids; i++) { -- if (skel->rodata->pair_cpu[i] < 0) { -- skel->rodata->pair_cpu[i] = i + stride; -- skel->rodata->pair_cpu[i + stride] = i; -- skel->rodata->pair_id[i] = i; -- skel->rodata->pair_id[i + stride] = i; -- skel->rodata->in_pair_idx[i] = 0; -- skel->rodata->in_pair_idx[i + stride] = 1; -- } -- } -- -- assert(!scx_example_pair__load(skel)); -- -- /* -- * Populate the cgrp_q_arr map which is an array containing per-cgroup -- * queues. It'd probably be better to do this from BPF but there are too -- * many to initialize statically and there's no way to dynamically -- * populate from BPF. -- */ -- outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr); -- assert(outer_fd >= 0); -- -- printf("Initializing"); -- for (i = 0; i < MAX_CGRPS; i++) { -- s32 inner_fd; -- -- if (exit_req) -- break; -- -- inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, -- sizeof(u32), MAX_QUEUED, NULL); -- assert(inner_fd >= 0); -- assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY)); -- close(inner_fd); -- -- if (!(i % 10)) -- printf("."); -- fflush(stdout); -- } -- printf("\n"); -- -- /* -- * Fully initialized, attach and run. -- */ -- link = bpf_map__attach_struct_ops(skel->maps.pair_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf(" total:%10lu dispatch:%10lu missing:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_dispatched, -- skel->bss->nr_missing); -- printf(" kicks:%10lu preemptions:%7lu\n", -- skel->bss->nr_kicks, -- skel->bss->nr_preemptions); -- printf(" exp:%10lu exp_wait:%10lu exp_empty:%10lu\n", -- skel->bss->nr_exps, -- skel->bss->nr_exp_waits, -- skel->bss->nr_exp_empty); -- printf("cgnext:%10lu cgcoll:%10lu cgempty:%10lu\n", -- skel->bss->nr_cgrp_next, -- skel->bss->nr_cgrp_coll, -- skel->bss->nr_cgrp_empty); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_pair__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_pair.h a/tools/sched_ext/scx_example_pair.h -deleted file mode 100644 -index f60b824272f7..000000000000 ---- b/tools/sched_ext/scx_example_pair.h -+++ /dev/null -@@ -1,10 +0,0 @@ --#ifndef __SCX_EXAMPLE_PAIR_H --#define __SCX_EXAMPLE_PAIR_H -- --enum { -- MAX_CPUS = 4096, -- MAX_QUEUED = 4096, -- MAX_CGRPS = 4096, --}; -- --#endif /* __SCX_EXAMPLE_PAIR_H */ -diff --git b/tools/sched_ext/scx_example_qmap.bpf.c a/tools/sched_ext/scx_example_qmap.bpf.c -deleted file mode 100644 -index 579ab21ae403..000000000000 ---- b/tools/sched_ext/scx_example_qmap.bpf.c -+++ /dev/null -@@ -1,401 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple five-level FIFO queue scheduler. -- * -- * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets -- * assigned to one depending on its compound weight. Each CPU round robins -- * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from -- * queue0, 2 from queue1, 4 from queue2 and so on. -- * -- * This scheduler demonstrates: -- * -- * - BPF-side queueing using PIDs. -- * - Sleepable per-task storage allocation using ops.prep_enable(). -- * - Using ops.cpu_release() to handle a higher priority scheduling class taking -- * the CPU away. -- * - Core-sched support. -- * -- * This scheduler is primarily for demonstration and testing of sched_ext -- * features and unlikely to be useful for actual workloads. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include -- --char _license[] SEC("license") = "GPL"; -- --const volatile u64 slice_ns = SCX_SLICE_DFL; --const volatile bool switch_partial; --const volatile u32 stall_user_nth; --const volatile u32 stall_kernel_nth; --const volatile u32 dsp_inf_loop_after; --const volatile s32 disallow_tgid; -- --u32 test_error_cnt; -- --struct user_exit_info uei; -- --struct qmap { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, u32); --} queue0 SEC(".maps"), -- queue1 SEC(".maps"), -- queue2 SEC(".maps"), -- queue3 SEC(".maps"), -- queue4 SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, 5); -- __type(key, int); -- __array(values, struct qmap); --} queue_arr SEC(".maps") = { -- .values = { -- [0] = &queue0, -- [1] = &queue1, -- [2] = &queue2, -- [3] = &queue3, -- [4] = &queue4, -- }, --}; -- --/* -- * Per-queue sequence numbers to implement core-sched ordering. -- * -- * Tail seq is assigned to each queued task and incremented. Head seq tracks the -- * sequence number of the latest dispatched task. The distance between the a -- * task's seq and the associated queue's head seq is called the queue distance -- * and used when comparing two tasks for ordering. See qmap_core_sched_before(). -- */ --static u64 core_sched_head_seqs[5]; --static u64 core_sched_tail_seqs[5]; -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local_dsq */ -- u64 core_sched_seq; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --/* Per-cpu dispatch index and remaining count */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(max_entries, 2); -- __type(key, u32); -- __type(value, u64); --} dispatch_idx_cnt SEC(".maps"); -- --/* Statistics */ --unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued; --unsigned long nr_core_sched_execed; -- --s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- struct task_ctx *tctx; -- s32 cpu; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- return cpu; -- -- return prev_cpu; --} -- --static int weight_to_idx(u32 weight) --{ -- /* Coarsely map the compound weight to a FIFO. */ -- if (weight <= 25) -- return 0; -- else if (weight <= 50) -- return 1; -- else if (weight < 200) -- return 2; -- else if (weight < 400) -- return 3; -- else -- return 4; --} -- --void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) --{ -- static u32 user_cnt, kernel_cnt; -- struct task_ctx *tctx; -- u32 pid = p->pid; -- int idx = weight_to_idx(p->scx.weight); -- void *ring; -- -- if (p->flags & PF_KTHREAD) { -- if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth)) -- return; -- } else { -- if (stall_user_nth && !(++user_cnt % stall_user_nth)) -- return; -- } -- -- if (test_error_cnt && !--test_error_cnt) -- scx_bpf_error("test triggering error"); -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * All enqueued tasks must have their core_sched_seq updated for correct -- * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in -- * qmap_ops.flags. -- */ -- tctx->core_sched_seq = core_sched_tail_seqs[idx]++; -- -- /* -- * If qmap_select_cpu() is telling us to or this is the last runnable -- * task on the CPU, enqueue locally. -- */ -- if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) { -- tctx->force_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); -- return; -- } -- -- /* -- * If the task was re-enqueued due to the CPU being preempted by a -- * higher priority scheduling class, just re-enqueue the task directly -- * on the global DSQ. As we want another CPU to pick it up, find and -- * kick an idle CPU. -- */ -- if (enq_flags & SCX_ENQ_REENQ) { -- s32 cpu; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags); -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- return; -- } -- -- ring = bpf_map_lookup_elem(&queue_arr, &idx); -- if (!ring) { -- scx_bpf_error("failed to find ring %d", idx); -- return; -- } -- -- /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */ -- if (bpf_map_push_elem(ring, &pid, 0)) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_enqueued, 1); --} -- --/* -- * The BPF queue map doesn't support removal and sched_ext can handle spurious -- * dispatches. qmap_dequeue() is only used to collect statistics. -- */ --void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags) --{ -- __sync_fetch_and_add(&nr_dequeued, 1); -- if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) -- __sync_fetch_and_add(&nr_core_sched_execed, 1); --} -- --static void update_core_sched_head_seq(struct task_struct *p) --{ -- struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- int idx = weight_to_idx(p->scx.weight); -- -- if (tctx) -- core_sched_head_seqs[idx] = tctx->core_sched_seq; -- else -- scx_bpf_error("task_ctx lookup failed"); --} -- --void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 zero = 0, one = 1; -- u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero); -- u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one); -- void *fifo; -- s32 pid; -- int i; -- -- if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) { -- struct task_struct *p; -- -- /* -- * PID 2 should be kthreadd which should mostly be idle and off -- * the scheduler. Let's keep dispatching it to force the kernel -- * to call this function over and over again. -- */ -- p = bpf_task_from_pid(2); -- if (p) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- if (!idx || !cnt) { -- scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt); -- return; -- } -- -- for (i = 0; i < 5; i++) { -- /* Advance the dispatch cursor and pick the fifo. */ -- if (!*cnt) { -- *idx = (*idx + 1) % 5; -- *cnt = 1 << *idx; -- } -- (*cnt)--; -- -- fifo = bpf_map_lookup_elem(&queue_arr, idx); -- if (!fifo) { -- scx_bpf_error("failed to find ring %llu", *idx); -- return; -- } -- -- /* Dispatch or advance. */ -- if (!bpf_map_pop_elem(fifo, &pid)) { -- struct task_struct *p; -- -- p = bpf_task_from_pid(pid); -- if (p) { -- update_core_sched_head_seq(p); -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- *cnt = 0; -- } --} -- --/* -- * The distance from the head of the queue scaled by the weight of the queue. -- * The lower the number, the older the task and the higher the priority. -- */ --static s64 task_qdist(struct task_struct *p) --{ -- int idx = weight_to_idx(p->scx.weight); -- struct task_ctx *tctx; -- s64 qdist; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return 0; -- } -- -- qdist = tctx->core_sched_seq - core_sched_head_seqs[idx]; -- -- /* -- * As queue index increments, the priority doubles. The queue w/ index 3 -- * is dispatched twice more frequently than 2. Reflect the difference by -- * scaling qdists accordingly. Note that the shift amount needs to be -- * flipped depending on the sign to avoid flipping priority direction. -- */ -- if (qdist >= 0) -- return qdist << (4 - idx); -- else -- return qdist << idx; --} -- --/* -- * This is called to determine the task ordering when core-sched is picking -- * tasks to execute on SMT siblings and should encode about the same ordering as -- * the regular scheduling path. Use the priority-scaled distances from the head -- * of the queues to compare the two tasks which should be consistent with the -- * dispatch path behavior. -- */ --bool BPF_STRUCT_OPS(qmap_core_sched_before, -- struct task_struct *a, struct task_struct *b) --{ -- return task_qdist(a) > task_qdist(b); --} -- --void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- u32 cnt; -- -- /* -- * Called when @cpu is taken by a higher priority scheduling class. This -- * makes @cpu no longer available for executing sched_ext tasks. As we -- * don't want the tasks in @cpu's local dsq to sit there until @cpu -- * becomes available again, re-enqueue them into the global dsq. See -- * %SCX_ENQ_REENQ handling in qmap_enqueue(). -- */ -- cnt = scx_bpf_reenqueue_local(); -- if (cnt) -- __sync_fetch_and_add(&nr_reenqueued, cnt); --} -- --s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (p->tgid == disallow_tgid) -- p->scx.disallow = true; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(qmap_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops qmap_ops = { -- .select_cpu = (void *)qmap_select_cpu, -- .enqueue = (void *)qmap_enqueue, -- .dequeue = (void *)qmap_dequeue, -- .dispatch = (void *)qmap_dispatch, -- .core_sched_before = (void *)qmap_core_sched_before, -- .cpu_release = (void *)qmap_cpu_release, -- .prep_enable = (void *)qmap_prep_enable, -- .init = (void *)qmap_init, -- .exit = (void *)qmap_exit, -- .flags = SCX_OPS_ENQ_LAST, -- .timeout_ms = 5000U, -- .name = "qmap", --}; -diff --git b/tools/sched_ext/scx_example_qmap.c a/tools/sched_ext/scx_example_qmap.c -deleted file mode 100644 -index ccb4814ee61b..000000000000 ---- b/tools/sched_ext/scx_example_qmap.c -+++ /dev/null -@@ -1,107 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_qmap.skel.h" -- --const char help_fmt[] = --"A simple five-level FIFO queue sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" --" -t COUNT Stall every COUNT'th user thread\n" --" -T COUNT Stall every COUNT'th kernel thread\n" --" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n" --" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_qmap *skel; -- struct bpf_link *link; -- int opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_qmap__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) { -- switch (opt) { -- case 's': -- skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; -- break; -- case 'e': -- skel->bss->test_error_cnt = strtoul(optarg, NULL, 0); -- break; -- case 't': -- skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0); -- break; -- case 'T': -- skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0); -- break; -- case 'l': -- skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0); -- break; -- case 'd': -- skel->rodata->disallow_tgid = strtol(optarg, NULL, 0); -- if (skel->rodata->disallow_tgid < 0) -- skel->rodata->disallow_tgid = getpid(); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_qmap__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.qmap_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- long nr_enqueued = skel->bss->nr_enqueued; -- long nr_dispatched = skel->bss->nr_dispatched; -- -- printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n", -- nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, -- skel->bss->nr_reenqueued, skel->bss->nr_dequeued, -- skel->bss->nr_core_sched_execed); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_qmap__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_simple.bpf.c a/tools/sched_ext/scx_example_simple.bpf.c -deleted file mode 100644 -index 4bccca3e2047..000000000000 ---- b/tools/sched_ext/scx_example_simple.bpf.c -+++ /dev/null -@@ -1,128 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple scheduler. -- * -- * By default, it operates as a simple global weighted vtime scheduler and can -- * be switched to FIFO scheduling. It also demonstrates the following niceties. -- * -- * - Statistics tracking how many tasks are queued to local and global dsq's. -- * - Termination notification for userspace. -- * -- * While very simple, this scheduler should work reasonably well on CPUs with a -- * uniform L3 cache topology. While preemption is not implemented, the fact that -- * the scheduling queue is shared across all CPUs means that whatever is at the -- * front of the queue is likely to be executed fairly quickly given enough -- * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads -- * but comes with the usual problems with FIFO scheduling where saturating -- * threads can easily drown out interactive ones. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --static u64 vtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, 2); /* [local, global] */ --} stats SEC(".maps"); -- --static void stat_inc(u32 idx) --{ -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); -- if (cnt_p) -- (*cnt_p)++; --} -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) --{ -- /* -- * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that -- * running @p on its CPU directly shouldn't affect fairness. Just queue -- * it on the local FIFO. -- */ -- if (enq_flags & SCX_ENQ_LOCAL) { -- stat_inc(0); /* count local queueing */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- return; -- } -- -- stat_inc(1); /* count global queueing */ -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) -- vtime = vtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --void BPF_STRUCT_OPS(simple_running, struct task_struct *p) --{ -- if (fifo_sched) -- return; -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(vtime_now, p->scx.dsq_vtime)) -- vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --s32 BPF_STRUCT_OPS(simple_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .running = (void *)simple_running, -- .stopping = (void *)simple_stopping, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", --}; -diff --git b/tools/sched_ext/scx_example_simple.c a/tools/sched_ext/scx_example_simple.c -deleted file mode 100644 -index 486b401f7c95..000000000000 ---- b/tools/sched_ext/scx_example_simple.c -+++ /dev/null -@@ -1,101 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_simple.skel.h" -- --const char help_fmt[] = --"A simple sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-f] [-p]\n" --"\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int simple) --{ -- exit_req = 1; --} -- --static void read_stats(struct scx_example_simple *skel, u64 *stats) --{ -- int nr_cpus = libbpf_num_possible_cpus(); -- u64 cnts[2][nr_cpus]; -- u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * 2); -- -- for (idx = 0; idx < 2; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_simple *skel; -- struct bpf_link *link; -- u32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_simple__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "fph")) != -1) { -- switch (opt) { -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_simple__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.simple_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- u64 stats[2]; -- -- read_stats(skel, stats); -- printf("local=%lu global=%lu\n", stats[0], stats[1]); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_simple__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_userland.bpf.c a/tools/sched_ext/scx_example_userland.bpf.c -deleted file mode 100644 -index a089bc6bbe86..000000000000 ---- b/tools/sched_ext/scx_example_userland.bpf.c -+++ /dev/null -@@ -1,269 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A minimal userland scheduler. -- * -- * In terms of scheduling, this provides two different types of behaviors: -- * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity. -- * All such tasks are direct-dispatched from the kernel, and are never -- * enqueued in user space. -- * 2. A primitive vruntime scheduler that is implemented in user space, for all -- * other tasks. -- * -- * Some parts of this example user space scheduler could be implemented more -- * efficiently using more complex and sophisticated data structures. For -- * example, rather than using BPF_MAP_TYPE_QUEUE's, -- * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between -- * user space and kernel space. Similarly, we use a simple vruntime-sorted list -- * in user space, but an rbtree could be used instead. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include --#include "scx_common.bpf.h" --#include "scx_example_userland_common.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; --const volatile s32 usersched_pid; -- --/* !0 for veristat, set during init */ --const volatile u32 num_possible_cpus = 64; -- --/* Stats that are printed by user space. */ --u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues; -- --struct user_exit_info uei; -- --/* -- * Whether the user space scheduler needs to be scheduled due to a task being -- * enqueued in user space. -- */ --static bool usersched_needed; -- --/* -- * The map containing tasks that are enqueued in user space from the kernel. -- * -- * This map is drained by the user space scheduler. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, struct scx_userland_enqueued_task); --} enqueued SEC(".maps"); -- --/* -- * The map containing tasks that are dispatched to the kernel from user space. -- * -- * Drained by the kernel in userland_dispatch(). -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, s32); --} dispatched SEC(".maps"); -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local DSQ */ --}; -- --/* Map that contains task-local storage. */ --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --static bool is_usersched_task(const struct task_struct *p) --{ -- return p->pid == usersched_pid; --} -- --static bool keep_in_kernel(const struct task_struct *p) --{ -- return p->nr_cpus_allowed < num_possible_cpus; --} -- --static struct task_struct *usersched_task(void) --{ -- struct task_struct *p; -- -- p = bpf_task_from_pid(usersched_pid); -- /* -- * Should never happen -- the usersched task should always be managed -- * by sched_ext. -- */ -- if (!p) { -- scx_bpf_error("Failed to find usersched task %d", usersched_pid); -- /* -- * We should never hit this path, and we error out of the -- * scheduler above just in case, so the scheduler will soon be -- * be evicted regardless. So as to simplify the logic in the -- * caller to not have to check for NULL, return an acquired -- * reference to the current task here rather than NULL. -- */ -- return bpf_task_acquire(bpf_get_current_task_btf()); -- } -- -- return p; --} -- --s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- if (keep_in_kernel(p)) { -- s32 cpu; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to look up task-local storage for %s", p->comm); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- tctx->force_local = true; -- return cpu; -- } -- } -- -- return prev_cpu; --} -- --static void dispatch_user_scheduler(void) --{ -- struct task_struct *p; -- -- usersched_needed = false; -- p = usersched_task(); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); --} -- --static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags) --{ -- struct scx_userland_enqueued_task task; -- -- memset(&task, 0, sizeof(task)); -- task.pid = p->pid; -- task.sum_exec_runtime = p->se.sum_exec_runtime; -- task.weight = p->scx.weight; -- -- if (bpf_map_push_elem(&enqueued, &task, 0)) { -- /* -- * If we fail to enqueue the task in user space, put it -- * directly on the global DSQ. -- */ -- __sync_fetch_and_add(&nr_failed_enqueues, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- __sync_fetch_and_add(&nr_user_enqueues, 1); -- usersched_needed = true; -- } --} -- --void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags) --{ -- if (keep_in_kernel(p)) { -- u64 dsq_id = SCX_DSQ_GLOBAL; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to lookup task ctx for %s", p->comm); -- return; -- } -- -- if (tctx->force_local) -- dsq_id = SCX_DSQ_LOCAL; -- tctx->force_local = false; -- scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags); -- __sync_fetch_and_add(&nr_kernel_enqueues, 1); -- return; -- } else if (!is_usersched_task(p)) { -- enqueue_task_in_user_space(p, enq_flags); -- } --} -- --void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (usersched_needed) -- dispatch_user_scheduler(); -- -- bpf_repeat(4096) { -- s32 pid; -- struct task_struct *p; -- -- if (bpf_map_pop_elem(&dispatched, &pid)) -- break; -- -- /* -- * The task could have exited by the time we get around to -- * dispatching it. Treat this as a normal occurrence, and simply -- * move onto the next iteration. -- */ -- p = bpf_task_from_pid(pid); -- if (!p) -- continue; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } --} -- --s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(userland_init) --{ -- if (num_possible_cpus == 0) { -- scx_bpf_error("User scheduler # CPUs uninitialized (%d)", -- num_possible_cpus); -- return -EINVAL; -- } -- -- if (usersched_pid <= 0) { -- scx_bpf_error("User scheduler pid uninitialized (%d)", -- usersched_pid); -- return -EINVAL; -- } -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops userland_ops = { -- .select_cpu = (void *)userland_select_cpu, -- .enqueue = (void *)userland_enqueue, -- .dispatch = (void *)userland_dispatch, -- .prep_enable = (void *)userland_prep_enable, -- .init = (void *)userland_init, -- .exit = (void *)userland_exit, -- .timeout_ms = 3000, -- .name = "userland", --}; -diff --git b/tools/sched_ext/scx_example_userland.c a/tools/sched_ext/scx_example_userland.c -deleted file mode 100644 -index 4152b1e65fe1..000000000000 ---- b/tools/sched_ext/scx_example_userland.c -+++ /dev/null -@@ -1,402 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext user space scheduler which provides vruntime semantics -- * using a simple ordered-list implementation. -- * -- * Each CPU in the system resides in a single, global domain. This precludes -- * the need to do any load balancing between domains. The scheduler could -- * easily be extended to support multiple domains, with load balancing -- * happening in user space. -- * -- * Any task which has any CPU affinity is scheduled entirely in BPF. This -- * program only schedules tasks which may run on any CPU. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include "user_exit_info.h" --#include "scx_example_userland_common.h" --#include "scx_example_userland.skel.h" -- --const char help_fmt[] = --"A minimal userland sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-b BATCH] [-p]\n" --"\n" --" -b BATCH The number of tasks to batch when dispatching (default: 8)\n" --" -p Don't switch all, switch only tasks on SCHED_EXT policy\n" --" -h Display this help and exit\n"; -- --/* Defined in UAPI */ --#define SCHED_EXT 7 -- --/* Number of tasks to batch when dispatching to user space. */ --static __u32 batch_size = 8; -- --static volatile int exit_req; --static int enqueued_fd, dispatched_fd; -- --static struct scx_example_userland *skel; --static struct bpf_link *ops_link; -- --/* Stats collected in user space. */ --static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches; -- --/* The data structure containing tasks that are enqueued in user space. */ --struct enqueued_task { -- LIST_ENTRY(enqueued_task) entries; -- __u64 sum_exec_runtime; -- double vruntime; --}; -- --/* -- * Use a vruntime-sorted list to store tasks. This could easily be extended to -- * a more optimal data structure, such as an rbtree as is done in CFS. We -- * currently elect to use a sorted list to simplify the example for -- * illustrative purposes. -- */ --LIST_HEAD(listhead, enqueued_task); -- --/* -- * A vruntime-sorted list of tasks. The head of the list contains the task with -- * the lowest vruntime. That is, the task that has the "highest" claim to be -- * scheduled. -- */ --static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head); -- --/* -- * The statically allocated array of tasks. We use a statically allocated list -- * here to avoid having to allocate on the enqueue path, which could cause a -- * deadlock. A more substantive user space scheduler could e.g. provide a hook -- * for newly enabled tasks that are passed to the scheduler from the -- * .prep_enable() callback to allows the scheduler to allocate on safe paths. -- */ --struct enqueued_task tasks[USERLAND_MAX_TASKS]; -- --static double min_vruntime; -- --static void sigint_handler(int userland) --{ -- exit_req = 1; --} -- --static __u32 task_pid(const struct enqueued_task *task) --{ -- return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task); --} -- --static int dispatch_task(s32 pid) --{ -- int err; -- -- err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d\n", pid); -- exit_req = 1; -- } else { -- nr_vruntime_dispatches++; -- } -- -- return err; --} -- --static struct enqueued_task *get_enqueued_task(__s32 pid) --{ -- if (pid >= USERLAND_MAX_TASKS) -- return NULL; -- -- return &tasks[pid]; --} -- --static double calc_vruntime_delta(__u64 weight, __u64 delta) --{ -- double weight_f = (double)weight / 100.0; -- double delta_f = (double)delta; -- -- return delta_f / weight_f; --} -- --static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task) --{ -- __u64 delta; -- -- delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime; -- -- enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta); -- if (min_vruntime > enqueued->vruntime) -- enqueued->vruntime = min_vruntime; -- enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime; --} -- --static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task) --{ -- struct enqueued_task *curr, *enqueued, *prev; -- -- curr = get_enqueued_task(bpf_task->pid); -- if (!curr) -- return ENOENT; -- -- update_enqueued(curr, bpf_task); -- nr_vruntime_enqueues++; -- -- /* -- * Enqueue the task in a vruntime-sorted list. A more optimal data -- * structure such as an rbtree could easily be used as well. We elect -- * to use a list here simply because it's less code, and thus the -- * example is less convoluted and better serves to illustrate what a -- * user space scheduler could look like. -- */ -- -- if (LIST_EMPTY(&vruntime_head)) { -- LIST_INSERT_HEAD(&vruntime_head, curr, entries); -- return 0; -- } -- -- LIST_FOREACH(enqueued, &vruntime_head, entries) { -- if (curr->vruntime <= enqueued->vruntime) { -- LIST_INSERT_BEFORE(enqueued, curr, entries); -- return 0; -- } -- prev = enqueued; -- } -- -- LIST_INSERT_AFTER(prev, curr, entries); -- -- return 0; --} -- --static void drain_enqueued_map(void) --{ -- while (1) { -- struct scx_userland_enqueued_task task; -- int err; -- -- if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) -- return; -- -- err = vruntime_enqueue(&task); -- if (err) { -- fprintf(stderr, "Failed to enqueue task %d: %s\n", -- task.pid, strerror(err)); -- exit_req = 1; -- return; -- } -- } --} -- --static void dispatch_batch(void) --{ -- __u32 i; -- -- for (i = 0; i < batch_size; i++) { -- struct enqueued_task *task; -- int err; -- __s32 pid; -- -- task = LIST_FIRST(&vruntime_head); -- if (!task) -- return; -- -- min_vruntime = task->vruntime; -- pid = task_pid(task); -- LIST_REMOVE(task, entries); -- err = dispatch_task(pid); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d in %u\n", -- pid, i); -- return; -- } -- } --} -- --static void *run_stats_printer(void *arg) --{ -- while (!exit_req) { -- __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; -- -- nr_failed_enqueues = skel->bss->nr_failed_enqueues; -- nr_kernel_enqueues = skel->bss->nr_kernel_enqueues; -- nr_user_enqueues = skel->bss->nr_user_enqueues; -- total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues; -- -- printf("o-----------------------o\n"); -- printf("| BPF ENQUEUES |\n"); -- printf("|-----------------------|\n"); -- printf("| kern: %10llu |\n", nr_kernel_enqueues); -- printf("| user: %10llu |\n", nr_user_enqueues); -- printf("| failed: %10llu |\n", nr_failed_enqueues); -- printf("| -------------------- |\n"); -- printf("| total: %10llu |\n", total); -- printf("| |\n"); -- printf("|-----------------------|\n"); -- printf("| VRUNTIME / USER |\n"); -- printf("|-----------------------|\n"); -- printf("| enq: %10llu |\n", nr_vruntime_enqueues); -- printf("| disp: %10llu |\n", nr_vruntime_dispatches); -- printf("o-----------------------o\n"); -- printf("\n\n"); -- sleep(1); -- } -- -- return NULL; --} -- --static int spawn_stats_thread(void) --{ -- pthread_t stats_printer; -- -- return pthread_create(&stats_printer, NULL, run_stats_printer, NULL); --} -- --static int bootstrap(int argc, char **argv) --{ -- int err; -- __u32 opt; -- struct sched_param sched_param = { -- .sched_priority = sched_get_priority_max(SCHED_EXT), -- }; -- bool switch_partial = false; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- /* -- * Enforce that the user scheduler task is managed by sched_ext. The -- * task eagerly drains the list of enqueued tasks in its main work -- * loop, and then yields the CPU. The BPF scheduler only schedules the -- * user space scheduler task when at least one other task in the system -- * needs to be scheduled. -- */ -- err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param); -- if (err) { -- fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err)); -- return err; -- } -- -- while ((opt = getopt(argc, argv, "b:ph")) != -1) { -- switch (opt) { -- case 'b': -- batch_size = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- exit(opt != 'h'); -- } -- } -- -- /* -- * It's not always safe to allocate in a user space scheduler, as an -- * enqueued task could hold a lock that we require in order to be able -- * to allocate. -- */ -- err = mlockall(MCL_CURRENT | MCL_FUTURE); -- if (err) { -- fprintf(stderr, "Failed to prefault and lock address space: %s\n", -- strerror(err)); -- return err; -- } -- -- skel = scx_example_userland__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno)); -- return errno; -- } -- skel->rodata->num_possible_cpus = libbpf_num_possible_cpus(); -- assert(skel->rodata->num_possible_cpus > 0); -- skel->rodata->usersched_pid = getpid(); -- assert(skel->rodata->usersched_pid > 0); -- skel->rodata->switch_partial = switch_partial; -- -- err = scx_example_userland__load(skel); -- if (err) { -- fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- enqueued_fd = bpf_map__fd(skel->maps.enqueued); -- dispatched_fd = bpf_map__fd(skel->maps.dispatched); -- assert(enqueued_fd > 0); -- assert(dispatched_fd > 0); -- -- err = spawn_stats_thread(); -- if (err) { -- fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops); -- if (!ops_link) { -- fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno)); -- err = errno; -- goto destroy_skel; -- } -- -- return 0; -- --destroy_skel: -- scx_example_userland__destroy(skel); -- exit_req = 1; -- return err; --} -- --static void sched_main_loop(void) --{ -- while (!exit_req) { -- /* -- * Perform the following work in the main user space scheduler -- * loop: -- * -- * 1. Drain all tasks from the enqueued map, and enqueue them -- * to the vruntime sorted list. -- * -- * 2. Dispatch a batch of tasks from the vruntime sorted list -- * down to the kernel. -- * -- * 3. Yield the CPU back to the system. The BPF scheduler will -- * reschedule the user space scheduler once another task has -- * been enqueued to user space. -- */ -- drain_enqueued_map(); -- dispatch_batch(); -- sched_yield(); -- } --} -- --int main(int argc, char **argv) --{ -- int err; -- -- err = bootstrap(argc, argv); -- if (err) { -- fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err)); -- return err; -- } -- -- sched_main_loop(); -- -- exit_req = 1; -- bpf_link__destroy(ops_link); -- uei_print(&skel->bss->uei); -- scx_example_userland__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_userland_common.h a/tools/sched_ext/scx_example_userland_common.h -deleted file mode 100644 -index 639c6809c5ff..000000000000 ---- b/tools/sched_ext/scx_example_userland_common.h -+++ /dev/null -@@ -1,19 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* Copyright (c) 2022 Meta, Inc */ -- --#ifndef __SCX_USERLAND_COMMON_H --#define __SCX_USERLAND_COMMON_H -- --#define USERLAND_MAX_TASKS 8192 -- --/* -- * An instance of a task that has been enqueued by the kernel for consumption -- * by a user space global scheduler thread. -- */ --struct scx_userland_enqueued_task { -- __s32 pid; -- u64 sum_exec_runtime; -- u64 weight; --}; -- --#endif // __SCX_USERLAND_COMMON_H -diff --git b/tools/sched_ext/user_exit_info.h a/tools/sched_ext/user_exit_info.h -deleted file mode 100644 -index e701ef0e0b86..000000000000 ---- b/tools/sched_ext/user_exit_info.h -+++ /dev/null -@@ -1,50 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Define struct user_exit_info which is shared between BPF and userspace parts -- * to communicate exit status and other information. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __USER_EXIT_INFO_H --#define __USER_EXIT_INFO_H -- --struct user_exit_info { -- int type; -- char reason[128]; -- char msg[1024]; --}; -- --#ifdef __bpf__ -- --#include "vmlinux.h" --#include -- --static inline void uei_record(struct user_exit_info *uei, -- const struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason); -- bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg); -- /* use __sync to force memory barrier */ -- __sync_val_compare_and_swap(&uei->type, uei->type, ei->type); --} -- --#else /* !__bpf__ */ -- --static inline bool uei_exited(struct user_exit_info *uei) --{ -- /* use __sync to force memory barrier */ -- return __sync_val_compare_and_swap(&uei->type, -1, -1); --} -- --static inline void uei_print(const struct user_exit_info *uei) --{ -- fprintf(stderr, "EXIT: %s", uei->reason); -- if (uei->msg[0] != '\0') -- fprintf(stderr, " (%s)", uei->msg); -- fputs("\n", stderr); --} -- --#endif /* __bpf__ */ --#endif /* __USER_EXIT_INFO_H */ diff --git a/oneplus/hmbird/fengchi_OP-PAD-3-SM8750_A16.patch b/oneplus/hmbird/fengchi_OP-PAD-3-SM8750_A16.patch index e6008b42..81dfa588 100644 --- a/oneplus/hmbird/fengchi_OP-PAD-3-SM8750_A16.patch +++ b/oneplus/hmbird/fengchi_OP-PAD-3-SM8750_A16.patch @@ -365,20 +365,6 @@ index ba49d7bd2b67..dd6b1d58af01 100755 #define MAX_SCHEDULE_TIMEOUT LONG_MAX extern long schedule_timeout(long timeout); -@@ -1523,13 +1520,8 @@ struct task_struct { - */ - struct callback_head l1d_flush_kill; - #endif --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, unsigned long sched_prop); -- ANDROID_KABI_USE(2, struct sched_ext_entity *scx); --#else - ANDROID_KABI_RESERVE(1); - ANDROID_KABI_RESERVE(2); --#endif - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); - ANDROID_KABI_RESERVE(5); @@ -1933,6 +1925,91 @@ static inline int task_nice(const struct task_struct *p) return PRIO_TO_NICE((p)->static_prio); } diff --git a/oneplus/hmbird/fengchi_OP-TURBO-6_A16.patch b/oneplus/hmbird/fengchi_OP-TURBO-6_A16.patch deleted file mode 100644 index e6008b42..00000000 --- a/oneplus/hmbird/fengchi_OP-TURBO-6_A16.patch +++ /dev/null @@ -1,19909 +0,0 @@ -diff --git b/Documentation/scheduler/index.rst a/Documentation/scheduler/index.rst -index 0b650bb550e6..3170747226f6 100644 ---- b/Documentation/scheduler/index.rst -+++ a/Documentation/scheduler/index.rst -@@ -19,7 +19,6 @@ Scheduler - sched-nice-design - sched-rt-group - sched-stats -- sched-ext - sched-debug - - text_files -diff --git b/Documentation/scheduler/sched-ext.rst a/Documentation/scheduler/sched-ext.rst -deleted file mode 100644 -index 84c30b44f104..000000000000 ---- b/Documentation/scheduler/sched-ext.rst -+++ /dev/null -@@ -1,230 +0,0 @@ --========================== --Extensible Scheduler Class --========================== -- --sched_ext is a scheduler class whose behavior can be defined by a set of BPF --programs - the BPF scheduler. -- --* sched_ext exports a full scheduling interface so that any scheduling -- algorithm can be implemented on top. -- --* The BPF scheduler can group CPUs however it sees fit and schedule them -- together, as tasks aren't tied to specific CPUs at the time of wakeup. -- --* The BPF scheduler can be turned on and off dynamically anytime. -- --* The system integrity is maintained no matter what the BPF scheduler does. -- The default scheduling behavior is restored anytime an error is detected, -- a runnable task stalls, or on invoking the SysRq key sequence -- :kbd:`SysRq-S`. -- --Switching to and from sched_ext --=============================== -- --``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and --``tools/sched_ext`` contains the example schedulers. -- --sched_ext is used only when the BPF scheduler is loaded and running. -- --If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be --treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is --loaded. On load, such tasks will be switched to and scheduled by sched_ext. -- --The BPF scheduler can choose to schedule all normal and lower class tasks by --calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this --case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and --``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers, --this mode can be selected with the ``-a`` option. -- --Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or --detection of any internal error including stalled runnable tasks aborts the --BPF scheduler and reverts all tasks back to CFS. -- --.. code-block:: none -- -- # make -j16 -C tools/sched_ext -- # tools/sched_ext/scx_example_simple -- local=0 global=3 -- local=5 global=24 -- local=9 global=44 -- local=13 global=56 -- local=17 global=72 -- ^CEXIT: BPF scheduler unregistered -- --If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler --and whether a given task is on sched_ext can be determined as follows: -- --.. code-block:: none -- -- # cat /sys/kernel/debug/sched/ext -- ops : simple -- enabled : 1 -- switching_all : 1 -- switched_all : 1 -- enable_state : enabled -- -- # grep ext /proc/self/sched -- ext.enabled : 1 -- --The Basics --========== -- --Userspace can implement an arbitrary BPF scheduler by loading a set of BPF --programs that implement ``struct sched_ext_ops``. The only mandatory field --is ``ops.name`` which must be a valid BPF object name. All operations are --optional. The following modified excerpt is from --``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO --scheduler. -- --.. code-block:: c -- -- s32 BPF_STRUCT_OPS(simple_init) -- { -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; -- } -- -- void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) -- { -- if (enq_flags & SCX_ENQ_LOCAL) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags); -- } -- -- void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) -- { -- exit_type = ei->type; -- } -- -- SEC(".struct_ops") -- struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", -- }; -- --Dispatch Queues ----------------- -- --To match the impedance between the scheduler core and the BPF scheduler, --sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a --priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``), --and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage --an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and --``scx_bpf_destroy_dsq()``. -- --A CPU always executes a task from its local DSQ. A task is "dispatched" to a --DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's --local DSQ. -- --When a CPU is looking for the next task to run, if the local DSQ is not --empty, the first task is picked. Otherwise, the CPU tries to consume the --global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()`` --is invoked. -- --Scheduling Cycle ------------------ -- --The following briefly shows how a waking task is scheduled and executed. -- --1. When a task is waking up, ``ops.select_cpu()`` is the first operation -- invoked. This serves two purposes. First, CPU selection optimization -- hint. Second, waking up the selected CPU if idle. -- -- The CPU selected by ``ops.select_cpu()`` is an optimization hint and not -- binding. The actual decision is made at the last step of scheduling. -- However, there is a small performance gain if the CPU -- ``ops.select_cpu()`` returns matches the CPU the task eventually runs on. -- -- A side-effect of selecting a CPU is waking it up from idle. While a BPF -- scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper, -- using ``ops.select_cpu()`` judiciously can be simpler and more efficient. -- -- Note that the scheduler core will ignore an invalid CPU selection, for -- example, if it's outside the allowed cpumask of the task. -- --2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can -- make one of the following decisions: -- -- * Immediately dispatch the task to either the global or local DSQ by -- calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or -- ``SCX_DSQ_LOCAL``, respectively. -- -- * Immediately dispatch the task to a custom DSQ by calling -- ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63. -- -- * Queue the task on the BPF side. -- --3. When a CPU is ready to schedule, it first looks at its local DSQ. If -- empty, it then looks at the global DSQ. If there still isn't a task to -- run, ``ops.dispatch()`` is invoked which can use the following two -- functions to populate the local DSQ. -- -- * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can -- be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``, -- ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()`` -- currently can't be called with BPF locks held, this is being worked on -- and will be supported. ``scx_bpf_dispatch()`` schedules dispatching -- rather than performing them immediately. There can be up to -- ``ops.dispatch_max_batch`` pending tasks. -- -- * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ -- to the dispatching DSQ. This function cannot be called with any BPF -- locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks -- before trying to consume the specified DSQ. -- --4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ, -- the CPU runs the first one. If empty, the following steps are taken: -- -- * Try to consume the global DSQ. If successful, run the task. -- -- * If ``ops.dispatch()`` has dispatched any tasks, retry #3. -- -- * If the previous task is an SCX task and still runnable, keep executing -- it (see ``SCX_OPS_ENQ_LAST``). -- -- * Go idle. -- --Note that the BPF scheduler can always choose to dispatch tasks immediately --in ``ops.enqueue()`` as illustrated in the above simple example. If only the --built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as --a task is never queued on the BPF scheduler and both the local and global --DSQs are consumed automatically. -- --``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use --``scx_bpf_dispatch_vtime()`` for the priority queue. See the function --documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for --more information. -- --Where to Look --============= -- --* ``include/linux/sched/ext.h`` defines the core data structures, ops table -- and constants. -- --* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. -- The functions prefixed with ``scx_bpf_`` can be called from the BPF -- scheduler. -- --* ``tools/sched_ext/`` hosts example BPF scheduler implementations. -- -- * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example -- using a custom DSQ. -- -- * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting -- five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. -- --ABI Instability --=============== -- --The APIs provided by sched_ext to BPF schedulers programs have no stability --guarantees. This includes the ops table callbacks and constants defined in --``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in --``kernel/sched/ext.c``. -- --While we will attempt to provide a relatively stable API surface when --possible, they are subject to change without warning between kernel --versions. -diff --git b/MAINTAINERS a/MAINTAINERS -index 9fc6108503c3..2384b55682ee 100644 ---- b/MAINTAINERS -+++ a/MAINTAINERS -@@ -19132,8 +19132,6 @@ R: Ben Segall (CONFIG_CFS_BANDWIDTH) - R: Mel Gorman (CONFIG_NUMA_BALANCING) - R: Daniel Bristot de Oliveira (SCHED_DEADLINE) - R: Valentin Schneider (TOPOLOGY) --R: Tejun Heo (SCHED_EXT) --R: David Vernet (SCHED_EXT) - L: linux-kernel@vger.kernel.org - S: Maintained - T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core -@@ -19142,7 +19140,6 @@ F: include/linux/sched.h - F: include/linux/wait.h - F: include/uapi/linux/sched.h - F: kernel/sched/ --F: tools/sched_ext/ - - SCSI LIBSAS SUBSYSTEM - R: John Garry -diff --git b/arch/arm64/configs/defconfig a/arch/arm64/configs/defconfig -index f93980c09d7f..c2d530535980 100644 ---- b/arch/arm64/configs/defconfig -+++ a/arch/arm64/configs/defconfig -@@ -6,8 +6,6 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_BPF_SYSCALL=y - CONFIG_BPF_JIT=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_BSD_PROCESS_ACCT=y - CONFIG_BSD_PROCESS_ACCT_V3=y -@@ -1587,3 +1585,4 @@ CONFIG_CORESIGHT_STM=m - CONFIG_CORESIGHT_CPU_DEBUG=m - CONFIG_CORESIGHT_CTI=m - CONFIG_MEMTEST=y -+CONFIG_HMBIRD_SCHED=y -diff --git b/arch/arm64/configs/gki_defconfig a/arch/arm64/configs/gki_defconfig -index 4f756dca5e05..6c1bb94f875d 100644 ---- b/arch/arm64/configs/gki_defconfig -+++ a/arch/arm64/configs/gki_defconfig -@@ -10,8 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y - # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set - CONFIG_BPF_LSM=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_TASKSTATS=y - CONFIG_TASK_DELAY_ACCT=y -diff --git b/drivers/gpu/drm/msm/msm_drv.c a/drivers/gpu/drm/msm/msm_drv.c -index 4bd028fa7500..5825ce9d6d7b 100755 ---- b/drivers/gpu/drm/msm/msm_drv.c -+++ a/drivers/gpu/drm/msm/msm_drv.c -@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) - } - - sched_set_fifo(ev_thread->worker->task); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3); -+#endif - } - - ret = drm_vblank_init(ddev, priv->num_crtcs); -diff --git b/drivers/tty/sysrq.c a/drivers/tty/sysrq.c -index bd001445815e..dcf2e1ebf9c5 100644 ---- b/drivers/tty/sysrq.c -+++ a/drivers/tty/sysrq.c -@@ -524,7 +524,6 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { - NULL, /* P */ - NULL, /* Q */ - NULL, /* R */ -- /* S: May be registered by sched_ext for resetting */ - NULL, /* S */ - NULL, /* T */ - NULL, /* U */ -diff --git b/include/asm-generic/vmlinux.lds.h a/include/asm-generic/vmlinux.lds.h -index c45078a445c8..6c15659f874e 100755 ---- b/include/asm-generic/vmlinux.lds.h -+++ a/include/asm-generic/vmlinux.lds.h -@@ -131,7 +131,7 @@ - *(__dl_sched_class) \ - *(__rt_sched_class) \ - *(__fair_sched_class) \ -- *(__ext_sched_class) \ -+ *(__hmbird_sched_class) \ - *(__idle_sched_class) \ - __sched_class_lowest = .; - -diff --git b/include/linux/sched.h a/include/linux/sched.h -index ba49d7bd2b67..dd6b1d58af01 100755 ---- b/include/linux/sched.h -+++ a/include/linux/sched.h -@@ -73,7 +73,9 @@ struct task_delay_info; - struct task_group; - struct user_event_mm; - --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - - /* - * Task state bitmask. NOTE! These bits are also -@@ -298,11 +300,6 @@ enum { - - extern void scheduler_tick(void); - --#ifdef CONFIG_SLIM_SCHED --extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer); --extern void stop_shadow_tick_timer(void); --#endif -- - #define MAX_SCHEDULE_TIMEOUT LONG_MAX - - extern long schedule_timeout(long timeout); -@@ -1523,13 +1520,8 @@ struct task_struct { - */ - struct callback_head l1d_flush_kill; - #endif --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, unsigned long sched_prop); -- ANDROID_KABI_USE(2, struct sched_ext_entity *scx); --#else - ANDROID_KABI_RESERVE(1); - ANDROID_KABI_RESERVE(2); --#endif - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); - ANDROID_KABI_RESERVE(5); -@@ -1933,6 +1925,91 @@ static inline int task_nice(const struct task_struct *p) - return PRIO_TO_NICE((p)->static_prio); - } - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline bool task_is_top_task(struct task_struct *p) -+{ -+ return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ ->top_task_prop & TOP_TASK_BITS_MASK); -+} -+ -+static inline int get_top_task_prop(struct task_struct *p) -+{ -+ return get_hmbird_ts(p)->top_task_prop; -+} -+ -+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear) -+{ -+ if (set) -+ get_hmbird_ts(p)->top_task_prop |= set; -+ if (clear) -+ get_hmbird_ts(p)->top_task_prop &= ~clear; -+ return 0; -+} -+ -+static inline void reset_top_task_prop(struct task_struct *p) -+{ -+ get_hmbird_ts(p)->top_task_prop = 0; -+} -+ -+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ entity->sched_prop = sp; -+ } -+ return 0; -+} -+ -+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->sched_prop; -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq) -+{ -+ unsigned long new_dsq; -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq; -+ entity->sched_prop = new_dsq; -+ } -+} -+ -+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK); -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ entity->dsq_sync_ux = val; -+} -+ -+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->dsq_sync_ux; -+ else -+ return 0; -+} -+#endif - - extern int can_nice(const struct task_struct *p, const int nice); - extern int task_curr(const struct task_struct *p); -diff --git b/include/linux/sched/ext.h a/include/linux/sched/ext.h -deleted file mode 100755 -index b737a00c1373..000000000000 ---- b/include/linux/sched/ext.h -+++ /dev/null -@@ -1,647 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef _LINUX_SCHED_EXT_H --#define _LINUX_SCHED_EXT_H -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --#include -- --enum scx_consts { -- SCX_OPS_NAME_LEN = 128, -- SCX_EXIT_REASON_LEN = 128, -- SCX_EXIT_BT_LEN = 64, -- SCX_EXIT_MSG_LEN = 1024, -- -- SCX_SLICE_DFL = 20 * NSEC_PER_MSEC, -- SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */ --}; -- --/* -- * DSQ (dispatch queue) IDs are 64bit of the format: -- * -- * Bits: [63] [62 .. 0] -- * [ B] [ ID ] -- * -- * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -- * ID: 63 bit ID -- * -- * Built-in IDs: -- * -- * Bits: [63] [62] [61..32] [31 .. 0] -- * [ 1] [ L] [ R ] [ V ] -- * -- * 1: 1 for built-in DSQs. -- * L: 1 for LOCAL_ON DSQ IDs, 0 for others -- * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -- */ --enum scx_dsq_id_flags { -- SCX_DSQ_FLAG_BUILTIN = 1LLU << 63, -- SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -- -- SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0, -- SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, -- SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, -- SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, -- SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, --}; -- --enum scx_exit_type { -- SCX_EXIT_NONE, -- SCX_EXIT_DONE, -- -- SCX_EXIT_UNREG = 64, /* BPF unregistration */ -- SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -- SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ -- SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ --}; -- --/* -- * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is -- * being disabled. -- */ --struct scx_exit_info { -- /* %SCX_EXIT_* - broad category of the exit reason */ -- enum scx_exit_type type; -- /* textual representation of the above */ -- char reason[SCX_EXIT_REASON_LEN]; -- /* number of entries in the backtrace */ -- u32 bt_len; -- /* backtrace if exiting due to an error */ -- unsigned long bt[SCX_EXIT_BT_LEN]; -- /* extra message */ -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --/* sched_ext_ops.flags */ --enum scx_ops_flags { -- /* -- * Keep built-in idle tracking even if ops.update_idle() is implemented. -- */ -- SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, -- -- /* -- * By default, if there are no other task to run on the CPU, ext core -- * keeps running the current task even after its slice expires. If this -- * flag is specified, such tasks are passed to ops.enqueue() with -- * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. -- */ -- SCX_OPS_ENQ_LAST = 1LLU << 1, -- -- /* -- * An exiting task may schedule after PF_EXITING is set. In such cases, -- * bpf_task_from_pid() may not be able to find the task and if the BPF -- * scheduler depends on pid lookup for dispatching, the task will be -- * lost leading to various issues including RCU grace period stalls. -- * -- * To mask this problem, by default, unhashed tasks are automatically -- * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't -- * depend on pid lookups and wants to handle these tasks directly, the -- * following flag can be used. -- */ -- SCX_OPS_ENQ_EXITING = 1LLU << 2, -- -- /* -- * CPU cgroup knob enable flags -- */ -- SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16, /* cpu.weight */ -- -- SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | -- SCX_OPS_ENQ_LAST | -- SCX_OPS_ENQ_EXITING | -- SCX_OPS_CGROUP_KNOB_WEIGHT, --}; -- --/* argument container for ops.enable() and friends */ --struct scx_enable_args { --}; -- --/* argument container for ops->cgroup_init() */ --struct scx_cgroup_init_args { -- /* the weight of the cgroup [1..10000] */ -- u32 weight; --}; -- --enum scx_cpu_preempt_reason { -- /* next task is being scheduled by &sched_class_rt */ -- SCX_CPU_PREEMPT_RT, -- /* next task is being scheduled by &sched_class_dl */ -- SCX_CPU_PREEMPT_DL, -- /* next task is being scheduled by &sched_class_stop */ -- SCX_CPU_PREEMPT_STOP, -- /* unknown reason for SCX being preempted */ -- SCX_CPU_PREEMPT_UNKNOWN, --}; -- --/* -- * Argument container for ops->cpu_acquire(). Currently empty, but may be -- * expanded in the future. -- */ --struct scx_cpu_acquire_args {}; -- --/* argument container for ops->cpu_release() */ --struct scx_cpu_release_args { -- /* the reason the CPU was preempted */ -- enum scx_cpu_preempt_reason reason; -- -- /* the task that's going to be scheduled on the CPU */ -- struct task_struct *task; --}; -- --/** -- * struct sched_ext_ops - Operation table for BPF scheduler implementation -- * -- * Userland can implement an arbitrary scheduling policy by implementing and -- * loading operations in this table. -- */ --struct sched_ext_ops { -- /** -- * select_cpu - Pick the target CPU for a task which is being woken up -- * @p: task being woken up -- * @prev_cpu: the cpu @p was on before sleeping -- * @wake_flags: SCX_WAKE_* -- * -- * Decision made here isn't final. @p may be moved to any CPU while it -- * is getting dispatched for execution later. However, as @p is not on -- * the rq at this point, getting the eventual execution CPU right here -- * saves a small bit of overhead down the line. -- * -- * If an idle CPU is returned, the CPU is kicked and will try to -- * dispatch. While an explicit custom mechanism can be added, -- * select_cpu() serves as the default way to wake up idle CPUs. -- */ -- s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); -- -- /** -- * enqueue - Enqueue a task on the BPF scheduler -- * @p: task being enqueued -- * @enq_flags: %SCX_ENQ_* -- * -- * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() -- * or enqueue on the BPF scheduler. If not directly dispatched, the bpf -- * scheduler owns @p and if it fails to dispatch @p, the task will -- * stall. -- */ -- void (*enqueue)(struct task_struct *p, u64 enq_flags); -- -- /** -- * dequeue - Remove a task from the BPF scheduler -- * @p: task being dequeued -- * @deq_flags: %SCX_DEQ_* -- * -- * Remove @p from the BPF scheduler. This is usually called to isolate -- * the task while updating its scheduling properties (e.g. priority). -- * -- * The ext core keeps track of whether the BPF side owns a given task or -- * not and can gracefully ignore spurious dispatches from BPF side, -- * which makes it safe to not implement this method. However, depending -- * on the scheduling logic, this can lead to confusing behaviors - e.g. -- * scheduling position not being updated across a priority change. -- */ -- void (*dequeue)(struct task_struct *p, u64 deq_flags); -- -- /** -- * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs -- * @cpu: CPU to dispatch tasks for -- * @prev: previous task being switched out -- * -- * Called when a CPU's local dsq is empty. The operation should dispatch -- * one or more tasks from the BPF scheduler into the DSQs using -- * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using -- * scx_bpf_consume(). -- * -- * The maximum number of times scx_bpf_dispatch() can be called without -- * an intervening scx_bpf_consume() is specified by -- * ops.dispatch_max_batch. See the comments on top of the two functions -- * for more details. -- * -- * When not %NULL, @prev is an SCX task with its slice depleted. If -- * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in -- * @prev->scx.flags, it is not enqueued yet and will be enqueued after -- * ops.dispatch() returns. To keep executing @prev, return without -- * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. -- */ -- void (*dispatch)(s32 cpu, struct task_struct *prev); -- -- /** -- * runnable - A task is becoming runnable on its associated CPU -- * @p: task becoming runnable -- * @enq_flags: %SCX_ENQ_* -- * -- * This and the following three functions can be used to track a task's -- * execution state transitions. A task becomes ->runnable() on a CPU, -- * and then goes through one or more ->running() and ->stopping() pairs -- * as it runs on the CPU, and eventually becomes ->quiescent() when it's -- * done running on the CPU. -- * -- * @p is becoming runnable on the CPU because it's -- * -- * - waking up (%SCX_ENQ_WAKEUP) -- * - being moved from another CPU -- * - being restored after temporarily taken off the queue for an -- * attribute change. -- * -- * This and ->enqueue() are related but not coupled. This operation -- * notifies @p's state transition and may not be followed by ->enqueue() -- * e.g. when @p is being dispatched to a remote CPU. Likewise, a task -- * may be ->enqueue()'d without being preceded by this operation e.g. -- * after exhausting its slice. -- */ -- void (*runnable)(struct task_struct *p, u64 enq_flags); -- -- /** -- * running - A task is starting to run on its associated CPU -- * @p: task starting to run -- * -- * See ->runnable() for explanation on the task state notifiers. -- */ -- void (*running)(struct task_struct *p); -- -- /** -- * stopping - A task is stopping execution -- * @p: task stopping to run -- * @runnable: is task @p still runnable? -- * -- * See ->runnable() for explanation on the task state notifiers. If -- * !@runnable, ->quiescent() will be invoked after this operation -- * returns. -- */ -- void (*stopping)(struct task_struct *p, bool runnable); -- -- /** -- * quiescent - A task is becoming not runnable on its associated CPU -- * @p: task becoming not runnable -- * @deq_flags: %SCX_DEQ_* -- * -- * See ->runnable() for explanation on the task state notifiers. -- * -- * @p is becoming quiescent on the CPU because it's -- * -- * - sleeping (%SCX_DEQ_SLEEP) -- * - being moved to another CPU -- * - being temporarily taken off the queue for an attribute change -- * (%SCX_DEQ_SAVE) -- * -- * This and ->dequeue() are related but not coupled. This operation -- * notifies @p's state transition and may not be preceded by ->dequeue() -- * e.g. when @p is being dispatched to a remote CPU. -- */ -- void (*quiescent)(struct task_struct *p, u64 deq_flags); -- -- /** -- * yield - Yield CPU -- * @from: yielding task -- * @to: optional yield target task -- * -- * If @to is NULL, @from is yielding the CPU to other runnable tasks. -- * The BPF scheduler should ensure that other available tasks are -- * dispatched before the yielding task. Return value is ignored in this -- * case. -- * -- * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf -- * scheduler can implement the request, return %true; otherwise, %false. -- */ -- bool (*yield)(struct task_struct *from, struct task_struct *to); -- -- /** -- * core_sched_before - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Used by core-sched to determine the ordering between two tasks. See -- * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on -- * core-sched. -- * -- * Both @a and @b are runnable and may or may not currently be queued on -- * the BPF scheduler. Should return %true if @a should run before @b. -- * %false if there's no required ordering or @b should run before @a. -- * -- * If not specified, the default is ordering them according to when they -- * became runnable. -- */ -- bool (*core_sched_before)(struct task_struct *a,struct task_struct *b); -- -- /** -- * set_weight - Set task weight -- * @p: task to set weight for -- * @weight: new eight [1..10000] -- * -- * Update @p's weight to @weight. -- */ -- void (*set_weight)(struct task_struct *p, u32 weight); -- -- /** -- * set_cpumask - Set CPU affinity -- * @p: task to set CPU affinity for -- * @cpumask: cpumask of cpus that @p can run on -- * -- * Update @p's CPU affinity to @cpumask. -- */ -- void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask); -- -- /** -- * update_idle - Update the idle state of a CPU -- * @cpu: CPU to udpate the idle state for -- * @idle: whether entering or exiting the idle state -- * -- * This operation is called when @rq's CPU goes or leaves the idle -- * state. By default, implementing this operation disables the built-in -- * idle CPU tracking and the following helpers become unavailable: -- * -- * - scx_bpf_select_cpu_dfl() -- * - scx_bpf_test_and_clear_cpu_idle() -- * - scx_bpf_pick_idle_cpu() -- * - scx_bpf_any_idle_cpu() -- * -- * The user also must implement ops.select_cpu() as the default -- * implementation relies on scx_bpf_select_cpu_dfl(). -- * -- * If you keep the built-in idle tracking, specify the -- * %SCX_OPS_KEEP_BUILTIN_IDLE flag. -- */ -- void (*update_idle)(s32 cpu, bool idle); -- -- /** -- * cpu_acquire - A CPU is becoming available to the BPF scheduler -- * @cpu: The CPU being acquired by the BPF scheduler. -- * @args: Acquire arguments, see the struct definition. -- * -- * A CPU that was previously released from the BPF scheduler is now once -- * again under its control. -- */ -- void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); -- -- /** -- * cpu_release - A CPU is taken away from the BPF scheduler -- * @cpu: The CPU being released by the BPF scheduler. -- * @args: Release arguments, see the struct definition. -- * -- * The specified CPU is no longer under the control of the BPF -- * scheduler. This could be because it was preempted by a higher -- * priority sched_class, though there may be other reasons as well. The -- * caller should consult @args->reason to determine the cause. -- */ -- void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -- -- /** -- * cpu_online - A CPU became online -- * @cpu: CPU which just came up -- * -- * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs beforehand. -- */ -- void (*cpu_online)(s32 cpu); -- -- /** -- * cpu_offline - A CPU is going offline -- * @cpu: CPU which is going offline -- * -- * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs afterwards. -- */ -- void (*cpu_offline)(s32 cpu); -- -- /** -- * prep_enable - Prepare to enable BPF scheduling for a task -- * @p: task to prepare BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Either we're loading a BPF scheduler or a new task is being forked. -- * Prepare BPF scheduling for @p. This operation may block and can be -- * used for allocations. -- * -- * Return 0 for success, -errno for failure. An error return while -- * loading will abort loading of the BPF scheduler. During a fork, will -- * abort the specific fork. -- */ -- s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * enable - Enable BPF scheduling for a task -- * @p: task to enable BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Enable @p for BPF scheduling. @p is now in the cgroup specified for -- * the preceding prep_enable() and will start running soon. -- */ -- void (*enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * cancel_enable - Cancel prep_enable() -- * @p: task being canceled -- * @args: enable arguments, see the struct definition -- * -- * @p was prep_enable()'d but failed before reaching enable(). Undo the -- * preparation. -- */ -- void (*cancel_enable)(struct task_struct *p, -- struct scx_enable_args *args); -- -- /** -- * disable - Disable BPF scheduling for a task -- * @p: task to disable BPF scheduling for -- * -- * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. -- * Disable BPF scheduling for @p. -- */ -- void (*disable)(struct task_struct *p); -- -- /* -- * All online ops must come before ops.init(). -- */ -- -- /** -- * init - Initialize the BPF scheduler -- */ -- s32 (*init)(void); -- -- /** -- * exit - Clean up after the BPF scheduler -- * @info: Exit info -- */ -- void (*exit)(struct scx_exit_info *info); -- -- /** -- * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch -- */ -- u32 dispatch_max_batch; -- -- /** -- * flags - %SCX_OPS_* flags -- */ -- u64 flags; -- -- /** -- * timeout_ms - The maximum amount of time, in milliseconds, that a -- * runnable task should be able to wait before being scheduled. The -- * maximum timeout may not exceed the default timeout of 30 seconds. -- * -- * Defaults to the maximum allowed timeout value of 30 seconds. -- */ -- u32 timeout_ms; -- -- /** -- * name - BPF scheduler's name -- * -- * Must be a non-zero valid BPF object name including only isalnum(), -- * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the -- * BPF scheduler is enabled. -- */ -- char name[SCX_OPS_NAME_LEN]; --}; -- --/* -- * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -- * scheduler core and the BPF scheduler. See the documentation for more details. -- */ --struct scx_dispatch_q { -- raw_spinlock_t lock; -- struct list_head fifo; /* processed in dispatching order */ -- struct rb_root_cached priq; /* processed in p->scx.dsq_vtime order */ -- u32 nr; -- u64 id; -- struct llist_node free_node; -- struct rcu_head rcu; --}; -- --/* scx_entity.flags */ --enum scx_ent_flags { -- SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ -- SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -- SCX_TASK_ENQ_LOCAL = 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */ -- -- SCX_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -- SCX_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -- -- SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -- SCX_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -- -- SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ --}; -- --/* scx_entity.dsq_flags */ --enum scx_ent_dsq_flags { -- SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ --}; -- --/* -- * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from -- * everywhere and the following bits track which kfunc sets are currently -- * allowed for %current. This simple per-task tracking works because SCX ops -- * nest in a limited way. BPF will likely implement a way to allow and disallow -- * kfuncs depending on the calling context which will replace this manual -- * mechanism. See scx_kf_allow(). -- */ --enum scx_kf_mask { -- SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ -- /* all non-sleepables may be nested inside INIT and SLEEPABLE */ -- SCX_KF_INIT = 1 << 0, /* running ops.init() */ -- SCX_KF_SLEEPABLE = 1 << 1, /* other sleepable init operations */ -- /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ -- SCX_KF_CPU_RELEASE = 1 << 2, /* ops.cpu_release() */ -- /* ops.dequeue (in REST) may be nested inside DISPATCH */ -- SCX_KF_DISPATCH = 1 << 3, /* ops.dispatch() */ -- SCX_KF_ENQUEUE = 1 << 4, /* ops.enqueue() */ -- SCX_KF_REST = 1 << 5, /* other rq-locked operations */ -- -- __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | -- SCX_KF_ENQUEUE | SCX_KF_REST, -- __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_REST, --}; -- --#define RAVG_HIST_SIZE 5 --struct scx_sched_task_stats { -- u64 mark_start; -- u64 window_start; -- u32 sum; -- u32 sum_history[RAVG_HIST_SIZE]; -- int cidx; -- -- u32 demand; -- u16 demand_scaled; -- void *sdsq; --}; -- -- -- --/* -- * The following is embedded in task_struct and contains all fields necessary -- * for a task to be scheduled by SCX. -- */ --struct sched_ext_entity { -- struct scx_dispatch_q *dsq; -- struct { -- struct list_head fifo; /* dispatch order */ -- struct rb_node priq; /* p->scx.dsq_vtime order */ -- } dsq_node; -- struct list_head watchdog_node; -- u32 flags; /* protected by rq lock */ -- u32 dsq_flags; /* protected by dsq lock */ -- u32 weight; -- s32 sticky_cpu; -- s32 holding_cpu; -- u32 kf_mask; /* see scx_kf_mask above */ -- struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ -- atomic64_t ops_state; -- unsigned long runnable_at; --#ifdef CONFIG_SCHED_CORE -- u64 core_sched_at; /* see scx_prio_less() */ --#endif -- -- /* BPF scheduler modifiable fields */ -- -- /* -- * Runtime budget in nsecs. This is usually set through -- * scx_bpf_dispatch() but can also be modified directly by the BPF -- * scheduler. Automatically decreased by SCX as the task executes. On -- * depletion, a scheduling event is triggered. -- * -- * This value is cleared to zero if the task is preempted by -- * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the -- * task ran. Use p->se.sum_exec_runtime instead. -- */ -- u64 slice; -- -- /* -- * Used to order tasks when dispatching to the vtime-ordered priority -- * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime() -- * but can also be modified directly by the BPF scheduler. Modifying it -- * while a task is queued on a dsq may mangle the ordering and is not -- * recommended. -- */ -- u64 dsq_vtime; -- -- /* -- * If set, reject future sched_setscheduler(2) calls updating the policy -- * to %SCHED_EXT with -%EACCES. -- * -- * If set from ops.prep_enable() and the task's policy is already -- * %SCHED_EXT, which can happen while the BPF scheduler is being loaded -- * or by inhering the parent's policy during fork, the task's policy is -- * rejected and forcefully reverted to %SCHED_NORMAL. The number of such -- * events are reported through /sys/kernel/debug/sched_ext::nr_rejected. -- */ -- bool disallow; /* reject switching into SCX */ -- -- /* cold fields */ -- struct list_head tasks_node; -- struct task_struct *task; -- struct scx_sched_task_stats sts; --}; -- --void sched_ext_free(struct task_struct *p); -- --#else /* !CONFIG_SCHED_CLASS_EXT */ -- --static inline void sched_ext_free(struct task_struct *p) {} -- --#endif /* CONFIG_SCHED_CLASS_EXT */ --#endif /* _LINUX_SCHED_EXT_H */ -diff --git b/include/linux/sched/hmbird_proc_val.h a/include/linux/sched/hmbird_proc_val.h -new file mode 100755 -index 000000000000..e59708b16ea3 ---- /dev/null -+++ a/include/linux/sched/hmbird_proc_val.h -@@ -0,0 +1,22 @@ -+#ifndef __HMBORD_PROC_VAL_H__ -+#define __HMBORD_PROC_VAL_H__ -+ -+extern int scx_enable; -+extern int partial_enable; -+extern int cpuctrl_high_ratio; -+extern int cpuctrl_low_ratio; -+extern int slim_stats; -+extern int hmbirdcore_debug; -+extern int slim_for_app; -+extern int misfit_ds; -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+extern int cpu7_tl; -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int slim_gov_debug; -+extern int scx_gov_ctrl; -+extern int sched_ravg_window_frame_per_sec; -+ -+#endif -\ No newline at end of file -diff --git b/include/linux/sched/hmbird_version.h a/include/linux/sched/hmbird_version.h -new file mode 100755 -index 000000000000..b2843881c26f ---- /dev/null -+++ a/include/linux/sched/hmbird_version.h -@@ -0,0 +1,52 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#ifndef _OPLUS_HMBIRD_VERSION_H_ -+#define _OPLUS_HMBIRD_VERSION_H_ -+#include -+#include -+#include -+ -+enum hmbird_version { -+ HMBIRD_UNINIT, -+ HMBIRD_GKI_VERSION, -+ HMBIRD_OGKI_VERSION, -+ HMBIRD_UNKNOW_VERSION, -+}; -+ -+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT; -+ -+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type" -+ -+static inline enum hmbird_version get_hmbird_version_type(void) -+{ -+ struct device_node *np = NULL; -+ const char *hmbird_version_str = NULL; -+ if (HMBIRD_UNINIT != hmbird_version_type) -+ return hmbird_version_type; -+ np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH); -+ if (np) { -+ of_property_read_string(np, "type", &hmbird_version_str); -+ if (NULL != hmbird_version_str) { -+ if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) { -+ hmbird_version_type = HMBIRD_OGKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi"); -+ } else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) { -+ hmbird_version_type = HMBIRD_GKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi"); -+ } else { -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi"); -+ } -+ return hmbird_version_type; -+ } -+ } -+ -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION"); -+ return hmbird_version_type; -+} -+ -+#endif /*_OPLUS_HMBIRD_VERSION_H_ */ -diff --git b/include/linux/sched/task.h a/include/linux/sched/task.h -index 03d35e3eda3c..6a5f0c0d74bd 100644 ---- b/include/linux/sched/task.h -+++ a/include/linux/sched/task.h -@@ -61,8 +61,10 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); - extern void init_idle(struct task_struct *idle, int cpu); - - extern int sched_fork(unsigned long clone_flags, struct task_struct *p); --extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); --extern void sched_cancel_fork(struct task_struct *p); -+extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); -+#ifdef CONFIG_HMBIRD_SCHED -+extern void hmbird_cancel_fork(struct task_struct *p); -+#endif - extern void sched_post_fork(struct task_struct *p); - extern void sched_dead(struct task_struct *p); - -diff --git b/include/uapi/linux/sched.h a/include/uapi/linux/sched.h -index 359a14cc76a4..969716ab4320 100755 ---- b/include/uapi/linux/sched.h -+++ a/include/uapi/linux/sched.h -@@ -118,7 +118,7 @@ struct clone_args { - /* SCHED_ISO: reserved but not implemented yet */ - #define SCHED_IDLE 5 - #define SCHED_DEADLINE 6 --#define SCHED_EXT 7 -+#define SCHED_HMBIRD 7 - - /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ - #define SCHED_RESET_ON_FORK 0x40000000 -diff --git b/init/Kconfig a/init/Kconfig -index 337276cbc7f8..bf052ca532b4 100644 ---- b/init/Kconfig -+++ a/init/Kconfig -@@ -1030,10 +1030,6 @@ config RT_GROUP_SCHED - realtime bandwidth for them. - See Documentation/scheduler/sched-rt-group.rst for more information. - --config EXT_GROUP_SCHED -- bool -- depends on SCHED_CLASS_EXT && CGROUP_SCHED -- default y - endif #CGROUP_SCHED - - config SCHED_MM_CID -diff --git b/init/init_task.c a/init/init_task.c -index 69a046388995..31ceb0e469f7 100755 ---- b/init/init_task.c -+++ a/init/init_task.c -@@ -6,7 +6,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -102,9 +101,6 @@ struct task_struct init_task - #endif - #ifdef CONFIG_CGROUP_SCHED - .sched_task_group = &root_task_group, --#endif --#ifdef CONFIG_SLIM_SCHED -- .scx = NULL, - #endif - .ptraced = LIST_HEAD_INIT(init_task.ptraced), - .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), -diff --git b/kernel/Kconfig.preempt a/kernel/Kconfig.preempt -index cf22ef6107b8..ca8de6eb1e16 100755 ---- b/kernel/Kconfig.preempt -+++ a/kernel/Kconfig.preempt -@@ -133,12 +133,8 @@ config SCHED_CORE - which is the likely usage by Linux distributions, there should - be no measurable impact on performance. - --config SLIM_SCHED -- bool "slim sched" -- default n --config SCHED_CLASS_EXT -- bool "Extensible Scheduling Class" -- depends on BPF_SYSCALL && BPF_JIT -+config HMBIRD_SCHED -+ bool "hmbird Scheduling Class(base on ext scheduler)" - help - This option enables a new scheduler class sched_ext (SCX), which - allows scheduling policies to be implemented as BPF programs to -diff --git b/kernel/bpf/bpf_struct_ops_types.h a/kernel/bpf/bpf_struct_ops_types.h -index 3618769d853d..5678a9ddf817 100644 ---- b/kernel/bpf/bpf_struct_ops_types.h -+++ a/kernel/bpf/bpf_struct_ops_types.h -@@ -9,8 +9,4 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) - #include - BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) - #endif --#ifdef CONFIG_SCHED_CLASS_EXT --#include --BPF_STRUCT_OPS_TYPE(sched_ext_ops) --#endif - #endif -diff --git b/kernel/fork.c a/kernel/fork.c -index a43f97302332..7a91505b9378 100755 ---- b/kernel/fork.c -+++ a/kernel/fork.c -@@ -23,7 +23,9 @@ - #include - #include - #include --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - #include - #include - #include -@@ -999,8 +1001,9 @@ void __put_task_struct(struct task_struct *tsk) - WARN_ON(!tsk->exit_state); - WARN_ON(refcount_read(&tsk->usage)); - WARN_ON(tsk == current); -- -- sched_ext_free(tsk); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_free(tsk); -+#endif - io_uring_free(tsk); - cgroup_free(tsk); - task_numa_free(tsk, true); -@@ -2517,7 +2520,11 @@ __latent_entropy struct task_struct *copy_process( - - retval = perf_event_init_task(p, clone_flags); - if (retval) -- goto bad_fork_sched_cancel_fork; -+#ifdef CONFIG_HMBIRD_SCHED -+ goto bad_fork_hmbird_cancel_fork; -+#else -+ goto bad_fork_cleanup_policy; -+#endif - retval = audit_alloc(p); - if (retval) - goto bad_fork_cleanup_perf; -@@ -2649,9 +2656,7 @@ __latent_entropy struct task_struct *copy_process( - * cgroup specific, it unconditionally needs to place the task on a - * runqueue. - */ -- retval = sched_cgroup_fork(p, args); -- if (retval) -- goto bad_fork_cancel_cgroup; -+ sched_cgroup_fork(p, args); - - /* - * From this point on we must avoid any synchronous user-space -@@ -2697,13 +2702,13 @@ __latent_entropy struct task_struct *copy_process( - /* Don't start children in a dying pid namespace */ - if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { - retval = -ENOMEM; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* Let kill terminate clone/fork in the middle */ - if (fatal_signal_pending(current)) { - retval = -EINTR; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* No more failure paths after this point. */ -@@ -2780,11 +2785,10 @@ __latent_entropy struct task_struct *copy_process( - - return p; - --bad_fork_core_free: -+bad_fork_cancel_cgroup: - sched_core_free(p); - spin_unlock(¤t->sighand->siglock); - write_unlock_irq(&tasklist_lock); --bad_fork_cancel_cgroup: - cgroup_cancel_fork(p, args); - bad_fork_put_pidfd: - if (clone_flags & CLONE_PIDFD) { -@@ -2823,8 +2827,10 @@ __latent_entropy struct task_struct *copy_process( - audit_free(p); - bad_fork_cleanup_perf: - perf_event_free_task(p); --bad_fork_sched_cancel_fork: -- sched_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+bad_fork_hmbird_cancel_fork: -+ hmbird_cancel_fork(p); -+#endif - bad_fork_cleanup_policy: - lockdep_free_task(p); - #ifdef CONFIG_NUMA -diff --git b/kernel/sched/build_policy.c a/kernel/sched/build_policy.c -index 005025f55bea..c091539a0f60 100755 ---- b/kernel/sched/build_policy.c -+++ a/kernel/sched/build_policy.c -@@ -28,8 +28,10 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED - #include - #include -+#endif - - #include - -@@ -54,6 +56,10 @@ - #include "cputime.c" - #include "deadline.c" - --#ifdef CONFIG_SCHED_CLASS_EXT --# include "ext.c" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/hmbird_util_track.c" -+#include "hmbird/hmbird_sched_proc.c" -+#include "hmbird/hmbird_shadow_tick.c" -+#include "hmbird/hmbird.c" -+#include "hmbird/hmbird_misc.c" - #endif -diff --git b/kernel/sched/core.c a/kernel/sched/core.c -index 25d5703df992..b894417ce7df 100755 ---- b/kernel/sched/core.c -+++ a/kernel/sched/core.c -@@ -96,6 +96,12 @@ - #include "../../io_uring/io-wq.h" - #include "../smpboot.h" - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/slim.h" -+#include "hmbird/hmbird_shadow_tick.h" -+#include "hmbird.h" -+#endif -+ - #include - #include - #include -@@ -170,7 +176,6 @@ __read_mostly int scheduler_running; - - DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); - -- - /* kernel prio, less is more */ - static inline int __task_prio(const struct task_struct *p) - { -@@ -183,11 +188,10 @@ static inline int __task_prio(const struct task_struct *p) - if (p->sched_class == &idle_sched_class) - return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - --#ifdef CONFIG_SCHED_CLASS_EXT -- if (p->sched_class == &ext_sched_class) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (p->sched_class == &hmbird_sched_class) - return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ - #endif -- - return MAX_RT_PRIO + MAX_NICE; /* 119, squash fair */ - } - -@@ -217,9 +221,9 @@ static inline bool prio_less(const struct task_struct *a, - if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ - return cfs_prio_less(a, b, in_fi); - --#ifdef CONFIG_SCHED_CLASS_EXT -+#ifdef CONFIG_HMBIRD_SCHED - if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ -- return scx_prio_less(a, b, in_fi); -+ return hmbird_prio_less(a, b, in_fi); - #endif - - return false; -@@ -1279,17 +1283,26 @@ bool sched_can_stop_tick(struct rq *rq) - if (fifo_nr_running) - return true; - -+#ifdef CONFIG_HMBIRD_SCHED - /* -- * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks -+ * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks - * left. For CFS, if there's more than one we need the tick for -- * involuntary preemption. For SCX, ask. -+ * involuntary preemption. For HMBIRD, ask. - */ -- if (!scx_switched_all() && rq->nr_running > 1) -+ if (!hmbird_enabled() && rq->nr_running > 1) - return false; - -- if (scx_enabled() && !scx_can_stop_tick(rq)) -+ if (hmbird_enabled() && !hmbird_can_stop_tick(rq)) - return false; -- -+#else -+ /* -+ * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; -+ * if there's more than one we need the tick for involuntary -+ * preemption. -+ */ -+ if (rq->nr_running > 1) -+ return false; -+#endif - /* - * If there is one task and it has CFS runtime bandwidth constraints - * and it's on the cpu now we don't want to stop the tick. -@@ -2222,10 +2235,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) - } - EXPORT_SYMBOL_GPL(deactivate_task); - --struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) -+#ifdef CONFIG_HMBIRD_SCHED -+struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - { -- struct sched_change_guard cg = { -+ struct hmbird_sched_change_guard cg = { - .rq = rq, - .p = p, - .queued = task_on_rq_queued(p), -@@ -2247,7 +2261,7 @@ sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - return cg; - } - --void sched_change_guard_fini(struct sched_change_guard *cg, int flags) -+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags) - { - if (cg->queued) - enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); -@@ -2255,6 +2269,7 @@ void sched_change_guard_fini(struct sched_change_guard *cg, int flags) - set_next_task(cg->rq, cg->p); - cg->done = true; - } -+#endif - - static inline int __normal_prio(int policy, int rt_prio, int nice) - { -@@ -2320,9 +2335,9 @@ inline int task_curr(const struct task_struct *p) - * this means any call to check_class_changed() must be followed by a call to - * balance_callback(). - */ --void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio) -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) - { - if (prev_class != p->sched_class) { - if (prev_class->switched_from) -@@ -2885,6 +2900,7 @@ static void - __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - { - struct rq *rq = task_rq(p); -+ bool queued, running; - - /* - * This here violates the locking rules for affinity, since we're only -@@ -2903,9 +2919,26 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - else - lockdep_assert_held(&p->pi_lock); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->sched_class->set_cpus_allowed(p, ctx); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ -+ if (queued) { -+ /* -+ * Because __kthread_bind() calls this on blocked tasks without -+ * holding rq->lock. -+ */ -+ lockdep_assert_rq_held(rq); -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); - } -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->sched_class->set_cpus_allowed(p, ctx); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - } - - /* -@@ -3772,7 +3805,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class)) -+#else - if (unlikely(!is_cpu_allowed(p, cpu))) -+#endif - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -@@ -4891,8 +4928,9 @@ late_initcall(sched_core_sysctl_init); - */ - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { -- -+#ifdef CONFIG_HMBIRD_SCHED - int ret; -+#endif - - trace_android_rvh_sched_fork(p); - -@@ -4933,21 +4971,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_reset_on_fork = 0; - } - -- scx_pre_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ ret = hmbird_pre_fork(p); -+ if (ret) -+ goto out_cancel; - - if (dl_prio(p->prio)) { - ret = -EAGAIN; - goto out_cancel; --#ifdef CONFIG_SCHED_CLASS_EXT -- } else if (task_on_scx(p)) { -- p->sched_class = &ext_sched_class; --#endif -+ } else if (task_on_hmbird(p)) { -+ p->sched_class = &hmbird_sched_class; - } else if (rt_prio(p->prio)) { - p->sched_class = &rt_sched_class; - } else { - p->sched_class = &fair_sched_class; - } -- -+#else -+ if (dl_prio(p->prio)) -+ return -EAGAIN; -+ else if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - init_entity_runnable_average(&p->se); - trace_android_rvh_finish_prio_fork(p); - -@@ -4966,12 +5012,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - #endif - return 0; - -+#ifdef CONFIG_HMBIRD_SCHED - out_cancel: -- scx_cancel_fork(p); -+ hmbird_cancel_fork(p); - return ret; -+#endif - } - --int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) -+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - { - unsigned long flags; - -@@ -4998,20 +5046,14 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -- -- return scx_fork(p); --} -- --void sched_cancel_fork(struct task_struct *p) --{ -- scx_cancel_fork(p); - } - - void sched_post_fork(struct task_struct *p) - { - uclamp_post_fork(p); -- -- scx_post_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_post_fork(p); -+#endif - } - - unsigned long to_ratio(u64 period, u64 runtime) -@@ -5875,21 +5917,28 @@ void scheduler_tick(void) - - if (sched_feat(LATENCY_WARN) && resched_latency) - resched_latency_warn(cpu, resched_latency); -- -- scx_notify_sched_tick(); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_notify_sched_tick(); -+#endif - perf_event_task_tick(); - - if (curr->flags & PF_WQ_WORKER) - wq_worker_tick(curr); - - #ifdef CONFIG_SMP -- if (!scx_switched_all()) { -+#ifdef CONFIG_HMBIRD_SCHED -+ if (!hmbird_enabled()) { -+#endif - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); -+#ifdef CONFIG_HMBIRD_SCHED - } -+#endif - #endif - trace_android_vh_scheduler_tick(rq); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ scheduler_tick_handler(NULL, NULL); -+#endif - } - - #ifdef CONFIG_NO_HZ_FULL -@@ -6191,7 +6240,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ -+#ifdef CONFIG_HMBIRD_SCHED - for_balance_class_range(class, prev->sched_class, &idle_sched_class) { -+#else -+ for_class_range(class, prev->sched_class, &idle_sched_class) { -+#endif - if (class->balance(rq, prev, rf)) - break; - } -@@ -6209,9 +6262,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - const struct sched_class *class; - struct task_struct *p; - -- if (scx_enabled()) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (hmbird_enabled()) - goto restart; -- -+#endif - /* - * Optimization: we know that if all tasks are in the fair class we can - * call that function directly, but only if the @prev task wasn't of a -@@ -6237,13 +6291,21 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - restart: - put_prev_task_balance(rq, prev, rf); - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { - p = class->pick_next_task(rq); - if (p) { -- scx_notify_pick_next_task(rq, p, class); -+ hmbird_notify_pick_next_task(rq, p, class); - return p; - } - } -+#else -+ for_each_class(class) { -+ p = class->pick_next_task(rq); -+ if (p) -+ return p; -+ } -+#endif - - BUG(); /* The idle class should always have a runnable task. */ - } -@@ -6272,7 +6334,11 @@ static inline struct task_struct *pick_task(struct rq *rq) - const struct sched_class *class; - struct task_struct *p; - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { -+#else -+ for_each_class(class) { -+#endif - p = class->pick_task(rq); - if (p) - return p; -@@ -6916,7 +6982,9 @@ static void __sched notrace __schedule(unsigned int sched_mode) - psi_sched_switch(prev, next, !task_on_rq_queued(prev)); - - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#endif - /* Also unlocks the rq: */ - rq = context_switch(rq, prev, next, &rf); - } else { -@@ -7244,24 +7312,31 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag - } - EXPORT_SYMBOL(default_wake_function); - --void __setscheduler_prio(struct task_struct *p, int prio) -+static void __setscheduler_prio(struct task_struct *p, int prio) - { -- /* -- * After switching all rt and fair class to ext, -- * stop class is switched to ext class too. Just -- * skip it. */ -+#ifdef CONFIG_HMBIRD_SCHED -+ bool on_hmbird = task_on_hmbird(p); -+ - if (p->sched_class == &stop_sched_class) -- ;/* do nothing */ -+ ; - else if (dl_prio(prio)) - p->sched_class = &dl_sched_class; --#ifdef CONFIG_SCHED_CLASS_EXT -- else if (task_on_scx(p)) -- p->sched_class = &ext_sched_class; --#endif -+ else if (rt_prio(prio) && on_hmbird) -+ p->sched_class = &hmbird_sched_class; - else if (rt_prio(prio)) - p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; - else - p->sched_class = &fair_sched_class; -+#else -+ if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - - p->prio = prio; - trace_android_rvh_setscheduler_prio(p); -@@ -7297,7 +7372,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - */ - void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - { -- int prio, oldprio, queue_flag = -+ int prio, oldprio, queued, running, queue_flag = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *prev_class; - struct rq_flags rf; -@@ -7360,42 +7435,52 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - queue_flag &= ~DEQUEUE_MOVE; - - prev_class = p->sched_class; -- SCHED_CHANGE_BLOCK(rq, p, queue_flag) { -- /* -- * Boosting condition are: -- * 1. -rt task is running and holds mutex A -- * --> -dl task blocks on mutex A -- * -- * 2. -dl task is running and holds mutex A -- * --> -dl task blocks on mutex A and could preempt the -- * running task -- */ -- if (dl_prio(prio)) { -- if (!dl_prio(p->normal_prio) || -- (pi_task && dl_prio(pi_task->prio) && -- dl_entity_preempt(&pi_task->dl, &p->dl))) { -- p->dl.pi_se = pi_task->dl.pi_se; -- queue_flag |= ENQUEUE_REPLENISH; -- } else { -- p->dl.pi_se = &p->dl; -- } -- } else if (rt_prio(prio)) { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- if (oldprio < prio) -- queue_flag |= ENQUEUE_HEAD; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flag); -+ if (running) -+ put_prev_task(rq, p); -+ -+ /* -+ * Boosting condition are: -+ * 1. -rt task is running and holds mutex A -+ * --> -dl task blocks on mutex A -+ * -+ * 2. -dl task is running and holds mutex A -+ * --> -dl task blocks on mutex A and could preempt the -+ * running task -+ */ -+ if (dl_prio(prio)) { -+ if (!dl_prio(p->normal_prio) || -+ (pi_task && dl_prio(pi_task->prio) && -+ dl_entity_preempt(&pi_task->dl, &p->dl))) { -+ p->dl.pi_se = pi_task->dl.pi_se; -+ queue_flag |= ENQUEUE_REPLENISH; - } else { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- else if (rt_prio(oldprio)) -- p->rt.timeout = 0; -- else if (!task_has_idle_policy(p)) -- reweight_task(p, prio - MAX_RT_PRIO); -+ p->dl.pi_se = &p->dl; - } -- -- __setscheduler_prio(p, prio); -+ } else if (rt_prio(prio)) { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ if (oldprio < prio) -+ queue_flag |= ENQUEUE_HEAD; -+ } else { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ else if (rt_prio(oldprio)) -+ p->rt.timeout = 0; -+ else if (!task_has_idle_policy(p)) -+ reweight_task(p, prio - MAX_RT_PRIO); - } - -+ __setscheduler_prio(p, prio); -+ -+ if (queued) -+ enqueue_task(rq, p, queue_flag); -+ if (running) -+ set_next_task(rq, p); -+ - check_class_changed(rq, p, prev_class, oldprio); - out_unlock: - /* Avoid rq from going away on us: */ -@@ -7416,7 +7501,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - - void set_user_nice(struct task_struct *p, long nice) - { -- bool allowed = false; -+ bool queued, running, allowed = false; - int old_prio; - struct rq_flags rf; - struct rq *rq; -@@ -7445,13 +7530,22 @@ void set_user_nice(struct task_struct *p, long nice) - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); -+ if (running) -+ put_prev_task(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->static_prio = NICE_TO_PRIO(nice); -- set_load_weight(p, true); -- old_prio = p->prio; -- p->prio = effective_prio(p); -- } -+ p->static_prio = NICE_TO_PRIO(nice); -+ set_load_weight(p, true); -+ old_prio = p->prio; -+ p->prio = effective_prio(p); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - - /* - * If the task increased its priority or is running and -@@ -7868,7 +7962,7 @@ static int __sched_setscheduler(struct task_struct *p, - bool user, bool pi) - { - int oldpolicy = -1, policy = attr->sched_policy; -- int retval, oldprio, newprio; -+ int retval, oldprio, newprio, queued, running; - const struct sched_class *prev_class; - struct balance_callback *head; - struct rq_flags rf; -@@ -7876,7 +7970,9 @@ static int __sched_setscheduler(struct task_struct *p, - int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct rq *rq; - bool cpuset_locked = false; -- -+#ifdef CONFIG_HMBIRD_SCHED -+ unsigned long flags; -+#endif - - /* The pi code expects interrupts enabled */ - BUG_ON(pi && in_interrupt()); -@@ -7942,7 +8038,9 @@ static int __sched_setscheduler(struct task_struct *p, - * To be able to change p->policy safely, the appropriate - * runqueue lock must be held. - */ -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+#endif - rq = task_rq_lock(p, &rf); - update_rq_clock(rq); - -@@ -7954,10 +8052,11 @@ static int __sched_setscheduler(struct task_struct *p, - goto unlock; - } - -- retval = scx_check_setscheduler(p, policy); -+#ifdef CONFIG_HMBIRD_SCHED -+ retval = hmbird_check_setscheduler(p, policy); - if (retval) - goto unlock; -- -+#endif - /* - * If not changing anything there's no need to proceed further, - * but store a possible modification of reset_on_fork. -@@ -8014,7 +8113,9 @@ static int __sched_setscheduler(struct task_struct *p, - if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { - policy = oldpolicy = -1; - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - goto recheck; -@@ -8047,16 +8148,23 @@ static int __sched_setscheduler(struct task_struct *p, - queue_flags &= ~DEQUEUE_MOVE; - } - -- SCHED_CHANGE_BLOCK(rq, p, queue_flags) { -- prev_class = p->sched_class; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flags); -+ if (running) -+ put_prev_task(rq, p); -+ -+ prev_class = p->sched_class; - -- if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -- __setscheduler_params(p, attr); -- __setscheduler_prio(p, newprio); -- trace_android_rvh_setscheduler(p); -- } -- __setscheduler_uclamp(p, attr); -+ if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -+ __setscheduler_params(p, attr); -+ __setscheduler_prio(p, newprio); -+ trace_android_rvh_setscheduler(p); -+ } -+ __setscheduler_uclamp(p, attr); - -+ if (queued) { - /* - * We enqueue to tail when the priority of a task is - * increased (user space view). -@@ -8064,7 +8172,10 @@ static int __sched_setscheduler(struct task_struct *p, - if (oldprio < p->prio) - queue_flags |= ENQUEUE_HEAD; - -+ enqueue_task(rq, p, queue_flags); - } -+ if (running) -+ set_next_task(rq, p); - - check_class_changed(rq, p, prev_class, oldprio); - -@@ -8072,7 +8183,9 @@ static int __sched_setscheduler(struct task_struct *p, - preempt_disable(); - head = splice_balance_callbacks(rq); - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (pi) { - if (cpuset_locked) - cpuset_unlock(); -@@ -8087,7 +8200,9 @@ static int __sched_setscheduler(struct task_struct *p, - - unlock: - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - return retval; -@@ -8785,6 +8900,9 @@ static void do_sched_yield(void) - long skip = 0; - - trace_android_rvh_before_do_sched_yield(&skip); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_skip_yield(&skip); -+#endif - if (skip) - return; - -@@ -9309,7 +9427,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - break; - } -@@ -9337,7 +9457,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - } - return ret; -@@ -9638,15 +9760,25 @@ int migrate_task_to(struct task_struct *p, int target_cpu) - */ - void sched_setnuma(struct task_struct *p, int nid) - { -+ bool queued, running; - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) { -- p->numa_preferred_nid = nid; -- } -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE); -+ if (running) -+ put_prev_task(rq, p); - -+ p->numa_preferred_nid = nid; -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - task_rq_unlock(rq, p, &rf); - } - #endif /* CONFIG_NUMA_BALANCING */ -@@ -10212,17 +10344,23 @@ void __init sched_init(void) - int i; - - /* Make sure the linker didn't screw up */ -+#ifdef CONFIG_HMBIRD_SCHED - #ifdef CONFIG_SMP -- BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+#endif -+ WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class)); -+#else -+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 || -+ &fair_sched_class != &rt_sched_class + 1 || -+ &rt_sched_class != &dl_sched_class + 1); -+#ifdef CONFIG_SMP -+ BUG_ON(&dl_sched_class != &stop_sched_class + 1); - #endif -- BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); -- BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); -- BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); --#ifdef CONFIG_SCHED_CLASS_EXT -- BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); -- BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); - #endif -- - wait_bit_init(); - - #ifdef CONFIG_FAIR_GROUP_SCHED -@@ -10392,8 +10530,9 @@ void __init sched_init(void) - balance_push_set(smp_processor_id(), false); - #endif - init_sched_fair_class(); -- init_sched_ext_class(); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ init_sched_hmbird_class(); -+#endif - psi_init(); - - init_uclamp(); -@@ -10863,6 +11002,11 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) - struct task_group *tg = css_tg(css); - struct task_group *parent = css_tg(css->parent); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_tg_online(tg); -+#endif -+ -+ - if (parent) - sched_online_group(tg, parent); - -@@ -10912,13 +11056,77 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) - } - #endif - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 dl) -+{ -+ int i; -+ -+ for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) { -+ if (dl < HMBIRD_BPF_DSQS_DEADLINE[i]) -+ break; -+ } -+ -+ i = max_t(int, i-1, MIN_CGROUP_DL_IDX); -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return 0; -+ update_cgroup_ids_table(css->cgroup->kn->id, i); -+ -+ return 0; -+} -+ -+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ u8 i; -+ -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX]; -+ i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1); -+ if (i < 0) { -+ pr_err(" <%s> i is %d, less than 0, name is %s\n", -+ __func__, i, css->cgroup->kn->name); -+ i = DEFAULT_CGROUP_DL_IDX; -+ } -+ -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[i]; -+} -+ -+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css, -+ struct task_struct *p, int prio) -+{ -+ if (!css || !rt_prio(prio)) -+ return; -+ -+ if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) { -+ if (!strcmp(css->cgroup->kn->name, "display")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3); -+ else if (!strcmp(css->cgroup->kn->name, "touch")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2); -+ else if (!strcmp(css->cgroup->kn->name, "multimedia")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+ } -+} -+#endif -+ - static void cpu_cgroup_attach(struct cgroup_taskset *tset) - { - struct task_struct *task; - struct cgroup_subsys_state *css; - - cgroup_taskset_for_each(task, css, tset) { -- -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_change_rt_sched_prop(css, task, task->prio); -+#endif - sched_move_task(task); - } - -@@ -11600,7 +11808,13 @@ static struct cftype cpu_legacy_files[] = { - .write_u64 = cpu_uclamp_ls_write_u64, - }, - #endif -- -+#ifdef CONFIG_HMBIRD_SCHED -+ { -+ .name = "hmbird.deadline", -+ .read_u64 = cgroup_read_hmbird_deadline, -+ .write_u64 = cgroup_write_hmbird_deadline, -+ }, -+#endif - { } /* Terminate */ - }; - -@@ -11649,7 +11863,6 @@ static int cpu_local_stat_show(struct seq_file *sf, - } - - #ifdef CONFIG_FAIR_GROUP_SCHED -- - static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, - struct cftype *cft) - { -diff --git b/kernel/sched/debug.c a/kernel/sched/debug.c -index a6c99df9edf9..e09904f4d6e5 100755 ---- b/kernel/sched/debug.c -+++ a/kernel/sched/debug.c -@@ -375,9 +375,8 @@ static __init int sched_init_debug(void) - #endif - - debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); -- --#ifdef CONFIG_SCHED_CLASS_EXT -- debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops); -+#ifdef CONFIG_HMBIRD_SCHED -+ debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops); - #endif - return 0; - } -@@ -1006,9 +1005,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - SEQ_printf(m, - "---------------------------------------------------------" - "----------\n"); --#ifdef CONFIG_SLIM_SCHED -- SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop); --#endif - - #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) - #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) -@@ -1104,8 +1100,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - } else if (fair_policy(p->policy)) { - P(se.slice); - } --#ifdef CONFIG_SCHED_CLASS_EXT -- __PS("ext.enabled", p->sched_class == &ext_sched_class); -+#ifdef CONFIG_HMBIRD_SCHED -+ __PS("hmbird.enabled", p->sched_class == &hmbird_sched_class); -+ __PS("sched_prop", get_hmbird_ts(p)->sched_prop); -+ __PS("top_task_prop", get_hmbird_ts(p)->top_task_prop); - #endif - #undef PN_SCHEDSTAT - #undef P_SCHEDSTAT -diff --git b/kernel/sched/ext.c a/kernel/sched/ext.c -deleted file mode 100755 -index 4845d441df2b..000000000000 ---- b/kernel/sched/ext.c -+++ /dev/null -@@ -1,3978 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -- --enum scx_internal_consts { -- SCX_NR_ONLINE_OPS = SCX_OP_IDX(init), -- SCX_DSP_DFL_MAX_BATCH = 32, -- SCX_DSP_MAX_LOOPS = 32, -- SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, --}; -- --enum scx_ops_enable_state { -- SCX_OPS_PREPPING, -- SCX_OPS_ENABLING, -- SCX_OPS_ENABLED, -- SCX_OPS_DISABLING, -- SCX_OPS_DISABLED, --}; -- --static inline struct task_group *css_tg(struct cgroup_subsys_state *css) --{ -- return css ? container_of(css, struct task_group, css) : NULL; --} -- --/* -- * sched_ext_entity->ops_state -- * -- * Used to track the task ownership between the SCX core and the BPF scheduler. -- * State transitions look as follows: -- * -- * NONE -> QUEUEING -> QUEUED -> DISPATCHING -- * ^ | | -- * | v v -- * \-------------------------------/ -- * -- * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -- * sites for explanations on the conditions being waited upon and why they are -- * safe. Transitions out of them into NONE or QUEUED must store_release and the -- * waiters should load_acquire. -- * -- * Tracking scx_ops_state enables sched_ext core to reliably determine whether -- * any given task can be dispatched by the BPF scheduler at all times and thus -- * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -- * to try to dispatch any task anytime regardless of its state as the SCX core -- * can safely reject invalid dispatches. -- */ --enum scx_ops_state { -- SCX_OPSS_NONE, /* owned by the SCX core */ -- SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -- SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ -- SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ -- -- /* -- * QSEQ brands each QUEUED instance so that, when dispatch races -- * dequeue/requeue, the dispatcher can tell whether it still has a claim -- * on the task being dispatched. -- */ -- SCX_OPSS_QSEQ_SHIFT = 2, -- SCX_OPSS_STATE_MASK = (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1, -- SCX_OPSS_QSEQ_MASK = ~SCX_OPSS_STATE_MASK, --}; -- --/* -- * During exit, a task may schedule after losing its PIDs. When disabling the -- * BPF scheduler, we need to be able to iterate tasks in every state to -- * guarantee system safety. Maintain a dedicated task list which contains every -- * task between its fork and eventual free. -- */ --static DEFINE_SPINLOCK(scx_tasks_lock); --static LIST_HEAD(scx_tasks); -- --/* ops enable/disable */ --static struct kthread_worker *scx_ops_helper; --static DEFINE_MUTEX(scx_ops_enable_mutex); --DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); --DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); --static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); --static bool scx_switch_all_req; --static bool scx_switching_all; --DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -- --static struct sched_ext_ops scx_ops; --static bool scx_warned_zero_slice; -- --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); --DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); --static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); -- --struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] = -- { [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT }; -- --static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE); --static struct scx_exit_info scx_exit_info; -- --static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0); -- --/* -- * The maximum amount of time in jiffies that a task may be runnable without -- * being scheduled on a CPU. If this timeout is exceeded, it will trigger -- * scx_ops_error(). -- */ --unsigned long scx_watchdog_timeout; -- --/* -- * The last time the delayed work was run. This delayed work relies on -- * ksoftirqd being able to run to service timer interrupts, so it's possible -- * that this work itself could get wedged. To account for this, we check that -- * it's not stalled in the timer tick, and trigger an error if it is. -- */ --unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; -- --static struct delayed_work scx_watchdog_work; -- --/* idle tracking */ --#ifdef CONFIG_SMP --#ifdef CONFIG_CPUMASK_OFFSTACK --#define CL_ALIGNED_IF_ONSTACK --#else --#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp --#endif -- --static struct { -- cpumask_var_t cpu; -- cpumask_var_t smt; --} idle_masks CL_ALIGNED_IF_ONSTACK; -- --static bool __cacheline_aligned_in_smp scx_has_idle_cpus; --#endif /* CONFIG_SMP */ -- --/* for %SCX_KICK_WAIT */ --static u64 __percpu *scx_kick_cpus_pnt_seqs; -- --/* -- * Direct dispatch marker. -- * -- * Non-NULL values are used for direct dispatch from enqueue path. A valid -- * pointer points to the task currently being enqueued. An ERR_PTR value is used -- * to indicate that direct dispatch has already happened. -- */ --static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); -- --/* dispatch queues */ --static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; -- --static LLIST_HEAD(dsqs_to_free); -- --/* dispatch buf */ --struct scx_dsp_buf_ent { -- struct task_struct *task; -- u64 qseq; -- u64 dsq_id; -- u64 enq_flags; --}; -- --static u32 scx_dsp_max_batch; --static struct scx_dsp_buf_ent __percpu *scx_dsp_buf; -- --struct scx_dsp_ctx { -- struct rq *rq; -- struct rq_flags *rf; -- u32 buf_cursor; -- u32 nr_tasks; --}; -- --static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx); -- --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags); --void scx_bpf_kick_cpu(s32 cpu, u64 flags); -- --struct scx_task_iter { -- struct sched_ext_entity cursor; -- struct task_struct *locked; -- struct rq *rq; -- struct rq_flags rf; --}; -- --#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) -- --/* if the highest set bit is N, return a mask with bits [N+1, 31] set */ --static u32 higher_bits(u32 flags) --{ -- return ~((1 << fls(flags)) - 1); --} -- --/* return the mask with only the highest bit set */ --static u32 highest_bit(u32 flags) --{ -- int bit = fls(flags); -- return bit ? 1 << (bit - 1) : 0; --} -- --/* -- * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX -- * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate -- * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check -- * whether it's running from an allowed context. -- * -- * @mask is constant, always inline to cull the mask calculations. -- */ --static __always_inline void scx_kf_allow(u32 mask) --{ -- /* nesting is allowed only in increasing scx_kf_mask order */ -- WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask, -- "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n", -- current->scx->kf_mask, mask); -- current->scx->kf_mask |= mask; --} -- --static void scx_kf_disallow(u32 mask) --{ -- current->scx->kf_mask &= ~mask; --} -- --#define SCX_CALL_OP(mask, op, args...) \ --do { \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- scx_ops.op(args); \ -- } \ --} while (0) -- --#define SCX_CALL_OP_RET(mask, op, args...) \ --({ \ -- __typeof__(scx_ops.op(args)) __ret; \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- __ret = scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- __ret = scx_ops.op(args); \ -- } \ -- __ret; \ --}) -- --/* -- * Some kfuncs are allowed only on the tasks that are subjects of the -- * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such -- * restrictions, the following SCX_CALL_OP_*() variants should be used when -- * invoking scx_ops operations that take task arguments. These can only be used -- * for non-nesting operations due to the way the tasks are tracked. -- * -- * kfuncs which can only operate on such tasks can in turn use -- * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on -- * the specific task. -- */ --#define SCX_CALL_OP_TASK(mask, op, task, args...) \ --do { \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- SCX_CALL_OP(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ --} while (0) -- --#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ --({ \ -- __typeof__(scx_ops.op(task, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- __ret; \ --}) -- --#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ --({ \ -- __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task0; \ -- current->scx->kf_tasks[1] = task1; \ -- __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- current->scx->kf_tasks[1] = NULL; \ -- __ret; \ --}) -- --//#include "./slim_walt.c" -- --/* @mask is constant, always inline to cull unnecessary branches */ --static __always_inline bool scx_kf_allowed(u32 mask) --{ -- if (unlikely(!(current->scx->kf_mask & mask))) { -- scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", -- mask, current->scx->kf_mask); -- return false; -- } -- -- if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) && -- in_interrupt())) { -- scx_ops_error("sleepable kfunc called from non-sleepable context"); -- return false; -- } -- -- /* -- * Enforce nesting boundaries. e.g. A kfunc which can be called from -- * DISPATCH must not be called if we're running DEQUEUE which is nested -- * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE -- * boundary thanks to the above in_interrupt() check. -- */ -- if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && -- (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { -- scx_ops_error("cpu_release kfunc called from a nested operation"); -- return false; -- } -- -- if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && -- (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) { -- scx_ops_error("dispatch kfunc called from a nested operation"); -- return false; -- } -- -- return true; --} -- --/* see SCX_CALL_OP_TASK() */ --static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, -- struct task_struct *p) --{ -- if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED)) -- return false; -- -- if (unlikely((p != current->scx->kf_tasks[0] && -- p != current->scx->kf_tasks[1]))) { -- scx_ops_error("called on a task not being operated on"); -- return false; -- } -- -- return true; --} -- --/** -- * scx_task_iter_init - Initialize a task iterator -- * @iter: iterator to init -- * -- * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized, -- * @iter must eventually be exited with scx_task_iter_exit(). -- * -- * scx_tasks_lock may be released between this and the first next() call or -- * between any two next() calls. If scx_tasks_lock is released between two -- * next() calls, the caller is responsible for ensuring that the task being -- * iterated remains accessible either through RCU read lock or obtaining a -- * reference count. -- * -- * All tasks which existed when the iteration started are guaranteed to be -- * visited as long as they still exist. -- */ --static void scx_task_iter_init(struct scx_task_iter *iter) --{ -- lockdep_assert_held(&scx_tasks_lock); -- -- iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; -- list_add(&iter->cursor.tasks_node, &scx_tasks); -- iter->locked = NULL; --} -- --/** -- * scx_task_iter_exit - Exit a task iterator -- * @iter: iterator to exit -- * -- * Exit a previously initialized @iter. Must be called with scx_tasks_lock held. -- * If the iterator holds a task's rq lock, that rq lock is released. See -- * scx_task_iter_init() for details. -- */ --static void scx_task_iter_exit(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- if (list_empty(cursor)) -- return; -- -- list_del_init(cursor); --} -- --/** -- * scx_task_iter_next - Next task -- * @iter: iterator to walk -- * -- * Visit the next task. See scx_task_iter_init() for details. -- */ --static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- struct sched_ext_entity *pos; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- list_for_each_entry(pos, cursor, tasks_node) { -- if (&pos->tasks_node == &scx_tasks) -- return NULL; -- if (!(pos->flags & SCX_TASK_CURSOR)) { -- list_move(cursor, &pos->tasks_node); -- return pos->task; -- } -- } -- -- /* can't happen, should always terminate at scx_tasks above */ -- BUG(); --} -- --/** -- * scx_task_iter_next_filtered - Next non-idle task -- * @iter: iterator to walk -- * -- * Visit the next non-idle task. See scx_task_iter_init() for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- while ((p = scx_task_iter_next(iter))) { -- if (!is_idle_task(p)) -- return p; -- } -- return NULL; --} -- --/** -- * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked -- * @iter: iterator to walk -- * -- * Visit the next non-idle task with its rq lock held. See scx_task_iter_init() -- * for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered_locked(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- p = scx_task_iter_next_filtered(iter); -- if (!p) -- return NULL; -- -- iter->rq = task_rq_lock(p, &iter->rf); -- iter->locked = p; -- return p; --} -- --static enum scx_ops_enable_state scx_ops_enable_state(void) --{ -- return atomic_read(&scx_ops_enable_state_var); --} -- --static enum scx_ops_enable_state --scx_ops_set_enable_state(enum scx_ops_enable_state to) --{ -- return atomic_xchg(&scx_ops_enable_state_var, to); --} -- --static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, -- enum scx_ops_enable_state from) --{ -- int from_v = from; -- -- return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); --} -- --static bool scx_ops_disabling(void) --{ -- return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING); --} -- --/** -- * wait_ops_state - Busy-wait the specified ops state to end -- * @p: target task -- * @opss: state to wait the end of -- * -- * Busy-wait for @p to transition out of @opss. This can only be used when the -- * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also -- * has load_acquire semantics to ensure that the caller can see the updates made -- * in the enqueueing and dispatching paths. -- */ --static void wait_ops_state(struct task_struct *p, u64 opss) --{ -- do { -- cpu_relax(); -- } while (atomic64_read_acquire(&p->scx->ops_state) == opss); --} -- --/** -- * ops_cpu_valid - Verify a cpu number -- * @cpu: cpu number which came from a BPF ops -- * -- * @cpu is a cpu number which came from the BPF scheduler and can be any value. -- * Verify that it is in range and one of the possible cpus. -- */ --static bool ops_cpu_valid(s32 cpu) --{ -- return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); --} -- --/** -- * ops_sanitize_err - Sanitize a -errno value -- * @ops_name: operation to blame on failure -- * @err: -errno value to sanitize -- * -- * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return -- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can -- * cause misbehaviors. For an example, a large negative return from -- * ops.prep_enable() triggers an oops when passed up the call chain because the -- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is -- * handled as a pointer. -- */ --static int ops_sanitize_err(const char *ops_name, s32 err) --{ -- if (err < 0 && err >= -MAX_ERRNO) -- return err; -- -- scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); -- return -EPROTO; --} -- --/** -- * touch_core_sched - Update timestamp used for core-sched task ordering -- * @rq: rq to read clock from, must be locked -- * @p: task to update the timestamp for -- * -- * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to -- * implement global or local-DSQ FIFO ordering for core-sched. Should be called -- * when a task becomes runnable and its turn on the CPU ends (e.g. slice -- * exhaustion). -- */ --static void touch_core_sched(struct rq *rq, struct task_struct *p) --{ --#ifdef CONFIG_SCHED_CORE -- /* -- * It's okay to update the timestamp spuriously. Use -- * sched_core_disabled() which is cheaper than enabled(). -- */ -- if (!sched_core_disabled()) -- p->scx->core_sched_at = rq_clock_task(rq); --#endif --} -- --/** -- * touch_core_sched_dispatch - Update core-sched timestamp on dispatch -- * @rq: rq to read clock from, must be locked -- * @p: task being dispatched -- * -- * If the BPF scheduler implements custom core-sched ordering via -- * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO -- * ordering within each local DSQ. This function is called from dispatch paths -- * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect. -- */ --static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- assert_clock_updated(rq); -- --#ifdef CONFIG_SCHED_CORE -- if (SCX_HAS_OP(core_sched_before)) -- touch_core_sched(rq, p); --#endif --} -- --static void update_curr_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- u64 now = rq_clock_task(rq); -- u64 delta_exec; -- -- if (time_before_eq64(now, curr->se.exec_start)) -- return; -- -- delta_exec = now - curr->se.exec_start; -- curr->se.exec_start = now; -- curr->se.sum_exec_runtime += delta_exec; -- account_group_exec_runtime(curr, delta_exec); -- cgroup_account_cputime(curr, delta_exec); -- -- if (curr->scx->slice != SCX_SLICE_INF) { -- curr->scx->slice -= min(curr->scx->slice, delta_exec); -- if (!curr->scx->slice) -- touch_core_sched(rq, curr); -- } --} -- --static bool scx_dsq_priq_less(struct rb_node *node_a, -- const struct rb_node *node_b) --{ -- const struct sched_ext_entity *a = -- container_of(node_a, struct sched_ext_entity, dsq_node.priq); -- const struct sched_ext_entity *b = -- container_of(node_b, struct sched_ext_entity, dsq_node.priq); -- -- return time_before64(a->dsq_vtime, b->dsq_vtime); --} -- --static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, -- u64 enq_flags) --{ -- bool is_local = dsq->id == SCX_DSQ_LOCAL; -- -- WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo)); -- WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq)); -- -- if (!is_local) { -- raw_spin_lock(&dsq->lock); -- if (unlikely(dsq->id == SCX_DSQ_INVALID)) { -- scx_ops_error("attempting to dispatch to a destroyed dsq"); -- /* fall back to the global dsq */ -- raw_spin_unlock(&dsq->lock); -- dsq = &scx_dsq_global; -- raw_spin_lock(&dsq->lock); -- } -- } -- -- if (enq_flags & SCX_ENQ_DSQ_PRIQ) { -- p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; -- rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq, -- scx_dsq_priq_less); -- } else { -- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) -- list_add(&p->scx->dsq_node.fifo, &dsq->fifo); -- else -- list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo); -- } -- dsq->nr++; -- p->scx->dsq = dsq; -- -- /* -- * We're transitioning out of QUEUEING or DISPATCHING. store_release to -- * match waiters' load_acquire. -- */ -- if (enq_flags & SCX_ENQ_CLEAR_OPSS) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- if (is_local) { -- struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq); -- struct rq *rq = scx->rq; -- bool preempt = false; -- -- if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && -- rq->curr->sched_class == &ext_sched_class) { -- rq->curr->scx->slice = 0; -- preempt = true; -- } -- -- if (preempt || sched_class_above(&ext_sched_class, -- rq->curr->sched_class)) -- resched_curr(rq); -- } else { -- raw_spin_unlock(&dsq->lock); -- } --} -- --static void task_unlink_from_dsq(struct task_struct *p, -- struct scx_dispatch_q *dsq) --{ -- if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { -- rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; -- } else { -- list_del_init(&p->scx->dsq_node.fifo); -- } -- --} -- --static bool task_linked_on_dsq(struct task_struct *p) --{ -- return !list_empty(&p->scx->dsq_node.fifo) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq); --} -- --static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p) --{ -- struct scx_dispatch_q *dsq = p->scx->dsq; -- bool is_local = dsq == &scx_rq->local_dsq; -- -- if (!dsq) { -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- /* -- * When dispatching directly from the BPF scheduler to a local -- * DSQ, the task isn't associated with any DSQ but -- * @p->scx->holding_cpu may be set under the protection of -- * %SCX_OPSS_DISPATCHING. -- */ -- if (p->scx->holding_cpu >= 0) -- p->scx->holding_cpu = -1; -- return; -- } -- -- if (!is_local) -- raw_spin_lock(&dsq->lock); -- -- /* -- * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node -- * can't change underneath us. -- */ -- if (p->scx->holding_cpu < 0) { -- /* @p must still be on @dsq, dequeue */ -- WARN_ON_ONCE(!task_linked_on_dsq(p)); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- } else { -- /* -- * We're racing against dispatch_to_local_dsq() which already -- * removed @p from @dsq and set @p->scx->holding_cpu. Clear the -- * holding_cpu which tells dispatch_to_local_dsq() that it lost -- * the race. -- */ -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- p->scx->holding_cpu = -1; -- } -- p->scx->dsq = NULL; -- -- if (!is_local) -- raw_spin_unlock(&dsq->lock); --} -- --static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) --{ -- lockdep_assert(rcu_read_lock_any_held()); -- -- return &scx_dsq_global; --} -- --static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, -- struct task_struct *p) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id == SCX_DSQ_LOCAL) -- return &rq->scx->local_dsq; -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", -- dsq_id, p->comm, p->pid); -- return &scx_dsq_global; -- } -- -- return dsq; --} -- --static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- -- /* @p must match the task which is being enqueued */ -- if (unlikely(p != ddsp_task)) { -- if (IS_ERR(ddsp_task)) -- scx_ops_error("%s[%d] already direct-dispatched", -- p->comm, p->pid); -- else -- scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]", -- ddsp_task->comm, ddsp_task->pid, -- p->comm, p->pid); -- return; -- } -- -- /* -- * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because -- * dispatching to the local DSQ of a different CPU requires unlocking -- * the current rq which isn't allowed in the enqueue path. Use -- * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL. -- */ -- if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) { -- scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch"); -- return; -- } -- -- touch_core_sched_dispatch(task_rq(p), p); -- -- dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- -- /* -- * Mark that dispatch already happened by spoiling direct_dispatch_task -- * with a non-NULL value which can never match a valid task pointer. -- */ -- __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); --} -- --static bool test_rq_online(struct rq *rq) --{ --#ifdef CONFIG_SMP -- return rq->online; --#else -- return true; --#endif --} -- --static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- int sticky_cpu) --{ -- struct task_struct **ddsp_taskp; -- u64 qseq; -- -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- if (p->scx->flags & SCX_TASK_ENQ_LOCAL) { -- enq_flags |= SCX_ENQ_LOCAL; -- p->scx->flags &= ~SCX_TASK_ENQ_LOCAL; -- } -- -- /* rq migration */ -- if (sticky_cpu == cpu_of(rq)) -- goto local_norefill; -- -- /* -- * If !rq->online, we already told the BPF scheduler that the CPU is -- * offline. We're just trying to on/offline the CPU. Don't bother the -- * BPF scheduler. -- */ -- if (unlikely(!test_rq_online(rq))) -- goto local; -- -- /* see %SCX_OPS_ENQ_EXITING */ -- if (!static_branch_unlikely(&scx_ops_enq_exiting) && -- unlikely(p->flags & PF_EXITING)) -- goto local; -- -- /* see %SCX_OPS_ENQ_LAST */ -- if (!static_branch_unlikely(&scx_ops_enq_last) && -- (enq_flags & SCX_ENQ_LAST)) -- goto local; -- -- if (!SCX_HAS_OP(enqueue)) { -- if (enq_flags & SCX_ENQ_LOCAL) -- goto local; -- else -- goto global; -- } -- -- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ -- qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; -- -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq); -- -- ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); -- WARN_ON_ONCE(*ddsp_taskp); -- *ddsp_taskp = p; -- -- SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); -- -- /* -- * If not directly dispatched, QUEUEING isn't clear yet and dispatch or -- * dequeue may be waiting. The store_release matches their load_acquire. -- */ -- if (*ddsp_taskp == p) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq); -- *ddsp_taskp = NULL; -- return; -- --local: -- /* -- * For task-ordering, slice refill must be treated as implying the end -- * of the current slice. Otherwise, the longer @p stays on the CPU, the -- * higher priority it becomes from scx_prio_less()'s POV. -- */ -- touch_core_sched(rq, p); -- p->scx->slice = SCX_SLICE_DFL; --local_norefill: -- dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags); -- return; -- --global: -- touch_core_sched(rq, p); /* see the comment in local: */ -- p->scx->slice = SCX_SLICE_DFL; -- dispatch_enqueue(&scx_dsq_global, p, enq_flags); --} -- --static bool watchdog_task_watched(const struct task_struct *p) --{ -- return !list_empty(&p->scx->watchdog_node); --} -- --static void watchdog_watch_task(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- if (p->scx->flags & SCX_TASK_WATCHDOG_RESET) -- p->scx->runnable_at = jiffies; -- p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET; -- list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list); --} -- --static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) --{ -- list_del_init(&p->scx->watchdog_node); -- if (reset_timeout) -- p->scx->flags |= SCX_TASK_WATCHDOG_RESET; --} -- --static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) --{ -- int sticky_cpu = p->scx->sticky_cpu; -- -- enq_flags |= rq->scx->extra_enq_flags; -- -- if (sticky_cpu >= 0) -- p->scx->sticky_cpu = -1; -- -- /* -- * Restoring a running task will be immediately followed by -- * set_next_task_scx() which expects the task to not be on the BPF -- * scheduler as tasks can only start running through local DSQs. Force -- * direct-dispatch into the local DSQ by setting the sticky_cpu. -- */ -- if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -- sticky_cpu = cpu_of(rq); -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- WARN_ON_ONCE(!watchdog_task_watched(p)); -- return; -- } -- -- watchdog_watch_task(rq, p); -- p->scx->flags |= SCX_TASK_QUEUED; -- rq->scx->nr_running++; -- add_nr_running(rq, 1); -- -- if (SCX_HAS_OP(runnable)) -- SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); -- -- if (enq_flags & SCX_ENQ_WAKEUP) -- touch_core_sched(rq, p); -- -- do_enqueue_task(rq, p, enq_flags, sticky_cpu); --} -- --static void ops_dequeue(struct task_struct *p, u64 deq_flags) --{ -- u64 opss; -- -- watchdog_unwatch_task(p, false); -- -- /* acquire ensures that we see the preceding updates on QUEUED */ -- opss = atomic64_read_acquire(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_NONE: -- break; -- case SCX_OPSS_QUEUEING: -- /* -- * QUEUEING is started and finished while holding @p's rq lock. -- * As we're holding the rq lock now, we shouldn't see QUEUEING. -- */ -- BUG(); -- case SCX_OPSS_QUEUED: -- if (SCX_HAS_OP(dequeue)) -- SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); -- -- if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_NONE)) -- break; -- fallthrough; -- case SCX_OPSS_DISPATCHING: -- /* -- * If @p is being dispatched from the BPF scheduler to a DSQ, -- * wait for the transfer to complete so that @p doesn't get -- * added to its DSQ after dequeueing is complete. -- * -- * As we're waiting on DISPATCHING with the rq locked, the -- * dispatching side shouldn't try to lock the rq while -- * DISPATCHING is set. See dispatch_to_local_dsq(). -- * -- * DISPATCHING shouldn't have qseq set and control can reach -- * here with NONE @opss from the above QUEUED case block. -- * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. -- */ -- wait_ops_state(p, SCX_OPSS_DISPATCHING); -- BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- break; -- } --} -- --static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) --{ -- struct scx_rq *scx_rq = rq->scx; -- -- if (!(p->scx->flags & SCX_TASK_QUEUED)) { -- WARN_ON_ONCE(watchdog_task_watched(p)); -- return; -- } -- -- ops_dequeue(p, deq_flags); -- -- /* -- * A currently running task which is going off @rq first gets dequeued -- * and then stops running. As we want running <-> stopping transitions -- * to be contained within runnable <-> quiescent transitions, trigger -- * ->stopping() early here instead of in put_prev_task_scx(). -- * -- * @p may go through multiple stopping <-> running transitions between -- * here and put_prev_task_scx() if task attribute changes occur while -- * balance_scx() leaves @rq unlocked. However, they don't contain any -- * information meaningful to the BPF scheduler and can be suppressed by -- * skipping the callbacks if the task is !QUEUED. -- */ -- if (SCX_HAS_OP(stopping) && task_current(rq, p)) { -- update_curr_scx(rq); -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); -- } -- -- //if(task_current(rq, p)) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- if (SCX_HAS_OP(quiescent)) -- SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); -- -- if (deq_flags & SCX_DEQ_SLEEP) -- p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP; -- else -- p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP; -- -- p->scx->flags &= ~SCX_TASK_QUEUED; -- BUG_ON(!scx_rq->nr_running); -- scx_rq->nr_running--; -- sub_nr_running(rq, 1); -- -- dispatch_dequeue(scx_rq, p); --} -- --static void yield_task_scx(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); -- else -- p->scx->slice = 0; --} -- --static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) --{ -- struct task_struct *from = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); -- else -- return false; --} -- --#ifdef CONFIG_SMP --/** -- * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -- * @rq: rq to move the task into, currently locked -- * @p: task to move -- * @enq_flags: %SCX_ENQ_* -- * -- * Move @p which is currently on a different rq to @rq's local DSQ. The caller -- * must: -- * -- * 1. Start with exclusive access to @p either through its DSQ lock or -- * %SCX_OPSS_DISPATCHING flag. -- * -- * 2. Set @p->scx->holding_cpu to raw_smp_processor_id(). -- * -- * 3. Remember task_rq(@p). Release the exclusive access so that we don't -- * deadlock with dequeue. -- * -- * 4. Lock @rq and the task_rq from #3. -- * -- * 5. Call this function. -- * -- * Returns %true if @p was successfully moved. %false after racing dequeue and -- * losing. -- */ --static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -- u64 enq_flags) --{ -- struct rq *task_rq; -- -- lockdep_assert_rq_held(rq); -- -- /* -- * If dequeue got to @p while we were trying to lock both rq's, it'd -- * have cleared @p->scx->holding_cpu to -1. While other cpus may have -- * updated it to different values afterwards, as this operation can't be -- * preempted or recurse, @p->scx->holding_cpu can never become -- * raw_smp_processor_id() again before we're done. Thus, we can tell -- * whether we lost to dequeue by testing whether @p->scx->holding_cpu is -- * still raw_smp_processor_id(). -- * -- * See dispatch_dequeue() for the counterpart. -- */ -- if (unlikely(p->scx->holding_cpu != raw_smp_processor_id())) -- return false; -- -- /* @p->rq couldn't have changed if we're still the holding cpu */ -- task_rq = task_rq(p); -- lockdep_assert_rq_held(task_rq); -- -- WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)); -- deactivate_task(task_rq, p, 0); -- set_task_cpu(p, cpu_of(rq)); -- p->scx->sticky_cpu = cpu_of(rq); -- -- /* -- * We want to pass scx-specific enq_flags but activate_task() will -- * truncate the upper 32 bit. As we own @rq, we can pass them through -- * @rq->scx->extra_enq_flags instead. -- */ -- WARN_ON_ONCE(rq->scx->extra_enq_flags); -- rq->scx->extra_enq_flags = enq_flags; -- activate_task(rq, p, 0); -- rq->scx->extra_enq_flags = 0; -- -- return true; --} -- --/** -- * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * We're holding @rq lock and trying to dispatch a task from @src_rq to -- * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether -- * @rq stays locked isn't important as long as the state is restored after -- * dispatch_to_local_dsq_unlock(). -- */ --static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- rq_unpin_lock(rq, rf); -- -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(rq); -- raw_spin_rq_lock(dst_rq); -- } else if (rq == src_rq) { -- double_lock_balance(rq, dst_rq); -- rq_repin_lock(rq, rf); -- } else if (rq == dst_rq) { -- double_lock_balance(rq, src_rq); -- rq_repin_lock(rq, rf); -- } else { -- raw_spin_rq_unlock(rq); -- double_rq_lock(src_rq, dst_rq); -- } --} -- --/** -- * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock() -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return. -- */ --static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } else if (rq == src_rq) { -- double_unlock_balance(rq, dst_rq); -- } else if (rq == dst_rq) { -- double_unlock_balance(rq, src_rq); -- } else { -- double_rq_unlock(src_rq, dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } --} --#endif /* CONFIG_SMP */ -- -- --static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq) --{ -- return likely(test_rq_online(rq)) && !is_migration_disabled(p) && -- cpumask_test_cpu(cpu_of(rq), p->cpus_ptr); --} -- --static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -- struct scx_dispatch_q *dsq) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rb_node *rb_node; -- struct rq *task_rq; -- bool moved = false; --retry: -- if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -- return false; -- -- raw_spin_lock(&dsq->lock); -- -- list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- for (rb_node = rb_first_cached(&dsq->priq); rb_node; -- rb_node = rb_next(rb_node)) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- raw_spin_unlock(&dsq->lock); -- return false; -- --this_rq: -- /* @dsq is locked and @p is on this rq */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo); -- dsq->nr--; -- scx_rq->local_dsq.nr++; -- p->scx->dsq = &scx_rq->local_dsq; -- raw_spin_unlock(&dsq->lock); -- return true; -- --remote_rq: --#ifdef CONFIG_SMP -- /* -- * @dsq is locked and @p is on a remote rq. @p is currently protected by -- * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -- * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -- * rq lock or fail, do a little dancing from our side. See -- * move_task_to_local_dsq(). -- */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- p->scx->holding_cpu = raw_smp_processor_id(); -- raw_spin_unlock(&dsq->lock); -- -- rq_unpin_lock(rq, rf); -- double_lock_balance(rq, task_rq); -- rq_repin_lock(rq, rf); -- -- moved = move_task_to_local_dsq(rq, p, 0); -- -- double_unlock_balance(rq, task_rq); --#endif /* CONFIG_SMP */ -- if (likely(moved)) -- return true; -- goto retry; --} -- --enum dispatch_to_local_dsq_ret { -- DTL_DISPATCHED, /* successfully dispatched */ -- DTL_LOST, /* lost race to dequeue */ -- DTL_NOT_LOCAL, /* destination is not a local DSQ */ -- DTL_INVALID, /* invalid local dsq_id */ --}; -- --/** -- * dispatch_to_local_dsq - Dispatch a task to a local dsq -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @dsq_id: destination dsq ID -- * @p: task to dispatch -- * @enq_flags: %SCX_ENQ_* -- * -- * We're holding @rq lock and want to dispatch @p to the local DSQ identified by -- * @dsq_id. This function performs all the synchronization dancing needed -- * because local DSQs are protected with rq locks. -- * -- * The caller must have exclusive ownership of @p (e.g. through -- * %SCX_OPSS_DISPATCHING). -- */ --static enum dispatch_to_local_dsq_ret --dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id, -- struct task_struct *p, u64 enq_flags) --{ -- struct rq *src_rq = task_rq(p); -- struct rq *dst_rq; -- -- /* -- * We're synchronized against dequeue through DISPATCHING. As @p can't -- * be dequeued, its task_rq and cpus_allowed are stable too. -- */ -- if (dsq_id == SCX_DSQ_LOCAL) { -- dst_rq = rq; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]", -- cpu, p->comm, p->pid); -- return DTL_INVALID; -- } -- dst_rq = cpu_rq(cpu); -- } else { -- return DTL_NOT_LOCAL; -- } -- -- /* if dispatching to @rq that @p is already on, no lock dancing needed */ -- if (rq == src_rq && rq == dst_rq) { -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags | SCX_ENQ_CLEAR_OPSS); -- return DTL_DISPATCHED; -- } -- --#ifdef CONFIG_SMP -- if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { -- struct rq *locked_dst_rq = dst_rq; -- bool dsp; -- -- /* -- * @p is on a possibly remote @src_rq which we need to lock to -- * move the task. If dequeue is in progress, it'd be locking -- * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq -- * lock while holding DISPATCHING. -- * -- * As DISPATCHING guarantees that @p is wholly ours, we can -- * pretend that we're moving from a DSQ and use the same -- * mechanism - mark the task under transfer with holding_cpu, -- * release DISPATCHING and then follow the same protocol. -- */ -- p->scx->holding_cpu = raw_smp_processor_id(); -- -- /* store_release ensures that dequeue sees the above */ -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq); -- -- /* -- * We don't require the BPF scheduler to avoid dispatching to -- * offline CPUs mostly for convenience but also because CPUs can -- * go offline between scx_bpf_dispatch() calls and here. If @p -- * is destined to an offline CPU, queue it on its current CPU -- * instead, which should always be safe. As this is an allowed -- * behavior, don't trigger an ops error. -- */ -- if (unlikely(!test_rq_online(dst_rq))) -- dst_rq = src_rq; -- -- if (src_rq == dst_rq) { -- /* -- * As @p is staying on the same rq, there's no need to -- * go through the full deactivate/activate cycle. -- * Optimize by abbreviating the operations in -- * move_task_to_local_dsq(). -- */ -- dsp = p->scx->holding_cpu == raw_smp_processor_id(); -- if (likely(dsp)) { -- p->scx->holding_cpu = -1; -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags); -- } -- } else { -- dsp = move_task_to_local_dsq(dst_rq, p, enq_flags); -- } -- -- /* if the destination CPU is idle, wake it up */ -- if (dsp && p->sched_class > dst_rq->curr->sched_class) -- resched_curr(dst_rq); -- -- dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq); -- -- return dsp ? DTL_DISPATCHED : DTL_LOST; -- } --#endif /* CONFIG_SMP */ -- -- scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", -- cpu_of(dst_rq), p->comm, p->pid); -- return DTL_INVALID; --} -- --/** -- * finish_dispatch - Asynchronously finish dispatching a task -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @p: task to finish dispatching -- * @qseq_at_dispatch: qseq when @p started getting dispatched -- * @dsq_id: destination DSQ ID -- * @enq_flags: %SCX_ENQ_* -- * -- * Dispatching to local DSQs may need to wait for queueing to complete or -- * require rq lock dancing. As we don't wanna do either while inside -- * ops.dispatch() to avoid locking order inversion, we split dispatching into -- * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the -- * task and its qseq. Once ops.dispatch() returns, this function is called to -- * finish up. -- * -- * There is no guarantee that @p is still valid for dispatching or even that it -- * was valid in the first place. Make sure that the task is still owned by the -- * BPF scheduler and claim the ownership before dispatching. -- */ --static void finish_dispatch(struct rq *rq, struct rq_flags *rf, -- struct task_struct *p, u64 qseq_at_dispatch, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- u64 opss; -- -- touch_core_sched_dispatch(rq, p); --retry: -- /* -- * No need for _acquire here. @p is accessed only after a successful -- * try_cmpxchg to DISPATCHING. -- */ -- opss = atomic64_read(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_DISPATCHING: -- case SCX_OPSS_NONE: -- /* someone else already got to it */ -- return; -- case SCX_OPSS_QUEUED: -- /* -- * If qseq doesn't match, @p has gone through at least one -- * dispatch/dequeue and re-enqueue cycle between -- * scx_bpf_dispatch() and here and we have no claim on it. -- */ -- if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) -- return; -- -- /* -- * While we know @p is accessible, we don't yet have a claim on -- * it - the BPF scheduler is allowed to dispatch tasks -- * spuriously and there can be a racing dequeue attempt. Let's -- * claim @p by atomically transitioning it from QUEUED to -- * DISPATCHING. -- */ -- if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_DISPATCHING))) -- break; -- goto retry; -- case SCX_OPSS_QUEUEING: -- /* -- * do_enqueue_task() is in the process of transferring the task -- * to the BPF scheduler while holding @p's rq lock. As we aren't -- * holding any kernel or BPF resource that the enqueue path may -- * depend upon, it's safe to wait. -- */ -- wait_ops_state(p, opss); -- goto retry; -- } -- -- BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) { -- case DTL_DISPATCHED: -- break; -- case DTL_LOST: -- break; -- case DTL_INVALID: -- dsq_id = SCX_DSQ_GLOBAL; -- fallthrough; -- case DTL_NOT_LOCAL: -- dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()), -- dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- break; -- } --} -- --static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- u32 u; -- -- for (u = 0; u < dspc->buf_cursor; u++) { -- struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u]; -- -- finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id, -- ent->enq_flags); -- } -- -- dspc->nr_tasks += dspc->buf_cursor; -- dspc->buf_cursor = 0; --} -- --static int balance_one(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf, bool local) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- bool prev_on_scx = prev->sched_class == &ext_sched_class; -- int nr_loops = SCX_DSP_MAX_LOOPS; -- -- lockdep_assert_rq_held(rq); -- -- if (static_branch_unlikely(&scx_ops_cpu_preempt) && -- unlikely(rq->scx->cpu_released)) { -- /* -- * If the previous sched_class for the current CPU was not SCX, -- * notify the BPF scheduler that it again has control of the -- * core. This callback complements ->cpu_release(), which is -- * emitted in scx_notify_pick_next_task(). -- */ -- if (SCX_HAS_OP(cpu_acquire)) -- SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq), -- NULL); -- rq->scx->cpu_released = false; -- } -- -- if (prev_on_scx) { -- WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP)); -- update_curr_scx(rq); -- -- /* -- * If @prev is runnable & has slice left, it has priority and -- * fetching more just increases latency for the fetched tasks. -- * Tell put_prev_task_scx() to put @prev on local_dsq. If the -- * BPF scheduler wants to handle this explicitly, it should -- * implement ->cpu_released(). -- * -- * See scx_ops_disable_workfn() for the explanation on the -- * disabling() test. -- * -- * When balancing a remote CPU for core-sched, there won't be a -- * following put_prev_task_scx() call and we don't own -- * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the -- * same conditions later and pick @rq->curr accordingly. -- */ -- if ((prev->scx->flags & SCX_TASK_QUEUED) && -- prev->scx->slice && !scx_ops_disabling()) { -- if (local) -- prev->scx->flags |= SCX_TASK_BAL_KEEP; -- return 1; -- } -- } -- -- /* if there already are tasks to run, nothing to do */ -- if (scx_rq->local_dsq.nr) -- return 1; -- -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- if (!SCX_HAS_OP(dispatch)) -- return 0; -- -- dspc->rq = rq; -- dspc->rf = rf; -- -- /* -- * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, -- * the local DSQ might still end up empty after a successful -- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() -- * produced some tasks, retry. The BPF scheduler may depend on this -- * looping behavior to simplify its implementation. -- */ -- do { -- dspc->nr_tasks = 0; -- -- SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), -- prev_on_scx ? prev : NULL); -- -- flush_dispatch_buf(rq, rf); -- -- if (scx_rq->local_dsq.nr) -- return 1; -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- /* -- * ops.dispatch() can trap us in this loop by repeatedly -- * dispatching ineligible tasks. Break out once in a while to -- * allow the watchdog to run. As IRQ can't be enabled in -- * balance(), we want to complete this scheduling cycle and then -- * start a new one. IOW, we want to call resched_curr() on the -- * next, most likely idle, task, not the current one. Use -- * scx_bpf_kick_cpu() for deferred kicking. -- */ -- if (unlikely(!--nr_loops)) { -- scx_bpf_kick_cpu(cpu_of(rq), 0); -- break; -- } -- } while (dspc->nr_tasks); -- -- return 0; --} -- --static int balance_scx(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf) --{ -- int ret; -- -- ret = balance_one(rq, prev, rf, true); --#ifdef CONFIG_SCHED_SMT -- /* -- * When core-sched is enabled, this ops.balance() call will be followed -- * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx() -- * on the SMT siblings. Balance the siblings too. -- */ -- if (sched_core_enabled(rq)) { -- const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); -- int scpu; -- -- for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { -- struct rq *srq = cpu_rq(scpu); -- struct rq_flags srf; -- struct task_struct *sprev = srq->curr; -- -- /* -- * While core-scheduling, rq lock is shared among -- * siblings but the debug annotations and rq clock -- * aren't. Do pinning dance to transfer the ownership. -- */ -- WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); -- rq_unpin_lock(rq, rf); -- rq_pin_lock(srq, &srf); -- -- update_rq_clock(srq); -- balance_one(srq, sprev, &srf, false); -- -- rq_unpin_lock(srq, &srf); -- rq_repin_lock(rq, rf); -- } -- } --#endif -- return ret; --} -- --static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) --{ -- if (p->scx->flags & SCX_TASK_QUEUED) { -- /* -- * Core-sched might decide to execute @p before it is -- * dispatched. Call ops_dequeue() to notify the BPF scheduler. -- */ -- ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); -- dispatch_dequeue(rq->scx, p); -- } -- -- p->se.exec_start = rq_clock_task(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, running, p); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock); -- -- watchdog_unwatch_task(p, true); -- -- /* -- * @p is getting newly scheduled or got kicked after someone updated its -- * slice. Refresh whether tick can be stopped. See can_stop_tick_scx(). -- */ -- if ((p->scx->slice == SCX_SLICE_INF) != -- (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) { -- if (p->scx->slice == SCX_SLICE_INF) -- rq->scx->flags |= SCX_RQ_CAN_STOP_TICK; -- else -- rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK; -- -- sched_update_tick_dependency(rq); -- } --} -- --static void put_prev_task_scx(struct rq *rq, struct task_struct *p) --{ --#ifndef CONFIG_SMP -- /* -- * UP workaround. -- * -- * Because SCX may transfer tasks across CPUs during dispatch, dispatch -- * is performed from its balance operation which isn't called in UP. -- * Let's work around by calling it from the operations which come right -- * after. -- * -- * 1. If the prev task is on SCX, pick_next_task() calls -- * .put_prev_task() right after. As .put_prev_task() is also called -- * from other places, we need to distinguish the calls which can be -- * done by looking at the previous task's state - if still queued or -- * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task(). -- * This case is handled here. -- * -- * 2. If the prev task is not on SCX, the first following call into SCX -- * will be .pick_next_task(), which is covered by calling -- * balance_scx() from pick_next_task_scx(). -- * -- * Note that we can't merge the first case into the second as -- * balance_scx() must be called before the previous SCX task goes -- * through put_prev_task_scx(). -- * -- * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf. -- * Pass in %NULL. -- */ -- if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP)) -- balance_scx(rq, p, NULL); --#endif -- -- update_curr_scx(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- -- /* -- * If we're being called from put_prev_task_balance(), balance_scx() may -- * have decided that @p should keep running. -- */ -- if (p->scx->flags & SCX_TASK_BAL_KEEP) { -- p->scx->flags &= ~SCX_TASK_BAL_KEEP; -- watchdog_watch_task(rq, p); -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- watchdog_watch_task(rq, p); -- -- /* -- * If @p has slice left and balance_scx() didn't tag it for -- * keeping, @p is getting preempted by a higher priority -- * scheduler class or core-sched forcing a different task. Leave -- * it at the head of the local DSQ. -- */ -- if (p->scx->slice && !scx_ops_disabling()) { -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- /* -- * If we're in the pick_next_task path, balance_scx() should -- * have already populated the local DSQ if there are any other -- * available tasks. If empty, tell ops.enqueue() that @p is the -- * only one available for this cpu. ops.enqueue() should put it -- * on the local DSQ so that the subsequent pick_next_task_scx() -- * can find the task unless it wants to trigger a separate -- * follow-up scheduling event. -- */ -- if (list_empty(&rq->scx->local_dsq.fifo)) -- do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1); -- else -- do_enqueue_task(rq, p, 0, -1); -- } --} -- --static struct task_struct *first_local_task(struct rq *rq) --{ -- struct rb_node *rb_node; -- struct sched_ext_entity *entity; -- -- if (!list_empty(&rq->scx->local_dsq.fifo)) { -- entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo); -- return entity->task; -- } -- -- rb_node = rb_first_cached(&rq->scx->local_dsq.priq); -- if (rb_node) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- return entity->task; -- } -- -- return NULL; --} -- --static struct task_struct *pick_next_task_scx(struct rq *rq) --{ -- struct task_struct *p; -- --#ifndef CONFIG_SMP -- /* UP workaround - see the comment at the head of put_prev_task_scx() */ -- if (unlikely(rq->curr->sched_class != &ext_sched_class)) -- balance_scx(rq, rq->curr, NULL); --#endif -- -- p = first_local_task(rq); -- if (!p) -- return NULL; -- -- if (unlikely(!p->scx->slice)) { -- if (!scx_ops_disabling() && !scx_warned_zero_slice) { -- printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n", -- p->comm, p->pid); -- scx_warned_zero_slice = true; -- } -- p->scx->slice = SCX_SLICE_DFL; -- } -- -- set_next_task_scx(rq, p, true); -- -- return p; --} -- --#ifdef CONFIG_SCHED_CORE --/** -- * scx_prio_less - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Core-sched is implemented as an additional scheduling layer on top of the -- * usual sched_class'es and needs to find out the expected task ordering. For -- * SCX, core-sched calls this function to interrogate the task ordering. -- * -- * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used -- * to implement the default task ordering. The older the timestamp, the higher -- * prority the task - the global FIFO ordering matching the default scheduling -- * behavior. -- * -- * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to -- * implement FIFO ordering within each local DSQ. See pick_task_scx(). -- */ --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi) --{ -- /* -- * The const qualifiers are dropped from task_struct pointers when -- * calling ops.core_sched_before(). Accesses are controlled by the -- * verifier. -- */ -- if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling()) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, -- (struct task_struct *)a, -- (struct task_struct *)b); -- else -- return time_after64(a->scx->core_sched_at, b->scx->core_sched_at); --} -- --/** -- * pick_task_scx - Pick a candidate task for core-sched -- * @rq: rq to pick the candidate task from -- * -- * Core-sched calls this function on each SMT sibling to determine the next -- * tasks to run on the SMT siblings. balance_one() has been called on all -- * siblings and put_prev_task_scx() has been called only for the current CPU. -- * -- * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look -- * at the first task in the local dsq. @rq->curr has to be considered explicitly -- * to mimic %SCX_TASK_BAL_KEEP. -- */ --static struct task_struct *pick_task_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- struct task_struct *first = first_local_task(rq); -- -- if (curr->scx->flags & SCX_TASK_QUEUED) { -- /* is curr the only runnable task? */ -- if (!first) -- return curr; -- -- /* -- * Does curr trump first? We can always go by core_sched_at for -- * this comparison as it represents global FIFO ordering when -- * the default core-sched ordering is used and local-DSQ FIFO -- * ordering otherwise. -- * -- * We can have a task with an earlier timestamp on the DSQ. For -- * example, when a current task is preempted by a sibling -- * picking a different cookie, the task would be requeued at the -- * head of the local DSQ with an earlier timestamp than the -- * core-sched picked next task. Besides, the BPF scheduler may -- * dispatch any tasks to the local DSQ anytime. -- */ -- if (curr->scx->slice && time_before64(curr->scx->core_sched_at, -- first->scx->core_sched_at)) -- return curr; -- } -- -- return first; /* this may be %NULL */ --} --#endif /* CONFIG_SCHED_CORE */ -- --static enum scx_cpu_preempt_reason --preempt_reason_from_class(const struct sched_class *class) --{ --#ifdef CONFIG_SMP -- if (class == &stop_sched_class) -- return SCX_CPU_PREEMPT_STOP; --#endif -- if (class == &dl_sched_class) -- return SCX_CPU_PREEMPT_DL; -- if (class == &rt_sched_class) -- return SCX_CPU_PREEMPT_RT; -- return SCX_CPU_PREEMPT_UNKNOWN; --} -- --void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task, -- const struct sched_class *active) --{ -- lockdep_assert_rq_held(rq); -- -- /* -- * The callback is conceptually meant to convey that the CPU is no -- * longer under the control of SCX. Therefore, don't invoke the -- * callback if the CPU is is staying on SCX, or going idle (in which -- * case the SCX scheduler has actively decided not to schedule any -- * tasks on the CPU). -- */ -- if (likely(active >= &ext_sched_class)) -- return; -- -- /* -- * At this point we know that SCX was preempted by a higher priority -- * sched_class, so invoke the ->cpu_release() callback if we have not -- * done so already. We only send the callback once between SCX being -- * preempted, and it regaining control of the CPU. -- * -- * ->cpu_release() complements ->cpu_acquire(), which is emitted the -- * next time that balance_scx() is invoked. -- */ -- if (!rq->scx->cpu_released) { -- if (SCX_HAS_OP(cpu_release)) { -- struct scx_cpu_release_args args = { -- .reason = preempt_reason_from_class(active), -- .task = task, -- }; -- -- SCX_CALL_OP(SCX_KF_CPU_RELEASE, -- cpu_release, cpu_of(rq), &args); -- } -- rq->scx->cpu_released = true; -- } --} -- --#ifdef CONFIG_SMP -- --static bool test_and_clear_cpu_idle(int cpu) --{ -- if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -- if (cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- return true; -- } else { -- return false; -- } --} -- --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- int cpu; -- -- do { -- cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); -- if (cpu < nr_cpu_ids) { -- const struct cpumask *sbm = topology_sibling_cpumask(cpu); -- -- /* -- * If offline, @cpu is not its own sibling and we can -- * get caught in an infinite loop as @cpu is never -- * cleared from idle_masks.smt. Clear @cpu directly in -- * such cases. -- */ -- if (likely(cpumask_test_cpu(cpu, sbm))) -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm); -- else -- cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu)); -- } else { -- cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -- if (cpu >= nr_cpu_ids) -- return -EBUSY; -- } -- } while (!test_and_clear_cpu_idle(cpu)); -- -- return cpu; --} -- --static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) --{ -- s32 cpu; -- -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return prev_cpu; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local DSQ of the waker. -- */ -- if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 && -- scx_has_idle_cpus && !(current->flags & PF_EXITING)) { -- cpu = smp_processor_id(); -- if (cpumask_test_cpu(cpu, p->cpus_ptr)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (test_and_clear_cpu_idle(prev_cpu)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return prev_cpu; -- } -- -- if (p->nr_cpus_allowed == 1) -- return prev_cpu; -- -- cpu = scx_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- -- return prev_cpu; --} -- --static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) --{ -- if (SCX_HAS_OP(select_cpu)) { -- s32 cpu; -- -- cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu, -- wake_flags); -- if (ops_cpu_valid(cpu)) { -- return cpu; -- } else { -- scx_ops_error("select_cpu returned invalid cpu %d", cpu); -- return prev_cpu; -- } -- } else { -- return scx_select_cpu_dfl(p, prev_cpu, wake_flags); -- } --} -- --static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx) --{ -- set_cpus_allowed_common(p, ctx); -- -- /* -- * The effective cpumask is stored in @p->cpus_ptr which may temporarily -- * differ from the configured one in @p->cpus_mask. Always tell the bpf -- * scheduler the effective one. -- * -- * Fine-grained memory write control is enforced by BPF making the const -- * designation pointless. Cast it away when calling the operation. -- */ -- if (SCX_HAS_OP(set_cpumask)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, -- (struct cpumask *)p->cpus_ptr); --} -- --static void reset_idle_masks(void) --{ -- /* consider all cpus idle, should converge to the actual state quickly */ -- cpumask_setall(idle_masks.cpu); -- cpumask_setall(idle_masks.smt); -- scx_has_idle_cpus = true; --} -- --void __scx_update_idle(struct rq *rq, bool idle) --{ -- int cpu = cpu_of(rq); -- struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -- -- if (SCX_HAS_OP(update_idle)) { -- SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); -- if (!static_branch_unlikely(&scx_builtin_idle_enabled)) -- return; -- } -- -- if (idle) { -- cpumask_set_cpu(cpu, idle_masks.cpu); -- if (!scx_has_idle_cpus) -- scx_has_idle_cpus = true; -- -- /* -- * idle_masks.smt handling is racy but that's fine as it's only -- * for optimization and self-correcting. -- */ -- for_each_cpu(cpu, sib_mask) { -- if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -- return; -- } -- cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -- } else { -- cpumask_clear_cpu(cpu, idle_masks.cpu); -- if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -- } --} -- --#else /* !CONFIG_SMP */ -- --static bool test_and_clear_cpu_idle(int cpu) { return false; } --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } --static void reset_idle_masks(void) {} -- --#endif /* CONFIG_SMP */ -- --static bool check_rq_for_timeouts(struct rq *rq) --{ -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rq_flags rf; -- bool timed_out = false; -- -- rq_lock_irqsave(rq, &rf); -- list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) { -- unsigned long last_runnable; -- -- p = entity->task; -- last_runnable = p->scx->runnable_at; -- -- if (unlikely(time_after(jiffies, -- last_runnable + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "%s[%d] failed to run for %u.%03us", -- p->comm, p->pid, -- dur_ms / 1000, dur_ms % 1000); -- timed_out = true; -- break; -- } -- } -- rq_unlock_irqrestore(rq, &rf); -- -- return timed_out; --} -- --static void scx_watchdog_workfn(struct work_struct *work) --{ -- int cpu; -- -- scx_watchdog_timestamp = jiffies; -- -- for_each_online_cpu(cpu) { -- if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) -- break; -- -- cond_resched(); -- } -- queue_delayed_work(system_unbound_wq, to_delayed_work(work), -- scx_watchdog_timeout / 2); --} -- --static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) --{ -- update_curr_scx(rq); -- //scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock); -- /* -- * While disabling, always resched and refresh core-sched timestamp as -- * we can't trust the slice management or ops.core_sched_before(). -- */ -- if (scx_ops_disabling()) { -- curr->scx->slice = 0; -- touch_core_sched(rq, curr); -- } -- -- if (!curr->scx->slice) -- resched_curr(rq); --} -- --#define SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- --static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg) --{ -- int ret; -- -- WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED); -- -- p->scx->disallow = false; -- -- if (SCX_HAS_OP(prep_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- }; -- -- ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args); -- if (unlikely(ret)) { -- ret = ops_sanitize_err("prep_enable", ret); -- return ret; -- } -- } -- //scx_sched_init_task(p); -- -- if (p->scx->disallow) { -- struct rq *rq; -- struct rq_flags rf; -- -- rq = task_rq_lock(p, &rf); -- -- /* -- * We're either in fork or load path and @p->policy will be -- * applied right after. Reverting @p->policy here and rejecting -- * %SCHED_EXT transitions from scx_check_setscheduler() -- * guarantees that if ops.prep_enable() sets @p->disallow, @p -- * can never be in SCX. -- */ -- if (p->policy == SCHED_EXT) { -- p->policy = SCHED_NORMAL; -- atomic64_inc(&scx_nr_rejected); -- } -- -- task_rq_unlock(rq, p, &rf); -- } -- -- p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET); -- return 0; --} -- --static void scx_ops_enable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED)); -- -- if (SCX_HAS_OP(enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- p->scx->flags |= SCX_TASK_OPS_ENABLED; --} -- --static void scx_ops_disable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- if (p->scx->flags & SCX_TASK_OPS_PREPPED) { -- if (SCX_HAS_OP(cancel_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- } else if (p->scx->flags & SCX_TASK_OPS_ENABLED) { -- if (SCX_HAS_OP(disable)) -- SCX_CALL_OP(SCX_KF_REST, disable, p); -- p->scx->flags &= ~SCX_TASK_OPS_ENABLED; -- } --} -- --static void set_task_scx_weight(struct task_struct *p) --{ -- u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -- -- p->scx->weight = sched_weight_to_cgroup(weight); --} -- --/** -- * refresh_scx_weight - Refresh a task's ext weight -- * @p: task to refresh ext weight for -- * -- * @p->scx->weight carries the task's static priority in cgroup weight scale to -- * enable easy access from the BPF scheduler. To keep it synchronized with the -- * current task priority, this function should be called when a new task is -- * created, priority is changed for a task on sched_ext, and a task is switched -- * to sched_ext from other classes. -- */ --static void refresh_scx_weight(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- set_task_scx_weight(p); -- if (SCX_HAS_OP(set_weight)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight); --} -- --void scx_pre_fork(struct task_struct *p) --{ -- p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL); -- if (!p->scx) { -- goto lock; -- } -- -- p->scx->dsq = NULL; -- INIT_LIST_HEAD(&p->scx->dsq_node.fifo); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- INIT_LIST_HEAD(&p->scx->watchdog_node); -- p->scx->flags = 0; -- p->scx->weight = 0; -- p->scx->sticky_cpu = -1; -- p->scx->holding_cpu = -1; -- p->scx->kf_mask = 0; -- atomic64_set(&p->scx->ops_state, 0); -- p->scx->runnable_at = INITIAL_JIFFIES; -- p->scx->slice = SCX_SLICE_DFL; -- p->scx->task = p; -- p->sched_prop = 0; -- -- /* -- * BPF scheduler enable/disable paths want to be able to iterate and -- * update all tasks which can become complex when racing forks. As -- * enable/disable are very cold paths, let's use a percpu_rwsem to -- * exclude forks. -- */ --lock: -- percpu_down_read(&scx_fork_rwsem); --} -- --int scx_fork(struct task_struct *p) --{ -- percpu_rwsem_assert_held(&scx_fork_rwsem); -- -- if (scx_enabled()) -- return scx_ops_prepare_task(p, task_group(p)); -- else -- return 0; --} -- --void scx_post_fork(struct task_struct *p) --{ -- if (scx_enabled()) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- /* -- * Set the weight manually before calling ops.enable() so that -- * the scheduler doesn't see a stale value if they inspect the -- * task struct. We'll invoke ops.set_weight() afterwards, as it -- * would be odd to receive a callback on the task before we -- * tell the scheduler that it's been fully enabled. -- */ -- set_task_scx_weight(p); -- scx_ops_enable_task(p); -- refresh_scx_weight(p); -- task_rq_unlock(rq, p, &rf); -- } -- -- spin_lock_irq(&scx_tasks_lock); -- list_add_tail(&p->scx->tasks_node, &scx_tasks); -- spin_unlock_irq(&scx_tasks_lock); -- -- percpu_up_read(&scx_fork_rwsem); --} -- --void scx_cancel_fork(struct task_struct *p) --{ -- if (scx_enabled()) -- scx_ops_disable_task(p); -- percpu_up_read(&scx_fork_rwsem); --} -- --void sched_ext_free(struct task_struct *p) --{ -- unsigned long flags; -- -- spin_lock_irqsave(&scx_tasks_lock, flags); -- list_del_init(&p->scx->tasks_node); -- spin_unlock_irqrestore(&scx_tasks_lock, flags); -- -- /* -- * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED -> -- * ENABLED transitions can't race us. Disable ops for @p. -- */ -- if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- scx_ops_disable_task(p); -- task_rq_unlock(rq, p, &rf); -- } --} -- --static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) --{ --} -- --static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} --static void switched_to_scx(struct rq *rq, struct task_struct *p) {} -- --int scx_check_setscheduler(struct task_struct *p, int policy) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- /* if disallow, reject transitioning into SCX */ -- if (scx_enabled() && READ_ONCE(p->scx->disallow) && -- p->policy != policy && policy == SCHED_EXT) -- return -EACCES; -- -- return 0; --} -- --#ifdef CONFIG_NO_HZ_FULL --bool scx_can_stop_tick(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (scx_ops_disabling()) -- return false; -- -- if (p->sched_class != &ext_sched_class) -- return true; -- -- /* -- * @rq can dispatch from different DSQs, so we can't tell whether it -- * needs the tick or not by looking at nr_running. Allow stopping ticks -- * iff the BPF scheduler indicated so. See set_next_task_scx(). -- */ -- return rq->scx->flags & SCX_RQ_CAN_STOP_TICK; --} --#endif -- --static inline void scx_cgroup_lock(void) {} --static inline void scx_cgroup_unlock(void) {} -- --/* -- * Omitted operations: -- * -- * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -- * task isn't tied to the CPU at that point. Preemption is implemented by -- * resetting the victim task's slice to 0 and triggering reschedule on the -- * target CPU. -- * -- * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -- * -- * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -- * their current sched_class. Call them directly from sched core instead. -- * -- * - task_woken, switched_from: Unnecessary. -- */ --DEFINE_SCHED_CLASS(ext) = { -- .enqueue_task = enqueue_task_scx, -- .dequeue_task = dequeue_task_scx, -- .yield_task = yield_task_scx, -- .yield_to_task = yield_to_task_scx, -- -- .check_preempt_curr = check_preempt_curr_scx, -- -- .pick_next_task = pick_next_task_scx, -- -- .put_prev_task = put_prev_task_scx, -- .set_next_task = set_next_task_scx, -- --#ifdef CONFIG_SMP -- .balance = balance_scx, -- .select_task_rq = select_task_rq_scx, -- .set_cpus_allowed = set_cpus_allowed_scx, --#endif -- --#ifdef CONFIG_SCHED_CORE -- .pick_task = pick_task_scx, --#endif -- -- .task_tick = task_tick_scx, -- -- .switched_to = switched_to_scx, -- .prio_changed = prio_changed_scx, -- -- .update_curr = update_curr_scx, -- --#ifdef CONFIG_UCLAMP_TASK -- .uclamp_enabled = 0, --#endif --}; -- --static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) --{ -- memset(dsq, 0, sizeof(*dsq)); -- -- raw_spin_lock_init(&dsq->lock); -- INIT_LIST_HEAD(&dsq->fifo); -- dsq->id = dsq_id; --} -- --static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id & SCX_DSQ_FLAG_BUILTIN) -- return ERR_PTR(-EINVAL); -- -- dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node); -- if (!dsq) -- return ERR_PTR(-ENOMEM); -- -- init_dsq(dsq, dsq_id); -- -- return dsq; --} -- --static void free_dsq_irq_workfn(struct irq_work *irq_work) --{ -- struct llist_node *to_free = llist_del_all(&dsqs_to_free); -- struct scx_dispatch_q *dsq, *tmp_dsq; -- -- llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) -- kfree_rcu(dsq, rcu); --} -- --static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); -- --static void destroy_dsq(u64 dsq_id) --{ --} -- --static void scx_cgroup_exit(void) {} --static int scx_cgroup_init(void) { return 0; } --static void scx_cgroup_config_knobs(void) {} -- --/* -- * Used by sched_fork() and __setscheduler_prio() to pick the matching -- * sched_class. dl/rt are already handled. -- */ --bool task_on_scx(struct task_struct *p) --{ -- if (!scx_enabled() || scx_ops_disabling()) -- return false; -- if (READ_ONCE(scx_switching_all)) -- return true; -- return p->policy == SCHED_EXT; --} -- --static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags) --{ -- if (enq_flags & SCX_ENQ_LAST) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); --} -- --static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {} -- --static void scx_ops_disable_workfn(struct kthread_work *work) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- struct scx_task_iter sti; -- struct task_struct *p; -- const char *reason; -- int i, cpu, type; -- -- type = atomic_read(&scx_exit_type); -- while (true) { -- /* -- * NONE indicates that a new scx_ops has been registered since -- * disable was scheduled - don't kill the new ops. DONE -- * indicates that the ops has already been disabled. -- */ -- if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE) -- return; -- if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE)) -- break; -- } -- -- cancel_delayed_work_sync(&scx_watchdog_work); -- -- switch (type) { -- case SCX_EXIT_UNREG: -- reason = "BPF scheduler unregistered"; -- break; -- case SCX_EXIT_SYSRQ: -- reason = "disabled by sysrq-S"; -- break; -- case SCX_EXIT_ERROR: -- reason = "runtime error"; -- break; -- case SCX_EXIT_ERROR_BPF: -- reason = "scx_bpf_error"; -- break; -- case SCX_EXIT_ERROR_STALL: -- reason = "runnable task stall"; -- break; -- default: -- reason = ""; -- } -- -- ei->type = type; -- strlcpy(ei->reason, reason, sizeof(ei->reason)); -- -- switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { -- case SCX_OPS_DISABLED: -- pr_warn("sched_ext: ops error detected without ops (%s)\n", -- scx_exit_info.msg); -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- return; -- case SCX_OPS_PREPPING: -- goto forward_progress_guaranteed; -- case SCX_OPS_DISABLING: -- /* shouldn't happen but handle it like ENABLING if it does */ -- WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); -- fallthrough; -- case SCX_OPS_ENABLING: -- case SCX_OPS_ENABLED: -- break; -- } -- -- /* -- * DISABLING is set and ops was either ENABLING or ENABLED indicating -- * that the ops and static branches are set. -- * -- * We must guarantee that all runnable tasks make forward progress -- * without trusting the BPF scheduler. We can't grab any mutexes or -- * rwsems as they might be held by tasks that the BPF scheduler is -- * forgetting to run, which unfortunately also excludes toggling the -- * static branches. -- * -- * Let's work around by overriding a couple ops and modifying behaviors -- * based on the DISABLING state and then cycling the tasks through -- * dequeue/enqueue to force global FIFO scheduling. -- * -- * a. ops.enqueue() and .dispatch() are overridden for simple global -- * FIFO scheduling. -- * -- * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value -- * can't be trusted. Whenever a tick triggers, the running task is -- * rotated to the tail of the queue with core_sched_at touched. -- * -- * c. pick_next_task() suppresses zero slice warning. -- * -- * d. scx_prio_less() reverts to the default core_sched_at order. -- */ -- scx_ops.enqueue = scx_ops_fallback_enqueue; -- scx_ops.dispatch = scx_ops_fallback_dispatch; -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- SCHED_CHANGE_BLOCK(task_rq(p), p, -- DEQUEUE_SAVE | DEQUEUE_MOVE) { -- /* cycling deq/enq is enough, see above */ -- } -- } -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* kick all CPUs to restore ticks */ -- for_each_possible_cpu(cpu) -- resched_cpu(cpu); -- --forward_progress_guaranteed: -- /* -- * Here, every runnable task is guaranteed to make forward progress and -- * we can safely use blocking synchronization constructs. Actually -- * disable ops. -- */ -- mutex_lock(&scx_ops_enable_mutex); -- -- static_branch_disable(&__scx_switched_all); -- WRITE_ONCE(scx_switching_all, false); -- -- /* avoid racing against fork and cgroup changes */ -- cpus_read_lock(); -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- bool alive = READ_ONCE(p->__state) != TASK_DEAD; -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL); -- -- __setscheduler_prio(p, p->prio); -- } -- -- if (alive) -- check_class_changed(task_rq(p), p, old_class, p->prio); -- -- scx_ops_disable_task(p); -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* no task is on scx, turn off all the switches and flush in-progress calls */ -- static_branch_disable_cpuslocked(&__scx_ops_enabled); -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- static_branch_disable_cpuslocked(&scx_has_op[i]); -- static_branch_disable_cpuslocked(&scx_ops_enq_last); -- static_branch_disable_cpuslocked(&scx_ops_enq_exiting); -- static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- synchronize_rcu(); -- -- scx_cgroup_exit(); -- -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- cpus_read_unlock(); -- -- if (ei->type >= SCX_EXIT_ERROR) { -- printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name); -- -- if (ei->msg[0] == '\0') -- printk(KERN_ERR "sched_ext: %s\n", ei->reason); -- else -- printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg); -- -- stack_trace_print(ei->bt, ei->bt_len, 2); -- } -- -- if (scx_ops.exit) -- SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); -- -- memset(&scx_ops, 0, sizeof(scx_ops)); -- -- free_percpu(scx_dsp_buf); -- scx_dsp_buf = NULL; -- scx_dsp_max_batch = 0; -- -- mutex_unlock(&scx_ops_enable_mutex); -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- -- scx_cgroup_config_knobs(); --} -- --static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); -- --static void schedule_scx_ops_disable_work(void) --{ -- struct kthread_worker *helper = READ_ONCE(scx_ops_helper); -- -- /* -- * We may be called spuriously before the first bpf_sched_ext_reg(). If -- * scx_ops_helper isn't set up yet, there's nothing to do. -- */ -- if (helper) -- kthread_queue_work(helper, &scx_ops_disable_work); --} -- --static void scx_ops_disable(enum scx_exit_type type) --{ -- int none = SCX_EXIT_NONE; -- -- if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)) -- type = SCX_EXIT_ERROR; -- -- atomic_try_cmpxchg(&scx_exit_type, &none, type); -- -- schedule_scx_ops_disable_work(); --} -- --static void scx_ops_error_irq_workfn(struct irq_work *irq_work) --{ -- schedule_scx_ops_disable_work(); --} -- --static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- int none = SCX_EXIT_NONE; -- va_list args; -- -- if (!atomic_try_cmpxchg(&scx_exit_type, &none, type)) -- return; -- -- ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1); -- -- va_start(args, fmt); -- vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args); -- va_end(args); -- -- irq_work_queue(&scx_ops_error_irq_work); --} -- --static struct kthread_worker *scx_create_rt_helper(const char *name) --{ -- struct kthread_worker *helper; -- -- helper = kthread_create_worker(0, name); -- if (helper) -- sched_set_fifo(helper->task); -- return helper; --} -- --static int scx_ops_enable(struct sched_ext_ops *ops) --{ -- struct scx_task_iter sti; -- struct task_struct *p; -- int i, ret; -- int tcnt = 0; -- unsigned long long start = 0; -- unsigned long long total_start = sched_clock(); -- -- mutex_lock(&scx_ops_enable_mutex); -- -- if (!scx_ops_helper) { -- WRITE_ONCE(scx_ops_helper, -- scx_create_rt_helper("sched_ext_ops_helper")); -- if (!scx_ops_helper) { -- ret = -ENOMEM; -- goto err_unlock; -- } -- } -- -- if (scx_ops_enable_state() != SCX_OPS_DISABLED) { -- ret = -EBUSY; -- goto err_unlock; -- } -- -- //slim_walt_enable(true); -- -- /* -- * Set scx_ops, transition to PREPPING and clear exit info to arm the -- * disable path. Failure triggers full disabling from here on. -- */ -- scx_ops = *ops; -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != -- SCX_OPS_DISABLED); -- -- memset(&scx_exit_info, 0, sizeof(scx_exit_info)); -- atomic_set(&scx_exit_type, SCX_EXIT_NONE); -- scx_warned_zero_slice = false; -- -- atomic64_set(&scx_nr_rejected, 0); -- -- /* -- * Keep CPUs stable during enable so that the BPF scheduler can track -- * online CPUs by watching ->on/offline_cpu() after ->init(). -- */ -- cpus_read_lock(); -- -- scx_switch_all_req = true; -- if (scx_ops.init) { -- ret = SCX_CALL_OP_RET(SCX_KF_INIT, init); -- if (ret) { -- ret = ops_sanitize_err("init", ret); -- goto err_disable; -- } -- -- /* -- * Exit early if ops.init() triggered scx_bpf_error(). Not -- * strictly necessary as we'll fail transitioning into ENABLING -- * later but that'd be after calling ops.prep_enable() on all -- * tasks and with -EBUSY which isn't very intuitive. Let's exit -- * early with success so that the condition is notified through -- * ops.exit() like other scx_bpf_error() invocations. -- */ -- if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE) -- goto err_disable; -- } -- -- WARN_ON_ONCE(scx_dsp_buf); -- scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; -- scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch, -- __alignof__(scx_dsp_buf[0])); -- if (!scx_dsp_buf) { -- ret = -ENOMEM; -- goto err_disable; -- } -- -- scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; -- if (ops->timeout_ms) -- scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); -- -- scx_watchdog_timestamp = jiffies; -- queue_delayed_work(system_unbound_wq, &scx_watchdog_work, -- scx_watchdog_timeout / 2); -- -- /* -- * Lock out forks, cgroup on/offlining and moves before opening the -- * floodgate so that they don't wander into the operations prematurely. -- */ -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- if (((void (**)(void))ops)[i]) -- static_branch_enable_cpuslocked(&scx_has_op[i]); -- -- if (ops->flags & SCX_OPS_ENQ_LAST) -- static_branch_enable_cpuslocked(&scx_ops_enq_last); -- -- if (ops->flags & SCX_OPS_ENQ_EXITING) -- static_branch_enable_cpuslocked(&scx_ops_enq_exiting); -- if (scx_ops.cpu_acquire || scx_ops.cpu_release) -- static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); -- -- if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { -- reset_idle_masks(); -- static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); -- } else { -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- } -- -- /* -- * All cgroups should be initialized before letting in tasks. cgroup -- * on/offlining and task migrations are already locked out. -- */ -- ret = scx_cgroup_init(); -- if (ret) -- goto err_disable_unlock; -- -- static_branch_enable_cpuslocked(&__scx_ops_enabled); -- -- /* -- * Enable ops for every task. Fork is excluded by scx_fork_rwsem -- * preventing new tasks from being added. No need to exclude tasks -- * leaving as sched_ext_free() can handle both prepped and enabled -- * tasks. Prep all tasks first and then enable them with preemption -- * disabled. -- */ -- spin_lock_irq(&scx_tasks_lock); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered(&sti))) { -- get_task_struct(p); -- spin_unlock_irq(&scx_tasks_lock); -- -- ret = scx_ops_prepare_task(p, task_group(p)); -- if (ret) { -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n", -- ret, p->comm, p->pid); -- goto err_disable_unlock; -- } -- -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- } -- scx_task_iter_exit(&sti); -- -- /* -- * All tasks are prepped but are still ops-disabled. Ensure that -- * %current can't be scheduled out and switch everyone. -- * preempt_disable() is necessary because we can't guarantee that -- * %current won't be starved if scheduled out while switching. -- */ -- preempt_disable(); -- -- /* -- * From here on, the disable path must assume that tasks have ops -- * enabled and need to be recovered. -- */ -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { -- preempt_enable(); -- spin_unlock_irq(&scx_tasks_lock); -- ret = -EBUSY; -- goto err_disable_unlock; -- } -- -- /* -- * We're fully committed and can't fail. The PREPPED -> ENABLED -- * transitions here are synchronized against sched_ext_free() through -- * scx_tasks_lock. -- */ -- WRITE_ONCE(scx_switching_all, scx_switch_all_req); -- start = sched_clock(); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- tcnt++; -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- scx_ops_enable_task(p); -- __setscheduler_prio(p, p->prio); -- } -- -- check_class_changed(task_rq(p), p, old_class, p->prio); -- } else { -- scx_ops_disable_task(p); -- } -- } -- printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id()); -- scx_task_iter_exit(&sti); -- -- spin_unlock_irq(&scx_tasks_lock); -- preempt_enable(); -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { -- ret = -EBUSY; -- goto err_disable; -- } -- -- if (scx_switch_all_req) -- static_branch_enable_cpuslocked(&__scx_switched_all); -- -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- -- scx_cgroup_config_knobs(); -- printk("\n total duration = %llu , current cpu = %d\n", sched_clock() - total_start, smp_processor_id()); -- -- return 0; -- --err_unlock: -- mutex_unlock(&scx_ops_enable_mutex); -- return ret; -- --err_disable_unlock: -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); --err_disable: -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- /* must be fully disabled before returning */ -- scx_ops_disable(SCX_EXIT_ERROR); -- kthread_flush_work(&scx_ops_disable_work); -- return ret; --} -- --#ifdef CONFIG_SCHED_DEBUG --static const char *scx_ops_enable_state_str[] = { -- [SCX_OPS_PREPPING] = "prepping", -- [SCX_OPS_ENABLING] = "enabling", -- [SCX_OPS_ENABLED] = "enabled", -- [SCX_OPS_DISABLING] = "disabling", -- [SCX_OPS_DISABLED] = "disabled", --}; -- --static int scx_debug_show(struct seq_file *m, void *v) --{ -- mutex_lock(&scx_ops_enable_mutex); -- seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name); -- seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled()); -- seq_printf(m, "%-30s: %d\n", "switching_all", -- READ_ONCE(scx_switching_all)); -- seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all()); -- seq_printf(m, "%-30s: %s\n", "enable_state", -- scx_ops_enable_state_str[scx_ops_enable_state()]); -- seq_printf(m, "%-30s: %llu\n", "nr_rejected", -- atomic64_read(&scx_nr_rejected)); -- mutex_unlock(&scx_ops_enable_mutex); -- return 0; --} -- --static int scx_debug_open(struct inode *inode, struct file *file) --{ -- return single_open(file, scx_debug_show, NULL); --} -- --const struct file_operations sched_ext_fops = { -- .open = scx_debug_open, -- .read = seq_read, -- .llseek = seq_lseek, -- .release = single_release, --}; --#endif -- --/******************************************************************************** -- * bpf_struct_ops plumbing. -- */ --#include --#include --#include -- --extern struct btf *btf_vmlinux; --static const struct btf_type *task_struct_type; -- --static bool bpf_scx_is_valid_access(int off, int size, -- enum bpf_access_type type, -- const struct bpf_prog *prog, -- struct bpf_insn_access_aux *info) --{ -- if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) -- return false; -- if (type != BPF_READ) -- return false; -- if (off % size != 0) -- return false; -- -- return btf_ctx_access(off, size, type, prog, info); --} -- --static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, -- const struct bpf_reg_state *reg, -- int off, int size) --{ -- return 0; --} -- --static const struct bpf_func_proto * --bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) --{ -- switch (func_id) { -- case BPF_FUNC_task_storage_get: -- return &bpf_task_storage_get_proto; -- case BPF_FUNC_task_storage_delete: -- return &bpf_task_storage_delete_proto; -- default: -- return bpf_base_func_proto(func_id); -- } --} -- --const struct bpf_verifier_ops bpf_scx_verifier_ops = { -- .get_func_proto = bpf_scx_get_func_proto, -- .is_valid_access = bpf_scx_is_valid_access, -- .btf_struct_access = bpf_scx_btf_struct_access, --}; -- --static int bpf_scx_init_member(const struct btf_type *t, -- const struct btf_member *member, -- void *kdata, const void *udata) --{ -- const struct sched_ext_ops *uops = udata; -- struct sched_ext_ops *ops = kdata; -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- int ret; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, dispatch_max_batch): -- if (*(u32 *)(udata + moff) > INT_MAX) -- return -E2BIG; -- ops->dispatch_max_batch = *(u32 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, flags): -- if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) -- return -EINVAL; -- ops->flags = *(u64 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, name): -- ret = bpf_obj_name_cpy(ops->name, uops->name, -- sizeof(ops->name)); -- if (ret < 0) -- return ret; -- if (ret == 0) -- return -EINVAL; -- return 1; -- case offsetof(struct sched_ext_ops, timeout_ms): -- if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT) -- return -E2BIG; -- ops->timeout_ms = *(u32 *)(udata + moff); -- return 1; -- } -- -- return 0; --} -- --static int bpf_scx_check_member(const struct btf_type *t, -- const struct btf_member *member, -- const struct bpf_prog *prog) --{ -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, prep_enable): -- case offsetof(struct sched_ext_ops, init): -- case offsetof(struct sched_ext_ops, exit): -- break; -- default: -- /* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */ -- /* if (prog->aux->sleepable) -- return -EINVAL; */ -- break; -- } -- -- return 0; --} -- --static int bpf_scx_reg(void *kdata) --{ -- return scx_ops_enable(kdata); --} -- --static void bpf_scx_unreg(void *kdata) --{ -- scx_ops_disable(SCX_EXIT_UNREG); -- kthread_flush_work(&scx_ops_disable_work); --} -- --static int bpf_scx_init(struct btf *btf) --{ -- u32 type_id; -- -- type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); -- if (type_id < 0) -- return -EINVAL; -- task_struct_type = btf_type_by_id(btf, type_id); -- -- return 0; --} -- --/* "extern" to avoid sparse warning, only used in this file */ --extern struct bpf_struct_ops bpf_sched_ext_ops; -- --struct bpf_struct_ops bpf_sched_ext_ops = { -- .verifier_ops = &bpf_scx_verifier_ops, -- .reg = bpf_scx_reg, -- .unreg = bpf_scx_unreg, -- .check_member = bpf_scx_check_member, -- .init_member = bpf_scx_init_member, -- .init = bpf_scx_init, -- .name = "sched_ext_ops", --}; -- --static void sysrq_handle_sched_ext_reset(u8 key) --{ -- if (scx_ops_helper) -- scx_ops_disable(SCX_EXIT_SYSRQ); -- else -- pr_info("sched_ext: BPF scheduler not yet used\n"); --} -- --static const struct sysrq_key_op sysrq_sched_ext_reset_op = { -- .handler = sysrq_handle_sched_ext_reset, -- .help_msg = "reset-sched-ext(S)", -- .action_msg = "Disable sched_ext and revert all tasks to CFS", -- .enable_mask = SYSRQ_ENABLE_RTNICE, --}; -- --static void kick_cpus_irq_workfn(struct irq_work *irq_work) --{ -- struct rq *this_rq = this_rq(); -- u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); -- int this_cpu = cpu_of(this_rq); -- int cpu; -- -- for_each_cpu(cpu, this_rq->scx->cpus_to_kick) { -- struct rq *rq = cpu_rq(cpu); -- unsigned long flags; -- -- raw_spin_rq_lock_irqsave(rq, flags); -- -- if (cpu_online(cpu) || cpu == this_cpu) { -- if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) && -- rq->curr->sched_class == &ext_sched_class) -- rq->curr->scx->slice = 0; -- pseqs[cpu] = rq->scx->pnt_seq; -- resched_curr(rq); -- } else { -- cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait); -- } -- -- raw_spin_rq_unlock_irqrestore(rq, flags); -- } -- -- for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait, -- cpumask_of(this_cpu)) { -- /* -- * Pairs with smp_store_release() issued by this CPU in -- * scx_notify_pick_next_task() on the resched path. -- * -- * We busy-wait here to guarantee that no other task can be -- * scheduled on our core before the target CPU has entered the -- * resched path. -- */ -- while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu]) -- cpu_relax(); -- } -- -- cpumask_clear(this_rq->scx->cpus_to_kick); -- cpumask_clear(this_rq->scx->cpus_to_preempt); -- cpumask_clear(this_rq->scx->cpus_to_wait); --} -- --void __init init_sched_ext_class(void) --{ -- int cpu; -- u32 v; -- -- /* -- * The following is to prevent the compiler from optimizing out the enum -- * definitions so that BPF scheduler implementations can use them -- * through the generated vmlinux.h. -- */ -- WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | -- SCX_TG_ONLINE | SCX_KICK_PREEMPT); -- -- init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); --#ifdef CONFIG_SMP -- BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -- BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); --#endif -- scx_kick_cpus_pnt_seqs = -- __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * -- num_possible_cpus(), -- __alignof__(scx_kick_cpus_pnt_seqs[0])); -- BUG_ON(!scx_kick_cpus_pnt_seqs); -- -- for_each_possible_cpu(cpu) { -- struct rq *rq = cpu_rq(cpu); -- -- rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL); -- if (rq->scx) -- rq->scx->rq = rq; -- else -- pr_err("fatal error : alloc rq->scx failed!!!\n"); -- -- init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL); -- INIT_LIST_HEAD(&rq->scx->watchdog_list); -- -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL)); -- init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn); -- } -- -- register_sysrq_key('S', &sysrq_sched_ext_reset_op); -- INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); -- scx_cgroup_config_knobs(); --} -- -- --/******************************************************************************** -- * Helpers that can be called from the BPF scheduler. -- */ --#include -- --/* Disables missing prototype warnings for kfuncs */ --__diag_push(); --__diag_ignore_all("-Wmissing-prototypes", -- "Global functions as their definitions will be in vmlinux BTF"); -- --/** -- * scx_bpf_switch_all - Switch all tasks into SCX -- * @into_scx: switch direction -- * -- * If @into_scx is %true, all existing and future non-dl/rt tasks are switched -- * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on -- * SCX. The actual switching is asynchronous. Can be called from ops.init(). -- */ --void scx_bpf_switch_all(void) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return; -- -- scx_switch_all_req = true; --} -- --BTF_SET8_START(scx_kfunc_ids_init) --BTF_ID_FLAGS(func, scx_bpf_switch_all) --BTF_SET8_END(scx_kfunc_ids_init) -- --static const struct btf_kfunc_id_set scx_kfunc_set_init = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_init, --}; -- --/** -- * scx_bpf_create_dsq - Create a custom DSQ -- * @dsq_id: DSQ to create -- * @node: NUMA node to allocate from -- * -- * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(), -- * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move(). -- */ --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return -EINVAL; -- -- if (unlikely(node >= (int)nr_node_ids || -- (node < 0 && node != NUMA_NO_NODE))) -- return -EINVAL; -- return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); --} -- --BTF_SET8_START(scx_kfunc_ids_sleepable) --BTF_ID_FLAGS(func, scx_bpf_create_dsq) --BTF_SET8_END(scx_kfunc_ids_sleepable) -- --static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_sleepable, --}; -- --static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) --{ -- if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) -- return false; -- -- lockdep_assert_irqs_disabled(); -- -- if (unlikely(!p)) { -- scx_ops_error("called with NULL task"); -- return false; -- } -- -- if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { -- scx_ops_error("invalid enq_flags 0x%llx", enq_flags); -- return false; -- } -- -- return true; --} -- --static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) --{ -- struct task_struct *ddsp_task; -- int idx; -- -- ddsp_task = __this_cpu_read(direct_dispatch_task); -- if (ddsp_task) { -- direct_dispatch(ddsp_task, p, dsq_id, enq_flags); -- return; -- } -- -- idx = __this_cpu_read(scx_dsp_ctx.buf_cursor); -- if (unlikely(idx >= scx_dsp_max_batch)) { -- scx_ops_error("dispatch buffer overflow"); -- return; -- } -- -- this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){ -- .task = p, -- .qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK, -- .dsq_id = dsq_id, -- .enq_flags = enq_flags, -- }; -- __this_cpu_inc(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe -- * to call this function spuriously. Can be called from ops.enqueue() and -- * ops.dispatch(). -- * -- * When called from ops.enqueue(), it's for direct dispatch and @p must match -- * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the -- * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be -- * on the target CPU in the first place. -- * -- * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id -- * and this function can be called upto ops.dispatch_max_batch times to dispatch -- * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the -- * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. -- * -- * This function doesn't have any locking restrictions and may be called under -- * BPF locks (in the future when BPF introduces more flexible locking). -- * -- * @p is allowed to run for @slice. The scheduling path is triggered on slice -- * exhaustion. If zero, the current residual slice is maintained. If -- * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with -- * scx_bpf_kick_cpu() to trigger scheduling. -- */ --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- scx_dispatch_commit(p, dsq_id, enq_flags); --} -- --/** -- * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. -- * Tasks queued into the priority queue are ordered by @vtime and always -- * consumed after the tasks in the FIFO queue. All other aspects are identical -- * to scx_bpf_dispatch(). -- * -- * @vtime ordering is according to time_before64() which considers wrapping. A -- * numerically larger vtime may indicate an earlier position in the ordering and -- * vice-versa. -- */ --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 vtime, u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- p->scx->dsq_vtime = vtime; -- -- scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); --} -- --BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime) --BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_enqueue_dispatch, --}; -- --/** -- * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots -- * -- * Can only be called from ops.dispatch(). -- */ --u32 scx_bpf_dispatch_nr_slots(void) --{ -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return 0; -- -- return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ -- * @dsq_id: DSQ to consume -- * -- * Consume a task from the non-local DSQ identified by @dsq_id and transfer it -- * to the current CPU's local DSQ for execution. Can only be called from -- * ops.dispatch(). -- * -- * This function flushes the in-flight dispatches from scx_bpf_dispatch() before -- * trying to consume the specified DSQ. It may also grab rq locks and thus can't -- * be called under any BPF locks. -- * -- * Returns %true if a task has been consumed, %false if there isn't any task to -- * consume. -- */ --bool scx_bpf_consume(u64 dsq_id) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- struct scx_dispatch_q *dsq; -- -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return false; -- -- flush_dispatch_buf(dspc->rq, dspc->rf); -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); -- return false; -- } -- -- if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) { -- /* -- * A successfully consumed task can be dequeued before it starts -- * running while the CPU is trying to migrate other dispatched -- * tasks. Bump nr_tasks to tell balance_scx() to retry on empty -- * local DSQ. -- */ -- dspc->nr_tasks++; -- return true; -- } else { -- return false; -- } --} -- --BTF_SET8_START(scx_kfunc_ids_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) --BTF_ID_FLAGS(func, scx_bpf_consume) --BTF_SET8_END(scx_kfunc_ids_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_dispatch, --}; -- --/** -- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ -- * -- * Iterate over all of the tasks currently enqueued on the local DSQ of the -- * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of -- * processed tasks. Can only be called from ops.cpu_release(). -- */ --u32 scx_bpf_reenqueue_local(void) --{ -- u32 nr_enqueued, i; -- struct rq *rq; -- struct scx_rq *scx_rq; -- -- if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) -- return 0; -- -- rq = cpu_rq(smp_processor_id()); -- lockdep_assert_rq_held(rq); -- scx_rq = rq->scx; -- -- /* -- * Get the number of tasks on the local DSQ before iterating over it to -- * pull off tasks. The enqueue callback below can signal that it wants -- * the task to stay on the local DSQ, and we want to prevent the BPF -- * scheduler from causing us to loop indefinitely. -- */ -- nr_enqueued = scx_rq->local_dsq.nr; -- for (i = 0; i < nr_enqueued; i++) { -- struct task_struct *p; -- -- p = first_local_task(rq); -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- WARN_ON_ONCE(p->scx->holding_cpu != -1); -- dispatch_dequeue(scx_rq, p); -- do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); -- } -- -- return nr_enqueued; --} -- --BTF_SET8_START(scx_kfunc_ids_cpu_release) --BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) --BTF_SET8_END(scx_kfunc_ids_cpu_release) -- --static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_cpu_release, --}; -- --/** -- * scx_bpf_kick_cpu - Trigger reschedule on a CPU -- * @cpu: cpu to kick -- * @flags: SCX_KICK_* flags -- * -- * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or -- * trigger rescheduling on a busy CPU. This can be called from any online -- * scx_ops operation and the actual kicking is performed asynchronously through -- * an irq work. -- */ --void scx_bpf_kick_cpu(s32 cpu, u64 flags) --{ -- struct rq *rq; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d", cpu); -- return; -- } -- -- preempt_disable(); -- rq = this_rq(); -- -- /* -- * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting -- * rq locks. We can probably be smarter and avoid bouncing if called -- * from ops which don't hold a rq lock. -- */ -- cpumask_set_cpu(cpu, rq->scx->cpus_to_kick); -- if (flags & SCX_KICK_PREEMPT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt); -- if (flags & SCX_KICK_WAIT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_wait); -- -- irq_work_queue(&rq->scx->kick_cpus_irq_work); -- preempt_enable(); --} -- --/** -- * scx_bpf_dsq_nr_queued - Return the number of queued tasks -- * @dsq_id: id of the DSQ -- * -- * Return the number of tasks in the DSQ matching @dsq_id. If not found, -- * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops -- * operations. -- */ --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) --{ -- struct scx_dispatch_q *dsq; -- -- lockdep_assert(rcu_read_lock_any_held()); -- -- if (dsq_id == SCX_DSQ_LOCAL) { -- return this_rq()->scx->local_dsq.nr; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (ops_cpu_valid(cpu)) -- return cpu_rq(cpu)->scx->local_dsq.nr; -- } else { -- dsq = find_non_local_dsq(dsq_id); -- if (dsq) -- return dsq->nr; -- } -- return -ENOENT; --} -- --/** -- * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state -- * @cpu: cpu to test and clear idle for -- * -- * Returns %true if @cpu was idle and its idle state was successfully cleared. -- * %false otherwise. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return false; -- } -- -- if (ops_cpu_valid(cpu)) -- return test_and_clear_cpu_idle(cpu); -- else -- return false; --} -- --/** -- * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu -- * @cpus_allowed: Allowed cpumask -- * -- * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked -- * idle cpu number on success. -%EBUSY if no matching cpu was found. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return -EBUSY; -- } -- -- return scx_pick_idle_cpu(cpus_allowed); --} -- --/** -- * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking -- * per-CPU cpumask. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_cpumask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.cpu; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, -- * per-physical-core cpumask. Can be used to determine if an entire physical -- * core is free. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_smtmask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.smt; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to -- * either the percpu, or SMT idle-tracking cpumask. -- */ --void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) --{ -- /* -- * Empty function body because we aren't actually acquiring or -- * releasing a reference to a global idle cpumask, which is read-only -- * in the caller and is never released. The acquire / release semantics -- * here are just used to make the cpumask is a trusted pointer in the -- * caller. -- */ --} -- --struct scx_bpf_error_bstr_bufs { -- u64 data[MAX_BPRINTF_VARARGS]; -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs); -- --/** -- * scx_bpf_error_bstr - Indicate fatal error -- * @fmt: error message format string -- * @data: format string parameters packaged using ___bpf_fill() macro -- * @data__sz: @data len, must end in '__sz' for the verifier -- * -- * Indicate that the BPF scheduler encountered a fatal error and initiate ops -- * disabling. -- */ --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) --{ -- struct scx_bpf_error_bstr_bufs *bufs; -- unsigned long flags; -- struct bpf_bprintf_data args = {}; -- int ret; -- -- local_irq_save(flags); -- bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs); -- -- if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || -- (data__sz && !data)) { -- scx_ops_error("invalid data=%p and data__sz=%u", -- (void *)data, data__sz); -- goto out_restore; -- } -- -- ret = copy_from_kernel_nofault(bufs->data, data, data__sz); -- if (ret) { -- scx_ops_error("failed to read data fields (%d)", ret); -- goto out_restore; -- } -- -- ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args); -- if (ret < 0) { -- scx_ops_error("failed to format prepration (%d)", ret); -- goto out_restore; -- } -- -- ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt, -- args.bin_args); -- bpf_bprintf_cleanup(&args); -- if (ret < 0) { -- scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format", -- fmt, data, data__sz); -- goto out_restore; -- } -- -- scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg); --out_restore: -- local_irq_restore(flags); --} -- --/** -- * scx_bpf_destroy_dsq - Destroy a custom DSQ -- * @dsq_id: DSQ to destroy -- * -- * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with -- * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is -- * empty and no further tasks are dispatched to it. Ignored if called on a DSQ -- * which doesn't exist. Can be called from any online scx_ops operations. -- */ --void scx_bpf_destroy_dsq(u64 dsq_id) --{ -- destroy_dsq(dsq_id); --} -- --/** -- * scx_bpf_task_running - Is task currently running? -- * @p: task of interest -- */ --bool scx_bpf_task_running(const struct task_struct *p) --{ -- return task_rq(p)->curr == p; --} -- --/** -- * scx_bpf_task_cpu - CPU a task is currently associated with -- * @p: task of interest -- */ --s32 scx_bpf_task_cpu(const struct task_struct *p) --{ -- return task_cpu(p); --} -- --/** -- * scx_bpf_task_cgroup - Return the sched cgroup of a task -- * @p: task of interest -- * -- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with -- * from the scheduler's POV. SCX operations should use this function to -- * determine @p's current cgroup as, unlike following @p->cgroups, -- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all -- * rq-locked operations. Can be called on the parameter tasks of rq-locked -- * operations. The restriction guarantees that @p's rq is locked by the caller. -- */ --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) --{ -- struct task_group *tg = p->sched_task_group; -- struct cgroup *cgrp = &cgrp_dfl_root.cgrp; -- -- if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) -- goto out; -- -- /* -- * A task_group may either be a cgroup or an autogroup. In the latter -- * case, @tg->css.cgroup is %NULL. A task_group can't become the other -- * kind once created. -- */ -- if (tg && tg->css.cgroup) -- cgrp = tg->css.cgroup; -- else -- cgrp = &cgrp_dfl_root.cgrp; --out: -- cgroup_get(cgrp); -- return cgrp; --} -- --BTF_SET8_START(scx_kfunc_ids_any) --BTF_ID_FLAGS(func, scx_bpf_kick_cpu) --BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) --BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) --BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu) --BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) --BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) --BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) --BTF_ID_FLAGS(func, scx_bpf_task_running) --BTF_ID_FLAGS(func, scx_bpf_task_cpu) --BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE) --BTF_SET8_END(scx_kfunc_ids_any) -- --static const struct btf_kfunc_id_set scx_kfunc_set_any = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_any, --}; -- --__diag_pop(); -- --/* -- * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set() -- * needs most of the system to be up. -- */ --static int __init register_ext_kfuncs(void) --{ -- int ret; -- -- /* -- * Some kfuncs are context-sensitive and can only be called from -- * specific SCX ops. They are grouped into BTF sets accordingly. -- * Unfortunately, BPF currently doesn't have a way of enforcing such -- * restrictions. Eventually, the verifier should be able to enforce -- * them. For now, register them the same and make each kfunc explicitly -- * check using scx_kf_allowed(). -- */ -- if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_init)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_sleepable)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_enqueue_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_cpu_release)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_any))) { -- pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret); -- return ret; -- } -- -- return 0; --} --__initcall(register_ext_kfuncs); -diff --git b/kernel/sched/ext.h a/kernel/sched/ext.h -deleted file mode 100755 -index fba8ed845f99..000000000000 ---- b/kernel/sched/ext.h -+++ /dev/null -@@ -1,257 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ -- -- --/* -- * Tag marking a kernel function as a kfunc. This is meant to minimize the -- * amount of copy-paste that kfunc authors have to include for correctness so -- * as to avoid issues such as the compiler inlining or eliding either a static -- * kfunc, or a global kfunc in an LTO build. -- */ --#define __bpf_kfunc __used noinline -- --enum scx_wake_flags { -- /* expose select WF_* flags as enums */ -- SCX_WAKE_EXEC = WF_EXEC, -- SCX_WAKE_FORK = WF_FORK, -- SCX_WAKE_TTWU = WF_TTWU, -- SCX_WAKE_SYNC = WF_SYNC, --}; -- --enum scx_enq_flags { -- /* expose select ENQUEUE_* flags as enums */ -- SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, -- SCX_ENQ_HEAD = ENQUEUE_HEAD, -- -- /* high 32bits are SCX specific */ -- -- /* -- * Set the following to trigger preemption when calling -- * scx_bpf_dispatch() with a local dsq as the target. The slice of the -- * current task is cleared to zero and the CPU is kicked into the -- * scheduling path. Implies %SCX_ENQ_HEAD. -- */ -- SCX_ENQ_PREEMPT = 1LLU << 32, -- -- /* -- * The task being enqueued was previously enqueued on the current CPU's -- * %SCX_DSQ_LOCAL, but was removed from it in a call to the -- * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was -- * invoked in a ->cpu_release() callback, and the task is again -- * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the -- * task will not be scheduled on the CPU until at least the next invocation -- * of the ->cpu_acquire() callback. -- */ -- SCX_ENQ_REENQ = 1LLU << 40, -- -- /* -- * The task being enqueued is the only task available for the cpu. By -- * default, ext core keeps executing such tasks but when -- * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with -- * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set. -- * -- * If the BPF scheduler wants to continue executing the task, -- * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately. -- * If the task gets queued on a different dsq or the BPF side, the BPF -- * scheduler is responsible for triggering a follow-up scheduling event. -- * Otherwise, Execution may stall. -- */ -- SCX_ENQ_LAST = 1LLU << 41, -- -- /* -- * A hint indicating that it's advisable to enqueue the task on the -- * local dsq of the currently selected CPU. Currently used by -- * select_cpu_dfl() and together with %SCX_ENQ_LAST. -- */ -- SCX_ENQ_LOCAL = 1LLU << 42, -- -- /* high 8 bits are internal */ -- __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, -- -- SCX_ENQ_CLEAR_OPSS = 1LLU << 56, -- SCX_ENQ_DSQ_PRIQ = 1LLU << 57, --}; -- --enum scx_deq_flags { -- /* expose select DEQUEUE_* flags as enums */ -- SCX_DEQ_SLEEP = DEQUEUE_SLEEP, -- -- /* high 32bits are SCX specific */ -- -- /* -- * The generic core-sched layer decided to execute the task even though -- * it hasn't been dispatched yet. Dequeue from the BPF side. -- */ -- SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, --}; -- --enum scx_tg_flags { -- SCX_TG_ONLINE = 1U << 0, -- SCX_TG_INITED = 1U << 1, --}; -- --enum scx_kick_flags { -- SCX_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -- SCX_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ --}; -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --extern const struct sched_class ext_sched_class; --extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops; --extern const struct file_operations sched_ext_fops; --extern unsigned long scx_watchdog_timeout; --extern unsigned long scx_watchdog_timestamp; -- --DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); --DECLARE_STATIC_KEY_FALSE(__scx_switched_all); --#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) --#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) -- --DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); -- --bool task_on_scx(struct task_struct *p); --void scx_pre_fork(struct task_struct *p); --int scx_fork(struct task_struct *p); --void scx_post_fork(struct task_struct *p); --void scx_cancel_fork(struct task_struct *p); --int scx_check_setscheduler(struct task_struct *p, int policy); --bool scx_can_stop_tick(struct rq *rq); --void init_sched_ext_class(void); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...); --#define scx_ops_error(fmt, args...) \ -- scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args) -- --void __scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active); -- --static inline void scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active) --{ -- if (!scx_enabled()) -- return; --#ifdef CONFIG_SMP -- /* -- * Pairs with the smp_load_acquire() issued by a CPU in -- * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -- * resched. -- */ -- smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1); --#endif -- if (!static_branch_unlikely(&scx_ops_cpu_preempt)) -- return; -- __scx_notify_pick_next_task(rq, p, active); --} -- --//extern void scx_scheduler_tick(void); --static inline void scx_notify_sched_tick(void) --{ -- unsigned long last_check; -- -- if (!scx_enabled()) -- return; -- //scx_scheduler_tick(); -- -- last_check = scx_watchdog_timestamp; -- if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "watchdog failed to check in for %u.%03us", -- dur_ms / 1000, dur_ms % 1000); -- } --} -- --static inline const struct sched_class *next_active_class(const struct sched_class *class) --{ -- class++; -- if (scx_switched_all() && class == &fair_sched_class) -- class++; -- if (!scx_enabled() && class == &ext_sched_class) -- class++; -- return class; --} -- --#define for_active_class_range(class, _from, _to) \ -- for (class = (_from); class != (_to); class = next_active_class(class)) -- --#define for_each_active_class(class) \ -- for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -- --/* -- * SCX requires a balance() call before every pick_next_task() call including -- * when waking up from idle. -- */ --#define for_balance_class_range(class, prev_class, end_class) \ -- for_active_class_range(class, (prev_class) > &ext_sched_class ? \ -- &ext_sched_class : (prev_class), (end_class)) -- --#ifdef CONFIG_SCHED_CORE --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi); --#endif -- --#else /* CONFIG_SCHED_CLASS_EXT */ -- --#define scx_enabled() false --#define scx_switched_all() false -- --static inline void scx_pre_fork(struct task_struct *p) {} --static inline int scx_fork(struct task_struct *p) { return 0; } --static inline void scx_post_fork(struct task_struct *p) {} --static inline void scx_cancel_fork(struct task_struct *p) {} --static inline int scx_check_setscheduler(struct task_struct *p, -- int policy) { return 0; } --static inline bool scx_can_stop_tick(struct rq *rq) { return true; } --static inline void init_sched_ext_class(void) {} --static inline void scx_notify_pick_next_task(struct rq *rq, -- const struct task_struct *p, -- const struct sched_class *active) {} --static inline void scx_notify_sched_tick(void) {} -- --#define for_each_active_class for_each_class --#define for_balance_class_range for_class_range -- --#endif /* CONFIG_SCHED_CLASS_EXT */ -- --#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) --void __scx_update_idle(struct rq *rq, bool idle); -- --static inline void scx_update_idle(struct rq *rq, bool idle) --{ -- if (scx_enabled()) -- __scx_update_idle(rq, idle); --} --#else --static inline void scx_update_idle(struct rq *rq, bool idle) {} --#endif -- --#ifdef CONFIG_CGROUP_SCHED --#ifdef CONFIG_EXT_GROUP_SCHED --int scx_tg_online(struct task_group *tg); --void scx_tg_offline(struct task_group *tg); --int scx_cgroup_can_attach(struct cgroup_taskset *tset); --void scx_move_task(struct task_struct *p); --void scx_cgroup_finish_attach(void); --void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); --void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); --#else /* CONFIG_EXT_GROUP_SCHED */ --static inline int scx_tg_online(struct task_group *tg) { return 0; } --static inline void scx_tg_offline(struct task_group *tg) {} --static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } --static inline void scx_move_task(struct task_struct *p) {} --static inline void scx_cgroup_finish_attach(void) {} --static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} --static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} --#endif /* CONFIG_EXT_GROUP_SCHED */ --#endif /* CONFIG_CGROUP_SCHED */ -diff --git b/kernel/sched/hmbird.h a/kernel/sched/hmbird.h -new file mode 100755 -index 000000000000..fa76c7f09977 ---- /dev/null -+++ a/kernel/sched/hmbird.h -@@ -0,0 +1,342 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#ifndef _HMBIRD_H_ -+#define _HMBIRD_H_ -+ -+/* -+ * Tag marking a kernel function as a kfunc. This is meant to minimize the -+ * amount of copy-paste that kfunc authors have to include for correctness so -+ * as to avoid issues such as the compiler inlining or eliding either a static -+ * kfunc, or a global kfunc in an LTO build. -+ */ -+ -+#define DEBUG_INTERNAL (1 << 0) -+#define DEBUG_INFO_TRACE (1 << 1) -+#define DEBUG_INFO_SYSTRACE (1 << 2) -+ -+#define NUMS_CGROUP_KINDS (512) -+#define SLIM_FOR_SGAME (1 << 0) -+#define SLIM_FOR_GENSHIN (1 << 1) -+ -+#ifdef MAX_TASK_NR -+#define MAX_KEY_THREAD_RECORD ((MAX_TASK_NR + 1) >> 1) -+#else -+#define MAX_KEY_THREAD_RECORD (1 << 3) -+#endif /* MAX_TASK_NR */ -+ -+#define TOP_TASK_SHIFT (8) -+#define TOP_TASK_MAX (1 << TOP_TASK_SHIFT) -+#ifndef TOP_TASK_BITS_MASK -+#define TOP_TASK_BITS_MASK (TOP_TASK_MAX - 1) -+#endif /* TOP_TASK_BITS_MASK */ -+ -+extern struct md_info_t *md_info; -+ -+#define debug_enabled() \ -+ (unlikely(hmbirdcore_debug)) -+ -+#define hmbird_debug(fmt, ...) \ -+ pr_info(":"fmt, ##__VA_ARGS__) -+ -+#define hmbird_err(id, fmt, ...) \ -+do { \ -+ pr_err(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_deferred_err(id, fmt, ...) \ -+do { \ -+ printk_deferred(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \ -+do { \ -+ if (unlikely(cond)) { \ -+ hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+ } \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_info_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_info_trace(fmt, ...) -+#endif -+ -+#define hmbird_info_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+ } \ -+} while (0) -+ -+#define hmbird_output_systrace(fmt, ...) \ -+do { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_internal_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_internal_trace(fmt, ...) -+#endif -+ -+#define hmbird_internal_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) { \ -+ hmbird_output_systrace(fmt, ##__VA_ARGS__); \ -+ } \ -+} while (0) -+ -+enum hmbird_wake_flags { -+ /* expose select WF_* flags as enums */ -+ HMBIRD_WAKE_EXEC = WF_EXEC, -+ HMBIRD_WAKE_FORK = WF_FORK, -+ HMBIRD_WAKE_TTWU = WF_TTWU, -+ HMBIRD_WAKE_SYNC = WF_SYNC, -+}; -+ -+enum hmbird_enq_flags { -+ /* expose select ENQUEUE_* flags as enums */ -+ HMBIRD_ENQ_WAKEUP = ENQUEUE_WAKEUP, -+ HMBIRD_ENQ_HEAD = ENQUEUE_HEAD, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * Set the following to trigger preemption when calling -+ * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the -+ * current task is cleared to zero and the CPU is kicked into the -+ * scheduling path. Implies %HMBIRD_ENQ_HEAD. -+ */ -+ HMBIRD_ENQ_PREEMPT = 1LLU << 32, -+ HMBIRD_ENQ_REENQ = 1LLU << 40, -+ HMBIRD_ENQ_LAST = 1LLU << 41, -+ -+ /* -+ * A hint indicating that it's advisable to enqueue the task on the -+ * local dsq of the currently selected CPU. Currently used by -+ * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST. -+ */ -+ HMBIRD_ENQ_LOCAL = 1LLU << 42, -+ -+ /* high 8 bits are internal */ -+ __HMBIRD_ENQ_INTERNAL_MASK = 0xffLLU << 56, -+ -+ HMBIRD_ENQ_CLEAR_OPSS = 1LLU << 56, -+ HMBIRD_ENQ_DSQ_PRIQ = 1LLU << 57, -+}; -+ -+enum hmbird_deq_flags { -+ /* expose select DEQUEUE_* flags as enums */ -+ HMBIRD_DEQ_SLEEP = DEQUEUE_SLEEP, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * The generic core-sched layer decided to execute the task even though -+ * it hasn't been dispatched yet. Dequeue from the BPF side. -+ */ -+ HMBIRD_DEQ_CORE_SCHED_EXEC = 1LLU << 32, -+}; -+ -+enum hmbird_kick_flags { -+ HMBIRD_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -+ HMBIRD_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ -+}; -+ -+enum hmbird_task_prop_type { -+ HMBIRD_TASK_PROP_COMMON, -+ HMBIRD_TASK_PROP_PIPELINE, -+ HMBIRD_TASK_PROP_DEBUG_OR_LOG, -+ HMBIRD_TASK_PROP_HIGH_LOAD, -+ HMBIRD_TASK_PROP_IO, -+ HMBIRD_TASK_PROP_NETWORK, -+ HMBIRD_TASK_PROP_PERIODIC, -+ HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL, -+ HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL, -+ HMBIRD_TASK_PROP_ISOLATE, -+ HMBIRD_TASK_PROP_MAX, -+}; -+ -+enum hmbird_preempt_policy_id { -+ HMBIRD_PREEMPT_POLICY_NONE, -+ HMBIRD_PREEMPT_POLICY_PRIO_BASED, -+}; -+ -+extern const struct sched_class hmbird_sched_class; -+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops; -+extern const struct file_operations sched_hmbird_fops; -+extern unsigned long hmbird_watchdog_timeout; -+extern unsigned long hmbird_watchdog_timestamp; -+ -+enum scx_rq_flags { -+ HMBIRD_RQ_CAN_STOP_TICK = 1 << 0, -+}; -+ -+struct hmbird_rq { -+ struct hmbird_dispatch_q local_dsq; -+ struct list_head watchdog_list; -+ u64 ops_qseq; -+ u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -+ u32 nr_running; -+ u32 flags; -+ bool cpu_released; -+ cpumask_var_t cpus_to_kick; -+ cpumask_var_t cpus_to_preempt; -+ cpumask_var_t cpus_to_wait; -+ u64 pnt_seq; -+ u64 *prev_runnable_sum_fixed; -+ u32 *prev_window_size; -+ struct irq_work kick_cpus_irq_work; -+ bool pipeline; -+ bool exclusive; -+ bool period_disallow; -+ bool nonperiod_disallow; -+ struct rq *rq; -+ struct hmbird_sched_rq_stats *srq; -+}; -+ -+struct hmbird_sched_change_guard { -+ struct task_struct *p; -+ struct rq *rq; -+ bool queued; -+ bool running; -+ bool done; -+}; -+ -+extern struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -+ -+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags); -+ -+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -+ for (struct hmbird_sched_change_guard __cg = \ -+ hmbird_sched_change_guard_init(__rq, __p, __flags); \ -+ !__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags)) -+ -+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+bool task_on_hmbird(struct task_struct *p); -+int hmbird_pre_fork(struct task_struct *p); -+void hmbird_post_fork(struct task_struct *p); -+void hmbird_cancel_fork(struct task_struct *p); -+int hmbird_check_setscheduler(struct task_struct *p, int policy); -+bool hmbird_can_stop_tick(struct rq *rq); -+void init_sched_hmbird_class(void); -+ -+void hmbird_ops_exit(void); -+#define hmbird_ops_error(fmt, ...) \ -+do { \ -+ hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__); \ -+ hmbird_ops_exit(); \ -+} while (0) -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active); -+ -+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight) -+{ -+ return clamp_t(unsigned long, -+ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -+ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); -+} -+ -+static inline void hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active) -+{ -+ if (!hmbird_enabled()) -+ return; -+#ifdef CONFIG_SMP -+ /* -+ * Pairs with the smp_load_acquire() issued by a CPU in -+ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -+ * resched. -+ */ -+ smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1); -+#endif -+ if (!static_branch_unlikely(&hmbird_ops_cpu_preempt)) -+ return; -+ __hmbird_notify_pick_next_task(rq, p, active); -+} -+ -+extern void hmbird_scheduler_tick(void); -+void scan_timeout(struct rq *rq); -+void hmbird_notify_sched_tick(void); -+ -+static inline const struct sched_class *next_active_class(const struct sched_class *class) -+{ -+ class++; -+ -+ if (hmbird_enabled() && class == &fair_sched_class) -+ class++; -+ if (!hmbird_enabled() && class == &hmbird_sched_class) -+ class++; -+ return class; -+} -+ -+#define for_active_class_range(class, _from, _to) \ -+ for (class = (_from); class != (_to); class = next_active_class(class)) -+ -+#define for_each_active_class(class) \ -+ for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -+ -+/* -+ * HMBIRD requires a balance() call before every pick_next_task() call including -+ * when waking up from idle. -+ */ -+#define for_balance_class_range(class, prev_class, end_class) \ -+ for_active_class_range(class, (prev_class) > &hmbird_sched_class ? \ -+ &hmbird_sched_class : (prev_class), (end_class)) -+ -+#define MIN_CGROUP_DL_IDX (5) /* 8ms */ -+#define DEFAULT_CGROUP_DL_IDX (8) /* 64ms */ -+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS]; -+ -+ -+void __hmbird_update_idle(struct rq *rq, bool idle); -+ -+static inline void hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ if (hmbird_enabled()) -+ __hmbird_update_idle(rq, idle); -+} -+ -+int hmbird_ctrl(bool enable); -+ -+int hmbird_tg_online(struct task_group *tg); -+ -+ -+static inline u16 hmbird_task_util(struct task_struct *p) -+{ -+ return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0; -+} -+u16 hmbird_cpu_util(int cpu); -+ -+bool get_hmbird_ops_enabled(void); -+bool get_non_hmbird_task(void); -+void set_cpu_cluster(u64 cpu_cluster); -+#endif /*_EXT_H_*/ -diff --git b/kernel/sched/hmbird/hmbird.c a/kernel/sched/hmbird/hmbird.c -new file mode 100755 -index 000000000000..86f9fd092424 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird.c -@@ -0,0 +1,4354 @@ -+// SPDX-License-Identifier: GPL-2.0 -+ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#include -+#include -+ -+#include "slim.h" -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+#include -+ -+#define CLUSTER_SEPARATE -+ -+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0); -+atomic_t non_hmbird_task; -+atomic_t hmbird_module_loaded = ATOMIC_INIT(0); -+int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0}; -+ -+enum hmbird_internal_consts { -+ HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ, -+}; -+ -+enum hmbird_ops_enable_state { -+ HMBIRD_OPS_PREPPING, -+ HMBIRD_OPS_ENABLING, -+ HMBIRD_OPS_ENABLED, -+ HMBIRD_OPS_DISABLING, -+ HMBIRD_OPS_DISABLED, -+}; -+ -+static inline void put_hmbird_ts(struct task_struct *p) -+{ -+ kfree((void *)p->android_oem_data1[HMBIRD_TS_IDX]); -+ p->android_oem_data1[HMBIRD_TS_IDX] = 0; -+} -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+/* -+ * hmbird_entity->ops_state -+ * -+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler. -+ * State transitions look as follows: -+ * -+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING -+ * ^ | | -+ * | v v -+ * \-------------------------------/ -+ * -+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -+ * sites for explanations on the conditions being waited upon and why they are -+ * safe. Transitions out of them into NONE or QUEUED must store_release and the -+ * waiters should load_acquire. -+ * -+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether -+ * any given task can be dispatched by the BPF scheduler at all times and thus -+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core -+ * can safely reject invalid dispatches. -+ */ -+enum hmbird_ops_state { -+ HMBIRD_OPSS_NONE, /* owned by the HMBIRD core */ -+ HMBIRD_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -+ HMBIRD_OPSS_QUEUED, /* owned by the BPF scheduler */ -+ HMBIRD_OPSS_DISPATCHING, /* in transit back to the HMBIRD core */ -+ -+ /* -+ * QSEQ brands each QUEUED instance so that, when dispatch races -+ * dequeue/requeue, the dispatcher can tell whether it still has a claim -+ * on the task being dispatched. -+ */ -+ HMBIRD_OPSS_QSEQ_SHIFT = 2, -+ HMBIRD_OPSS_STATE_MASK = (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1, -+ HMBIRD_OPSS_QSEQ_MASK = ~HMBIRD_OPSS_STATE_MASK, -+}; -+ -+enum switch_stat { -+ HMBIRD_DISABLED, -+ HMBIRD_SWITCH_PREP, -+ HMBIRD_RQ_SWITCH_BEGIN, -+ HMBIRD_RQ_SWITCH_DONE, -+ HMBIRD_ENABLED, -+}; -+enum switch_stat curr_ss; -+ -+/* -+ * During exit, a task may schedule after losing its PIDs. When disabling the -+ * BPF scheduler, we need to be able to iterate tasks in every state to -+ * guarantee system safety. Maintain a dedicated task list which contains every -+ * task between its fork and eventual free. -+ */ -+DEFINE_SPINLOCK(hmbird_tasks_lock); -+static LIST_HEAD(hmbird_tasks); -+ -+/* ops enable/disable */ -+static struct kthread_worker *hmbird_ops_helper; -+static DEFINE_MUTEX(hmbird_ops_enable_mutex); -+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem); -+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED); -+ -+static bool hmbird_warned_zero_slice; -+enum hmbird_switch_type sw_type; -+static int skip_num[MAX_GLOBAL_DSQS]; -+static int big_distribute_mask_prev; -+static int little_distribute_mask_prev; -+ -+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0); -+ -+/* -+ * The maximum amount of time in jiffies that a task may be runnable without -+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger -+ * hmbird_ops_error(). -+ */ -+unsigned long hmbird_watchdog_timeout; -+ -+/* -+ * The last time the delayed work was run. This delayed work relies on -+ * ksoftirqd being able to run to service timer interrupts, so it's possible -+ * that this work itself could get wedged. To account for this, we check that -+ * it's not stalled in the timer tick, and trigger an error if it is. -+ */ -+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES; -+ -+static struct delayed_work hmbird_watchdog_work; -+static struct work_struct hmbird_err_exit_work; -+ -+/* idle tracking */ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_CPUMASK_OFFSTACK -+#define CL_ALIGNED_IF_ONSTACK -+#else -+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp -+#endif -+ -+static struct { -+ cpumask_var_t cpu; -+ cpumask_var_t smt; -+} idle_masks CL_ALIGNED_IF_ONSTACK; -+ -+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus; -+#endif /* CONFIG_SMP */ -+ -+/* dispatch queues */ -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global; -+ -+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128}; -+u32 pcp_dsq_deadline = 20; -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS]; -+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq); -+ -+static u64 max_hmbird_dsq_internal_id; -+ -+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/ -+#define CLUSTER_SEPARATE_IDX (8) -+#define GDSQS_ID_BASE (3) -+#define UX_COMPATIBLE_IDX (4) -+#define NON_PERIOD_START (5) -+#define NON_PERIOD_END (MAX_GLOBAL_DSQS) -+#define CREATE_DSQ_LEVEL_WITHIN (1) -+ -+struct hmbird_sched_info { -+ spinlock_t lock; -+ int curr_idx[2]; -+ int rtime[MAX_GLOBAL_DSQS]; -+}; -+ -+struct pcp_sched_info { -+ s64 pcp_seq; -+ int rtime; -+ bool pcp_round; -+}; -+ -+/* -+ * pcp_info may rw by another cpu. -+ * protected by rq->lock. -+ */ -+atomic64_t pcp_dsq_round; -+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info); -+ -+struct md_info_t *md_info; -+ -+static int b_rescue_l, l_rescue_b; -+static struct hmbird_sched_info sinfo; -+ -+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC; -+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = { -+ 0, 0, 0, 0, 0, -+ 32 * NSEC_PER_MSEC, -+ 20 * NSEC_PER_MSEC, -+ 14 * NSEC_PER_MSEC, -+ 8 * NSEC_PER_MSEC, -+ 6 * NSEC_PER_MSEC -+}; -+ -+struct cluster_ctx { -+ /* cpu-dsq map must within [lower, upper) */ -+ int upper; -+ int lower; -+ int tidx; -+}; -+ -+enum stat_items { -+ GLOBAL_STAT, -+ CPU_ALLOW_FAIL, -+ RT_CNT, -+ KEY_TASK_CNT, -+ SWITCH_IDX, -+ TIMEOUT_CNT, -+ -+ TOTAL_DSP_CNT, -+ MOVE_RQ_CNT, -+ SELECT_CPU, -+ -+ DWORD_STAT_END = SELECT_CPU, -+ -+ GDSQ_CNT, -+ ERR_IDX, -+ PCP_TIMEOUT_CNT, -+ PCP_LDSQ_CNT, -+ PCP_ENQL_CNT, -+ -+ MAX_ITEMS, -+}; -+static DEFINE_SPINLOCK(stats_lock); -+static char *stats_str[MAX_ITEMS] = { -+ "global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt", -+ "switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt", -+ "select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt", -+ "pcp_ldsq_cnt", "pcp_enql_cnt" -+}; -+ -+ -+struct stats_struct { -+ u64 global_stat[2]; -+ u64 cpu_allow_fail[2]; -+ u64 rt_cnt[2]; -+ u64 key_task_cnt[2]; -+ u64 switch_idx[2]; -+ u64 timeout_cnt[2]; -+ -+ u64 total_dsp_cnt[2]; -+ u64 move_rq_cnt[2]; -+ u64 select_cpu[2]; -+ -+ u64 gdsq_count[MAX_GLOBAL_DSQS][2]; -+ u64 err_idx[5]; -+ u64 pcp_timeout_cnt[NR_CPUS]; -+ u64 pcp_ldsq_count[NR_CPUS][2]; -+ u64 pcp_enql_cnt[NR_CPUS]; -+} stats_data; -+ -+static struct { -+ cpumask_var_t ex_free; -+ cpumask_var_t exclusive; -+ cpumask_var_t partial; -+ cpumask_var_t big; -+ cpumask_var_t little; -+} iso_masks __read_mostly; -+ -+/* -+ * Need more synchronization for these two variables? -+ * I choose not to. -+ */ -+static int l_need_rescue, b_need_rescue; -+ -+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...) \ -+{ \ -+ char buf[MAX_FATAL_INFO]; \ -+ \ -+ scnprintf(buf, MAX_FATAL_INFO, fmt, ##args); \ -+ hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf); \ -+ trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable), \ -+ READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf); \ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); \ -+} \ -+ -+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2) -+{ -+ int c1, c2; -+ struct cpumask mask = {.bits = {0}}; -+ struct cpumask tmp = {.bits = {0}}; -+ -+ if (!slim_stats) -+ return false; -+ -+ if (!rq1 || !rq2) -+ return false; -+ -+ c1 = cpu_of(rq1); -+ c2 = cpu_of(rq2); -+ cpumask_set_cpu(c1, &mask); -+ cpumask_set_cpu(c2, &mask); -+ -+ if (cpumask_and(&tmp, iso_masks.little, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.big, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.partial, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ return false; -+} -+ -+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu) -+{ -+ unsigned long flags; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ if (!slim_stats) -+ return; -+ -+ switch (item) { -+ case GLOBAL_STAT: -+ fallthrough; -+ case CPU_ALLOW_FAIL: -+ fallthrough; -+ case RT_CNT: -+ fallthrough; -+ case KEY_TASK_CNT: -+ fallthrough; -+ case SWITCH_IDX: -+ fallthrough; -+ case TIMEOUT_CNT: -+ fallthrough; -+ case TOTAL_DSP_CNT: -+ fallthrough; -+ case MOVE_RQ_CNT: -+ fallthrough; -+ case SELECT_CPU: -+ pval = pbase + item * 2 + idx; -+ break; -+ case GDSQ_CNT: -+ pval = &stats_data.gdsq_count[dsq_id][idx]; -+ break; -+ case ERR_IDX: -+ pval = &stats_data.err_idx[idx]; -+ break; -+ case PCP_TIMEOUT_CNT: -+ pval = &stats_data.pcp_timeout_cnt[cpu]; -+ break; -+ case PCP_LDSQ_CNT: -+ pval = &stats_data.pcp_ldsq_count[cpu][idx]; -+ break; -+ case PCP_ENQL_CNT: -+ pval = &stats_data.pcp_enql_cnt[cpu]; -+ break; -+ default: -+ return; -+ } -+ -+ spin_lock_irqsave(&stats_lock, flags); -+ *pval += 1; -+ spin_unlock_irqrestore(&stats_lock, flags); -+} -+ -+static inline bool handle_ret(int ret, int *idx, int len) -+{ -+ if (ret < 0 || ret >= len - *idx) -+ return true; -+ *idx += ret; -+ return false; -+} -+ -+#define PRINT_INTV (5 * HZ) -+void stats_print(char *buf, int len) -+{ -+ int idx = 0, i, j, ret; -+ int item = 0; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ for (item = 0; item < MAX_ITEMS; item++) { -+ if (item <= DWORD_STAT_END) { -+ pval = pbase + item * 2; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n", -+ stats_str[item], pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == GDSQ_CNT) { -+ for (j = 0; j < MAX_GLOBAL_DSQS; j++) { -+ pval = (u64 *)&stats_data.gdsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == ERR_IDX) { -+ pval = (u64 *)&stats_data.err_idx; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n", -+ stats_str[item], pval[0], -+ pval[1], pval[2], pval[3], pval[4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == PCP_TIMEOUT_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_timeout_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_LDSQ_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_ldsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_ENQL_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_enql_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } -+ } -+ -+ if (!md_info) { -+ buf[idx] = '\0'; -+ return; -+ } -+ -+ ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_SWITCHS; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "sw_rec[%d] = %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.sw_rec[i].switch_at, -+ md_info->kern_dump.sw_rec[i].is_success, -+ md_info->kern_dump.sw_rec[i].end_state, -+ md_info->kern_dump.sw_rec[i].switch_reason); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_EXCEP_ID; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "excep[%d] = %llu %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.excep_rec[i][0], -+ md_info->kern_dump.excep_rec[i][1], -+ md_info->kern_dump.excep_rec[i][2], -+ md_info->kern_dump.excep_rec[i][3], -+ md_info->kern_dump.excep_rec[i][4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n", -+ i, md_info->kern_dump.excep_idx[i]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ -+ buf[idx] = '\0'; -+} -+ -+enum cpu_type { -+ LITTLE, -+ BIG, -+ PARTIAL, -+ EXCLUSIVE, -+ EX_FREE, -+ INVALID -+}; -+ -+enum dsq_type { -+ GLOBAL_DSQ, -+ PCP_DSQ, -+ OTHER, -+ MAX_DSQ_TYPE, -+}; -+ -+static enum cpu_type cpu_cluster(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (get_hmbird_rq(rq)->exclusive) { -+ return EXCLUSIVE; -+ } else { -+ if (cpumask_test_cpu(cpu, iso_masks.little)) -+ return LITTLE; -+ else if (cpumask_test_cpu(cpu, iso_masks.big)) -+ return BIG; -+ else if (cpumask_test_cpu(cpu, iso_masks.partial)) -+ return PARTIAL; -+ else if (cpumask_test_cpu(cpu, iso_masks.exclusive)) -+ return EXCLUSIVE; -+ else if (cpumask_test_cpu(cpu, iso_masks.ex_free)) -+ return EX_FREE; -+ } -+ return INVALID; -+} -+ -+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq) -+{ -+ if (!dsq) -+ return OTHER; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= GDSQS_ID_BASE) && -+ ((dsq->id & 0xff) < MAX_GLOBAL_DSQS)) -+ return GLOBAL_DSQ; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) && -+ ((dsq->id & 0xff) < max_hmbird_dsq_internal_id)) -+ return PCP_DSQ; -+ -+ return OTHER; -+} -+ -+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq) -+{ -+ enum dsq_type type; -+ -+ type = get_dsq_type(dsq); -+ switch (type) { -+ case GLOBAL_DSQ: -+ case PCP_DSQ: -+ return (dsq->id & 0xff) - GDSQS_ID_BASE; -+ default: -+ return -1; -+ } -+ return -1; -+} -+ -+static void update_cpus_idle(bool set, struct cpumask *mask) -+{ -+ int cpu; -+ struct rq *rq; -+ -+ if (set) { -+ for_each_cpu(cpu, mask) { -+ rq = cpu_rq(cpu); -+ if (is_idle_task(rq->curr)) -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ } -+ } else -+ cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask); -+} -+ -+static void set_partial_status(bool enable, bool little, bool big) -+{ -+ WRITE_ONCE(partial_enable, enable); -+ WRITE_ONCE(l_need_rescue, little); -+ WRITE_ONCE(b_need_rescue, big); -+} -+ -+static bool is_little_need_rescue(void) -+{ -+ return READ_ONCE(l_need_rescue); -+} -+ -+static bool is_big_need_rescue(void) -+{ -+ return READ_ONCE(b_need_rescue); -+} -+ -+static bool is_partial_enabled(void) -+{ -+ return READ_ONCE(partial_enable); -+} -+ -+static bool is_partial_cpu(int cpu) -+{ -+ return cpumask_test_cpu(cpu, iso_masks.partial); -+} -+ -+static void set_iso_par_free(bool enable) -+{ -+ WRITE_ONCE(isolate_ctrl, enable); -+} -+ -+static bool is_iso_par_free(void) -+{ -+ return READ_ONCE(isolate_ctrl); -+} -+ -+static bool skip_update_idle(void) -+{ -+ int cpu = smp_processor_id(); -+ enum cpu_type type = cpu_cluster(cpu); -+ -+ if ((type == EXCLUSIVE && !is_iso_par_free()) || -+ /* partial enable may changed during idle, it doesn't matter. */ -+ (!is_partial_enabled() && type == PARTIAL)) -+ return true; -+ -+ return false; -+} -+ -+static void init_isolate_cpus(void) -+{ -+ WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -+ cpumask_set_cpu(0, iso_masks.big); -+ cpumask_set_cpu(1, iso_masks.big); -+ cpumask_set_cpu(2, iso_masks.big); -+ cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(4, iso_masks.big); -+ cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(6, iso_masks.partial); -+ cpumask_set_cpu(7, iso_masks.ex_free); -+} -+ -+extern spinlock_t css_set_lock; -+ -+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp) -+{ -+ int i; -+ struct cgroup *anc; -+ -+ spin_lock_irq(&css_set_lock); -+ for (i = 0; i < cgrp->level; i++) { -+ anc = cgrp->ancestors[i]; -+ if (anc->level != CREATE_DSQ_LEVEL_WITHIN) -+ continue; -+ cgroup_get(anc); -+ spin_unlock_irq(&css_set_lock); -+ return anc; -+ } -+ spin_unlock_irq(&css_set_lock); -+ hmbird_err(NO_CGROUP_L1, ":error cgroup = %s\n", cgrp->kn->name); -+ return NULL; -+} -+ -+#define PCP_IDX_BIT (1 << 31) -+ -+static bool is_pcp_rt(struct task_struct *p) -+{ -+ return rt_prio(p->prio) && (p->nr_cpus_allowed == 1); -+} -+ -+static bool is_pcp_idx(int idx) -+{ -+ return idx >= MAX_GLOBAL_DSQS; -+} -+ -+static bool is_critical_system_task(struct task_struct *p) -+{ -+ int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK; -+ -+ return (p->prio < (MAX_RT_PRIO >> 1) && -+ (sp_dl < SCHED_PROP_DEADLINE_LEVEL3)); -+} -+ -+#define ISOLATE_TASK_TOP_BIT (1 << 17) -+#define PIPELINE_TASK_TOP_BIT (1 << 9) -+static bool is_pipeline_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT); -+} -+ -+static bool is_isolate_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT); -+} -+ -+static bool is_critical_app_task_without_isolate(struct task_struct *p) -+{ -+ return task_is_top_task(p) && !is_isolate_task(p); -+} -+ -+static int find_idx_from_task(struct task_struct *p) -+{ -+ int idx, cpu; -+ int sp_dl; -+ struct task_group *tg = p->sched_task_group; -+ -+ if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) { -+ cpu = cpumask_any(p->cpus_ptr); -+ idx = cpu + MAX_GLOBAL_DSQS; -+ return idx; -+ } -+ -+ if (is_critical_system_task(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL0; -+ goto done; -+ } -+ -+ if (is_critical_app_task_without_isolate(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL1; -+ goto done; -+ } -+ -+ if (p->pid == scx_systemui_pid) { -+ idx = SCHED_PROP_DEADLINE_LEVEL4; -+ goto done; -+ } -+ -+ sp_dl = hmbird_get_dsq_id(p); -+ if (sp_dl) { -+ idx = sp_dl; -+ goto done; -+ } -+ -+ if (rt_prio(p->prio)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL3; -+ goto done; -+ } -+ -+ if (tg && tg->css.cgroup && tg->css.cgroup->kn) { -+ if (likely(tg->css.cgroup->kn->id >= 0 && -+ tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS)) -+ idx = cgroup_ids_table[tg->css.cgroup->kn->id]; -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ -+done: -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) { -+ hmbird_err(DSQ_ID_ERR, " : idx error, idx = %d-----\n", idx); -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } -+ return idx; -+} -+ -+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p) -+{ -+ int idx; -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq; -+ -+ if (!p) -+ return NULL; -+ -+ idx = find_idx_from_task(p); -+ if (is_pcp_idx(idx)) { -+ idx -= MAX_GLOBAL_DSQS; -+ dsq = &per_cpu(pcp_ldsq, idx); -+ get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq); -+ slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx); -+ } else { -+ dsq = &gdsqs[idx]; -+ get_hmbird_ts(p)->gdsq_idx = idx; -+ slim_stats_record(GDSQ_CNT, 0, idx, 0); -+ } -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ dsq->last_consume_at = jiffies; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ return dsq; -+} -+ -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq); -+ -+static void set_partial_rescue(bool p_state, bool l_over, bool b_over) -+{ -+ set_partial_status(p_state, l_over, b_over); -+ update_cpus_idle(p_state, iso_masks.partial); -+ hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled()); -+ hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue()); -+ hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue()); -+} -+ -+static void free_isocpu(bool enable) -+{ -+ set_iso_par_free(enable); -+ update_cpus_idle(enable, iso_masks.ex_free); -+ hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free()); -+} -+ -+inline u64 get_hmbird_cpu_util(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed) -+ return 0; -+ u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed); -+ u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size); -+ -+ do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT); -+ -+ return prev_runnable_sum_fixed; -+} -+ -+static inline unsigned int get_scaling_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->max; -+} -+ -+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static u64 get_cpus_max_util(struct cpumask *mask) -+{ -+ int cpu; -+ u64 max = 0; -+ u64 util, ratio; -+ unsigned long effective_cap = 0; -+ -+ for_each_cpu(cpu, mask) { -+ if (slim_walt_ctrl) -+ slim_get_cpu_util(cpu, &util); -+ else -+ util = get_hmbird_cpu_util(cpu); -+ -+ /* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/ -+ if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu))) -+ effective_cap = arch_scale_cpu_capacity(cpu); -+ else -+ effective_cap = arch_scale_cpu_capacity(cpu) * -+ get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu); -+ -+ ratio = util * 100 / effective_cap; -+ hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util); -+ hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n", -+ cpu, (u64)arch_scale_cpu_capacity(cpu)); -+ hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n", -+ cpu, (u64)effective_cap); -+ -+ if (ratio > 100) -+ ratio = 100; -+ -+ if (ratio > max) -+ max = ratio; -+ } -+ return max; -+} -+ -+static bool cluster_need_rescue(struct cpumask *mask, int hres) -+{ -+ return get_cpus_max_util(mask) > hres; -+} -+ -+void partial_dynamic_ctrl(void) -+{ -+ u64 lmax = 0, bmax = 0; -+ bool l_over, l_under, b_over, b_under; -+ static bool last_l_over, last_b_over; -+ static unsigned long last_check; -+ -+ /* Check partial every jiffies. need lock? */ -+ if (time_before_eq(jiffies, READ_ONCE(last_check))) -+ return; -+ -+ WRITE_ONCE(last_check, jiffies); -+ -+ lmax = get_cpus_max_util(iso_masks.little); -+ l_over = lmax > parctrl_high_ratio_l; -+ l_under = lmax < parctrl_low_ratio_l; -+ -+ bmax = get_cpus_max_util(iso_masks.big); -+ b_over = bmax > parctrl_high_ratio; -+ b_under = bmax < parctrl_low_ratio; -+ -+ if (is_partial_enabled() && (l_over || b_over)) { -+ if (last_l_over != l_over || last_b_over != b_over) -+ set_partial_rescue(true, l_over, b_over); -+ } else if (!is_partial_enabled() && (l_over || b_over)) { -+ set_partial_rescue(true, l_over, b_over); -+ } else if (is_partial_enabled() && l_under && b_under) { -+ set_partial_rescue(false, false, false); -+ } -+ last_l_over = l_over; -+ last_b_over = b_over; -+ -+ if (is_partial_enabled()) { -+ if (cpumask_empty(iso_masks.partial)) { -+ bmax = bmax > lmax ? bmax : lmax; -+ } else { -+ bmax = get_cpus_max_util(iso_masks.partial); -+ } -+ if (!is_iso_par_free() && bmax > isoctrl_high_ratio) { -+ hmbird_info_trace("partial max = %llu\n", bmax); -+ free_isocpu(true); -+ } else if (is_iso_par_free() && bmax < isoctrl_low_ratio) -+ free_isocpu(false); -+ } else if (is_iso_par_free()) { -+ free_isocpu(false); -+ } -+} -+ -+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx) -+{ -+ hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx); -+} -+ -+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx) -+{ -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[idx])) { -+ slim_stats_record(GDSQ_CNT, 1, idx, 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ int i; -+ -+ for (i = 0; i < UX_COMPATIBLE_IDX; i++) { -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ slim_stats_record(GDSQ_CNT, 1, i, 0); -+ return 1; -+ } -+ } -+ return 0; -+} -+ -+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) { -+ slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ goto clear_timeout; -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) -+ goto clear_timeout; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ hmbird_info_trace( -+ "dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n", -+ dsq_id_to_internal(dsq), entity->task->comm, -+ jiffies, msecs_to_jiffies(deadline), entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1); -+ return; -+ -+clear_timeout: -+ hmbird_info_trace("dsq[%d] clear timeout\n", -+ dsq_id_to_internal(dsq)); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0); -+ dsq->is_timeout = false; -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq)); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime) -+{ -+ hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime); -+} -+ -+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any) -+{ -+ bool is_timeout; -+ int cpu = cpu_of(rq); -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu); -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ is_timeout = dsq->is_timeout; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ /* -+ * dsq->is_timeout may change here, let it be. -+ * it won't cause serious problems. -+ * the same for consume_dispatch_q later. -+ */ -+ if (!is_timeout && !any) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, dsq)) { -+ if (is_timeout) { -+ hmbird_info_trace("dsq[%d] consume a pcp timeout task\n", -+ dsq_id_to_internal(dsq)); -+ update_timeout_stats(rq, dsq, pcp_dsq_deadline); -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu); -+ } -+ slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu); -+ return 1; -+ } -+ /* -+ * No pcp task, clear quota. -+ */ -+ if (any) { -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+ } -+ return 0; -+} -+ -+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf) -+{ -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ } -+ return 0; -+} -+ -+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf, -+ int tmp, int cidx, int tidx) -+{ -+ unsigned long flags; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) { -+ if (tmp != cidx) { -+ spin_lock(&sinfo.lock); -+ sinfo.curr_idx[tidx] = tmp; -+ spin_unlock(&sinfo.lock); -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]); -+ slim_stats_record(SWITCH_IDX, 0, 0, 0); -+ } -+ slim_stats_record(GDSQ_CNT, 1, tmp, 0); -+ -+ raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags); -+ gdsqs[tmp].last_consume_at = jiffies; -+ raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags); -+ return 1; -+ } -+ return 0; -+ -+} -+ -+static int get_cidx(struct cluster_ctx *ctx) -+{ -+ int cidx; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ if (cidx < ctx->lower || cidx >= ctx->upper) { -+ sinfo.curr_idx[ctx->tidx] = ctx->lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]); -+ slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ } -+ spin_unlock(&sinfo.lock); -+ -+ return cidx; -+} -+ -+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = CLUSTER_SEPARATE_IDX; -+ if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) { -+ pr_debug(" %s iso_masks.little first is %d\n", -+ __func__, cpumask_first(iso_masks.little)); -+ ctx->upper = NON_PERIOD_END; -+ } -+ ctx->tidx = 0; -+ break; -+ case LITTLE: -+ ctx->lower = CLUSTER_SEPARATE_IDX; -+ ctx->upper = NON_PERIOD_END; -+ if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) { -+ pr_debug(" %s iso_masks.big first is %d", -+ __func__, cpumask_first(iso_masks.big)); -+ ctx->lower = NON_PERIOD_START; -+ } -+ ctx->tidx = 1; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ case LITTLE: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = NON_PERIOD_END; -+ ctx->tidx = 0; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ if (cluster_separate) { -+ return gen_cluster_ctx_separate(ctx, type); -+ } else { -+ return gen_cluster_ctx_common(ctx, type); -+ } -+} -+ -+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ int i; -+ bool is_timeout; -+ unsigned long flags; -+ struct cluster_ctx ctx; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ /* Third param means only consume timeout pcp dsq. */ -+ if (consume_pcp_dsq(rq, rf, false)) -+ return 1; -+ -+ for (i = ctx.lower; i < ctx.upper; i++) { -+ raw_spin_lock_irqsave(&gdsqs[i].lock, flags); -+ is_timeout = gdsqs[i].is_timeout; -+ raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags); -+ /* gdsqs[i].is_timeout may change here, let it be... */ -+ if (is_timeout) { -+ /* -+ * consume_dispatch_q will acquire dsq-lock, -+ * So cannot keep lock here, annoy enough. -+ * may rewrite a consume_dispatch_q_locked, TODO. -+ */ -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ hmbird_info_trace("dsq[%d] consume a timeout task\n", i); -+ slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0); -+ update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]); -+ return 1; -+ } -+ } -+ } -+ return 0; -+} -+ -+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ struct cluster_ctx ctx; -+ int cidx; -+ int tmp; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ cidx = get_cidx(&ctx); -+ tmp = cidx; -+ do { -+ if (check_pcp_dsq_round(rq, rf)) -+ return 1; -+ if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx)) -+ return 1; -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[tmp] = 0; -+ systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]); -+ tmp++; -+ if (tmp >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ tmp = ctx.lower; -+ } -+ spin_unlock(&sinfo.lock); -+ } while (tmp != cidx); -+ -+ return consume_pcp_dsq(rq, rf, true); -+} -+ -+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ enum cpu_type type = cpu_cluster(cpu_of(rq)); -+ bool period_allowed = !get_hmbird_rq(rq)->period_disallow; -+ bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow; -+ -+ switch (type) { -+ case EX_FREE: -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ break; -+ case EXCLUSIVE: -+ if (!is_iso_par_free()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ } -+ /* Only non-period task can run on isolate cpus */ -+ if (!READ_ONCE(iso_free_rescue)) { -+ if (is_big_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ } else { -+ /* Free run, can run on any task. */ -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ case PARTIAL: -+ if (!is_partial_enabled()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ return 0; -+ } -+ if (is_big_need_rescue()) { -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ -+ case BIG: -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ if (is_iso_par_free() || (is_little_need_rescue() && -+ !cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) { -+ hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++); -+ return 1; -+ } -+ } -+ break; -+ -+ case LITTLE: -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ -+ if (is_iso_par_free() || (is_big_need_rescue() && -+ !cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) { -+ hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++); -+ return 1; -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return 0; -+} -+ -+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf) -+{ -+ return consume_hmbird_global_dsq(rq, rf); -+} -+ -+ -+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time) -+{ -+ int idx; -+ -+ /* which dsq belongs to while task enqueue, task will consume its running time. */ -+ idx = get_hmbird_ts(p)->gdsq_idx; -+ /* Only non-period dsq share running time between each other. */ -+ if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id) -+ return; -+ -+ if (idx >= MAX_GLOBAL_DSQS) { -+ per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } else { -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[idx] += exec_time; -+ spin_unlock(&sinfo.lock); -+ systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]); -+ } -+} -+ -+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type) -+{ -+ int cidx; -+ struct cluster_ctx ctx; -+ int cpu = cpu_of(rq); -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ if (cidx < ctx.lower || cidx >= ctx.upper) { -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ slim_stats_record(ERR_IDX, ctx.tidx, 0, 0); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ } -+ -+ while (1) { -+ if (per_cpu(pcp_info, cpu).pcp_round) { -+ if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) { -+ hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n", -+ cpu, per_cpu(pcp_info, cpu).rtime); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } -+ } -+ if (sinfo.rtime[cidx] < dsq_quota[cidx]) -+ break; -+ -+ /* clear current dsq rtime */ -+ sinfo.rtime[cidx] = 0; -+ systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]); -+ -+ sinfo.curr_idx[ctx.tidx]++; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", -+ ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ } -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ slim_stats_record(SWITCH_IDX, 1, 0, 0); -+ } -+ spin_unlock(&sinfo.lock); -+} -+ -+ -+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p) -+{ -+ enum cpu_type type; -+ -+ if (!rq || !p) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ switch (type) { -+ case PARTIAL: -+ return; -+ case EXCLUSIVE: -+ return; -+ case EX_FREE: -+ return; -+ default: -+ break; -+ } -+ update_dsq_idx(rq, p, type); -+} -+ -+ -+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ int dsq_id; -+ -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo) || dsq->is_timeout) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ hmbird_deferred_err(SCAN_ENTITY_NULL, -+ " : entity is NULL, dsq->id = %llu\n", dsq->id); -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ dsq->is_timeout = true; -+ dsq_id = dsq_id_to_internal(dsq); -+ hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n", -+ dsq_id, entity->task->comm, jiffies, entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1); -+ raw_spin_unlock(&dsq->lock); -+ -+ return true; -+} -+ -+void scan_timeout(struct rq *rq) -+{ -+ int i; -+ int cpu = cpu_of(rq); -+ struct hmbird_dispatch_q *dsq; -+ static u64 last_scan_at; -+ static DEFINE_PER_CPU(u64, pcp_last_scan_at); -+ -+ if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu))) -+ return; -+ per_cpu(pcp_last_scan_at, cpu) = jiffies; -+ -+ dsq = &per_cpu(pcp_ldsq, cpu); -+ scan_dsq_timeout(rq, dsq, pcp_dsq_deadline); -+ -+ if (time_before_eq(jiffies, (unsigned long)last_scan_at)) -+ return; -+ last_scan_at = jiffies; -+ -+ for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) { -+ dsq = &gdsqs[i]; -+ scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]); -+ } -+} -+ -+/*******************************Initialize***********************************/ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id); -+static void init_dsq_at_boot(void) -+{ -+ int i, cpu; -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i)); -+ } -+ for_each_possible_cpu(cpu) -+ init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i + cpu)); -+ -+ max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu; -+ spin_lock_init(&sinfo.lock); -+} -+ -+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgrp_name_to_idx(struct cgroup *cgrp) -+{ -+ int idx; -+ -+ if (!cgrp) -+ return -1; -+ -+ if (!strcmp(cgrp->kn->name, "display") -+ || !strcmp(cgrp->kn->name, "multimedia")) -+ idx = 5; /* 8ms */ -+ else if (!strcmp(cgrp->kn->name, "top-app") -+ || !strcmp(cgrp->kn->name, "ss-top")) -+ idx = 6; /* 16ms */ -+ else if (!strcmp(cgrp->kn->name, "ssfg") -+ || !strcmp(cgrp->kn->name, "foreground")) -+ idx = 7; /* 32ms */ -+ else if (!strcmp(cgrp->kn->name, "bg") -+ || !strcmp(cgrp->kn->name, "log") -+ || !strcmp(cgrp->kn->name, "dex2oat") -+ || !strcmp(cgrp->kn->name, "background")) -+ idx = 9; /* 128ms */ -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */ -+ -+ return idx; -+} -+ -+static void init_root_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX); -+} -+ -+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) { -+ pr_err("%s idx err!\n", __func__); -+ return; -+ } -+ -+ if (cgroup_ids_table[cgrp->kn->id] == -1) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp)); -+} -+ -+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ struct cgroup *l1cgrp; -+ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ l1cgrp = cgroup_ancestor_l1(cgrp); -+ if (l1cgrp) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp)); -+ cgroup_put(l1cgrp); -+} -+ -+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg) -+{ -+ switch (cgrp->level) { -+ case 0: -+ init_root_tg(cgrp, tg); -+ break; -+ case 1: -+ init_level1_tg(cgrp, tg); -+ break; -+ default: -+ init_child_tg(cgrp, tg); -+ break; -+ } -+} -+ -+/**************************************************************************/ -+ -+struct hmbird_task_iter { -+ struct hmbird_entity cursor; -+ struct task_struct *locked; -+ struct rq *rq; -+ struct rq_flags rf; -+}; -+ -+/** -+ * hmbird_task_iter_init - Initialize a task iterator -+ * @iter: iterator to init -+ * -+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized, -+ * @iter must eventually be exited with hmbird_task_iter_exit(). -+ * -+ * hmbird_tasks_lock may be released between this and the first next() call or -+ * between any two next() calls. If hmbird_tasks_lock is released between two -+ * next() calls, the caller is responsible for ensuring that the task being -+ * iterated remains accessible either through RCU read lock or obtaining a -+ * reference count. -+ * -+ * All tasks which existed when the iteration started are guaranteed to be -+ * visited as long as they still exist. -+ */ -+static void hmbird_task_iter_init(struct hmbird_task_iter *iter) -+{ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR }; -+ list_add(&iter->cursor.tasks_node, &hmbird_tasks); -+ iter->locked = NULL; -+} -+ -+/** -+ * hmbird_task_iter_exit - Exit a task iterator -+ * @iter: iterator to exit -+ * -+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held. -+ * If the iterator holds a task's rq lock, that rq lock is released. See -+ * hmbird_task_iter_init() for details. -+ */ -+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ if (list_empty(cursor)) -+ return; -+ -+ list_del_init(cursor); -+} -+ -+/** -+ * hmbird_task_iter_next - Next task -+ * @iter: iterator to walk -+ * -+ * Visit the next task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ struct hmbird_entity *pos; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ list_for_each_entry(pos, cursor, tasks_node) { -+ if (&pos->tasks_node == &hmbird_tasks) -+ return NULL; -+ if (!(pos->flags & HMBIRD_TASK_CURSOR)) { -+ list_move(cursor, &pos->tasks_node); -+ return pos->task; -+ } -+ } -+ -+ /* can't happen, should always terminate at hmbird_tasks above */ -+ hmbird_deferred_err(ITER_RET_NULL, " : unreachable path in scx_task_iter_next\n"); -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered - Next non-idle task -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ while ((p = hmbird_task_iter_next(iter))) { -+ if (!is_idle_task(p)) -+ return p; -+ } -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init() -+ * for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ p = hmbird_task_iter_next_filtered(iter); -+ if (!p) -+ return NULL; -+ -+ iter->rq = task_rq_lock(p, &iter->rf); -+ iter->locked = p; -+ return p; -+} -+ -+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void) -+{ -+ return atomic_read(&hmbird_ops_enable_state_var); -+} -+ -+static enum hmbird_ops_enable_state -+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to) -+{ -+ return atomic_xchg(&hmbird_ops_enable_state_var, to); -+} -+ -+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to, -+ enum hmbird_ops_enable_state from) -+{ -+ int from_v = from; -+ -+ return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to); -+} -+ -+static bool hmbird_ops_disabling(void) -+{ -+ return false; -+} -+ -+/** -+ * wait_ops_state - Busy-wait the specified ops state to end -+ * @p: target task -+ * @opss: state to wait the end of -+ * -+ * Busy-wait for @p to transition out of @opss. This can only be used when the -+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also -+ * has load_acquire semantics to ensure that the caller can see the updates made -+ * in the enqueueing and dispatching paths. -+ */ -+static void wait_ops_state(struct task_struct *p, u64 opss) -+{ -+ do { -+ cpu_relax(); -+ } while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss); -+} -+ -+ -+static void update_curr_hmbird(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ u64 now = rq_clock_task(rq); -+ u64 delta_exec; -+ -+ if (time_before_eq64(now, curr->se.exec_start)) -+ return; -+ -+ delta_exec = now - curr->se.exec_start; -+ curr->se.exec_start = now; -+ update_runningtime(rq, curr, delta_exec); -+ curr->se.sum_exec_runtime += delta_exec; -+ account_group_exec_runtime(curr, delta_exec); -+ cgroup_account_cputime(curr, delta_exec); -+ -+ if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF) -+ get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec); -+ -+ trace_sched_stat_runtime(curr, delta_exec, 0); -+} -+ -+static bool hmbird_dsq_priq_less(struct rb_node *node_a, -+ const struct rb_node *node_b) -+{ -+ const struct hmbird_entity *a = -+ container_of(node_a, struct hmbird_entity, dsq_node.priq); -+ const struct hmbird_entity *b = -+ container_of(node_b, struct hmbird_entity, dsq_node.priq); -+ -+ return time_before64(a->dsq_vtime, b->dsq_vtime); -+} -+ -+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ bool is_local = dsq->id == HMBIRD_DSQ_LOCAL; -+ unsigned long flags; -+ -+ hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq || -+ !list_empty(&get_hmbird_ts(p)->dsq_node.fifo), -+ "task = %s, dsq->id = %llu\n", p->comm, dsq->id); -+ hmbird_cond_deferred_err(ENQ_EXIST2, -+ (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq), -+ "task = %s\n", p->comm); -+ -+ if (!is_local) { -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ); -+ hmbird_ops_error(": %s\n", -+ "attempting to dispatch to a destroyed dsq"); -+ /* fall back to the global dsq */ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ dsq = &hmbird_dsq_global; -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ } -+ } -+ -+ if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) { -+ get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ; -+ rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq, -+ hmbird_dsq_priq_less); -+ } else { -+ if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT)) -+ list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ else -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ } -+ dsq->nr++; -+ get_hmbird_ts(p)->dsq = dsq; -+ -+ /* -+ * We're transitioning out of QUEUEING or DISPATCHING. store_release to -+ * match waiters' load_acquire. -+ */ -+ if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS) -+ atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE); -+ -+ if (is_local) { -+ struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq); -+ struct rq *rq = hmbird->rq; -+ bool preempt = false; -+ -+ if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr && -+ rq->curr->sched_class == &hmbird_sched_class) { -+ get_hmbird_ts(rq->curr)->slice = 0; -+ preempt = true; -+ } -+ -+ if (preempt || sched_class_above(&hmbird_sched_class, -+ rq->curr->sched_class)) -+ resched_curr(rq); -+ } else { -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ } -+} -+ -+static void task_unlink_from_dsq(struct task_struct *p, -+ struct hmbird_dispatch_q *dsq) -+{ -+ if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) { -+ rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ; -+ } else { -+ list_del_init(&get_hmbird_ts(p)->dsq_node.fifo); -+ } -+} -+ -+static bool task_linked_on_dsq(struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+} -+ -+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq; -+ bool is_local = dsq == &hmbird_rq->local_dsq; -+ -+ if (!dsq) { -+ hmbird_cond_deferred_err(TASK_LINKED1, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ /* -+ * When dispatching directly from the BPF scheduler to a local -+ * DSQ, the task isn't associated with any DSQ but -+ * @get_hmbird_ts(p)->holding_cpu may be set under the protection of -+ * %HMBIRD_OPSS_DISPATCHING. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu >= 0) -+ get_hmbird_ts(p)->holding_cpu = -1; -+ return; -+ } -+ -+ if (!is_local) -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ /* -+ * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node -+ * can't change underneath us. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu < 0) { -+ /* @p must still be on @dsq, dequeue */ -+ hmbird_cond_deferred_err(TASK_UNLINKED, -+ !task_linked_on_dsq(p), "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ } else { -+ /* -+ * We're racing against dispatch_to_local_dsq() which already -+ * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the -+ * holding_cpu which tells dispatch_to_local_dsq() that it lost -+ * the race. -+ */ -+ hmbird_cond_deferred_err(TASK_LINKED2, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ get_hmbird_ts(p)->holding_cpu = -1; -+ } -+ get_hmbird_ts(p)->dsq = NULL; -+ -+ if (!is_local) -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+ -+static bool test_rq_online(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ return rq->online; -+#else -+ return true; -+#endif -+} -+ -+static void refill_task_slice(struct task_struct *p) -+{ -+ if (is_isolate_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO; -+ else if (is_pipeline_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2; -+ else -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+} -+ -+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -+ int sticky_cpu) -+{ -+ struct hmbird_dispatch_q *d; -+ s32 cpu = -1; -+ -+ hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), -+ "task = %s\n", p->comm); -+ -+ if (is_pcp_rt(p)) { -+ /* Enqueue percpu rt task to local directly. */ -+ /* Or cause a bug when disable dispatch. */ -+ if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ } -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (cpu >= 0) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ -+ if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ /* rq migration */ -+ if (sticky_cpu == cpu_of(rq)) -+ goto local_norefill; -+ /* -+ * If !rq->online, we already told the BPF scheduler that the CPU is -+ * offline. We're just trying to on/offline the CPU. Don't bother the -+ * BPF scheduler. -+ */ -+ if (unlikely(!test_rq_online(rq))) -+ goto local; -+ -+ /* see %HMBIRD_OPS_ENQ_LAST */ -+ if (enq_flags & HMBIRD_ENQ_LAST) -+ goto local; -+ -+ if (enq_flags & HMBIRD_ENQ_LOCAL) -+ goto local; -+ else -+ goto global; -+local: -+ /* -+ * For task-ordering, slice refill must be treated as implying the end -+ * of the current slice. Otherwise, the longer @p stays on the CPU, the -+ * higher priority it becomes from hmbird_prio_less()'s POV. -+ */ -+ refill_task_slice(p); -+local_norefill: -+ dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags); -+ slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq)); -+ return; -+ -+global: -+ d = find_dsq_from_task(p); -+ if (d) { -+ refill_task_slice(p); -+ dispatch_enqueue(d, p, enq_flags); -+ return; -+ } -+ slim_stats_record(GLOBAL_STAT, 0, 0, 0); -+ refill_task_slice(p); -+ dispatch_enqueue(&hmbird_dsq_global, p, enq_flags); -+} -+ -+static bool watchdog_task_watched(const struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->watchdog_node); -+} -+ -+static void watchdog_watch_task(struct rq *rq, struct task_struct *p) -+{ -+ lockdep_assert_rq_held(rq); -+ if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ get_hmbird_ts(p)->runnable_at = jiffies; -+ clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list); -+} -+ -+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) -+{ -+ list_del_init(&get_hmbird_ts(p)->watchdog_node); -+ if (reset_timeout) -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags) -+{ -+ int sticky_cpu = get_hmbird_ts(p)->sticky_cpu; -+ -+ enq_flags |= get_hmbird_rq(rq)->extra_enq_flags; -+ -+ if (sticky_cpu >= 0) -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ -+ /* -+ * Restoring a running task will be immediately followed by -+ * set_next_task_hmbird() which expects the task to not be on the BPF -+ * scheduler as tasks can only start running through local DSQs. Force -+ * direct-dispatch into the local DSQ by setting the sticky_cpu. -+ */ -+ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -+ sticky_cpu = cpu_of(rq); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_UNWATCHED, -+ !watchdog_task_watched(p), "task = %s\n", p->comm); -+ return; -+ } -+ -+ watchdog_watch_task(rq, p); -+ set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ get_hmbird_rq(rq)->nr_running++; -+ add_nr_running(rq, 1); -+ -+ do_enqueue_task(rq, p, enq_flags, sticky_cpu); -+} -+ -+static void ops_dequeue(struct task_struct *p, u64 deq_flags) -+{ -+ u64 opss; -+ -+ watchdog_unwatch_task(p, false); -+ -+ /* acquire ensures that we see the preceding updates on QUEUED */ -+ opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state); -+ -+ switch (opss & HMBIRD_OPSS_STATE_MASK) { -+ case HMBIRD_OPSS_NONE: -+ break; -+ case HMBIRD_OPSS_QUEUEING: -+ /* -+ * QUEUEING is started and finished while holding @p's rq lock. -+ * As we're holding the rq lock now, we shouldn't see QUEUEING. -+ */ -+ hmbird_deferred_err(DEQ_DEQING, " : unreachable path in %s\n", __func__); -+ break; -+ case HMBIRD_OPSS_QUEUED: -+ if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss, -+ HMBIRD_OPSS_NONE)) -+ break; -+ fallthrough; -+ case HMBIRD_OPSS_DISPATCHING: -+ /* -+ * If @p is being dispatched from the BPF scheduler to a DSQ, -+ * wait for the transfer to complete so that @p doesn't get -+ * added to its DSQ after dequeueing is complete. -+ * -+ * As we're waiting on DISPATCHING with the rq locked, the -+ * dispatching side shouldn't try to lock the rq while -+ * DISPATCHING is set. See dispatch_to_local_dsq(). -+ * -+ * DISPATCHING shouldn't have qseq set and control can reach -+ * here with NONE @opss from the above QUEUED case block. -+ * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss. -+ */ -+ wait_ops_state(p, HMBIRD_OPSS_DISPATCHING); -+ hmbird_cond_deferred_err(HMBIRD_OPN, -+ atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE, -+ "task = %s\n", p->comm); -+ break; -+ } -+} -+ -+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ -+ if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p), -+ "task = %s\n", p->comm); -+ return; -+ } -+ -+ ops_dequeue(p, deq_flags); -+ -+ if (slim_walt_ctrl) { -+ if (task_current(rq, p)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ if (deq_flags & HMBIRD_DEQ_SLEEP) -+ set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else -+ clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), -+ (unsigned long *)&get_hmbird_ts(p)->flags); -+ -+ clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm); -+ hmbird_rq->nr_running--; -+ sub_nr_running(rq, 1); -+ dispatch_dequeue(hmbird_rq, p); -+} -+ -+static void yield_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ get_hmbird_ts(p)->slice = 0; -+} -+ -+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to) -+{ -+ return false; -+} -+ -+#ifdef CONFIG_SMP -+/** -+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -+ * @rq: rq to move the task into, currently locked -+ * @p: task to move -+ * @enq_flags: %HMBIRD_ENQ_* -+ * -+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller -+ * must: -+ * -+ * 1. Start with exclusive access to @p either through its DSQ lock or -+ * %HMBIRD_OPSS_DISPATCHING flag. -+ * -+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id(). -+ * -+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't -+ * deadlock with dequeue. -+ * -+ * 4. Lock @rq and the task_rq from #3. -+ * -+ * 5. Call this function. -+ * -+ * Returns %true if @p was successfully moved. %false after racing dequeue and -+ * losing. -+ */ -+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ struct rq *task_rq; -+ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * If dequeue got to @p while we were trying to lock both rq's, it'd -+ * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have -+ * updated it to different values afterwards, as this operation can't be -+ * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become -+ * raw_smp_processor_id() again before we're done. Thus, we can tell -+ * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is -+ * still raw_smp_processor_id(). -+ * -+ * See dispatch_dequeue() for the counterpart. -+ */ -+ if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id())) -+ return false; -+ -+ /* @p->rq couldn't have changed if we're still the holding cpu */ -+ task_rq = task_rq(p); -+ lockdep_assert_rq_held(task_rq); -+ deactivate_task(task_rq, p, 0); -+ set_task_cpu(p, cpu_of(rq)); -+ get_hmbird_ts(p)->sticky_cpu = cpu_of(rq); -+ -+ /* -+ * We want to pass hmbird-specific enq_flags but activate_task() will -+ * truncate the upper 32 bit. As we own @rq, we can pass them through -+ * @get_hmbird_rq(rq)->extra_enq_flags instead. -+ */ -+ hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags, -+ "task = %s\n", p->comm); -+ get_hmbird_rq(rq)->extra_enq_flags = enq_flags; -+ activate_task(rq, p, 0); -+ get_hmbird_rq(rq)->extra_enq_flags = 0; -+ -+ return true; -+} -+ -+#endif /* CONFIG_SMP */ -+ -+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu) -+{ -+ int fitable = 1; -+ -+ return fitable; -+} -+ -+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq, -+ struct hmbird_dispatch_q *dsq) -+{ -+ bool dsq_misfit; -+ int cpu = cpu_of(rq); -+ u64 task_util = 0; -+ struct cluster_ctx ctx; -+ int dsq_int = dsq_id_to_internal(dsq); -+ -+ if (!cpumask_test_cpu(cpu, iso_masks.little)) -+ return false; -+ if (p->pid == scx_systemui_pid) -+ return true; -+ -+ gen_cluster_ctx(&ctx, BIG); -+ dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 && -+ dsq_int <= SCHED_PROP_DEADLINE_LEVEL4); -+#ifdef CLUSTER_SEPARATE -+ /* In rescue mode, little will consume big cluster's dsq.*/ -+ dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper); -+#endif -+ if (!dsq_misfit) -+ return false; -+ -+ if (p) { -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &task_util); -+ else -+ task_util = get_hmbird_ts(p)->demand_scaled; -+ } -+ -+ if (task_util <= misfit_ds) -+ return false; -+ -+ hmbird_info_trace(":task %s can't run on cpu%d, util = %llu\n", -+ p->comm, cpu, task_util); -+ return true; -+} -+ -+static int check_misfit_task_on_fake_big(struct task_struct *p, struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (likely(p->pid != scx_systemui_pid)) -+ return false; -+ -+ if (topology_cluster_id(num_possible_cpus() - 1) > 1 && -+ arch_scale_cpu_capacity(cpu) == arch_scale_cpu_capacity(0) && -+ (parctrl_high_ratio <= 0 || parctrl_high_ratio_l <= 0)) -+ return true; -+ -+ return false; -+} -+ -+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq) -+{ -+ if (!cpumask_test_cpu(cpu_of(rq), task_cpu_possible_mask(p))) -+ return false; -+ -+ if (!task_fits_cpu_hmbird(p, cpu_of(rq))) -+ return false; -+ -+ if (check_misfit_task_on_little(p, rq, dsq)) -+ return false; -+ -+ if (check_misfit_task_on_fake_big(p, rq)) -+ return false; -+ -+ return likely(test_rq_online(rq)); -+} -+ -+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return; -+ -+ if (add) -+ skipn[idx]++; -+ else -+ skipn[idx] = 0; -+} -+ -+static bool skip_too_much(struct hmbird_dispatch_q *dsq) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return false; -+ -+ if (skip_num[idx] > 3) { -+ skip_num[idx] = 0; -+ return true; -+ } -+ -+ return false; -+} -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rb_node *rb_node; -+ struct rq *task_rq; -+ unsigned long flags; -+ bool moved = false; -+ struct task_struct *may_fit = NULL; -+ int skip = 0; -+ -+retry: -+ if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -+ return false; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) { -+ set_skip_num(dsq, skip_num, (bool)may_fit); -+ goto this_rq; -+ } -+ if (skip_too_much(dsq)) -+ goto remote_rq; -+ if (!may_fit) -+ may_fit = p; -+ if (++skip <= 3) -+ continue; -+ /* -+ * If the recent 3 tasks not fit, use the first one. -+ * and clear the skip, because the first one is consumed. -+ */ -+ set_skip_num(dsq, skip_num, false); -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ /* No more task, use the first may fit task.*/ -+ if (may_fit) { -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ -+ for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) -+ goto this_rq; -+ goto remote_rq; -+ } -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ return false; -+ -+this_rq: -+ /* @dsq is locked and @p is on this rq */ -+ hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo); -+ dsq->nr--; -+ hmbird_rq->local_dsq.nr++; -+ get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ -+remote_rq: -+#ifdef CONFIG_SMP -+ if (cpu_same_cluster_stat(p, rq, task_rq)) -+ slim_stats_record(MOVE_RQ_CNT, 0, 0, 0); -+ else -+ slim_stats_record(MOVE_RQ_CNT, 1, 0, 0); -+ /* -+ * @dsq is locked and @p is on a remote rq. @p is currently protected by -+ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -+ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -+ * rq lock or fail, do a little dancing from our side. See -+ * move_task_to_local_dsq(). -+ */ -+ hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id(); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ rq_unpin_lock(rq, rf); -+ double_lock_balance(rq, task_rq); -+ rq_repin_lock(rq, rf); -+ -+ moved = move_task_to_local_dsq(rq, p, 0); -+ -+ double_unlock_balance(rq, task_rq); -+#endif /* CONFIG_SMP */ -+ if (likely(moved)) { -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ } -+ may_fit = NULL; -+ goto retry; -+} -+ -+ -+static int balance_one(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf, bool local) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class; -+ -+ if (!hmbird_rq) -+ return 1; -+ -+ lockdep_assert_rq_held(rq); -+ -+ if (static_branch_unlikely(&hmbird_ops_cpu_preempt) && -+ unlikely(get_hmbird_rq(rq)->cpu_released)) { -+ /* -+ * If the previous sched_class for the current CPU was not HMBIRD, -+ * notify the BPF scheduler that it again has control of the -+ * core. This callback complements ->cpu_release(), which is -+ * emitted in hmbird_notify_pick_next_task(). -+ */ -+ get_hmbird_rq(rq)->cpu_released = false; -+ } -+ -+ if (prev_on_hmbird) -+ update_curr_hmbird(rq); -+ /* if there already are tasks to run, nothing to do */ -+ if (hmbird_rq->local_dsq.nr) -+ return 1; -+ -+ if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) { -+ slim_stats_record(GLOBAL_STAT, 1, 0, 0); -+ return 1; -+ } -+ -+ if (consume_dispatch_global(rq, rf)) -+ return 1; -+ -+ return 0; -+} -+ -+static int balance_hmbird(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf) -+{ -+ return balance_one(rq, prev, rf, true); -+} -+ -+/* -+ * output task util to systrace, only for debug mode. -+ * we can not output too many logs to systrace buffer even in debug mode -+ * only output debug-info while it exceed misfit_ds. -+ */ -+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p) -+{ -+ static DEFINE_PER_CPU(int, is_last_exceed); -+ int cpu = cpu_of(rq); -+ u64 util = 0; -+ -+ if (likely(!debug_enabled())) -+ return; -+ -+ if (!p) -+ return; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ util = uclamp_rq_util_with(rq, util, p); -+ -+ if (util >= misfit_ds) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util); -+ per_cpu(is_last_exceed, cpu) = true; -+ } else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0); -+ per_cpu(is_last_exceed, cpu) = false; -+ } else { -+ } -+} -+ -+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first) -+{ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ /* -+ * Core-sched might decide to execute @p before it is -+ * dispatched. Call ops_dequeue() to notify the BPF scheduler. -+ */ -+ ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC); -+ dispatch_dequeue(get_hmbird_rq(rq), p); -+ } -+ -+ p->se.exec_start = rq_clock_task(rq); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK); -+ } -+ -+ watchdog_unwatch_task(p, true); -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx); -+ systrace_output_cpu_ds(rq, p); -+ /* -+ * @p is getting newly scheduled or got kicked after someone updated its -+ * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird(). -+ */ -+ if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) != -+ (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) { -+ if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) -+ get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK; -+ else -+ get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK; -+ -+ sched_update_tick_dependency(rq); -+ } -+ -+ p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime; -+} -+ -+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p) -+{ -+ update_curr_hmbird(rq); -+ -+ update_dispatch_dsq_info(rq, p); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ watchdog_watch_task(rq, p); -+ -+ if (is_pipeline_task(p)) { -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1); -+ return; -+ } -+ -+ /* -+ * If we're in the pick_next_task path, balance_hmbird() should -+ * have already populated the local DSQ if there are any other -+ * available tasks. If empty, tell ops.enqueue() that @p is the -+ * only one available for this cpu. ops.enqueue() should put it -+ * on the local DSQ so that the subsequent pick_next_task_hmbird() -+ * can find the task unless it wants to trigger a separate -+ * follow-up scheduling event. -+ */ -+ if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1); -+ else -+ do_enqueue_task(rq, p, 0, -1); -+ } -+} -+ -+static struct task_struct *first_local_task(struct rq *rq) -+{ -+ struct rb_node *rb_node; -+ struct hmbird_entity *entity; -+ -+ if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) { -+ entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo, -+ struct hmbird_entity, dsq_node.fifo); -+ return entity->task; -+ } -+ -+ rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq); -+ if (rb_node) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ return entity->task; -+ } -+ return NULL; -+} -+ -+static struct task_struct *pick_next_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p; -+ -+ p = first_local_task(rq); -+ if (!p) -+ return NULL; -+ -+ if (unlikely(!get_hmbird_ts(p)->slice)) { -+ if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice) -+ hmbird_warned_zero_slice = true; -+ -+ refill_task_slice(p); -+ } -+ -+ set_next_task_hmbird(rq, p, true); -+ -+ return p; -+} -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task, -+ const struct sched_class *active) -+{ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * The callback is conceptually meant to convey that the CPU is no -+ * longer under the control of HMBIRD. Therefore, don't invoke the -+ * callback if the CPU is staying on HMBIRD, or going idle (in which -+ * case the HMBIRD scheduler has actively decided not to schedule any -+ * tasks on the CPU). -+ */ -+ if (likely(active >= &hmbird_sched_class)) -+ return; -+ -+ /* -+ * At this point we know that HMBIRD was preempted by a higher priority -+ * sched_class, so invoke the ->cpu_release() callback if we have not -+ * done so already. We only send the callback once between HMBIRD being -+ * preempted, and it regaining control of the CPU. -+ * -+ * ->cpu_release() complements ->cpu_acquire(), which is emitted the -+ * next time that balance_hmbird() is invoked. -+ */ -+ if (!get_hmbird_rq(rq)->cpu_released) -+ get_hmbird_rq(rq)->cpu_released = true; -+} -+ -+#ifdef CONFIG_SMP -+ -+static bool test_and_clear_cpu_idle(int cpu) -+{ -+ if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -+ if (cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ return true; -+ } else { -+ return false; -+ } -+} -+ -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) -+{ -+ int cpu; -+ -+ do { -+ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -+ if (cpu >= nr_cpu_ids) -+ return -EBUSY; -+ } while (!test_and_clear_cpu_idle(cpu)); -+ -+ return cpu; -+} -+ -+ -+static bool prev_cpu_misfit(int prev) -+{ -+ if (!is_partial_enabled() && is_partial_cpu(prev)) -+ return true; -+ -+ return false; -+} -+ -+static int heavy_rt_placement(struct task_struct *p, int prev) -+{ -+ struct cpumask tmp = {.bits = {0}}; -+ int cpu; -+ u64 util = 0; -+ -+ if (!rt_prio(p->prio)) -+ return -EFAULT; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ -+ if (util < misfit_ds) -+ return -EFAULT; -+ -+ if (is_partial_enabled()) -+ cpumask_or(&tmp, iso_masks.big, iso_masks.partial); -+ else -+ cpumask_copy(&tmp, iso_masks.big); -+ -+ cpu = hmbird_pick_idle_cpu(&tmp); -+ if (cpu >= 0) -+ return cpu; -+ -+ if (cpumask_test_cpu(prev, iso_masks.big) || -+ (is_partial_enabled() && is_partial_cpu(prev))) -+ return prev; -+ -+ return cpumask_first(&tmp); -+} -+ -+static int spec_task_before_pick_idle(struct task_struct *p, int prev) -+{ -+ int cpu; -+ -+ cpu = heavy_rt_placement(p, prev); -+ if (cpu >= 0) -+ return cpu; -+ return -EFAULT; -+} -+ -+static int cpumask_distribute_next(struct cpumask *mask, int *prev) -+{ -+ int p = *prev, n; -+ -+ n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1); -+ if (n < nr_cpu_ids) -+ WRITE_ONCE(*prev, n); -+ return n; -+} -+ -+static int repick_fallback_cpu(void) -+{ -+ /* -+ * partial cpu follow big cluster's Scheduling policy, -+ * simply return first bit cpu. -+ */ -+ return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev); -+} -+ -+/* -+ * Must return a valid cpu num, as this task's cpu. -+ */ -+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu, -+ struct cpumask *mask, int *prev_mask) -+{ -+ int cpu; -+ -+ cpu = hmbird_pick_idle_cpu(mask); -+ if (cpu >= 0) -+ return cpu; -+ return cpumask_distribute_next(mask, prev_mask); -+} -+ -+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type) -+{ -+ int idx; -+ struct cluster_ctx ctx; -+ -+ idx = find_idx_from_task(p); -+ if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS) -+ return false; -+ -+ gen_cluster_ctx(&ctx, type); -+ if (idx >= ctx.lower && idx < ctx.upper) -+ return true; -+ return false; -+} -+ -+static bool is_valid_cpu(int cpu) -+{ -+ return (cpu >= 0) && (cpu < nr_cpu_ids); -+} -+ -+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) -+{ -+ s32 cpu = -1; -+ struct cpumask mask = {.bits = {0}}; -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (is_valid_cpu(cpu)) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ partial_dynamic_ctrl(); -+ -+ if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ iso_masks.big, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ if (p->nr_cpus_allowed == 1) { -+ cpu = cpumask_any(p->cpus_ptr); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* For non-period global dsq, not contain pcp task. */ -+ if (task_only_blongs_to_cluster(p, LITTLE)) { -+ cpumask_copy(&mask, iso_masks.little); -+ if (unlikely(l_need_rescue)) { -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ cpumask_or(&mask, iso_masks.ex_free, &mask); -+ } -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &little_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ if (task_only_blongs_to_cluster(p, BIG)) { -+ cpumask_copy(&mask, iso_masks.big); -+ if (unlikely(b_need_rescue)) { -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ cpumask_or(&mask, iso_masks.ex_free, &mask); -+ } -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ cpu = spec_task_before_pick_idle(p, prev_cpu); -+ if (is_valid_cpu(cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* if the previous CPU is idle, dispatch directly to it */ -+ if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+ } -+ -+ cpu = hmbird_pick_idle_cpu(cpu_possible_mask); -+ if (is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ if (prev_cpu_misfit(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ cpu = repick_fallback_cpu(); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+} -+ -+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags) -+{ -+ return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags); -+} -+ -+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx) -+{ -+ set_cpus_allowed_common(p, ctx); -+} -+ -+static void reset_idle_masks(void) -+{ -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little); -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big); -+ if (is_partial_enabled()) -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial); -+ hmbird_has_idle_cpus = true; -+} -+ -+void __hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ int cpu = cpu_of(rq); -+ struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -+ -+ if (skip_update_idle()) -+ return; -+ -+ if (idle) { -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ if (!hmbird_has_idle_cpus) -+ hmbird_has_idle_cpus = true; -+ -+ /* -+ * idle_masks.smt handling is racy but that's fine as it's only -+ * for optimization and self-correcting. -+ */ -+ for_each_cpu(cpu, sib_mask) { -+ if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -+ return; -+ } -+ cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -+ } else { -+ cpumask_clear_cpu(cpu, idle_masks.cpu); -+ if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ -+ cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -+ } -+} -+ -+#else /* !CONFIG_SMP */ -+ -+static bool test_and_clear_cpu_idle(int cpu) { return false; } -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } -+static void reset_idle_masks(void) {} -+ -+#endif /* CONFIG_SMP */ -+ -+static bool check_rq_for_timeouts(struct rq *rq) -+{ -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rq_flags rf; -+ -+ rq_lock_irqsave(rq, &rf); -+ list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) { -+ unsigned long last_runnable; -+ -+ p = entity->task; -+ last_runnable = get_hmbird_ts(p)->runnable_at; -+ -+ if (unlikely(time_after(jiffies, -+ last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -+ -+ rq_unlock_irqrestore(rq, &rf); -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb", -+ p->comm, p->pid, -+ dur_ms / 1000, dur_ms % 1000, -+ get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0, -+ cpumask_pr_args(&p->cpus_mask)); -+ return 1; -+ } -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ return 0; -+} -+ -+static void hmbird_watchdog_workfn(struct work_struct *work) -+{ -+ int cpu; -+ bool timeout; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ -+ for_each_online_cpu(cpu) { -+ timeout = check_rq_for_timeouts(cpu_rq(cpu)); -+ if (unlikely(timeout)) -+ break; -+ -+ cond_resched(); -+ } -+ if (!timeout) -+ queue_delayed_work(system_unbound_wq, to_delayed_work(work), -+ hmbird_watchdog_timeout / 2); -+} -+ -+static void set_pcp_round(struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) { -+ per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round); -+ per_cpu(pcp_info, cpu).pcp_round = true; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+} -+ -+ -+/* -+ * Just for debug: output hmbird on/off state per 10s. -+ */ -+#define OUTPUT_INTVAL (msecs_to_jiffies(10 * 1000)) -+static void inform_hmbird_onoff_from_systrace(void) -+{ -+ static unsigned long __read_mostly next_print; -+ -+ if (time_before(jiffies, READ_ONCE(next_print))) -+ return; -+ -+ WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL); -+ hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio); -+} -+ -+void hmbird_notify_sched_tick(void) -+{ -+ unsigned long last_check; -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ hmbird_scheduler_tick(); -+ -+ if (!hmbird_enabled()) -+ return; -+ -+ last_check = hmbird_watchdog_timestamp; -+ if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -+ -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "watchdog failed to check in for %u.%03us", -+ dur_ms / 1000, dur_ms % 1000); -+ } -+ scan_timeout(rq); -+} -+ -+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued) -+{ -+ update_curr_hmbird(rq); -+ -+ set_pcp_round(rq); -+ -+ if (slim_walt_ctrl) -+ hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE); -+ /* -+ * While disabling, always resched and refresh core-sched timestamp as -+ * we can't trust the slice management or ops.core_sched_before(). -+ */ -+ if (hmbird_ops_disabling()) -+ get_hmbird_ts(curr)->slice = 0; -+ -+ if (!get_hmbird_ts(curr)->slice) -+ resched_curr(rq); -+ -+ inform_hmbird_onoff_from_systrace(); -+} -+ -+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg) -+{ -+ hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ get_hmbird_ts(p)->disallow = false; -+ -+ hmbird_sched_init_task(p); -+ -+ set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ return 0; -+} -+ -+static void hmbird_ops_enable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void hmbird_ops_disable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void set_task_hmbird_weight(struct task_struct *p) -+{ -+ u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -+ -+ get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight); -+} -+ -+/** -+ * refresh_hmbird_weight - Refresh a task's hmbird weight -+ * @p: task to refresh hmbird weight for -+ * -+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to -+ * enable easy access from the BPF scheduler. To keep it synchronized with the -+ * current task priority, this function should be called when a new task is -+ * created, priority is changed for a task on hmbird, and a task is switched -+ * to hmbird from other classes. -+ */ -+static void refresh_hmbird_weight(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ set_task_hmbird_weight(p); -+} -+ -+int hmbird_pre_fork(struct task_struct *p) -+{ -+ int ret = 0; -+ -+ p->android_oem_data1[HMBIRD_TS_IDX] = -+ (u64)(kzalloc(sizeof(struct hmbird_entity), GFP_KERNEL)); -+ if (!get_hmbird_ts(p)) -+ return -1; -+ -+ get_hmbird_ts(p)->dsq = NULL; -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node); -+ get_hmbird_ts(p)->flags = 0; -+ get_hmbird_ts(p)->weight = 0; -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ get_hmbird_ts(p)->holding_cpu = -1; -+ get_hmbird_ts(p)->kf_mask = 0; -+ atomic64_set(&get_hmbird_ts(p)->ops_state, 0); -+ get_hmbird_ts(p)->runnable_at = INITIAL_JIFFIES; -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+ get_hmbird_ts(p)->task = p; -+ hmbird_set_sched_prop(p, 0); -+ -+ get_hmbird_ts(p)->critical_affinity_cpu = -1; -+ get_hmbird_ts(p)->sched_class = &hmbird_sched_class; -+ get_hmbird_ts(p)->tick_hit_count = 0; -+ get_hmbird_ts(p)->start_jiffies = 0; -+ -+ /* -+ * BPF scheduler enable/disable paths want to be able to iterate and -+ * update all tasks which can become complex when racing forks. As -+ * enable/disable are very cold paths, let's use a percpu_rwsem to -+ * exclude forks. -+ */ -+ -+ return ret; -+} -+ -+void hmbird_post_fork(struct task_struct *p) -+{ -+ percpu_down_read(&hmbird_fork_rwsem); -+ -+ if (hmbird_enabled()) { -+ struct rq_flags rf; -+ struct rq *rq; -+ p->sched_class = &hmbird_sched_class; -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ rq = task_rq_lock(p, &rf); -+ /* -+ * Set the weight manually before calling ops.enable() so that -+ * the scheduler doesn't see a stale value if they inspect the -+ * task struct. We'll invoke ops.set_weight() afterwards, as it -+ * would be odd to receive a callback on the task before we -+ * tell the scheduler that it's been fully enabled. -+ */ -+ set_task_hmbird_weight(p); -+ hmbird_ops_enable_task(p); -+ refresh_hmbird_weight(p); -+ task_rq_unlock(rq, p, &rf); -+ } else { -+ if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ } -+ -+ spin_lock_irq(&hmbird_tasks_lock); -+ list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_cancel_fork(struct task_struct *p) -+{ -+ if (hmbird_enabled()) -+ hmbird_ops_disable_task(p); -+ put_hmbird_ts(p); -+} -+ -+void hmbird_free(struct task_struct *p) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+ list_del_init(&get_hmbird_ts(p)->tasks_node); -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+ -+ /* -+ * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED -> -+ * ENABLED transitions can't race us. Disable ops for @p. -+ */ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) || -+ test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ hmbird_ops_disable_task(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ put_hmbird_ts(p); -+} -+ -+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio) -+{ -+} -+ -+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type) -+{ -+ return (prop >> TOP_TASK_SHIFT) & (1 << type); -+} -+ -+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p) -+{ -+ uint32_t prop = get_top_task_prop(p); -+ -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE)) -+ return HMBIRD_TASK_PROP_PIPELINE; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) || -+ !task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) { -+ return HMBIRD_TASK_PROP_COMMON; -+ } -+ return HMBIRD_TASK_PROP_DEBUG_OR_LOG; -+} -+ -+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b) -+{ -+ int type_a = hmbird_get_task_type(a); -+ int type_b = hmbird_get_task_type(b); -+ -+ return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b]; -+} -+ -+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ enum cpu_type type; -+ int sp_dl; -+ struct task_struct *curr = NULL; -+ -+ switch (hmbird_preempt_policy) { -+ case HMBIRD_PREEMPT_POLICY_PRIO_BASED: -+ curr = rq->curr; -+ if (curr && hmbird_prio_higher(p, curr)) -+ goto preempt; -+ break; -+ default: -+ break; -+ } -+ if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) || -+ (is_critical_system_task(p) && !is_critical_system_task(rq->curr))) -+ goto preempt; -+ -+ sp_dl = find_idx_from_task(p); -+ if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled())) -+ return; -+ -+ if (rq->curr->prio > p->prio) -+ goto preempt; -+ -+ return; -+ -+preempt: -+ resched_curr(rq); -+} -+ -+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {} -+ -+int hmbird_check_setscheduler(struct task_struct *p, int policy) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ /* if disallow, reject transitioning into HMBIRD */ -+ if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) && -+ p->policy != policy && policy == SCHED_HMBIRD) -+ return -EACCES; -+ -+ return 0; -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool hmbird_can_stop_tick(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ if (hmbird_ops_disabling()) -+ return false; -+ -+ if (p->sched_class != &hmbird_sched_class) -+ return true; -+ -+ return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK; -+} -+#endif -+ -+int hmbird_tg_online(struct task_group *tg) -+{ -+ struct cgroup *cgrp; -+ -+ if (!tg) -+ return 0; -+ cgrp = tg->css.cgroup; -+ if (!cgrp || !(cgrp->kn)) -+ return 0; -+ update_cgroup_ids_table(cgrp->kn->id, -1); -+ return 0; -+} -+ -+/* -+ * Omitted operations: -+ * -+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -+ * task isn't tied to the CPU at that point. Preemption is implemented by -+ * resetting the victim task's slice to 0 and triggering reschedule on the -+ * target CPU. -+ * -+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -+ * -+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -+ * their current sched_class. Call them directly from sched core instead. -+ * -+ * - task_woken, switched_from: Unnecessary. -+ */ -+DEFINE_SCHED_CLASS(hmbird) = { -+ .enqueue_task = enqueue_task_hmbird, -+ .dequeue_task = dequeue_task_hmbird, -+ .yield_task = yield_task_hmbird, -+ .yield_to_task = yield_to_task_hmbird, -+ -+ .check_preempt_curr = check_preempt_curr_hmbird, -+ -+ .pick_next_task = pick_next_task_hmbird, -+ -+ .put_prev_task = put_prev_task_hmbird, -+ .set_next_task = set_next_task_hmbird, -+ -+#ifdef CONFIG_SMP -+ .balance = balance_hmbird, -+ .select_task_rq = select_task_rq_hmbird, -+ .set_cpus_allowed = set_cpus_allowed_hmbird, -+#endif -+ .task_tick = task_tick_hmbird, -+ -+ .switched_to = switched_to_hmbird, -+ .prio_changed = prio_changed_hmbird, -+ -+ .update_curr = update_curr_hmbird, -+ -+#ifdef CONFIG_UCLAMP_TASK -+ .uclamp_enabled = 0, -+#endif -+}; -+ -+/* -+ * Must with rq lock held. -+ */ -+bool task_is_hmbird(struct task_struct *p) -+{ -+ return p->sched_class == &hmbird_sched_class; -+} -+ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id) -+{ -+ memset(dsq, 0, sizeof(*dsq)); -+ -+ raw_spin_lock_init(&dsq->lock); -+ INIT_LIST_HEAD(&dsq->fifo); -+ dsq->id = dsq_id; -+} -+ -+static int hmbird_cgroup_init(void) -+{ -+ struct cgroup_subsys_state *css; -+ -+ css_for_each_descendant_pre(css, &root_task_group.css) { -+ struct task_group *tg = css_tg(css); -+ -+ cgrp_dsq_idx_init(css->cgroup, tg); -+ } -+ return 0; -+} -+ -+/* -+ * Used by sched_fork() and __setscheduler_prio() to pick the matching -+ * sched_class. dl/rt are already handled. -+ */ -+bool task_on_hmbird(struct task_struct *p) -+{ -+ return hmbird_enabled(); -+} -+ -+static void __setscheduler_prio(struct task_struct *p, int prio) -+{ -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) { -+ p->prio = prio; -+ return; -+ } else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (on_hmbird && rt_prio(prio)) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ -+ p->prio = prio; -+} -+ -+/* -+ * Heartbeat, avoid humbird keep running while APP already exit. -+ * Check whether APP send alive-signal periodly. -+ */ -+#define HEARTBEAT_TIMEOUT (msecs_to_jiffies(2500)) -+#define HEARTBEAT_CHECK_INTERVAL (msecs_to_jiffies(1000)) -+static struct timer_list hb_timer; -+static unsigned long next_hb; -+ -+void hb_timer_handler(struct timer_list *timer) -+{ -+ if (!heartbeat_enable) -+ goto refill; -+ -+ pr_info(": enter timer.\n"); -+ if (heartbeat) { -+ heartbeat = 0; -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ } -+ -+ /* can't detect heartbeat, disable ext. */ -+ if (time_after(jiffies, READ_ONCE(next_hb))) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT, -+ "can't detect heartbeat, disable ext\n"); -+ } -+refill: -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_start(void) -+{ -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_init(void) -+{ -+ timer_setup(&hb_timer, hb_timer_handler, 0); -+} -+ -+static void hb_timer_exit(void) -+{ -+ del_timer(&hb_timer); -+} -+ -+static bool check_and_disable_cpuhp(void) -+{ -+ struct rq *rq; -+ struct rq_flags rf; -+ int cpu; -+ -+ cpu_hotplug_disable(); -+ cpus_read_lock(); -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ rq_lock_irqsave(rq, &rf); -+ if (!rq->online) { -+ rq_unlock_irqrestore(rq, &rf); -+ goto err_offline; -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ cpus_read_unlock(); -+ return true; -+ -+err_offline: -+ cpus_read_unlock(); -+ cpu_hotplug_enable(); -+ return false; -+} -+ -+static void reenable_cpuhp(void) -+{ -+ cpu_hotplug_enable(); -+} -+ -+static void scheduler_switch_done(bool final_state) -+{ -+ /* set scx_enable again, switch may fail. */ -+ scx_enable = final_state; -+ /* reset heartbeat*/ -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ -+ if (final_state) -+ hb_timer_start(); -+ else -+ hb_timer_exit(); -+} -+ -+/** -+ * ss : curr switch state -+ * finish : success or fail -+ * enable : curr operation is diabling or enabling? -+ * fail_reason : string output when failed, empty string when success. -+ */ -+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason) -+{ -+ char *s1 = finish ? "finished" : "failed"; -+ char *s2 = enable ? "enabled" : "disabled"; -+ -+ hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss); -+ if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) { -+ sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type)); -+ hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n", -+ s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason); -+ } -+ curr_ss = ss; -+} -+ -+bool get_hmbird_ops_enabled(void) -+{ -+ return atomic_read(&__hmbird_ops_enabled); -+} -+ -+bool get_non_hmbird_task(void) -+{ -+ return atomic_read(&non_hmbird_task); -+} -+ -+static void hmbird_ops_disable_workfn(struct kthread_work *work) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int cpu; -+ -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, ""); -+ cancel_delayed_work_sync(&hmbird_watchdog_work); -+ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) { -+ case HMBIRD_OPS_DISABLED: -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled"); -+ scheduler_switch_done(false); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return; -+ case HMBIRD_OPS_PREPPING: -+ fallthrough; -+ case HMBIRD_OPS_DISABLING: -+ /* shouldn't happen but handle it like ENABLING if it does */ -+ WARN_ONCE(true, "hmbird: duplicate disabling instance?"); -+ fallthrough; -+ case HMBIRD_OPS_ENABLING: -+ case HMBIRD_OPS_ENABLED: -+ break; -+ } -+ -+ /* kick all CPUs to restore ticks */ -+ for_each_possible_cpu(cpu) -+ resched_cpu(cpu); -+ -+ /* avoid racing against fork and cgroup changes */ -+ cpus_read_lock(); -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, ""); -+ spin_lock_irq(&hmbird_tasks_lock); -+ atomic_set(&__hmbird_ops_enabled, false); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ bool alive = READ_ONCE(p->__state) != TASK_DEAD; -+ -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ get_hmbird_ts(p)->slice = min_t(u64, -+ get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL); -+ -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ if (alive) -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ -+ hmbird_ops_disable_task(p); -+ } -+ hmbird_task_iter_exit(&sti); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, ""); -+ -+ atomic_set(&non_hmbird_task, true); -+ /* no task is on hmbird, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt); -+ synchronize_rcu(); -+ -+ percpu_up_write(&hmbird_fork_rwsem); -+ cpus_read_unlock(); -+ -+ if (slim_walt_ctrl) -+ slim_walt_enable(false); -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ -+ hmbird_switch_log(HMBIRD_DISABLED, 1, 0, ""); -+ scheduler_switch_done(false); -+ reenable_cpuhp(); -+} -+ -+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn); -+ -+static void schedule_hmbird_ops_disable_work(void) -+{ -+ struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper); -+ -+ /* -+ * We may be called spuriously before the first bpf_hmbird_reg(). If -+ * hmbird_ops_helper isn't set up yet, there's nothing to do. -+ */ -+ if (helper) -+ kthread_queue_work(helper, &hmbird_ops_disable_work); -+} -+ -+static void hmbird_ops_disable(void) -+{ -+ schedule_hmbird_ops_disable_work(); -+} -+ -+static void hmbird_err_exit_workfn(struct work_struct *work) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ hmbird_ctrl(false); -+ -+ for_each_present_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy) -+ continue; -+ if (cpu != policy->cpu) -+ goto put; -+ down_write(&policy->rwsem); -+ WARN_ON(store_scaling_governor(policy, -+ saved_gov[cpu], strlen(saved_gov[cpu])) <= 0); -+ up_write(&policy->rwsem); -+ hmbird_info_trace(":restore origin gov : %s\n", saved_gov[cpu]); -+put: -+ cpufreq_cpu_put(policy); -+ } -+ memset((char *)saved_gov, 0, sizeof(saved_gov)); -+} -+ -+void hmbird_ops_exit(void) -+{ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); -+} -+ -+static struct kthread_worker *hmbird_create_rt_helper(const char *name) -+{ -+ struct kthread_worker *helper; -+ -+ helper = kthread_create_worker(KTW_FREEZABLE, name); -+ if (helper) -+ sched_set_fifo(helper->task); -+ return helper; -+} -+ -+static inline void set_audio_thread_sched_prop(struct task_struct *p) -+{ -+ struct cgroup_subsys_state *css; -+ -+ if (likely(p->prio >= MAX_RT_PRIO)) -+ return; -+ -+ rcu_read_lock(); -+ css = task_css(p, cpuset_cgrp_id); -+ if (!css) { -+ rcu_read_unlock(); -+ return; -+ } -+ rcu_read_unlock(); -+ -+ if (!strcmp(css->cgroup->kn->name, "audio-app")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+} -+ -+int scx_systemui_pid = -1; -+void set_systemui_thread_pid(struct task_struct *p) -+{ -+ if ((strcmp(p->comm, "ndroid.systemui") == 0) && (p->pid == p->tgid)) -+ scx_systemui_pid = p->pid; -+} -+ -+static int hmbird_ops_enable(void *unused) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int ret; -+ int tcnt = 0; -+ unsigned long long start = 0; -+ -+ if (!check_and_disable_cpuhp()) { -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline"); -+ return -EBUSY; -+ } -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, ""); -+ mutex_lock(&hmbird_ops_enable_mutex); -+ -+ if (!hmbird_ops_helper) { -+ WRITE_ONCE(hmbird_ops_helper, -+ hmbird_create_rt_helper("hmbird_ops_helper")); -+ if (!hmbird_ops_helper) { -+ ret = -ENOMEM; -+ goto err_unlock; -+ } -+ } -+ -+ if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) { -+ ret = -EBUSY; -+ goto err_unlock; -+ } -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) != -+ HMBIRD_OPS_DISABLED); -+ -+ hmbird_warned_zero_slice = false; -+ -+ atomic64_set(&hmbird_nr_rejected, 0); -+ -+ /* -+ * Keep CPUs stable during enable so that the BPF scheduler can track -+ * online CPUs by watching ->on/offline_cpu() after ->init(). -+ */ -+ cpus_read_lock(); -+ -+ hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work, -+ hmbird_watchdog_timeout / 2); -+ -+ /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ */ -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ reset_idle_masks(); -+ -+ /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. -+ */ -+ ret = hmbird_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock; -+ -+ /* -+ * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem -+ * preventing new tasks from being added. No need to exclude tasks -+ * leaving as hmbird_free() can handle both prepped and enabled -+ * tasks. Prep all tasks first and then enable them with preemption -+ * disabled. -+ */ -+ spin_lock_irq(&hmbird_tasks_lock); -+ -+ atomic_set(&non_hmbird_task, false); -+ atomic_set(&__hmbird_ops_enabled, true); -+ -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered(&sti))) -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ hmbird_task_iter_exit(&sti); -+ -+ /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. -+ */ -+ preempt_disable(); -+ -+ /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ */ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) { -+ atomic_set(&non_hmbird_task, true); -+ atomic_set(&__hmbird_ops_enabled, false); -+ preempt_enable(); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ ret = -EBUSY; -+ goto err_disable_unlock; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED -+ * transitions here are synchronized against hmbird_free() through -+ * hmbird_tasks_lock. -+ */ -+ start = sched_clock(); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, ""); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ tcnt++; -+ if (READ_ONCE(p->__state) != TASK_DEAD) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ -+ set_audio_thread_sched_prop(p); -+ set_systemui_thread_pid(p); -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ hmbird_ops_enable_task(p); -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ } else { -+ hmbird_ops_disable_task(p); -+ } -+ } -+ hmbird_task_iter_exit(&sti); -+ -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, ""); -+ preempt_enable(); -+ percpu_up_write(&hmbird_fork_rwsem); -+ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) { -+ ret = -EBUSY; -+ goto err_disable; -+ } -+ -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_ENABLED, 1, 1, ""); -+ scheduler_switch_done(true); -+ -+ return 0; -+ -+err_unlock: -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock"); -+ scheduler_switch_done(false); -+ return ret; -+ -+err_disable_unlock: -+ percpu_up_write(&hmbird_fork_rwsem); -+err_disable: -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ /* must be fully disabled before returning */ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable"); -+ scheduler_switch_done(false); -+ return ret; -+} -+ -+#ifdef CONFIG_SCHED_DEBUG -+static const char *hmbird_ops_enable_state_str[] = { -+ [HMBIRD_OPS_PREPPING] = "prepping", -+ [HMBIRD_OPS_ENABLING] = "enabling", -+ [HMBIRD_OPS_ENABLED] = "enabled", -+ [HMBIRD_OPS_DISABLING] = "disabling", -+ [HMBIRD_OPS_DISABLED] = "disabled", -+}; -+ -+static int hmbird_debug_show(struct seq_file *m, void *v) -+{ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled()); -+ seq_printf(m, "%-30s: %s\n", "enable_state", -+ hmbird_ops_enable_state_str[hmbird_ops_enable_state()]); -+ seq_printf(m, "%-30s: %llu\n", "nr_rejected", -+ atomic64_read(&hmbird_nr_rejected)); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return 0; -+} -+ -+static int hmbird_debug_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_debug_show, NULL); -+} -+ -+const struct file_operations sched_hmbird_fops = { -+ .open = hmbird_debug_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif -+ -+ -+static int bpf_hmbird_reg(void *kdata) -+{ -+ return hmbird_ops_enable(kdata); -+} -+ -+static int bpf_hmbird_unreg(void *kdata) -+{ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ return 0; -+} -+ -+void set_hmbird_module_loaded(int is_loaded) -+{ -+ atomic_set(&hmbird_module_loaded, is_loaded); -+} -+ -+/* -+ * MUST load hmbird module before enable hmbird scheduler -+ * load track & hmbird gover implemented in hmbird module -+ */ -+int hmbird_ctrl(bool enable) -+{ -+ if (!atomic_read(&hmbird_module_loaded)) { -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "ext module unloaded\n"); -+ return -EINVAL; -+ } -+ -+ if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED -+ || hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING -+ || hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already enabled(ing)\n"); -+ hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING || -+ hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already disabled(ing)\n"); -+ hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (enable) -+ return bpf_hmbird_reg(NULL); -+ else -+ return bpf_hmbird_unreg(NULL); -+} -+ -+void set_cpu_isomask(int cpu, cpumask_var_t *mask) -+{ -+ cpumask_clear_cpu(cpu, iso_masks.ex_free); -+ cpumask_clear_cpu(cpu, iso_masks.exclusive); -+ cpumask_clear_cpu(cpu, iso_masks.partial); -+ cpumask_clear_cpu(cpu, iso_masks.big); -+ cpumask_clear_cpu(cpu, iso_masks.little); -+ cpumask_set_cpu(cpu, *mask); -+} -+ -+void set_cpu_cluster(u64 cpu_cluster) -+{ -+ int cpu; -+ -+ for_each_present_cpu(cpu) { -+ u64 pos = 1 << cpu; -+ -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.ex_free)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.exclusive)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.partial)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.big)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) -+ set_cpu_isomask(cpu, &(iso_masks.little)); -+ } -+} -+ -+static void get_hmbird_snapshot(struct panic_snapshot_t *p) -+{ -+ struct hmbird_dispatch_q *dsq; -+ struct rq *rq; -+ int cpu, i; -+ -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ p->rq_nr[cpu] = rq->nr_running; -+ p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running; -+ } -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ struct hmbird_entity *entity; -+ -+ dsq = &gdsqs[i]; -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo)) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ p->runnable_at[i] = entity->runnable_at; -+ raw_spin_unlock(&dsq->lock); -+ -+ } -+ -+ p->snap_misc.hmbird_enabled = hmbird_enabled(); -+ p->snap_misc.curr_ss = curr_ss; -+ p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state(); -+ p->snap_misc.parctrl_high_ratio = parctrl_high_ratio; -+ p->snap_misc.parctrl_low_ratio = parctrl_low_ratio; -+ p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l; -+ p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l; -+ p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio; -+ p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio; -+ p->snap_misc.misfit_ds = misfit_ds; -+ p->snap_misc.partial_enable = partial_enable; -+ p->snap_misc.iso_free_rescue = iso_free_rescue; -+ p->snap_misc.isolate_ctrl = isolate_ctrl; -+ p->snap_misc.snap_jiffies = jiffies; -+ p->snap_misc.snap_time = local_clock(); -+} -+ -+// MTK minidump begin -+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3) -+{ -+ strscpy(m->desc_str, str, DESC_STR_LEN); -+ m->len = d1 * d2 * d3; -+ m->parse[0] = d1; -+ m->parse[1] = d2; -+ m->parse[2] = d3; -+} -+ -+static void init_desc_metas(struct md_info_t *m) -+{ -+ init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :", -+ 1, MAX_SWITCHS, SWITCH_ITEMS); -+ -+ init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1); -+ -+ init_desc_meta(&m->kern_dump.excep_rec_meta, -+ "excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS); -+ -+ init_desc_meta(&m->kern_dump.excep_idx_meta, -+ "excep idx :", 1, 1, MAX_EXCEP_ID); -+ -+ init_desc_meta(&m->kern_dump.snap.runnable_at_meta, -+ "each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS); -+ -+ init_desc_meta(&m->kern_dump.snap.rq_nr_meta, -+ "rq runnable task nr:", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta, -+ "scxrq runnable task nr :", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.snap_misc_meta, -+ "misc snap params :", 1, 1, SNAP_ITEMS); -+} -+ -+static void init_md_meta(struct md_info_t *m) -+{ -+ m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64); -+ m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64); -+ m->meta.unit_size = sizeof(u64); -+ m->meta.switches = MAX_SWITCHS; -+ m->meta.exceps = MAX_EXCEPS; -+ m->meta.global_dsqs = MAX_GLOBAL_DSQS; -+ m->meta.parse_dimens = PARSE_DIMENS; -+ m->meta.nr_cpus = num_possible_cpus(); -+ m->meta.real_cpus = nr_cpu_ids; -+ m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64); -+ m->meta.nr_meta_desc = 8; -+ m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64); -+ -+ init_desc_metas(m); -+} -+ -+#define MINIDUMP_DFL_SIZE (4 * 1024) -+ -+struct notifier_block hmbird_panic_blk; -+static int hmbird_panic_handler(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ if (!md_info) -+ return NOTIFY_DONE; -+ -+ get_hmbird_snapshot(&md_info->kern_dump.snap); -+ -+ return NOTIFY_DONE; -+} -+ -+static void panic_blk_init(void) -+{ -+ int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE); -+ -+ md_info = kzalloc(dump_size, GFP_KERNEL); -+ if (!md_info) -+ return; -+ init_md_meta(md_info); -+ -+ hmbird_panic_blk.notifier_call = hmbird_panic_handler; -+ /* make sure to execute before minidump. */ -+ hmbird_panic_blk.priority = INT_MAX; -+ atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk); -+ -+ hmbird_debug("register minidump.\n"); -+} -+ -+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size) -+{ -+ *vaddr = (unsigned long)md_info; -+ *size = sizeof(struct md_info_t); -+} -+// MTK minidump end -+ -+static inline void init_sched_prop_to_preempt_prio(void) -+{ -+ for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) { -+ switch (i) { -+ case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 5; -+ break; -+ -+ case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 4; -+ break; -+ -+ case HMBIRD_TASK_PROP_PIPELINE: -+ case HMBIRD_TASK_PROP_ISOLATE: -+ sched_prop_to_preempt_prio[i] = 3; -+ break; -+ -+ case HMBIRD_TASK_PROP_COMMON: -+ sched_prop_to_preempt_prio[i] = 1; -+ break; -+ -+ case HMBIRD_TASK_PROP_DEBUG_OR_LOG: -+ sched_prop_to_preempt_prio[i] = 0; -+ break; -+ -+ default: -+ sched_prop_to_preempt_prio[i] = 2; -+ break; -+ } -+ } -+} -+ -+void __init init_sched_hmbird_class(void) -+{ -+ int cpu; -+ u32 v; -+ struct hmbird_entity *init_hmbird; -+ -+ /* -+ * The following is to prevent the compiler from optimizing out the enum -+ * definitions so that BPF scheduler implementations can use them -+ * through the generated vmlinux.h. -+ */ -+ WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP | -+ HMBIRD_KICK_PREEMPT); -+ -+ init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL); -+ init_dsq_at_boot(); -+ init_isolate_cpus(); -+ hb_timer_init(); -+ init_sched_prop_to_preempt_prio(); -+#ifdef CONFIG_SMP -+ WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); -+#endif -+ -+ /* -+ * we can't static init init_task's hmbird struct, init here. -+ * init_task->hmbird would not use during boot. -+ */ -+ init_hmbird = kzalloc(sizeof(struct hmbird_entity), GFP_KERNEL); -+ init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird; -+ if (init_hmbird) { -+ INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo); -+ INIT_LIST_HEAD(&init_hmbird->watchdog_node); -+ init_hmbird->sticky_cpu = -1; -+ init_hmbird->holding_cpu = -1; -+ atomic64_set(&init_hmbird->ops_state, 0); -+ init_hmbird->runnable_at = jiffies; -+ init_hmbird->slice = HMBIRD_SLICE_DFL; -+ init_hmbird->task = &init_task; -+ hmbird_set_sched_prop(&init_task, 0); -+ } else { -+ hmbird_err(INIT_TASK_FAIL, ":alloc init_task.scx failed!!!\n"); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ /* -+ * exec during boot phase, no need to care about alloc failed. -+ * lifecycle same to rq, no need to free. -+ */ -+ rq->android_oem_data1[HMBIRD_RQ_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_rq), GFP_KERNEL); -+ if (get_hmbird_rq(rq)) { -+ get_hmbird_rq(rq)->rq = rq; -+ get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window; -+ } else { -+ hmbird_err(ALLOC_RQSCX_FAIL, ":alloc rq->scx failed!!!\n"); -+ } -+ -+ rq->android_oem_data1[HMBIRD_OPS_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL); -+ if (!get_hmbird_ops(rq)) -+ pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n"); -+ init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL); -+ INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list); -+ -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL)); -+ -+ hmbird_ops_init(get_hmbird_ops(rq)); -+ } -+ -+ INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn); -+ INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn); -+ hmbird_misc_init(); -+ -+ panic_blk_init(); -+} -+ -diff --git b/kernel/sched/hmbird/hmbird_misc.c a/kernel/sched/hmbird/hmbird_misc.c -new file mode 100755 -index 000000000000..895e8a29df33 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_misc.c -@@ -0,0 +1,148 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * @Description: -+ * @Version: -+ * @Author: wangrui8 -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include "hmbird_sched.h" -+#include -+#include -+#include "slim.h" -+ -+noinline int tracing_mark_write(const char *buf) -+{ -+ trace_printk(buf); -+ return 0; -+} -+ -+struct yield_opt_params yield_opt_params = { -+ .enable = 0, -+ .frame_per_sec = 120, -+ .frame_time_ns = NSEC_PER_SEC / 120, -+ .yield_headroom = 10, -+}; -+ -+DEFINE_PER_CPU(struct sched_yield_state, ystate); -+ -+static inline void hmbird_yield_state_update(struct sched_yield_state *ys) -+{ -+ if (!raw_spin_is_locked(&ys->lock)) -+ return; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ -+ if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1 -+ || ys->yield_cnt_after_sleep > yield_headroom) { -+ ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP); -+ } else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) { -+ ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP); -+ } -+ ys->yield_cnt = 0; -+ ys->sleep_times = 0; -+ ys->yield_cnt_after_sleep = 0; -+} -+ -+void hmbird_skip_yield(long *skip) -+{ -+ if (!get_hmbird_ops_enabled() || !yield_opt_params.enable) -+ return; -+ unsigned long flags, sleep_now = 0; -+ struct sched_yield_state *ys; -+ int cpu = raw_smp_processor_id(), cont_yield, new_frame; -+ int frame_time_ns = yield_opt_params.frame_time_ns; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ u64 wc; -+ -+ if (!(*skip)) { -+ wc = sched_clock(); -+ ys = &per_cpu(ystate, cpu); -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ -+ cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP; -+ new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1); -+ -+ if (!cont_yield && new_frame) { -+ hmbird_yield_state_update(ys); -+ ys->last_update_time = wc; -+ ys->sleep_end = ys->last_yield_time + frame_time_ns -+ - yield_headroom * YIELD_DURATION; -+ } -+ -+ if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) { -+ *skip = true; -+ -+ sleep_now = ys->sleep_times ? -+ max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep; -+ if (wc + sleep_now > ys->sleep_end) { -+ u64 delta = ys->sleep_end - wc; -+ -+ if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION) -+ sleep_now = delta; -+ else -+ sleep_now = 0; -+ } -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ if (sleep_now) { -+ sleep_now = div64_u64(sleep_now, 1000); -+ usleep_range_state(sleep_now, sleep_now, TASK_IDLE); -+ } -+ ys->sleep_times++; -+ ys->last_yield_time = sched_clock(); -+ return; -+ } -+ if (ys->sleep_times) -+ ys->yield_cnt_after_sleep++; -+ else -+ (ys->yield_cnt)++; -+ ys->last_yield_time = wc; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+} -+ -+struct boost_policy_params boost_policy_params = { -+ .enable = 0, -+ .bottom_freq = 1200000, -+ .boost_weight = 120, -+}; -+ -+int hmbird_get_boost_enable(void) -+{ -+ return boost_policy_params.enable; -+} -+ -+unsigned int hmbird_get_boost_bottom_freq(void) -+{ -+ return boost_policy_params.bottom_freq; -+} -+ -+int hmbird_get_boost_weight(void) -+{ -+ return boost_policy_params.boost_weight; -+} -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops) -+{ -+ hmbird_ops->scx_enable = get_hmbird_ops_enabled; -+ hmbird_ops->check_non_task = get_non_hmbird_task; -+ hmbird_ops->hmbird_get_md_info = hmbird_get_md_info; -+ hmbird_ops->hmbird_get_boost_enable = hmbird_get_boost_enable; -+ hmbird_ops->hmbird_get_boost_bottom_freq = hmbird_get_boost_bottom_freq; -+ hmbird_ops->hmbird_get_boost_weight = hmbird_get_boost_weight; -+} -+ -+void hmbird_misc_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_init(&ys->lock); -+ } -+ -+ set_hmbird_module_loaded(1); -+} -+ -diff --git b/kernel/sched/hmbird/hmbird_sched.h a/kernel/sched/hmbird/hmbird_sched.h -new file mode 100755 -index 000000000000..76acbe9685e4 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef __HMBIRD_SCHED__ -+#define __HMBIRD_SCHED__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include <../../kernel/time/tick-sched.h> -+#include <../../kernel/sched/sched.h> -+#include -+#include -+#include -+ -+#define REGISTER_TRACE_VH(vender_hook, handler) \ -+{ \ -+ ret = register_trace_##vender_hook(handler, NULL); \ -+ if (ret) { \ -+ pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \ -+ } \ -+} -+#define REGISTER_TRACE(vendor_hook, handler, data, err) \ -+do { \ -+ ret = register_trace_##vendor_hook(handler, data); \ -+ if (ret) { \ -+ pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret); \ -+ goto err; \ -+ } \ -+} while (0) -+ -+#define UNREGISTER_TRACE(vendor_hook, handler, data) \ -+ unregister_trace_##vendor_hook(handler, data) \ -+ -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+ -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int sched_ravg_window_frame_per_sec; -+extern int slim_gov_debug; -+extern int cpu7_tl; -+extern int scx_gov_ctrl; -+extern spinlock_t new_sched_ravg_window_lock; -+extern int cluster_separate; -+ -+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER BIT(31) -+#define MAX_YIELD_SLEEP (2000000ULL) -+#define MIN_YIELD_SLEEP (200000ULL) -+#define YIELD_DURATION (5000ULL) -+#define DEFAULT_YIELD_SLEEP_TH (10) -+ -+struct sched_yield_state { -+ raw_spinlock_t lock; -+ u64 last_yield_time; -+ u64 last_update_time; -+ u64 sleep_end; -+ unsigned long yield_cnt; -+ unsigned long yield_cnt_after_sleep; -+ unsigned long sleep; -+ int sleep_times; -+}; -+ -+DECLARE_PER_CPU(struct sched_yield_state, ystate); -+ -+void hmbird_window_rollover_run_once(struct rq *rq); -+void hmbird_yield_state_update_per_frame(void); -+void hmbird_misc_init(void); -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops); -+#endif /*__HMBIRD_SCHED__*/ -diff --git b/kernel/sched/hmbird/hmbird_sched_proc.c a/kernel/sched/hmbird/hmbird_sched_proc.c -new file mode 100755 -index 000000000000..295d45404608 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched_proc.c -@@ -0,0 +1,640 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "hmbird_sched_proc.h" -+#include -+ -+#include "hmbird_util_track.h" -+#include "slim.h" -+ -+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched" -+#define SLIM_FREQ_GOV_DIR "slim_freq_gov" -+#define LOAD_TRACK_DIR "slim_walt" -+#define HMBIRD_PROC_PERMISSION 0666 -+ -+int scx_enable; -+int partial_enable; -+int cpuctrl_high_ratio = 55; -+int cpuctrl_low_ratio = 40; -+int slim_stats; -+int hmbirdcore_debug; -+int slim_for_app; -+int misfit_ds = 90; -+unsigned int highres_tick_ctrl; -+unsigned int highres_tick_ctrl_dbg; -+int cpu7_tl = 70; -+int slim_walt_ctrl; -+int slim_walt_dump; -+int slim_walt_policy; -+int slim_gov_debug; -+int scx_gov_ctrl = 1; -+int sched_ravg_window_frame_per_sec = 125; -+int parctrl_high_ratio = 55; -+int parctrl_low_ratio = 40; -+int parctrl_high_ratio_l = 65; -+int parctrl_low_ratio_l = 50; -+int isoctrl_high_ratio = 75; -+int isoctrl_low_ratio = 60; -+int isolate_ctrl; -+int iso_free_rescue; -+int heartbeat; -+int heartbeat_enable = 1; -+int watchdog_enable; -+int save_gov; -+u64 cpu_cluster_masks; -+int hmbird_preempt_policy; -+int cluster_separate; -+ -+char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= 32) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtoint(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoint\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+static int set_proc_buf_val_u64(struct file *file, const char __user *buf, -+ size_t count, u64 *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= sizeof(kbuf)) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtou64(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoul\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/* common ops begin */ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ return count; -+} -+ -+static int hmbird_common_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%d\n", *(int *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_show, pde_data(inode)); -+} -+ -+static int hmbird_common_ul_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%lu\n", *(unsigned long *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_ul_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_ul_show, pde_data(inode)); -+} -+ -+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write); -+/* common ops end */ -+ -+/* scx_enable ops begin */ -+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC); -+ if (hmbird_ctrl(*pval)) -+ return -EFAULT; -+ -+ return count; -+} -+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write); -+/* scx_enable ops end */ -+ -+/* hmbird_stats ops begin */ -+#define MAX_STATS_BUF (4096) -+static int hmbird_stats_proc_show(struct seq_file *m, void *v) -+{ -+ char *buf; -+ -+ buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ stats_print(buf, MAX_STATS_BUF); -+ -+ seq_printf(m, "%s\n", buf); -+ -+ kfree(buf); -+ return 0; -+} -+ -+static int hmbird_stats_proc_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_stats_proc_show, inode); -+} -+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL); -+/* hmbird_stats ops end */ -+ -+/* sched_ravg_window_frame_per_sec ops begin */ -+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file, -+ const char __user *buf, size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ sched_ravg_window_change(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open, -+ sched_ravg_window_frame_per_sec_proc_write); -+/* sched_ravg_window_frame_per_sec ops end */ -+ -+static ssize_t save_gov_str(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ for_each_possible_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy || (cpu != policy->cpu)) -+ continue; -+ WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0); -+ hmbird_info_systrace(":save origin gov : %s\n", saved_gov[cpu]); -+ } -+ return count; -+} -+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str); -+ -+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ u64 *pval = (u64 *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val_u64(file, buf, count, pval)) -+ return -EFAULT; -+ -+ if (scx_enable == 0) -+ set_cpu_cluster(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write); -+ -+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ int tmp_val; -+ -+ if (set_proc_buf_val(file, buf, count, &tmp_val)) -+ return -EFAULT; -+ -+ slim_walt_enable(tmp_val); -+ *pval = tmp_val; -+ -+ return count; -+} -+ -+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write); -+ -+/* yield_opt ops begin */ -+static int yield_opt_show(struct seq_file *m, void *v) -+{ -+ struct yield_opt_params *data = m->private; -+ -+ seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n", -+ data->enable, data->frame_per_sec, data->yield_headroom); -+ return 0; -+} -+ -+static int yield_opt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, yield_opt_show, pde_data(inode)); -+} -+ -+static ssize_t yield_opt_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu; -+ unsigned long flags; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if (copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp -+ != 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) || -+ (yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp; -+ yield_opt_params.frame_per_sec = frame_per_sec_tmp; -+ yield_opt_params.yield_headroom = yield_headroom_tmp; -+ yield_opt_params.enable = enable_tmp; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ ys->last_yield_time = 0; -+ ys->last_update_time = 0; -+ ys->sleep_end = 0; -+ ys->yield_cnt = 0; -+ ys->yield_cnt_after_sleep = 0; -+ ys->sleep = 0; -+ ys->sleep_times = 0; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write); -+ -+/* boost_policy ops begin */ -+static int boost_policy_show(struct seq_file *m, void *v) -+{ -+ struct boost_policy_params *data = m->private; -+ -+ seq_printf(m, "boost_policy:{\"enable\":%d; \"bottom_freq\":%u; \"boost_weight\":%d}\n", -+ data->enable, data->bottom_freq, data->boost_weight); -+ return 0; -+} -+ -+static int boost_policy_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, boost_policy_show, pde_data(inode)); -+} -+ -+static ssize_t boost_policy_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, bottom_freq_tmp, boost_weight_tmp; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if(copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &bottom_freq_tmp, &boost_weight_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || -+ (boost_weight_tmp < 50 || boost_weight_tmp > 300) || -+ (bottom_freq_tmp < 400000 || bottom_freq_tmp > 2200000)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ boost_policy_params.bottom_freq = bottom_freq_tmp; -+ boost_policy_params.boost_weight = boost_weight_tmp; -+ boost_policy_params.enable = enable_tmp; -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(boost_policy, boost_policy_open, boost_policy_write); -+ -+/* tick_hit ops begin */ -+static int tick_hit_show(struct seq_file *m, void *v) -+{ -+ struct tick_hit_params *data = m->private; -+ -+ seq_printf(m, "tick_hit:{\"enable\":%d; \"hit_count_thres\":%d; \"jiffies_num\":%lu}\n", -+ data->enable, data->hit_count_thres, data->jiffies_num); -+ return 0; -+} -+ -+static int tick_hit_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, tick_hit_show, pde_data(inode)); -+} -+ -+static ssize_t tick_hit_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, hit_count_thres_tmp, jiffies_num_tmp; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if(copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &hit_count_thres_tmp, &jiffies_num_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ tick_hit_params.hit_count_thres = hit_count_thres_tmp; -+ tick_hit_params.jiffies_num = jiffies_num_tmp; -+ tick_hit_params.enable = enable_tmp; -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(tick_hit, tick_hit_open, tick_hit_write); -+ -+static int __init hmbird_proc_init(void) -+{ -+ struct proc_dir_entry *hmbird_dir; -+ struct proc_dir_entry *load_track_dir; -+ struct proc_dir_entry *freq_gov_dir; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ -+ /* mkdir /proc/hmbird_sched */ -+ hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL); -+ if (!hmbird_dir) { -+ pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &scx_enable_proc_ops, -+ &scx_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &partial_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_stats); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbirdcore_debug); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_for_app); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &misfit_ds); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl_dbg); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpu7_tl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &cpu_cluster_masks_proc_ops, -+ &cpu_cluster_masks); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &save_gov_proc_ops, -+ &save_gov); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &watchdog_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isolate_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &iso_free_rescue); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_stats_proc_ops); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &yield_opt_proc_ops, -+ &yield_opt_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("boost_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &boost_policy_proc_ops, -+ &boost_policy_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("tick_hit", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &tick_hit_proc_ops, -+ &tick_hit_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbird_preempt_policy); -+ /* /proc/hmbird_sched--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_walt */ -+ load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir); -+ if (!load_track_dir) { -+ pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_walt--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &slim_walt_ctrl_proc_ops, -+ &slim_walt_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_dump); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_policy); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &sched_ravg_window_frame_per_sec_proc_ops, -+ &sched_ravg_window_frame_per_sec); -+ /* /proc/hmbird_sched/slim_walt--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_freq_gov */ -+ freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir); -+ if (!freq_gov_dir) { -+ pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_freq_gov--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &slim_gov_debug); -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &scx_gov_ctrl); -+ /* /proc/hmbird_sched/slim_freq_gov--end */ -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cluster_separate); -+ -+ return 0; -+} -+ -+device_initcall(hmbird_proc_init); -diff --git b/kernel/sched/hmbird/hmbird_sched_proc.h a/kernel/sched/hmbird/hmbird_sched_proc.h -new file mode 100755 -index 000000000000..e22bc75f1dd7 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched_proc.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SCHED_PROC_H__ -+#define __HMBIRD_SCHED_PROC_H__ -+ -+#include -+#include -+ -+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \ -+ do { \ -+ if (!proc_create(name, mode, parent, proc_ops)) { \ -+ pr_err("Error creating proc entry %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \ -+ do { \ -+ if (!proc_create_data(name, mode, parent, proc_ops, data)) { \ -+ pr_err("Error creating proc entry with data %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_PROC_OPS(name, open_func, write_func) \ -+ static const struct proc_ops name##_proc_ops = { \ -+ .proc_open = open_func, \ -+ .proc_write = write_func, \ -+ .proc_read = seq_read, \ -+ .proc_lseek = seq_lseek, \ -+ .proc_release = single_release, \ -+ } -+ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos); -+static int hmbird_common_show(struct seq_file *m, void *v); -+static int hmbird_common_open(struct inode *inode, struct file *file); -+ -+#endif -diff --git b/kernel/sched/hmbird/hmbird_shadow_tick.c a/kernel/sched/hmbird/hmbird_shadow_tick.c -new file mode 100755 -index 000000000000..2744cd42bbfd ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_shadow_tick.c -@@ -0,0 +1,198 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include -+#include <../../kernel/time/tick-sched.h> -+#include -+ -+#include "hmbird_shadow_tick.h" -+#include "slim.h" -+ -+#define HIGHRES_WATCH_CPU 0 -+ -+#include -+static bool shadow_tick_enable(void) {return highres_tick_ctrl; } -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; } -+#endif -+static bool shadow_tick_timer_init_flag; -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define shadow_tick_printk(fmt, args...) \ -+do { \ -+ int cpu = smp_processor_id(); \ -+ if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU) \ -+ trace_printk("hmbird shadow tick :"fmt, args); \ -+} while (0) -+#else -+#define shadow_tick_printk(fmt, args...) -+#endif -+ -+DEFINE_PER_CPU(struct hrtimer, stt); -+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu))) -+#define STOP_IDLE_TRIGGER (1) -+#define PERIODIC_TICK_TRIGGER (2) -+#define TICK_INTVAL (1000000ULL) -+#define HMBIRD_TICK_HIT_BOOST BIT(29) -+/* -+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms, -+ * so we can't restart hrtimer in scheduler tick. -+ */ -+static DEFINE_PER_CPU(u8, trigger_event); -+ -+/* -+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick. -+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler. -+ */ -+static DEFINE_PER_CPU(u8, tick_phase); -+ -+static inline void highres_timer_ctrl(bool enable, int cpu) -+{ -+ if (enable && hmbird_enabled()) { -+ if (!hrtimer_active(shadow_tick_timer(cpu))) -+ hrtimer_start(shadow_tick_timer(cpu), -+ ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED); -+ } else { -+ if (!enable) -+ hrtimer_cancel(shadow_tick_timer(cpu)); -+ } -+} -+ -+static inline void high_res_clear_phase(int cpu) -+{ -+ per_cpu(tick_phase, cpu) = 0; -+} -+ -+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer) -+{ -+ per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3; -+ if (per_cpu(tick_phase, cpu)) { -+ hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) { -+ per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+struct tick_hit_params tick_hit_params = { -+ .enable = 0, -+ .jiffies_num = 2, -+ .hit_count_thres = 6, -+}; -+ -+static void tick_hit_critical_task(struct task_struct *curr, struct rq *rq) -+{ -+ struct hmbird_entity *see = get_hmbird_ts(curr); -+ if (!tick_hit_params.enable || !see) -+ return; -+ -+ if ((!task_is_top_task(curr) || curr->pid != curr->tgid) && (curr->pid != scx_systemui_pid)) -+ return; -+ -+ if (see->tick_hit_count == 0) { -+ if (see->start_jiffies == 0) { -+ see->start_jiffies = jiffies; -+ see->tick_hit_count = 1; -+ } else { -+ see->start_jiffies = 0; /* status reset */ -+ see->tick_hit_count = 0; -+ } -+ } else { -+ if (time_before_eq(jiffies, see->start_jiffies + tick_hit_params.jiffies_num)) { -+ see->tick_hit_count++; -+ if (see->tick_hit_count >= tick_hit_params.hit_count_thres) { -+ cpufreq_update_util(rq, HMBIRD_TICK_HIT_BOOST); -+ } -+ } else { -+ see->tick_hit_count = 1; -+ see->start_jiffies = jiffies; -+ } -+ } -+} -+ -+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ struct rq_flags rf; -+ -+ rq_lock(rq, &rf); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ tick_hit_critical_task(curr, rq); -+ rq_unlock(rq, &rf); -+ -+ return highres_next_phase(cpu, timer); -+} -+ -+void shadow_tick_timer_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance; -+ } -+} -+ -+void start_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable()) { -+ if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER) -+ highres_timer_ctrl(false, cpu); -+ per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static void stop_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ per_cpu(trigger_event, cpu) = 0; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(false, cpu); -+} -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ stop_shadow_tick_timer(); -+} -+ -+void scheduler_tick_handler(void *unused, struct rq *rq) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ start_shadow_tick_timer(); -+} -+ -+static int __init hmbird_shadow_tick_init(void) -+{ -+ int ret = 0; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ shadow_tick_timer_init(); -+ shadow_tick_timer_init_flag = true; -+ return ret; -+} -+ -+device_initcall(hmbird_shadow_tick_init); -diff --git b/kernel/sched/hmbird/hmbird_shadow_tick.h a/kernel/sched/hmbird/hmbird_shadow_tick.h -new file mode 100755 -index 000000000000..0c26fd984f0a ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_shadow_tick.h -@@ -0,0 +1,11 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SHADOW_TICK_H__ -+#define __HMBIRD_SHADOW_TICK_H__ -+ -+#include -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+void scheduler_tick_handler(void *unused, struct rq *rq); -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state); -+#endif -diff --git b/kernel/sched/hmbird/hmbird_trace.h a/kernel/sched/hmbird/hmbird_trace.h -new file mode 100755 -index 000000000000..8468b406f347 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_trace.h -@@ -0,0 +1,92 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hmbird -+ -+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HMBIRD_H -+ -+#include -+#include -+#include -+ -+#define MAX_FATAL_INFO (64) -+ -+TRACE_EVENT(hmbird_fatal_info, -+ -+ TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info), -+ -+ TP_ARGS(type, pe, lnr, bnr, info), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, type) -+ __field(int, pe) -+ __field(int, lnr) -+ __field(int, bnr) -+ __array(char, info, MAX_FATAL_INFO)), -+ -+ TP_fast_assign( -+ __entry->type = type; -+ __entry->pe = pe; -+ __entry->lnr = lnr; -+ __entry->bnr = bnr; -+ memcpy(__entry->info, info, MAX_FATAL_INFO);), -+ -+ TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s", -+ __entry->type, __entry->pe, __entry->lnr, __entry->bnr, -+ __entry->info) -+); -+ -+TRACE_EVENT(hmbird_update_history, -+ -+ TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq, -+ struct task_struct *p, u32 runtime, int samples, int event), -+ -+ TP_ARGS(hmbird, rq, p, runtime, samples, event), -+ -+ TP_STRUCT__entry( -+ __array(char, comm, TASK_COMM_LEN) -+ __field(pid_t, pid) -+ __field(unsigned int, runtime) -+ __field(int, samples) -+ __field(int, event) -+ __field(unsigned int, demand) -+ __array(u32, hist, RAVG_HIST_SIZE) -+ __field(u16, task_util) -+ __field(int, cpu)), -+ -+ TP_fast_assign( -+ memcpy(__entry->comm, p->comm, TASK_COMM_LEN); -+ __entry->pid = p->pid; -+ __entry->runtime = runtime; -+ __entry->samples = samples; -+ __entry->event = event; -+ __entry->demand = hmbird->sts.demand; -+ memcpy(__entry->hist, hmbird->sts.sum_history, -+ RAVG_HIST_SIZE * sizeof(u32)); -+ __entry->task_util = hmbird->sts.demand_scaled; -+ __entry->cpu = rq->cpu;), -+ -+ TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d", -+ __entry->comm, __entry->pid, -+ __entry->runtime, __entry->samples, -+ __entry->event, -+ __entry->demand, -+ __entry->hist[0], __entry->hist[1], -+ __entry->hist[2], __entry->hist[3], -+ __entry->hist[4], -+ __entry->task_util, -+ __entry->cpu) -+); -+ -+#endif /*_TRACE_HMBIRD_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird -+ -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE hmbird_trace -+/* This part must be outside protection */ -+#include -diff --git b/kernel/sched/hmbird/hmbird_util_track.c a/kernel/sched/hmbird/hmbird_util_track.c -new file mode 100755 -index 000000000000..d520a8403401 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_util_track.c -@@ -0,0 +1,626 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+ -+#define CREATE_TRACE_POINTS -+#include "hmbird_trace.h" -+#undef CREATE_TRACE_POINTS -+ -+#define HMBIRD_DEBUG_PANIC (1 << 3) -+static bool init_irq_work_inited; -+ -+extern noinline int tracing_mark_write(const char *buf); -+ -+int hmbird_sched_ravg_window = 8000000; -+int new_hmbird_sched_ravg_window = 8000000; -+DEFINE_SPINLOCK(new_sched_ravg_window_lock); -+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+ -+static struct irq_work hmbird_slim_walt_irq_work; -+ -+/*Sysctl related interface*/ -+#define WINDOW_STATS_RECENT 0 -+#define WINDOW_STATS_MAX 1 -+#define WINDOW_STATS_MAX_RECENT_AVG 2 -+#define WINDOW_STATS_AVG 3 -+#define WINDOW_STATS_INVALID_POLICY 4 -+ -+ -+#define SCX_SCHED_CAPACITY_SHIFT 10 -+#define SCHED_ACCOUNT_WAIT_TIME 0 -+ -+atomic64_t hmbird_irq_work_lastq_ws; -+static u64 tick_sched_clock; -+ -+static inline u64 scale_exec_time(u64 delta, struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT; -+} -+ -+static inline u64 scale_time_to_util(u64 d) -+{ -+ do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT); -+ return d; -+} -+ -+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ delta = scale_exec_time(delta, rq); -+ sts->sum += delta; -+ if (unlikely(sts->sum > hmbird_sched_ravg_window)) -+ sts->sum = hmbird_sched_ravg_window; -+ -+ return delta; -+} -+ -+ -+static int -+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) -+{ -+ /* -+ * No need to bother updating task demand for the idle task. -+ */ -+ if (is_idle_task(p)) -+ return 0; -+ -+ /* -+ * When a task is waking up it is completing a segment of non-busy -+ * time. Likewise, if wait time is not treated as busy time, then -+ * when a task begins to run or is migrated, it is not running and -+ * is completing a segment of non-busy time. -+ */ -+ if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && -+ (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) -+ return 0; -+ -+ /* -+ * The idle exit time is not accounted for the first task _picked_ up to -+ * run on the idle CPU. -+ */ -+ if (event == PICK_NEXT_TASK && rq->curr == rq->idle) -+ return 0; -+ -+ /* -+ * TASK_UPDATE can be called on sleeping task, when its moved between -+ * related groups -+ */ -+ if (event == TASK_UPDATE) { -+ if (rq->curr == p) -+ return 1; -+ -+ return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; -+ } -+ -+ return 1; -+} -+ -+static void rollover_cpu_window(struct rq *rq, bool full_window) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 curr_sum = srq->curr_runnable_sum; -+ -+ if (unlikely(full_window)) -+ curr_sum = 0; -+ -+ srq->prev_runnable_sum = curr_sum; -+ srq->curr_runnable_sum = 0; -+} -+ -+static u64 -+update_window_start(struct rq *rq, u64 wallclock, int event) -+{ -+ s64 delta; -+ int nr_windows; -+ bool full_window; -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start = srq->window_start; -+ -+ if (wallclock < srq->latest_clock) -+ wallclock = srq->latest_clock; -+ delta = wallclock - srq->window_start; -+ if (delta < 0) { -+ delta = 0; -+ wallclock = srq->window_start; -+ } -+ srq->latest_clock = wallclock; -+ if (delta < hmbird_sched_ravg_window) -+ return old_window_start; -+ -+ nr_windows = div64_u64(delta, hmbird_sched_ravg_window); -+ srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window; -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ full_window = nr_windows > 1; -+ rollover_cpu_window(rq, full_window); -+ -+ return old_window_start; -+} -+ -+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -+ -+static inline unsigned int get_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static inline unsigned int cpu_cur_freq(int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cur; -+} -+ -+static void -+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, -+ u64 wallclock) -+{ -+ int cpu = cpu_of(rq); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * -+ arch_scale_cpu_capacity(cpu), get_max_freq(cpu)); -+} -+ -+/* -+ * Called when new window is starting for a task, to record cpu usage over -+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 -+ * when, say, a real-time task runs without preemption for several windows at a -+ * stretch. -+ */ -+static void update_history(struct rq *rq, struct task_struct *p, -+ u32 runtime, int samples, int event) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u32 *hist = &sts->sum_history[0]; -+ int i; -+ u32 max = 0, avg, demand; -+ u64 sum = 0; -+ u16 demand_scaled; -+ -+ /* Ignore windows where task had no activity */ -+ if (!runtime || is_idle_task(p) || !samples) -+ goto done; -+ -+ /* Push new 'runtime' value onto stack */ -+ for (; samples > 0; samples--) { -+ hist[sts->cidx] = runtime; -+ sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE; -+ } -+ -+ for (i = 0; i < RAVG_HIST_SIZE; i++) { -+ sum += hist[i]; -+ if (hist[i] > max) -+ max = hist[i]; -+ } -+ -+ sts->sum = 0; -+ avg = div64_u64(sum, RAVG_HIST_SIZE); -+ -+ switch (slim_walt_policy) { -+ case WINDOW_STATS_RECENT: -+ demand = runtime; -+ break; -+ case WINDOW_STATS_MAX: -+ demand = max; -+ break; -+ case WINDOW_STATS_AVG: -+ demand = avg; -+ break; -+ default: -+ demand = max(avg, runtime); -+ } -+ -+ demand_scaled = scale_time_to_util(demand); -+ -+ sts->demand = demand; -+ sts->demand_scaled = demand_scaled; -+ -+done: -+ return; -+} -+ -+ -+static u64 update_task_demand(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ -+ u64 delta, window_start = srq->window_start; -+ int new_window, nr_full_windows; -+ u32 window_size = hmbird_sched_ravg_window; -+ u64 runtime; -+ -+ new_window = mark_start < window_start; -+ if (!account_busy_for_task_demand(rq, p, event)) { -+ if (new_window) -+ /* -+ * If the time accounted isn't being accounted as -+ * busy time, and a new window started, only the -+ * previous window need be closed out with the -+ * pre-existing demand. Multiple windows may have -+ * elapsed, but since empty windows are dropped, -+ * it is not necessary to account those. -+ */ -+ update_history(rq, p, sts->sum, 1, event); -+ return 0; -+ } -+ -+ if (!new_window) { -+ /* -+ * The simple case - busy time contained within the existing -+ * window. -+ */ -+ return add_to_task_demand(rq, p, wallclock - mark_start); -+ } -+ -+ /* -+ * Busy time spans at least two windows. Temporarily rewind -+ * window_start to first window boundary after mark_start. -+ */ -+ delta = window_start - mark_start; -+ nr_full_windows = div64_u64(delta, window_size); -+ window_start -= (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (window_start - mark_start) first */ -+ runtime = add_to_task_demand(rq, p, window_start - mark_start); -+ -+ /* Push new sample(s) into task's demand history */ -+ update_history(rq, p, sts->sum, 1, event); -+ if (nr_full_windows) { -+ u64 scaled_window = scale_exec_time(window_size, rq); -+ -+ update_history(rq, p, scaled_window, nr_full_windows, event); -+ runtime += nr_full_windows * scaled_window; -+ } -+ -+ /* -+ * Roll window_start back to current to process any remainder -+ * in current window. -+ */ -+ window_start += (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (wallclock - window_start) next */ -+ mark_start = window_start; -+ runtime += add_to_task_demand(rq, p, wallclock - mark_start); -+ -+ return runtime; -+} -+ -+u16 slim_walt_cpu_util(int cpu) -+{ -+ u64 prev_runnable_sum; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ prev_runnable_sum = srq->prev_runnable_sum; -+ do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT); -+ -+ return (u16)prev_runnable_sum; -+} -+ -+static DEFINE_PER_CPU(u16, prev_cpu_util); -+static inline void cpu_util_update_systrace_c(int cpu) -+{ -+ char buf[256]; -+ u16 cpu_util = slim_walt_cpu_util(cpu); -+ -+ if (cpu_util != per_cpu(prev_cpu_util, cpu)) { -+ snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n", -+ cpu, cpu_util); -+ tracing_mark_write(buf); -+ per_cpu(prev_cpu_util, cpu) = cpu_util; -+ } -+} -+ -+ -+static inline int account_busy_for_cpu_time(struct rq *rq, -+ struct task_struct *p, -+ int event) -+{ -+ return !is_idle_task(p) && (event == PUT_PREV_TASK -+ || event == TASK_UPDATE); -+} -+ -+ -+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ int new_window, full_window = 0; -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 window_start = srq->window_start; -+ u32 window_size = srq->prev_window_size; -+ u64 delta; -+ u64 *curr_runnable_sum = &srq->curr_runnable_sum; -+ u64 *prev_runnable_sum = &srq->prev_runnable_sum; -+ -+ new_window = mark_start < window_start; -+ if (new_window) -+ full_window = (window_start - mark_start) >= window_size; -+ -+ -+ if (!account_busy_for_cpu_time(rq, p, event)) -+ goto done; -+ -+ -+ if (!new_window) { -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. No rollover -+ * since we didn't start a new window. An example of this is -+ * when a task starts execution and then sleeps within the -+ * same window. -+ */ -+ delta = wallclock - mark_start; -+ -+ delta = scale_exec_time(delta, rq); -+ *curr_runnable_sum += delta; -+ -+ goto done; -+ } -+ -+ /* -+ * situations below this need window rollover, -+ * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done -+ * in update_window_start() -+ * -+ * For task counters curr/prev_window[_cpu] are rolled over in the early part of -+ * this function. If full_window(s) have expired and time since last update needs -+ * to be accounted as busy time, set the prev to a complete window size time, else -+ * add the prev window portion. -+ * -+ * For task curr counters a new window has begun, always assign -+ */ -+ -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. A new window -+ * must have been started in udpate_window_start() -+ * If any of these three above conditions are true -+ * then this busy time can't be accounted as irqtime. -+ * -+ * Busy time for the idle task need not be accounted. -+ * -+ * An example of this would be a task that starts execution -+ * and then sleeps once a new window has begun. -+ */ -+ -+ /* -+ * A full window hasn't elapsed, account partial -+ * contribution to previous completed window. -+ */ -+ -+ -+ delta = full_window ? scale_exec_time(window_size, rq) : -+ scale_exec_time(window_start - mark_start, rq); -+ -+ *prev_runnable_sum += delta; -+ -+ /* Account piece of busy time in the current window. */ -+ delta = scale_exec_time(wallclock - window_start, rq); -+ *curr_runnable_sum += delta; -+ -+done: -+ if (slim_walt_dump && new_window) -+ cpu_util_update_systrace_c(rq->cpu); -+} -+ -+ -+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq) -+{ -+ u64 result; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 new_window_start = srq->window_start; -+ -+ if (old_window_start == new_window_start) -+ return; -+ -+ result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws, -+ old_window_start, new_window_start); -+ if (result != old_window_start) -+ return; -+ -+ if (likely(cpu_online(raw_smp_processor_id()))) -+ irq_work_queue(&hmbird_slim_walt_irq_work); -+ else -+ irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask)); -+} -+ -+/* -+ * In the core scheduler, most of the load update points update the rq_clock after -+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of -+ * obtaining the time, but to prevent the subsequent migration of the load update -+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock -+ * update here. -+ */ -+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p, -+ struct rq *rq, int event) -+{ -+ if (!(rq->clock_update_flags & RQCF_UPDATED)) -+ update_rq_clock(rq); -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock)); -+} -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start; -+ -+ if (!slim_walt_ctrl) -+ return; -+ -+ if (!srq->window_start || sts->mark_start == wallclock) -+ return; -+ -+ old_window_start = update_window_start(rq, wallclock, event); -+ -+ if (!sts->window_start) -+ sts->window_start = srq->window_start; -+ -+ if (!sts->mark_start) -+ goto done; -+ -+ update_task_rq_cpu_cycles(p, rq, event, wallclock); -+ update_task_demand(p, rq, event, wallclock); -+ update_cpu_busy_time(p, rq, event, wallclock); -+ -+ sts->window_start = srq->window_start; -+ -+done: -+ sts->mark_start = wallclock; -+ slim_walt_window_rollover_run_once(old_window_start, rq); -+} -+ -+static void slim_walt_irq_work(struct irq_work *irq_work) -+{ -+ cpumask_t lock_cpus; -+ struct hmbird_sched_rq_stats *srq; -+ struct rq *rq; -+ int cpu; -+ int level = 0; -+ u64 wc; -+ unsigned long flags; -+ -+ cpumask_copy(&lock_cpus, cpu_possible_mask); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ if (level == 0) -+ raw_spin_lock(&cpu_rq(cpu)->__lock); -+ else -+ raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); -+ level++; -+ } -+ -+ wc = sched_clock(); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ rq = cpu_rq(cpu); -+ hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc); -+ } -+ -+ cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER); -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) { -+ srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id()); -+ if (wc < srq->window_start + new_hmbird_sched_ravg_window) -+ hmbird_sched_ravg_window = new_hmbird_sched_ravg_window; -+ } -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ raw_spin_unlock(&cpu_rq(cpu)->__lock); -+ } -+} -+static void hmbird_sched_init_rq(struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ srq->task_exec_scale = 1024; -+ srq->window_start = 0; -+} -+ -+void hmbird_sched_init_task(struct task_struct *p) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ memset(sts, 0, sizeof(struct hmbird_sched_task_stats)); -+} -+ -+static void hmbird_sched_stats_init(void) -+{ -+ unsigned long flags; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ raw_spin_lock_irqsave(&rq->__lock, flags); -+ hmbird_sched_init_rq(rq); -+ raw_spin_unlock_irqrestore(&rq->__lock, flags); -+ } -+ slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG; -+ -+ if (false == init_irq_work_inited) { -+ init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work); -+ init_irq_work_inited = true; -+ } -+} -+ -+void hmbird_scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (unlikely(!tick_sched_clock)) { -+ /* -+ * Let the window begin 20us prior to the tick, -+ * that way we are guaranteed a rollover when the tick occurs. -+ * Use rq->clock directly instead of rq_clock() since -+ * we do not have the rq lock and -+ * rq->clock was updated in the tick callpath. -+ */ -+ if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000)) -+ return; -+ for_each_possible_cpu(cpu) { -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ srq->window_start = tick_sched_clock; -+ } -+ atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock); -+ } -+} -+ -+void slim_walt_enable(int enable) -+{ -+ if (1 == !!enable) { -+ hmbird_sched_stats_init(); -+ WRITE_ONCE(tick_sched_clock, 0); -+ } else -+ slim_walt_ctrl = 0; -+} -+ -+void slim_get_cpu_util(int cpu, u64 *util) -+{ -+ if (cpu < 0) -+ return; -+ -+ *util = slim_walt_cpu_util(cpu); -+} -+ -+void slim_get_task_util(struct task_struct *p, u64 *util) -+{ -+ if (p == NULL) -+ return; -+ -+ *util = get_hmbird_ts(p)->sts.demand_scaled; -+} -+ -+void sched_ravg_window_change(int frame_per_sec) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec; -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+} -diff --git b/kernel/sched/hmbird/hmbird_util_track.h a/kernel/sched/hmbird/hmbird_util_track.h -new file mode 100644 -index 000000000000..17b85df7f83b ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_util_track.h -@@ -0,0 +1,33 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef __HMBIRD_UTIL_TRACK_H__ -+#define __HMBIRD_UTIL_TRACK_H__ -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock); -+void hmbird_sched_init_task(struct task_struct *p); -+void slim_walt_enable(int enable); -+void slim_get_cpu_util(int cpu, u64 *util); -+void slim_get_task_util(struct task_struct *p, u64 *util); -+ -+extern atomic64_t hmbird_irq_work_lastq_ws; -+ -+enum task_event { -+ PUT_PREV_TASK = 0, -+ PICK_NEXT_TASK = 1, -+ TASK_WAKE = 2, -+ TASK_MIGRATE = 3, -+ TASK_UPDATE = 4, -+ IRQ_UPDATE = 5, -+}; -+ -+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+#endif /* __HMBIRD_UTIL_TRACK_H__ */ -diff --git b/kernel/sched/hmbird/slim.h a/kernel/sched/hmbird/slim.h -new file mode 100755 -index 000000000000..dffe6506658e ---- /dev/null -+++ a/kernel/sched/hmbird/slim.h -@@ -0,0 +1,56 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+ -+#ifndef __SLIM_H -+#define __SLIM_H -+ -+extern atomic_t __hmbird_ops_enabled; -+extern atomic_t non_hmbird_task; -+extern int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+extern int heartbeat; -+extern int heartbeat_enable; -+extern int watchdog_enable; -+extern int isolate_ctrl; -+extern int parctrl_high_ratio; -+extern int parctrl_low_ratio; -+extern int isoctrl_high_ratio; -+extern int isoctrl_low_ratio; -+extern int iso_free_rescue; -+extern int yield_opt; -+ -+extern enum hmbird_switch_type sw_type; -+extern noinline int tracing_mark_write(const char *buf); -+int task_top_id(struct task_struct *p); -+void stats_print(char *buf, int len); -+void hmbird_skip_yield(long *skip); -+extern spinlock_t hmbird_tasks_lock; -+extern int scx_systemui_pid; -+ -+struct yield_opt_params { -+ int enable; -+ int frame_per_sec; -+ u64 frame_time_ns; -+ int yield_headroom; -+}; -+ -+extern struct yield_opt_params yield_opt_params; -+ -+struct tick_hit_params { -+ int enable; -+ unsigned long jiffies_num; -+ int hit_count_thres; -+}; -+ -+extern struct tick_hit_params tick_hit_params; -+ -+struct boost_policy_params { -+ int enable; -+ unsigned int bottom_freq; -+ int boost_weight; -+}; -+ -+extern struct boost_policy_params boost_policy_params; -+ -+#define MAX_GOV_LEN (16) -+extern char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+#endif -diff --git b/kernel/sched/idle.c a/kernel/sched/idle.c -index b33cefeb4188..487255ac6832 100755 ---- b/kernel/sched/idle.c -+++ a/kernel/sched/idle.c -@@ -408,13 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl - - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) - { -- scx_update_idle(rq, false); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, false); -+#endif - } - - static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) - { - update_idle_core(rq); -- scx_update_idle(rq, true); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, true); -+#endif - schedstat_inc(rq->sched_goidle); - } - -diff --git b/kernel/sched/sched.h a/kernel/sched/sched.h -index 509e8dc616ab..a7e9caea1efe 100755 ---- b/kernel/sched/sched.h -+++ a/kernel/sched/sched.h -@@ -188,18 +188,13 @@ static inline int idle_policy(int policy) - return policy == SCHED_IDLE; - } - --static inline int normal_policy(int policy) --{ --#ifdef CONFIG_SCHED_CLASS_EXT -- if (policy == SCHED_EXT) -- return true; --#endif -- return policy == SCHED_NORMAL; --} -- - static inline int fair_policy(int policy) - { -- return normal_policy(policy) || policy == SCHED_BATCH; -+#ifdef CONFIG_HMBIRD_SCHED -+ return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#else -+ return policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#endif - } - - static inline int rt_policy(int policy) -@@ -247,25 +242,7 @@ static inline void update_avg(u64 *avg, u64 sample) - #define shr_bound(val, shift) \ - (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) - --/* -- * cgroup weight knobs should use the common MIN, DFL and MAX values which are -- * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it -- * maps pretty well onto the shares value used by scheduler and the round-trip -- * conversions preserve the original value over the entire range. -- */ --static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) --{ -- return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); --} -- --static inline unsigned long sched_weight_to_cgroup(unsigned long weight) --{ -- return clamp_t(unsigned long, -- DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -- CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); --} -- --/* -+/* - * !! For sched_setattr_nocheck() (kernel) only !! - * - * This is actually gross. :( -@@ -532,10 +509,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) { } - #endif /* CONFIG_SMP */ - #else /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_HMBIRD_SCHED - static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) - { - return 0; - } -+#endif - #endif /* CONFIG_FAIR_GROUP_SCHED */ - - #else /* CONFIG_CGROUP_SCHED */ -@@ -700,29 +679,6 @@ struct cfs_rq { - #endif /* CONFIG_FAIR_GROUP_SCHED */ - }; - --#ifdef CONFIG_SCHED_CLASS_EXT --/* scx_rq->flags, protected by the rq lock */ --enum scx_rq_flags { -- SCX_RQ_CAN_STOP_TICK = 1 << 0, --}; -- --struct scx_rq { -- struct scx_dispatch_q local_dsq; -- struct list_head watchdog_list; -- u64 ops_qseq; -- u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -- u32 nr_running; -- u32 flags; -- bool cpu_released; -- cpumask_var_t cpus_to_kick; -- cpumask_var_t cpus_to_preempt; -- cpumask_var_t cpus_to_wait; -- u64 pnt_seq; -- struct irq_work kick_cpus_irq_work; -- struct rq *rq; --}; --#endif /* CONFIG_SCHED_CLASS_EXT */ -- - static inline int rt_bandwidth_enabled(void) - { - return sysctl_sched_rt_runtime >= 0; -@@ -1258,11 +1214,7 @@ struct rq { - - ANDROID_OEM_DATA_ARRAY(1, 16); - --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, struct scx_rq *scx); --#else - ANDROID_KABI_RESERVE(1); --#endif - ANDROID_KABI_RESERVE(2); - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); -@@ -2340,11 +2292,6 @@ struct affinity_context { - unsigned int flags; - }; - --enum rq_onoff_reason { -- RQ_ONOFF_HOTPLUG, /* CPU is going on/offline */ -- RQ_ONOFF_TOPOLOGY, /* sched domain topology update */ --}; -- - struct sched_class { - - #ifdef CONFIG_UCLAMP_TASK -@@ -2551,7 +2498,6 @@ extern void init_sched_rt_class(void); - extern void init_sched_fair_class(void); - - extern void reweight_task(struct task_struct *p, int prio); --extern void __setscheduler_prio(struct task_struct *p, int prio); - - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); -@@ -2632,53 +2578,6 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) - extern void activate_task(struct rq *rq, struct task_struct *p, int flags); - extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); - --struct sched_change_guard { -- struct task_struct *p; -- struct rq *rq; -- bool queued; -- bool running; -- bool done; --}; -- --extern struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -- --extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags); -- --/** -- * SCHED_CHANGE_BLOCK - Nested block for task attribute updates -- * @__rq: Runqueue the target task belongs to -- * @__p: Target task -- * @__flags: DEQUEUE/ENQUEUE_* flags -- * -- * A task may need to be dequeued and put_prev_task'd for attribute updates and -- * set_next_task'd and re-enqueued afterwards. This helper defines a nested -- * block which automatically handles these preparation and cleanup operations. -- * -- * SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- * update_attribute(p); -- * ... -- * } -- * -- * If @__flags is a variable, the variable may be updated in the block body and -- * the updated value will be used when re-enqueueing @p. -- * -- * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling -- * update_rq_clock() beforehand. Otherwise, the rq clock is automatically -- * updated iff the task needs to be dequeued and re-enqueued. Only the former -- * case guarantees that the rq clock is up-to-date inside and after the block. -- */ --#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -- for (struct sched_change_guard __cg = \ -- sched_change_guard_init(__rq, __p, __flags); \ -- !__cg.done; sched_change_guard_fini(&__cg, __flags)) -- --extern void check_class_changing(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class); --extern void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio); -- - extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); - - #ifdef CONFIG_PREEMPT_RT -@@ -3710,6 +3609,8 @@ static inline bool cpu_busy_with_softirqs(int cpu) - } - #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ - --#include "ext.h" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird.h" -+#endif - - #endif /* _KERNEL_SCHED_SCHED_H */ -diff --git b/kernel/time/tick-sched.c a/kernel/time/tick-sched.c -index 998c2eefe173..c13772ee823c 100755 ---- b/kernel/time/tick-sched.c -+++ a/kernel/time/tick-sched.c -@@ -34,6 +34,10 @@ - - #include - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "../sched/hmbird/hmbird_shadow_tick.h" -+#endif -+ - /* - * Per-CPU nohz control structure - */ -@@ -1124,6 +1128,9 @@ void tick_nohz_idle_stop_tick(void) - ktime_t expires; - - trace_android_vh_tick_nohz_idle_stop_tick(NULL); -+#ifdef CONFIG_HMBIRD_SCHED -+ android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL); -+#endif - /* - * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the - * tick timer expiration time is known already. -diff --git b/tools/sched_ext/.gitignore a/tools/sched_ext/.gitignore -deleted file mode 100644 -index a3240f9f7eba..000000000000 ---- b/tools/sched_ext/.gitignore -+++ /dev/null -@@ -1,9 +0,0 @@ --scx_example_simple --scx_example_qmap --scx_example_central --scx_example_pair --scx_example_flatcg --scx_example_userland --*.skel.h --*.subskel.h --/tools/ -diff --git b/tools/sched_ext/Makefile a/tools/sched_ext/Makefile -deleted file mode 100644 -index 73c43782837d..000000000000 ---- b/tools/sched_ext/Makefile -+++ /dev/null -@@ -1,216 +0,0 @@ --# SPDX-License-Identifier: GPL-2.0 --# Copyright (c) 2022 Meta Platforms, Inc. and affiliates. --include ../build/Build.include --include ../scripts/Makefile.arch --include ../scripts/Makefile.include -- --ifneq ($(LLVM),) --ifneq ($(filter %/,$(LLVM)),) --LLVM_PREFIX := $(LLVM) --else ifneq ($(filter -%,$(LLVM)),) --LLVM_SUFFIX := $(LLVM) --endif -- --CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi --CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu --CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl --CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu --CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu --CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu --CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu --CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu --CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu --CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) -- --ifeq ($(CROSS_COMPILE),) --ifeq ($(CLANG_TARGET_FLAGS),) --$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk --else --CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) --endif # CLANG_TARGET_FLAGS --else --CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) --endif # CROSS_COMPILE -- --CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as --else --CC := $(CROSS_COMPILE)gcc --endif # LLVM -- --CURDIR := $(abspath .) --TOOLSDIR := $(abspath ..) --LIBDIR := $(TOOLSDIR)/lib --BPFDIR := $(LIBDIR)/bpf --TOOLSINCDIR := $(TOOLSDIR)/include --BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool --APIDIR := $(TOOLSINCDIR)/uapi --GENDIR := $(abspath ../../include/generated) --GENHDR := $(GENDIR)/autoconf.h -- --SCRATCH_DIR := $(CURDIR)/tools --BUILD_DIR := $(SCRATCH_DIR)/build --INCLUDE_DIR := $(SCRATCH_DIR)/include --BPFOBJ_DIR := $(BUILD_DIR)/libbpf --BPFOBJ := $(BPFOBJ_DIR)/libbpf.a --ifneq ($(CROSS_COMPILE),) --HOST_BUILD_DIR := $(BUILD_DIR)/host --HOST_SCRATCH_DIR := host-tools --HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include --else --HOST_BUILD_DIR := $(BUILD_DIR) --HOST_SCRATCH_DIR := $(SCRATCH_DIR) --HOST_INCLUDE_DIR := $(INCLUDE_DIR) --endif --HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a --RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids --DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -- --VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ -- $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ -- ../../vmlinux \ -- /sys/kernel/btf/vmlinux \ -- /boot/vmlinux-$(shell uname -r) --VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) --ifeq ($(VMLINUX_BTF),) --$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") --endif -- --BPFTOOL ?= $(DEFAULT_BPFTOOL) -- --ifneq ($(wildcard $(GENHDR)),) -- GENFLAGS := -DHAVE_GENHDR --endif -- --CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ -- -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -- -I$(TOOLSINCDIR) -I$(APIDIR) -- --CARGOFLAGS := --release -- --# Silence some warnings when compiled with clang --ifneq ($(LLVM),) --CFLAGS += -Wno-unused-command-line-argument --endif -- --LDFLAGS = -lelf -lz -lpthread -- --IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - &1 \ -- | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ --$(shell $(1) -dM -E - $@ --else -- $(call msg,CP,,$@) -- $(Q)cp "$(VMLINUX_H)" $@ --endif -- --%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \ -- | $(BPFOBJ) -- $(call msg,CLNG-BPF,,$@) -- $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ -- --%.skel.h: %.bpf.o $(BPFTOOL) -- $(call msg,GEN-SKEL,,$@) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) -- $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) -- $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@ -- $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h) -- --scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_userland: scx_example_userland.c scx_example_userland.skel.h \ -- scx_example_userland_common.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR) --atropos: export ATROPOS_CLANG = $(CLANG) --atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS) --atropos: $(INCLUDE_DIR)/vmlinux.h -- cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS) -- --clean: -- cargo clean --manifest-path=atropos/Cargo.toml -- rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) -- rm -f *.o *.bpf.o *.skel.h *.subskel.h -- rm -f scx_example_simple scx_example_qmap scx_example_central \ -- scx_example_pair scx_example_flatcg scx_example_userland -- --.PHONY: all atropos clean -- --# delete failed targets --.DELETE_ON_ERROR: -- --# keep intermediate (.skel.h, .bpf.o, etc) targets --.SECONDARY: -diff --git b/tools/sched_ext/atropos/.gitignore a/tools/sched_ext/atropos/.gitignore -deleted file mode 100644 -index 186dba259ec2..000000000000 ---- b/tools/sched_ext/atropos/.gitignore -+++ /dev/null -@@ -1,3 +0,0 @@ --src/bpf/.output --Cargo.lock --target -diff --git b/tools/sched_ext/atropos/Cargo.toml a/tools/sched_ext/atropos/Cargo.toml -deleted file mode 100644 -index 7462a836d53d..000000000000 ---- b/tools/sched_ext/atropos/Cargo.toml -+++ /dev/null -@@ -1,28 +0,0 @@ --[package] --name = "scx_atropos" --version = "0.5.0" --authors = ["Dan Schatzberg ", "Meta"] --edition = "2021" --description = "Userspace scheduling with BPF" --license = "GPL-2.0-only" -- --[dependencies] --anyhow = "1.0.65" --bitvec = { version = "1.0", features = ["serde"] } --clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } --ctrlc = { version = "3.1", features = ["termination"] } --fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"} --hex = "0.4.3" --libbpf-rs = "0.19.1" --libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] } --libc = "0.2.137" --log = "0.4.17" --ordered-float = "3.4.0" --simplelog = "0.12.0" -- --[build-dependencies] --bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false } --libbpf-cargo = "0.13.0" -- --[features] --enable_backtrace = [] -diff --git b/tools/sched_ext/atropos/build.rs a/tools/sched_ext/atropos/build.rs -deleted file mode 100644 -index 26e792c5e17e..000000000000 ---- b/tools/sched_ext/atropos/build.rs -+++ /dev/null -@@ -1,70 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --extern crate bindgen; -- --use std::env; --use std::fs::create_dir_all; --use std::path::Path; --use std::path::PathBuf; -- --use libbpf_cargo::SkeletonBuilder; -- --const HEADER_PATH: &str = "src/bpf/atropos.h"; -- --fn bindgen_atropos() { -- // Tell cargo to invalidate the built crate whenever the wrapper changes -- println!("cargo:rerun-if-changed={}", HEADER_PATH); -- -- // The bindgen::Builder is the main entry point -- // to bindgen, and lets you build up options for -- // the resulting bindings. -- let bindings = bindgen::Builder::default() -- // The input header we would like to generate -- // bindings for. -- .header(HEADER_PATH) -- // Tell cargo to invalidate the built crate whenever any of the -- // included header files changed. -- .parse_callbacks(Box::new(bindgen::CargoCallbacks)) -- // Finish the builder and generate the bindings. -- .generate() -- // Unwrap the Result and panic on failure. -- .expect("Unable to generate bindings"); -- -- // Write the bindings to the $OUT_DIR/bindings.rs file. -- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); -- bindings -- .write_to_file(out_path.join("atropos-sys.rs")) -- .expect("Couldn't write bindings!"); --} -- --fn gen_bpf_sched(name: &str) { -- let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap(); -- let clang = env::var("ATROPOS_CLANG").unwrap(); -- eprintln!("{}", clang); -- let outpath = format!("./src/bpf/.output/{}.skel.rs", name); -- let skel = Path::new(&outpath); -- let src = format!("./src/bpf/{}.bpf.c", name); -- SkeletonBuilder::new() -- .source(src.clone()) -- .clang(clang) -- .clang_args(bpf_cflags) -- .build_and_generate(&skel) -- .unwrap(); -- println!("cargo:rerun-if-changed={}", src); --} -- --fn main() { -- bindgen_atropos(); -- // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton. -- // Reasons are because the generated skeleton contains compiler attributes -- // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]` -- // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside -- // the path attribute either (see https://github.com/rust-lang/rust/pull/83366). -- // -- // However, there is hope! When the above feature stabilizes we can clean this -- // all up. -- create_dir_all("./src/bpf/.output").unwrap(); -- gen_bpf_sched("atropos"); --} -diff --git b/tools/sched_ext/atropos/rustfmt.toml a/tools/sched_ext/atropos/rustfmt.toml -deleted file mode 100644 -index b7258ed0a8d8..000000000000 ---- b/tools/sched_ext/atropos/rustfmt.toml -+++ /dev/null -@@ -1,8 +0,0 @@ --# Get help on options with `rustfmt --help=config` --# Please keep these in alphabetical order. --edition = "2021" --group_imports = "StdExternalCrate" --imports_granularity = "Item" --merge_derives = false --use_field_init_shorthand = true --version = "Two" -diff --git b/tools/sched_ext/atropos/src/atropos_sys.rs a/tools/sched_ext/atropos/src/atropos_sys.rs -deleted file mode 100644 -index bbeaf856d40e..000000000000 ---- b/tools/sched_ext/atropos/src/atropos_sys.rs -+++ /dev/null -@@ -1,10 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#![allow(non_upper_case_globals)] --#![allow(non_camel_case_types)] --#![allow(non_snake_case)] --#![allow(dead_code)] -- --include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs")); -diff --git b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -deleted file mode 100644 -index 3905a403e940..000000000000 ---- b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -+++ /dev/null -@@ -1,743 +0,0 @@ --/* Copyright (c) Meta Platforms, Inc. and affiliates. */ --/* -- * This software may be used and distributed according to the terms of the -- * GNU General Public License version 2. -- * -- * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF -- * part does simple round robin in each domain and the userspace part -- * calculates the load factor of each domain and tells the BPF part how to load -- * balance the domains. -- * -- * Every task has an entry in the task_data map which lists which domain the -- * task belongs to. When a task first enters the system (atropos_prep_enable), -- * they are round-robined to a domain. -- * -- * atropos_select_cpu is the primary scheduling logic, invoked when a task -- * becomes runnable. The lb_data map is populated by userspace to inform the BPF -- * scheduler that a task should be migrated to a new domain. Otherwise, the task -- * is scheduled in priority order as follows: -- * * The current core if the task was woken up synchronously and there are idle -- * cpus in the system -- * * The previous core, if idle -- * * The pinned-to core if the task is pinned to a specific core -- * * Any idle cpu in the domain -- * -- * If none of the above conditions are met, then the task is enqueued to a -- * dispatch queue corresponding to the domain (atropos_enqueue). -- * -- * atropos_dispatch will attempt to consume a task from its domain's -- * corresponding dispatch queue (this occurs after scheduling any tasks directly -- * assigned to it due to the logic in atropos_select_cpu). If no task is found, -- * then greedy load stealing will attempt to find a task on another dispatch -- * queue to run. -- * -- * Load balancing is almost entirely handled by userspace. BPF populates the -- * task weight, dom mask and current dom in the task_data map and executes the -- * load balance based on userspace populating the lb_data map. -- */ --#include "../../../scx_common.bpf.h" --#include "atropos.h" -- --#include --#include --#include --#include --#include --#include -- --char _license[] SEC("license") = "GPL"; -- --/* -- * const volatiles are set during initialization and treated as consts by the -- * jit compiler. -- */ -- --/* -- * Domains and cpus -- */ --const volatile __u32 nr_doms = 32; /* !0 for veristat, set during init */ --const volatile __u32 nr_cpus = 64; /* !0 for veristat, set during init */ --const volatile __u32 cpu_dom_id_map[MAX_CPUS]; --const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; -- --const volatile bool kthreads_local; --const volatile bool fifo_sched; --const volatile bool switch_partial; --const volatile __u32 greedy_threshold; -- --/* base slice duration */ --const volatile __u64 slice_us = 20000; -- --/* -- * Exit info -- */ --int exit_type = SCX_EXIT_NONE; --char exit_msg[SCX_EXIT_MSG_LEN]; -- --struct pcpu_ctx { -- __u32 dom_rr_cur; /* used when scanning other doms */ -- -- /* libbpf-rs does not respect the alignment, so pad out the struct explicitly */ -- __u8 _padding[CACHELINE_SIZE - sizeof(u64)]; --} __attribute__((aligned(CACHELINE_SIZE))); -- --struct pcpu_ctx pcpu_ctx[MAX_CPUS]; -- --/* -- * Domain context -- */ --struct dom_ctx { -- struct bpf_cpumask __kptr *cpumask; -- u64 vtime_now; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __type(key, u32); -- __type(value, struct dom_ctx); -- __uint(max_entries, MAX_DOMS); -- __uint(map_flags, 0); --} dom_ctx SEC(".maps"); -- --/* -- * Statistics -- */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, ATROPOS_NR_STATS); --} stats SEC(".maps"); -- --static inline void stat_add(enum stat_idx idx, u64 addend) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p) += addend; --} -- --/* Map pid -> task_ctx */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, struct task_ctx); -- __uint(max_entries, 1000000); -- __uint(map_flags, 0); --} task_data SEC(".maps"); -- --/* -- * This is populated from userspace to indicate which pids should be reassigned -- * to new doms. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, u32); -- __uint(max_entries, 1000); -- __uint(map_flags, 0); --} lb_data SEC(".maps"); -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p, -- u32 new_dom_id) --{ -- struct dom_ctx *old_domc, *new_domc; -- struct bpf_cpumask *d_cpumask, *t_cpumask; -- u32 old_dom_id = task_ctx->dom_id; -- s64 vtime_delta; -- -- old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id); -- if (!old_domc) { -- scx_bpf_error("No dom%u", old_dom_id); -- return false; -- } -- -- vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now; -- -- new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id); -- if (!new_domc) { -- scx_bpf_error("No dom%u", new_dom_id); -- return false; -- } -- -- d_cpumask = new_domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to get domain %u cpumask kptr", -- new_dom_id); -- return false; -- } -- -- t_cpumask = task_ctx->cpumask; -- if (!t_cpumask) { -- scx_bpf_error("Failed to look up task cpumask"); -- return false; -- } -- -- /* -- * set_cpumask might have happened between userspace requesting LB and -- * here and @p might not be able to run in @dom_id anymore. Verify. -- */ -- if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- p->cpus_ptr)) { -- p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta; -- task_ctx->dom_id = new_dom_id; -- bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask, -- p->cpus_ptr); -- } -- -- return task_ctx->dom_id == new_dom_id; --} -- --s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu, -- u32 wake_flags) --{ -- s32 cpu; -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- struct bpf_cpumask *p_cpumask; -- -- if (!task_ctx) -- return -ENOENT; -- -- if (kthreads_local && -- (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- cpu = prev_cpu; -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local dsq of the waker. -- */ -- if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) { -- struct task_struct *current = (void *)bpf_get_current_task(); -- -- if (!(BPF_CORE_READ(current, flags) & PF_EXITING) && -- task_ctx->dom_id < MAX_DOMS) { -- struct dom_ctx *domc; -- struct bpf_cpumask *d_cpumask; -- const struct cpumask *idle_cpumask; -- bool has_idle; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id); -- if (!domc) { -- scx_bpf_error("Failed to find dom%u", -- task_ctx->dom_id); -- return prev_cpu; -- } -- d_cpumask = domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to acquire domain %u cpumask kptr", -- task_ctx->dom_id); -- return prev_cpu; -- } -- -- idle_cpumask = scx_bpf_get_idle_cpumask(); -- -- has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- idle_cpumask); -- -- scx_bpf_put_idle_cpumask(idle_cpumask); -- -- if (has_idle) { -- cpu = bpf_get_smp_processor_id(); -- if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- stat_add(ATROPOS_STAT_WAKE_SYNC, 1); -- goto local; -- } -- } -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- stat_add(ATROPOS_STAT_PREV_IDLE, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- /* If only one core is allowed, dispatch */ -- if (p->nr_cpus_allowed == 1) { -- stat_add(ATROPOS_STAT_PINNED, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) -- return -ENOENT; -- -- /* If there is an eligible idle CPU, dispatch directly */ -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- if (cpu >= 0) { -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * @prev_cpu may be in a different domain. Returning an out-of-domain -- * CPU can lead to stalls as all in-domain CPUs may be idle by the time -- * @p gets enqueued. -- */ -- if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask)) -- cpu = prev_cpu; -- else -- cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask); -- -- return cpu; -- --local: -- task_ctx->dispatch_local = true; -- return cpu; --} -- --void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- u32 *new_dom; -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- new_dom = bpf_map_lookup_elem(&lb_data, &pid); -- if (new_dom && *new_dom != task_ctx->dom_id && -- task_set_dsq(task_ctx, p, *new_dom)) { -- struct bpf_cpumask *p_cpumask; -- s32 cpu; -- -- stat_add(ATROPOS_STAT_LOAD_BALANCE, 1); -- -- /* -- * If dispatch_local is set, We own @p's idle state but we are -- * not gonna put the task in the associated local dsq which can -- * cause the CPU to stall. Kick it. -- */ -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0); -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) { -- scx_bpf_error("Failed to get task_ctx->cpumask"); -- return; -- } -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- } -- -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags); -- return; -- } -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000, -- enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- u32 dom_id = task_ctx->dom_id; -- struct dom_ctx *domc; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, domc->vtime_now - slice_us * 1000)) -- vtime = domc->vtime_now - slice_us * 1000; -- -- scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --static u32 cpu_to_dom_id(s32 cpu) --{ -- const volatile u32 *dom_idp; -- -- if (nr_doms <= 1) -- return 0; -- -- dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]); -- if (!dom_idp) -- return MAX_DOMS; -- -- return *dom_idp; --} -- --static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) --{ -- s32 cpu; -- -- if (dom_id >= MAX_DOMS) -- return false; -- -- bpf_for(cpu, 0, nr_cpus) { -- if (bpf_cpumask_test_cpu(cpu, cpumask) && -- (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64)))) -- return true; -- } -- return false; --} -- --static u32 dom_rr_next(s32 cpu) --{ -- struct pcpu_ctx *pcpuc; -- u32 dom_id; -- -- pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]); -- if (!pcpuc) -- return 0; -- -- dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms; -- -- if (dom_id == cpu_to_dom_id(cpu)) -- dom_id = (dom_id + 1) % nr_doms; -- -- pcpuc->dom_rr_cur = dom_id; -- return dom_id; --} -- --void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 dom = cpu_to_dom_id(cpu); -- -- if (scx_bpf_consume(dom)) { -- stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1); -- return; -- } -- -- if (!greedy_threshold) -- return; -- -- bpf_repeat(nr_doms - 1) { -- u32 dom_id = dom_rr_next(cpu); -- -- if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold && -- scx_bpf_consume(dom_id)) { -- stat_add(ATROPOS_STAT_GREEDY, 1); -- break; -- } -- } --} -- --void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_at = bpf_ktime_get_ns(); --} -- --void BPF_STRUCT_OPS(atropos_running, struct task_struct *p) --{ -- struct task_ctx *taskc; -- struct dom_ctx *domc; -- pid_t pid = p->pid; -- u32 dom_id; -- -- if (fifo_sched) -- return; -- -- taskc = bpf_map_lookup_elem(&task_data, &pid); -- if (!taskc) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- dom_id = taskc->dom_id; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(domc->vtime_now, p->scx.dsq_vtime)) -- domc->vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at; -- task_ctx->runnable_at = 0; --} -- --void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->weight = weight; --} -- --struct pick_task_domain_loop_ctx { -- struct task_struct *p; -- const struct cpumask *cpumask; -- u64 dom_mask; -- u32 dom_rr_base; -- u32 dom_id; --}; -- --static int pick_task_domain_loopfn(u32 idx, void *data) --{ -- struct pick_task_domain_loop_ctx *lctx = data; -- u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms; -- -- if (dom_id >= MAX_DOMS) -- return 1; -- -- if (cpumask_intersects_domain(lctx->cpumask, dom_id)) { -- lctx->dom_mask |= 1LLU << dom_id; -- if (lctx->dom_id == MAX_DOMS) -- lctx->dom_id = dom_id; -- } -- return 0; --} -- --static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- struct pick_task_domain_loop_ctx lctx = { -- .p = p, -- .cpumask = cpumask, -- .dom_id = MAX_DOMS, -- }; -- s32 cpu = bpf_get_smp_processor_id(); -- -- if (cpu < 0 || cpu >= MAX_CPUS) -- return MAX_DOMS; -- -- lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur); -- -- bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0); -- task_ctx->dom_mask = lctx.dom_mask; -- -- return lctx.dom_id; --} -- --static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- u32 dom_id = 0; -- -- if (nr_doms > 1) -- dom_id = pick_task_domain(task_ctx, p, cpumask); -- -- if (!task_set_dsq(task_ctx, p, dom_id)) -- scx_bpf_error("Failed to set domain %d for %s[%d]", -- dom_id, p->comm, p->pid); --} -- --void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_set_domain(task_ctx, p, cpumask); --} -- --s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct bpf_cpumask *cpumask; -- struct task_ctx task_ctx, *map_value; -- long ret; -- pid_t pid; -- -- memset(&task_ctx, 0, sizeof(task_ctx)); -- -- pid = p->pid; -- ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return ret; -- } -- -- /* -- * Read the entry from the map immediately so we can add the cpumask -- * with bpf_kptr_xchg(). -- */ -- map_value = bpf_map_lookup_elem(&task_data, &pid); -- if (!map_value) -- /* Should never happen -- it was just inserted above. */ -- return -EINVAL; -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- bpf_map_delete_elem(&task_data, &pid); -- return -ENOMEM; -- } -- -- cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask); -- if (cpumask) { -- /* Should never happen as we just inserted it above. */ -- bpf_cpumask_release(cpumask); -- bpf_map_delete_elem(&task_data, &pid); -- return -EINVAL; -- } -- -- task_set_domain(map_value, p, p->cpus_ptr); -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p) --{ -- pid_t pid = p->pid; -- long ret = bpf_map_delete_elem(&task_data, &pid); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return; -- } --} -- --static int create_dom_dsq(u32 idx, void *data) --{ -- struct dom_ctx domc_init = {}, *domc; -- struct bpf_cpumask *cpumask; -- u32 cpu, dom_id = idx; -- s32 ret; -- -- ret = scx_bpf_create_dsq(dom_id, -1); -- if (ret < 0) { -- scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret); -- return 1; -- } -- -- ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0); -- if (ret) { -- scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret); -- return 1; -- } -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- /* Should never happen, we just inserted it above. */ -- scx_bpf_error("No dom%u", dom_id); -- return 1; -- } -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id); -- return 1; -- } -- -- for (cpu = 0; cpu < MAX_CPUS; cpu++) { -- const volatile __u64 *dmask; -- -- dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]); -- if (!dmask) { -- scx_bpf_error("array index error"); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- if (*dmask & (1LLU << (cpu % 64))) -- bpf_cpumask_set_cpu(cpu, cpumask); -- } -- -- cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask); -- if (cpumask) { -- scx_bpf_error("Domain %u was already present", dom_id); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(atropos_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- bpf_loop(nr_doms, create_dom_dsq, NULL, 0); -- -- for (u32 i = 0; i < nr_cpus; i++) -- pcpu_ctx[i].dom_rr_cur = i; -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg); -- exit_type = ei->type; --} -- --SEC(".struct_ops") --struct sched_ext_ops atropos = { -- .select_cpu = (void *)atropos_select_cpu, -- .enqueue = (void *)atropos_enqueue, -- .dispatch = (void *)atropos_dispatch, -- .runnable = (void *)atropos_runnable, -- .running = (void *)atropos_running, -- .stopping = (void *)atropos_stopping, -- .quiescent = (void *)atropos_quiescent, -- .set_weight = (void *)atropos_set_weight, -- .set_cpumask = (void *)atropos_set_cpumask, -- .prep_enable = (void *)atropos_prep_enable, -- .disable = (void *)atropos_disable, -- .init = (void *)atropos_init, -- .exit = (void *)atropos_exit, -- .flags = 0, -- .name = "atropos", --}; -diff --git b/tools/sched_ext/atropos/src/bpf/atropos.h a/tools/sched_ext/atropos/src/bpf/atropos.h -deleted file mode 100644 -index addf29ca104a..000000000000 ---- b/tools/sched_ext/atropos/src/bpf/atropos.h -+++ /dev/null -@@ -1,44 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#ifndef __ATROPOS_H --#define __ATROPOS_H -- --#include --#ifndef __kptr --#ifdef __KERNEL__ --#error "__kptr_ref not defined in the kernel" --#endif --#define __kptr --#endif -- --#define MAX_CPUS 512 --#define MAX_DOMS 64 /* limited to avoid complex bitmask ops */ --#define CACHELINE_SIZE 64 -- --/* Statistics */ --enum stat_idx { -- ATROPOS_STAT_TASK_GET_ERR, -- ATROPOS_STAT_WAKE_SYNC, -- ATROPOS_STAT_PREV_IDLE, -- ATROPOS_STAT_PINNED, -- ATROPOS_STAT_DIRECT_DISPATCH, -- ATROPOS_STAT_DSQ_DISPATCH, -- ATROPOS_STAT_GREEDY, -- ATROPOS_STAT_LOAD_BALANCE, -- ATROPOS_STAT_LAST_TASK, -- ATROPOS_NR_STATS, --}; -- --struct task_ctx { -- unsigned long long dom_mask; /* the domains this task can run on */ -- struct bpf_cpumask __kptr *cpumask; -- unsigned int dom_id; -- unsigned int weight; -- unsigned long long runnable_at; -- unsigned long long runnable_for; -- bool dispatch_local; --}; -- --#endif /* __ATROPOS_H */ -diff --git b/tools/sched_ext/atropos/src/main.rs a/tools/sched_ext/atropos/src/main.rs -deleted file mode 100644 -index 0d313662f713..000000000000 ---- b/tools/sched_ext/atropos/src/main.rs -+++ /dev/null -@@ -1,942 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#[path = "bpf/.output/atropos.skel.rs"] --mod atropos; --pub use atropos::*; --pub mod atropos_sys; -- --use std::cell::Cell; --use std::collections::{BTreeMap, BTreeSet}; --use std::ffi::CStr; --use std::ops::Bound::{Included, Unbounded}; --use std::sync::atomic::{AtomicBool, Ordering}; --use std::sync::Arc; --use std::time::{Duration, SystemTime}; -- --use ::fb_procfs as procfs; --use anyhow::{anyhow, bail, Context, Result}; --use bitvec::prelude::*; --use clap::Parser; --use log::{info, trace, warn}; --use ordered_float::OrderedFloat; -- --/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF --/// part does simple round robin in each domain and the userspace part --/// calculates the load factor of each domain and tells the BPF part how to load --/// balance the domains. -- --/// This scheduler demonstrates dividing scheduling logic between BPF and --/// userspace and using rust to build the userspace part. An earlier variant of --/// this scheduler was used to balance across six domains, each representing a --/// chiplet in a six-chiplet AMD processor, and could match the performance of --/// production setup using CFS. --#[derive(Debug, Parser)] --struct Opts { -- /// Scheduling slice duration in microseconds. -- #[clap(short, long, default_value = "20000")] -- slice_us: u64, -- -- /// Monitoring and load balance interval in seconds. -- #[clap(short, long, default_value = "2.0")] -- interval: f64, -- -- /// Build domains according to how CPUs are grouped at this cache level -- /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id. -- #[clap(short = 'c', long, default_value = "3")] -- cache_level: u32, -- -- /// Instead of using cache locality, set the cpumask for each domain -- /// manually, provide multiple --cpumasks, one for each domain. E.g. -- /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with -- /// the corresponding CPUs belonging to each domain. Each CPU must -- /// belong to precisely one domain. -- #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")] -- cpumasks: Vec, -- -- /// When non-zero, enable greedy task stealing. When a domain is idle, a -- /// cpu will attempt to steal tasks from a domain with at least -- /// greedy_threshold tasks enqueued. These tasks aren't permanently -- /// stolen from the domain. -- #[clap(short, long, default_value = "4")] -- greedy_threshold: u32, -- -- /// The load decay factor. Every interval, the existing load is decayed -- /// by this factor and new load is added. Must be in the range [0.0, -- /// 0.99]. The smaller the value, the more sensitive load calculation -- /// is to recent changes. When 0.0, history is ignored and the load -- /// value from the latest period is used directly. -- #[clap(short, long, default_value = "0.5")] -- load_decay_factor: f64, -- -- /// Disable load balancing. Unless disabled, periodically userspace will -- /// calculate the load factor of each domain and instruct BPF which -- /// processes to move. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- no_load_balance: bool, -- -- /// Put per-cpu kthreads directly into local dsq's. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- kthreads_local: bool, -- -- /// Use FIFO scheduling instead of weighted vtime scheduling. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- fifo_sched: bool, -- -- /// If specified, only tasks which have their scheduling policy set to -- /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all -- /// tasks are switched. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- partial: bool, -- -- /// Enable verbose output including libbpf details. Specify multiple -- /// times to increase verbosity. -- #[clap(short, long, action = clap::ArgAction::Count)] -- verbose: u8, --} -- --fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result { -- Ok(reader -- .read_stat() -- .context("Failed to read procfs")? -- .total_cpu -- .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?) --} -- --fn now_monotonic() -> u64 { -- let mut time = libc::timespec { -- tv_sec: 0, -- tv_nsec: 0, -- }; -- let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) }; -- assert!(ret == 0); -- time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 --} -- --fn clear_map(map: &mut libbpf_rs::Map) { -- // XXX: libbpf_rs has some design flaw that make it impossible to -- // delete while iterating despite it being safe so we alias it here -- let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) }; -- for key in map.keys() { -- let _ = deleter.delete(&key); -- } --} -- --#[derive(Debug)] --struct TaskLoad { -- runnable_for: u64, -- load: f64, --} -- --#[derive(Debug)] --struct TaskInfo { -- pid: i32, -- dom_mask: u64, -- migrated: Cell, --} -- --struct LoadBalancer<'a, 'b, 'c> { -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- -- tasks_by_load: Vec, TaskInfo>>, -- load_avg: f64, -- dom_loads: Vec, -- -- imbal: Vec, -- doms_to_push: BTreeMap, u32>, -- doms_to_pull: BTreeMap, u32>, -- -- nr_lb_data_errors: &'c mut u64, --} -- --impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> { -- const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10; -- const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1; -- const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50; -- -- fn new( -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- nr_lb_data_errors: &'c mut u64, -- ) -> Self { -- Self { -- maps, -- task_loads, -- nr_doms, -- load_decay_factor, -- -- tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(), -- load_avg: 0f64, -- dom_loads: vec![0.0; nr_doms], -- -- imbal: vec![0.0; nr_doms], -- doms_to_pull: BTreeMap::new(), -- doms_to_push: BTreeMap::new(), -- -- nr_lb_data_errors, -- } -- } -- -- fn read_task_loads(&mut self, period: Duration) -> Result<()> { -- let now_mono = now_monotonic(); -- let task_data = self.maps.task_data(); -- let mut this_task_loads = BTreeMap::::new(); -- let mut load_sum = 0.0f64; -- self.dom_loads = vec![0f64; self.nr_doms]; -- -- for key in task_data.keys() { -- if let Some(task_ctx_vec) = task_data -- .lookup(&key, libbpf_rs::MapFlags::ANY) -- .context("Failed to lookup task_data")? -- { -- let task_ctx = -- unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) }; -- let pid = i32::from_ne_bytes( -- key.as_slice() -- .try_into() -- .context("Invalid key length in task_data map")?, -- ); -- -- let (this_at, this_for, weight) = unsafe { -- ( -- std::ptr::read_volatile(&task_ctx.runnable_at as *const u64), -- std::ptr::read_volatile(&task_ctx.runnable_for as *const u64), -- std::ptr::read_volatile(&task_ctx.weight as *const u32), -- ) -- }; -- -- let (mut delta, prev_load) = match self.task_loads.get(&pid) { -- Some(prev) => (this_for - prev.runnable_for, Some(prev.load)), -- None => (this_for, None), -- }; -- -- // Non-zero this_at indicates that the task is currently -- // runnable. Note that we read runnable_at and runnable_for -- // without any synchronization and there is a small window -- // where we end up misaccounting. While this can cause -- // temporary error, it's unlikely to cause any noticeable -- // misbehavior especially given the load value clamping. -- if this_at > 0 && this_at < now_mono { -- delta += now_mono - this_at; -- } -- -- delta = delta.min(period.as_nanos() as u64); -- let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64) -- .clamp(0.0, weight as f64); -- -- let this_load = match prev_load { -- Some(prev_load) => { -- prev_load * self.load_decay_factor -- + this_load * (1.0 - self.load_decay_factor) -- } -- None => this_load, -- }; -- -- this_task_loads.insert( -- pid, -- TaskLoad { -- runnable_for: this_for, -- load: this_load, -- }, -- ); -- -- load_sum += this_load; -- self.dom_loads[task_ctx.dom_id as usize] += this_load; -- // Only record pids that are eligible for load balancing -- if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) { -- continue; -- } -- self.tasks_by_load[task_ctx.dom_id as usize].insert( -- OrderedFloat(this_load), -- TaskInfo { -- pid, -- dom_mask: task_ctx.dom_mask, -- migrated: Cell::new(false), -- }, -- ); -- } -- } -- -- self.load_avg = load_sum / self.nr_doms as f64; -- *self.task_loads = this_task_loads; -- Ok(()) -- } -- -- // To balance dom loads we identify doms with lower and higher load than average -- fn calculate_dom_load_balance(&mut self) -> Result<()> { -- for (dom, dom_load) in self.dom_loads.iter().enumerate() { -- let imbal = dom_load - self.load_avg; -- if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO { -- if imbal > 0f64 { -- self.doms_to_push.insert(OrderedFloat(imbal), dom as u32); -- } else { -- self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32); -- } -- self.imbal[dom] = imbal; -- } -- } -- Ok(()) -- } -- -- // Find the first candidate pid which hasn't already been migrated and -- // can run in @pull_dom. -- fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)> -- where -- I: IntoIterator, &'d TaskInfo)>, -- { -- match tasks_by_load -- .into_iter() -- .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0) -- .next() -- { -- Some((OrderedFloat(load), task)) => Some((*load, task)), -- None => None, -- } -- } -- -- fn pick_victim( -- &self, -- (push_dom, to_push): (u32, f64), -- (pull_dom, to_pull): (u32, f64), -- ) -> Option<(&TaskInfo, f64)> { -- let to_xfer = to_pull.min(to_push); -- -- trace!( -- "considering dom {}@{:.2} -> {}@{:.2}", -- push_dom, -- to_push, -- pull_dom, -- to_pull -- ); -- -- let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs(); -- -- trace!( -- "to_xfer={:.2} tasks_by_load={:?}", -- to_xfer, -- &self.tasks_by_load[push_dom as usize] -- ); -- -- // We want to pick a task to transfer from push_dom to pull_dom to -- // maximize the reduction of load imbalance between the two. IOW, -- // pick a task which has the closest load value to $to_xfer that can -- // be migrated. Find such task by locating the first migratable task -- // while scanning left from $to_xfer and the counterpart while -- // scanning right and picking the better of the two. -- let (load, task, new_imbal) = match ( -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Unbounded, Included(&OrderedFloat(to_xfer)))) -- .rev(), -- pull_dom, -- ), -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Included(&OrderedFloat(to_xfer)), Unbounded)), -- pull_dom, -- ), -- ) { -- (None, None) => return None, -- (Some((load, task)), None) | (None, Some((load, task))) => { -- (load, task, calc_new_imbal(load)) -- } -- (Some((load0, task0)), Some((load1, task1))) => { -- let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1)); -- if new_imbal0 <= new_imbal1 { -- (load0, task0, new_imbal0) -- } else { -- (load1, task1, new_imbal1) -- } -- } -- }; -- -- // If the best candidate can't reduce the imbalance, there's nothing -- // to do for this pair. -- let old_imbal = to_push + to_pull; -- if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal { -- trace!( -- "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal -- ); -- return None; -- } -- -- trace!( -- "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal, -- ); -- -- Some((task, load)) -- } -- -- // Actually execute the load balancing. Concretely this writes pid -> dom -- // entries into the lb_data map for bpf side to consume. -- fn load_balance(&mut self) -> Result<()> { -- clear_map(self.maps.lb_data()); -- -- trace!("imbal={:?}", &self.imbal); -- trace!("doms_to_push={:?}", &self.doms_to_push); -- trace!("doms_to_pull={:?}", &self.doms_to_pull); -- -- // Push from the most imbalanced to least. -- while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() { -- let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO; -- let mut pushed = 0f64; -- -- // Transfer tasks from push_dom to reduce imbalance. -- loop { -- let last_pushed = pushed; -- -- // Pull from the most imbalaned to least. -- let mut doms_to_pull = BTreeMap::<_, _>::new(); -- std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull); -- let mut pull_doms = doms_to_pull.into_iter().rev().collect::>(); -- -- for (to_pull, pull_dom) in pull_doms.iter_mut() { -- if let Some((task, load)) = -- self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull))) -- { -- // Execute migration. -- task.migrated.set(true); -- to_push -= load; -- *to_pull -= load; -- pushed += load; -- -- // Ask BPF code to execute the migration. -- let pid = task.pid; -- let cpid = (pid as libc::pid_t).to_ne_bytes(); -- if let Err(e) = self.maps.lb_data().update( -- &cpid, -- &pull_dom.to_ne_bytes(), -- libbpf_rs::MapFlags::NO_EXIST, -- ) { -- warn!( -- "Failed to update lb_data map for pid={} error={:?}", -- pid, &e -- ); -- *self.nr_lb_data_errors += 1; -- } -- -- // Always break after a successful migration so that -- // the pulling domains are always considered in the -- // descending imbalance order. -- break; -- } -- } -- -- pull_doms -- .into_iter() -- .map(|(k, v)| self.doms_to_pull.insert(k, v)) -- .count(); -- -- // Stop repeating if nothing got transferred or pushed enough. -- if pushed == last_pushed || pushed >= push_max { -- break; -- } -- } -- } -- Ok(()) -- } --} -- --struct Scheduler<'a> { -- skel: AtroposSkel<'a>, -- struct_ops: Option, -- -- nr_cpus: usize, -- nr_doms: usize, -- load_decay_factor: f64, -- balance_load: bool, -- -- proc_reader: procfs::ProcReader, -- -- prev_at: SystemTime, -- prev_total_cpu: procfs::CpuStat, -- task_loads: BTreeMap, -- -- nr_lb_data_errors: u64, --} -- --impl<'a> Scheduler<'a> { -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn parse_cpusets( -- cpumasks: &[String], -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- if cpumasks.len() > atropos_sys::MAX_DOMS as usize { -- bail!( -- "Number of requested DSQs ({}) is greater than MAX_DOMS ({})", -- cpumasks.len(), -- atropos_sys::MAX_DOMS -- ); -- } -- let mut cpus = vec![-1i32; nr_cpus]; -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()]; -- for (dq, cpumask) in cpumasks.iter().enumerate() { -- let hex_str = { -- let mut tmp_str = cpumask -- .strip_prefix("0x") -- .unwrap_or(cpumask) -- .replace('_', ""); -- if tmp_str.len() % 2 != 0 { -- tmp_str = "0".to_string() + &tmp_str; -- } -- tmp_str -- }; -- let byte_vec = hex::decode(&hex_str) -- .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?; -- -- for (index, &val) in byte_vec.iter().rev().enumerate() { -- let mut v = val; -- while v != 0 { -- let lsb = v.trailing_zeros() as usize; -- v &= !(1 << lsb); -- let cpu = index * 8 + lsb; -- if cpu > nr_cpus { -- bail!( -- concat!( -- "Found cpu ({}) in cpumask ({}) which is larger", -- " than the number of cpus on the machine ({})" -- ), -- cpu, -- cpumask, -- nr_cpus -- ); -- } -- if cpus[cpu] != -1 { -- bail!( -- "Found cpu ({}) with dq ({}) but also in cpumask ({})", -- cpu, -- cpus[cpu], -- cpumask -- ); -- } -- cpus[cpu] = dq as i32; -- cpusets[dq].set(cpu, true); -- } -- } -- cpusets[dq].set_uninitialized(false); -- } -- -- for (cpu, &dq) in cpus.iter().enumerate() { -- if dq < 0 { -- bail!( -- "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.", -- cpu -- ); -- } -- } -- -- Ok((cpusets, cpus)) -- } -- -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn cpusets_from_cache( -- level: u32, -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- let mut cpu_to_cache = vec![]; // (cpu_id, cache_id) -- let mut cache_ids = BTreeSet::::new(); -- let mut nr_not_found = 0; -- -- // Build cpu -> cache ID mapping. -- for cpu in 0..nr_cpus { -- let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level); -- let id = match std::fs::read_to_string(&path) { -- Ok(val) => val -- .trim() -- .parse::() -- .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?, -- Err(e) if e.kind() == std::io::ErrorKind::NotFound => { -- nr_not_found += 1; -- 0 -- } -- Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)), -- }; -- -- cpu_to_cache.push(id); -- cache_ids.insert(id); -- } -- -- if nr_not_found > 1 { -- warn!( -- "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0", -- level, nr_not_found, nr_cpus -- ); -- } -- -- // Cache IDs may have holes. Assign consecutive domain IDs to -- // existing cache IDs. -- let mut cache_to_dom = BTreeMap::::new(); -- let mut nr_doms = 0; -- for cache_id in cache_ids.iter() { -- cache_to_dom.insert(*cache_id, nr_doms); -- nr_doms += 1; -- } -- -- if nr_doms > atropos_sys::MAX_DOMS { -- bail!( -- "Total number of doms {} is greater than MAX_DOMS ({})", -- nr_doms, -- atropos_sys::MAX_DOMS -- ); -- } -- -- // Build and return dom -> cpumask and cpu -> dom mappings. -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize]; -- let mut cpu_to_dom = vec![]; -- -- for cpu in 0..nr_cpus { -- let dom_id = cache_to_dom[&cpu_to_cache[cpu]]; -- cpusets[dom_id as usize].set(cpu, true); -- cpu_to_dom.push(dom_id as i32); -- } -- -- Ok((cpusets, cpu_to_dom)) -- } -- -- fn init(opts: &Opts) -> Result { -- // Open the BPF prog first for verification. -- let mut skel_builder = AtroposSkelBuilder::default(); -- skel_builder.obj_builder.debug(opts.verbose > 0); -- let mut skel = skel_builder.open().context("Failed to open BPF program")?; -- -- let nr_cpus = libbpf_rs::num_possible_cpus().unwrap(); -- if nr_cpus > atropos_sys::MAX_CPUS as usize { -- bail!( -- "nr_cpus ({}) is greater than MAX_CPUS ({})", -- nr_cpus, -- atropos_sys::MAX_CPUS -- ); -- } -- -- // Initialize skel according to @opts. -- let (cpusets, cpus) = if opts.cpumasks.len() > 0 { -- Self::parse_cpusets(&opts.cpumasks, nr_cpus)? -- } else { -- Self::cpusets_from_cache(opts.cache_level, nr_cpus)? -- }; -- let nr_doms = cpusets.len(); -- skel.rodata().nr_doms = nr_doms as u32; -- skel.rodata().nr_cpus = nr_cpus as u32; -- -- for (cpu, dom) in cpus.iter().enumerate() { -- skel.rodata().cpu_dom_id_map[cpu] = *dom as u32; -- } -- -- for (dom, cpuset) in cpusets.iter().enumerate() { -- let raw_cpuset_slice = cpuset.as_raw_slice(); -- let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom]; -- let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len()); -- left.clone_from_slice(cpuset.as_raw_slice()); -- let cpumask_str = dom_cpumask_slice -- .iter() -- .take((nr_cpus + 63) / 64) -- .rev() -- .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x)); -- info!( -- "DOM[{:02}] cpumask{} ({} cpus)", -- dom, -- &cpumask_str, -- cpuset.count_ones() -- ); -- } -- -- skel.rodata().slice_us = opts.slice_us; -- skel.rodata().kthreads_local = opts.kthreads_local; -- skel.rodata().fifo_sched = opts.fifo_sched; -- skel.rodata().switch_partial = opts.partial; -- skel.rodata().greedy_threshold = opts.greedy_threshold; -- -- // Attach. -- let mut skel = skel.load().context("Failed to load BPF program")?; -- skel.attach().context("Failed to attach BPF program")?; -- let struct_ops = Some( -- skel.maps_mut() -- .atropos() -- .attach_struct_ops() -- .context("Failed to attach atropos struct ops")?, -- ); -- info!("Atropos Scheduler Attached"); -- -- // Other stuff. -- let mut proc_reader = procfs::ProcReader::new(); -- let prev_total_cpu = read_total_cpu(&mut proc_reader)?; -- -- Ok(Self { -- skel, -- struct_ops, // should be held to keep it attached -- -- nr_cpus, -- nr_doms, -- load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99), -- balance_load: !opts.no_load_balance, -- -- proc_reader, -- -- prev_at: SystemTime::now(), -- prev_total_cpu, -- task_loads: BTreeMap::new(), -- -- nr_lb_data_errors: 0, -- }) -- } -- -- fn get_cpu_busy(&mut self) -> Result { -- let total_cpu = read_total_cpu(&mut self.proc_reader)?; -- let busy = match (&self.prev_total_cpu, &total_cpu) { -- ( -- procfs::CpuStat { -- user_usec: Some(prev_user), -- nice_usec: Some(prev_nice), -- system_usec: Some(prev_system), -- idle_usec: Some(prev_idle), -- iowait_usec: Some(prev_iowait), -- irq_usec: Some(prev_irq), -- softirq_usec: Some(prev_softirq), -- stolen_usec: Some(prev_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- procfs::CpuStat { -- user_usec: Some(curr_user), -- nice_usec: Some(curr_nice), -- system_usec: Some(curr_system), -- idle_usec: Some(curr_idle), -- iowait_usec: Some(curr_iowait), -- irq_usec: Some(curr_irq), -- softirq_usec: Some(curr_softirq), -- stolen_usec: Some(curr_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- ) => { -- let idle_usec = curr_idle - prev_idle; -- let iowait_usec = curr_iowait - prev_iowait; -- let user_usec = curr_user - prev_user; -- let system_usec = curr_system - prev_system; -- let nice_usec = curr_nice - prev_nice; -- let irq_usec = curr_irq - prev_irq; -- let softirq_usec = curr_softirq - prev_softirq; -- let stolen_usec = curr_stolen - prev_stolen; -- -- let busy_usec = -- user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec; -- let total_usec = idle_usec + busy_usec + iowait_usec; -- busy_usec as f64 / total_usec as f64 -- } -- _ => { -- bail!("Some procfs stats are not populated!"); -- } -- }; -- -- self.prev_total_cpu = total_cpu; -- Ok(busy) -- } -- -- fn read_bpf_stats(&mut self) -> Result> { -- let mut maps = self.skel.maps_mut(); -- let stats_map = maps.stats(); -- let mut stats: Vec = Vec::new(); -- let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus]; -- -- for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS { -- let cpu_stat_vec = stats_map -- .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY) -- .with_context(|| format!("Failed to lookup stat {}", stat))? -- .expect("per-cpu stat should exist"); -- let sum = cpu_stat_vec -- .iter() -- .map(|val| { -- u64::from_ne_bytes( -- val.as_slice() -- .try_into() -- .expect("Invalid value length in stat map"), -- ) -- }) -- .sum(); -- stats_map -- .update_percpu( -- &(stat as u32).to_ne_bytes(), -- &zero_vec, -- libbpf_rs::MapFlags::ANY, -- ) -- .context("Failed to zero stat")?; -- stats.push(sum); -- } -- Ok(stats) -- } -- -- fn report( -- &self, -- stats: &Vec, -- cpu_busy: f64, -- processing_dur: Duration, -- load_avg: f64, -- dom_loads: &Vec, -- imbal: &Vec, -- ) { -- let stat = |idx| stats[idx as usize]; -- let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK); -- -- info!( -- "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms", -- cpu_busy * 100.0, -- load_avg, -- stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize], -- stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize], -- self.nr_lb_data_errors, -- processing_dur.as_millis(), -- ); -- -- let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0; -- -- info!( -- "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}", -- total, -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY), -- ); -- -- for i in 0..self.nr_doms { -- info!( -- "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}", -- i, -- dom_loads[i], -- if imbal[i] < 0.0 { -imbal[i] } else { 0.0 }, -- if imbal[i] > 0.0 { imbal[i] } else { 0.0 }, -- ); -- } -- } -- -- fn step(&mut self) -> Result<()> { -- let started_at = std::time::SystemTime::now(); -- let bpf_stats = self.read_bpf_stats()?; -- let cpu_busy = self.get_cpu_busy()?; -- -- let mut lb = LoadBalancer::new( -- self.skel.maps_mut(), -- &mut self.task_loads, -- self.nr_doms, -- self.load_decay_factor, -- &mut self.nr_lb_data_errors, -- ); -- -- lb.read_task_loads(started_at.duration_since(self.prev_at)?)?; -- lb.calculate_dom_load_balance()?; -- -- if self.balance_load { -- lb.load_balance()?; -- } -- -- // Extract fields needed for reporting and drop lb to release -- // mutable borrows. -- let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal); -- -- self.report( -- &bpf_stats, -- cpu_busy, -- std::time::SystemTime::now().duration_since(started_at)?, -- load_avg, -- &dom_loads, -- &imbal, -- ); -- -- self.prev_at = started_at; -- Ok(()) -- } -- -- fn read_bpf_exit_type(&mut self) -> i32 { -- unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) } -- } -- -- fn report_bpf_exit_type(&mut self) -> Result<()> { -- // Report msg if EXT_OPS_EXIT_ERROR. -- match self.read_bpf_exit_type() { -- 0 => Ok(()), -- etype if etype == 2 => { -- let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) }; -- let msg = cstr -- .to_str() -- .context("Failed to convert exit msg to string") -- .unwrap(); -- bail!("BPF exit_type={} msg={}", etype, msg); -- } -- etype => { -- info!("BPF exit_type={}", etype); -- Ok(()) -- } -- } -- } --} -- --impl<'a> Drop for Scheduler<'a> { -- fn drop(&mut self) { -- if let Some(struct_ops) = self.struct_ops.take() { -- drop(struct_ops); -- } -- } --} -- --fn main() -> Result<()> { -- let opts = Opts::parse(); -- -- let llv = match opts.verbose { -- 0 => simplelog::LevelFilter::Info, -- 1 => simplelog::LevelFilter::Debug, -- _ => simplelog::LevelFilter::Trace, -- }; -- let mut lcfg = simplelog::ConfigBuilder::new(); -- lcfg.set_time_level(simplelog::LevelFilter::Error) -- .set_location_level(simplelog::LevelFilter::Off) -- .set_target_level(simplelog::LevelFilter::Off) -- .set_thread_level(simplelog::LevelFilter::Off); -- simplelog::TermLogger::init( -- llv, -- lcfg.build(), -- simplelog::TerminalMode::Stderr, -- simplelog::ColorChoice::Auto, -- )?; -- -- let shutdown = Arc::new(AtomicBool::new(false)); -- let shutdown_clone = shutdown.clone(); -- ctrlc::set_handler(move || { -- shutdown_clone.store(true, Ordering::Relaxed); -- }) -- .context("Error setting Ctrl-C handler")?; -- -- let mut sched = Scheduler::init(&opts)?; -- -- while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 { -- std::thread::sleep(Duration::from_secs_f64(opts.interval)); -- sched.step()?; -- } -- -- sched.report_bpf_exit_type() --} -diff --git b/tools/sched_ext/gnu/stubs.h a/tools/sched_ext/gnu/stubs.h -deleted file mode 100644 -index 719225b16626..000000000000 ---- b/tools/sched_ext/gnu/stubs.h -+++ /dev/null -@@ -1 +0,0 @@ --/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ -diff --git b/tools/sched_ext/scx_common.bpf.h a/tools/sched_ext/scx_common.bpf.h -deleted file mode 100644 -index e56de9dc86f2..000000000000 ---- b/tools/sched_ext/scx_common.bpf.h -+++ /dev/null -@@ -1,288 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __SCHED_EXT_COMMON_BPF_H --#define __SCHED_EXT_COMMON_BPF_H -- --#include "vmlinux.h" --#include --#include --#include --#include "user_exit_info.h" -- --#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ --#define PF_EXITING 0x00000004 --#define CLOCK_MONOTONIC 1 -- --/* -- * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can -- * lead to really confusing misbehaviors. Let's trigger a build failure. -- */ --static inline void ___vmlinux_h_sanity_check___(void) --{ -- _Static_assert(SCX_DSQ_FLAG_BUILTIN, -- "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); --} -- --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; -- --static inline __attribute__((format(printf, 1, 2))) --void ___scx_bpf_error_format_checker(const char *fmt, ...) {} -- --/* -- * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments -- * instead of an array of u64. Note that __param[] must have at least one -- * element to keep the verifier happy. -- */ --#define scx_bpf_error(fmt, args...) \ --({ \ -- static char ___fmt[] = fmt; \ -- unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ -- \ -- _Pragma("GCC diagnostic push") \ -- _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ -- ___bpf_fill(___param, args); \ -- _Pragma("GCC diagnostic pop") \ -- \ -- scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ -- \ -- ___scx_bpf_error_format_checker(fmt, ##args); \ --}) -- --void scx_bpf_switch_all(void) __ksym; --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; --bool scx_bpf_consume(u64 dsq_id) __ksym; --u32 scx_bpf_dispatch_nr_slots(void) __ksym; --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym; --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym; --void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; --s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym; --const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; --const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; --void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; --void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; --bool scx_bpf_task_running(const struct task_struct *p) __ksym; --s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym; --u32 scx_bpf_reenqueue_local(void) __ksym; -- --#define BPF_STRUCT_OPS(name, args...) \ --SEC("struct_ops/"#name) \ --BPF_PROG(name, ##args) -- --#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ --SEC("struct_ops.s/"#name) \ --BPF_PROG(name, ##args) -- --/** -- * MEMBER_VPTR - Obtain the verified pointer to a struct or array member -- * @base: struct or array to index -- * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...) -- * -- * The verifier often gets confused by the instruction sequence the compiler -- * generates for indexing struct fields or arrays. This macro forces the -- * compiler to generate a code sequence which first calculates the byte offset, -- * checks it against the struct or array size and add that byte offset to -- * generate the pointer to the member to help the verifier. -- * -- * Ideally, we want to abort if the calculated offset is out-of-bounds. However, -- * BPF currently doesn't support abort, so evaluate to NULL instead. The caller -- * must check for NULL and take appropriate action to appease the verifier. To -- * avoid confusing the verifier, it's best to check for NULL and dereference -- * immediately. -- * -- * vptr = MEMBER_VPTR(my_array, [i][j]); -- * if (!vptr) -- * return error; -- * *vptr = new_value; -- */ --#define MEMBER_VPTR(base, member) (typeof(base member) *)({ \ -- u64 __base = (u64)base; \ -- u64 __addr = (u64)&(base member) - __base; \ -- asm volatile ( \ -- "if %0 <= %[max] goto +2\n" \ -- "%0 = 0\n" \ -- "goto +1\n" \ -- "%0 += %1\n" \ -- : "+r"(__addr) \ -- : "r"(__base), \ -- [max]"i"(sizeof(base) - sizeof(base member))); \ -- __addr; \ --}) -- --/* -- * BPF core and other generic helpers -- */ -- --/* list and rbtree */ --#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) --#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) -- --void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; --void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; -- --#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) --#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) -- --void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; --struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; --struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, -- struct bpf_rb_node *node) __ksym; --void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, -- bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym; --struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; -- --/* task */ --struct task_struct *bpf_task_from_pid(s32 pid) __ksym; --struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; --void bpf_task_release(struct task_struct *p) __ksym; -- --/* cgroup */ --struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; --void bpf_cgroup_release(struct cgroup *cgrp) __ksym; --struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; -- --/* cpumask */ --struct bpf_cpumask *bpf_cpumask_create(void) __ksym; --struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; --u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; --u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym; -- --/* rcu */ --void bpf_rcu_read_lock(void) __ksym; --void bpf_rcu_read_unlock(void) __ksym; -- --/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */ --struct bpf_iter_num; -- --extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym; --extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym; --extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym; -- --#ifndef bpf_for_each --/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for -- * using BPF open-coded iterators without having to write mundane explicit -- * low-level loop logic. Instead, it provides for()-like generic construct -- * that can be used pretty naturally. E.g., for some hypothetical cgroup -- * iterator, you'd write: -- * -- * struct cgroup *cg, *parent_cg = <...>; -- * -- * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { -- * bpf_printk("Child cgroup id = %d", cg->cgroup_id); -- * if (cg->cgroup_id == 123) -- * break; -- * } -- * -- * I.e., it looks almost like high-level for each loop in other languages, -- * supports continue/break, and is verifiable by BPF verifier. -- * -- * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) -- * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to -- * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int -- * *`, not just `int`. So for integers bpf_for() is more convenient. -- * -- * Note: this macro relies on C99 feature of allowing to declare variables -- * inside for() loop, bound to for() loop lifetime. It also utilizes GCC -- * extension: __attribute__((cleanup())), supported by both GCC and -- * Clang. -- */ --#define bpf_for_each(type, cur, args...) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ -- cleanup(bpf_iter_##type##_destroy))), \ -- /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_##type##_new(&___it, ##args), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_##type##_destroy, (void *)0); \ -- /* iteration and termination check */ \ -- (((cur) = bpf_iter_##type##_next(&___it))); \ --) --#endif /* bpf_for_each */ -- --#ifndef bpf_for --/* bpf_for(i, start, end) implements a for()-like looping construct that sets -- * provided integer variable *i* to values starting from *start* through, -- * but not including, *end*. It also proves to BPF verifier that *i* belongs -- * to range [start, end), so this can be used for accessing arrays without -- * extra checks. -- * -- * Note: *start* and *end* are assumed to be expressions with no side effects -- * and whose values do not change throughout bpf_for() loop execution. They do -- * not have to be statically known or constant, though. -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_for(i, start, end) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, (start), (end)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- ({ \ -- /* iteration step */ \ -- int *___t = bpf_iter_num_next(&___it); \ -- /* termination and bounds check */ \ -- (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ -- }); \ --) --#endif /* bpf_for */ -- --#ifndef bpf_repeat --/* bpf_repeat(N) performs N iterations without exposing iteration number -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_repeat(N) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, 0, (N)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- bpf_iter_num_next(&___it); \ -- /* nothing here */ \ --) --#endif /* bpf_repeat */ -- --#endif /* __SCHED_EXT_COMMON_BPF_H */ -diff --git b/tools/sched_ext/scx_example_central.bpf.c a/tools/sched_ext/scx_example_central.bpf.c -deleted file mode 100644 -index 4cec04b4c2ed..000000000000 ---- b/tools/sched_ext/scx_example_central.bpf.c -+++ /dev/null -@@ -1,334 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A central FIFO sched_ext scheduler which demonstrates the followings: -- * -- * a. Making all scheduling decisions from one CPU: -- * -- * The central CPU is the only one making scheduling decisions. All other -- * CPUs kick the central CPU when they run out of tasks to run. -- * -- * There is one global BPF queue and the central CPU schedules all CPUs by -- * dispatching from the global queue to each CPU's local dsq from dispatch(). -- * This isn't the most straightforward. e.g. It'd be easier to bounce -- * through per-CPU BPF queues. The current design is chosen to maximally -- * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching. -- * -- * b. Tickless operation -- * -- * All tasks are dispatched with the infinite slice which allows stopping the -- * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full -- * parameter. The tickless operation can be observed through -- * /proc/interrupts. -- * -- * Periodic switching is enforced by a periodic timer checking all CPUs and -- * preempting them as necessary. Unfortunately, BPF timer currently doesn't -- * have a way to pin to a specific CPU, so the periodic timer isn't pinned to -- * the central CPU. -- * -- * c. Preemption -- * -- * Kthreads are unconditionally queued to the head of a matching local dsq -- * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always -- * prioritized over user threads, which is required for ensuring forward -- * progress as e.g. the periodic timer may run on a ksoftirqd and if the -- * ksoftirqd gets starved by a user thread, there may not be anything else to -- * vacate that user thread. -- * -- * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the -- * next tasks. -- * -- * This scheduler is designed to maximize usage of various SCX mechanisms. A -- * more practical implementation would likely put the scheduling loop outside -- * the central CPU's dispatch() path and add some form of priority mechanism. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --enum { -- FALLBACK_DSQ_ID = 0, -- MAX_CPUS = 4096, -- MS_TO_NS = 1000LLU * 1000, -- TIMER_INTERVAL_NS = 1 * MS_TO_NS, --}; -- --const volatile bool switch_partial; --const volatile s32 central_cpu; --const volatile u32 nr_cpu_ids = 64; /* !0 for veristat, set during init */ -- --u64 nr_total, nr_locals, nr_queued, nr_lost_pids; --u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries; --u64 nr_overflows; -- --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, s32); --} central_q SEC(".maps"); -- --/* can't use percpu map due to bad lookups */ --static bool cpu_gimme_task[MAX_CPUS]; --static u64 cpu_started_at[MAX_CPUS]; -- --struct central_timer { -- struct bpf_timer timer; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, 1); -- __type(key, u32); -- __type(value, struct central_timer); --} central_timer SEC(".maps"); -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- /* -- * Steer wakeups to the central CPU as much as possible to avoid -- * disturbing other CPUs. It's safe to blindly return the central cpu as -- * select_cpu() is a hint and if @p can't be on it, the kernel will -- * automatically pick a fallback CPU. -- */ -- return central_cpu; --} -- --void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags) --{ -- s32 pid = p->pid; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- /* -- * Push per-cpu kthreads at the head of local dsq's and preempt the -- * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked -- * behind other threads which is necessary for forward progress -- * guarantee as we depend on the BPF timer which may run from ksoftirqd. -- */ -- if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- __sync_fetch_and_add(&nr_locals, 1); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF, -- enq_flags | SCX_ENQ_PREEMPT); -- return; -- } -- -- if (bpf_map_push_elem(¢ral_q, &pid, 0)) { -- __sync_fetch_and_add(&nr_overflows, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_queued, 1); -- -- if (!scx_bpf_task_running(p)) -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); --} -- --static bool dispatch_to_cpu(s32 cpu) --{ -- struct task_struct *p; -- s32 pid; -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (bpf_map_pop_elem(¢ral_q, &pid)) -- break; -- -- __sync_fetch_and_sub(&nr_queued, 1); -- -- p = bpf_task_from_pid(pid); -- if (!p) { -- __sync_fetch_and_add(&nr_lost_pids, 1); -- continue; -- } -- -- /* -- * If we can't run the task at the top, do the dumb thing and -- * bounce it to the fallback dsq. -- */ -- if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- __sync_fetch_and_add(&nr_mismatches, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0); -- bpf_task_release(p); -- continue; -- } -- -- /* dispatch to local and mark that @cpu doesn't need more */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0); -- -- if (cpu != central_cpu) -- scx_bpf_kick_cpu(cpu, 0); -- -- bpf_task_release(p); -- return true; -- } -- -- return false; --} -- --void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (cpu == central_cpu) { -- /* dispatch for all other CPUs first */ -- __sync_fetch_and_add(&nr_dispatches, 1); -- -- bpf_for(cpu, 0, nr_cpu_ids) { -- bool *gimme; -- -- if (!scx_bpf_dispatch_nr_slots()) -- break; -- -- /* central's gimme is never set */ -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme && !*gimme) -- continue; -- -- if (dispatch_to_cpu(cpu)) -- *gimme = false; -- } -- -- /* -- * Retry if we ran out of dispatch buffer slots as we might have -- * skipped some CPUs and also need to dispatch for self. The ext -- * core automatically retries if the local dsq is empty but we -- * can't rely on that as we're dispatching for other CPUs too. -- * Kick self explicitly to retry. -- */ -- if (!scx_bpf_dispatch_nr_slots()) { -- __sync_fetch_and_add(&nr_retries, 1); -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- return; -- } -- -- /* look for a task to run on the central CPU */ -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- dispatch_to_cpu(central_cpu); -- } else { -- bool *gimme; -- -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme) -- *gimme = true; -- -- /* -- * Force dispatch on the scheduling CPU so that it finds a task -- * to run for us. -- */ -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- } --} -- --void BPF_STRUCT_OPS(central_running, struct task_struct *p) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ --} -- --void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = 0; --} -- --static int central_timerfn(void *map, int *key, struct bpf_timer *timer) --{ -- u64 now = bpf_ktime_get_ns(); -- u64 nr_to_kick = nr_queued; -- s32 i; -- -- bpf_for(i, 0, nr_cpu_ids) { -- s32 cpu = (nr_timers + i) % nr_cpu_ids; -- u64 *started_at; -- -- if (cpu == central_cpu) -- continue; -- -- /* kick iff the current one exhausted its slice */ -- started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at && *started_at && -- vtime_before(now, *started_at + SCX_SLICE_DFL)) -- continue; -- -- /* and there's something pending */ -- if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) || -- scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu)) -- ; -- else if (nr_to_kick) -- nr_to_kick--; -- else -- continue; -- -- scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); -- } -- -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- -- bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- __sync_fetch_and_add(&nr_timers, 1); -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(central_init) --{ -- u32 key = 0; -- struct bpf_timer *timer; -- int ret; -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1); -- if (ret) -- return ret; -- -- timer = bpf_map_lookup_elem(¢ral_timer, &key); -- if (!timer) -- return -ESRCH; -- -- bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); -- bpf_timer_set_callback(timer, central_timerfn); -- ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- return ret; --} -- --void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops central_ops = { -- /* -- * We are offloading all scheduling decisions to the central CPU and -- * thus being the last task on a given CPU doesn't mean anything -- * special. Enqueue the last tasks like any other tasks. -- */ -- .flags = SCX_OPS_ENQ_LAST, -- -- .select_cpu = (void *)central_select_cpu, -- .enqueue = (void *)central_enqueue, -- .dispatch = (void *)central_dispatch, -- .running = (void *)central_running, -- .stopping = (void *)central_stopping, -- .init = (void *)central_init, -- .exit = (void *)central_exit, -- .name = "central", --}; -diff --git b/tools/sched_ext/scx_example_central.c a/tools/sched_ext/scx_example_central.c -deleted file mode 100644 -index 7ad591cbdc65..000000000000 ---- b/tools/sched_ext/scx_example_central.c -+++ /dev/null -@@ -1,94 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_central.skel.h" -- --const char help_fmt[] = --"A central FIFO sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-c CPU] [-p]\n" --"\n" --" -c CPU Override the central CPU (default: 0)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_central *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_central__open(); -- assert(skel); -- -- skel->rodata->central_cpu = 0; -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "c:ph")) != -1) { -- switch (opt) { -- case 'c': -- skel->rodata->central_cpu = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_central__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.central_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf("total :%10lu local:%10lu queued:%10lu lost:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_locals, -- skel->bss->nr_queued, -- skel->bss->nr_lost_pids); -- printf("timer :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n", -- skel->bss->nr_timers, -- skel->bss->nr_dispatches, -- skel->bss->nr_mismatches, -- skel->bss->nr_retries); -- printf("overflow:%10lu\n", -- skel->bss->nr_overflows); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_central__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_flatcg.bpf.c a/tools/sched_ext/scx_example_flatcg.bpf.c -deleted file mode 100644 -index f6078b9a681f..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.bpf.c -+++ /dev/null -@@ -1,872 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext flattened cgroup hierarchy scheduler. It implements -- * hierarchical weight-based cgroup CPU control by flattening the cgroup -- * hierarchy into a single layer by compounding the active weight share at each -- * level. Consider the following hierarchy with weights in parentheses: -- * -- * R + A (100) + B (100) -- * | \ C (100) -- * \ D (200) -- * -- * Ignoring the root and threaded cgroups, only B, C and D can contain tasks. -- * Let's say all three have runnable tasks. The total share that each of these -- * three cgroups is entitled to can be calculated by compounding its share at -- * each level. -- * -- * For example, B is competing against C and in that competition its share is -- * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's -- * share in that competition is 200/(200+100) == 1/3. B's eventual share in the -- * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's -- * eventual shaer is the same at 1/6. D is only competing at the top level and -- * its share is 200/(100+200) == 2/3. -- * -- * So, instead of hierarchically scheduling level-by-level, we can consider it -- * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3 -- * and keep updating the eventual shares as the cgroups' runnable states change. -- * -- * This flattening of hierarchy can bring a substantial performance gain when -- * the cgroup hierarchy is nested multiple levels. in a simple benchmark using -- * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it -- * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two -- * apache instances competing with 2:1 weight ratio nested four level deep. -- * -- * However, the gain comes at the cost of not being able to properly handle -- * thundering herd of cgroups. For example, if many cgroups which are nested -- * behind a low priority parent cgroup wake up around the same time, they may be -- * able to consume more CPU cycles than they are entitled to. In many use cases, -- * this isn't a real concern especially given the performance gain. Also, there -- * are ways to mitigate the problem further by e.g. introducing an extra -- * scheduling layer on cgroup delegation boundaries. -- * -- * The scheduler first picks the cgroup to run and then schedule the tasks -- * within by using nested weighted vtime scheduling by default. The -- * cgroup-internal scheduling can be switched to FIFO with the -f option. -- */ --#include "scx_common.bpf.h" --#include "user_exit_info.h" --#include "scx_example_flatcg.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */ --const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL; --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --u64 cvtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, u64); -- __uint(max_entries, FCG_NR_STATS); --} stats SEC(".maps"); -- --static void stat_inc(enum fcg_stat_idx idx) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p)++; --} -- --struct fcg_cpu_ctx { -- u64 cur_cgid; -- u64 cur_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, struct fcg_cpu_ctx); -- __uint(max_entries, 1); --} cpu_ctx SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_cgrp_ctx); --} cgrp_ctx SEC(".maps"); -- --struct cgv_node { -- struct bpf_rb_node rb_node; -- __u64 cvtime; -- __u64 cgid; --}; -- --private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock; --private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node); -- --struct cgv_node_stash { -- struct cgv_node __kptr *node; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, 16384); -- __type(key, __u64); -- __type(value, struct cgv_node_stash); --} cgv_node_stash SEC(".maps"); -- --struct fcg_task_ctx { -- u64 bypassed_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_task_ctx); --} task_ctx SEC(".maps"); -- --/* gets inc'd on weight tree changes to expire the cached hweights */ --unsigned long hweight_gen = 1; -- --static u64 div_round_up(u64 dividend, u64 divisor) --{ -- return (dividend + divisor - 1) / divisor; --} -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) --{ -- struct cgv_node *cgc_a, *cgc_b; -- -- cgc_a = container_of(a, struct cgv_node, rb_node); -- cgc_b = container_of(b, struct cgv_node, rb_node); -- -- return cgc_a->cvtime < cgc_b->cvtime; --} -- --static struct fcg_cpu_ctx *find_cpu_ctx(void) --{ -- struct fcg_cpu_ctx *cpuc; -- u32 idx = 0; -- -- cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx); -- if (!cpuc) { -- scx_bpf_error("cpu_ctx lookup failed"); -- return NULL; -- } -- return cpuc; --} -- --static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id); -- return NULL; -- } -- return cgc; --} -- --static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgrp = bpf_cgroup_ancestor(cgrp, level); -- if (!cgrp) { -- scx_bpf_error("ancestor cgroup lookup failed"); -- return NULL; -- } -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- scx_bpf_error("ancestor cgrp_ctx lookup failed"); -- bpf_cgroup_release(cgrp); -- return cgc; --} -- --static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- int level; -- -- if (!cgc->nr_active) { -- stat_inc(FCG_STAT_HWT_SKIP); -- return; -- } -- -- if (cgc->hweight_gen == hweight_gen) { -- stat_inc(FCG_STAT_HWT_CACHE); -- return; -- } -- -- stat_inc(FCG_STAT_HWT_UPDATES); -- bpf_for(level, 0, cgrp->level + 1) { -- struct fcg_cgrp_ctx *cgc; -- bool is_active; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- -- if (!level) { -- cgc->hweight = FCG_HWEIGHT_ONE; -- cgc->hweight_gen = hweight_gen; -- } else { -- struct fcg_cgrp_ctx *pcgc; -- -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- -- /* -- * We can be oppotunistic here and not grab the -- * cgv_tree_lock and deal with the occasional races. -- * However, hweight updates are already cached and -- * relatively low-frequency. Let's just do the -- * straightforward thing. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- is_active = cgc->nr_active; -- if (is_active) { -- cgc->hweight_gen = pcgc->hweight_gen; -- cgc->hweight = -- div_round_up(pcgc->hweight * cgc->weight, -- pcgc->child_weight_sum); -- } -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!is_active) { -- stat_inc(FCG_STAT_HWT_RACE); -- break; -- } -- } -- } --} -- --static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) --{ -- u64 delta, cvtime, max_budget; -- -- /* -- * A node which is on the rbtree can't be pointed to from elsewhere yet -- * and thus can't be updated and repositioned. Instead, we collect the -- * vtime deltas separately and apply it asynchronously here. -- */ -- delta = cgc->cvtime_delta; -- __sync_fetch_and_sub(&cgc->cvtime_delta, delta); -- cvtime = cgv_node->cvtime + delta; -- -- /* -- * Allow a cgroup to carry the maximum budget proportional to its -- * hweight such that a full-hweight cgroup can immediately take up half -- * of the CPUs at the most while staying at the front of the rbtree. -- */ -- max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / -- (2 * FCG_HWEIGHT_ONE); -- if (vtime_before(cvtime, cvtime_now - max_budget)) -- cvtime = cvtime_now - max_budget; -- -- cgv_node->cvtime = cvtime; --} -- --static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- u64 cgid = cgrp->kn->id; -- -- /* paired with cmpxchg in try_pick_next_cgroup() */ -- if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { -- stat_inc(FCG_STAT_ENQ_SKIP); -- return; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid); -- return; -- } -- -- /* NULL if the node is already on the rbtree */ -- cgv_node = bpf_kptr_xchg(&stash->node, NULL); -- if (!cgv_node) { -- stat_inc(FCG_STAT_ENQ_RACE); -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); --} -- --void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * If select_cpu_dfl() is recommending local enqueue, the target CPU is -- * idle. Follow it and charge the cgroup later in fcg_stopping() after -- * the fact. Use the same mechanism to deal with tasks with custom -- * affinities so that we don't have to worry about per-cgroup dq's -- * containing tasks that can't be executed from some CPUs. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) { -- /* -- * Tell fcg_stopping() that this bypassed the regular scheduling -- * path and should be force charged to the cgroup. 0 is used to -- * indicate that the task isn't bypassing, so if the current -- * runtime is 0, go back by one nanosecond. -- */ -- taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1; -- -- /* -- * The global dq is deprioritized as we don't want to let tasks -- * to boost themselves by constraining its cpumask. The -- * deprioritization is rather severe, so let's not apply that to -- * per-cpu kernel threads. This is ham-fisted. We probably wanna -- * implement per-cgroup fallback dq's instead so that we have -- * more control over when tasks with custom cpumask get issued. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || -- (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) { -- stat_inc(FCG_STAT_LOCAL); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- } else { -- stat_inc(FCG_STAT_GLOBAL); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } -- return; -- } -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- goto out_release; -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 tvtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) -- tvtime = cgc->tvtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, -- tvtime, enq_flags); -- } -- -- cgrp_enqueued(cgrp, cgc); --out_release: -- bpf_cgroup_release(cgrp); --} -- --/* -- * Walk the cgroup tree to update the active weight sums as tasks wake up and -- * sleep. The weight sums are used as the base when calculating the proportion a -- * given cgroup or task is entitled to at each level. -- */ --static void update_active_weight_sums(struct cgroup *cgrp, bool runnable) --{ -- struct fcg_cgrp_ctx *cgc; -- bool updated = false; -- int idx; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- /* -- * In most cases, a hot cgroup would have multiple threads going to -- * sleep and waking up while the whole cgroup stays active. In leaf -- * cgroups, ->nr_runnable which is updated with __sync operations gates -- * ->nr_active updates, so that we don't have to grab the cgv_tree_lock -- * repeatedly for a busy cgroup which is staying active. -- */ -- if (runnable) { -- if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_ACT); -- } else { -- if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_DEACT); -- } -- -- /* -- * If @cgrp is becoming runnable, its hweight should be refreshed after -- * it's added to the weight tree so that enqueue has the up-to-date -- * value. If @cgrp is becoming quiescent, the hweight should be -- * refreshed before it's removed from the weight tree so that the usage -- * charging which happens afterwards has access to the latest value. -- */ -- if (!runnable) -- cgrp_refresh_hweight(cgrp, cgc); -- -- /* propagate upwards */ -- bpf_for(idx, 0, cgrp->level) { -- int level = cgrp->level - idx; -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- bool propagate = false; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- if (level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- } -- -- /* -- * We need the propagation protected by a lock to synchronize -- * against weight changes. There's no reason to drop the lock at -- * each level but bpf_spin_lock() doesn't want any function -- * calls while locked. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- -- if (runnable) { -- if (!cgc->nr_active++) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum += cgc->weight; -- } -- } -- } else { -- if (!--cgc->nr_active) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum -= cgc->weight; -- } -- } -- } -- -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!propagate) -- break; -- } -- -- if (updated) -- __sync_fetch_and_add(&hweight_gen, 1); -- -- if (runnable) -- cgrp_refresh_hweight(cgrp, cgc); --} -- --void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, true); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) --{ -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- if (fifo_sched) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- /* -- * @cgc->tvtime_now always progresses forward as tasks start -- * executing. The test and update can be performed concurrently -- * from multiple CPUs and thus racy. Any error should be -- * contained and temporary. Let's just live with it. -- */ -- if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) -- cgc->tvtime_now = p->scx.dsq_vtime; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- /* scale the execution time by the inverse of the weight and charge */ -- if (!fifo_sched) -- p->scx.dsq_vtime += -- (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- if (!taskc->bypassed_at) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- __sync_fetch_and_add(&cgc->cvtime_delta, -- p->se.sum_exec_runtime - taskc->bypassed_at); -- taskc->bypassed_at = 0; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, false); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) --{ -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- if (cgrp->level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1); -- if (!pcgc) -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- if (pcgc && cgc->nr_active) -- pcgc->child_weight_sum += (s64)weight - cgc->weight; -- cgc->weight = weight; -- bpf_spin_unlock(&cgv_tree_lock); --} -- --static bool try_pick_next_cgroup(u64 *cgidp) --{ -- struct bpf_rb_node *rb_node; -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 cgid; -- -- /* pop the front cgroup and wind cvtime_now accordingly */ -- bpf_spin_lock(&cgv_tree_lock); -- -- rb_node = bpf_rbtree_first(&cgv_tree); -- if (!rb_node) { -- bpf_spin_unlock(&cgv_tree_lock); -- stat_inc(FCG_STAT_PNC_NO_CGRP); -- *cgidp = 0; -- return true; -- } -- -- rb_node = bpf_rbtree_remove(&cgv_tree, rb_node); -- bpf_spin_unlock(&cgv_tree_lock); -- -- cgv_node = container_of(rb_node, struct cgv_node, rb_node); -- cgid = cgv_node->cgid; -- -- if (vtime_before(cvtime_now, cgv_node->cvtime)) -- cvtime_now = cgv_node->cvtime; -- -- /* -- * If lookup fails, the cgroup's gone. Free and move on. See -- * fcg_cgroup_exit(). -- */ -- cgrp = bpf_cgroup_from_id(cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- if (!scx_bpf_consume(cgid)) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_EMPTY); -- goto out_stash; -- } -- -- /* -- * Successfully consumed from the cgroup. This will be our current -- * cgroup for the new slice. Refresh its hweight. -- */ -- cgrp_refresh_hweight(cgrp, cgc); -- -- bpf_cgroup_release(cgrp); -- -- /* -- * As the cgroup may have more tasks, add it back to the rbtree. Note -- * that here we charge the full slice upfront and then exact later -- * according to the actual consumption. This prevents lowpri thundering -- * herd from saturating the machine. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- -- *cgidp = cgid; -- stat_inc(FCG_STAT_PNC_NEXT); -- return true; -- --out_stash: -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- /* -- * Paired with cmpxchg in cgrp_enqueued(). If they see the following -- * transition, they'll enqueue the cgroup. If they are earlier, we'll -- * see their task in the dq below and requeue the cgroup. -- */ -- __sync_val_compare_and_swap(&cgc->queued, 1, 0); -- -- if (scx_bpf_dsq_nr_queued(cgid)) { -- bpf_spin_lock(&cgv_tree_lock); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- goto out_free; -- } -- } -- -- return false; -- --out_free: -- bpf_obj_drop(cgv_node); -- return false; --} -- --void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) --{ -- struct fcg_cpu_ctx *cpuc; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 now = bpf_ktime_get_ns(); -- -- cpuc = find_cpu_ctx(); -- if (!cpuc) -- return; -- -- if (!cpuc->cur_cgid) -- goto pick_next_cgroup; -- -- if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { -- if (scx_bpf_consume(cpuc->cur_cgid)) { -- stat_inc(FCG_STAT_CNS_KEEP); -- return; -- } -- stat_inc(FCG_STAT_CNS_EMPTY); -- } else { -- stat_inc(FCG_STAT_CNS_EXPIRE); -- } -- -- /* -- * The current cgroup is expiring. It was already charged a full slice. -- * Calculate the actual usage and accumulate the delta. -- */ -- cgrp = bpf_cgroup_from_id(cpuc->cur_cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_CNS_GONE); -- goto pick_next_cgroup; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (cgc) { -- /* -- * We want to update the vtime delta and then look for the next -- * cgroup to execute but the latter needs to be done in a loop -- * and we can't keep the lock held. Oh well... -- */ -- bpf_spin_lock(&cgv_tree_lock); -- __sync_fetch_and_add(&cgc->cvtime_delta, -- (cpuc->cur_at + cgrp_slice_ns - now) * -- FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- stat_inc(FCG_STAT_CNS_GONE); -- } -- -- bpf_cgroup_release(cgrp); -- --pick_next_cgroup: -- cpuc->cur_at = now; -- -- if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { -- cpuc->cur_cgid = 0; -- return; -- } -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_pick_next_cgroup(&cpuc->cur_cgid)) -- break; -- } --} -- --s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct fcg_task_ctx *taskc; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- taskc = bpf_task_storage_get(&task_ctx, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!taskc) -- return -ENOMEM; -- -- taskc->bypassed_at = 0; -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp, -- struct scx_cgroup_init_args *args) --{ -- struct fcg_cgrp_ctx *cgc; -- struct cgv_node *cgv_node; -- struct cgv_node_stash empty_stash = {}, *stash; -- u64 cgid = cgrp->kn->id; -- int ret; -- -- /* -- * Technically incorrect as cgroup ID is full 64bit while dq ID is -- * 63bit. Should not be a problem in practice and easy to spot in the -- * unlikely case that it breaks. -- */ -- ret = scx_bpf_create_dsq(cgid, -1); -- if (ret) -- return ret; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!cgc) { -- ret = -ENOMEM; -- goto err_destroy_dsq; -- } -- -- cgc->weight = args->weight; -- cgc->hweight = FCG_HWEIGHT_ONE; -- -- ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash, -- BPF_NOEXIST); -- if (ret) { -- if (ret != -ENOMEM) -- scx_bpf_error("unexpected stash creation error (%d)", -- ret); -- goto err_destroy_dsq; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("unexpected cgv_node stash lookup failure"); -- ret = -ENOENT; -- goto err_destroy_dsq; -- } -- -- cgv_node = bpf_obj_new(struct cgv_node); -- if (!cgv_node) { -- ret = -ENOMEM; -- goto err_del_cgv_node; -- } -- -- cgv_node->cgid = cgid; -- cgv_node->cvtime = cvtime_now; -- -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- ret = -EBUSY; -- goto err_drop; -- } -- -- return 0; -- --err_drop: -- bpf_obj_drop(cgv_node); --err_del_cgv_node: -- bpf_map_delete_elem(&cgv_node_stash, &cgid); --err_destroy_dsq: -- scx_bpf_destroy_dsq(cgid); -- return ret; --} -- --void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- -- /* -- * For now, there's no way find and remove the cgv_node if it's on the -- * cgv_tree. Let's drain them in the dispatch path as they get popped -- * off the front of the tree. -- */ -- bpf_map_delete_elem(&cgv_node_stash, &cgid); -- scx_bpf_destroy_dsq(cgid); --} -- --s32 BPF_STRUCT_OPS(fcg_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops flatcg_ops = { -- .enqueue = (void *)fcg_enqueue, -- .dispatch = (void *)fcg_dispatch, -- .runnable = (void *)fcg_runnable, -- .running = (void *)fcg_running, -- .stopping = (void *)fcg_stopping, -- .quiescent = (void *)fcg_quiescent, -- .prep_enable = (void *)fcg_prep_enable, -- .cgroup_set_weight = (void *)fcg_cgroup_set_weight, -- .cgroup_init = (void *)fcg_cgroup_init, -- .cgroup_exit = (void *)fcg_cgroup_exit, -- .init = (void *)fcg_init, -- .exit = (void *)fcg_exit, -- .flags = SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING, -- .name = "flatcg", --}; -diff --git b/tools/sched_ext/scx_example_flatcg.c a/tools/sched_ext/scx_example_flatcg.c -deleted file mode 100644 -index f9c8a5b84a70..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.c -+++ /dev/null -@@ -1,232 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2023 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2023 Tejun Heo -- * Copyright (c) 2023 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_flatcg.h" --#include "scx_example_flatcg.skel.h" -- --#ifndef FILEID_KERNFS --#define FILEID_KERNFS 0xfe --#endif -- --const char help_fmt[] = --"A flattened cgroup hierarchy sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -i INTERVAL Report interval\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --static float read_cpu_util(__u64 *last_sum, __u64 *last_idle) --{ -- FILE *fp; -- char buf[4096]; -- char *line, *cur = NULL, *tok; -- __u64 sum = 0, idle = 0; -- __u64 delta_sum, delta_idle; -- int idx; -- -- fp = fopen("/proc/stat", "r"); -- if (!fp) { -- perror("fopen(\"/proc/stat\")"); -- return 0.0; -- } -- -- if (!fgets(buf, sizeof(buf), fp)) { -- perror("fgets(\"/proc/stat\")"); -- fclose(fp); -- return 0.0; -- } -- fclose(fp); -- -- line = buf; -- for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) { -- char *endp = NULL; -- __u64 v; -- -- if (idx == 0) { -- line = NULL; -- continue; -- } -- v = strtoull(tok, &endp, 0); -- if (!endp || *endp != '\0') { -- fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n", -- idx, tok); -- continue; -- } -- sum += v; -- if (idx == 4) -- idle = v; -- } -- -- delta_sum = sum - *last_sum; -- delta_idle = idle - *last_idle; -- *last_sum = sum; -- *last_idle = idle; -- -- return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0; --} -- --static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats) --{ -- __u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus]; -- __u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); -- -- for (idx = 0; idx < FCG_NR_STATS; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_flatcg *skel; -- struct bpf_link *link; -- struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 }; -- bool dump_cgrps = false; -- __u64 last_cpu_sum = 0, last_cpu_idle = 0; -- __u64 last_stats[FCG_NR_STATS] = {}; -- unsigned long seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_flatcg__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open: %s\n", strerror(errno)); -- return 1; -- } -- -- skel->rodata->nr_cpus = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) { -- double v; -- -- switch (opt) { -- case 's': -- v = strtod(optarg, NULL); -- skel->rodata->cgrp_slice_ns = v * 1000; -- break; -- case 'i': -- v = strtod(optarg, NULL); -- intv_ts.tv_sec = v; -- intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000; -- break; -- case 'd': -- dump_cgrps = true; -- break; -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- case 'h': -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d", -- (double)skel->rodata->cgrp_slice_ns / 1000000.0, -- (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0, -- dump_cgrps); -- -- if (scx_example_flatcg__load(skel)) { -- fprintf(stderr, "Failed to load: %s\n", strerror(errno)); -- return 1; -- } -- -- link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops); -- if (!link) { -- fprintf(stderr, "Failed to attach_struct_ops: %s\n", -- strerror(errno)); -- return 1; -- } -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- __u64 acc_stats[FCG_NR_STATS]; -- __u64 stats[FCG_NR_STATS]; -- float cpu_util; -- int i; -- -- cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle); -- -- fcg_read_stats(skel, acc_stats); -- for (i = 0; i < FCG_NR_STATS; i++) -- stats[i] = acc_stats[i] - last_stats[i]; -- -- memcpy(last_stats, acc_stats, sizeof(acc_stats)); -- -- printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n", -- seq++, cpu_util * 100.0, skel->data->hweight_gen); -- printf(" act:%6llu deact:%6llu local:%6llu global:%6llu\n", -- stats[FCG_STAT_ACT], -- stats[FCG_STAT_DEACT], -- stats[FCG_STAT_LOCAL], -- stats[FCG_STAT_GLOBAL]); -- printf("HWT skip:%6llu race:%6llu cache:%6llu update:%6llu\n", -- stats[FCG_STAT_HWT_SKIP], -- stats[FCG_STAT_HWT_RACE], -- stats[FCG_STAT_HWT_CACHE], -- stats[FCG_STAT_HWT_UPDATES]); -- printf("ENQ skip:%6llu race:%6llu\n", -- stats[FCG_STAT_ENQ_SKIP], -- stats[FCG_STAT_ENQ_RACE]); -- printf("CNS keep:%6llu expire:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_CNS_KEEP], -- stats[FCG_STAT_CNS_EXPIRE], -- stats[FCG_STAT_CNS_EMPTY], -- stats[FCG_STAT_CNS_GONE]); -- printf("PNC nocgrp:%6llu next:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_PNC_NO_CGRP], -- stats[FCG_STAT_PNC_NEXT], -- stats[FCG_STAT_PNC_EMPTY], -- stats[FCG_STAT_PNC_GONE]); -- printf("BAD remove:%6llu\n", -- acc_stats[FCG_STAT_BAD_REMOVAL]); -- -- nanosleep(&intv_ts, NULL); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_flatcg__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_flatcg.h a/tools/sched_ext/scx_example_flatcg.h -deleted file mode 100644 -index 490758ed41f0..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.h -+++ /dev/null -@@ -1,49 +0,0 @@ --#ifndef __SCX_EXAMPLE_FLATCG_H --#define __SCX_EXAMPLE_FLATCG_H -- --enum { -- FCG_HWEIGHT_ONE = 1LLU << 16, --}; -- --enum fcg_stat_idx { -- FCG_STAT_ACT, -- FCG_STAT_DEACT, -- FCG_STAT_LOCAL, -- FCG_STAT_GLOBAL, -- -- FCG_STAT_HWT_UPDATES, -- FCG_STAT_HWT_CACHE, -- FCG_STAT_HWT_SKIP, -- FCG_STAT_HWT_RACE, -- -- FCG_STAT_ENQ_SKIP, -- FCG_STAT_ENQ_RACE, -- -- FCG_STAT_CNS_KEEP, -- FCG_STAT_CNS_EXPIRE, -- FCG_STAT_CNS_EMPTY, -- FCG_STAT_CNS_GONE, -- -- FCG_STAT_PNC_NO_CGRP, -- FCG_STAT_PNC_NEXT, -- FCG_STAT_PNC_EMPTY, -- FCG_STAT_PNC_GONE, -- -- FCG_STAT_BAD_REMOVAL, -- -- FCG_NR_STATS, --}; -- --struct fcg_cgrp_ctx { -- u32 nr_active; -- u32 nr_runnable; -- u32 queued; -- u32 weight; -- u32 hweight; -- u64 child_weight_sum; -- u64 hweight_gen; -- s64 cvtime_delta; -- u64 tvtime_now; --}; -- --#endif /* __SCX_EXAMPLE_FLATCG_H */ -diff --git b/tools/sched_ext/scx_example_pair.bpf.c a/tools/sched_ext/scx_example_pair.bpf.c -deleted file mode 100644 -index 279efe58b777..000000000000 ---- b/tools/sched_ext/scx_example_pair.bpf.c -+++ /dev/null -@@ -1,627 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext core-scheduler which always makes every sibling CPU pair -- * execute from the same CPU cgroup. -- * -- * This scheduler is a minimal implementation and would need some form of -- * priority handling both inside each cgroup and across the cgroups to be -- * practically useful. -- * -- * Each CPU in the system is paired with exactly one other CPU, according to a -- * "stride" value that can be specified when the BPF scheduler program is first -- * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee -- * that they will only ever schedule tasks that belong to the same CPU cgroup. -- * -- * Scheduler Initialization -- * ------------------------ -- * -- * The scheduler BPF program is first initialized from user space, before it is -- * enabled. During this initialization process, each CPU on the system is -- * assigned several values that are constant throughout its runtime: -- * -- * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling -- * decisions. Paired CPUs always schedule tasks from the same -- * CPU cgroup, and synchronize with each other to guarantee -- * that this constraint is not violated. -- * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access -- * a struct pair_ctx object that is shared between the pair. -- * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the -- * pair. Each struct pair_ctx has an active_mask field, -- * which is a bitmap used to indicate whether each core -- * in the pair currently has an actively running task. -- * This index specifies which entry in the bitmap corresponds -- * to each CPU in the pair. -- * -- * During this initialization, the CPUs are paired according to a "stride" that -- * may be specified when invoking the user space program that initializes and -- * loads the scheduler. By default, the stride is 1/2 the total number of CPUs. -- * -- * Tasks and cgroups -- * ----------------- -- * -- * Every cgroup in the system is registered with the scheduler using the -- * pair_cgroup_init() callback, and every task in the system is associated with -- * exactly one cgroup. At a high level, the idea with the pair scheduler is to -- * always schedule tasks from the same cgroup within a given CPU pair. When a -- * task is enqueued (i.e. passed to the pair_enqueue() callback function), its -- * cgroup ID is read from its task struct, and then a corresponding queue map -- * is used to FIFO-enqueue the task for that cgroup. -- * -- * If you look through the implementation of the scheduler, you'll notice that -- * there is quite a bit of complexity involved with looking up the per-cgroup -- * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash -- * BPF hash map that is used to map a cgroup ID to a globally unique ID that's -- * allocated in the BPF program. This is done because we use separate maps to -- * store the FIFO queue of tasks, and the length of that map, per cgroup. This -- * complexity is only present because of current deficiencies in BPF that will -- * soon be addressed. The main point to keep in mind is that newly enqueued -- * tasks are added to their cgroup's FIFO queue. -- * -- * Dispatching tasks -- * ----------------- -- * -- * This section will describe how enqueued tasks are dispatched and scheduled. -- * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is -- * as follows: -- * -- * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is -- * the structure that's used to synchronize amongst the two pair CPUs in their -- * scheduling decisions. After any of the following events have occurred: -- * -- * - The cgroup's slice run has expired, or -- * - The cgroup becomes empty, or -- * - Either CPU in the pair is preempted by a higher priority scheduling class -- * -- * The cgroup transitions to the draining state and stops executing new tasks -- * from the cgroup. -- * -- * 2. If the pair is still executing a task, mark the pair_ctx as draining, and -- * wait for the pair CPU to be preempted. -- * -- * 3. Otherwise, if the pair CPU is not running a task, we can move onto -- * scheduling new tasks. Pop the next cgroup id from the top_q queue. -- * -- * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it. -- * -- * Note again that this scheduling behavior is simple, but the implementation -- * is complex mostly because this it hits several BPF shortcomings and has to -- * work around in often awkward ways. Most of the shortcomings are expected to -- * be resolved in the near future which should allow greatly simplifying this -- * scheduler. -- * -- * Dealing with preemption -- * ----------------------- -- * -- * SCX is the lowest priority sched_class, and could be preempted by them at -- * any time. To address this, the scheduler implements pair_cpu_release() and -- * pair_cpu_acquire() callbacks which are invoked by the core scheduler when -- * the scheduler loses and gains control of the CPU respectively. -- * -- * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and -- * then invoke: -- * -- * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- * -- * This preempts the pair CPU, and waits until it has re-entered the scheduler -- * before returning. This is necessary to ensure that the higher priority -- * sched_class that preempted our scheduler does not schedule a task -- * concurrently with our pair CPU. -- * -- * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption -- * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable -- * pair scheduling. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include "scx_example_pair.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; -- --/* !0 for veristat, set during init */ --const volatile u32 nr_cpu_ids = 64; -- --/* a pair of CPUs stay on a cgroup for this duration */ --const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL; -- --/* cpu ID -> pair cpu ID */ --const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 }; -- --/* cpu ID -> pair_id */ --const volatile u32 pair_id[MAX_CPUS]; -- --/* CPU ID -> CPU # in the pair (0 or 1) */ --const volatile u32 in_pair_idx[MAX_CPUS]; -- --struct pair_ctx { -- struct bpf_spin_lock lock; -- -- /* the cgroup the pair is currently executing */ -- u64 cgid; -- -- /* the pair started executing the current cgroup at */ -- u64 started_at; -- -- /* whether the current cgroup is draining */ -- bool draining; -- -- /* the CPUs that are currently active on the cgroup */ -- u32 active_mask; -- -- /* -- * the CPUs that are currently preempted and running tasks in a -- * different scheduler. -- */ -- u32 preempted_mask; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, MAX_CPUS / 2); -- __type(key, u32); -- __type(value, struct pair_ctx); --} pair_ctx SEC(".maps"); -- --/* queue of cgrp_q's possibly with tasks on them */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- /* -- * Because it's difficult to build strong synchronization encompassing -- * multiple non-trivial operations in BPF, this queue is managed in an -- * opportunistic way so that we guarantee that a cgroup w/ active tasks -- * is always on it but possibly multiple times. Once we have more robust -- * synchronization constructs and e.g. linked list, we should be able to -- * do this in a prettier way but for now just size it big enough. -- */ -- __uint(max_entries, 4 * MAX_CGRPS); -- __type(value, u64); --} top_q SEC(".maps"); -- --/* per-cgroup q which FIFOs the tasks from the cgroup */ --struct cgrp_q { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, MAX_QUEUED); -- __type(value, u32); --}; -- --/* -- * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local -- * storage; however, a cgroup local storage can only be accessed from the BPF -- * progs attached to the cgroup. For now, work around by allocating array of -- * cgrp_q's and then allocating per-cgroup indices. -- * -- * Another caveat: It's difficult to populate a large array of maps statically -- * or from BPF. Initialize it from userland. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, MAX_CGRPS); -- __type(key, s32); -- __array(values, struct cgrp_q); --} cgrp_q_arr SEC(".maps"); -- --static u64 cgrp_q_len[MAX_CGRPS]; -- --/* -- * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be -- * useful to have as a map type. -- */ --static u32 cgrp_q_idx_cursor; --static u64 cgrp_q_idx_busy[MAX_CGRPS]; -- --/* -- * All added up, the following is what we do: -- * -- * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking -- * for a free ID. If not found, fail cgroup creation with -EBUSY. -- * -- * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following -- * cgrp_q_idx_hash. -- * -- * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from -- * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr. -- * -- * This is sadly complicated for something pretty simple. Hopefully, we should -- * be able to simplify in the future. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, MAX_CGRPS); -- __uint(key_size, sizeof(u64)); /* cgrp ID */ -- __uint(value_size, sizeof(s32)); /* cgrp_q idx */ --} cgrp_q_idx_hash SEC(".maps"); -- --/* statistics */ --u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions; --u64 nr_exps, nr_exp_waits, nr_exp_empty; --u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty; -- --struct user_exit_info uei; -- --static bool time_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- struct cgrp_q *cgq; -- s32 pid = p->pid; -- u64 cgid; -- u32 *q_idx; -- u64 *cgq_len; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- cgrp = scx_bpf_task_cgroup(p); -- cgid = cgrp->kn->id; -- bpf_cgroup_release(cgrp); -- -- /* find the cgroup's q and push @p into it */ -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!q_idx) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return; -- } -- -- cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx); -- if (!cgq) { -- scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]", -- cgid, *q_idx); -- return; -- } -- -- if (bpf_map_push_elem(cgq, &pid, 0)) { -- scx_bpf_error("cgroup[%llu] queue overflow", cgid); -- return; -- } -- -- /* bump q len, if going 0 -> 1, queue cgroup into the top_q */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len) { -- scx_bpf_error("MEMBER_VTPR malfunction"); -- return; -- } -- -- if (!__sync_fetch_and_add(cgq_len, 1) && -- bpf_map_push_elem(&top_q, &cgid, 0)) { -- scx_bpf_error("top_q overflow"); -- return; -- } --} -- --static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) --{ -- u32 *vptr; -- -- vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *pairc = bpf_map_lookup_elem(&pair_ctx, vptr); -- if (!(*pairc)) -- return -EINVAL; -- -- vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *mask = 1U << *vptr; -- -- return 0; --} -- --static int try_dispatch(s32 cpu) --{ -- struct pair_ctx *pairc; -- struct bpf_map *cgq_map; -- struct task_struct *p; -- u64 now = bpf_ktime_get_ns(); -- bool kick_pair = false; -- bool expired, pair_preempted; -- u32 *vptr, in_pair_mask; -- s32 pid, q_idx; -- u64 cgid; -- int ret; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) { -- scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]", -- cpu); -- return -ENOENT; -- } -- -- bpf_spin_lock(&pairc->lock); -- pairc->active_mask &= ~in_pair_mask; -- -- expired = time_before(pairc->started_at + pair_batch_dur_ns, now); -- if (expired || pairc->draining) { -- u64 new_cgid = 0; -- -- __sync_fetch_and_add(&nr_exps, 1); -- -- /* -- * We're done with the current cgid. An obvious optimization -- * would be not draining if the next cgroup is the current one. -- * For now, be dumb and always expire. -- */ -- pairc->draining = true; -- -- pair_preempted = pairc->preempted_mask; -- if (pairc->active_mask || pair_preempted) { -- /* -- * The other CPU is still active, or is no longer under -- * our control due to e.g. being preempted by a higher -- * priority sched_class. We want to wait until this -- * cgroup expires, or until control of our pair CPU has -- * been returned to us. -- * -- * If the pair controls its CPU, and the time already -- * expired, kick. When the other CPU arrives at -- * dispatch and clears its active mask, it'll push the -- * pair to the next cgroup and kick this CPU. -- */ -- __sync_fetch_and_add(&nr_exp_waits, 1); -- bpf_spin_unlock(&pairc->lock); -- if (expired && !pair_preempted) -- kick_pair = true; -- goto out_maybe_kick; -- } -- -- bpf_spin_unlock(&pairc->lock); -- -- /* -- * Pick the next cgroup. It'd be easier / cleaner to not drop -- * pairc->lock and use stronger synchronization here especially -- * given that we'll be switching cgroups significantly less -- * frequently than tasks. Unfortunately, bpf_spin_lock can't -- * really protect anything non-trivial. Let's do opportunistic -- * operations instead. -- */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u32 *q_idx; -- u64 *cgq_len; -- -- if (bpf_map_pop_elem(&top_q, &new_cgid)) { -- /* no active cgroup, go idle */ -- __sync_fetch_and_add(&nr_exp_empty, 1); -- return 0; -- } -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid); -- if (!q_idx) -- continue; -- -- /* -- * This is the only place where empty cgroups are taken -- * off the top_q. -- */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len || !*cgq_len) -- continue; -- -- /* -- * If it has any tasks, requeue as we may race and not -- * execute it. -- */ -- bpf_map_push_elem(&top_q, &new_cgid, 0); -- break; -- } -- -- bpf_spin_lock(&pairc->lock); -- -- /* -- * The other CPU may already have started on a new cgroup while -- * we dropped the lock. Make sure that we're still draining and -- * start on the new cgroup. -- */ -- if (pairc->draining && !pairc->active_mask) { -- __sync_fetch_and_add(&nr_cgrp_next, 1); -- pairc->cgid = new_cgid; -- pairc->started_at = now; -- pairc->draining = false; -- kick_pair = true; -- } else { -- __sync_fetch_and_add(&nr_cgrp_coll, 1); -- } -- } -- -- cgid = pairc->cgid; -- pairc->active_mask |= in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- -- /* again, it'd be better to do all these with the lock held, oh well */ -- vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!vptr) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return -ENOENT; -- } -- q_idx = *vptr; -- -- /* claim one task from cgrp_q w/ q_idx */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u64 *cgq_len, len; -- -- cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]); -- if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) { -- /* the cgroup must be empty, expire and repeat */ -- __sync_fetch_and_add(&nr_cgrp_empty, 1); -- bpf_spin_lock(&pairc->lock); -- pairc->draining = true; -- pairc->active_mask &= ~in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- return -EAGAIN; -- } -- -- if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len) -- continue; -- -- break; -- } -- -- cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx); -- if (!cgq_map) { -- scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- if (bpf_map_pop_elem(cgq_map, &pid)) { -- scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- p = bpf_task_from_pid(pid); -- if (p) { -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } else { -- /* we don't handle dequeues, retry on lost tasks */ -- __sync_fetch_and_add(&nr_missing, 1); -- return -EAGAIN; -- } -- --out_maybe_kick: -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } -- return 0; --} -- --void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) --{ -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_dispatch(cpu) != -EAGAIN) -- break; -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask &= ~in_pair_mask; -- /* Kick the pair CPU, unless it was also preempted. */ -- kick_pair = !pairc->preempted_mask; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask |= in_pair_mask; -- pairc->active_mask &= ~in_pair_mask; -- /* Kick the pair CPU if it's still running. */ -- kick_pair = pairc->active_mask; -- pairc->draining = true; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- } -- } -- __sync_fetch_and_add(&nr_preemptions, 1); --} -- --s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 i, q_idx; -- -- bpf_for(i, 0, MAX_CGRPS) { -- q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS; -- if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1)) -- break; -- } -- if (i == MAX_CGRPS) -- return -EBUSY; -- -- if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]); -- if (busy) -- *busy = 0; -- return -EBUSY; -- } -- -- return 0; --} -- --void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 *q_idx; -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (q_idx) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]); -- if (busy) -- *busy = 0; -- bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid); -- } --} -- --s32 BPF_STRUCT_OPS(pair_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops pair_ops = { -- .enqueue = (void *)pair_enqueue, -- .dispatch = (void *)pair_dispatch, -- .cpu_acquire = (void *)pair_cpu_acquire, -- .cpu_release = (void *)pair_cpu_release, -- .cgroup_init = (void *)pair_cgroup_init, -- .cgroup_exit = (void *)pair_cgroup_exit, -- .init = (void *)pair_init, -- .exit = (void *)pair_exit, -- .name = "pair", --}; -diff --git b/tools/sched_ext/scx_example_pair.c a/tools/sched_ext/scx_example_pair.c -deleted file mode 100644 -index 18e032bbc173..000000000000 ---- b/tools/sched_ext/scx_example_pair.c -+++ /dev/null -@@ -1,143 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_pair.h" --#include "scx_example_pair.skel.h" -- --const char help_fmt[] = --"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n" --"execute from the same CPU cgroup.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-S STRIDE] [-p]\n" --"\n" --" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_pair *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 stride, i, opt, outer_fd; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_pair__open(); -- assert(skel); -- -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- /* pair up the earlier half to the latter by default, override with -s */ -- stride = skel->rodata->nr_cpu_ids / 2; -- -- while ((opt = getopt(argc, argv, "S:ph")) != -1) { -- switch (opt) { -- case 'S': -- stride = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- for (i = 0; i < skel->rodata->nr_cpu_ids; i++) { -- if (skel->rodata->pair_cpu[i] < 0) { -- skel->rodata->pair_cpu[i] = i + stride; -- skel->rodata->pair_cpu[i + stride] = i; -- skel->rodata->pair_id[i] = i; -- skel->rodata->pair_id[i + stride] = i; -- skel->rodata->in_pair_idx[i] = 0; -- skel->rodata->in_pair_idx[i + stride] = 1; -- } -- } -- -- assert(!scx_example_pair__load(skel)); -- -- /* -- * Populate the cgrp_q_arr map which is an array containing per-cgroup -- * queues. It'd probably be better to do this from BPF but there are too -- * many to initialize statically and there's no way to dynamically -- * populate from BPF. -- */ -- outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr); -- assert(outer_fd >= 0); -- -- printf("Initializing"); -- for (i = 0; i < MAX_CGRPS; i++) { -- s32 inner_fd; -- -- if (exit_req) -- break; -- -- inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, -- sizeof(u32), MAX_QUEUED, NULL); -- assert(inner_fd >= 0); -- assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY)); -- close(inner_fd); -- -- if (!(i % 10)) -- printf("."); -- fflush(stdout); -- } -- printf("\n"); -- -- /* -- * Fully initialized, attach and run. -- */ -- link = bpf_map__attach_struct_ops(skel->maps.pair_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf(" total:%10lu dispatch:%10lu missing:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_dispatched, -- skel->bss->nr_missing); -- printf(" kicks:%10lu preemptions:%7lu\n", -- skel->bss->nr_kicks, -- skel->bss->nr_preemptions); -- printf(" exp:%10lu exp_wait:%10lu exp_empty:%10lu\n", -- skel->bss->nr_exps, -- skel->bss->nr_exp_waits, -- skel->bss->nr_exp_empty); -- printf("cgnext:%10lu cgcoll:%10lu cgempty:%10lu\n", -- skel->bss->nr_cgrp_next, -- skel->bss->nr_cgrp_coll, -- skel->bss->nr_cgrp_empty); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_pair__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_pair.h a/tools/sched_ext/scx_example_pair.h -deleted file mode 100644 -index f60b824272f7..000000000000 ---- b/tools/sched_ext/scx_example_pair.h -+++ /dev/null -@@ -1,10 +0,0 @@ --#ifndef __SCX_EXAMPLE_PAIR_H --#define __SCX_EXAMPLE_PAIR_H -- --enum { -- MAX_CPUS = 4096, -- MAX_QUEUED = 4096, -- MAX_CGRPS = 4096, --}; -- --#endif /* __SCX_EXAMPLE_PAIR_H */ -diff --git b/tools/sched_ext/scx_example_qmap.bpf.c a/tools/sched_ext/scx_example_qmap.bpf.c -deleted file mode 100644 -index 579ab21ae403..000000000000 ---- b/tools/sched_ext/scx_example_qmap.bpf.c -+++ /dev/null -@@ -1,401 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple five-level FIFO queue scheduler. -- * -- * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets -- * assigned to one depending on its compound weight. Each CPU round robins -- * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from -- * queue0, 2 from queue1, 4 from queue2 and so on. -- * -- * This scheduler demonstrates: -- * -- * - BPF-side queueing using PIDs. -- * - Sleepable per-task storage allocation using ops.prep_enable(). -- * - Using ops.cpu_release() to handle a higher priority scheduling class taking -- * the CPU away. -- * - Core-sched support. -- * -- * This scheduler is primarily for demonstration and testing of sched_ext -- * features and unlikely to be useful for actual workloads. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include -- --char _license[] SEC("license") = "GPL"; -- --const volatile u64 slice_ns = SCX_SLICE_DFL; --const volatile bool switch_partial; --const volatile u32 stall_user_nth; --const volatile u32 stall_kernel_nth; --const volatile u32 dsp_inf_loop_after; --const volatile s32 disallow_tgid; -- --u32 test_error_cnt; -- --struct user_exit_info uei; -- --struct qmap { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, u32); --} queue0 SEC(".maps"), -- queue1 SEC(".maps"), -- queue2 SEC(".maps"), -- queue3 SEC(".maps"), -- queue4 SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, 5); -- __type(key, int); -- __array(values, struct qmap); --} queue_arr SEC(".maps") = { -- .values = { -- [0] = &queue0, -- [1] = &queue1, -- [2] = &queue2, -- [3] = &queue3, -- [4] = &queue4, -- }, --}; -- --/* -- * Per-queue sequence numbers to implement core-sched ordering. -- * -- * Tail seq is assigned to each queued task and incremented. Head seq tracks the -- * sequence number of the latest dispatched task. The distance between the a -- * task's seq and the associated queue's head seq is called the queue distance -- * and used when comparing two tasks for ordering. See qmap_core_sched_before(). -- */ --static u64 core_sched_head_seqs[5]; --static u64 core_sched_tail_seqs[5]; -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local_dsq */ -- u64 core_sched_seq; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --/* Per-cpu dispatch index and remaining count */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(max_entries, 2); -- __type(key, u32); -- __type(value, u64); --} dispatch_idx_cnt SEC(".maps"); -- --/* Statistics */ --unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued; --unsigned long nr_core_sched_execed; -- --s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- struct task_ctx *tctx; -- s32 cpu; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- return cpu; -- -- return prev_cpu; --} -- --static int weight_to_idx(u32 weight) --{ -- /* Coarsely map the compound weight to a FIFO. */ -- if (weight <= 25) -- return 0; -- else if (weight <= 50) -- return 1; -- else if (weight < 200) -- return 2; -- else if (weight < 400) -- return 3; -- else -- return 4; --} -- --void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) --{ -- static u32 user_cnt, kernel_cnt; -- struct task_ctx *tctx; -- u32 pid = p->pid; -- int idx = weight_to_idx(p->scx.weight); -- void *ring; -- -- if (p->flags & PF_KTHREAD) { -- if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth)) -- return; -- } else { -- if (stall_user_nth && !(++user_cnt % stall_user_nth)) -- return; -- } -- -- if (test_error_cnt && !--test_error_cnt) -- scx_bpf_error("test triggering error"); -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * All enqueued tasks must have their core_sched_seq updated for correct -- * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in -- * qmap_ops.flags. -- */ -- tctx->core_sched_seq = core_sched_tail_seqs[idx]++; -- -- /* -- * If qmap_select_cpu() is telling us to or this is the last runnable -- * task on the CPU, enqueue locally. -- */ -- if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) { -- tctx->force_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); -- return; -- } -- -- /* -- * If the task was re-enqueued due to the CPU being preempted by a -- * higher priority scheduling class, just re-enqueue the task directly -- * on the global DSQ. As we want another CPU to pick it up, find and -- * kick an idle CPU. -- */ -- if (enq_flags & SCX_ENQ_REENQ) { -- s32 cpu; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags); -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- return; -- } -- -- ring = bpf_map_lookup_elem(&queue_arr, &idx); -- if (!ring) { -- scx_bpf_error("failed to find ring %d", idx); -- return; -- } -- -- /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */ -- if (bpf_map_push_elem(ring, &pid, 0)) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_enqueued, 1); --} -- --/* -- * The BPF queue map doesn't support removal and sched_ext can handle spurious -- * dispatches. qmap_dequeue() is only used to collect statistics. -- */ --void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags) --{ -- __sync_fetch_and_add(&nr_dequeued, 1); -- if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) -- __sync_fetch_and_add(&nr_core_sched_execed, 1); --} -- --static void update_core_sched_head_seq(struct task_struct *p) --{ -- struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- int idx = weight_to_idx(p->scx.weight); -- -- if (tctx) -- core_sched_head_seqs[idx] = tctx->core_sched_seq; -- else -- scx_bpf_error("task_ctx lookup failed"); --} -- --void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 zero = 0, one = 1; -- u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero); -- u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one); -- void *fifo; -- s32 pid; -- int i; -- -- if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) { -- struct task_struct *p; -- -- /* -- * PID 2 should be kthreadd which should mostly be idle and off -- * the scheduler. Let's keep dispatching it to force the kernel -- * to call this function over and over again. -- */ -- p = bpf_task_from_pid(2); -- if (p) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- if (!idx || !cnt) { -- scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt); -- return; -- } -- -- for (i = 0; i < 5; i++) { -- /* Advance the dispatch cursor and pick the fifo. */ -- if (!*cnt) { -- *idx = (*idx + 1) % 5; -- *cnt = 1 << *idx; -- } -- (*cnt)--; -- -- fifo = bpf_map_lookup_elem(&queue_arr, idx); -- if (!fifo) { -- scx_bpf_error("failed to find ring %llu", *idx); -- return; -- } -- -- /* Dispatch or advance. */ -- if (!bpf_map_pop_elem(fifo, &pid)) { -- struct task_struct *p; -- -- p = bpf_task_from_pid(pid); -- if (p) { -- update_core_sched_head_seq(p); -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- *cnt = 0; -- } --} -- --/* -- * The distance from the head of the queue scaled by the weight of the queue. -- * The lower the number, the older the task and the higher the priority. -- */ --static s64 task_qdist(struct task_struct *p) --{ -- int idx = weight_to_idx(p->scx.weight); -- struct task_ctx *tctx; -- s64 qdist; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return 0; -- } -- -- qdist = tctx->core_sched_seq - core_sched_head_seqs[idx]; -- -- /* -- * As queue index increments, the priority doubles. The queue w/ index 3 -- * is dispatched twice more frequently than 2. Reflect the difference by -- * scaling qdists accordingly. Note that the shift amount needs to be -- * flipped depending on the sign to avoid flipping priority direction. -- */ -- if (qdist >= 0) -- return qdist << (4 - idx); -- else -- return qdist << idx; --} -- --/* -- * This is called to determine the task ordering when core-sched is picking -- * tasks to execute on SMT siblings and should encode about the same ordering as -- * the regular scheduling path. Use the priority-scaled distances from the head -- * of the queues to compare the two tasks which should be consistent with the -- * dispatch path behavior. -- */ --bool BPF_STRUCT_OPS(qmap_core_sched_before, -- struct task_struct *a, struct task_struct *b) --{ -- return task_qdist(a) > task_qdist(b); --} -- --void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- u32 cnt; -- -- /* -- * Called when @cpu is taken by a higher priority scheduling class. This -- * makes @cpu no longer available for executing sched_ext tasks. As we -- * don't want the tasks in @cpu's local dsq to sit there until @cpu -- * becomes available again, re-enqueue them into the global dsq. See -- * %SCX_ENQ_REENQ handling in qmap_enqueue(). -- */ -- cnt = scx_bpf_reenqueue_local(); -- if (cnt) -- __sync_fetch_and_add(&nr_reenqueued, cnt); --} -- --s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (p->tgid == disallow_tgid) -- p->scx.disallow = true; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(qmap_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops qmap_ops = { -- .select_cpu = (void *)qmap_select_cpu, -- .enqueue = (void *)qmap_enqueue, -- .dequeue = (void *)qmap_dequeue, -- .dispatch = (void *)qmap_dispatch, -- .core_sched_before = (void *)qmap_core_sched_before, -- .cpu_release = (void *)qmap_cpu_release, -- .prep_enable = (void *)qmap_prep_enable, -- .init = (void *)qmap_init, -- .exit = (void *)qmap_exit, -- .flags = SCX_OPS_ENQ_LAST, -- .timeout_ms = 5000U, -- .name = "qmap", --}; -diff --git b/tools/sched_ext/scx_example_qmap.c a/tools/sched_ext/scx_example_qmap.c -deleted file mode 100644 -index ccb4814ee61b..000000000000 ---- b/tools/sched_ext/scx_example_qmap.c -+++ /dev/null -@@ -1,107 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_qmap.skel.h" -- --const char help_fmt[] = --"A simple five-level FIFO queue sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" --" -t COUNT Stall every COUNT'th user thread\n" --" -T COUNT Stall every COUNT'th kernel thread\n" --" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n" --" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_qmap *skel; -- struct bpf_link *link; -- int opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_qmap__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) { -- switch (opt) { -- case 's': -- skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; -- break; -- case 'e': -- skel->bss->test_error_cnt = strtoul(optarg, NULL, 0); -- break; -- case 't': -- skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0); -- break; -- case 'T': -- skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0); -- break; -- case 'l': -- skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0); -- break; -- case 'd': -- skel->rodata->disallow_tgid = strtol(optarg, NULL, 0); -- if (skel->rodata->disallow_tgid < 0) -- skel->rodata->disallow_tgid = getpid(); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_qmap__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.qmap_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- long nr_enqueued = skel->bss->nr_enqueued; -- long nr_dispatched = skel->bss->nr_dispatched; -- -- printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n", -- nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, -- skel->bss->nr_reenqueued, skel->bss->nr_dequeued, -- skel->bss->nr_core_sched_execed); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_qmap__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_simple.bpf.c a/tools/sched_ext/scx_example_simple.bpf.c -deleted file mode 100644 -index 4bccca3e2047..000000000000 ---- b/tools/sched_ext/scx_example_simple.bpf.c -+++ /dev/null -@@ -1,128 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple scheduler. -- * -- * By default, it operates as a simple global weighted vtime scheduler and can -- * be switched to FIFO scheduling. It also demonstrates the following niceties. -- * -- * - Statistics tracking how many tasks are queued to local and global dsq's. -- * - Termination notification for userspace. -- * -- * While very simple, this scheduler should work reasonably well on CPUs with a -- * uniform L3 cache topology. While preemption is not implemented, the fact that -- * the scheduling queue is shared across all CPUs means that whatever is at the -- * front of the queue is likely to be executed fairly quickly given enough -- * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads -- * but comes with the usual problems with FIFO scheduling where saturating -- * threads can easily drown out interactive ones. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --static u64 vtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, 2); /* [local, global] */ --} stats SEC(".maps"); -- --static void stat_inc(u32 idx) --{ -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); -- if (cnt_p) -- (*cnt_p)++; --} -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) --{ -- /* -- * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that -- * running @p on its CPU directly shouldn't affect fairness. Just queue -- * it on the local FIFO. -- */ -- if (enq_flags & SCX_ENQ_LOCAL) { -- stat_inc(0); /* count local queueing */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- return; -- } -- -- stat_inc(1); /* count global queueing */ -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) -- vtime = vtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --void BPF_STRUCT_OPS(simple_running, struct task_struct *p) --{ -- if (fifo_sched) -- return; -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(vtime_now, p->scx.dsq_vtime)) -- vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --s32 BPF_STRUCT_OPS(simple_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .running = (void *)simple_running, -- .stopping = (void *)simple_stopping, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", --}; -diff --git b/tools/sched_ext/scx_example_simple.c a/tools/sched_ext/scx_example_simple.c -deleted file mode 100644 -index 486b401f7c95..000000000000 ---- b/tools/sched_ext/scx_example_simple.c -+++ /dev/null -@@ -1,101 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_simple.skel.h" -- --const char help_fmt[] = --"A simple sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-f] [-p]\n" --"\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int simple) --{ -- exit_req = 1; --} -- --static void read_stats(struct scx_example_simple *skel, u64 *stats) --{ -- int nr_cpus = libbpf_num_possible_cpus(); -- u64 cnts[2][nr_cpus]; -- u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * 2); -- -- for (idx = 0; idx < 2; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_simple *skel; -- struct bpf_link *link; -- u32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_simple__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "fph")) != -1) { -- switch (opt) { -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_simple__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.simple_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- u64 stats[2]; -- -- read_stats(skel, stats); -- printf("local=%lu global=%lu\n", stats[0], stats[1]); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_simple__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_userland.bpf.c a/tools/sched_ext/scx_example_userland.bpf.c -deleted file mode 100644 -index a089bc6bbe86..000000000000 ---- b/tools/sched_ext/scx_example_userland.bpf.c -+++ /dev/null -@@ -1,269 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A minimal userland scheduler. -- * -- * In terms of scheduling, this provides two different types of behaviors: -- * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity. -- * All such tasks are direct-dispatched from the kernel, and are never -- * enqueued in user space. -- * 2. A primitive vruntime scheduler that is implemented in user space, for all -- * other tasks. -- * -- * Some parts of this example user space scheduler could be implemented more -- * efficiently using more complex and sophisticated data structures. For -- * example, rather than using BPF_MAP_TYPE_QUEUE's, -- * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between -- * user space and kernel space. Similarly, we use a simple vruntime-sorted list -- * in user space, but an rbtree could be used instead. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include --#include "scx_common.bpf.h" --#include "scx_example_userland_common.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; --const volatile s32 usersched_pid; -- --/* !0 for veristat, set during init */ --const volatile u32 num_possible_cpus = 64; -- --/* Stats that are printed by user space. */ --u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues; -- --struct user_exit_info uei; -- --/* -- * Whether the user space scheduler needs to be scheduled due to a task being -- * enqueued in user space. -- */ --static bool usersched_needed; -- --/* -- * The map containing tasks that are enqueued in user space from the kernel. -- * -- * This map is drained by the user space scheduler. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, struct scx_userland_enqueued_task); --} enqueued SEC(".maps"); -- --/* -- * The map containing tasks that are dispatched to the kernel from user space. -- * -- * Drained by the kernel in userland_dispatch(). -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, s32); --} dispatched SEC(".maps"); -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local DSQ */ --}; -- --/* Map that contains task-local storage. */ --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --static bool is_usersched_task(const struct task_struct *p) --{ -- return p->pid == usersched_pid; --} -- --static bool keep_in_kernel(const struct task_struct *p) --{ -- return p->nr_cpus_allowed < num_possible_cpus; --} -- --static struct task_struct *usersched_task(void) --{ -- struct task_struct *p; -- -- p = bpf_task_from_pid(usersched_pid); -- /* -- * Should never happen -- the usersched task should always be managed -- * by sched_ext. -- */ -- if (!p) { -- scx_bpf_error("Failed to find usersched task %d", usersched_pid); -- /* -- * We should never hit this path, and we error out of the -- * scheduler above just in case, so the scheduler will soon be -- * be evicted regardless. So as to simplify the logic in the -- * caller to not have to check for NULL, return an acquired -- * reference to the current task here rather than NULL. -- */ -- return bpf_task_acquire(bpf_get_current_task_btf()); -- } -- -- return p; --} -- --s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- if (keep_in_kernel(p)) { -- s32 cpu; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to look up task-local storage for %s", p->comm); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- tctx->force_local = true; -- return cpu; -- } -- } -- -- return prev_cpu; --} -- --static void dispatch_user_scheduler(void) --{ -- struct task_struct *p; -- -- usersched_needed = false; -- p = usersched_task(); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); --} -- --static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags) --{ -- struct scx_userland_enqueued_task task; -- -- memset(&task, 0, sizeof(task)); -- task.pid = p->pid; -- task.sum_exec_runtime = p->se.sum_exec_runtime; -- task.weight = p->scx.weight; -- -- if (bpf_map_push_elem(&enqueued, &task, 0)) { -- /* -- * If we fail to enqueue the task in user space, put it -- * directly on the global DSQ. -- */ -- __sync_fetch_and_add(&nr_failed_enqueues, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- __sync_fetch_and_add(&nr_user_enqueues, 1); -- usersched_needed = true; -- } --} -- --void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags) --{ -- if (keep_in_kernel(p)) { -- u64 dsq_id = SCX_DSQ_GLOBAL; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to lookup task ctx for %s", p->comm); -- return; -- } -- -- if (tctx->force_local) -- dsq_id = SCX_DSQ_LOCAL; -- tctx->force_local = false; -- scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags); -- __sync_fetch_and_add(&nr_kernel_enqueues, 1); -- return; -- } else if (!is_usersched_task(p)) { -- enqueue_task_in_user_space(p, enq_flags); -- } --} -- --void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (usersched_needed) -- dispatch_user_scheduler(); -- -- bpf_repeat(4096) { -- s32 pid; -- struct task_struct *p; -- -- if (bpf_map_pop_elem(&dispatched, &pid)) -- break; -- -- /* -- * The task could have exited by the time we get around to -- * dispatching it. Treat this as a normal occurrence, and simply -- * move onto the next iteration. -- */ -- p = bpf_task_from_pid(pid); -- if (!p) -- continue; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } --} -- --s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(userland_init) --{ -- if (num_possible_cpus == 0) { -- scx_bpf_error("User scheduler # CPUs uninitialized (%d)", -- num_possible_cpus); -- return -EINVAL; -- } -- -- if (usersched_pid <= 0) { -- scx_bpf_error("User scheduler pid uninitialized (%d)", -- usersched_pid); -- return -EINVAL; -- } -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops userland_ops = { -- .select_cpu = (void *)userland_select_cpu, -- .enqueue = (void *)userland_enqueue, -- .dispatch = (void *)userland_dispatch, -- .prep_enable = (void *)userland_prep_enable, -- .init = (void *)userland_init, -- .exit = (void *)userland_exit, -- .timeout_ms = 3000, -- .name = "userland", --}; -diff --git b/tools/sched_ext/scx_example_userland.c a/tools/sched_ext/scx_example_userland.c -deleted file mode 100644 -index 4152b1e65fe1..000000000000 ---- b/tools/sched_ext/scx_example_userland.c -+++ /dev/null -@@ -1,402 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext user space scheduler which provides vruntime semantics -- * using a simple ordered-list implementation. -- * -- * Each CPU in the system resides in a single, global domain. This precludes -- * the need to do any load balancing between domains. The scheduler could -- * easily be extended to support multiple domains, with load balancing -- * happening in user space. -- * -- * Any task which has any CPU affinity is scheduled entirely in BPF. This -- * program only schedules tasks which may run on any CPU. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include "user_exit_info.h" --#include "scx_example_userland_common.h" --#include "scx_example_userland.skel.h" -- --const char help_fmt[] = --"A minimal userland sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-b BATCH] [-p]\n" --"\n" --" -b BATCH The number of tasks to batch when dispatching (default: 8)\n" --" -p Don't switch all, switch only tasks on SCHED_EXT policy\n" --" -h Display this help and exit\n"; -- --/* Defined in UAPI */ --#define SCHED_EXT 7 -- --/* Number of tasks to batch when dispatching to user space. */ --static __u32 batch_size = 8; -- --static volatile int exit_req; --static int enqueued_fd, dispatched_fd; -- --static struct scx_example_userland *skel; --static struct bpf_link *ops_link; -- --/* Stats collected in user space. */ --static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches; -- --/* The data structure containing tasks that are enqueued in user space. */ --struct enqueued_task { -- LIST_ENTRY(enqueued_task) entries; -- __u64 sum_exec_runtime; -- double vruntime; --}; -- --/* -- * Use a vruntime-sorted list to store tasks. This could easily be extended to -- * a more optimal data structure, such as an rbtree as is done in CFS. We -- * currently elect to use a sorted list to simplify the example for -- * illustrative purposes. -- */ --LIST_HEAD(listhead, enqueued_task); -- --/* -- * A vruntime-sorted list of tasks. The head of the list contains the task with -- * the lowest vruntime. That is, the task that has the "highest" claim to be -- * scheduled. -- */ --static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head); -- --/* -- * The statically allocated array of tasks. We use a statically allocated list -- * here to avoid having to allocate on the enqueue path, which could cause a -- * deadlock. A more substantive user space scheduler could e.g. provide a hook -- * for newly enabled tasks that are passed to the scheduler from the -- * .prep_enable() callback to allows the scheduler to allocate on safe paths. -- */ --struct enqueued_task tasks[USERLAND_MAX_TASKS]; -- --static double min_vruntime; -- --static void sigint_handler(int userland) --{ -- exit_req = 1; --} -- --static __u32 task_pid(const struct enqueued_task *task) --{ -- return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task); --} -- --static int dispatch_task(s32 pid) --{ -- int err; -- -- err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d\n", pid); -- exit_req = 1; -- } else { -- nr_vruntime_dispatches++; -- } -- -- return err; --} -- --static struct enqueued_task *get_enqueued_task(__s32 pid) --{ -- if (pid >= USERLAND_MAX_TASKS) -- return NULL; -- -- return &tasks[pid]; --} -- --static double calc_vruntime_delta(__u64 weight, __u64 delta) --{ -- double weight_f = (double)weight / 100.0; -- double delta_f = (double)delta; -- -- return delta_f / weight_f; --} -- --static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task) --{ -- __u64 delta; -- -- delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime; -- -- enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta); -- if (min_vruntime > enqueued->vruntime) -- enqueued->vruntime = min_vruntime; -- enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime; --} -- --static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task) --{ -- struct enqueued_task *curr, *enqueued, *prev; -- -- curr = get_enqueued_task(bpf_task->pid); -- if (!curr) -- return ENOENT; -- -- update_enqueued(curr, bpf_task); -- nr_vruntime_enqueues++; -- -- /* -- * Enqueue the task in a vruntime-sorted list. A more optimal data -- * structure such as an rbtree could easily be used as well. We elect -- * to use a list here simply because it's less code, and thus the -- * example is less convoluted and better serves to illustrate what a -- * user space scheduler could look like. -- */ -- -- if (LIST_EMPTY(&vruntime_head)) { -- LIST_INSERT_HEAD(&vruntime_head, curr, entries); -- return 0; -- } -- -- LIST_FOREACH(enqueued, &vruntime_head, entries) { -- if (curr->vruntime <= enqueued->vruntime) { -- LIST_INSERT_BEFORE(enqueued, curr, entries); -- return 0; -- } -- prev = enqueued; -- } -- -- LIST_INSERT_AFTER(prev, curr, entries); -- -- return 0; --} -- --static void drain_enqueued_map(void) --{ -- while (1) { -- struct scx_userland_enqueued_task task; -- int err; -- -- if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) -- return; -- -- err = vruntime_enqueue(&task); -- if (err) { -- fprintf(stderr, "Failed to enqueue task %d: %s\n", -- task.pid, strerror(err)); -- exit_req = 1; -- return; -- } -- } --} -- --static void dispatch_batch(void) --{ -- __u32 i; -- -- for (i = 0; i < batch_size; i++) { -- struct enqueued_task *task; -- int err; -- __s32 pid; -- -- task = LIST_FIRST(&vruntime_head); -- if (!task) -- return; -- -- min_vruntime = task->vruntime; -- pid = task_pid(task); -- LIST_REMOVE(task, entries); -- err = dispatch_task(pid); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d in %u\n", -- pid, i); -- return; -- } -- } --} -- --static void *run_stats_printer(void *arg) --{ -- while (!exit_req) { -- __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; -- -- nr_failed_enqueues = skel->bss->nr_failed_enqueues; -- nr_kernel_enqueues = skel->bss->nr_kernel_enqueues; -- nr_user_enqueues = skel->bss->nr_user_enqueues; -- total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues; -- -- printf("o-----------------------o\n"); -- printf("| BPF ENQUEUES |\n"); -- printf("|-----------------------|\n"); -- printf("| kern: %10llu |\n", nr_kernel_enqueues); -- printf("| user: %10llu |\n", nr_user_enqueues); -- printf("| failed: %10llu |\n", nr_failed_enqueues); -- printf("| -------------------- |\n"); -- printf("| total: %10llu |\n", total); -- printf("| |\n"); -- printf("|-----------------------|\n"); -- printf("| VRUNTIME / USER |\n"); -- printf("|-----------------------|\n"); -- printf("| enq: %10llu |\n", nr_vruntime_enqueues); -- printf("| disp: %10llu |\n", nr_vruntime_dispatches); -- printf("o-----------------------o\n"); -- printf("\n\n"); -- sleep(1); -- } -- -- return NULL; --} -- --static int spawn_stats_thread(void) --{ -- pthread_t stats_printer; -- -- return pthread_create(&stats_printer, NULL, run_stats_printer, NULL); --} -- --static int bootstrap(int argc, char **argv) --{ -- int err; -- __u32 opt; -- struct sched_param sched_param = { -- .sched_priority = sched_get_priority_max(SCHED_EXT), -- }; -- bool switch_partial = false; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- /* -- * Enforce that the user scheduler task is managed by sched_ext. The -- * task eagerly drains the list of enqueued tasks in its main work -- * loop, and then yields the CPU. The BPF scheduler only schedules the -- * user space scheduler task when at least one other task in the system -- * needs to be scheduled. -- */ -- err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param); -- if (err) { -- fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err)); -- return err; -- } -- -- while ((opt = getopt(argc, argv, "b:ph")) != -1) { -- switch (opt) { -- case 'b': -- batch_size = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- exit(opt != 'h'); -- } -- } -- -- /* -- * It's not always safe to allocate in a user space scheduler, as an -- * enqueued task could hold a lock that we require in order to be able -- * to allocate. -- */ -- err = mlockall(MCL_CURRENT | MCL_FUTURE); -- if (err) { -- fprintf(stderr, "Failed to prefault and lock address space: %s\n", -- strerror(err)); -- return err; -- } -- -- skel = scx_example_userland__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno)); -- return errno; -- } -- skel->rodata->num_possible_cpus = libbpf_num_possible_cpus(); -- assert(skel->rodata->num_possible_cpus > 0); -- skel->rodata->usersched_pid = getpid(); -- assert(skel->rodata->usersched_pid > 0); -- skel->rodata->switch_partial = switch_partial; -- -- err = scx_example_userland__load(skel); -- if (err) { -- fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- enqueued_fd = bpf_map__fd(skel->maps.enqueued); -- dispatched_fd = bpf_map__fd(skel->maps.dispatched); -- assert(enqueued_fd > 0); -- assert(dispatched_fd > 0); -- -- err = spawn_stats_thread(); -- if (err) { -- fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops); -- if (!ops_link) { -- fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno)); -- err = errno; -- goto destroy_skel; -- } -- -- return 0; -- --destroy_skel: -- scx_example_userland__destroy(skel); -- exit_req = 1; -- return err; --} -- --static void sched_main_loop(void) --{ -- while (!exit_req) { -- /* -- * Perform the following work in the main user space scheduler -- * loop: -- * -- * 1. Drain all tasks from the enqueued map, and enqueue them -- * to the vruntime sorted list. -- * -- * 2. Dispatch a batch of tasks from the vruntime sorted list -- * down to the kernel. -- * -- * 3. Yield the CPU back to the system. The BPF scheduler will -- * reschedule the user space scheduler once another task has -- * been enqueued to user space. -- */ -- drain_enqueued_map(); -- dispatch_batch(); -- sched_yield(); -- } --} -- --int main(int argc, char **argv) --{ -- int err; -- -- err = bootstrap(argc, argv); -- if (err) { -- fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err)); -- return err; -- } -- -- sched_main_loop(); -- -- exit_req = 1; -- bpf_link__destroy(ops_link); -- uei_print(&skel->bss->uei); -- scx_example_userland__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_userland_common.h a/tools/sched_ext/scx_example_userland_common.h -deleted file mode 100644 -index 639c6809c5ff..000000000000 ---- b/tools/sched_ext/scx_example_userland_common.h -+++ /dev/null -@@ -1,19 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* Copyright (c) 2022 Meta, Inc */ -- --#ifndef __SCX_USERLAND_COMMON_H --#define __SCX_USERLAND_COMMON_H -- --#define USERLAND_MAX_TASKS 8192 -- --/* -- * An instance of a task that has been enqueued by the kernel for consumption -- * by a user space global scheduler thread. -- */ --struct scx_userland_enqueued_task { -- __s32 pid; -- u64 sum_exec_runtime; -- u64 weight; --}; -- --#endif // __SCX_USERLAND_COMMON_H -diff --git b/tools/sched_ext/user_exit_info.h a/tools/sched_ext/user_exit_info.h -deleted file mode 100644 -index e701ef0e0b86..000000000000 ---- b/tools/sched_ext/user_exit_info.h -+++ /dev/null -@@ -1,50 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Define struct user_exit_info which is shared between BPF and userspace parts -- * to communicate exit status and other information. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __USER_EXIT_INFO_H --#define __USER_EXIT_INFO_H -- --struct user_exit_info { -- int type; -- char reason[128]; -- char msg[1024]; --}; -- --#ifdef __bpf__ -- --#include "vmlinux.h" --#include -- --static inline void uei_record(struct user_exit_info *uei, -- const struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason); -- bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg); -- /* use __sync to force memory barrier */ -- __sync_val_compare_and_swap(&uei->type, uei->type, ei->type); --} -- --#else /* !__bpf__ */ -- --static inline bool uei_exited(struct user_exit_info *uei) --{ -- /* use __sync to force memory barrier */ -- return __sync_val_compare_and_swap(&uei->type, -1, -1); --} -- --static inline void uei_print(const struct user_exit_info *uei) --{ -- fprintf(stderr, "EXIT: %s", uei->reason); -- if (uei->msg[0] != '\0') -- fprintf(stderr, " (%s)", uei->msg); -- fputs("\n", stderr); --} -- --#endif /* __bpf__ */ --#endif /* __USER_EXIT_INFO_H */ diff --git a/oneplus/hmbird/fengchi_OP13-CPH_A15.patch b/oneplus/hmbird/fengchi_OP13-CPH_A15.patch deleted file mode 100644 index 6d07da62..00000000 --- a/oneplus/hmbird/fengchi_OP13-CPH_A15.patch +++ /dev/null @@ -1,21358 +0,0 @@ -diff --git a/Documentation/scheduler/index.rst b/Documentation/scheduler/index.rst -index 0b650bb550e6..3170747226f6 100644 ---- a/Documentation/scheduler/index.rst -+++ b/Documentation/scheduler/index.rst -@@ -19,7 +19,6 @@ Scheduler - sched-nice-design - sched-rt-group - sched-stats -- sched-ext - sched-debug - - text_files -diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst -deleted file mode 100644 -index 84c30b44f104..000000000000 ---- a/Documentation/scheduler/sched-ext.rst -+++ /dev/null -@@ -1,230 +0,0 @@ --========================== --Extensible Scheduler Class --========================== -- --sched_ext is a scheduler class whose behavior can be defined by a set of BPF --programs - the BPF scheduler. -- --* sched_ext exports a full scheduling interface so that any scheduling -- algorithm can be implemented on top. -- --* The BPF scheduler can group CPUs however it sees fit and schedule them -- together, as tasks aren't tied to specific CPUs at the time of wakeup. -- --* The BPF scheduler can be turned on and off dynamically anytime. -- --* The system integrity is maintained no matter what the BPF scheduler does. -- The default scheduling behavior is restored anytime an error is detected, -- a runnable task stalls, or on invoking the SysRq key sequence -- :kbd:`SysRq-S`. -- --Switching to and from sched_ext --=============================== -- --``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and --``tools/sched_ext`` contains the example schedulers. -- --sched_ext is used only when the BPF scheduler is loaded and running. -- --If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be --treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is --loaded. On load, such tasks will be switched to and scheduled by sched_ext. -- --The BPF scheduler can choose to schedule all normal and lower class tasks by --calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this --case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and --``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers, --this mode can be selected with the ``-a`` option. -- --Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or --detection of any internal error including stalled runnable tasks aborts the --BPF scheduler and reverts all tasks back to CFS. -- --.. code-block:: none -- -- # make -j16 -C tools/sched_ext -- # tools/sched_ext/scx_example_simple -- local=0 global=3 -- local=5 global=24 -- local=9 global=44 -- local=13 global=56 -- local=17 global=72 -- ^CEXIT: BPF scheduler unregistered -- --If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler --and whether a given task is on sched_ext can be determined as follows: -- --.. code-block:: none -- -- # cat /sys/kernel/debug/sched/ext -- ops : simple -- enabled : 1 -- switching_all : 1 -- switched_all : 1 -- enable_state : enabled -- -- # grep ext /proc/self/sched -- ext.enabled : 1 -- --The Basics --========== -- --Userspace can implement an arbitrary BPF scheduler by loading a set of BPF --programs that implement ``struct sched_ext_ops``. The only mandatory field --is ``ops.name`` which must be a valid BPF object name. All operations are --optional. The following modified excerpt is from --``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO --scheduler. -- --.. code-block:: c -- -- s32 BPF_STRUCT_OPS(simple_init) -- { -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; -- } -- -- void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) -- { -- if (enq_flags & SCX_ENQ_LOCAL) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags); -- } -- -- void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) -- { -- exit_type = ei->type; -- } -- -- SEC(".struct_ops") -- struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", -- }; -- --Dispatch Queues ----------------- -- --To match the impedance between the scheduler core and the BPF scheduler, --sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a --priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``), --and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage --an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and --``scx_bpf_destroy_dsq()``. -- --A CPU always executes a task from its local DSQ. A task is "dispatched" to a --DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's --local DSQ. -- --When a CPU is looking for the next task to run, if the local DSQ is not --empty, the first task is picked. Otherwise, the CPU tries to consume the --global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()`` --is invoked. -- --Scheduling Cycle ------------------ -- --The following briefly shows how a waking task is scheduled and executed. -- --1. When a task is waking up, ``ops.select_cpu()`` is the first operation -- invoked. This serves two purposes. First, CPU selection optimization -- hint. Second, waking up the selected CPU if idle. -- -- The CPU selected by ``ops.select_cpu()`` is an optimization hint and not -- binding. The actual decision is made at the last step of scheduling. -- However, there is a small performance gain if the CPU -- ``ops.select_cpu()`` returns matches the CPU the task eventually runs on. -- -- A side-effect of selecting a CPU is waking it up from idle. While a BPF -- scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper, -- using ``ops.select_cpu()`` judiciously can be simpler and more efficient. -- -- Note that the scheduler core will ignore an invalid CPU selection, for -- example, if it's outside the allowed cpumask of the task. -- --2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can -- make one of the following decisions: -- -- * Immediately dispatch the task to either the global or local DSQ by -- calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or -- ``SCX_DSQ_LOCAL``, respectively. -- -- * Immediately dispatch the task to a custom DSQ by calling -- ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63. -- -- * Queue the task on the BPF side. -- --3. When a CPU is ready to schedule, it first looks at its local DSQ. If -- empty, it then looks at the global DSQ. If there still isn't a task to -- run, ``ops.dispatch()`` is invoked which can use the following two -- functions to populate the local DSQ. -- -- * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can -- be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``, -- ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()`` -- currently can't be called with BPF locks held, this is being worked on -- and will be supported. ``scx_bpf_dispatch()`` schedules dispatching -- rather than performing them immediately. There can be up to -- ``ops.dispatch_max_batch`` pending tasks. -- -- * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ -- to the dispatching DSQ. This function cannot be called with any BPF -- locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks -- before trying to consume the specified DSQ. -- --4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ, -- the CPU runs the first one. If empty, the following steps are taken: -- -- * Try to consume the global DSQ. If successful, run the task. -- -- * If ``ops.dispatch()`` has dispatched any tasks, retry #3. -- -- * If the previous task is an SCX task and still runnable, keep executing -- it (see ``SCX_OPS_ENQ_LAST``). -- -- * Go idle. -- --Note that the BPF scheduler can always choose to dispatch tasks immediately --in ``ops.enqueue()`` as illustrated in the above simple example. If only the --built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as --a task is never queued on the BPF scheduler and both the local and global --DSQs are consumed automatically. -- --``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use --``scx_bpf_dispatch_vtime()`` for the priority queue. See the function --documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for --more information. -- --Where to Look --============= -- --* ``include/linux/sched/ext.h`` defines the core data structures, ops table -- and constants. -- --* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. -- The functions prefixed with ``scx_bpf_`` can be called from the BPF -- scheduler. -- --* ``tools/sched_ext/`` hosts example BPF scheduler implementations. -- -- * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example -- using a custom DSQ. -- -- * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting -- five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. -- --ABI Instability --=============== -- --The APIs provided by sched_ext to BPF schedulers programs have no stability --guarantees. This includes the ops table callbacks and constants defined in --``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in --``kernel/sched/ext.c``. -- --While we will attempt to provide a relatively stable API surface when --possible, they are subject to change without warning between kernel --versions. -diff --git a/MAINTAINERS b/MAINTAINERS -index 9fc6108503c3..2384b55682ee 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -19132,8 +19132,6 @@ R: Ben Segall (CONFIG_CFS_BANDWIDTH) - R: Mel Gorman (CONFIG_NUMA_BALANCING) - R: Daniel Bristot de Oliveira (SCHED_DEADLINE) - R: Valentin Schneider (TOPOLOGY) --R: Tejun Heo (SCHED_EXT) --R: David Vernet (SCHED_EXT) - L: linux-kernel@vger.kernel.org - S: Maintained - T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core -@@ -19142,7 +19140,6 @@ F: include/linux/sched.h - F: include/linux/wait.h - F: include/uapi/linux/sched.h - F: kernel/sched/ --F: tools/sched_ext/ - - SCSI LIBSAS SUBSYSTEM - R: John Garry -diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig -index f93980c09d7f..5ba590235a8c 100644 ---- a/arch/arm64/configs/defconfig -+++ b/arch/arm64/configs/defconfig -@@ -6,8 +6,7 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_BPF_SYSCALL=y - CONFIG_BPF_JIT=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_BSD_PROCESS_ACCT=y - CONFIG_BSD_PROCESS_ACCT_V3=y -diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig -index 54fded4659ee..6ab8ce58a0c4 100644 ---- a/arch/arm64/configs/gki_defconfig -+++ b/arch/arm64/configs/gki_defconfig -@@ -10,8 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y - # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set - CONFIG_BPF_LSM=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_TASKSTATS=y - CONFIG_TASK_DELAY_ACCT=y -diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c -index 4bd028fa7500..5825ce9d6d7b 100644 ---- a/drivers/gpu/drm/msm/msm_drv.c -+++ b/drivers/gpu/drm/msm/msm_drv.c -@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) - } - - sched_set_fifo(ev_thread->worker->task); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3); -+#endif - } - - ret = drm_vblank_init(ddev, priv->num_crtcs); -diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c -index bd001445815e..dcf2e1ebf9c5 100644 ---- a/drivers/tty/sysrq.c -+++ b/drivers/tty/sysrq.c -@@ -524,7 +524,6 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { - NULL, /* P */ - NULL, /* Q */ - NULL, /* R */ -- /* S: May be registered by sched_ext for resetting */ - NULL, /* S */ - NULL, /* T */ - NULL, /* U */ -diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h -index c45078a445c8..6c15659f874e 100644 ---- a/include/asm-generic/vmlinux.lds.h -+++ b/include/asm-generic/vmlinux.lds.h -@@ -131,7 +131,7 @@ - *(__dl_sched_class) \ - *(__rt_sched_class) \ - *(__fair_sched_class) \ -- *(__ext_sched_class) \ -+ *(__hmbird_sched_class) \ - *(__idle_sched_class) \ - __sched_class_lowest = .; - -diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h -index 63452733a2f9..3a2f906ed3ce 100644 ---- a/include/linux/cpufreq.h -+++ b/include/linux/cpufreq.h -@@ -44,6 +44,10 @@ enum cpufreq_table_sorting { - CPUFREQ_TABLE_SORTED_DESCENDING - }; - -+ssize_t store_scaling_governor(struct cpufreq_policy *policy, -+ const char *buf, size_t count); -+ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf); -+ - struct cpufreq_cpuinfo { - unsigned int max_freq; - unsigned int min_freq; -diff --git a/include/linux/sched.h b/include/linux/sched.h -index 2cbc342db12d..dd6b1d58af01 100644 ---- a/include/linux/sched.h -+++ b/include/linux/sched.h -@@ -73,7 +73,9 @@ struct task_delay_info; - struct task_group; - struct user_event_mm; - --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - - /* - * Task state bitmask. NOTE! These bits are also -@@ -298,11 +300,6 @@ enum { - - extern void scheduler_tick(void); - --#ifdef CONFIG_SLIM_SCHED --extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer); --extern void stop_shadow_tick_timer(void); --#endif -- - #define MAX_SCHEDULE_TIMEOUT LONG_MAX - - extern long schedule_timeout(long timeout); -@@ -1523,13 +1520,8 @@ struct task_struct { - */ - struct callback_head l1d_flush_kill; - #endif --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, unsigned long sched_prop); -- ANDROID_KABI_USE(2, struct sched_ext_entity *scx); --#else - ANDROID_KABI_RESERVE(1); - ANDROID_KABI_RESERVE(2); --#endif - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); - ANDROID_KABI_RESERVE(5); -@@ -1933,6 +1925,92 @@ static inline int task_nice(const struct task_struct *p) - return PRIO_TO_NICE((p)->static_prio); - } - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline bool task_is_top_task(struct task_struct *p) -+{ -+ return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ ->top_task_prop & TOP_TASK_BITS_MASK); -+} -+ -+static inline int get_top_task_prop(struct task_struct *p) -+{ -+ return get_hmbird_ts(p)->top_task_prop; -+} -+ -+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear) -+{ -+ if (set) -+ get_hmbird_ts(p)->top_task_prop |= set; -+ if (clear) -+ get_hmbird_ts(p)->top_task_prop &= ~clear; -+ return 0; -+} -+ -+static inline void reset_top_task_prop(struct task_struct *p) -+{ -+ get_hmbird_ts(p)->top_task_prop = 0; -+} -+ -+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ entity->sched_prop = sp; -+ } -+ return 0; -+} -+ -+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->sched_prop; -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq) -+{ -+ unsigned long new_dsq; -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq; -+ entity->sched_prop = new_dsq; -+ } -+} -+ -+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK); -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ entity->dsq_sync_ux = val; -+} -+ -+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->dsq_sync_ux; -+ else -+ return 0; -+} -+#endif -+ - extern int can_nice(const struct task_struct *p, const int nice); - extern int task_curr(const struct task_struct *p); - extern int idle_cpu(int cpu); -diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h -deleted file mode 100755 -index b737a00c1373..000000000000 ---- a/include/linux/sched/ext.h -+++ /dev/null -@@ -1,647 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef _LINUX_SCHED_EXT_H --#define _LINUX_SCHED_EXT_H -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --#include -- --enum scx_consts { -- SCX_OPS_NAME_LEN = 128, -- SCX_EXIT_REASON_LEN = 128, -- SCX_EXIT_BT_LEN = 64, -- SCX_EXIT_MSG_LEN = 1024, -- -- SCX_SLICE_DFL = 20 * NSEC_PER_MSEC, -- SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */ --}; -- --/* -- * DSQ (dispatch queue) IDs are 64bit of the format: -- * -- * Bits: [63] [62 .. 0] -- * [ B] [ ID ] -- * -- * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -- * ID: 63 bit ID -- * -- * Built-in IDs: -- * -- * Bits: [63] [62] [61..32] [31 .. 0] -- * [ 1] [ L] [ R ] [ V ] -- * -- * 1: 1 for built-in DSQs. -- * L: 1 for LOCAL_ON DSQ IDs, 0 for others -- * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -- */ --enum scx_dsq_id_flags { -- SCX_DSQ_FLAG_BUILTIN = 1LLU << 63, -- SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -- -- SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0, -- SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, -- SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, -- SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, -- SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, --}; -- --enum scx_exit_type { -- SCX_EXIT_NONE, -- SCX_EXIT_DONE, -- -- SCX_EXIT_UNREG = 64, /* BPF unregistration */ -- SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -- SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ -- SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ --}; -- --/* -- * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is -- * being disabled. -- */ --struct scx_exit_info { -- /* %SCX_EXIT_* - broad category of the exit reason */ -- enum scx_exit_type type; -- /* textual representation of the above */ -- char reason[SCX_EXIT_REASON_LEN]; -- /* number of entries in the backtrace */ -- u32 bt_len; -- /* backtrace if exiting due to an error */ -- unsigned long bt[SCX_EXIT_BT_LEN]; -- /* extra message */ -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --/* sched_ext_ops.flags */ --enum scx_ops_flags { -- /* -- * Keep built-in idle tracking even if ops.update_idle() is implemented. -- */ -- SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, -- -- /* -- * By default, if there are no other task to run on the CPU, ext core -- * keeps running the current task even after its slice expires. If this -- * flag is specified, such tasks are passed to ops.enqueue() with -- * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. -- */ -- SCX_OPS_ENQ_LAST = 1LLU << 1, -- -- /* -- * An exiting task may schedule after PF_EXITING is set. In such cases, -- * bpf_task_from_pid() may not be able to find the task and if the BPF -- * scheduler depends on pid lookup for dispatching, the task will be -- * lost leading to various issues including RCU grace period stalls. -- * -- * To mask this problem, by default, unhashed tasks are automatically -- * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't -- * depend on pid lookups and wants to handle these tasks directly, the -- * following flag can be used. -- */ -- SCX_OPS_ENQ_EXITING = 1LLU << 2, -- -- /* -- * CPU cgroup knob enable flags -- */ -- SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16, /* cpu.weight */ -- -- SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | -- SCX_OPS_ENQ_LAST | -- SCX_OPS_ENQ_EXITING | -- SCX_OPS_CGROUP_KNOB_WEIGHT, --}; -- --/* argument container for ops.enable() and friends */ --struct scx_enable_args { --}; -- --/* argument container for ops->cgroup_init() */ --struct scx_cgroup_init_args { -- /* the weight of the cgroup [1..10000] */ -- u32 weight; --}; -- --enum scx_cpu_preempt_reason { -- /* next task is being scheduled by &sched_class_rt */ -- SCX_CPU_PREEMPT_RT, -- /* next task is being scheduled by &sched_class_dl */ -- SCX_CPU_PREEMPT_DL, -- /* next task is being scheduled by &sched_class_stop */ -- SCX_CPU_PREEMPT_STOP, -- /* unknown reason for SCX being preempted */ -- SCX_CPU_PREEMPT_UNKNOWN, --}; -- --/* -- * Argument container for ops->cpu_acquire(). Currently empty, but may be -- * expanded in the future. -- */ --struct scx_cpu_acquire_args {}; -- --/* argument container for ops->cpu_release() */ --struct scx_cpu_release_args { -- /* the reason the CPU was preempted */ -- enum scx_cpu_preempt_reason reason; -- -- /* the task that's going to be scheduled on the CPU */ -- struct task_struct *task; --}; -- --/** -- * struct sched_ext_ops - Operation table for BPF scheduler implementation -- * -- * Userland can implement an arbitrary scheduling policy by implementing and -- * loading operations in this table. -- */ --struct sched_ext_ops { -- /** -- * select_cpu - Pick the target CPU for a task which is being woken up -- * @p: task being woken up -- * @prev_cpu: the cpu @p was on before sleeping -- * @wake_flags: SCX_WAKE_* -- * -- * Decision made here isn't final. @p may be moved to any CPU while it -- * is getting dispatched for execution later. However, as @p is not on -- * the rq at this point, getting the eventual execution CPU right here -- * saves a small bit of overhead down the line. -- * -- * If an idle CPU is returned, the CPU is kicked and will try to -- * dispatch. While an explicit custom mechanism can be added, -- * select_cpu() serves as the default way to wake up idle CPUs. -- */ -- s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); -- -- /** -- * enqueue - Enqueue a task on the BPF scheduler -- * @p: task being enqueued -- * @enq_flags: %SCX_ENQ_* -- * -- * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() -- * or enqueue on the BPF scheduler. If not directly dispatched, the bpf -- * scheduler owns @p and if it fails to dispatch @p, the task will -- * stall. -- */ -- void (*enqueue)(struct task_struct *p, u64 enq_flags); -- -- /** -- * dequeue - Remove a task from the BPF scheduler -- * @p: task being dequeued -- * @deq_flags: %SCX_DEQ_* -- * -- * Remove @p from the BPF scheduler. This is usually called to isolate -- * the task while updating its scheduling properties (e.g. priority). -- * -- * The ext core keeps track of whether the BPF side owns a given task or -- * not and can gracefully ignore spurious dispatches from BPF side, -- * which makes it safe to not implement this method. However, depending -- * on the scheduling logic, this can lead to confusing behaviors - e.g. -- * scheduling position not being updated across a priority change. -- */ -- void (*dequeue)(struct task_struct *p, u64 deq_flags); -- -- /** -- * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs -- * @cpu: CPU to dispatch tasks for -- * @prev: previous task being switched out -- * -- * Called when a CPU's local dsq is empty. The operation should dispatch -- * one or more tasks from the BPF scheduler into the DSQs using -- * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using -- * scx_bpf_consume(). -- * -- * The maximum number of times scx_bpf_dispatch() can be called without -- * an intervening scx_bpf_consume() is specified by -- * ops.dispatch_max_batch. See the comments on top of the two functions -- * for more details. -- * -- * When not %NULL, @prev is an SCX task with its slice depleted. If -- * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in -- * @prev->scx.flags, it is not enqueued yet and will be enqueued after -- * ops.dispatch() returns. To keep executing @prev, return without -- * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. -- */ -- void (*dispatch)(s32 cpu, struct task_struct *prev); -- -- /** -- * runnable - A task is becoming runnable on its associated CPU -- * @p: task becoming runnable -- * @enq_flags: %SCX_ENQ_* -- * -- * This and the following three functions can be used to track a task's -- * execution state transitions. A task becomes ->runnable() on a CPU, -- * and then goes through one or more ->running() and ->stopping() pairs -- * as it runs on the CPU, and eventually becomes ->quiescent() when it's -- * done running on the CPU. -- * -- * @p is becoming runnable on the CPU because it's -- * -- * - waking up (%SCX_ENQ_WAKEUP) -- * - being moved from another CPU -- * - being restored after temporarily taken off the queue for an -- * attribute change. -- * -- * This and ->enqueue() are related but not coupled. This operation -- * notifies @p's state transition and may not be followed by ->enqueue() -- * e.g. when @p is being dispatched to a remote CPU. Likewise, a task -- * may be ->enqueue()'d without being preceded by this operation e.g. -- * after exhausting its slice. -- */ -- void (*runnable)(struct task_struct *p, u64 enq_flags); -- -- /** -- * running - A task is starting to run on its associated CPU -- * @p: task starting to run -- * -- * See ->runnable() for explanation on the task state notifiers. -- */ -- void (*running)(struct task_struct *p); -- -- /** -- * stopping - A task is stopping execution -- * @p: task stopping to run -- * @runnable: is task @p still runnable? -- * -- * See ->runnable() for explanation on the task state notifiers. If -- * !@runnable, ->quiescent() will be invoked after this operation -- * returns. -- */ -- void (*stopping)(struct task_struct *p, bool runnable); -- -- /** -- * quiescent - A task is becoming not runnable on its associated CPU -- * @p: task becoming not runnable -- * @deq_flags: %SCX_DEQ_* -- * -- * See ->runnable() for explanation on the task state notifiers. -- * -- * @p is becoming quiescent on the CPU because it's -- * -- * - sleeping (%SCX_DEQ_SLEEP) -- * - being moved to another CPU -- * - being temporarily taken off the queue for an attribute change -- * (%SCX_DEQ_SAVE) -- * -- * This and ->dequeue() are related but not coupled. This operation -- * notifies @p's state transition and may not be preceded by ->dequeue() -- * e.g. when @p is being dispatched to a remote CPU. -- */ -- void (*quiescent)(struct task_struct *p, u64 deq_flags); -- -- /** -- * yield - Yield CPU -- * @from: yielding task -- * @to: optional yield target task -- * -- * If @to is NULL, @from is yielding the CPU to other runnable tasks. -- * The BPF scheduler should ensure that other available tasks are -- * dispatched before the yielding task. Return value is ignored in this -- * case. -- * -- * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf -- * scheduler can implement the request, return %true; otherwise, %false. -- */ -- bool (*yield)(struct task_struct *from, struct task_struct *to); -- -- /** -- * core_sched_before - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Used by core-sched to determine the ordering between two tasks. See -- * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on -- * core-sched. -- * -- * Both @a and @b are runnable and may or may not currently be queued on -- * the BPF scheduler. Should return %true if @a should run before @b. -- * %false if there's no required ordering or @b should run before @a. -- * -- * If not specified, the default is ordering them according to when they -- * became runnable. -- */ -- bool (*core_sched_before)(struct task_struct *a,struct task_struct *b); -- -- /** -- * set_weight - Set task weight -- * @p: task to set weight for -- * @weight: new eight [1..10000] -- * -- * Update @p's weight to @weight. -- */ -- void (*set_weight)(struct task_struct *p, u32 weight); -- -- /** -- * set_cpumask - Set CPU affinity -- * @p: task to set CPU affinity for -- * @cpumask: cpumask of cpus that @p can run on -- * -- * Update @p's CPU affinity to @cpumask. -- */ -- void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask); -- -- /** -- * update_idle - Update the idle state of a CPU -- * @cpu: CPU to udpate the idle state for -- * @idle: whether entering or exiting the idle state -- * -- * This operation is called when @rq's CPU goes or leaves the idle -- * state. By default, implementing this operation disables the built-in -- * idle CPU tracking and the following helpers become unavailable: -- * -- * - scx_bpf_select_cpu_dfl() -- * - scx_bpf_test_and_clear_cpu_idle() -- * - scx_bpf_pick_idle_cpu() -- * - scx_bpf_any_idle_cpu() -- * -- * The user also must implement ops.select_cpu() as the default -- * implementation relies on scx_bpf_select_cpu_dfl(). -- * -- * If you keep the built-in idle tracking, specify the -- * %SCX_OPS_KEEP_BUILTIN_IDLE flag. -- */ -- void (*update_idle)(s32 cpu, bool idle); -- -- /** -- * cpu_acquire - A CPU is becoming available to the BPF scheduler -- * @cpu: The CPU being acquired by the BPF scheduler. -- * @args: Acquire arguments, see the struct definition. -- * -- * A CPU that was previously released from the BPF scheduler is now once -- * again under its control. -- */ -- void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); -- -- /** -- * cpu_release - A CPU is taken away from the BPF scheduler -- * @cpu: The CPU being released by the BPF scheduler. -- * @args: Release arguments, see the struct definition. -- * -- * The specified CPU is no longer under the control of the BPF -- * scheduler. This could be because it was preempted by a higher -- * priority sched_class, though there may be other reasons as well. The -- * caller should consult @args->reason to determine the cause. -- */ -- void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -- -- /** -- * cpu_online - A CPU became online -- * @cpu: CPU which just came up -- * -- * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs beforehand. -- */ -- void (*cpu_online)(s32 cpu); -- -- /** -- * cpu_offline - A CPU is going offline -- * @cpu: CPU which is going offline -- * -- * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs afterwards. -- */ -- void (*cpu_offline)(s32 cpu); -- -- /** -- * prep_enable - Prepare to enable BPF scheduling for a task -- * @p: task to prepare BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Either we're loading a BPF scheduler or a new task is being forked. -- * Prepare BPF scheduling for @p. This operation may block and can be -- * used for allocations. -- * -- * Return 0 for success, -errno for failure. An error return while -- * loading will abort loading of the BPF scheduler. During a fork, will -- * abort the specific fork. -- */ -- s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * enable - Enable BPF scheduling for a task -- * @p: task to enable BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Enable @p for BPF scheduling. @p is now in the cgroup specified for -- * the preceding prep_enable() and will start running soon. -- */ -- void (*enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * cancel_enable - Cancel prep_enable() -- * @p: task being canceled -- * @args: enable arguments, see the struct definition -- * -- * @p was prep_enable()'d but failed before reaching enable(). Undo the -- * preparation. -- */ -- void (*cancel_enable)(struct task_struct *p, -- struct scx_enable_args *args); -- -- /** -- * disable - Disable BPF scheduling for a task -- * @p: task to disable BPF scheduling for -- * -- * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. -- * Disable BPF scheduling for @p. -- */ -- void (*disable)(struct task_struct *p); -- -- /* -- * All online ops must come before ops.init(). -- */ -- -- /** -- * init - Initialize the BPF scheduler -- */ -- s32 (*init)(void); -- -- /** -- * exit - Clean up after the BPF scheduler -- * @info: Exit info -- */ -- void (*exit)(struct scx_exit_info *info); -- -- /** -- * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch -- */ -- u32 dispatch_max_batch; -- -- /** -- * flags - %SCX_OPS_* flags -- */ -- u64 flags; -- -- /** -- * timeout_ms - The maximum amount of time, in milliseconds, that a -- * runnable task should be able to wait before being scheduled. The -- * maximum timeout may not exceed the default timeout of 30 seconds. -- * -- * Defaults to the maximum allowed timeout value of 30 seconds. -- */ -- u32 timeout_ms; -- -- /** -- * name - BPF scheduler's name -- * -- * Must be a non-zero valid BPF object name including only isalnum(), -- * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the -- * BPF scheduler is enabled. -- */ -- char name[SCX_OPS_NAME_LEN]; --}; -- --/* -- * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -- * scheduler core and the BPF scheduler. See the documentation for more details. -- */ --struct scx_dispatch_q { -- raw_spinlock_t lock; -- struct list_head fifo; /* processed in dispatching order */ -- struct rb_root_cached priq; /* processed in p->scx.dsq_vtime order */ -- u32 nr; -- u64 id; -- struct llist_node free_node; -- struct rcu_head rcu; --}; -- --/* scx_entity.flags */ --enum scx_ent_flags { -- SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ -- SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -- SCX_TASK_ENQ_LOCAL = 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */ -- -- SCX_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -- SCX_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -- -- SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -- SCX_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -- -- SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ --}; -- --/* scx_entity.dsq_flags */ --enum scx_ent_dsq_flags { -- SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ --}; -- --/* -- * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from -- * everywhere and the following bits track which kfunc sets are currently -- * allowed for %current. This simple per-task tracking works because SCX ops -- * nest in a limited way. BPF will likely implement a way to allow and disallow -- * kfuncs depending on the calling context which will replace this manual -- * mechanism. See scx_kf_allow(). -- */ --enum scx_kf_mask { -- SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ -- /* all non-sleepables may be nested inside INIT and SLEEPABLE */ -- SCX_KF_INIT = 1 << 0, /* running ops.init() */ -- SCX_KF_SLEEPABLE = 1 << 1, /* other sleepable init operations */ -- /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ -- SCX_KF_CPU_RELEASE = 1 << 2, /* ops.cpu_release() */ -- /* ops.dequeue (in REST) may be nested inside DISPATCH */ -- SCX_KF_DISPATCH = 1 << 3, /* ops.dispatch() */ -- SCX_KF_ENQUEUE = 1 << 4, /* ops.enqueue() */ -- SCX_KF_REST = 1 << 5, /* other rq-locked operations */ -- -- __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | -- SCX_KF_ENQUEUE | SCX_KF_REST, -- __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_REST, --}; -- --#define RAVG_HIST_SIZE 5 --struct scx_sched_task_stats { -- u64 mark_start; -- u64 window_start; -- u32 sum; -- u32 sum_history[RAVG_HIST_SIZE]; -- int cidx; -- -- u32 demand; -- u16 demand_scaled; -- void *sdsq; --}; -- -- -- --/* -- * The following is embedded in task_struct and contains all fields necessary -- * for a task to be scheduled by SCX. -- */ --struct sched_ext_entity { -- struct scx_dispatch_q *dsq; -- struct { -- struct list_head fifo; /* dispatch order */ -- struct rb_node priq; /* p->scx.dsq_vtime order */ -- } dsq_node; -- struct list_head watchdog_node; -- u32 flags; /* protected by rq lock */ -- u32 dsq_flags; /* protected by dsq lock */ -- u32 weight; -- s32 sticky_cpu; -- s32 holding_cpu; -- u32 kf_mask; /* see scx_kf_mask above */ -- struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ -- atomic64_t ops_state; -- unsigned long runnable_at; --#ifdef CONFIG_SCHED_CORE -- u64 core_sched_at; /* see scx_prio_less() */ --#endif -- -- /* BPF scheduler modifiable fields */ -- -- /* -- * Runtime budget in nsecs. This is usually set through -- * scx_bpf_dispatch() but can also be modified directly by the BPF -- * scheduler. Automatically decreased by SCX as the task executes. On -- * depletion, a scheduling event is triggered. -- * -- * This value is cleared to zero if the task is preempted by -- * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the -- * task ran. Use p->se.sum_exec_runtime instead. -- */ -- u64 slice; -- -- /* -- * Used to order tasks when dispatching to the vtime-ordered priority -- * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime() -- * but can also be modified directly by the BPF scheduler. Modifying it -- * while a task is queued on a dsq may mangle the ordering and is not -- * recommended. -- */ -- u64 dsq_vtime; -- -- /* -- * If set, reject future sched_setscheduler(2) calls updating the policy -- * to %SCHED_EXT with -%EACCES. -- * -- * If set from ops.prep_enable() and the task's policy is already -- * %SCHED_EXT, which can happen while the BPF scheduler is being loaded -- * or by inhering the parent's policy during fork, the task's policy is -- * rejected and forcefully reverted to %SCHED_NORMAL. The number of such -- * events are reported through /sys/kernel/debug/sched_ext::nr_rejected. -- */ -- bool disallow; /* reject switching into SCX */ -- -- /* cold fields */ -- struct list_head tasks_node; -- struct task_struct *task; -- struct scx_sched_task_stats sts; --}; -- --void sched_ext_free(struct task_struct *p); -- --#else /* !CONFIG_SCHED_CLASS_EXT */ -- --static inline void sched_ext_free(struct task_struct *p) {} -- --#endif /* CONFIG_SCHED_CLASS_EXT */ --#endif /* _LINUX_SCHED_EXT_H */ -diff --git a/include/linux/sched/hmbird.h b/include/linux/sched/hmbird.h -new file mode 100755 -index 000000000000..3c7861e37ade ---- /dev/null -+++ b/include/linux/sched/hmbird.h -@@ -0,0 +1,381 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class: Documentation/scheduler/hmbird.rst -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef _LINUX_SCHED_HMBIRD_H -+#define _LINUX_SCHED_HMBIRD_H -+ -+#define HMBIRD_TS_IDX 1 -+#define HMBIRD_OPS_IDX 14 -+#define HMBIRD_RQ_IDX 15 -+ -+#define get_hmbird_ts(p) \ -+ ((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ -+#define get_hmbird_rq(rq) \ -+ ((struct hmbird_rq *)(rq->android_oem_data1[HMBIRD_RQ_IDX])) -+ -+#define get_hmbird_ops(rq) \ -+ ((struct hmbird_ops *)(rq->android_oem_data1[HMBIRD_OPS_IDX])) -+ -+#define SCHED_PROP_DEADLINE_MASK (0xFF) /* deadline for ext sched class */ -+/* -+ * Every task has a DEADLINE_LEVEL which stands for -+ * max schedule latency this task can afford. LEVEL1~5 -+ * for user-aware tasks, LEVEL6~9 for other tasks. -+ */ -+#define SCHED_PROP_DEADLINE_LEVEL0 (0) -+#define SCHED_PROP_DEADLINE_LEVEL1 (1) -+#define SCHED_PROP_DEADLINE_LEVEL2 (2) -+#define SCHED_PROP_DEADLINE_LEVEL3 (3) -+#define SCHED_PROP_DEADLINE_LEVEL4 (4) -+#define SCHED_PROP_DEADLINE_LEVEL5 (5) -+#define SCHED_PROP_DEADLINE_LEVEL6 (6) -+#define SCHED_PROP_DEADLINE_LEVEL7 (7) -+#define SCHED_PROP_DEADLINE_LEVEL8 (8) -+#define SCHED_PROP_DEADLINE_LEVEL9 (9) -+/* -+ * Distinguish tasks into periodical tasks which requires -+ * low schedule latency and non-periodical tasks which are -+ * not sensitive to schedule latency. -+ */ -+#define SCHED_HMBIRD_DSQ_TYPE_PERIOD (0) /* period dsq of hmbird */ -+#define SCHED_HMBIRD_DSQ_TYPE_NON_PERIOD (1) /* non period dsq of hmbird */ -+ -+#define TOP_TASK_BITS_MASK (0xFF) -+#define TOP_TASK_BITS (8) -+#include -+ -+extern atomic_t non_hmbird_task; -+extern atomic_t __hmbird_ops_enabled; -+#define hmbird_enabled() atomic_read(&__hmbird_ops_enabled) -+#define MAX_GLOBAL_DSQS (10) -+ -+enum hmbird_consts { -+ HMBIRD_SLICE_DFL = 1 * NSEC_PER_MSEC, -+ HMBIRD_SLICE_ISO = 8 * HMBIRD_SLICE_DFL, -+ HMBIRD_SLICE_INF = U64_MAX, /* infinite, implies nohz */ -+}; -+ -+/* -+ * DSQ (dispatch queue) IDs are 64bit of the format: -+ * -+ * Bits: [63] [62 .. 0] -+ * [ B] [ ID ] -+ * -+ * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -+ * ID: 63 bit ID -+ * -+ * Built-in IDs: -+ * -+ * Bits: [63] [62] [61..32] [31 .. 0] -+ * [ 1] [ L] [ R ] [ V ] -+ * -+ * 1: 1 for built-in DSQs. -+ * L: 1 for LOCAL_ON DSQ IDs, 0 for others -+ * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -+ */ -+enum hmbird_dsq_id_flags { -+ HMBIRD_DSQ_FLAG_BUILTIN = 1LLU << 63, -+ HMBIRD_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -+ -+ HMBIRD_DSQ_INVALID = HMBIRD_DSQ_FLAG_BUILTIN | 0, -+ HMBIRD_DSQ_GLOBAL = HMBIRD_DSQ_FLAG_BUILTIN | 1, -+ HMBIRD_DSQ_LOCAL = HMBIRD_DSQ_FLAG_BUILTIN | 2, -+ HMBIRD_DSQ_LOCAL_ON = HMBIRD_DSQ_FLAG_BUILTIN | HMBIRD_DSQ_FLAG_LOCAL_ON, -+ HMBIRD_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, -+}; -+ -+enum hmbird_switch_type { -+ HMBIRD_SWITCH_PROC, -+ HMBIRD_SWITCH_ERR_WDT, -+ HMBIRD_SWITCH_ERR_HB, -+ HMBIRD_SWITCH_ERR_DSQ, -+ HMBIRD_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ -+ HMBIRD_EXIT_ERROR_HEARTBEAT, /* heart beat has stopped */ -+}; -+ -+/* -+ * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -+ * scheduler core and the BPF scheduler. See the documentation for more details. -+ */ -+struct hmbird_dispatch_q { -+ raw_spinlock_t lock; -+ struct list_head fifo; /* processed in dispatching order */ -+ struct rb_root_cached priq; -+ u32 nr; -+ u64 id; -+ struct llist_node free_node; -+ struct rcu_head rcu; -+ u64 last_consume_at; -+ bool is_timeout; -+}; -+ -+/* hmbird_entity.flags */ -+enum hmbird_ent_flags { -+ HMBIRD_TASK_QUEUED = 1 << 0, /* on hmbird runqueue */ -+ HMBIRD_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -+ HMBIRD_TASK_ENQ_LOCAL = 1 << 2, /* used by hmbird_select_cpu_dfl, set HMBIRD_ENQ_LOCAL */ -+ -+ HMBIRD_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -+ HMBIRD_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -+ -+ HMBIRD_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -+ HMBIRD_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -+ -+ HMBIRD_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ -+}; -+ -+/* hmbird_entity.dsq_flags */ -+enum hmbird_ent_dsq_flags { -+ HMBIRD_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ -+}; -+ -+#define RAVG_HIST_SIZE 5 -+struct hmbird_sched_task_stats { -+ u64 mark_start; -+ u64 window_start; -+ u32 sum; -+ u32 sum_history[RAVG_HIST_SIZE]; -+ int cidx; -+ -+ u32 demand; -+ u16 demand_scaled; -+ void *sdsq; -+}; -+ -+struct hmbird_sched_rq_stats { -+ u64 window_start; -+ u64 latest_clock; -+ u32 prev_window_size; -+ u64 task_exec_scale; -+ u64 prev_runnable_sum; -+ u64 curr_runnable_sum; -+ int *sched_ravg_window_ptr; -+}; -+ -+/* -+ * The following is embedded in task_struct and contains all fields necessary -+ * for a task to be scheduled by HMBIRD. -+ */ -+struct hmbird_entity { -+ struct hmbird_dispatch_q *dsq; -+ struct { -+ struct list_head fifo; /* dispatch order */ -+ struct rb_node priq; -+ } dsq_node; -+ struct list_head watchdog_node; -+ u32 flags; /* protected by rq lock */ -+ u32 dsq_flags; /* protected by dsq lock */ -+ u32 weight; -+ s32 sticky_cpu; -+ s32 holding_cpu; -+ u32 kf_mask; /* see hmbird_kf_mask above */ -+ struct task_struct *kf_tasks[2]; /* see HMBIRD_CALL_OP_TASK() */ -+ atomic64_t ops_state; -+ unsigned long runnable_at; -+ u64 slice; -+ u64 dsq_vtime; -+ bool disallow; /* reject switching into HMBIRD */ -+ u16 demand_scaled; -+ -+ /* cold fields */ -+ struct list_head tasks_node; -+ struct task_struct *task; -+ const struct sched_class *sched_class; -+ unsigned long sched_prop; -+ unsigned long top_task_prop; -+ struct hmbird_sched_task_stats sts; -+ unsigned long running_at; -+ int gdsq_idx; -+ -+ s32 critical_affinity_cpu; -+ int dsq_sync_ux; -+}; -+ -+ -+/* -+ * All variables use 64bits width, -+ * Avoid parsing problems caused by automatic alignment(with padding) of structures. -+ */ -+ -+/* NOTING : Must align to 64bits. */ -+#define DESC_STR_LEN (32) -+/* Supporti up to three-dimensional arrays. */ -+#define PARSE_DIMENS (3) -+struct meta_desc_t { -+ char desc_str[DESC_STR_LEN]; -+ u64 len; -+ u64 parse[PARSE_DIMENS]; -+}; -+ -+#define MAX_SWITCHS (5) -+struct hmbird_switch_t { -+ u64 switch_at; -+ u64 is_success; -+ u64 end_state; -+ u64 switch_reason; -+}; -+#define SWITCH_ITEMS (sizeof(struct hmbird_switch_t) / sizeof(u64)) -+ -+#define MAX_EXCEPS (5) -+enum excep_id { -+ NO_CGROUP_L1, -+ MODULE_UNLOAD, -+ ALREADY_ENABLED, -+ ALREADY_DISABLED, -+ INIT_TASK_FAIL, -+ ALLOC_RQSCX_FAIL, -+ DSQ_ID_ERR, -+ CPU_NO_MASK, -+ SCAN_ENTITY_NULL, -+ ITER_RET_NULL, -+ DEQ_DEQING, -+ ENQ_EXIST1, -+ ENQ_EXIST2, -+ TASK_LINKED1, -+ TASK_UNLINKED, -+ TASK_LINKED2, -+ TASK_UNQUED, -+ TASK_UNWATCHED, -+ HMBIRD_OPN, -+ TASK_WATCHED, -+ RQ_NO_RUNNING, -+ EXTRA_FLAGS, -+ HOLDING_CPU1, -+ HOLDING_CPU2, -+ TASK_OPS_PREPPED, -+ TASK_OPS_UNPREPPED, -+ HMBIRD_OPS_ERR, -+ MAX_EXCEP_ID, -+}; -+ -+struct snap_misc_t { -+ u64 hmbird_enabled; -+ u64 curr_ss; -+ u64 hmbird_ops_enable_state_var; -+ u64 non_ext_task; -+ u64 parctrl_high_ratio; -+ u64 parctrl_low_ratio; -+ u64 parctrl_high_ratio_l; -+ u64 parctrl_low_ratio_l; -+ u64 isoctrl_high_ratio; -+ u64 isoctrl_low_ratio; -+ u64 misfit_ds; -+ u64 partial_enable; -+ u64 iso_free_rescue; -+ u64 isolate_ctrl; -+ u64 snap_jiffies; -+ u64 snap_time; -+}; -+#define SNAP_ITEMS (sizeof(struct snap_misc_t) / sizeof(u64)) -+ -+struct panic_snapshot_t { -+ /* what time dose the first task of dsq turn in runnable, check starvation */ -+ struct meta_desc_t runnable_at_meta; -+ u64 runnable_at[MAX_GLOBAL_DSQS]; -+ -+ struct meta_desc_t rq_nr_meta; -+ u64 rq_nr[NR_CPUS]; -+ -+ struct meta_desc_t scxrq_nr_meta; -+ u64 scxrq_nr[NR_CPUS]; -+ -+ struct meta_desc_t snap_misc_meta; -+ struct snap_misc_t snap_misc; -+}; -+ -+/* -+ * Do not record info in hot paths unless absolutely necessary, -+ * The impact on performance should be minimized. -+ */ -+struct kernel_info_t { -+ struct meta_desc_t sw_rec_meta; -+ struct hmbird_switch_t sw_rec[MAX_SWITCHS]; -+ -+ struct meta_desc_t sw_idx_meta; -+ u64 sw_idx; -+ -+ struct meta_desc_t excep_rec_meta; -+ u64 excep_rec[MAX_EXCEP_ID][MAX_EXCEPS]; -+ -+ struct meta_desc_t excep_idx_meta; -+ u64 excep_idx[MAX_EXCEP_ID]; -+ -+ /* snapshot while panic. */ -+ struct panic_snapshot_t snap; -+}; -+ -+struct ko_info_t {}; -+ -+struct md_meta_t { -+ u64 self_len; -+ u64 unit_size; -+ u64 desc_meta_len; -+ u64 desc_str_len; -+ u64 switches; -+ u64 exceps; -+ u64 global_dsqs; -+ u64 parse_dimens; -+ u64 nr_cpus; -+ u64 real_cpus; -+ u64 nr_meta_desc; -+ u64 dump_real_size; -+}; -+ -+struct md_info_t { -+ struct md_meta_t meta; -+ struct kernel_info_t kern_dump; -+ struct ko_info_t ko_dump; -+}; -+ -+static inline void exceps_update(struct md_info_t *rec, int id, unsigned long jiffies) -+{ -+ u64 *idx; -+ -+ if (!rec) -+ return; -+ -+ idx = &rec->kern_dump.excep_idx[id]; -+ rec->kern_dump.excep_rec[id][*idx] = jiffies; -+ *idx = ++(*idx) % MAX_EXCEPS; -+} -+ -+static inline void sw_update(struct md_info_t *rec, u64 switch_at, -+ u64 is_success, u64 end_state, u64 switch_reason) -+{ -+ u64 *idx; -+ -+ if (!rec) -+ return; -+ -+ idx = &rec->kern_dump.sw_idx; -+ rec->kern_dump.sw_rec[*idx].switch_at = switch_at; -+ rec->kern_dump.sw_rec[*idx].is_success = is_success; -+ rec->kern_dump.sw_rec[*idx].end_state = end_state; -+ rec->kern_dump.sw_rec[*idx].switch_reason = switch_reason; -+ *idx = ++(*idx) % MAX_SWITCHS; -+} -+ -+struct hmbird_ops { -+ bool (*scx_enable)(void); -+ bool (*check_non_task)(void); -+ void (*do_sched_yield_before)(long *skip); -+ void (*window_rollover_run_once)(struct rq *rq); -+ void (*hmbird_get_md_info)(unsigned long *vaddr, unsigned long *size); -+}; -+ -+void hmbird_free(struct task_struct *p); -+ -+enum DSQ_SYNC_UX_FLAG { -+ DSQ_SYNC_UX_NONE = 0, -+ DSQ_SYNC_STATIC_UX = 1, -+ DSQ_SYNC_INHERIT_UX = 1 << 1, -+}; -+ -+#endif /* _LINUX_SCHED_HMBIRD_H */ -diff --git a/include/linux/sched/hmbird_proc_val.h b/include/linux/sched/hmbird_proc_val.h -new file mode 100755 -index 000000000000..e59708b16ea3 ---- /dev/null -+++ b/include/linux/sched/hmbird_proc_val.h -@@ -0,0 +1,22 @@ -+#ifndef __HMBORD_PROC_VAL_H__ -+#define __HMBORD_PROC_VAL_H__ -+ -+extern int scx_enable; -+extern int partial_enable; -+extern int cpuctrl_high_ratio; -+extern int cpuctrl_low_ratio; -+extern int slim_stats; -+extern int hmbirdcore_debug; -+extern int slim_for_app; -+extern int misfit_ds; -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+extern int cpu7_tl; -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int slim_gov_debug; -+extern int scx_gov_ctrl; -+extern int sched_ravg_window_frame_per_sec; -+ -+#endif -\ No newline at end of file -diff --git a/include/linux/sched/hmbird_version.h b/include/linux/sched/hmbird_version.h -new file mode 100755 -index 000000000000..b2843881c26f ---- /dev/null -+++ b/include/linux/sched/hmbird_version.h -@@ -0,0 +1,52 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#ifndef _OPLUS_HMBIRD_VERSION_H_ -+#define _OPLUS_HMBIRD_VERSION_H_ -+#include -+#include -+#include -+ -+enum hmbird_version { -+ HMBIRD_UNINIT, -+ HMBIRD_GKI_VERSION, -+ HMBIRD_OGKI_VERSION, -+ HMBIRD_UNKNOW_VERSION, -+}; -+ -+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT; -+ -+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type" -+ -+static inline enum hmbird_version get_hmbird_version_type(void) -+{ -+ struct device_node *np = NULL; -+ const char *hmbird_version_str = NULL; -+ if (HMBIRD_UNINIT != hmbird_version_type) -+ return hmbird_version_type; -+ np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH); -+ if (np) { -+ of_property_read_string(np, "type", &hmbird_version_str); -+ if (NULL != hmbird_version_str) { -+ if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) { -+ hmbird_version_type = HMBIRD_OGKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi"); -+ } else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) { -+ hmbird_version_type = HMBIRD_GKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi"); -+ } else { -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi"); -+ } -+ return hmbird_version_type; -+ } -+ } -+ -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION"); -+ return hmbird_version_type; -+} -+ -+#endif /*_OPLUS_HMBIRD_VERSION_H_ */ -diff --git a/include/linux/sched/sa_common.h b/include/linux/sched/sa_common.h -new file mode 100755 -index 000000000000..6f76d7cfd7bf ---- /dev/null -+++ b/include/linux/sched/sa_common.h -@@ -0,0 +1,797 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_COMMON_H_ -+#define _OPLUS_SA_COMMON_H_ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+#include -+#endif -+#include -+#include -+#include -+#include -+#include -+ -+#include "sa_oemdata.h" -+#include "sa_common_struct.h" -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) -+#define OPLUS_UX_EEVDF_COMPATIBLE 1 -+#endif -+ -+#define SA_DEBUG_ON 0 -+ -+#if (SA_DEBUG_ON >= 1) -+#define DEBUG_BUG_ON(x) BUG_ON(x) -+#define DEBUG_WARN_ON(x) WARN_ON(x) -+#else -+#define DEBUG_BUG_ON(x) -+#define DEBUG_WARN_ON(x) -+#endif -+ -+#define ux_err(fmt, ...) \ -+ pr_err("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_warn(fmt, ...) \ -+ pr_warn("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_debug(fmt, ...) \ -+ pr_info("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+ -+#define UX_MSG_LEN 64 -+#define UX_DEPTH_MAX 5 -+ -+/* define for debug */ -+#define DEBUG_SYSTRACE (1 << 0) -+#define DEBUG_FTRACE (1 << 1) -+#define DEBUG_KMSG (1 << 2) /* used for frameboost */ -+#define DEBUG_PIPELINE (1 << 3) -+#define DEBUG_DYNAMIC_HZ (1 << 4) -+#define DEBUG_DYNAMIC_PREEMPT (1 << 5) -+#define DEBUG_AMU_INSTRUCTION (1 << 6) -+#define DEBUG_VERBOSE (1 << 10) /* used for frameboost */ -+#define DEBUG_SET_DSQ_ID (1 << 11) /* used for hmbird */ -+ -+/* define for sched assist feature */ -+#define FEATURE_COMMON (1 << 0) -+#define FEATURE_SPREAD (1 << 1) -+ -+#define UX_EXEC_SLICE (4000000U) -+ -+/* define for sched assist thread type, keep same as the define in java file */ -+#define SA_OPT_CLEAR (0) -+#define SA_TYPE_LIGHT (1 << 0) -+#define SA_TYPE_HEAVY (1 << 1) -+#define SA_TYPE_ANIMATOR (1 << 2) -+/* SA_TYPE_LISTPICK for camera */ -+#define SA_TYPE_LISTPICK (1 << 3) -+#define SA_TYPE_MQ_VIP (1 << 4) -+#define SA_OPT_SET (1 << 7) -+#define SA_OPT_RESET (1 << 8) -+#define SA_OPT_SET_PRIORITY (1 << 9) -+ -+/* The following ux value only used in kernel */ -+#define SA_TYPE_SWIFT (1 << 14) -+/* clear ux type when dequeue */ -+#define SA_TYPE_ONCE (1 << 15) -+#define SA_TYPE_INHERIT (1 << 16) -+/* SA_TYPE_COMBINED isn't marked in ux_state, it only exists in return value of function */ -+#define SA_TYPE_COMBINED (1 << 17) -+#define SA_TYPE_URGENT_MASK (SA_TYPE_LIGHT|SA_TYPE_ANIMATOR|SA_TYPE_SWIFT) -+#define SCHED_ASSIST_UX_MASK (SA_TYPE_LIGHT|SA_TYPE_HEAVY|SA_TYPE_ANIMATOR|SA_TYPE_LISTPICK|SA_TYPE_SWIFT) -+ -+/* load balance operation is performed on the following ux types */ -+#define SCHED_ASSIST_LB_UX (SA_TYPE_SWIFT | SA_TYPE_ANIMATOR | SA_TYPE_LIGHT | SA_TYPE_HEAVY) -+#define POSSIBLE_UX_MASK (SA_TYPE_SWIFT | \ -+ SA_TYPE_LIGHT | \ -+ SA_TYPE_HEAVY | \ -+ SA_TYPE_ANIMATOR | \ -+ SA_TYPE_LISTPICK | \ -+ SA_TYPE_ONCE | \ -+ SA_TYPE_INHERIT) -+ -+#define SCHED_ASSIST_UX_PRIORITY_MASK (0xFF000000) -+#define SCHED_ASSIST_UX_PRIORITY_SHIFT 24 -+ -+#define UX_PRIORITY_TOP_APP 0x0A000000 -+#define UX_PRIORITY_AUDIO 0x0A000000 -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+#define UX_PRIORITY_PIPELINE_UI 0x06000000 -+#define UX_PRIORITY_PIPELINE 0x05000000 -+#endif -+ -+/* define for sched assist scene type, keep same as the define in java file */ -+#define SA_SCENE_OPT_CLEAR (0) -+#define SA_LAUNCH (1 << 0) -+#define SA_SLIDE (1 << 1) -+#define SA_CAMERA (1 << 2) -+#define SA_ANIM_START (1 << 3) /* we care about both launcher and top app */ -+#define SA_ANIM (1 << 4) /* we only care about launcher */ -+#define SA_INPUT (1 << 5) -+#define SA_LAUNCHER_SI (1 << 6) -+#define SA_SCENE_OPT_SET (1 << 7) -+ -+#define ROOT_UID 0 -+#define SYSTEM_UID 1000 -+#define FIRST_APPLICATION_UID 10000 -+#define LAST_APPLICATION_UID 19999 -+#define PER_USER_RANGE 100000 -+/* define for clear IM_FLAG -+if im_flag is 70, it should clear im_flag_audio (70 - 64 = 6) -+eg: gerrit patchset "30438485" -+ */ -+#define IM_FLAG_CLEAR 64 -+ -+extern pid_t save_audio_tgid; -+extern pid_t save_top_app_tgid; -+extern unsigned int top_app_type; -+extern int global_lowend_plat_opt; -+ -+#ifdef CONFIG_OPLUS_SCHED_HALT_MASK_PRT -+/* This must be the same as the definition of pause_type in walt_halt.c */ -+enum oplus_pause_type { -+ OPLUS_HALT, -+ OPLUS_PARTIAL_HALT, -+ -+ OPLUS_MAX_PAUSE_TYPE -+}; -+extern cpumask_t cur_cpus_halt_mask; -+extern cpumask_t cur_cpus_phalt_mask; -+DECLARE_PER_CPU(int[OPLUS_MAX_PAUSE_TYPE], oplus_cur_pause_client); -+extern void sa_corectl_systrace_c(void); -+#endif /* CONFIG_OPLUS_SCHED_HALT_MASK_PRT */ -+ -+/* define for boost threshold unit */ -+#define BOOST_THRESHOLD_UNIT (51) -+ -+ -+enum UX_STATE_TYPE { -+ UX_STATE_INVALID = 0, -+ UX_STATE_NONE, -+ UX_STATE_STATIC, -+ UX_STATE_INHERIT, -+ UX_STATE_COMBINED, -+ MAX_UX_STATE_TYPE, -+}; -+ -+enum INHERIT_UX_TYPE { -+ INHERIT_UX_BINDER = 0, -+ INHERIT_UX_RWSEM, -+ INHERIT_UX_MUTEX, -+ INHERIT_UX_FUTEX, -+ INHERIT_UX_PIFUTEX, -+ INHERIT_UX_MAX, -+}; -+ -+/* -+ * WANNING: -+ * new flag should be add before MAX_IM_FLAG_TYPE, never change -+ * the value of those existed flag type. -+ */ -+enum IM_FLAG_TYPE { -+ IM_FLAG_NONE = 0, -+ IM_FLAG_SURFACEFLINGER, -+ IM_FLAG_HWC, /* Discarded */ -+ IM_FLAG_RENDERENGINE, -+ IM_FLAG_WEBVIEW, -+ IM_FLAG_CAMERA_HAL, -+ IM_FLAG_AUDIO, -+ IM_FLAG_HWBINDER, -+ IM_FLAG_LAUNCHER, -+ IM_FLAG_LAUNCHER_NON_UX_RENDER, -+ IM_FLAG_SS_LOCK_OWNER, -+ IM_FLAG_FORBID_SET_CPU_AFFINITY = 11, /* forbid setting cpu affinity from app */ -+ IM_FLAG_SYSTEMSERVER_PID, -+ IM_FLAG_MIDASD, -+ IM_FLAG_AUDIO_CAMERA_HAL, /* audio mode disable camera hal ux */ -+ IM_FLAG_AFFINITY_THREAD, -+ IM_FLAG_TPD_SET_CPU_AFFINITY = 16, -+ IM_FLAG_COMPRESS_THREAD = 17, /* compress thread skips locking protect */ -+ IM_FLAG_RENDER_THREAD = 18, -+ MAX_IM_FLAG_TYPE, -+}; -+ -+#define MAX_IM_FLAG_PRIO MAX_IM_FLAG_TYPE -+ -+enum ots_state { -+ OTS_STATE_SET_AFFINITY, -+ OTS_STATE_DDL_ACTIVE, -+ OTS_STATE_DDL_ACTIVE_PREEMPTED, -+ OTS_STATE_MAX, -+}; -+ -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+DECLARE_PER_CPU(u64, retired_instrs); -+DECLARE_PER_CPU(u64, nvcsw); -+DECLARE_PER_CPU(u64, nivcsw); -+#endif -+ -+struct ux_sched_cluster { -+ struct cpumask cpus; -+ unsigned long capacity; -+}; -+ -+#define OPLUS_NR_CPUS (8) -+#define OPLUS_MAX_CLS (5) -+struct ux_sched_cputopo { -+ int cls_nr; -+ struct ux_sched_cluster sched_cls[OPLUS_NR_CPUS]; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ cpumask_t oplus_cpu_array[2*OPLUS_MAX_CLS][OPLUS_MAX_CLS]; -+#endif -+}; -+ -+struct oplus_rq { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_root_cached ux_list; -+ /* a tree to track minimum exec vruntime */ -+ struct rb_root_cached exec_timeline; -+ /* malloc this spinlock_t instead of built-in to shrank size of oplus_rq */ -+ spinlock_t *ux_list_lock; -+ int nr_running; -+ u64 min_vruntime; -+ u64 load_weight; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) -+ struct rb_root_cached ddl_root; -+ spinlock_t *ddl_lock; -+#endif -+ -+#ifdef CONFIG_LOCKING_PROTECT -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ struct list_head locking_thread_list; -+ spinlock_t *locking_list_lock; -+ int rq_locking_task; -+#else -+ struct sched_entity *last_entity; -+#endif -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ struct oplus_lb lb; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+ struct hrtimer *resched_timer; -+ int cpu; -+#endif -+}; -+ -+extern int global_debug_enabled; -+extern int global_sched_assist_enabled; -+extern int global_sched_assist_scene; -+extern int global_silver_perf_core; -+extern int global_sched_group_enabled; -+ -+struct rq; -+ -+#ifdef CONFIG_LOCKING_PROTECT -+struct sched_assist_locking_ops { -+ void (*check_preempt_tick)(struct task_struct *p, -+ unsigned long *ideal_runtime, bool *skip_preempt, -+ unsigned long delta_exec, struct cfs_rq *cfs_rq, -+ struct sched_entity *curr, unsigned int granularity); -+ void (*check_preempt_wakeup)(struct rq *rq, struct task_struct *p, bool *preempt, bool *nopreempt); -+ void (*state_systrace_c)(unsigned int cpu, struct task_struct *p); -+ void (*locking_tick_hit)(struct task_struct *prev, struct task_struct *next); -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ void (*replace_next_task_fair)(struct rq *rq, -+ struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*enqueue_entity)(struct rq *rq, struct task_struct *p); -+ void (*dequeue_entity)(struct rq *rq, struct task_struct *p); -+#else -+ void (*set_last_entity)(struct task_struct *prev, struct task_struct *next, struct rq *rq); -+ void (*pick_last_entity)(struct rq *rq, struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*clear_last_entity)(struct rq *rq, struct task_struct *p); -+#endif -+ void (*opt_ss_lock_contention)(struct task_struct *p, int old_im, int new_im); -+}; -+ -+extern struct sched_assist_locking_ops *locking_ops; -+ -+ -+#define LOCKING_CALL_OP(op, args...) \ -+do { \ -+ if (locking_ops && locking_ops->op) { \ -+ locking_ops->op(args); \ -+ } \ -+} while (0) -+ -+#define LOCKING_CALL_OP_RET(op, args...) \ -+({ \ -+ __typeof__(locking_ops->op(args)) __ret = 0; \ -+ if (locking_ops && locking_ops->op) \ -+ __ret = locking_ops->op(args); \ -+ __ret; \ -+}) -+ -+void register_sched_assist_locking_ops(struct sched_assist_locking_ops *ops); -+#endif -+ -+/* attention: before insert .ko, task's list->prev/next is init with 0 */ -+static inline bool oplus_list_empty(struct list_head *list) -+{ -+ return list_empty(list) || (list->prev == 0 && list->next == 0); -+} -+ -+static inline bool oplus_rbtree_empty(struct rb_root_cached *list) -+{ -+ return (rb_first_cached(list) == NULL); -+} -+ -+/** -+ * Check if there are ux tasks waiting to run on the specified cpu -+ */ -+static inline bool orq_has_ux_tasks(struct oplus_rq *orq) -+{ -+ return !oplus_rbtree_empty(&orq->ux_list); -+} -+ -+/* attention: before insert .ko, task's node->__rb_parent_color is init with 0 */ -+static inline bool oplus_rbnode_empty(struct rb_node *node) -+{ -+ return RB_EMPTY_NODE(node) || (node->__rb_parent_color == 0); -+} -+ -+static inline struct oplus_task_struct *ux_list_first_entry(struct rb_root_cached *list) -+{ -+ struct rb_node *leftmost = rb_first_cached(list); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, ux_entry); -+} -+ -+static inline struct oplus_task_struct *exec_timeline_first_entry(struct rb_root_cached *tree) -+{ -+ struct rb_node *leftmost = rb_first_cached(tree); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, exec_time_node); -+} -+ -+typedef bool (*migrate_task_callback_t)(struct task_struct *tsk, int src_cpu, int dst_cpu); -+extern migrate_task_callback_t fbg_migrate_task_callback; -+typedef void (*android_rvh_schedule_handler_t)(struct task_struct *prev, -+ struct task_struct *next, struct rq *rq); -+extern android_rvh_schedule_handler_t fbg_android_rvh_schedule_callback; -+ -+extern struct kmem_cache *oplus_task_struct_cachep; -+ -+#define ots_to_ts(ots) (ots->task) -+#define OTS_IDX 0 -+#define ORQ_IDX 0 -+ -+static inline bool test_task_is_fair(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid CFS priority is MAX_RT_PRIO..MAX_PRIO-1 */ -+ if ((task->prio >= MAX_RT_PRIO) && (task->prio <= MAX_PRIO-1)) -+ return true; -+ return false; -+} -+ -+static inline bool test_task_is_rt(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid RT priority is 0..MAX_RT_PRIO-1 */ -+ if (rt_prio(task->prio)) -+ return true; -+ -+ return false; -+} -+ -+static inline struct oplus_task_struct *get_oplus_task_struct(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = NULL; -+ -+ /* not Skip idle thread */ -+ if (!t) -+ return NULL; -+ -+ ots = (struct oplus_task_struct *) READ_ONCE(t->android_oem_data1[OTS_IDX]); -+ if (IS_ERR_OR_NULL(ots)) -+ return NULL; -+ -+ return ots; -+} -+ -+static inline unsigned long oplus_get_im_flag(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return IM_FLAG_NONE; -+ -+ return ots->im_flag; -+} -+ -+static inline bool is_optimized_audio_thread(struct task_struct *t) -+{ -+ unsigned long im_flag; -+ -+ im_flag = oplus_get_im_flag(t); -+ if (test_bit(IM_FLAG_AUDIO, &im_flag)) -+ return true; -+ -+ return false; -+} -+ -+static inline void oplus_set_im_flag(struct task_struct *t, int im_flag) -+{ -+ struct oplus_task_struct *ots = NULL; -+ ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ set_bit(im_flag, &ots->im_flag); -+} -+ -+static inline int get_ots_ux_state(struct oplus_task_struct *ots) -+{ -+ int ux_state; -+ int sub_ux_state; -+ int ux_prio; -+ int ret; -+ -+ ux_prio = ots->ux_state & SCHED_ASSIST_UX_PRIORITY_MASK; -+ ux_state = ots->ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ sub_ux_state = ots->sub_ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ -+ if (sub_ux_state) { -+ ret = ux_state | (sub_ux_state & ~SA_TYPE_INHERIT) | SA_TYPE_COMBINED; -+ } else { -+ ret = ux_state; -+ } -+ -+ if (ret & SCHED_ASSIST_UX_MASK) { -+ return ux_prio | ret; -+ } -+ -+ return 0; -+} -+ -+bool is_multiple_ux(struct oplus_task_struct *ots); -+ -+static inline int oplus_get_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return get_ots_ux_state(ots); -+} -+ -+static inline int oplus_get_static_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return 0; -+ } -+ return ots->ux_state; -+} -+ -+static inline int oplus_get_sub_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->sub_ux_state; -+} -+ -+static inline int oplus_get_inherited_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return ots->ux_state; -+ } -+ return ots->sub_ux_state; -+} -+ -+void oplus_set_ux_state_lock(struct task_struct *t, int ux_state, int inherit_type, bool need_lock_rq); -+ -+static inline s64 oplus_get_inherit_ux(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return atomic64_read(&ots->inherit_ux); -+} -+ -+static inline void oplus_set_inherit_ux(struct task_struct *t, s64 inherit_ux) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ atomic64_set(&ots->inherit_ux, inherit_ux); -+} -+ -+static inline int oplus_get_ux_depth(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->ux_depth; -+} -+ -+static inline void oplus_set_ux_depth(struct task_struct *t, int ux_depth) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->ux_depth = ux_depth; -+} -+ -+static inline u64 oplus_get_enqueue_time(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->enqueue_time; -+} -+ -+static inline void oplus_set_enqueue_time(struct task_struct *t, u64 enqueue_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->enqueue_time = enqueue_time; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* -+ * Record the number of task context switches -+ * and scheduling delays when enqueueing a task. -+ */ -+ ots->snap_pcount = t->sched_info.pcount; -+ ots->snap_run_delay = t->sched_info.run_delay; -+#endif -+} -+ -+static inline void oplus_set_inherit_ux_start(struct task_struct *t, u64 start_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->inherit_ux_start = t->se.sum_exec_runtime; -+} -+ -+static inline void init_task_ux_info(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ RB_CLEAR_NODE(&ots->ux_entry); -+ RB_CLEAR_NODE(&ots->exec_time_node); -+ ots->ux_state = 0; -+ ots->sub_ux_state = 0; -+ atomic64_set(&ots->inherit_ux, 0); -+ ots->ux_depth = 0; -+ ots->enqueue_time = 0; -+ ots->inherit_ux_start = 0; -+ ots->ux_priority = -1; -+ ots->ux_nice = -1; -+ ots->vruntime = 0; -+ ots->preset_vruntime = 0; -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG) -+ ots->abnormal_flag = 0; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_SCHED_SPREAD -+ ots->lb_state = 0; -+ ots->ld_flag = 0; -+#endif -+ ots->exec_calc_runtime = 0; -+ ots->is_update_runtime = 0; -+ ots->target_process = -1; -+ ots->wake_tid = 0; -+ ots->running_start_time = 0; -+ ots->update_running_start_time = false; -+ ots->last_wake_ts = 0; -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ memset(&ots->lkinfo, 0, sizeof(struct locking_info)); -+ INIT_LIST_HEAD(&ots->lkinfo.node); -+/*#endif*/ -+ ots->block_start_time = 0; -+#ifdef CONFIG_LOCKING_PROTECT -+ INIT_LIST_HEAD(&ots->locking_entry); -+ ots->locking_start_time = 0; -+ ots->locking_depth = 0; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ ots->snap_pcount = 0; -+ ots->snap_run_delay = 0; -+ plist_node_init(&ots->rtb, MAX_IM_FLAG_PRIO); -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+ atomic_set(&ots->pipeline_cpu, -1); -+#endif -+ -+#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO) -+ ots->amu_cycle = 0; -+ ots->amu_instruct = 0; -+#endif -+}; -+ -+static inline bool test_ux_type(struct task_struct *task, int ux_type) -+{ -+ return oplus_get_ux_state(task) & ux_type; -+} -+ -+static inline bool is_heavy_ux_task(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return false; -+ -+ return get_ots_ux_state(ots) & SA_TYPE_HEAVY; -+} -+ -+static inline bool sched_assist_scene(unsigned int scene) -+{ -+ if (unlikely(!global_sched_assist_enabled)) -+ return false; -+ -+ return global_sched_assist_scene & scene; -+} -+ -+static inline unsigned long oplus_task_util(struct task_struct *p) -+{ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+ struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; -+ -+ return wts->demand_scaled; -+#else -+ return READ_ONCE(p->se.avg.util_avg); -+#endif -+} -+ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+static inline u32 task_wts_sum(struct task_struct *tsk) -+{ -+ struct walt_task_struct *wts = (struct walt_task_struct *) tsk->android_vendor_data1; -+ -+ return wts->sum; -+} -+#endif -+ -+bool is_min_cluster(int cpu); -+bool is_max_cluster(int cpu); -+bool is_mid_cluster(int cpu); -+bool im_mali(const char *comm); -+bool is_top(struct task_struct *p); -+bool task_is_runnable(struct task_struct *task); -+struct oplus_rq *get_oplus_rq(struct rq *rq); -+ -+typedef unsigned long (*kallsyms_lookup_name_t)(const char *name); -+extern kallsyms_lookup_name_t _kallsyms_lookup_name; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+int ux_mask_to_prio(int ux_mask); -+int ux_prio_to_mask(int prio); -+#endif -+ -+noinline int tracing_mark_write(const char *buf); -+void hwbinder_systrace_c(unsigned int cpu, int flag); -+void sched_assist_init_oplus_rq(void); -+void queue_ux_thread(struct rq *rq, struct task_struct *p, int enqueue); -+ -+void inherit_ux_inc(struct task_struct *task, int type); -+void inherit_ux_sub(struct task_struct *task, int type, int value); -+void set_inherit_ux(struct task_struct *task, int type, int depth, int inherit_val); -+void reset_inherit_ux(struct task_struct *inherit_task, struct task_struct *ux_task, int reset_type); -+void unset_inherit_ux(struct task_struct *task, int type); -+void unset_inherit_ux_value(struct task_struct *task, int type, int value); -+void inc_inherit_ux_refs(struct task_struct *task, int type); -+void clear_all_inherit_type(struct task_struct *p); -+int get_max_inherit_gran(struct task_struct *p); -+ -+bool is_heavy_load_top_task(struct task_struct *p); -+bool test_task_is_fair(struct task_struct *task); -+bool test_task_is_rt(struct task_struct *task); -+ -+bool test_task_ux(struct task_struct *task); -+bool test_task_ux_depth(int ux_depth); -+bool test_inherit_ux(struct task_struct *task, int type); -+bool test_set_inherit_ux(struct task_struct *task); -+int get_ux_state_type(struct task_struct *task); -+void sched_assist_target_comm(struct task_struct *task, const char *comm); -+unsigned int ux_task_exec_limit(struct task_struct *p); -+ -+void update_ux_sched_cputopo(void); -+bool is_task_util_over(struct task_struct *tsk, int threshold); -+bool oplus_task_misfit(struct task_struct *tsk, int cpu); -+ssize_t oplus_show_cpus(const struct cpumask *mask, char *buf); -+void adjust_rt_lowest_mask(struct task_struct *p, struct cpumask *local_cpu_mask, int ret, bool force_adjust); -+bool sa_skip_rt_sync(struct rq *rq, struct task_struct *p, bool *sync); -+void ux_state_systrace_c(unsigned int cpu, struct task_struct *p); -+bool sa_rt_skip_ux_cpu(int cpu); -+int is_vip_mvp(struct task_struct *p); -+ -+/* s64 account_ux_runtime(struct rq *rq, struct task_struct *curr); */ -+void opt_ss_lock_contention(struct task_struct *p, unsigned long old_im, int new_im); -+ -+/* register vender hook in kernel/sched/topology.c */ -+void android_vh_build_sched_domains_handler(void *unused, bool has_asym); -+ -+/* register vender hook in kernel/sched/rt.c */ -+void android_rvh_select_task_rq_rt_handler(void *unused, struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags, int *new_cpu); -+void android_rvh_find_lowest_rq_handler(void *unused, struct task_struct *p, struct cpumask *local_cpu_mask, int ret, int *best_cpu); -+ -+/* register vender hook in kernel/sched/core.c */ -+void android_rvh_sched_fork_handler(void *unused, struct task_struct *p); -+void android_rvh_after_enqueue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_dequeue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_schedule_handler(void *unused, struct task_struct *prev, struct task_struct *next, struct rq *rq); -+void android_vh_scheduler_tick_handler(void *unused, struct rq *rq); -+ -+void android_vh_account_process_tick_gran_handler(void *unused, struct task_struct *p, struct rq *rq, int user_tick, int *ticks); -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+void sa_sched_switch_handler(void *unused, bool preempt, struct task_struct *prev, struct task_struct *next, unsigned int prev_state); -+#endif -+ -+void set_im_flag_with_bit(int im_flag, struct task_struct *task); -+ -+/* register vendor hook in kernel/cgroup/cgroup-v1.c */ -+void android_vh_cgroup_set_task_handler(void *unused, int ret, struct task_struct *task); -+/* register vendor hook in kernel/signal.c */ -+void android_vh_exit_signal_handler(void *unused, struct task_struct *p); -+void android_rvh_set_cpus_allowed_by_task_handler(void *unused, const struct cpumask *cpu_valid_mask, -+ const struct cpumask *new_mask, struct task_struct *task, unsigned int *dest_cpu); -+void android_rvh_setscheduler_handler(void *unused, struct task_struct *p); -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_BAN_APP_SET_AFFINITY) -+void android_vh_sched_setaffinity_early_handler(void *unused, struct task_struct *task, const struct cpumask *new_mask, bool *skip); -+#endif -+ -+#ifdef CONFIG_OPLUS_SCHED_GROUP_OPT -+void android_vh_reweight_entity_handler(void *unused, struct sched_entity *se); -+#endif -+ -+#ifdef CONFIG_BLOCKIO_UX_OPT -+int sa_blockio_init(void); -+void sa_blockio_exit(void); -+#endif -+extern struct notifier_block process_exit_notifier_block; -+#endif /* _OPLUS_SA_COMMON_H_ */ -diff --git a/include/linux/sched/sa_common_struct.h b/include/linux/sched/sa_common_struct.h -new file mode 100755 -index 000000000000..876feff48b36 ---- /dev/null -+++ b/include/linux/sched/sa_common_struct.h -@@ -0,0 +1,277 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+/* -+this file is splited from the sa_common.h to adapt the OKI, -+IS_ENABLED is not allowed in here, because the macro will not work in OKI. -+*/ -+#ifndef _OPLUS_SA_COMMON_STRUCT_H_ -+#define _OPLUS_SA_COMMON_STRUCT_H_ -+ -+#define MAX_CLUSTER (4) -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+/* hot-thread */ -+struct task_record { -+#define RECOED_WINSIZE (1 << 8) -+#define RECOED_WINIDX_MASK (RECOED_WINSIZE - 1) -+ u8 winidx; -+ u8 count; -+}; -+ -+#define MAX_TASK_COMM_LEN 256 -+struct uid_struct { -+ uid_t uid; -+ u64 uid_total_cycle; -+ u64 uid_total_inst; -+ spinlock_t lock; -+ char leader_comm[TASK_COMM_LEN]; -+ char cmdline[MAX_TASK_COMM_LEN]; -+}; -+ -+struct amu_uid_entry { -+ uid_t uid; -+ struct uid_struct *uid_struct; -+ struct hlist_node node; -+}; -+ -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+struct locking_info { -+ u64 waittime_stamp; -+ u64 holdtime_stamp; -+ /* Used in torture acquire latency statistic.*/ -+ u64 acquire_stamp; -+ /* -+ * mutex or rwsem optimistic spin start time. Because a task -+ * can't spin both on mutex and rwsem at one time, use one common -+ * threshold time is OK. -+ */ -+ u64 opt_spin_start_time; -+ struct task_struct *holder; -+ u32 waittype; -+ bool ux_contrib; -+ /* -+ * Whether task is ux when it's going to be added to mutex or -+ * rwsem waiter list. It helps us check whether there is ux -+ * task on mutex or rwsem waiter list. Also, a task can't be -+ * added to both mutex and rwsem at one time, so use one common -+ * field is OK. -+ */ -+ bool is_block_ux; -+ u32 kill_flag; -+ /* for cfs enqueue smoothly.*/ -+ struct list_head node; -+ struct task_struct *owner; -+ struct list_head lock_head; -+ u64 clear_seq; -+ atomic_t lock_depth; -+}; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+#define RAVG_HIST_SIZE 5 -+#define SCX_SLICE_DFL (1 * NSEC_PER_MSEC) -+#define SCX_SLICE_INF U64_MAX -+#define DEFAULT_CGROUP_DL_IDX (8) -+#define EXT_FLAG_RT_CHANGED (1 << 0) -+#define EXT_FLAG_CFS_CHANGED (1 << 1) -+struct scx_task_stats { -+ u64 mark_start; -+ u64 window_start; -+ u32 sum; -+ u32 sum_history[RAVG_HIST_SIZE]; -+ int cidx; -+ u32 demand; -+ u16 demand_scaled; -+ void *sdsq; -+}; -+/* -+ * The following is embedded in task_struct and contains all fields necessary -+ * for a task to be scheduled by SCX. -+ */ -+struct scx_entity { -+ struct scx_dispatch_q *dsq; -+ struct { -+ struct list_head fifo; /* dispatch order */ -+ struct rb_node priq; /* p->scx.dsq_vtime order */ -+ } dsq_node; -+ u32 flags; /* protected by rq lock */ -+ u32 dsq_flags; /* protected by dsq lock */ -+ s32 sticky_cpu; -+ unsigned long runnable_at; -+ u64 slice; -+ u64 dsq_vtime; -+ int gdsq_idx; -+ int ext_flags; -+ int prio_backup; -+ unsigned long sched_prop; -+ struct scx_task_stats sts; -+}; -+/*#endif*/ -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ -+#define MAX_CPU_FREQ_STATE 32 -+#define MAX_CPU_CNT 8 -+ -+#define STATE_IN_POOL 0 -+#define STATE_ACTIVE 1 -+ -+struct powermodel_freq_task_state { -+ u8 powermodel_enqueued:1; -+ u8 powermodel_index:7; -+}; -+ -+struct powermodel_cpu_task_state { -+ struct oplus_task_struct *ots; -+ struct powermodel_freq_task_state powermodel_freq_task_states[MAX_CPU_FREQ_STATE]; -+ u64 powermodel_last_seq; -+ u64 last_seq; -+ struct list_head node; -+ int state; -+ int pool_id; -+}; -+ -+#endif -+ -+ -+/* Please add your own members of task_struct here :) */ -+struct oplus_task_struct { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_node ux_entry; -+ struct rb_node exec_time_node; -+ struct task_struct *task; -+ atomic64_t inherit_ux; -+ u64 enqueue_time; -+ u64 inherit_ux_start; -+ /* u64 sum_exec_baseline; */ -+ u64 total_exec; -+ u64 vruntime; -+ u64 preset_vruntime; -+ /* contains ux state -+ 1. if static and inherited ux both exist, static ux stores in ux_state, inherited ux in sub_ux_state. -+ 2. if only static ux exists, static ux stores in ux_state. -+ 2. if only inherited ux exists, inherited ux stores in ux_state */ -+ int ux_state; -+ int sub_ux_state; -+ u8 ux_depth; -+ s8 ux_priority; -+ s8 ux_nice; -+ pid_t affinity_pid; -+ pid_t affinity_tgid; -+ unsigned long state; -+ unsigned long im_flag; -+ atomic_t is_vip_mvp; -+ -+/* #if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) */ -+ u64 ddl; -+ u64 ddl_active_ts; -+ u64 runnable_ts; -+ struct rb_node ddl_node; -+/* #endif */ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG)*/ -+ int abnormal_flag; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_SCHED_SPREAD */ -+ int lb_state; -+ int ld_flag:1; -+ /* CONFIG_OPLUS_FEATURE_TASK_LOAD */ -+ int is_update_runtime:1; -+ int target_process; -+ u64 wake_tid; -+ u64 running_start_time; -+ bool update_running_start_time; -+ u64 exec_calc_runtime; -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct task_record record[MAX_CLUSTER]; /* 2*u64 */ -+ u64 block_start_time; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_FRAME_BOOST */ -+ struct list_head fbg_list; -+ raw_spinlock_t fbg_list_entry_lock; -+ bool fbg_running; /* task belongs to a group, and in running */ -+ u16 fbg_state; -+ s8 preferred_cluster_id; -+ s8 fbg_depth; -+ u64 last_wake_ts; -+ int fbg_cur_group; -+/*#ifdef CONFIG_LOCKING_PROTECT*/ -+ unsigned long locking_start_time; -+ unsigned long last_jiffies; -+ struct list_head locking_entry; -+ int locking_depth; -+ int lk_tick_hit; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ struct locking_info lkinfo; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+ struct scx_entity scx; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_FDLEAK_CHECK)*/ -+ u8 fdleak_flag; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+ /* for loadbalance */ -+ struct plist_node rtb; /* rt boost task */ -+ -+ /* -+ * The following variables are used to calculate the time -+ * a task spends in the running/runnable state. -+ */ -+ u64 snap_run_delay; -+ unsigned long snap_pcount; -+/*#endif*/ -+#if IS_ENABLED(CONFIG_OPLUS_SCHED_TUNE) -+ int stune_idx; -+#endif -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE)*/ -+ atomic_t pipeline_cpu; -+/*#endif*/ -+ -+ /* for oplus secure guard */ -+ int sg_flag; -+ int sg_scno; -+ uid_t sg_uid; -+ uid_t sg_euid; -+ gid_t sg_gid; -+ gid_t sg_egid; -+/*#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct uid_struct *uid_struct; -+ u64 amu_instruct; -+ u64 amu_cycle; -+/*#endif*/ -+ /* for binder ux */ -+ int binder_async_ux_enable; -+ bool binder_async_ux_sts; -+ int binder_thread_mode; -+ struct binder_node *binder_thread_node; -+ -+/* for powermodel */ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ struct powermodel_cpu_task_state *powermodel_cpu_task_states[MAX_CPU_CNT]; -+ u64 exec_runtime; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_CFBT) -+ int cfbt_cur_group; -+ bool cfbt_running; -+#endif /* CONFIG_OPLUS_FEATURE_SCHED_CFBT */ -+} ____cacheline_aligned; -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+#define INVALID_PID (-1) -+struct oplus_lb { -+ /* used for active_balance to record the running task. */ -+ pid_t pid; -+}; -+/*#endif*/ -+ -+#endif /* _OPLUS_SA_COMMON_STRUCT_H_ */ -+ -diff --git a/include/linux/sched/sa_oemdata.h b/include/linux/sched/sa_oemdata.h -new file mode 100755 -index 000000000000..ebbbc754e391 ---- /dev/null -+++ b/include/linux/sched/sa_oemdata.h -@@ -0,0 +1,13 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_OEMDATA_H_ -+#define _OPLUS_SA_OEMDATA_H_ -+ -+int sa_oemdata_init(void); -+void sa_oemdata_deinit(void); -+ -+#endif /* _OPLUS_SA_OEMDATA_H_ */ -diff --git a/include/linux/sched/sched_ext.h b/include/linux/sched/sched_ext.h -new file mode 100755 -index 000000000000..9f1afdb8a04b ---- /dev/null -+++ b/include/linux/sched/sched_ext.h -@@ -0,0 +1,57 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef _OPLUS_SCHED_EXT_H -+#define _OPLUS_SCHED_EXT_H -+#include "sa_common.h" -+ -+#define SCHED_PROP_TOP_THREAD_SHIFT (8) -+#define SCHED_PROP_TOP_THREAD_MASK (0xf << SCHED_PROP_TOP_THREAD_SHIFT) -+#define SCHED_PROP_DEADLINE_MASK (0xFF) /* deadline for ext sched class */ -+#define SCHED_PROP_DEADLINE_LEVEL1 (1) /* 1ms for user-aware audio tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL2 (2) /* 2ms for user-aware touch tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL3 (3) /* 4ms for user aware dispaly tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL4 (4) /* 6ms */ -+#define SCHED_PROP_DEADLINE_LEVEL5 (5) /* 8ms */ -+#define SCHED_PROP_DEADLINE_LEVEL6 (6) /* 16ms */ -+#define SCHED_PROP_DEADLINE_LEVEL7 (7) /* 32ms */ -+#define SCHED_PROP_DEADLINE_LEVEL8 (8) /* 64ms */ -+#define SCHED_PROP_DEADLINE_LEVEL9 (9) /* 128ms */ -+ -+static inline int sched_prop_get_top_thread_id(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ return -EPERM; -+ } -+ -+ return ((ots->scx.sched_prop & SCHED_PROP_TOP_THREAD_MASK) >> SCHED_PROP_TOP_THREAD_SHIFT); -+} -+ -+static inline int sched_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_set_sched_prop failed! fn=%s\n", __func__); -+ return -EPERM; -+ } -+ -+ ots->scx.sched_prop = sp; -+ return 0; -+} -+ -+static inline unsigned long sched_get_sched_prop(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_get_sched_prop failed! fn=%s\n", __func__); -+ return (unsigned long)-1; -+ } -+ return ots->scx.sched_prop; -+} -+ -+#endif /*_OPLUS_SCHED_EXT_H */ -diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h -index 03d35e3eda3c..6a5f0c0d74bd 100644 ---- a/include/linux/sched/task.h -+++ b/include/linux/sched/task.h -@@ -61,8 +61,10 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); - extern void init_idle(struct task_struct *idle, int cpu); - - extern int sched_fork(unsigned long clone_flags, struct task_struct *p); --extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); --extern void sched_cancel_fork(struct task_struct *p); -+extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); -+#ifdef CONFIG_HMBIRD_SCHED -+extern void hmbird_cancel_fork(struct task_struct *p); -+#endif - extern void sched_post_fork(struct task_struct *p); - extern void sched_dead(struct task_struct *p); - -diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h -index 359a14cc76a4..969716ab4320 100644 ---- a/include/uapi/linux/sched.h -+++ b/include/uapi/linux/sched.h -@@ -118,7 +118,7 @@ struct clone_args { - /* SCHED_ISO: reserved but not implemented yet */ - #define SCHED_IDLE 5 - #define SCHED_DEADLINE 6 --#define SCHED_EXT 7 -+#define SCHED_HMBIRD 7 - - /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ - #define SCHED_RESET_ON_FORK 0x40000000 -diff --git a/init/Kconfig b/init/Kconfig -index edd1d990c45f..47384a9666eb 100644 ---- a/init/Kconfig -+++ b/init/Kconfig -@@ -1030,10 +1030,6 @@ config RT_GROUP_SCHED - realtime bandwidth for them. - See Documentation/scheduler/sched-rt-group.rst for more information. - --config EXT_GROUP_SCHED -- bool -- depends on SCHED_CLASS_EXT && CGROUP_SCHED -- default y - endif #CGROUP_SCHED - - config SCHED_MM_CID -diff --git a/init/init_task.c b/init/init_task.c -index 69a046388995..31ceb0e469f7 100644 ---- a/init/init_task.c -+++ b/init/init_task.c -@@ -6,7 +6,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -102,9 +101,6 @@ struct task_struct init_task - #endif - #ifdef CONFIG_CGROUP_SCHED - .sched_task_group = &root_task_group, --#endif --#ifdef CONFIG_SLIM_SCHED -- .scx = NULL, - #endif - .ptraced = LIST_HEAD_INIT(init_task.ptraced), - .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), -diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt -index cf22ef6107b8..b131d5662b48 100644 ---- a/kernel/Kconfig.preempt -+++ b/kernel/Kconfig.preempt -@@ -133,12 +133,8 @@ config SCHED_CORE - which is the likely usage by Linux distributions, there should - be no measurable impact on performance. - --config SLIM_SCHED -- bool "slim sched" -- default n --config SCHED_CLASS_EXT -- bool "Extensible Scheduling Class" -- depends on BPF_SYSCALL && BPF_JIT -+config HMBIRD_SCHED -+ bool "hmbird Scheduling Class(base on ext scheduler)" - help - This option enables a new scheduler class sched_ext (SCX), which - allows scheduling policies to be implemented as BPF programs to -@@ -159,3 +155,10 @@ config SCHED_CLASS_EXT - similar to struct sched_class. - - See Documentation/scheduler/sched-ext.rst for more details. -+ -+config DYNAMIC_WALT_SUPPORT -+ bool "Dynamic WALT Support for Extensible Scheduling Class" -+ depends on HMBIRD_SCHED -+ default n -+ help -+ Enable dynamic WALT support for Extensible Scheduling Class. -diff --git a/kernel/bpf/bpf_struct_ops_types.h b/kernel/bpf/bpf_struct_ops_types.h -index 3618769d853d..5678a9ddf817 100644 ---- a/kernel/bpf/bpf_struct_ops_types.h -+++ b/kernel/bpf/bpf_struct_ops_types.h -@@ -9,8 +9,4 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) - #include - BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) - #endif --#ifdef CONFIG_SCHED_CLASS_EXT --#include --BPF_STRUCT_OPS_TYPE(sched_ext_ops) --#endif - #endif -diff --git a/kernel/fork.c b/kernel/fork.c -index a43f97302332..9165a473b628 100644 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -23,7 +23,9 @@ - #include - #include - #include --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - #include - #include - #include -@@ -1000,7 +1002,9 @@ void __put_task_struct(struct task_struct *tsk) - WARN_ON(refcount_read(&tsk->usage)); - WARN_ON(tsk == current); - -- sched_ext_free(tsk); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_free(tsk); -+#endif - io_uring_free(tsk); - cgroup_free(tsk); - task_numa_free(tsk, true); -@@ -2517,7 +2521,11 @@ __latent_entropy struct task_struct *copy_process( - - retval = perf_event_init_task(p, clone_flags); - if (retval) -- goto bad_fork_sched_cancel_fork; -+#ifdef CONFIG_HMBIRD_SCHED -+ goto bad_fork_hmbird_cancel_fork; -+#else -+ goto bad_fork_cleanup_policy; -+#endif - retval = audit_alloc(p); - if (retval) - goto bad_fork_cleanup_perf; -@@ -2649,9 +2657,7 @@ __latent_entropy struct task_struct *copy_process( - * cgroup specific, it unconditionally needs to place the task on a - * runqueue. - */ -- retval = sched_cgroup_fork(p, args); -- if (retval) -- goto bad_fork_cancel_cgroup; -+ sched_cgroup_fork(p, args); - - /* - * From this point on we must avoid any synchronous user-space -@@ -2697,13 +2703,13 @@ __latent_entropy struct task_struct *copy_process( - /* Don't start children in a dying pid namespace */ - if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { - retval = -ENOMEM; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* Let kill terminate clone/fork in the middle */ - if (fatal_signal_pending(current)) { - retval = -EINTR; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* No more failure paths after this point. */ -@@ -2780,11 +2786,10 @@ __latent_entropy struct task_struct *copy_process( - - return p; - --bad_fork_core_free: -+bad_fork_cancel_cgroup: - sched_core_free(p); - spin_unlock(¤t->sighand->siglock); - write_unlock_irq(&tasklist_lock); --bad_fork_cancel_cgroup: - cgroup_cancel_fork(p, args); - bad_fork_put_pidfd: - if (clone_flags & CLONE_PIDFD) { -@@ -2823,8 +2828,10 @@ __latent_entropy struct task_struct *copy_process( - audit_free(p); - bad_fork_cleanup_perf: - perf_event_free_task(p); --bad_fork_sched_cancel_fork: -- sched_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+bad_fork_hmbird_cancel_fork: -+ hmbird_cancel_fork(p); -+#endif - bad_fork_cleanup_policy: - lockdep_free_task(p); - #ifdef CONFIG_NUMA -diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c -index 005025f55bea..c091539a0f60 100644 ---- a/kernel/sched/build_policy.c -+++ b/kernel/sched/build_policy.c -@@ -28,8 +28,10 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED - #include - #include -+#endif - - #include - -@@ -54,6 +56,10 @@ - #include "cputime.c" - #include "deadline.c" - --#ifdef CONFIG_SCHED_CLASS_EXT --# include "ext.c" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/hmbird_util_track.c" -+#include "hmbird/hmbird_sched_proc.c" -+#include "hmbird/hmbird_shadow_tick.c" -+#include "hmbird/hmbird.c" -+#include "hmbird/hmbird_misc.c" - #endif -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index 3e3e89be6508..692c9aa0ffa6 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -96,6 +96,12 @@ - #include "../../io_uring/io-wq.h" - #include "../smpboot.h" - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/slim.h" -+#include "hmbird/hmbird_shadow_tick.h" -+#include "hmbird.h" -+#endif -+ - #include - #include - #include -@@ -170,7 +176,6 @@ __read_mostly int scheduler_running; - - DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); - -- - /* kernel prio, less is more */ - static inline int __task_prio(const struct task_struct *p) - { -@@ -183,8 +188,8 @@ static inline int __task_prio(const struct task_struct *p) - if (p->sched_class == &idle_sched_class) - return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - --#ifdef CONFIG_SCHED_CLASS_EXT -- if (p->sched_class == &ext_sched_class) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (p->sched_class == &hmbird_sched_class) - return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ - #endif - -@@ -217,9 +222,9 @@ static inline bool prio_less(const struct task_struct *a, - if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ - return cfs_prio_less(a, b, in_fi); - --#ifdef CONFIG_SCHED_CLASS_EXT -+#ifdef CONFIG_HMBIRD_SCHED - if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ -- return scx_prio_less(a, b, in_fi); -+ return hmbird_prio_less(a, b, in_fi); - #endif - - return false; -@@ -1279,17 +1284,26 @@ bool sched_can_stop_tick(struct rq *rq) - if (fifo_nr_running) - return true; - -+#ifdef CONFIG_HMBIRD_SCHED - /* -- * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks -+ * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks - * left. For CFS, if there's more than one we need the tick for -- * involuntary preemption. For SCX, ask. -+ * involuntary preemption. For HMBIRD, ask. - */ -- if (!scx_switched_all() && rq->nr_running > 1) -+ if (!hmbird_enabled() && rq->nr_running > 1) - return false; - -- if (scx_enabled() && !scx_can_stop_tick(rq)) -+ if (hmbird_enabled() && !hmbird_can_stop_tick(rq)) - return false; -- -+#else -+ /* -+ * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; -+ * if there's more than one we need the tick for involuntary -+ * preemption. -+ */ -+ if (rq->nr_running > 1) -+ return false; -+#endif - /* - * If there is one task and it has CFS runtime bandwidth constraints - * and it's on the cpu now we don't want to stop the tick. -@@ -2222,10 +2236,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) - } - EXPORT_SYMBOL_GPL(deactivate_task); - --struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) -+#ifdef CONFIG_HMBIRD_SCHED -+struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - { -- struct sched_change_guard cg = { -+ struct hmbird_sched_change_guard cg = { - .rq = rq, - .p = p, - .queued = task_on_rq_queued(p), -@@ -2247,7 +2262,7 @@ sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - return cg; - } - --void sched_change_guard_fini(struct sched_change_guard *cg, int flags) -+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags) - { - if (cg->queued) - enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); -@@ -2255,6 +2270,7 @@ void sched_change_guard_fini(struct sched_change_guard *cg, int flags) - set_next_task(cg->rq, cg->p); - cg->done = true; - } -+#endif - - static inline int __normal_prio(int policy, int rt_prio, int nice) - { -@@ -2320,9 +2336,9 @@ inline int task_curr(const struct task_struct *p) - * this means any call to check_class_changed() must be followed by a call to - * balance_callback(). - */ --void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio) -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) - { - if (prev_class != p->sched_class) { - if (prev_class->switched_from) -@@ -2885,6 +2901,7 @@ static void - __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - { - struct rq *rq = task_rq(p); -+ bool queued, running; - - /* - * This here violates the locking rules for affinity, since we're only -@@ -2903,9 +2920,26 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - else - lockdep_assert_held(&p->pi_lock); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->sched_class->set_cpus_allowed(p, ctx); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ -+ if (queued) { -+ /* -+ * Because __kthread_bind() calls this on blocked tasks without -+ * holding rq->lock. -+ */ -+ lockdep_assert_rq_held(rq); -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); - } -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->sched_class->set_cpus_allowed(p, ctx); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - } - - /* -@@ -3772,7 +3806,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class)) -+#else - if (unlikely(!is_cpu_allowed(p, cpu))) -+#endif - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -@@ -4891,7 +4929,9 @@ late_initcall(sched_core_sysctl_init); - */ - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { -+#ifdef CONFIG_HMBIRD_SCHED - int ret; -+#endif - - trace_android_rvh_sched_fork(p); - -@@ -4932,21 +4972,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_reset_on_fork = 0; - } - -- scx_pre_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ ret = hmbird_pre_fork(p); -+ if (ret) -+ goto out_cancel; - - if (dl_prio(p->prio)) { - ret = -EAGAIN; - goto out_cancel; --#ifdef CONFIG_SCHED_CLASS_EXT -- } else if (task_on_scx(p)) { -- p->sched_class = &ext_sched_class; --#endif -+ } else if (task_on_hmbird(p)) { -+ p->sched_class = &hmbird_sched_class; - } else if (rt_prio(p->prio)) { - p->sched_class = &rt_sched_class; - } else { - p->sched_class = &fair_sched_class; - } -- -+#else -+ if (dl_prio(p->prio)) -+ return -EAGAIN; -+ else if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - init_entity_runnable_average(&p->se); - trace_android_rvh_finish_prio_fork(p); - -@@ -4965,12 +5013,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - #endif - return 0; - -+#ifdef CONFIG_HMBIRD_SCHED - out_cancel: -- scx_cancel_fork(p); -+ hmbird_cancel_fork(p); - return ret; -+#endif - } - --int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) -+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - { - unsigned long flags; - -@@ -4997,19 +5047,17 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -- -- return scx_fork(p); --} -- --void sched_cancel_fork(struct task_struct *p) --{ -- scx_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_fork(p); -+#endif - } - - void sched_post_fork(struct task_struct *p) - { - uclamp_post_fork(p); -- scx_post_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_post_fork(p); -+#endif - } - - unsigned long to_ratio(u64 period, u64 runtime) -@@ -5874,20 +5922,28 @@ void scheduler_tick(void) - if (sched_feat(LATENCY_WARN) && resched_latency) - resched_latency_warn(cpu, resched_latency); - -- scx_notify_sched_tick(); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_notify_sched_tick(); -+#endif - perf_event_task_tick(); - - if (curr->flags & PF_WQ_WORKER) - wq_worker_tick(curr); - - #ifdef CONFIG_SMP -- if (!scx_switched_all()) { -+#ifdef CONFIG_HMBIRD_SCHED -+ if (!hmbird_enabled()) { -+#endif - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); -+#ifdef CONFIG_HMBIRD_SCHED - } - #endif -- -+#endif - trace_android_vh_scheduler_tick(rq); -+#ifdef CONFIG_HMBIRD_SCHED -+ scheduler_tick_handler(NULL, NULL); -+#endif - } - - #ifdef CONFIG_NO_HZ_FULL -@@ -6189,7 +6245,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ -+#ifdef CONFIG_HMBIRD_SCHED - for_balance_class_range(class, prev->sched_class, &idle_sched_class) { -+#else -+ for_class_range(class, prev->sched_class, &idle_sched_class) { -+#endif - if (class->balance(rq, prev, rf)) - break; - } -@@ -6207,9 +6267,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - const struct sched_class *class; - struct task_struct *p; - -- if (scx_enabled()) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (hmbird_enabled()) - goto restart; -- -+#endif - /* - * Optimization: we know that if all tasks are in the fair class we can - * call that function directly, but only if the @prev task wasn't of a -@@ -6235,13 +6296,21 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - restart: - put_prev_task_balance(rq, prev, rf); - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { - p = class->pick_next_task(rq); - if (p) { -- scx_notify_pick_next_task(rq, p, class); -+ hmbird_notify_pick_next_task(rq, p, class); - return p; - } - } -+#else -+ for_each_class(class) { -+ p = class->pick_next_task(rq); -+ if (p) -+ return p; -+ } -+#endif - - BUG(); /* The idle class should always have a runnable task. */ - } -@@ -6270,7 +6339,11 @@ static inline struct task_struct *pick_task(struct rq *rq) - const struct sched_class *class; - struct task_struct *p; - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { -+#else -+ for_each_class(class) { -+#endif - p = class->pick_task(rq); - if (p) - return p; -@@ -6914,6 +6987,9 @@ static void __sched notrace __schedule(unsigned int sched_mode) - psi_sched_switch(prev, next, !task_on_rq_queued(prev)); - - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#ifdef CONFIG_HMBIRD_SCHED -+ sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#endif - - /* Also unlocks the rq: */ - rq = context_switch(rq, prev, next, &rf); -@@ -7242,24 +7318,31 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag - } - EXPORT_SYMBOL(default_wake_function); - --void __setscheduler_prio(struct task_struct *p, int prio) -+static void __setscheduler_prio(struct task_struct *p, int prio) - { -- /* -- * After switching all rt and fair class to ext, -- * stop class is switched to ext class too. Just -- * skip it. */ -+#ifdef CONFIG_HMBIRD_SCHED -+ bool on_hmbird = task_on_hmbird(p); -+ - if (p->sched_class == &stop_sched_class) -- ;/* do nothing */ -+ ; - else if (dl_prio(prio)) - p->sched_class = &dl_sched_class; --#ifdef CONFIG_SCHED_CLASS_EXT -- else if (task_on_scx(p)) -- p->sched_class = &ext_sched_class; --#endif -+ else if (rt_prio(prio) && on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#else -+ if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; - else if (rt_prio(prio)) - p->sched_class = &rt_sched_class; - else - p->sched_class = &fair_sched_class; -+#endif - - p->prio = prio; - trace_android_rvh_setscheduler_prio(p); -@@ -7295,7 +7378,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - */ - void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - { -- int prio, oldprio, queue_flag = -+ int prio, oldprio, queued, running, queue_flag = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *prev_class; - struct rq_flags rf; -@@ -7358,42 +7441,52 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - queue_flag &= ~DEQUEUE_MOVE; - - prev_class = p->sched_class; -- SCHED_CHANGE_BLOCK(rq, p, queue_flag) { -- /* -- * Boosting condition are: -- * 1. -rt task is running and holds mutex A -- * --> -dl task blocks on mutex A -- * -- * 2. -dl task is running and holds mutex A -- * --> -dl task blocks on mutex A and could preempt the -- * running task -- */ -- if (dl_prio(prio)) { -- if (!dl_prio(p->normal_prio) || -- (pi_task && dl_prio(pi_task->prio) && -- dl_entity_preempt(&pi_task->dl, &p->dl))) { -- p->dl.pi_se = pi_task->dl.pi_se; -- queue_flag |= ENQUEUE_REPLENISH; -- } else { -- p->dl.pi_se = &p->dl; -- } -- } else if (rt_prio(prio)) { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- if (oldprio < prio) -- queue_flag |= ENQUEUE_HEAD; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flag); -+ if (running) -+ put_prev_task(rq, p); -+ -+ /* -+ * Boosting condition are: -+ * 1. -rt task is running and holds mutex A -+ * --> -dl task blocks on mutex A -+ * -+ * 2. -dl task is running and holds mutex A -+ * --> -dl task blocks on mutex A and could preempt the -+ * running task -+ */ -+ if (dl_prio(prio)) { -+ if (!dl_prio(p->normal_prio) || -+ (pi_task && dl_prio(pi_task->prio) && -+ dl_entity_preempt(&pi_task->dl, &p->dl))) { -+ p->dl.pi_se = pi_task->dl.pi_se; -+ queue_flag |= ENQUEUE_REPLENISH; - } else { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- else if (rt_prio(oldprio)) -- p->rt.timeout = 0; -- else if (!task_has_idle_policy(p)) -- reweight_task(p, prio - MAX_RT_PRIO); -+ p->dl.pi_se = &p->dl; - } -- -- __setscheduler_prio(p, prio); -+ } else if (rt_prio(prio)) { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ if (oldprio < prio) -+ queue_flag |= ENQUEUE_HEAD; -+ } else { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ else if (rt_prio(oldprio)) -+ p->rt.timeout = 0; -+ else if (!task_has_idle_policy(p)) -+ reweight_task(p, prio - MAX_RT_PRIO); - } - -+ __setscheduler_prio(p, prio); -+ -+ if (queued) -+ enqueue_task(rq, p, queue_flag); -+ if (running) -+ set_next_task(rq, p); -+ - check_class_changed(rq, p, prev_class, oldprio); - out_unlock: - /* Avoid rq from going away on us: */ -@@ -7414,7 +7507,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - - void set_user_nice(struct task_struct *p, long nice) - { -- bool allowed = false; -+ bool queued, running, allowed = false; - int old_prio; - struct rq_flags rf; - struct rq *rq; -@@ -7443,13 +7536,22 @@ void set_user_nice(struct task_struct *p, long nice) - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); -+ if (running) -+ put_prev_task(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->static_prio = NICE_TO_PRIO(nice); -- set_load_weight(p, true); -- old_prio = p->prio; -- p->prio = effective_prio(p); -- } -+ p->static_prio = NICE_TO_PRIO(nice); -+ set_load_weight(p, true); -+ old_prio = p->prio; -+ p->prio = effective_prio(p); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - - /* - * If the task increased its priority or is running and -@@ -7866,7 +7968,7 @@ static int __sched_setscheduler(struct task_struct *p, - bool user, bool pi) - { - int oldpolicy = -1, policy = attr->sched_policy; -- int retval, oldprio, newprio; -+ int retval, oldprio, newprio, queued, running; - const struct sched_class *prev_class; - struct balance_callback *head; - struct rq_flags rf; -@@ -7874,6 +7976,9 @@ static int __sched_setscheduler(struct task_struct *p, - int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct rq *rq; - bool cpuset_locked = false; -+#ifdef CONFIG_HMBIRD_SCHED -+ unsigned long flags; -+#endif - - /* The pi code expects interrupts enabled */ - BUG_ON(pi && in_interrupt()); -@@ -7939,6 +8044,9 @@ static int __sched_setscheduler(struct task_struct *p, - * To be able to change p->policy safely, the appropriate - * runqueue lock must be held. - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+#endif - rq = task_rq_lock(p, &rf); - update_rq_clock(rq); - -@@ -7950,10 +8058,11 @@ static int __sched_setscheduler(struct task_struct *p, - goto unlock; - } - -- retval = scx_check_setscheduler(p, policy); -+#ifdef CONFIG_HMBIRD_SCHED -+ retval = hmbird_check_setscheduler(p, policy); - if (retval) - goto unlock; -- -+#endif - /* - * If not changing anything there's no need to proceed further, - * but store a possible modification of reset_on_fork. -@@ -8010,6 +8119,9 @@ static int __sched_setscheduler(struct task_struct *p, - if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { - policy = oldpolicy = -1; - task_rq_unlock(rq, p, &rf); -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - goto recheck; -@@ -8042,16 +8154,23 @@ static int __sched_setscheduler(struct task_struct *p, - queue_flags &= ~DEQUEUE_MOVE; - } - -- SCHED_CHANGE_BLOCK(rq, p, queue_flags) { -- prev_class = p->sched_class; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flags); -+ if (running) -+ put_prev_task(rq, p); - -- if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -- __setscheduler_params(p, attr); -- __setscheduler_prio(p, newprio); -- trace_android_rvh_setscheduler(p); -- } -- __setscheduler_uclamp(p, attr); -+ prev_class = p->sched_class; - -+ if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -+ __setscheduler_params(p, attr); -+ __setscheduler_prio(p, newprio); -+ trace_android_rvh_setscheduler(p); -+ } -+ __setscheduler_uclamp(p, attr); -+ -+ if (queued) { - /* - * We enqueue to tail when the priority of a task is - * increased (user space view). -@@ -8059,7 +8178,10 @@ static int __sched_setscheduler(struct task_struct *p, - if (oldprio < p->prio) - queue_flags |= ENQUEUE_HEAD; - -+ enqueue_task(rq, p, queue_flags); - } -+ if (running) -+ set_next_task(rq, p); - - check_class_changed(rq, p, prev_class, oldprio); - -@@ -8067,7 +8189,9 @@ static int __sched_setscheduler(struct task_struct *p, - preempt_disable(); - head = splice_balance_callbacks(rq); - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (pi) { - if (cpuset_locked) - cpuset_unlock(); -@@ -8082,6 +8206,9 @@ static int __sched_setscheduler(struct task_struct *p, - - unlock: - task_rq_unlock(rq, p, &rf); -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - return retval; -@@ -9303,7 +9430,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - break; - } -@@ -9331,7 +9460,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - } - return ret; -@@ -9632,15 +9763,25 @@ int migrate_task_to(struct task_struct *p, int target_cpu) - */ - void sched_setnuma(struct task_struct *p, int nid) - { -+ bool queued, running; - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) { -- p->numa_preferred_nid = nid; -- } -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE); -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->numa_preferred_nid = nid; - -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - task_rq_unlock(rq, p, &rf); - } - #endif /* CONFIG_NUMA_BALANCING */ -@@ -10206,15 +10347,22 @@ void __init sched_init(void) - int i; - - /* Make sure the linker didn't screw up */ -+#ifdef CONFIG_HMBIRD_SCHED - #ifdef CONFIG_SMP -- BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+#endif -+ WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class)); -+#else -+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 || -+ &fair_sched_class != &rt_sched_class + 1 || -+ &rt_sched_class != &dl_sched_class + 1); -+#ifdef CONFIG_SMP -+ BUG_ON(&dl_sched_class != &stop_sched_class + 1); - #endif -- BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); -- BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); -- BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); --#ifdef CONFIG_SCHED_CLASS_EXT -- BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); -- BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); - #endif - - wait_bit_init(); -@@ -10386,8 +10534,9 @@ void __init sched_init(void) - balance_push_set(smp_processor_id(), false); - #endif - init_sched_fair_class(); -- init_sched_ext_class(); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ init_sched_hmbird_class(); -+#endif - psi_init(); - - init_uclamp(); -@@ -10791,6 +10940,9 @@ static void sched_change_group(struct task_struct *tsk) - */ - void sched_move_task(struct task_struct *tsk) - { -+ int queued, running, queue_flags = -+ DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; -+ struct task_group *group; - struct rq_flags rf; - struct rq *rq; - -@@ -10798,18 +10950,27 @@ void sched_move_task(struct task_struct *tsk) - rq = task_rq_lock(tsk, &rf); - update_rq_clock(rq); - -- SCHED_CHANGE_BLOCK(rq, tsk, -- DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { -- sched_change_group(tsk); -- } -+ running = task_current(rq, tsk); -+ queued = task_on_rq_queued(tsk); - -- /* -- * After changing group, the running task may have joined a throttled -- * one but it's still the running task. Trigger a resched to make sure -- * that task can still run. -- */ -- if (task_current(rq, tsk)) -+ if (queued) -+ dequeue_task(rq, tsk, queue_flags); -+ if (running) -+ put_prev_task(rq, tsk); -+ -+ sched_change_group(tsk, group); -+ -+ if (queued) -+ enqueue_task(rq, tsk, queue_flags); -+ if (running) { -+ set_next_task(rq, tsk); -+ /* -+ * After changing group, the running task may have joined a -+ * throttled one but it's still the running task. Trigger a -+ * resched to make sure that task can still run. -+ */ - resched_curr(rq); -+ } - - task_rq_unlock(rq, tsk, &rf); - } -@@ -10846,6 +11007,10 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) - struct task_group *tg = css_tg(css); - struct task_group *parent = css_tg(css->parent); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_tg_online(tg); -+#endif -+ - if (parent) - sched_online_group(tg, parent); - -@@ -10895,13 +11060,79 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) - } - #endif - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 dl) -+{ -+ int i; -+ -+ for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) { -+ if (dl < HMBIRD_BPF_DSQS_DEADLINE[i]) -+ break; -+ } -+ -+ i = max_t(int, i-1, MIN_CGROUP_DL_IDX); -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return 0; -+ update_cgroup_ids_table(css->cgroup->kn->id, i); -+ -+ return 0; -+} -+ -+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ u8 i; -+ -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX]; -+ i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1); -+ if (i < 0) { -+ pr_err(" <%s> i is %d, less than 0, name is %s\n", -+ __func__, i, css->cgroup->kn->name); -+ i = DEFAULT_CGROUP_DL_IDX; -+ } -+ -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[i]; -+} -+ -+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css, -+ struct task_struct *p, int prio) -+{ -+ if (!css || !rt_prio(prio)) -+ return; -+ -+ if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) { -+ if (!strcmp(css->cgroup->kn->name, "display")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3); -+ else if (!strcmp(css->cgroup->kn->name, "touch")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2); -+ else if (!strcmp(css->cgroup->kn->name, "multimedia")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+ } -+} -+#endif -+ - static void cpu_cgroup_attach(struct cgroup_taskset *tset) - { - struct task_struct *task; - struct cgroup_subsys_state *css; - -- cgroup_taskset_for_each(task, css, tset) -+ cgroup_taskset_for_each(task, css, tset) { -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_change_rt_sched_prop(css, task, task->prio); -+#endif - sched_move_task(task); -+ } - - trace_android_rvh_cpu_cgroup_attach(tset); - } -@@ -11580,6 +11811,13 @@ static struct cftype cpu_legacy_files[] = { - .read_u64 = cpu_uclamp_ls_read_u64, - .write_u64 = cpu_uclamp_ls_write_u64, - }, -+#endif -+#ifdef CONFIG_HMBIRD_SCHED -+ { -+ .name = "hmbird.deadline", -+ .read_u64 = cgroup_read_hmbird_deadline, -+ .write_u64 = cgroup_write_hmbird_deadline, -+ }, - #endif - { } /* Terminate */ - }; -@@ -11629,7 +11867,6 @@ static int cpu_local_stat_show(struct seq_file *sf, - } - - #ifdef CONFIG_FAIR_GROUP_SCHED -- - static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, - struct cftype *cft) - { -diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c -index a6c99df9edf9..f647dacc22e2 100644 ---- a/kernel/sched/debug.c -+++ b/kernel/sched/debug.c -@@ -376,8 +376,8 @@ static __init int sched_init_debug(void) - - debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); - --#ifdef CONFIG_SCHED_CLASS_EXT -- debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops); -+#ifdef CONFIG_HMBIRD_SCHED -+ debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops); - #endif - return 0; - } -@@ -1006,9 +1006,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - SEQ_printf(m, - "---------------------------------------------------------" - "----------\n"); --#ifdef CONFIG_SLIM_SCHED -- SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop); --#endif - - #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) - #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) -@@ -1104,8 +1101,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - } else if (fair_policy(p->policy)) { - P(se.slice); - } --#ifdef CONFIG_SCHED_CLASS_EXT -- __PS("ext.enabled", p->sched_class == &ext_sched_class); -+#ifdef CONFIG_HMBIRD_SCHED -+ __PS("hmbird.enabled", p->sched_class == &hmbird_sched_class); -+ __PS("sched_prop", get_hmbird_ts(p)->sched_prop); -+ __PS("top_task_prop", get_hmbird_ts(p)->top_task_prop); - #endif - #undef PN_SCHEDSTAT - #undef P_SCHEDSTAT -diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c -deleted file mode 100755 -index 4845d441df2b..000000000000 ---- a/kernel/sched/ext.c -+++ /dev/null -@@ -1,3978 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -- --enum scx_internal_consts { -- SCX_NR_ONLINE_OPS = SCX_OP_IDX(init), -- SCX_DSP_DFL_MAX_BATCH = 32, -- SCX_DSP_MAX_LOOPS = 32, -- SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, --}; -- --enum scx_ops_enable_state { -- SCX_OPS_PREPPING, -- SCX_OPS_ENABLING, -- SCX_OPS_ENABLED, -- SCX_OPS_DISABLING, -- SCX_OPS_DISABLED, --}; -- --static inline struct task_group *css_tg(struct cgroup_subsys_state *css) --{ -- return css ? container_of(css, struct task_group, css) : NULL; --} -- --/* -- * sched_ext_entity->ops_state -- * -- * Used to track the task ownership between the SCX core and the BPF scheduler. -- * State transitions look as follows: -- * -- * NONE -> QUEUEING -> QUEUED -> DISPATCHING -- * ^ | | -- * | v v -- * \-------------------------------/ -- * -- * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -- * sites for explanations on the conditions being waited upon and why they are -- * safe. Transitions out of them into NONE or QUEUED must store_release and the -- * waiters should load_acquire. -- * -- * Tracking scx_ops_state enables sched_ext core to reliably determine whether -- * any given task can be dispatched by the BPF scheduler at all times and thus -- * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -- * to try to dispatch any task anytime regardless of its state as the SCX core -- * can safely reject invalid dispatches. -- */ --enum scx_ops_state { -- SCX_OPSS_NONE, /* owned by the SCX core */ -- SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -- SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ -- SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ -- -- /* -- * QSEQ brands each QUEUED instance so that, when dispatch races -- * dequeue/requeue, the dispatcher can tell whether it still has a claim -- * on the task being dispatched. -- */ -- SCX_OPSS_QSEQ_SHIFT = 2, -- SCX_OPSS_STATE_MASK = (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1, -- SCX_OPSS_QSEQ_MASK = ~SCX_OPSS_STATE_MASK, --}; -- --/* -- * During exit, a task may schedule after losing its PIDs. When disabling the -- * BPF scheduler, we need to be able to iterate tasks in every state to -- * guarantee system safety. Maintain a dedicated task list which contains every -- * task between its fork and eventual free. -- */ --static DEFINE_SPINLOCK(scx_tasks_lock); --static LIST_HEAD(scx_tasks); -- --/* ops enable/disable */ --static struct kthread_worker *scx_ops_helper; --static DEFINE_MUTEX(scx_ops_enable_mutex); --DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); --DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); --static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); --static bool scx_switch_all_req; --static bool scx_switching_all; --DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -- --static struct sched_ext_ops scx_ops; --static bool scx_warned_zero_slice; -- --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); --DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); --static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); -- --struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] = -- { [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT }; -- --static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE); --static struct scx_exit_info scx_exit_info; -- --static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0); -- --/* -- * The maximum amount of time in jiffies that a task may be runnable without -- * being scheduled on a CPU. If this timeout is exceeded, it will trigger -- * scx_ops_error(). -- */ --unsigned long scx_watchdog_timeout; -- --/* -- * The last time the delayed work was run. This delayed work relies on -- * ksoftirqd being able to run to service timer interrupts, so it's possible -- * that this work itself could get wedged. To account for this, we check that -- * it's not stalled in the timer tick, and trigger an error if it is. -- */ --unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; -- --static struct delayed_work scx_watchdog_work; -- --/* idle tracking */ --#ifdef CONFIG_SMP --#ifdef CONFIG_CPUMASK_OFFSTACK --#define CL_ALIGNED_IF_ONSTACK --#else --#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp --#endif -- --static struct { -- cpumask_var_t cpu; -- cpumask_var_t smt; --} idle_masks CL_ALIGNED_IF_ONSTACK; -- --static bool __cacheline_aligned_in_smp scx_has_idle_cpus; --#endif /* CONFIG_SMP */ -- --/* for %SCX_KICK_WAIT */ --static u64 __percpu *scx_kick_cpus_pnt_seqs; -- --/* -- * Direct dispatch marker. -- * -- * Non-NULL values are used for direct dispatch from enqueue path. A valid -- * pointer points to the task currently being enqueued. An ERR_PTR value is used -- * to indicate that direct dispatch has already happened. -- */ --static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); -- --/* dispatch queues */ --static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; -- --static LLIST_HEAD(dsqs_to_free); -- --/* dispatch buf */ --struct scx_dsp_buf_ent { -- struct task_struct *task; -- u64 qseq; -- u64 dsq_id; -- u64 enq_flags; --}; -- --static u32 scx_dsp_max_batch; --static struct scx_dsp_buf_ent __percpu *scx_dsp_buf; -- --struct scx_dsp_ctx { -- struct rq *rq; -- struct rq_flags *rf; -- u32 buf_cursor; -- u32 nr_tasks; --}; -- --static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx); -- --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags); --void scx_bpf_kick_cpu(s32 cpu, u64 flags); -- --struct scx_task_iter { -- struct sched_ext_entity cursor; -- struct task_struct *locked; -- struct rq *rq; -- struct rq_flags rf; --}; -- --#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) -- --/* if the highest set bit is N, return a mask with bits [N+1, 31] set */ --static u32 higher_bits(u32 flags) --{ -- return ~((1 << fls(flags)) - 1); --} -- --/* return the mask with only the highest bit set */ --static u32 highest_bit(u32 flags) --{ -- int bit = fls(flags); -- return bit ? 1 << (bit - 1) : 0; --} -- --/* -- * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX -- * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate -- * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check -- * whether it's running from an allowed context. -- * -- * @mask is constant, always inline to cull the mask calculations. -- */ --static __always_inline void scx_kf_allow(u32 mask) --{ -- /* nesting is allowed only in increasing scx_kf_mask order */ -- WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask, -- "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n", -- current->scx->kf_mask, mask); -- current->scx->kf_mask |= mask; --} -- --static void scx_kf_disallow(u32 mask) --{ -- current->scx->kf_mask &= ~mask; --} -- --#define SCX_CALL_OP(mask, op, args...) \ --do { \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- scx_ops.op(args); \ -- } \ --} while (0) -- --#define SCX_CALL_OP_RET(mask, op, args...) \ --({ \ -- __typeof__(scx_ops.op(args)) __ret; \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- __ret = scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- __ret = scx_ops.op(args); \ -- } \ -- __ret; \ --}) -- --/* -- * Some kfuncs are allowed only on the tasks that are subjects of the -- * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such -- * restrictions, the following SCX_CALL_OP_*() variants should be used when -- * invoking scx_ops operations that take task arguments. These can only be used -- * for non-nesting operations due to the way the tasks are tracked. -- * -- * kfuncs which can only operate on such tasks can in turn use -- * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on -- * the specific task. -- */ --#define SCX_CALL_OP_TASK(mask, op, task, args...) \ --do { \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- SCX_CALL_OP(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ --} while (0) -- --#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ --({ \ -- __typeof__(scx_ops.op(task, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- __ret; \ --}) -- --#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ --({ \ -- __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task0; \ -- current->scx->kf_tasks[1] = task1; \ -- __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- current->scx->kf_tasks[1] = NULL; \ -- __ret; \ --}) -- --//#include "./slim_walt.c" -- --/* @mask is constant, always inline to cull unnecessary branches */ --static __always_inline bool scx_kf_allowed(u32 mask) --{ -- if (unlikely(!(current->scx->kf_mask & mask))) { -- scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", -- mask, current->scx->kf_mask); -- return false; -- } -- -- if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) && -- in_interrupt())) { -- scx_ops_error("sleepable kfunc called from non-sleepable context"); -- return false; -- } -- -- /* -- * Enforce nesting boundaries. e.g. A kfunc which can be called from -- * DISPATCH must not be called if we're running DEQUEUE which is nested -- * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE -- * boundary thanks to the above in_interrupt() check. -- */ -- if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && -- (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { -- scx_ops_error("cpu_release kfunc called from a nested operation"); -- return false; -- } -- -- if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && -- (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) { -- scx_ops_error("dispatch kfunc called from a nested operation"); -- return false; -- } -- -- return true; --} -- --/* see SCX_CALL_OP_TASK() */ --static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, -- struct task_struct *p) --{ -- if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED)) -- return false; -- -- if (unlikely((p != current->scx->kf_tasks[0] && -- p != current->scx->kf_tasks[1]))) { -- scx_ops_error("called on a task not being operated on"); -- return false; -- } -- -- return true; --} -- --/** -- * scx_task_iter_init - Initialize a task iterator -- * @iter: iterator to init -- * -- * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized, -- * @iter must eventually be exited with scx_task_iter_exit(). -- * -- * scx_tasks_lock may be released between this and the first next() call or -- * between any two next() calls. If scx_tasks_lock is released between two -- * next() calls, the caller is responsible for ensuring that the task being -- * iterated remains accessible either through RCU read lock or obtaining a -- * reference count. -- * -- * All tasks which existed when the iteration started are guaranteed to be -- * visited as long as they still exist. -- */ --static void scx_task_iter_init(struct scx_task_iter *iter) --{ -- lockdep_assert_held(&scx_tasks_lock); -- -- iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; -- list_add(&iter->cursor.tasks_node, &scx_tasks); -- iter->locked = NULL; --} -- --/** -- * scx_task_iter_exit - Exit a task iterator -- * @iter: iterator to exit -- * -- * Exit a previously initialized @iter. Must be called with scx_tasks_lock held. -- * If the iterator holds a task's rq lock, that rq lock is released. See -- * scx_task_iter_init() for details. -- */ --static void scx_task_iter_exit(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- if (list_empty(cursor)) -- return; -- -- list_del_init(cursor); --} -- --/** -- * scx_task_iter_next - Next task -- * @iter: iterator to walk -- * -- * Visit the next task. See scx_task_iter_init() for details. -- */ --static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- struct sched_ext_entity *pos; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- list_for_each_entry(pos, cursor, tasks_node) { -- if (&pos->tasks_node == &scx_tasks) -- return NULL; -- if (!(pos->flags & SCX_TASK_CURSOR)) { -- list_move(cursor, &pos->tasks_node); -- return pos->task; -- } -- } -- -- /* can't happen, should always terminate at scx_tasks above */ -- BUG(); --} -- --/** -- * scx_task_iter_next_filtered - Next non-idle task -- * @iter: iterator to walk -- * -- * Visit the next non-idle task. See scx_task_iter_init() for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- while ((p = scx_task_iter_next(iter))) { -- if (!is_idle_task(p)) -- return p; -- } -- return NULL; --} -- --/** -- * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked -- * @iter: iterator to walk -- * -- * Visit the next non-idle task with its rq lock held. See scx_task_iter_init() -- * for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered_locked(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- p = scx_task_iter_next_filtered(iter); -- if (!p) -- return NULL; -- -- iter->rq = task_rq_lock(p, &iter->rf); -- iter->locked = p; -- return p; --} -- --static enum scx_ops_enable_state scx_ops_enable_state(void) --{ -- return atomic_read(&scx_ops_enable_state_var); --} -- --static enum scx_ops_enable_state --scx_ops_set_enable_state(enum scx_ops_enable_state to) --{ -- return atomic_xchg(&scx_ops_enable_state_var, to); --} -- --static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, -- enum scx_ops_enable_state from) --{ -- int from_v = from; -- -- return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); --} -- --static bool scx_ops_disabling(void) --{ -- return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING); --} -- --/** -- * wait_ops_state - Busy-wait the specified ops state to end -- * @p: target task -- * @opss: state to wait the end of -- * -- * Busy-wait for @p to transition out of @opss. This can only be used when the -- * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also -- * has load_acquire semantics to ensure that the caller can see the updates made -- * in the enqueueing and dispatching paths. -- */ --static void wait_ops_state(struct task_struct *p, u64 opss) --{ -- do { -- cpu_relax(); -- } while (atomic64_read_acquire(&p->scx->ops_state) == opss); --} -- --/** -- * ops_cpu_valid - Verify a cpu number -- * @cpu: cpu number which came from a BPF ops -- * -- * @cpu is a cpu number which came from the BPF scheduler and can be any value. -- * Verify that it is in range and one of the possible cpus. -- */ --static bool ops_cpu_valid(s32 cpu) --{ -- return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); --} -- --/** -- * ops_sanitize_err - Sanitize a -errno value -- * @ops_name: operation to blame on failure -- * @err: -errno value to sanitize -- * -- * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return -- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can -- * cause misbehaviors. For an example, a large negative return from -- * ops.prep_enable() triggers an oops when passed up the call chain because the -- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is -- * handled as a pointer. -- */ --static int ops_sanitize_err(const char *ops_name, s32 err) --{ -- if (err < 0 && err >= -MAX_ERRNO) -- return err; -- -- scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); -- return -EPROTO; --} -- --/** -- * touch_core_sched - Update timestamp used for core-sched task ordering -- * @rq: rq to read clock from, must be locked -- * @p: task to update the timestamp for -- * -- * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to -- * implement global or local-DSQ FIFO ordering for core-sched. Should be called -- * when a task becomes runnable and its turn on the CPU ends (e.g. slice -- * exhaustion). -- */ --static void touch_core_sched(struct rq *rq, struct task_struct *p) --{ --#ifdef CONFIG_SCHED_CORE -- /* -- * It's okay to update the timestamp spuriously. Use -- * sched_core_disabled() which is cheaper than enabled(). -- */ -- if (!sched_core_disabled()) -- p->scx->core_sched_at = rq_clock_task(rq); --#endif --} -- --/** -- * touch_core_sched_dispatch - Update core-sched timestamp on dispatch -- * @rq: rq to read clock from, must be locked -- * @p: task being dispatched -- * -- * If the BPF scheduler implements custom core-sched ordering via -- * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO -- * ordering within each local DSQ. This function is called from dispatch paths -- * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect. -- */ --static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- assert_clock_updated(rq); -- --#ifdef CONFIG_SCHED_CORE -- if (SCX_HAS_OP(core_sched_before)) -- touch_core_sched(rq, p); --#endif --} -- --static void update_curr_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- u64 now = rq_clock_task(rq); -- u64 delta_exec; -- -- if (time_before_eq64(now, curr->se.exec_start)) -- return; -- -- delta_exec = now - curr->se.exec_start; -- curr->se.exec_start = now; -- curr->se.sum_exec_runtime += delta_exec; -- account_group_exec_runtime(curr, delta_exec); -- cgroup_account_cputime(curr, delta_exec); -- -- if (curr->scx->slice != SCX_SLICE_INF) { -- curr->scx->slice -= min(curr->scx->slice, delta_exec); -- if (!curr->scx->slice) -- touch_core_sched(rq, curr); -- } --} -- --static bool scx_dsq_priq_less(struct rb_node *node_a, -- const struct rb_node *node_b) --{ -- const struct sched_ext_entity *a = -- container_of(node_a, struct sched_ext_entity, dsq_node.priq); -- const struct sched_ext_entity *b = -- container_of(node_b, struct sched_ext_entity, dsq_node.priq); -- -- return time_before64(a->dsq_vtime, b->dsq_vtime); --} -- --static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, -- u64 enq_flags) --{ -- bool is_local = dsq->id == SCX_DSQ_LOCAL; -- -- WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo)); -- WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq)); -- -- if (!is_local) { -- raw_spin_lock(&dsq->lock); -- if (unlikely(dsq->id == SCX_DSQ_INVALID)) { -- scx_ops_error("attempting to dispatch to a destroyed dsq"); -- /* fall back to the global dsq */ -- raw_spin_unlock(&dsq->lock); -- dsq = &scx_dsq_global; -- raw_spin_lock(&dsq->lock); -- } -- } -- -- if (enq_flags & SCX_ENQ_DSQ_PRIQ) { -- p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; -- rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq, -- scx_dsq_priq_less); -- } else { -- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) -- list_add(&p->scx->dsq_node.fifo, &dsq->fifo); -- else -- list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo); -- } -- dsq->nr++; -- p->scx->dsq = dsq; -- -- /* -- * We're transitioning out of QUEUEING or DISPATCHING. store_release to -- * match waiters' load_acquire. -- */ -- if (enq_flags & SCX_ENQ_CLEAR_OPSS) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- if (is_local) { -- struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq); -- struct rq *rq = scx->rq; -- bool preempt = false; -- -- if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && -- rq->curr->sched_class == &ext_sched_class) { -- rq->curr->scx->slice = 0; -- preempt = true; -- } -- -- if (preempt || sched_class_above(&ext_sched_class, -- rq->curr->sched_class)) -- resched_curr(rq); -- } else { -- raw_spin_unlock(&dsq->lock); -- } --} -- --static void task_unlink_from_dsq(struct task_struct *p, -- struct scx_dispatch_q *dsq) --{ -- if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { -- rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; -- } else { -- list_del_init(&p->scx->dsq_node.fifo); -- } -- --} -- --static bool task_linked_on_dsq(struct task_struct *p) --{ -- return !list_empty(&p->scx->dsq_node.fifo) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq); --} -- --static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p) --{ -- struct scx_dispatch_q *dsq = p->scx->dsq; -- bool is_local = dsq == &scx_rq->local_dsq; -- -- if (!dsq) { -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- /* -- * When dispatching directly from the BPF scheduler to a local -- * DSQ, the task isn't associated with any DSQ but -- * @p->scx->holding_cpu may be set under the protection of -- * %SCX_OPSS_DISPATCHING. -- */ -- if (p->scx->holding_cpu >= 0) -- p->scx->holding_cpu = -1; -- return; -- } -- -- if (!is_local) -- raw_spin_lock(&dsq->lock); -- -- /* -- * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node -- * can't change underneath us. -- */ -- if (p->scx->holding_cpu < 0) { -- /* @p must still be on @dsq, dequeue */ -- WARN_ON_ONCE(!task_linked_on_dsq(p)); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- } else { -- /* -- * We're racing against dispatch_to_local_dsq() which already -- * removed @p from @dsq and set @p->scx->holding_cpu. Clear the -- * holding_cpu which tells dispatch_to_local_dsq() that it lost -- * the race. -- */ -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- p->scx->holding_cpu = -1; -- } -- p->scx->dsq = NULL; -- -- if (!is_local) -- raw_spin_unlock(&dsq->lock); --} -- --static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) --{ -- lockdep_assert(rcu_read_lock_any_held()); -- -- return &scx_dsq_global; --} -- --static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, -- struct task_struct *p) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id == SCX_DSQ_LOCAL) -- return &rq->scx->local_dsq; -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", -- dsq_id, p->comm, p->pid); -- return &scx_dsq_global; -- } -- -- return dsq; --} -- --static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- -- /* @p must match the task which is being enqueued */ -- if (unlikely(p != ddsp_task)) { -- if (IS_ERR(ddsp_task)) -- scx_ops_error("%s[%d] already direct-dispatched", -- p->comm, p->pid); -- else -- scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]", -- ddsp_task->comm, ddsp_task->pid, -- p->comm, p->pid); -- return; -- } -- -- /* -- * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because -- * dispatching to the local DSQ of a different CPU requires unlocking -- * the current rq which isn't allowed in the enqueue path. Use -- * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL. -- */ -- if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) { -- scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch"); -- return; -- } -- -- touch_core_sched_dispatch(task_rq(p), p); -- -- dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- -- /* -- * Mark that dispatch already happened by spoiling direct_dispatch_task -- * with a non-NULL value which can never match a valid task pointer. -- */ -- __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); --} -- --static bool test_rq_online(struct rq *rq) --{ --#ifdef CONFIG_SMP -- return rq->online; --#else -- return true; --#endif --} -- --static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- int sticky_cpu) --{ -- struct task_struct **ddsp_taskp; -- u64 qseq; -- -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- if (p->scx->flags & SCX_TASK_ENQ_LOCAL) { -- enq_flags |= SCX_ENQ_LOCAL; -- p->scx->flags &= ~SCX_TASK_ENQ_LOCAL; -- } -- -- /* rq migration */ -- if (sticky_cpu == cpu_of(rq)) -- goto local_norefill; -- -- /* -- * If !rq->online, we already told the BPF scheduler that the CPU is -- * offline. We're just trying to on/offline the CPU. Don't bother the -- * BPF scheduler. -- */ -- if (unlikely(!test_rq_online(rq))) -- goto local; -- -- /* see %SCX_OPS_ENQ_EXITING */ -- if (!static_branch_unlikely(&scx_ops_enq_exiting) && -- unlikely(p->flags & PF_EXITING)) -- goto local; -- -- /* see %SCX_OPS_ENQ_LAST */ -- if (!static_branch_unlikely(&scx_ops_enq_last) && -- (enq_flags & SCX_ENQ_LAST)) -- goto local; -- -- if (!SCX_HAS_OP(enqueue)) { -- if (enq_flags & SCX_ENQ_LOCAL) -- goto local; -- else -- goto global; -- } -- -- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ -- qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; -- -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq); -- -- ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); -- WARN_ON_ONCE(*ddsp_taskp); -- *ddsp_taskp = p; -- -- SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); -- -- /* -- * If not directly dispatched, QUEUEING isn't clear yet and dispatch or -- * dequeue may be waiting. The store_release matches their load_acquire. -- */ -- if (*ddsp_taskp == p) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq); -- *ddsp_taskp = NULL; -- return; -- --local: -- /* -- * For task-ordering, slice refill must be treated as implying the end -- * of the current slice. Otherwise, the longer @p stays on the CPU, the -- * higher priority it becomes from scx_prio_less()'s POV. -- */ -- touch_core_sched(rq, p); -- p->scx->slice = SCX_SLICE_DFL; --local_norefill: -- dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags); -- return; -- --global: -- touch_core_sched(rq, p); /* see the comment in local: */ -- p->scx->slice = SCX_SLICE_DFL; -- dispatch_enqueue(&scx_dsq_global, p, enq_flags); --} -- --static bool watchdog_task_watched(const struct task_struct *p) --{ -- return !list_empty(&p->scx->watchdog_node); --} -- --static void watchdog_watch_task(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- if (p->scx->flags & SCX_TASK_WATCHDOG_RESET) -- p->scx->runnable_at = jiffies; -- p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET; -- list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list); --} -- --static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) --{ -- list_del_init(&p->scx->watchdog_node); -- if (reset_timeout) -- p->scx->flags |= SCX_TASK_WATCHDOG_RESET; --} -- --static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) --{ -- int sticky_cpu = p->scx->sticky_cpu; -- -- enq_flags |= rq->scx->extra_enq_flags; -- -- if (sticky_cpu >= 0) -- p->scx->sticky_cpu = -1; -- -- /* -- * Restoring a running task will be immediately followed by -- * set_next_task_scx() which expects the task to not be on the BPF -- * scheduler as tasks can only start running through local DSQs. Force -- * direct-dispatch into the local DSQ by setting the sticky_cpu. -- */ -- if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -- sticky_cpu = cpu_of(rq); -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- WARN_ON_ONCE(!watchdog_task_watched(p)); -- return; -- } -- -- watchdog_watch_task(rq, p); -- p->scx->flags |= SCX_TASK_QUEUED; -- rq->scx->nr_running++; -- add_nr_running(rq, 1); -- -- if (SCX_HAS_OP(runnable)) -- SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); -- -- if (enq_flags & SCX_ENQ_WAKEUP) -- touch_core_sched(rq, p); -- -- do_enqueue_task(rq, p, enq_flags, sticky_cpu); --} -- --static void ops_dequeue(struct task_struct *p, u64 deq_flags) --{ -- u64 opss; -- -- watchdog_unwatch_task(p, false); -- -- /* acquire ensures that we see the preceding updates on QUEUED */ -- opss = atomic64_read_acquire(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_NONE: -- break; -- case SCX_OPSS_QUEUEING: -- /* -- * QUEUEING is started and finished while holding @p's rq lock. -- * As we're holding the rq lock now, we shouldn't see QUEUEING. -- */ -- BUG(); -- case SCX_OPSS_QUEUED: -- if (SCX_HAS_OP(dequeue)) -- SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); -- -- if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_NONE)) -- break; -- fallthrough; -- case SCX_OPSS_DISPATCHING: -- /* -- * If @p is being dispatched from the BPF scheduler to a DSQ, -- * wait for the transfer to complete so that @p doesn't get -- * added to its DSQ after dequeueing is complete. -- * -- * As we're waiting on DISPATCHING with the rq locked, the -- * dispatching side shouldn't try to lock the rq while -- * DISPATCHING is set. See dispatch_to_local_dsq(). -- * -- * DISPATCHING shouldn't have qseq set and control can reach -- * here with NONE @opss from the above QUEUED case block. -- * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. -- */ -- wait_ops_state(p, SCX_OPSS_DISPATCHING); -- BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- break; -- } --} -- --static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) --{ -- struct scx_rq *scx_rq = rq->scx; -- -- if (!(p->scx->flags & SCX_TASK_QUEUED)) { -- WARN_ON_ONCE(watchdog_task_watched(p)); -- return; -- } -- -- ops_dequeue(p, deq_flags); -- -- /* -- * A currently running task which is going off @rq first gets dequeued -- * and then stops running. As we want running <-> stopping transitions -- * to be contained within runnable <-> quiescent transitions, trigger -- * ->stopping() early here instead of in put_prev_task_scx(). -- * -- * @p may go through multiple stopping <-> running transitions between -- * here and put_prev_task_scx() if task attribute changes occur while -- * balance_scx() leaves @rq unlocked. However, they don't contain any -- * information meaningful to the BPF scheduler and can be suppressed by -- * skipping the callbacks if the task is !QUEUED. -- */ -- if (SCX_HAS_OP(stopping) && task_current(rq, p)) { -- update_curr_scx(rq); -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); -- } -- -- //if(task_current(rq, p)) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- if (SCX_HAS_OP(quiescent)) -- SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); -- -- if (deq_flags & SCX_DEQ_SLEEP) -- p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP; -- else -- p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP; -- -- p->scx->flags &= ~SCX_TASK_QUEUED; -- BUG_ON(!scx_rq->nr_running); -- scx_rq->nr_running--; -- sub_nr_running(rq, 1); -- -- dispatch_dequeue(scx_rq, p); --} -- --static void yield_task_scx(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); -- else -- p->scx->slice = 0; --} -- --static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) --{ -- struct task_struct *from = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); -- else -- return false; --} -- --#ifdef CONFIG_SMP --/** -- * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -- * @rq: rq to move the task into, currently locked -- * @p: task to move -- * @enq_flags: %SCX_ENQ_* -- * -- * Move @p which is currently on a different rq to @rq's local DSQ. The caller -- * must: -- * -- * 1. Start with exclusive access to @p either through its DSQ lock or -- * %SCX_OPSS_DISPATCHING flag. -- * -- * 2. Set @p->scx->holding_cpu to raw_smp_processor_id(). -- * -- * 3. Remember task_rq(@p). Release the exclusive access so that we don't -- * deadlock with dequeue. -- * -- * 4. Lock @rq and the task_rq from #3. -- * -- * 5. Call this function. -- * -- * Returns %true if @p was successfully moved. %false after racing dequeue and -- * losing. -- */ --static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -- u64 enq_flags) --{ -- struct rq *task_rq; -- -- lockdep_assert_rq_held(rq); -- -- /* -- * If dequeue got to @p while we were trying to lock both rq's, it'd -- * have cleared @p->scx->holding_cpu to -1. While other cpus may have -- * updated it to different values afterwards, as this operation can't be -- * preempted or recurse, @p->scx->holding_cpu can never become -- * raw_smp_processor_id() again before we're done. Thus, we can tell -- * whether we lost to dequeue by testing whether @p->scx->holding_cpu is -- * still raw_smp_processor_id(). -- * -- * See dispatch_dequeue() for the counterpart. -- */ -- if (unlikely(p->scx->holding_cpu != raw_smp_processor_id())) -- return false; -- -- /* @p->rq couldn't have changed if we're still the holding cpu */ -- task_rq = task_rq(p); -- lockdep_assert_rq_held(task_rq); -- -- WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)); -- deactivate_task(task_rq, p, 0); -- set_task_cpu(p, cpu_of(rq)); -- p->scx->sticky_cpu = cpu_of(rq); -- -- /* -- * We want to pass scx-specific enq_flags but activate_task() will -- * truncate the upper 32 bit. As we own @rq, we can pass them through -- * @rq->scx->extra_enq_flags instead. -- */ -- WARN_ON_ONCE(rq->scx->extra_enq_flags); -- rq->scx->extra_enq_flags = enq_flags; -- activate_task(rq, p, 0); -- rq->scx->extra_enq_flags = 0; -- -- return true; --} -- --/** -- * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * We're holding @rq lock and trying to dispatch a task from @src_rq to -- * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether -- * @rq stays locked isn't important as long as the state is restored after -- * dispatch_to_local_dsq_unlock(). -- */ --static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- rq_unpin_lock(rq, rf); -- -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(rq); -- raw_spin_rq_lock(dst_rq); -- } else if (rq == src_rq) { -- double_lock_balance(rq, dst_rq); -- rq_repin_lock(rq, rf); -- } else if (rq == dst_rq) { -- double_lock_balance(rq, src_rq); -- rq_repin_lock(rq, rf); -- } else { -- raw_spin_rq_unlock(rq); -- double_rq_lock(src_rq, dst_rq); -- } --} -- --/** -- * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock() -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return. -- */ --static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } else if (rq == src_rq) { -- double_unlock_balance(rq, dst_rq); -- } else if (rq == dst_rq) { -- double_unlock_balance(rq, src_rq); -- } else { -- double_rq_unlock(src_rq, dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } --} --#endif /* CONFIG_SMP */ -- -- --static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq) --{ -- return likely(test_rq_online(rq)) && !is_migration_disabled(p) && -- cpumask_test_cpu(cpu_of(rq), p->cpus_ptr); --} -- --static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -- struct scx_dispatch_q *dsq) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rb_node *rb_node; -- struct rq *task_rq; -- bool moved = false; --retry: -- if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -- return false; -- -- raw_spin_lock(&dsq->lock); -- -- list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- for (rb_node = rb_first_cached(&dsq->priq); rb_node; -- rb_node = rb_next(rb_node)) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- raw_spin_unlock(&dsq->lock); -- return false; -- --this_rq: -- /* @dsq is locked and @p is on this rq */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo); -- dsq->nr--; -- scx_rq->local_dsq.nr++; -- p->scx->dsq = &scx_rq->local_dsq; -- raw_spin_unlock(&dsq->lock); -- return true; -- --remote_rq: --#ifdef CONFIG_SMP -- /* -- * @dsq is locked and @p is on a remote rq. @p is currently protected by -- * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -- * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -- * rq lock or fail, do a little dancing from our side. See -- * move_task_to_local_dsq(). -- */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- p->scx->holding_cpu = raw_smp_processor_id(); -- raw_spin_unlock(&dsq->lock); -- -- rq_unpin_lock(rq, rf); -- double_lock_balance(rq, task_rq); -- rq_repin_lock(rq, rf); -- -- moved = move_task_to_local_dsq(rq, p, 0); -- -- double_unlock_balance(rq, task_rq); --#endif /* CONFIG_SMP */ -- if (likely(moved)) -- return true; -- goto retry; --} -- --enum dispatch_to_local_dsq_ret { -- DTL_DISPATCHED, /* successfully dispatched */ -- DTL_LOST, /* lost race to dequeue */ -- DTL_NOT_LOCAL, /* destination is not a local DSQ */ -- DTL_INVALID, /* invalid local dsq_id */ --}; -- --/** -- * dispatch_to_local_dsq - Dispatch a task to a local dsq -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @dsq_id: destination dsq ID -- * @p: task to dispatch -- * @enq_flags: %SCX_ENQ_* -- * -- * We're holding @rq lock and want to dispatch @p to the local DSQ identified by -- * @dsq_id. This function performs all the synchronization dancing needed -- * because local DSQs are protected with rq locks. -- * -- * The caller must have exclusive ownership of @p (e.g. through -- * %SCX_OPSS_DISPATCHING). -- */ --static enum dispatch_to_local_dsq_ret --dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id, -- struct task_struct *p, u64 enq_flags) --{ -- struct rq *src_rq = task_rq(p); -- struct rq *dst_rq; -- -- /* -- * We're synchronized against dequeue through DISPATCHING. As @p can't -- * be dequeued, its task_rq and cpus_allowed are stable too. -- */ -- if (dsq_id == SCX_DSQ_LOCAL) { -- dst_rq = rq; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]", -- cpu, p->comm, p->pid); -- return DTL_INVALID; -- } -- dst_rq = cpu_rq(cpu); -- } else { -- return DTL_NOT_LOCAL; -- } -- -- /* if dispatching to @rq that @p is already on, no lock dancing needed */ -- if (rq == src_rq && rq == dst_rq) { -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags | SCX_ENQ_CLEAR_OPSS); -- return DTL_DISPATCHED; -- } -- --#ifdef CONFIG_SMP -- if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { -- struct rq *locked_dst_rq = dst_rq; -- bool dsp; -- -- /* -- * @p is on a possibly remote @src_rq which we need to lock to -- * move the task. If dequeue is in progress, it'd be locking -- * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq -- * lock while holding DISPATCHING. -- * -- * As DISPATCHING guarantees that @p is wholly ours, we can -- * pretend that we're moving from a DSQ and use the same -- * mechanism - mark the task under transfer with holding_cpu, -- * release DISPATCHING and then follow the same protocol. -- */ -- p->scx->holding_cpu = raw_smp_processor_id(); -- -- /* store_release ensures that dequeue sees the above */ -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq); -- -- /* -- * We don't require the BPF scheduler to avoid dispatching to -- * offline CPUs mostly for convenience but also because CPUs can -- * go offline between scx_bpf_dispatch() calls and here. If @p -- * is destined to an offline CPU, queue it on its current CPU -- * instead, which should always be safe. As this is an allowed -- * behavior, don't trigger an ops error. -- */ -- if (unlikely(!test_rq_online(dst_rq))) -- dst_rq = src_rq; -- -- if (src_rq == dst_rq) { -- /* -- * As @p is staying on the same rq, there's no need to -- * go through the full deactivate/activate cycle. -- * Optimize by abbreviating the operations in -- * move_task_to_local_dsq(). -- */ -- dsp = p->scx->holding_cpu == raw_smp_processor_id(); -- if (likely(dsp)) { -- p->scx->holding_cpu = -1; -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags); -- } -- } else { -- dsp = move_task_to_local_dsq(dst_rq, p, enq_flags); -- } -- -- /* if the destination CPU is idle, wake it up */ -- if (dsp && p->sched_class > dst_rq->curr->sched_class) -- resched_curr(dst_rq); -- -- dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq); -- -- return dsp ? DTL_DISPATCHED : DTL_LOST; -- } --#endif /* CONFIG_SMP */ -- -- scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", -- cpu_of(dst_rq), p->comm, p->pid); -- return DTL_INVALID; --} -- --/** -- * finish_dispatch - Asynchronously finish dispatching a task -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @p: task to finish dispatching -- * @qseq_at_dispatch: qseq when @p started getting dispatched -- * @dsq_id: destination DSQ ID -- * @enq_flags: %SCX_ENQ_* -- * -- * Dispatching to local DSQs may need to wait for queueing to complete or -- * require rq lock dancing. As we don't wanna do either while inside -- * ops.dispatch() to avoid locking order inversion, we split dispatching into -- * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the -- * task and its qseq. Once ops.dispatch() returns, this function is called to -- * finish up. -- * -- * There is no guarantee that @p is still valid for dispatching or even that it -- * was valid in the first place. Make sure that the task is still owned by the -- * BPF scheduler and claim the ownership before dispatching. -- */ --static void finish_dispatch(struct rq *rq, struct rq_flags *rf, -- struct task_struct *p, u64 qseq_at_dispatch, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- u64 opss; -- -- touch_core_sched_dispatch(rq, p); --retry: -- /* -- * No need for _acquire here. @p is accessed only after a successful -- * try_cmpxchg to DISPATCHING. -- */ -- opss = atomic64_read(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_DISPATCHING: -- case SCX_OPSS_NONE: -- /* someone else already got to it */ -- return; -- case SCX_OPSS_QUEUED: -- /* -- * If qseq doesn't match, @p has gone through at least one -- * dispatch/dequeue and re-enqueue cycle between -- * scx_bpf_dispatch() and here and we have no claim on it. -- */ -- if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) -- return; -- -- /* -- * While we know @p is accessible, we don't yet have a claim on -- * it - the BPF scheduler is allowed to dispatch tasks -- * spuriously and there can be a racing dequeue attempt. Let's -- * claim @p by atomically transitioning it from QUEUED to -- * DISPATCHING. -- */ -- if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_DISPATCHING))) -- break; -- goto retry; -- case SCX_OPSS_QUEUEING: -- /* -- * do_enqueue_task() is in the process of transferring the task -- * to the BPF scheduler while holding @p's rq lock. As we aren't -- * holding any kernel or BPF resource that the enqueue path may -- * depend upon, it's safe to wait. -- */ -- wait_ops_state(p, opss); -- goto retry; -- } -- -- BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) { -- case DTL_DISPATCHED: -- break; -- case DTL_LOST: -- break; -- case DTL_INVALID: -- dsq_id = SCX_DSQ_GLOBAL; -- fallthrough; -- case DTL_NOT_LOCAL: -- dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()), -- dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- break; -- } --} -- --static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- u32 u; -- -- for (u = 0; u < dspc->buf_cursor; u++) { -- struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u]; -- -- finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id, -- ent->enq_flags); -- } -- -- dspc->nr_tasks += dspc->buf_cursor; -- dspc->buf_cursor = 0; --} -- --static int balance_one(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf, bool local) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- bool prev_on_scx = prev->sched_class == &ext_sched_class; -- int nr_loops = SCX_DSP_MAX_LOOPS; -- -- lockdep_assert_rq_held(rq); -- -- if (static_branch_unlikely(&scx_ops_cpu_preempt) && -- unlikely(rq->scx->cpu_released)) { -- /* -- * If the previous sched_class for the current CPU was not SCX, -- * notify the BPF scheduler that it again has control of the -- * core. This callback complements ->cpu_release(), which is -- * emitted in scx_notify_pick_next_task(). -- */ -- if (SCX_HAS_OP(cpu_acquire)) -- SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq), -- NULL); -- rq->scx->cpu_released = false; -- } -- -- if (prev_on_scx) { -- WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP)); -- update_curr_scx(rq); -- -- /* -- * If @prev is runnable & has slice left, it has priority and -- * fetching more just increases latency for the fetched tasks. -- * Tell put_prev_task_scx() to put @prev on local_dsq. If the -- * BPF scheduler wants to handle this explicitly, it should -- * implement ->cpu_released(). -- * -- * See scx_ops_disable_workfn() for the explanation on the -- * disabling() test. -- * -- * When balancing a remote CPU for core-sched, there won't be a -- * following put_prev_task_scx() call and we don't own -- * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the -- * same conditions later and pick @rq->curr accordingly. -- */ -- if ((prev->scx->flags & SCX_TASK_QUEUED) && -- prev->scx->slice && !scx_ops_disabling()) { -- if (local) -- prev->scx->flags |= SCX_TASK_BAL_KEEP; -- return 1; -- } -- } -- -- /* if there already are tasks to run, nothing to do */ -- if (scx_rq->local_dsq.nr) -- return 1; -- -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- if (!SCX_HAS_OP(dispatch)) -- return 0; -- -- dspc->rq = rq; -- dspc->rf = rf; -- -- /* -- * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, -- * the local DSQ might still end up empty after a successful -- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() -- * produced some tasks, retry. The BPF scheduler may depend on this -- * looping behavior to simplify its implementation. -- */ -- do { -- dspc->nr_tasks = 0; -- -- SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), -- prev_on_scx ? prev : NULL); -- -- flush_dispatch_buf(rq, rf); -- -- if (scx_rq->local_dsq.nr) -- return 1; -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- /* -- * ops.dispatch() can trap us in this loop by repeatedly -- * dispatching ineligible tasks. Break out once in a while to -- * allow the watchdog to run. As IRQ can't be enabled in -- * balance(), we want to complete this scheduling cycle and then -- * start a new one. IOW, we want to call resched_curr() on the -- * next, most likely idle, task, not the current one. Use -- * scx_bpf_kick_cpu() for deferred kicking. -- */ -- if (unlikely(!--nr_loops)) { -- scx_bpf_kick_cpu(cpu_of(rq), 0); -- break; -- } -- } while (dspc->nr_tasks); -- -- return 0; --} -- --static int balance_scx(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf) --{ -- int ret; -- -- ret = balance_one(rq, prev, rf, true); --#ifdef CONFIG_SCHED_SMT -- /* -- * When core-sched is enabled, this ops.balance() call will be followed -- * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx() -- * on the SMT siblings. Balance the siblings too. -- */ -- if (sched_core_enabled(rq)) { -- const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); -- int scpu; -- -- for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { -- struct rq *srq = cpu_rq(scpu); -- struct rq_flags srf; -- struct task_struct *sprev = srq->curr; -- -- /* -- * While core-scheduling, rq lock is shared among -- * siblings but the debug annotations and rq clock -- * aren't. Do pinning dance to transfer the ownership. -- */ -- WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); -- rq_unpin_lock(rq, rf); -- rq_pin_lock(srq, &srf); -- -- update_rq_clock(srq); -- balance_one(srq, sprev, &srf, false); -- -- rq_unpin_lock(srq, &srf); -- rq_repin_lock(rq, rf); -- } -- } --#endif -- return ret; --} -- --static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) --{ -- if (p->scx->flags & SCX_TASK_QUEUED) { -- /* -- * Core-sched might decide to execute @p before it is -- * dispatched. Call ops_dequeue() to notify the BPF scheduler. -- */ -- ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); -- dispatch_dequeue(rq->scx, p); -- } -- -- p->se.exec_start = rq_clock_task(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, running, p); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock); -- -- watchdog_unwatch_task(p, true); -- -- /* -- * @p is getting newly scheduled or got kicked after someone updated its -- * slice. Refresh whether tick can be stopped. See can_stop_tick_scx(). -- */ -- if ((p->scx->slice == SCX_SLICE_INF) != -- (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) { -- if (p->scx->slice == SCX_SLICE_INF) -- rq->scx->flags |= SCX_RQ_CAN_STOP_TICK; -- else -- rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK; -- -- sched_update_tick_dependency(rq); -- } --} -- --static void put_prev_task_scx(struct rq *rq, struct task_struct *p) --{ --#ifndef CONFIG_SMP -- /* -- * UP workaround. -- * -- * Because SCX may transfer tasks across CPUs during dispatch, dispatch -- * is performed from its balance operation which isn't called in UP. -- * Let's work around by calling it from the operations which come right -- * after. -- * -- * 1. If the prev task is on SCX, pick_next_task() calls -- * .put_prev_task() right after. As .put_prev_task() is also called -- * from other places, we need to distinguish the calls which can be -- * done by looking at the previous task's state - if still queued or -- * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task(). -- * This case is handled here. -- * -- * 2. If the prev task is not on SCX, the first following call into SCX -- * will be .pick_next_task(), which is covered by calling -- * balance_scx() from pick_next_task_scx(). -- * -- * Note that we can't merge the first case into the second as -- * balance_scx() must be called before the previous SCX task goes -- * through put_prev_task_scx(). -- * -- * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf. -- * Pass in %NULL. -- */ -- if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP)) -- balance_scx(rq, p, NULL); --#endif -- -- update_curr_scx(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- -- /* -- * If we're being called from put_prev_task_balance(), balance_scx() may -- * have decided that @p should keep running. -- */ -- if (p->scx->flags & SCX_TASK_BAL_KEEP) { -- p->scx->flags &= ~SCX_TASK_BAL_KEEP; -- watchdog_watch_task(rq, p); -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- watchdog_watch_task(rq, p); -- -- /* -- * If @p has slice left and balance_scx() didn't tag it for -- * keeping, @p is getting preempted by a higher priority -- * scheduler class or core-sched forcing a different task. Leave -- * it at the head of the local DSQ. -- */ -- if (p->scx->slice && !scx_ops_disabling()) { -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- /* -- * If we're in the pick_next_task path, balance_scx() should -- * have already populated the local DSQ if there are any other -- * available tasks. If empty, tell ops.enqueue() that @p is the -- * only one available for this cpu. ops.enqueue() should put it -- * on the local DSQ so that the subsequent pick_next_task_scx() -- * can find the task unless it wants to trigger a separate -- * follow-up scheduling event. -- */ -- if (list_empty(&rq->scx->local_dsq.fifo)) -- do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1); -- else -- do_enqueue_task(rq, p, 0, -1); -- } --} -- --static struct task_struct *first_local_task(struct rq *rq) --{ -- struct rb_node *rb_node; -- struct sched_ext_entity *entity; -- -- if (!list_empty(&rq->scx->local_dsq.fifo)) { -- entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo); -- return entity->task; -- } -- -- rb_node = rb_first_cached(&rq->scx->local_dsq.priq); -- if (rb_node) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- return entity->task; -- } -- -- return NULL; --} -- --static struct task_struct *pick_next_task_scx(struct rq *rq) --{ -- struct task_struct *p; -- --#ifndef CONFIG_SMP -- /* UP workaround - see the comment at the head of put_prev_task_scx() */ -- if (unlikely(rq->curr->sched_class != &ext_sched_class)) -- balance_scx(rq, rq->curr, NULL); --#endif -- -- p = first_local_task(rq); -- if (!p) -- return NULL; -- -- if (unlikely(!p->scx->slice)) { -- if (!scx_ops_disabling() && !scx_warned_zero_slice) { -- printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n", -- p->comm, p->pid); -- scx_warned_zero_slice = true; -- } -- p->scx->slice = SCX_SLICE_DFL; -- } -- -- set_next_task_scx(rq, p, true); -- -- return p; --} -- --#ifdef CONFIG_SCHED_CORE --/** -- * scx_prio_less - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Core-sched is implemented as an additional scheduling layer on top of the -- * usual sched_class'es and needs to find out the expected task ordering. For -- * SCX, core-sched calls this function to interrogate the task ordering. -- * -- * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used -- * to implement the default task ordering. The older the timestamp, the higher -- * prority the task - the global FIFO ordering matching the default scheduling -- * behavior. -- * -- * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to -- * implement FIFO ordering within each local DSQ. See pick_task_scx(). -- */ --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi) --{ -- /* -- * The const qualifiers are dropped from task_struct pointers when -- * calling ops.core_sched_before(). Accesses are controlled by the -- * verifier. -- */ -- if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling()) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, -- (struct task_struct *)a, -- (struct task_struct *)b); -- else -- return time_after64(a->scx->core_sched_at, b->scx->core_sched_at); --} -- --/** -- * pick_task_scx - Pick a candidate task for core-sched -- * @rq: rq to pick the candidate task from -- * -- * Core-sched calls this function on each SMT sibling to determine the next -- * tasks to run on the SMT siblings. balance_one() has been called on all -- * siblings and put_prev_task_scx() has been called only for the current CPU. -- * -- * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look -- * at the first task in the local dsq. @rq->curr has to be considered explicitly -- * to mimic %SCX_TASK_BAL_KEEP. -- */ --static struct task_struct *pick_task_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- struct task_struct *first = first_local_task(rq); -- -- if (curr->scx->flags & SCX_TASK_QUEUED) { -- /* is curr the only runnable task? */ -- if (!first) -- return curr; -- -- /* -- * Does curr trump first? We can always go by core_sched_at for -- * this comparison as it represents global FIFO ordering when -- * the default core-sched ordering is used and local-DSQ FIFO -- * ordering otherwise. -- * -- * We can have a task with an earlier timestamp on the DSQ. For -- * example, when a current task is preempted by a sibling -- * picking a different cookie, the task would be requeued at the -- * head of the local DSQ with an earlier timestamp than the -- * core-sched picked next task. Besides, the BPF scheduler may -- * dispatch any tasks to the local DSQ anytime. -- */ -- if (curr->scx->slice && time_before64(curr->scx->core_sched_at, -- first->scx->core_sched_at)) -- return curr; -- } -- -- return first; /* this may be %NULL */ --} --#endif /* CONFIG_SCHED_CORE */ -- --static enum scx_cpu_preempt_reason --preempt_reason_from_class(const struct sched_class *class) --{ --#ifdef CONFIG_SMP -- if (class == &stop_sched_class) -- return SCX_CPU_PREEMPT_STOP; --#endif -- if (class == &dl_sched_class) -- return SCX_CPU_PREEMPT_DL; -- if (class == &rt_sched_class) -- return SCX_CPU_PREEMPT_RT; -- return SCX_CPU_PREEMPT_UNKNOWN; --} -- --void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task, -- const struct sched_class *active) --{ -- lockdep_assert_rq_held(rq); -- -- /* -- * The callback is conceptually meant to convey that the CPU is no -- * longer under the control of SCX. Therefore, don't invoke the -- * callback if the CPU is is staying on SCX, or going idle (in which -- * case the SCX scheduler has actively decided not to schedule any -- * tasks on the CPU). -- */ -- if (likely(active >= &ext_sched_class)) -- return; -- -- /* -- * At this point we know that SCX was preempted by a higher priority -- * sched_class, so invoke the ->cpu_release() callback if we have not -- * done so already. We only send the callback once between SCX being -- * preempted, and it regaining control of the CPU. -- * -- * ->cpu_release() complements ->cpu_acquire(), which is emitted the -- * next time that balance_scx() is invoked. -- */ -- if (!rq->scx->cpu_released) { -- if (SCX_HAS_OP(cpu_release)) { -- struct scx_cpu_release_args args = { -- .reason = preempt_reason_from_class(active), -- .task = task, -- }; -- -- SCX_CALL_OP(SCX_KF_CPU_RELEASE, -- cpu_release, cpu_of(rq), &args); -- } -- rq->scx->cpu_released = true; -- } --} -- --#ifdef CONFIG_SMP -- --static bool test_and_clear_cpu_idle(int cpu) --{ -- if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -- if (cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- return true; -- } else { -- return false; -- } --} -- --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- int cpu; -- -- do { -- cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); -- if (cpu < nr_cpu_ids) { -- const struct cpumask *sbm = topology_sibling_cpumask(cpu); -- -- /* -- * If offline, @cpu is not its own sibling and we can -- * get caught in an infinite loop as @cpu is never -- * cleared from idle_masks.smt. Clear @cpu directly in -- * such cases. -- */ -- if (likely(cpumask_test_cpu(cpu, sbm))) -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm); -- else -- cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu)); -- } else { -- cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -- if (cpu >= nr_cpu_ids) -- return -EBUSY; -- } -- } while (!test_and_clear_cpu_idle(cpu)); -- -- return cpu; --} -- --static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) --{ -- s32 cpu; -- -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return prev_cpu; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local DSQ of the waker. -- */ -- if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 && -- scx_has_idle_cpus && !(current->flags & PF_EXITING)) { -- cpu = smp_processor_id(); -- if (cpumask_test_cpu(cpu, p->cpus_ptr)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (test_and_clear_cpu_idle(prev_cpu)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return prev_cpu; -- } -- -- if (p->nr_cpus_allowed == 1) -- return prev_cpu; -- -- cpu = scx_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- -- return prev_cpu; --} -- --static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) --{ -- if (SCX_HAS_OP(select_cpu)) { -- s32 cpu; -- -- cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu, -- wake_flags); -- if (ops_cpu_valid(cpu)) { -- return cpu; -- } else { -- scx_ops_error("select_cpu returned invalid cpu %d", cpu); -- return prev_cpu; -- } -- } else { -- return scx_select_cpu_dfl(p, prev_cpu, wake_flags); -- } --} -- --static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx) --{ -- set_cpus_allowed_common(p, ctx); -- -- /* -- * The effective cpumask is stored in @p->cpus_ptr which may temporarily -- * differ from the configured one in @p->cpus_mask. Always tell the bpf -- * scheduler the effective one. -- * -- * Fine-grained memory write control is enforced by BPF making the const -- * designation pointless. Cast it away when calling the operation. -- */ -- if (SCX_HAS_OP(set_cpumask)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, -- (struct cpumask *)p->cpus_ptr); --} -- --static void reset_idle_masks(void) --{ -- /* consider all cpus idle, should converge to the actual state quickly */ -- cpumask_setall(idle_masks.cpu); -- cpumask_setall(idle_masks.smt); -- scx_has_idle_cpus = true; --} -- --void __scx_update_idle(struct rq *rq, bool idle) --{ -- int cpu = cpu_of(rq); -- struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -- -- if (SCX_HAS_OP(update_idle)) { -- SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); -- if (!static_branch_unlikely(&scx_builtin_idle_enabled)) -- return; -- } -- -- if (idle) { -- cpumask_set_cpu(cpu, idle_masks.cpu); -- if (!scx_has_idle_cpus) -- scx_has_idle_cpus = true; -- -- /* -- * idle_masks.smt handling is racy but that's fine as it's only -- * for optimization and self-correcting. -- */ -- for_each_cpu(cpu, sib_mask) { -- if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -- return; -- } -- cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -- } else { -- cpumask_clear_cpu(cpu, idle_masks.cpu); -- if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -- } --} -- --#else /* !CONFIG_SMP */ -- --static bool test_and_clear_cpu_idle(int cpu) { return false; } --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } --static void reset_idle_masks(void) {} -- --#endif /* CONFIG_SMP */ -- --static bool check_rq_for_timeouts(struct rq *rq) --{ -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rq_flags rf; -- bool timed_out = false; -- -- rq_lock_irqsave(rq, &rf); -- list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) { -- unsigned long last_runnable; -- -- p = entity->task; -- last_runnable = p->scx->runnable_at; -- -- if (unlikely(time_after(jiffies, -- last_runnable + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "%s[%d] failed to run for %u.%03us", -- p->comm, p->pid, -- dur_ms / 1000, dur_ms % 1000); -- timed_out = true; -- break; -- } -- } -- rq_unlock_irqrestore(rq, &rf); -- -- return timed_out; --} -- --static void scx_watchdog_workfn(struct work_struct *work) --{ -- int cpu; -- -- scx_watchdog_timestamp = jiffies; -- -- for_each_online_cpu(cpu) { -- if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) -- break; -- -- cond_resched(); -- } -- queue_delayed_work(system_unbound_wq, to_delayed_work(work), -- scx_watchdog_timeout / 2); --} -- --static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) --{ -- update_curr_scx(rq); -- //scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock); -- /* -- * While disabling, always resched and refresh core-sched timestamp as -- * we can't trust the slice management or ops.core_sched_before(). -- */ -- if (scx_ops_disabling()) { -- curr->scx->slice = 0; -- touch_core_sched(rq, curr); -- } -- -- if (!curr->scx->slice) -- resched_curr(rq); --} -- --#define SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- --static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg) --{ -- int ret; -- -- WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED); -- -- p->scx->disallow = false; -- -- if (SCX_HAS_OP(prep_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- }; -- -- ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args); -- if (unlikely(ret)) { -- ret = ops_sanitize_err("prep_enable", ret); -- return ret; -- } -- } -- //scx_sched_init_task(p); -- -- if (p->scx->disallow) { -- struct rq *rq; -- struct rq_flags rf; -- -- rq = task_rq_lock(p, &rf); -- -- /* -- * We're either in fork or load path and @p->policy will be -- * applied right after. Reverting @p->policy here and rejecting -- * %SCHED_EXT transitions from scx_check_setscheduler() -- * guarantees that if ops.prep_enable() sets @p->disallow, @p -- * can never be in SCX. -- */ -- if (p->policy == SCHED_EXT) { -- p->policy = SCHED_NORMAL; -- atomic64_inc(&scx_nr_rejected); -- } -- -- task_rq_unlock(rq, p, &rf); -- } -- -- p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET); -- return 0; --} -- --static void scx_ops_enable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED)); -- -- if (SCX_HAS_OP(enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- p->scx->flags |= SCX_TASK_OPS_ENABLED; --} -- --static void scx_ops_disable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- if (p->scx->flags & SCX_TASK_OPS_PREPPED) { -- if (SCX_HAS_OP(cancel_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- } else if (p->scx->flags & SCX_TASK_OPS_ENABLED) { -- if (SCX_HAS_OP(disable)) -- SCX_CALL_OP(SCX_KF_REST, disable, p); -- p->scx->flags &= ~SCX_TASK_OPS_ENABLED; -- } --} -- --static void set_task_scx_weight(struct task_struct *p) --{ -- u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -- -- p->scx->weight = sched_weight_to_cgroup(weight); --} -- --/** -- * refresh_scx_weight - Refresh a task's ext weight -- * @p: task to refresh ext weight for -- * -- * @p->scx->weight carries the task's static priority in cgroup weight scale to -- * enable easy access from the BPF scheduler. To keep it synchronized with the -- * current task priority, this function should be called when a new task is -- * created, priority is changed for a task on sched_ext, and a task is switched -- * to sched_ext from other classes. -- */ --static void refresh_scx_weight(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- set_task_scx_weight(p); -- if (SCX_HAS_OP(set_weight)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight); --} -- --void scx_pre_fork(struct task_struct *p) --{ -- p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL); -- if (!p->scx) { -- goto lock; -- } -- -- p->scx->dsq = NULL; -- INIT_LIST_HEAD(&p->scx->dsq_node.fifo); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- INIT_LIST_HEAD(&p->scx->watchdog_node); -- p->scx->flags = 0; -- p->scx->weight = 0; -- p->scx->sticky_cpu = -1; -- p->scx->holding_cpu = -1; -- p->scx->kf_mask = 0; -- atomic64_set(&p->scx->ops_state, 0); -- p->scx->runnable_at = INITIAL_JIFFIES; -- p->scx->slice = SCX_SLICE_DFL; -- p->scx->task = p; -- p->sched_prop = 0; -- -- /* -- * BPF scheduler enable/disable paths want to be able to iterate and -- * update all tasks which can become complex when racing forks. As -- * enable/disable are very cold paths, let's use a percpu_rwsem to -- * exclude forks. -- */ --lock: -- percpu_down_read(&scx_fork_rwsem); --} -- --int scx_fork(struct task_struct *p) --{ -- percpu_rwsem_assert_held(&scx_fork_rwsem); -- -- if (scx_enabled()) -- return scx_ops_prepare_task(p, task_group(p)); -- else -- return 0; --} -- --void scx_post_fork(struct task_struct *p) --{ -- if (scx_enabled()) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- /* -- * Set the weight manually before calling ops.enable() so that -- * the scheduler doesn't see a stale value if they inspect the -- * task struct. We'll invoke ops.set_weight() afterwards, as it -- * would be odd to receive a callback on the task before we -- * tell the scheduler that it's been fully enabled. -- */ -- set_task_scx_weight(p); -- scx_ops_enable_task(p); -- refresh_scx_weight(p); -- task_rq_unlock(rq, p, &rf); -- } -- -- spin_lock_irq(&scx_tasks_lock); -- list_add_tail(&p->scx->tasks_node, &scx_tasks); -- spin_unlock_irq(&scx_tasks_lock); -- -- percpu_up_read(&scx_fork_rwsem); --} -- --void scx_cancel_fork(struct task_struct *p) --{ -- if (scx_enabled()) -- scx_ops_disable_task(p); -- percpu_up_read(&scx_fork_rwsem); --} -- --void sched_ext_free(struct task_struct *p) --{ -- unsigned long flags; -- -- spin_lock_irqsave(&scx_tasks_lock, flags); -- list_del_init(&p->scx->tasks_node); -- spin_unlock_irqrestore(&scx_tasks_lock, flags); -- -- /* -- * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED -> -- * ENABLED transitions can't race us. Disable ops for @p. -- */ -- if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- scx_ops_disable_task(p); -- task_rq_unlock(rq, p, &rf); -- } --} -- --static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) --{ --} -- --static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} --static void switched_to_scx(struct rq *rq, struct task_struct *p) {} -- --int scx_check_setscheduler(struct task_struct *p, int policy) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- /* if disallow, reject transitioning into SCX */ -- if (scx_enabled() && READ_ONCE(p->scx->disallow) && -- p->policy != policy && policy == SCHED_EXT) -- return -EACCES; -- -- return 0; --} -- --#ifdef CONFIG_NO_HZ_FULL --bool scx_can_stop_tick(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (scx_ops_disabling()) -- return false; -- -- if (p->sched_class != &ext_sched_class) -- return true; -- -- /* -- * @rq can dispatch from different DSQs, so we can't tell whether it -- * needs the tick or not by looking at nr_running. Allow stopping ticks -- * iff the BPF scheduler indicated so. See set_next_task_scx(). -- */ -- return rq->scx->flags & SCX_RQ_CAN_STOP_TICK; --} --#endif -- --static inline void scx_cgroup_lock(void) {} --static inline void scx_cgroup_unlock(void) {} -- --/* -- * Omitted operations: -- * -- * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -- * task isn't tied to the CPU at that point. Preemption is implemented by -- * resetting the victim task's slice to 0 and triggering reschedule on the -- * target CPU. -- * -- * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -- * -- * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -- * their current sched_class. Call them directly from sched core instead. -- * -- * - task_woken, switched_from: Unnecessary. -- */ --DEFINE_SCHED_CLASS(ext) = { -- .enqueue_task = enqueue_task_scx, -- .dequeue_task = dequeue_task_scx, -- .yield_task = yield_task_scx, -- .yield_to_task = yield_to_task_scx, -- -- .check_preempt_curr = check_preempt_curr_scx, -- -- .pick_next_task = pick_next_task_scx, -- -- .put_prev_task = put_prev_task_scx, -- .set_next_task = set_next_task_scx, -- --#ifdef CONFIG_SMP -- .balance = balance_scx, -- .select_task_rq = select_task_rq_scx, -- .set_cpus_allowed = set_cpus_allowed_scx, --#endif -- --#ifdef CONFIG_SCHED_CORE -- .pick_task = pick_task_scx, --#endif -- -- .task_tick = task_tick_scx, -- -- .switched_to = switched_to_scx, -- .prio_changed = prio_changed_scx, -- -- .update_curr = update_curr_scx, -- --#ifdef CONFIG_UCLAMP_TASK -- .uclamp_enabled = 0, --#endif --}; -- --static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) --{ -- memset(dsq, 0, sizeof(*dsq)); -- -- raw_spin_lock_init(&dsq->lock); -- INIT_LIST_HEAD(&dsq->fifo); -- dsq->id = dsq_id; --} -- --static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id & SCX_DSQ_FLAG_BUILTIN) -- return ERR_PTR(-EINVAL); -- -- dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node); -- if (!dsq) -- return ERR_PTR(-ENOMEM); -- -- init_dsq(dsq, dsq_id); -- -- return dsq; --} -- --static void free_dsq_irq_workfn(struct irq_work *irq_work) --{ -- struct llist_node *to_free = llist_del_all(&dsqs_to_free); -- struct scx_dispatch_q *dsq, *tmp_dsq; -- -- llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) -- kfree_rcu(dsq, rcu); --} -- --static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); -- --static void destroy_dsq(u64 dsq_id) --{ --} -- --static void scx_cgroup_exit(void) {} --static int scx_cgroup_init(void) { return 0; } --static void scx_cgroup_config_knobs(void) {} -- --/* -- * Used by sched_fork() and __setscheduler_prio() to pick the matching -- * sched_class. dl/rt are already handled. -- */ --bool task_on_scx(struct task_struct *p) --{ -- if (!scx_enabled() || scx_ops_disabling()) -- return false; -- if (READ_ONCE(scx_switching_all)) -- return true; -- return p->policy == SCHED_EXT; --} -- --static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags) --{ -- if (enq_flags & SCX_ENQ_LAST) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); --} -- --static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {} -- --static void scx_ops_disable_workfn(struct kthread_work *work) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- struct scx_task_iter sti; -- struct task_struct *p; -- const char *reason; -- int i, cpu, type; -- -- type = atomic_read(&scx_exit_type); -- while (true) { -- /* -- * NONE indicates that a new scx_ops has been registered since -- * disable was scheduled - don't kill the new ops. DONE -- * indicates that the ops has already been disabled. -- */ -- if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE) -- return; -- if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE)) -- break; -- } -- -- cancel_delayed_work_sync(&scx_watchdog_work); -- -- switch (type) { -- case SCX_EXIT_UNREG: -- reason = "BPF scheduler unregistered"; -- break; -- case SCX_EXIT_SYSRQ: -- reason = "disabled by sysrq-S"; -- break; -- case SCX_EXIT_ERROR: -- reason = "runtime error"; -- break; -- case SCX_EXIT_ERROR_BPF: -- reason = "scx_bpf_error"; -- break; -- case SCX_EXIT_ERROR_STALL: -- reason = "runnable task stall"; -- break; -- default: -- reason = ""; -- } -- -- ei->type = type; -- strlcpy(ei->reason, reason, sizeof(ei->reason)); -- -- switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { -- case SCX_OPS_DISABLED: -- pr_warn("sched_ext: ops error detected without ops (%s)\n", -- scx_exit_info.msg); -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- return; -- case SCX_OPS_PREPPING: -- goto forward_progress_guaranteed; -- case SCX_OPS_DISABLING: -- /* shouldn't happen but handle it like ENABLING if it does */ -- WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); -- fallthrough; -- case SCX_OPS_ENABLING: -- case SCX_OPS_ENABLED: -- break; -- } -- -- /* -- * DISABLING is set and ops was either ENABLING or ENABLED indicating -- * that the ops and static branches are set. -- * -- * We must guarantee that all runnable tasks make forward progress -- * without trusting the BPF scheduler. We can't grab any mutexes or -- * rwsems as they might be held by tasks that the BPF scheduler is -- * forgetting to run, which unfortunately also excludes toggling the -- * static branches. -- * -- * Let's work around by overriding a couple ops and modifying behaviors -- * based on the DISABLING state and then cycling the tasks through -- * dequeue/enqueue to force global FIFO scheduling. -- * -- * a. ops.enqueue() and .dispatch() are overridden for simple global -- * FIFO scheduling. -- * -- * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value -- * can't be trusted. Whenever a tick triggers, the running task is -- * rotated to the tail of the queue with core_sched_at touched. -- * -- * c. pick_next_task() suppresses zero slice warning. -- * -- * d. scx_prio_less() reverts to the default core_sched_at order. -- */ -- scx_ops.enqueue = scx_ops_fallback_enqueue; -- scx_ops.dispatch = scx_ops_fallback_dispatch; -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- SCHED_CHANGE_BLOCK(task_rq(p), p, -- DEQUEUE_SAVE | DEQUEUE_MOVE) { -- /* cycling deq/enq is enough, see above */ -- } -- } -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* kick all CPUs to restore ticks */ -- for_each_possible_cpu(cpu) -- resched_cpu(cpu); -- --forward_progress_guaranteed: -- /* -- * Here, every runnable task is guaranteed to make forward progress and -- * we can safely use blocking synchronization constructs. Actually -- * disable ops. -- */ -- mutex_lock(&scx_ops_enable_mutex); -- -- static_branch_disable(&__scx_switched_all); -- WRITE_ONCE(scx_switching_all, false); -- -- /* avoid racing against fork and cgroup changes */ -- cpus_read_lock(); -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- bool alive = READ_ONCE(p->__state) != TASK_DEAD; -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL); -- -- __setscheduler_prio(p, p->prio); -- } -- -- if (alive) -- check_class_changed(task_rq(p), p, old_class, p->prio); -- -- scx_ops_disable_task(p); -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* no task is on scx, turn off all the switches and flush in-progress calls */ -- static_branch_disable_cpuslocked(&__scx_ops_enabled); -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- static_branch_disable_cpuslocked(&scx_has_op[i]); -- static_branch_disable_cpuslocked(&scx_ops_enq_last); -- static_branch_disable_cpuslocked(&scx_ops_enq_exiting); -- static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- synchronize_rcu(); -- -- scx_cgroup_exit(); -- -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- cpus_read_unlock(); -- -- if (ei->type >= SCX_EXIT_ERROR) { -- printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name); -- -- if (ei->msg[0] == '\0') -- printk(KERN_ERR "sched_ext: %s\n", ei->reason); -- else -- printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg); -- -- stack_trace_print(ei->bt, ei->bt_len, 2); -- } -- -- if (scx_ops.exit) -- SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); -- -- memset(&scx_ops, 0, sizeof(scx_ops)); -- -- free_percpu(scx_dsp_buf); -- scx_dsp_buf = NULL; -- scx_dsp_max_batch = 0; -- -- mutex_unlock(&scx_ops_enable_mutex); -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- -- scx_cgroup_config_knobs(); --} -- --static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); -- --static void schedule_scx_ops_disable_work(void) --{ -- struct kthread_worker *helper = READ_ONCE(scx_ops_helper); -- -- /* -- * We may be called spuriously before the first bpf_sched_ext_reg(). If -- * scx_ops_helper isn't set up yet, there's nothing to do. -- */ -- if (helper) -- kthread_queue_work(helper, &scx_ops_disable_work); --} -- --static void scx_ops_disable(enum scx_exit_type type) --{ -- int none = SCX_EXIT_NONE; -- -- if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)) -- type = SCX_EXIT_ERROR; -- -- atomic_try_cmpxchg(&scx_exit_type, &none, type); -- -- schedule_scx_ops_disable_work(); --} -- --static void scx_ops_error_irq_workfn(struct irq_work *irq_work) --{ -- schedule_scx_ops_disable_work(); --} -- --static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- int none = SCX_EXIT_NONE; -- va_list args; -- -- if (!atomic_try_cmpxchg(&scx_exit_type, &none, type)) -- return; -- -- ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1); -- -- va_start(args, fmt); -- vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args); -- va_end(args); -- -- irq_work_queue(&scx_ops_error_irq_work); --} -- --static struct kthread_worker *scx_create_rt_helper(const char *name) --{ -- struct kthread_worker *helper; -- -- helper = kthread_create_worker(0, name); -- if (helper) -- sched_set_fifo(helper->task); -- return helper; --} -- --static int scx_ops_enable(struct sched_ext_ops *ops) --{ -- struct scx_task_iter sti; -- struct task_struct *p; -- int i, ret; -- int tcnt = 0; -- unsigned long long start = 0; -- unsigned long long total_start = sched_clock(); -- -- mutex_lock(&scx_ops_enable_mutex); -- -- if (!scx_ops_helper) { -- WRITE_ONCE(scx_ops_helper, -- scx_create_rt_helper("sched_ext_ops_helper")); -- if (!scx_ops_helper) { -- ret = -ENOMEM; -- goto err_unlock; -- } -- } -- -- if (scx_ops_enable_state() != SCX_OPS_DISABLED) { -- ret = -EBUSY; -- goto err_unlock; -- } -- -- //slim_walt_enable(true); -- -- /* -- * Set scx_ops, transition to PREPPING and clear exit info to arm the -- * disable path. Failure triggers full disabling from here on. -- */ -- scx_ops = *ops; -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != -- SCX_OPS_DISABLED); -- -- memset(&scx_exit_info, 0, sizeof(scx_exit_info)); -- atomic_set(&scx_exit_type, SCX_EXIT_NONE); -- scx_warned_zero_slice = false; -- -- atomic64_set(&scx_nr_rejected, 0); -- -- /* -- * Keep CPUs stable during enable so that the BPF scheduler can track -- * online CPUs by watching ->on/offline_cpu() after ->init(). -- */ -- cpus_read_lock(); -- -- scx_switch_all_req = true; -- if (scx_ops.init) { -- ret = SCX_CALL_OP_RET(SCX_KF_INIT, init); -- if (ret) { -- ret = ops_sanitize_err("init", ret); -- goto err_disable; -- } -- -- /* -- * Exit early if ops.init() triggered scx_bpf_error(). Not -- * strictly necessary as we'll fail transitioning into ENABLING -- * later but that'd be after calling ops.prep_enable() on all -- * tasks and with -EBUSY which isn't very intuitive. Let's exit -- * early with success so that the condition is notified through -- * ops.exit() like other scx_bpf_error() invocations. -- */ -- if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE) -- goto err_disable; -- } -- -- WARN_ON_ONCE(scx_dsp_buf); -- scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; -- scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch, -- __alignof__(scx_dsp_buf[0])); -- if (!scx_dsp_buf) { -- ret = -ENOMEM; -- goto err_disable; -- } -- -- scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; -- if (ops->timeout_ms) -- scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); -- -- scx_watchdog_timestamp = jiffies; -- queue_delayed_work(system_unbound_wq, &scx_watchdog_work, -- scx_watchdog_timeout / 2); -- -- /* -- * Lock out forks, cgroup on/offlining and moves before opening the -- * floodgate so that they don't wander into the operations prematurely. -- */ -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- if (((void (**)(void))ops)[i]) -- static_branch_enable_cpuslocked(&scx_has_op[i]); -- -- if (ops->flags & SCX_OPS_ENQ_LAST) -- static_branch_enable_cpuslocked(&scx_ops_enq_last); -- -- if (ops->flags & SCX_OPS_ENQ_EXITING) -- static_branch_enable_cpuslocked(&scx_ops_enq_exiting); -- if (scx_ops.cpu_acquire || scx_ops.cpu_release) -- static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); -- -- if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { -- reset_idle_masks(); -- static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); -- } else { -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- } -- -- /* -- * All cgroups should be initialized before letting in tasks. cgroup -- * on/offlining and task migrations are already locked out. -- */ -- ret = scx_cgroup_init(); -- if (ret) -- goto err_disable_unlock; -- -- static_branch_enable_cpuslocked(&__scx_ops_enabled); -- -- /* -- * Enable ops for every task. Fork is excluded by scx_fork_rwsem -- * preventing new tasks from being added. No need to exclude tasks -- * leaving as sched_ext_free() can handle both prepped and enabled -- * tasks. Prep all tasks first and then enable them with preemption -- * disabled. -- */ -- spin_lock_irq(&scx_tasks_lock); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered(&sti))) { -- get_task_struct(p); -- spin_unlock_irq(&scx_tasks_lock); -- -- ret = scx_ops_prepare_task(p, task_group(p)); -- if (ret) { -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n", -- ret, p->comm, p->pid); -- goto err_disable_unlock; -- } -- -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- } -- scx_task_iter_exit(&sti); -- -- /* -- * All tasks are prepped but are still ops-disabled. Ensure that -- * %current can't be scheduled out and switch everyone. -- * preempt_disable() is necessary because we can't guarantee that -- * %current won't be starved if scheduled out while switching. -- */ -- preempt_disable(); -- -- /* -- * From here on, the disable path must assume that tasks have ops -- * enabled and need to be recovered. -- */ -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { -- preempt_enable(); -- spin_unlock_irq(&scx_tasks_lock); -- ret = -EBUSY; -- goto err_disable_unlock; -- } -- -- /* -- * We're fully committed and can't fail. The PREPPED -> ENABLED -- * transitions here are synchronized against sched_ext_free() through -- * scx_tasks_lock. -- */ -- WRITE_ONCE(scx_switching_all, scx_switch_all_req); -- start = sched_clock(); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- tcnt++; -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- scx_ops_enable_task(p); -- __setscheduler_prio(p, p->prio); -- } -- -- check_class_changed(task_rq(p), p, old_class, p->prio); -- } else { -- scx_ops_disable_task(p); -- } -- } -- printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id()); -- scx_task_iter_exit(&sti); -- -- spin_unlock_irq(&scx_tasks_lock); -- preempt_enable(); -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { -- ret = -EBUSY; -- goto err_disable; -- } -- -- if (scx_switch_all_req) -- static_branch_enable_cpuslocked(&__scx_switched_all); -- -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- -- scx_cgroup_config_knobs(); -- printk("\n total duration = %llu , current cpu = %d\n", sched_clock() - total_start, smp_processor_id()); -- -- return 0; -- --err_unlock: -- mutex_unlock(&scx_ops_enable_mutex); -- return ret; -- --err_disable_unlock: -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); --err_disable: -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- /* must be fully disabled before returning */ -- scx_ops_disable(SCX_EXIT_ERROR); -- kthread_flush_work(&scx_ops_disable_work); -- return ret; --} -- --#ifdef CONFIG_SCHED_DEBUG --static const char *scx_ops_enable_state_str[] = { -- [SCX_OPS_PREPPING] = "prepping", -- [SCX_OPS_ENABLING] = "enabling", -- [SCX_OPS_ENABLED] = "enabled", -- [SCX_OPS_DISABLING] = "disabling", -- [SCX_OPS_DISABLED] = "disabled", --}; -- --static int scx_debug_show(struct seq_file *m, void *v) --{ -- mutex_lock(&scx_ops_enable_mutex); -- seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name); -- seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled()); -- seq_printf(m, "%-30s: %d\n", "switching_all", -- READ_ONCE(scx_switching_all)); -- seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all()); -- seq_printf(m, "%-30s: %s\n", "enable_state", -- scx_ops_enable_state_str[scx_ops_enable_state()]); -- seq_printf(m, "%-30s: %llu\n", "nr_rejected", -- atomic64_read(&scx_nr_rejected)); -- mutex_unlock(&scx_ops_enable_mutex); -- return 0; --} -- --static int scx_debug_open(struct inode *inode, struct file *file) --{ -- return single_open(file, scx_debug_show, NULL); --} -- --const struct file_operations sched_ext_fops = { -- .open = scx_debug_open, -- .read = seq_read, -- .llseek = seq_lseek, -- .release = single_release, --}; --#endif -- --/******************************************************************************** -- * bpf_struct_ops plumbing. -- */ --#include --#include --#include -- --extern struct btf *btf_vmlinux; --static const struct btf_type *task_struct_type; -- --static bool bpf_scx_is_valid_access(int off, int size, -- enum bpf_access_type type, -- const struct bpf_prog *prog, -- struct bpf_insn_access_aux *info) --{ -- if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) -- return false; -- if (type != BPF_READ) -- return false; -- if (off % size != 0) -- return false; -- -- return btf_ctx_access(off, size, type, prog, info); --} -- --static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, -- const struct bpf_reg_state *reg, -- int off, int size) --{ -- return 0; --} -- --static const struct bpf_func_proto * --bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) --{ -- switch (func_id) { -- case BPF_FUNC_task_storage_get: -- return &bpf_task_storage_get_proto; -- case BPF_FUNC_task_storage_delete: -- return &bpf_task_storage_delete_proto; -- default: -- return bpf_base_func_proto(func_id); -- } --} -- --const struct bpf_verifier_ops bpf_scx_verifier_ops = { -- .get_func_proto = bpf_scx_get_func_proto, -- .is_valid_access = bpf_scx_is_valid_access, -- .btf_struct_access = bpf_scx_btf_struct_access, --}; -- --static int bpf_scx_init_member(const struct btf_type *t, -- const struct btf_member *member, -- void *kdata, const void *udata) --{ -- const struct sched_ext_ops *uops = udata; -- struct sched_ext_ops *ops = kdata; -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- int ret; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, dispatch_max_batch): -- if (*(u32 *)(udata + moff) > INT_MAX) -- return -E2BIG; -- ops->dispatch_max_batch = *(u32 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, flags): -- if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) -- return -EINVAL; -- ops->flags = *(u64 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, name): -- ret = bpf_obj_name_cpy(ops->name, uops->name, -- sizeof(ops->name)); -- if (ret < 0) -- return ret; -- if (ret == 0) -- return -EINVAL; -- return 1; -- case offsetof(struct sched_ext_ops, timeout_ms): -- if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT) -- return -E2BIG; -- ops->timeout_ms = *(u32 *)(udata + moff); -- return 1; -- } -- -- return 0; --} -- --static int bpf_scx_check_member(const struct btf_type *t, -- const struct btf_member *member, -- const struct bpf_prog *prog) --{ -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, prep_enable): -- case offsetof(struct sched_ext_ops, init): -- case offsetof(struct sched_ext_ops, exit): -- break; -- default: -- /* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */ -- /* if (prog->aux->sleepable) -- return -EINVAL; */ -- break; -- } -- -- return 0; --} -- --static int bpf_scx_reg(void *kdata) --{ -- return scx_ops_enable(kdata); --} -- --static void bpf_scx_unreg(void *kdata) --{ -- scx_ops_disable(SCX_EXIT_UNREG); -- kthread_flush_work(&scx_ops_disable_work); --} -- --static int bpf_scx_init(struct btf *btf) --{ -- u32 type_id; -- -- type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); -- if (type_id < 0) -- return -EINVAL; -- task_struct_type = btf_type_by_id(btf, type_id); -- -- return 0; --} -- --/* "extern" to avoid sparse warning, only used in this file */ --extern struct bpf_struct_ops bpf_sched_ext_ops; -- --struct bpf_struct_ops bpf_sched_ext_ops = { -- .verifier_ops = &bpf_scx_verifier_ops, -- .reg = bpf_scx_reg, -- .unreg = bpf_scx_unreg, -- .check_member = bpf_scx_check_member, -- .init_member = bpf_scx_init_member, -- .init = bpf_scx_init, -- .name = "sched_ext_ops", --}; -- --static void sysrq_handle_sched_ext_reset(u8 key) --{ -- if (scx_ops_helper) -- scx_ops_disable(SCX_EXIT_SYSRQ); -- else -- pr_info("sched_ext: BPF scheduler not yet used\n"); --} -- --static const struct sysrq_key_op sysrq_sched_ext_reset_op = { -- .handler = sysrq_handle_sched_ext_reset, -- .help_msg = "reset-sched-ext(S)", -- .action_msg = "Disable sched_ext and revert all tasks to CFS", -- .enable_mask = SYSRQ_ENABLE_RTNICE, --}; -- --static void kick_cpus_irq_workfn(struct irq_work *irq_work) --{ -- struct rq *this_rq = this_rq(); -- u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); -- int this_cpu = cpu_of(this_rq); -- int cpu; -- -- for_each_cpu(cpu, this_rq->scx->cpus_to_kick) { -- struct rq *rq = cpu_rq(cpu); -- unsigned long flags; -- -- raw_spin_rq_lock_irqsave(rq, flags); -- -- if (cpu_online(cpu) || cpu == this_cpu) { -- if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) && -- rq->curr->sched_class == &ext_sched_class) -- rq->curr->scx->slice = 0; -- pseqs[cpu] = rq->scx->pnt_seq; -- resched_curr(rq); -- } else { -- cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait); -- } -- -- raw_spin_rq_unlock_irqrestore(rq, flags); -- } -- -- for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait, -- cpumask_of(this_cpu)) { -- /* -- * Pairs with smp_store_release() issued by this CPU in -- * scx_notify_pick_next_task() on the resched path. -- * -- * We busy-wait here to guarantee that no other task can be -- * scheduled on our core before the target CPU has entered the -- * resched path. -- */ -- while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu]) -- cpu_relax(); -- } -- -- cpumask_clear(this_rq->scx->cpus_to_kick); -- cpumask_clear(this_rq->scx->cpus_to_preempt); -- cpumask_clear(this_rq->scx->cpus_to_wait); --} -- --void __init init_sched_ext_class(void) --{ -- int cpu; -- u32 v; -- -- /* -- * The following is to prevent the compiler from optimizing out the enum -- * definitions so that BPF scheduler implementations can use them -- * through the generated vmlinux.h. -- */ -- WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | -- SCX_TG_ONLINE | SCX_KICK_PREEMPT); -- -- init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); --#ifdef CONFIG_SMP -- BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -- BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); --#endif -- scx_kick_cpus_pnt_seqs = -- __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * -- num_possible_cpus(), -- __alignof__(scx_kick_cpus_pnt_seqs[0])); -- BUG_ON(!scx_kick_cpus_pnt_seqs); -- -- for_each_possible_cpu(cpu) { -- struct rq *rq = cpu_rq(cpu); -- -- rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL); -- if (rq->scx) -- rq->scx->rq = rq; -- else -- pr_err("fatal error : alloc rq->scx failed!!!\n"); -- -- init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL); -- INIT_LIST_HEAD(&rq->scx->watchdog_list); -- -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL)); -- init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn); -- } -- -- register_sysrq_key('S', &sysrq_sched_ext_reset_op); -- INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); -- scx_cgroup_config_knobs(); --} -- -- --/******************************************************************************** -- * Helpers that can be called from the BPF scheduler. -- */ --#include -- --/* Disables missing prototype warnings for kfuncs */ --__diag_push(); --__diag_ignore_all("-Wmissing-prototypes", -- "Global functions as their definitions will be in vmlinux BTF"); -- --/** -- * scx_bpf_switch_all - Switch all tasks into SCX -- * @into_scx: switch direction -- * -- * If @into_scx is %true, all existing and future non-dl/rt tasks are switched -- * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on -- * SCX. The actual switching is asynchronous. Can be called from ops.init(). -- */ --void scx_bpf_switch_all(void) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return; -- -- scx_switch_all_req = true; --} -- --BTF_SET8_START(scx_kfunc_ids_init) --BTF_ID_FLAGS(func, scx_bpf_switch_all) --BTF_SET8_END(scx_kfunc_ids_init) -- --static const struct btf_kfunc_id_set scx_kfunc_set_init = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_init, --}; -- --/** -- * scx_bpf_create_dsq - Create a custom DSQ -- * @dsq_id: DSQ to create -- * @node: NUMA node to allocate from -- * -- * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(), -- * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move(). -- */ --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return -EINVAL; -- -- if (unlikely(node >= (int)nr_node_ids || -- (node < 0 && node != NUMA_NO_NODE))) -- return -EINVAL; -- return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); --} -- --BTF_SET8_START(scx_kfunc_ids_sleepable) --BTF_ID_FLAGS(func, scx_bpf_create_dsq) --BTF_SET8_END(scx_kfunc_ids_sleepable) -- --static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_sleepable, --}; -- --static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) --{ -- if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) -- return false; -- -- lockdep_assert_irqs_disabled(); -- -- if (unlikely(!p)) { -- scx_ops_error("called with NULL task"); -- return false; -- } -- -- if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { -- scx_ops_error("invalid enq_flags 0x%llx", enq_flags); -- return false; -- } -- -- return true; --} -- --static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) --{ -- struct task_struct *ddsp_task; -- int idx; -- -- ddsp_task = __this_cpu_read(direct_dispatch_task); -- if (ddsp_task) { -- direct_dispatch(ddsp_task, p, dsq_id, enq_flags); -- return; -- } -- -- idx = __this_cpu_read(scx_dsp_ctx.buf_cursor); -- if (unlikely(idx >= scx_dsp_max_batch)) { -- scx_ops_error("dispatch buffer overflow"); -- return; -- } -- -- this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){ -- .task = p, -- .qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK, -- .dsq_id = dsq_id, -- .enq_flags = enq_flags, -- }; -- __this_cpu_inc(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe -- * to call this function spuriously. Can be called from ops.enqueue() and -- * ops.dispatch(). -- * -- * When called from ops.enqueue(), it's for direct dispatch and @p must match -- * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the -- * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be -- * on the target CPU in the first place. -- * -- * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id -- * and this function can be called upto ops.dispatch_max_batch times to dispatch -- * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the -- * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. -- * -- * This function doesn't have any locking restrictions and may be called under -- * BPF locks (in the future when BPF introduces more flexible locking). -- * -- * @p is allowed to run for @slice. The scheduling path is triggered on slice -- * exhaustion. If zero, the current residual slice is maintained. If -- * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with -- * scx_bpf_kick_cpu() to trigger scheduling. -- */ --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- scx_dispatch_commit(p, dsq_id, enq_flags); --} -- --/** -- * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. -- * Tasks queued into the priority queue are ordered by @vtime and always -- * consumed after the tasks in the FIFO queue. All other aspects are identical -- * to scx_bpf_dispatch(). -- * -- * @vtime ordering is according to time_before64() which considers wrapping. A -- * numerically larger vtime may indicate an earlier position in the ordering and -- * vice-versa. -- */ --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 vtime, u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- p->scx->dsq_vtime = vtime; -- -- scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); --} -- --BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime) --BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_enqueue_dispatch, --}; -- --/** -- * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots -- * -- * Can only be called from ops.dispatch(). -- */ --u32 scx_bpf_dispatch_nr_slots(void) --{ -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return 0; -- -- return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ -- * @dsq_id: DSQ to consume -- * -- * Consume a task from the non-local DSQ identified by @dsq_id and transfer it -- * to the current CPU's local DSQ for execution. Can only be called from -- * ops.dispatch(). -- * -- * This function flushes the in-flight dispatches from scx_bpf_dispatch() before -- * trying to consume the specified DSQ. It may also grab rq locks and thus can't -- * be called under any BPF locks. -- * -- * Returns %true if a task has been consumed, %false if there isn't any task to -- * consume. -- */ --bool scx_bpf_consume(u64 dsq_id) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- struct scx_dispatch_q *dsq; -- -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return false; -- -- flush_dispatch_buf(dspc->rq, dspc->rf); -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); -- return false; -- } -- -- if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) { -- /* -- * A successfully consumed task can be dequeued before it starts -- * running while the CPU is trying to migrate other dispatched -- * tasks. Bump nr_tasks to tell balance_scx() to retry on empty -- * local DSQ. -- */ -- dspc->nr_tasks++; -- return true; -- } else { -- return false; -- } --} -- --BTF_SET8_START(scx_kfunc_ids_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) --BTF_ID_FLAGS(func, scx_bpf_consume) --BTF_SET8_END(scx_kfunc_ids_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_dispatch, --}; -- --/** -- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ -- * -- * Iterate over all of the tasks currently enqueued on the local DSQ of the -- * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of -- * processed tasks. Can only be called from ops.cpu_release(). -- */ --u32 scx_bpf_reenqueue_local(void) --{ -- u32 nr_enqueued, i; -- struct rq *rq; -- struct scx_rq *scx_rq; -- -- if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) -- return 0; -- -- rq = cpu_rq(smp_processor_id()); -- lockdep_assert_rq_held(rq); -- scx_rq = rq->scx; -- -- /* -- * Get the number of tasks on the local DSQ before iterating over it to -- * pull off tasks. The enqueue callback below can signal that it wants -- * the task to stay on the local DSQ, and we want to prevent the BPF -- * scheduler from causing us to loop indefinitely. -- */ -- nr_enqueued = scx_rq->local_dsq.nr; -- for (i = 0; i < nr_enqueued; i++) { -- struct task_struct *p; -- -- p = first_local_task(rq); -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- WARN_ON_ONCE(p->scx->holding_cpu != -1); -- dispatch_dequeue(scx_rq, p); -- do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); -- } -- -- return nr_enqueued; --} -- --BTF_SET8_START(scx_kfunc_ids_cpu_release) --BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) --BTF_SET8_END(scx_kfunc_ids_cpu_release) -- --static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_cpu_release, --}; -- --/** -- * scx_bpf_kick_cpu - Trigger reschedule on a CPU -- * @cpu: cpu to kick -- * @flags: SCX_KICK_* flags -- * -- * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or -- * trigger rescheduling on a busy CPU. This can be called from any online -- * scx_ops operation and the actual kicking is performed asynchronously through -- * an irq work. -- */ --void scx_bpf_kick_cpu(s32 cpu, u64 flags) --{ -- struct rq *rq; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d", cpu); -- return; -- } -- -- preempt_disable(); -- rq = this_rq(); -- -- /* -- * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting -- * rq locks. We can probably be smarter and avoid bouncing if called -- * from ops which don't hold a rq lock. -- */ -- cpumask_set_cpu(cpu, rq->scx->cpus_to_kick); -- if (flags & SCX_KICK_PREEMPT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt); -- if (flags & SCX_KICK_WAIT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_wait); -- -- irq_work_queue(&rq->scx->kick_cpus_irq_work); -- preempt_enable(); --} -- --/** -- * scx_bpf_dsq_nr_queued - Return the number of queued tasks -- * @dsq_id: id of the DSQ -- * -- * Return the number of tasks in the DSQ matching @dsq_id. If not found, -- * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops -- * operations. -- */ --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) --{ -- struct scx_dispatch_q *dsq; -- -- lockdep_assert(rcu_read_lock_any_held()); -- -- if (dsq_id == SCX_DSQ_LOCAL) { -- return this_rq()->scx->local_dsq.nr; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (ops_cpu_valid(cpu)) -- return cpu_rq(cpu)->scx->local_dsq.nr; -- } else { -- dsq = find_non_local_dsq(dsq_id); -- if (dsq) -- return dsq->nr; -- } -- return -ENOENT; --} -- --/** -- * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state -- * @cpu: cpu to test and clear idle for -- * -- * Returns %true if @cpu was idle and its idle state was successfully cleared. -- * %false otherwise. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return false; -- } -- -- if (ops_cpu_valid(cpu)) -- return test_and_clear_cpu_idle(cpu); -- else -- return false; --} -- --/** -- * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu -- * @cpus_allowed: Allowed cpumask -- * -- * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked -- * idle cpu number on success. -%EBUSY if no matching cpu was found. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return -EBUSY; -- } -- -- return scx_pick_idle_cpu(cpus_allowed); --} -- --/** -- * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking -- * per-CPU cpumask. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_cpumask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.cpu; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, -- * per-physical-core cpumask. Can be used to determine if an entire physical -- * core is free. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_smtmask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.smt; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to -- * either the percpu, or SMT idle-tracking cpumask. -- */ --void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) --{ -- /* -- * Empty function body because we aren't actually acquiring or -- * releasing a reference to a global idle cpumask, which is read-only -- * in the caller and is never released. The acquire / release semantics -- * here are just used to make the cpumask is a trusted pointer in the -- * caller. -- */ --} -- --struct scx_bpf_error_bstr_bufs { -- u64 data[MAX_BPRINTF_VARARGS]; -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs); -- --/** -- * scx_bpf_error_bstr - Indicate fatal error -- * @fmt: error message format string -- * @data: format string parameters packaged using ___bpf_fill() macro -- * @data__sz: @data len, must end in '__sz' for the verifier -- * -- * Indicate that the BPF scheduler encountered a fatal error and initiate ops -- * disabling. -- */ --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) --{ -- struct scx_bpf_error_bstr_bufs *bufs; -- unsigned long flags; -- struct bpf_bprintf_data args = {}; -- int ret; -- -- local_irq_save(flags); -- bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs); -- -- if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || -- (data__sz && !data)) { -- scx_ops_error("invalid data=%p and data__sz=%u", -- (void *)data, data__sz); -- goto out_restore; -- } -- -- ret = copy_from_kernel_nofault(bufs->data, data, data__sz); -- if (ret) { -- scx_ops_error("failed to read data fields (%d)", ret); -- goto out_restore; -- } -- -- ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args); -- if (ret < 0) { -- scx_ops_error("failed to format prepration (%d)", ret); -- goto out_restore; -- } -- -- ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt, -- args.bin_args); -- bpf_bprintf_cleanup(&args); -- if (ret < 0) { -- scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format", -- fmt, data, data__sz); -- goto out_restore; -- } -- -- scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg); --out_restore: -- local_irq_restore(flags); --} -- --/** -- * scx_bpf_destroy_dsq - Destroy a custom DSQ -- * @dsq_id: DSQ to destroy -- * -- * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with -- * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is -- * empty and no further tasks are dispatched to it. Ignored if called on a DSQ -- * which doesn't exist. Can be called from any online scx_ops operations. -- */ --void scx_bpf_destroy_dsq(u64 dsq_id) --{ -- destroy_dsq(dsq_id); --} -- --/** -- * scx_bpf_task_running - Is task currently running? -- * @p: task of interest -- */ --bool scx_bpf_task_running(const struct task_struct *p) --{ -- return task_rq(p)->curr == p; --} -- --/** -- * scx_bpf_task_cpu - CPU a task is currently associated with -- * @p: task of interest -- */ --s32 scx_bpf_task_cpu(const struct task_struct *p) --{ -- return task_cpu(p); --} -- --/** -- * scx_bpf_task_cgroup - Return the sched cgroup of a task -- * @p: task of interest -- * -- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with -- * from the scheduler's POV. SCX operations should use this function to -- * determine @p's current cgroup as, unlike following @p->cgroups, -- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all -- * rq-locked operations. Can be called on the parameter tasks of rq-locked -- * operations. The restriction guarantees that @p's rq is locked by the caller. -- */ --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) --{ -- struct task_group *tg = p->sched_task_group; -- struct cgroup *cgrp = &cgrp_dfl_root.cgrp; -- -- if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) -- goto out; -- -- /* -- * A task_group may either be a cgroup or an autogroup. In the latter -- * case, @tg->css.cgroup is %NULL. A task_group can't become the other -- * kind once created. -- */ -- if (tg && tg->css.cgroup) -- cgrp = tg->css.cgroup; -- else -- cgrp = &cgrp_dfl_root.cgrp; --out: -- cgroup_get(cgrp); -- return cgrp; --} -- --BTF_SET8_START(scx_kfunc_ids_any) --BTF_ID_FLAGS(func, scx_bpf_kick_cpu) --BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) --BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) --BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu) --BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) --BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) --BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) --BTF_ID_FLAGS(func, scx_bpf_task_running) --BTF_ID_FLAGS(func, scx_bpf_task_cpu) --BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE) --BTF_SET8_END(scx_kfunc_ids_any) -- --static const struct btf_kfunc_id_set scx_kfunc_set_any = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_any, --}; -- --__diag_pop(); -- --/* -- * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set() -- * needs most of the system to be up. -- */ --static int __init register_ext_kfuncs(void) --{ -- int ret; -- -- /* -- * Some kfuncs are context-sensitive and can only be called from -- * specific SCX ops. They are grouped into BTF sets accordingly. -- * Unfortunately, BPF currently doesn't have a way of enforcing such -- * restrictions. Eventually, the verifier should be able to enforce -- * them. For now, register them the same and make each kfunc explicitly -- * check using scx_kf_allowed(). -- */ -- if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_init)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_sleepable)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_enqueue_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_cpu_release)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_any))) { -- pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret); -- return ret; -- } -- -- return 0; --} --__initcall(register_ext_kfuncs); -diff --git a/kernel/sched/ext.h b/kernel/sched/ext.h -deleted file mode 100755 -index fba8ed845f99..000000000000 ---- a/kernel/sched/ext.h -+++ /dev/null -@@ -1,257 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ -- -- --/* -- * Tag marking a kernel function as a kfunc. This is meant to minimize the -- * amount of copy-paste that kfunc authors have to include for correctness so -- * as to avoid issues such as the compiler inlining or eliding either a static -- * kfunc, or a global kfunc in an LTO build. -- */ --#define __bpf_kfunc __used noinline -- --enum scx_wake_flags { -- /* expose select WF_* flags as enums */ -- SCX_WAKE_EXEC = WF_EXEC, -- SCX_WAKE_FORK = WF_FORK, -- SCX_WAKE_TTWU = WF_TTWU, -- SCX_WAKE_SYNC = WF_SYNC, --}; -- --enum scx_enq_flags { -- /* expose select ENQUEUE_* flags as enums */ -- SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, -- SCX_ENQ_HEAD = ENQUEUE_HEAD, -- -- /* high 32bits are SCX specific */ -- -- /* -- * Set the following to trigger preemption when calling -- * scx_bpf_dispatch() with a local dsq as the target. The slice of the -- * current task is cleared to zero and the CPU is kicked into the -- * scheduling path. Implies %SCX_ENQ_HEAD. -- */ -- SCX_ENQ_PREEMPT = 1LLU << 32, -- -- /* -- * The task being enqueued was previously enqueued on the current CPU's -- * %SCX_DSQ_LOCAL, but was removed from it in a call to the -- * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was -- * invoked in a ->cpu_release() callback, and the task is again -- * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the -- * task will not be scheduled on the CPU until at least the next invocation -- * of the ->cpu_acquire() callback. -- */ -- SCX_ENQ_REENQ = 1LLU << 40, -- -- /* -- * The task being enqueued is the only task available for the cpu. By -- * default, ext core keeps executing such tasks but when -- * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with -- * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set. -- * -- * If the BPF scheduler wants to continue executing the task, -- * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately. -- * If the task gets queued on a different dsq or the BPF side, the BPF -- * scheduler is responsible for triggering a follow-up scheduling event. -- * Otherwise, Execution may stall. -- */ -- SCX_ENQ_LAST = 1LLU << 41, -- -- /* -- * A hint indicating that it's advisable to enqueue the task on the -- * local dsq of the currently selected CPU. Currently used by -- * select_cpu_dfl() and together with %SCX_ENQ_LAST. -- */ -- SCX_ENQ_LOCAL = 1LLU << 42, -- -- /* high 8 bits are internal */ -- __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, -- -- SCX_ENQ_CLEAR_OPSS = 1LLU << 56, -- SCX_ENQ_DSQ_PRIQ = 1LLU << 57, --}; -- --enum scx_deq_flags { -- /* expose select DEQUEUE_* flags as enums */ -- SCX_DEQ_SLEEP = DEQUEUE_SLEEP, -- -- /* high 32bits are SCX specific */ -- -- /* -- * The generic core-sched layer decided to execute the task even though -- * it hasn't been dispatched yet. Dequeue from the BPF side. -- */ -- SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, --}; -- --enum scx_tg_flags { -- SCX_TG_ONLINE = 1U << 0, -- SCX_TG_INITED = 1U << 1, --}; -- --enum scx_kick_flags { -- SCX_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -- SCX_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ --}; -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --extern const struct sched_class ext_sched_class; --extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops; --extern const struct file_operations sched_ext_fops; --extern unsigned long scx_watchdog_timeout; --extern unsigned long scx_watchdog_timestamp; -- --DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); --DECLARE_STATIC_KEY_FALSE(__scx_switched_all); --#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) --#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) -- --DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); -- --bool task_on_scx(struct task_struct *p); --void scx_pre_fork(struct task_struct *p); --int scx_fork(struct task_struct *p); --void scx_post_fork(struct task_struct *p); --void scx_cancel_fork(struct task_struct *p); --int scx_check_setscheduler(struct task_struct *p, int policy); --bool scx_can_stop_tick(struct rq *rq); --void init_sched_ext_class(void); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...); --#define scx_ops_error(fmt, args...) \ -- scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args) -- --void __scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active); -- --static inline void scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active) --{ -- if (!scx_enabled()) -- return; --#ifdef CONFIG_SMP -- /* -- * Pairs with the smp_load_acquire() issued by a CPU in -- * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -- * resched. -- */ -- smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1); --#endif -- if (!static_branch_unlikely(&scx_ops_cpu_preempt)) -- return; -- __scx_notify_pick_next_task(rq, p, active); --} -- --//extern void scx_scheduler_tick(void); --static inline void scx_notify_sched_tick(void) --{ -- unsigned long last_check; -- -- if (!scx_enabled()) -- return; -- //scx_scheduler_tick(); -- -- last_check = scx_watchdog_timestamp; -- if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "watchdog failed to check in for %u.%03us", -- dur_ms / 1000, dur_ms % 1000); -- } --} -- --static inline const struct sched_class *next_active_class(const struct sched_class *class) --{ -- class++; -- if (scx_switched_all() && class == &fair_sched_class) -- class++; -- if (!scx_enabled() && class == &ext_sched_class) -- class++; -- return class; --} -- --#define for_active_class_range(class, _from, _to) \ -- for (class = (_from); class != (_to); class = next_active_class(class)) -- --#define for_each_active_class(class) \ -- for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -- --/* -- * SCX requires a balance() call before every pick_next_task() call including -- * when waking up from idle. -- */ --#define for_balance_class_range(class, prev_class, end_class) \ -- for_active_class_range(class, (prev_class) > &ext_sched_class ? \ -- &ext_sched_class : (prev_class), (end_class)) -- --#ifdef CONFIG_SCHED_CORE --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi); --#endif -- --#else /* CONFIG_SCHED_CLASS_EXT */ -- --#define scx_enabled() false --#define scx_switched_all() false -- --static inline void scx_pre_fork(struct task_struct *p) {} --static inline int scx_fork(struct task_struct *p) { return 0; } --static inline void scx_post_fork(struct task_struct *p) {} --static inline void scx_cancel_fork(struct task_struct *p) {} --static inline int scx_check_setscheduler(struct task_struct *p, -- int policy) { return 0; } --static inline bool scx_can_stop_tick(struct rq *rq) { return true; } --static inline void init_sched_ext_class(void) {} --static inline void scx_notify_pick_next_task(struct rq *rq, -- const struct task_struct *p, -- const struct sched_class *active) {} --static inline void scx_notify_sched_tick(void) {} -- --#define for_each_active_class for_each_class --#define for_balance_class_range for_class_range -- --#endif /* CONFIG_SCHED_CLASS_EXT */ -- --#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) --void __scx_update_idle(struct rq *rq, bool idle); -- --static inline void scx_update_idle(struct rq *rq, bool idle) --{ -- if (scx_enabled()) -- __scx_update_idle(rq, idle); --} --#else --static inline void scx_update_idle(struct rq *rq, bool idle) {} --#endif -- --#ifdef CONFIG_CGROUP_SCHED --#ifdef CONFIG_EXT_GROUP_SCHED --int scx_tg_online(struct task_group *tg); --void scx_tg_offline(struct task_group *tg); --int scx_cgroup_can_attach(struct cgroup_taskset *tset); --void scx_move_task(struct task_struct *p); --void scx_cgroup_finish_attach(void); --void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); --void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); --#else /* CONFIG_EXT_GROUP_SCHED */ --static inline int scx_tg_online(struct task_group *tg) { return 0; } --static inline void scx_tg_offline(struct task_group *tg) {} --static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } --static inline void scx_move_task(struct task_struct *p) {} --static inline void scx_cgroup_finish_attach(void) {} --static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} --static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} --#endif /* CONFIG_EXT_GROUP_SCHED */ --#endif /* CONFIG_CGROUP_SCHED */ -diff --git a/kernel/sched/hmbird.h b/kernel/sched/hmbird.h -new file mode 100755 -index 000000000000..d0d84ddee13d ---- /dev/null -+++ b/kernel/sched/hmbird.h -@@ -0,0 +1,343 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#ifndef _HMBIRD_H_ -+#define _HMBIRD_H_ -+ -+/* -+ * Tag marking a kernel function as a kfunc. This is meant to minimize the -+ * amount of copy-paste that kfunc authors have to include for correctness so -+ * as to avoid issues such as the compiler inlining or eliding either a static -+ * kfunc, or a global kfunc in an LTO build. -+ */ -+ -+#define DEBUG_INTERNAL (1 << 0) -+#define DEBUG_INFO_TRACE (1 << 1) -+#define DEBUG_INFO_SYSTRACE (1 << 2) -+ -+#define NUMS_CGROUP_KINDS (512) -+#define SLIM_FOR_SGAME (1 << 0) -+#define SLIM_FOR_GENSHIN (1 << 1) -+ -+#ifdef MAX_TASK_NR -+#define MAX_KEY_THREAD_RECORD ((MAX_TASK_NR + 1) >> 1) -+#else -+#define MAX_KEY_THREAD_RECORD (1 << 3) -+#endif /* MAX_TASK_NR */ -+ -+#define TOP_TASK_SHIFT (8) -+#define TOP_TASK_MAX (1 << TOP_TASK_SHIFT) -+#ifndef TOP_TASK_BITS_MASK -+#define TOP_TASK_BITS_MASK (TOP_TASK_MAX - 1) -+#endif /* TOP_TASK_BITS_MASK */ -+ -+extern struct md_info_t *md_info; -+ -+#define debug_enabled() \ -+ (unlikely(hmbirdcore_debug)) -+ -+#define hmbird_debug(fmt, ...) \ -+ pr_info(":"fmt, ##__VA_ARGS__) -+ -+#define hmbird_err(id, fmt, ...) \ -+do { \ -+ pr_err(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_deferred_err(id, fmt, ...) \ -+do { \ -+ printk_deferred(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \ -+do { \ -+ if (unlikely(cond)) { \ -+ hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+ } \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_info_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_info_trace(fmt, ...) -+#endif -+ -+#define hmbird_info_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+ } \ -+} while (0) -+ -+#define hmbird_output_systrace(fmt, ...) \ -+do { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_internal_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_internal_trace(fmt, ...) -+#endif -+ -+#define hmbird_internal_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) { \ -+ hmbird_output_systrace(fmt, ##__VA_ARGS__); \ -+ } \ -+} while (0) -+ -+enum hmbird_wake_flags { -+ /* expose select WF_* flags as enums */ -+ HMBIRD_WAKE_EXEC = WF_EXEC, -+ HMBIRD_WAKE_FORK = WF_FORK, -+ HMBIRD_WAKE_TTWU = WF_TTWU, -+ HMBIRD_WAKE_SYNC = WF_SYNC, -+}; -+ -+enum hmbird_enq_flags { -+ /* expose select ENQUEUE_* flags as enums */ -+ HMBIRD_ENQ_WAKEUP = ENQUEUE_WAKEUP, -+ HMBIRD_ENQ_HEAD = ENQUEUE_HEAD, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * Set the following to trigger preemption when calling -+ * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the -+ * current task is cleared to zero and the CPU is kicked into the -+ * scheduling path. Implies %HMBIRD_ENQ_HEAD. -+ */ -+ HMBIRD_ENQ_PREEMPT = 1LLU << 32, -+ HMBIRD_ENQ_REENQ = 1LLU << 40, -+ HMBIRD_ENQ_LAST = 1LLU << 41, -+ -+ /* -+ * A hint indicating that it's advisable to enqueue the task on the -+ * local dsq of the currently selected CPU. Currently used by -+ * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST. -+ */ -+ HMBIRD_ENQ_LOCAL = 1LLU << 42, -+ -+ /* high 8 bits are internal */ -+ __HMBIRD_ENQ_INTERNAL_MASK = 0xffLLU << 56, -+ -+ HMBIRD_ENQ_CLEAR_OPSS = 1LLU << 56, -+ HMBIRD_ENQ_DSQ_PRIQ = 1LLU << 57, -+}; -+ -+enum hmbird_deq_flags { -+ /* expose select DEQUEUE_* flags as enums */ -+ HMBIRD_DEQ_SLEEP = DEQUEUE_SLEEP, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * The generic core-sched layer decided to execute the task even though -+ * it hasn't been dispatched yet. Dequeue from the BPF side. -+ */ -+ HMBIRD_DEQ_CORE_SCHED_EXEC = 1LLU << 32, -+}; -+ -+enum hmbird_kick_flags { -+ HMBIRD_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -+ HMBIRD_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ -+}; -+ -+enum hmbird_task_prop_type { -+ HMBIRD_TASK_PROP_COMMON, -+ HMBIRD_TASK_PROP_PIPELINE, -+ HMBIRD_TASK_PROP_DEBUG_OR_LOG, -+ HMBIRD_TASK_PROP_HIGH_LOAD, -+ HMBIRD_TASK_PROP_IO, -+ HMBIRD_TASK_PROP_NETWORK, -+ HMBIRD_TASK_PROP_PERIODIC, -+ HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL, -+ HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL, -+ HMBIRD_TASK_PROP_ISOLATE, -+ HMBIRD_TASK_PROP_MAX, -+}; -+ -+enum hmbird_preempt_policy_id { -+ HMBIRD_PREEMPT_POLICY_NONE, -+ HMBIRD_PREEMPT_POLICY_PRIO_BASED, -+}; -+ -+extern const struct sched_class hmbird_sched_class; -+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops; -+extern const struct file_operations sched_hmbird_fops; -+extern unsigned long hmbird_watchdog_timeout; -+extern unsigned long hmbird_watchdog_timestamp; -+ -+enum scx_rq_flags { -+ HMBIRD_RQ_CAN_STOP_TICK = 1 << 0, -+}; -+ -+struct hmbird_rq { -+ struct hmbird_dispatch_q local_dsq; -+ struct list_head watchdog_list; -+ u64 ops_qseq; -+ u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -+ u32 nr_running; -+ u32 flags; -+ bool cpu_released; -+ cpumask_var_t cpus_to_kick; -+ cpumask_var_t cpus_to_preempt; -+ cpumask_var_t cpus_to_wait; -+ u64 pnt_seq; -+ u64 *prev_runnable_sum_fixed; -+ u32 *prev_window_size; -+ struct irq_work kick_cpus_irq_work; -+ bool pipeline; -+ bool exclusive; -+ bool period_disallow; -+ bool nonperiod_disallow; -+ struct rq *rq; -+ struct hmbird_sched_rq_stats *srq; -+}; -+ -+struct hmbird_sched_change_guard { -+ struct task_struct *p; -+ struct rq *rq; -+ bool queued; -+ bool running; -+ bool done; -+}; -+ -+extern struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -+ -+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags); -+ -+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -+ for (struct hmbird_sched_change_guard __cg = \ -+ hmbird_sched_change_guard_init(__rq, __p, __flags); \ -+ !__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags)) -+ -+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+bool task_on_hmbird(struct task_struct *p); -+int hmbird_pre_fork(struct task_struct *p); -+int hmbird_fork(struct task_struct *p); -+void hmbird_post_fork(struct task_struct *p); -+void hmbird_cancel_fork(struct task_struct *p); -+int hmbird_check_setscheduler(struct task_struct *p, int policy); -+bool hmbird_can_stop_tick(struct rq *rq); -+void init_sched_hmbird_class(void); -+ -+void hmbird_ops_exit(void); -+#define hmbird_ops_error(fmt, ...) \ -+do { \ -+ hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__); \ -+ hmbird_ops_exit(); \ -+} while (0) -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active); -+ -+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight) -+{ -+ return clamp_t(unsigned long, -+ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -+ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); -+} -+ -+static inline void hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active) -+{ -+ if (!hmbird_enabled()) -+ return; -+#ifdef CONFIG_SMP -+ /* -+ * Pairs with the smp_load_acquire() issued by a CPU in -+ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -+ * resched. -+ */ -+ smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1); -+#endif -+ if (!static_branch_unlikely(&hmbird_ops_cpu_preempt)) -+ return; -+ __hmbird_notify_pick_next_task(rq, p, active); -+} -+ -+extern void hmbird_scheduler_tick(void); -+void scan_timeout(struct rq *rq); -+void hmbird_notify_sched_tick(void); -+ -+static inline const struct sched_class *next_active_class(const struct sched_class *class) -+{ -+ class++; -+ -+ if (hmbird_enabled() && class == &fair_sched_class) -+ class++; -+ if (!hmbird_enabled() && class == &hmbird_sched_class) -+ class++; -+ return class; -+} -+ -+#define for_active_class_range(class, _from, _to) \ -+ for (class = (_from); class != (_to); class = next_active_class(class)) -+ -+#define for_each_active_class(class) \ -+ for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -+ -+/* -+ * HMBIRD requires a balance() call before every pick_next_task() call including -+ * when waking up from idle. -+ */ -+#define for_balance_class_range(class, prev_class, end_class) \ -+ for_active_class_range(class, (prev_class) > &hmbird_sched_class ? \ -+ &hmbird_sched_class : (prev_class), (end_class)) -+ -+#define MIN_CGROUP_DL_IDX (5) /* 8ms */ -+#define DEFAULT_CGROUP_DL_IDX (8) /* 64ms */ -+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS]; -+ -+ -+void __hmbird_update_idle(struct rq *rq, bool idle); -+ -+static inline void hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ if (hmbird_enabled()) -+ __hmbird_update_idle(rq, idle); -+} -+ -+int hmbird_ctrl(bool enable); -+ -+int hmbird_tg_online(struct task_group *tg); -+ -+ -+static inline u16 hmbird_task_util(struct task_struct *p) -+{ -+ return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0; -+} -+u16 hmbird_cpu_util(int cpu); -+ -+bool get_hmbird_ops_enabled(void); -+bool get_non_hmbird_task(void); -+void set_cpu_cluster(u64 cpu_cluster); -+#endif /*_EXT_H_*/ -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -new file mode 100755 -index 000000000000..ce05928392bf ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird.c -@@ -0,0 +1,4313 @@ -+// SPDX-License-Identifier: GPL-2.0 -+ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#include -+#include -+ -+#include "slim.h" -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+#include -+ -+#define CLUSTER_SEPARATE -+ -+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0); -+atomic_t non_hmbird_task; -+atomic_t hmbird_module_loaded = ATOMIC_INIT(0); -+int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0}; -+ -+enum hmbird_internal_consts { -+ HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ, -+}; -+ -+enum hmbird_ops_enable_state { -+ HMBIRD_OPS_PREPPING, -+ HMBIRD_OPS_ENABLING, -+ HMBIRD_OPS_ENABLED, -+ HMBIRD_OPS_DISABLING, -+ HMBIRD_OPS_DISABLED, -+}; -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+/* -+ * hmbird_entity->ops_state -+ * -+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler. -+ * State transitions look as follows: -+ * -+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING -+ * ^ | | -+ * | v v -+ * \-------------------------------/ -+ * -+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -+ * sites for explanations on the conditions being waited upon and why they are -+ * safe. Transitions out of them into NONE or QUEUED must store_release and the -+ * waiters should load_acquire. -+ * -+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether -+ * any given task can be dispatched by the BPF scheduler at all times and thus -+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core -+ * can safely reject invalid dispatches. -+ */ -+enum hmbird_ops_state { -+ HMBIRD_OPSS_NONE, /* owned by the HMBIRD core */ -+ HMBIRD_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -+ HMBIRD_OPSS_QUEUED, /* owned by the BPF scheduler */ -+ HMBIRD_OPSS_DISPATCHING, /* in transit back to the HMBIRD core */ -+ -+ /* -+ * QSEQ brands each QUEUED instance so that, when dispatch races -+ * dequeue/requeue, the dispatcher can tell whether it still has a claim -+ * on the task being dispatched. -+ */ -+ HMBIRD_OPSS_QSEQ_SHIFT = 2, -+ HMBIRD_OPSS_STATE_MASK = (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1, -+ HMBIRD_OPSS_QSEQ_MASK = ~HMBIRD_OPSS_STATE_MASK, -+}; -+ -+enum switch_stat { -+ HMBIRD_DISABLED, -+ HMBIRD_SWITCH_PREP, -+ HMBIRD_RQ_SWITCH_BEGIN, -+ HMBIRD_RQ_SWITCH_DONE, -+ HMBIRD_ENABLED, -+}; -+enum switch_stat curr_ss; -+ -+/* -+ * During exit, a task may schedule after losing its PIDs. When disabling the -+ * BPF scheduler, we need to be able to iterate tasks in every state to -+ * guarantee system safety. Maintain a dedicated task list which contains every -+ * task between its fork and eventual free. -+ */ -+DEFINE_SPINLOCK(hmbird_tasks_lock); -+static LIST_HEAD(hmbird_tasks); -+ -+/* ops enable/disable */ -+static struct kthread_worker *hmbird_ops_helper; -+static DEFINE_MUTEX(hmbird_ops_enable_mutex); -+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem); -+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED); -+ -+static bool hmbird_warned_zero_slice; -+enum hmbird_switch_type sw_type; -+static int skip_num[MAX_GLOBAL_DSQS]; -+static int big_distribute_mask_prev; -+static int little_distribute_mask_prev; -+ -+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0); -+ -+/* -+ * The maximum amount of time in jiffies that a task may be runnable without -+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger -+ * hmbird_ops_error(). -+ */ -+unsigned long hmbird_watchdog_timeout; -+ -+/* -+ * The last time the delayed work was run. This delayed work relies on -+ * ksoftirqd being able to run to service timer interrupts, so it's possible -+ * that this work itself could get wedged. To account for this, we check that -+ * it's not stalled in the timer tick, and trigger an error if it is. -+ */ -+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES; -+ -+static struct delayed_work hmbird_watchdog_work; -+static struct work_struct hmbird_err_exit_work; -+ -+/* idle tracking */ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_CPUMASK_OFFSTACK -+#define CL_ALIGNED_IF_ONSTACK -+#else -+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp -+#endif -+ -+static struct { -+ cpumask_var_t cpu; -+ cpumask_var_t smt; -+} idle_masks CL_ALIGNED_IF_ONSTACK; -+ -+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus; -+#endif /* CONFIG_SMP */ -+ -+/* dispatch queues */ -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global; -+ -+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128}; -+u32 pcp_dsq_deadline = 20; -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS]; -+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq); -+ -+static u64 max_hmbird_dsq_internal_id; -+ -+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/ -+#define CLUSTER_SEPARATE_IDX (8) -+#define GDSQS_ID_BASE (3) -+#define UX_COMPATIBLE_IDX (4) -+#define NON_PERIOD_START (5) -+#define NON_PERIOD_END (MAX_GLOBAL_DSQS) -+#define CREATE_DSQ_LEVEL_WITHIN (1) -+ -+struct hmbird_sched_info { -+ spinlock_t lock; -+ int curr_idx[2]; -+ int rtime[MAX_GLOBAL_DSQS]; -+}; -+ -+struct pcp_sched_info { -+ s64 pcp_seq; -+ int rtime; -+ bool pcp_round; -+}; -+ -+/* -+ * pcp_info may rw by another cpu. -+ * protected by rq->lock. -+ */ -+atomic64_t pcp_dsq_round; -+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info); -+ -+struct md_info_t *md_info; -+ -+static int b_rescue_l, l_rescue_b; -+static struct hmbird_sched_info sinfo; -+ -+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC; -+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = { -+ 0, 0, 0, 0, 0, -+ 32 * NSEC_PER_MSEC, -+ 20 * NSEC_PER_MSEC, -+ 14 * NSEC_PER_MSEC, -+ 8 * NSEC_PER_MSEC, -+ 6 * NSEC_PER_MSEC -+}; -+ -+struct cluster_ctx { -+ /* cpu-dsq map must within [lower, upper) */ -+ int upper; -+ int lower; -+ int tidx; -+}; -+ -+enum stat_items { -+ GLOBAL_STAT, -+ CPU_ALLOW_FAIL, -+ RT_CNT, -+ KEY_TASK_CNT, -+ SWITCH_IDX, -+ TIMEOUT_CNT, -+ -+ TOTAL_DSP_CNT, -+ MOVE_RQ_CNT, -+ SELECT_CPU, -+ -+ DWORD_STAT_END = SELECT_CPU, -+ -+ GDSQ_CNT, -+ ERR_IDX, -+ PCP_TIMEOUT_CNT, -+ PCP_LDSQ_CNT, -+ PCP_ENQL_CNT, -+ -+ MAX_ITEMS, -+}; -+static DEFINE_SPINLOCK(stats_lock); -+static char *stats_str[MAX_ITEMS] = { -+ "global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt", -+ "switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt", -+ "select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt", -+ "pcp_ldsq_cnt", "pcp_enql_cnt" -+}; -+ -+ -+struct stats_struct { -+ u64 global_stat[2]; -+ u64 cpu_allow_fail[2]; -+ u64 rt_cnt[2]; -+ u64 key_task_cnt[2]; -+ u64 switch_idx[2]; -+ u64 timeout_cnt[2]; -+ -+ u64 total_dsp_cnt[2]; -+ u64 move_rq_cnt[2]; -+ u64 select_cpu[2]; -+ -+ u64 gdsq_count[MAX_GLOBAL_DSQS][2]; -+ u64 err_idx[5]; -+ u64 pcp_timeout_cnt[NR_CPUS]; -+ u64 pcp_ldsq_count[NR_CPUS][2]; -+ u64 pcp_enql_cnt[NR_CPUS]; -+} stats_data; -+ -+static struct { -+ cpumask_var_t ex_free; -+ cpumask_var_t exclusive; -+ cpumask_var_t partial; -+ cpumask_var_t big; -+ cpumask_var_t little; -+} iso_masks __read_mostly; -+ -+/* -+ * Need more synchronization for these two variables? -+ * I choose not to. -+ */ -+static int l_need_rescue, b_need_rescue; -+ -+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...) \ -+{ \ -+ char buf[MAX_FATAL_INFO]; \ -+ \ -+ scnprintf(buf, MAX_FATAL_INFO, fmt, ##args); \ -+ hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf); \ -+ trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable), \ -+ READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf); \ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); \ -+} \ -+ -+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2) -+{ -+ int c1, c2; -+ struct cpumask mask = {.bits = {0}}; -+ struct cpumask tmp = {.bits = {0}}; -+ -+ if (!slim_stats) -+ return false; -+ -+ if (!rq1 || !rq2) -+ return false; -+ -+ c1 = cpu_of(rq1); -+ c2 = cpu_of(rq2); -+ cpumask_set_cpu(c1, &mask); -+ cpumask_set_cpu(c2, &mask); -+ -+ if (cpumask_and(&tmp, iso_masks.little, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.big, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.partial, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ return false; -+} -+ -+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu) -+{ -+ unsigned long flags; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ if (!slim_stats) -+ return; -+ -+ switch (item) { -+ case GLOBAL_STAT: -+ fallthrough; -+ case CPU_ALLOW_FAIL: -+ fallthrough; -+ case RT_CNT: -+ fallthrough; -+ case KEY_TASK_CNT: -+ fallthrough; -+ case SWITCH_IDX: -+ fallthrough; -+ case TIMEOUT_CNT: -+ fallthrough; -+ case TOTAL_DSP_CNT: -+ fallthrough; -+ case MOVE_RQ_CNT: -+ fallthrough; -+ case SELECT_CPU: -+ pval = pbase + item * 2 + idx; -+ break; -+ case GDSQ_CNT: -+ pval = &stats_data.gdsq_count[dsq_id][idx]; -+ break; -+ case ERR_IDX: -+ pval = &stats_data.err_idx[idx]; -+ break; -+ case PCP_TIMEOUT_CNT: -+ pval = &stats_data.pcp_timeout_cnt[cpu]; -+ break; -+ case PCP_LDSQ_CNT: -+ pval = &stats_data.pcp_ldsq_count[cpu][idx]; -+ break; -+ case PCP_ENQL_CNT: -+ pval = &stats_data.pcp_enql_cnt[cpu]; -+ break; -+ default: -+ return; -+ } -+ -+ spin_lock_irqsave(&stats_lock, flags); -+ *pval += 1; -+ spin_unlock_irqrestore(&stats_lock, flags); -+} -+ -+static inline bool handle_ret(int ret, int *idx, int len) -+{ -+ if (ret < 0 || ret >= len - *idx) -+ return true; -+ *idx += ret; -+ return false; -+} -+ -+#define PRINT_INTV (5 * HZ) -+void stats_print(char *buf, int len) -+{ -+ int idx = 0, i, j, ret; -+ int item = 0; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ for (item = 0; item < MAX_ITEMS; item++) { -+ if (item <= DWORD_STAT_END) { -+ pval = pbase + item * 2; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n", -+ stats_str[item], pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == GDSQ_CNT) { -+ for (j = 0; j < MAX_GLOBAL_DSQS; j++) { -+ pval = (u64 *)&stats_data.gdsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == ERR_IDX) { -+ pval = (u64 *)&stats_data.err_idx; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n", -+ stats_str[item], pval[0], -+ pval[1], pval[2], pval[3], pval[4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == PCP_TIMEOUT_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_timeout_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_LDSQ_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_ldsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_ENQL_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_enql_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } -+ } -+ -+ if (!md_info) { -+ buf[idx] = '\0'; -+ return; -+ } -+ -+ ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_SWITCHS; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "sw_rec[%d] = %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.sw_rec[i].switch_at, -+ md_info->kern_dump.sw_rec[i].is_success, -+ md_info->kern_dump.sw_rec[i].end_state, -+ md_info->kern_dump.sw_rec[i].switch_reason); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_EXCEP_ID; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "excep[%d] = %llu %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.excep_rec[i][0], -+ md_info->kern_dump.excep_rec[i][1], -+ md_info->kern_dump.excep_rec[i][2], -+ md_info->kern_dump.excep_rec[i][3], -+ md_info->kern_dump.excep_rec[i][4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n", -+ i, md_info->kern_dump.excep_idx[i]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ -+ buf[idx] = '\0'; -+} -+ -+enum cpu_type { -+ LITTLE, -+ BIG, -+ PARTIAL, -+ EXCLUSIVE, -+ EX_FREE, -+ INVALID -+}; -+ -+enum dsq_type { -+ GLOBAL_DSQ, -+ PCP_DSQ, -+ OTHER, -+ MAX_DSQ_TYPE, -+}; -+ -+static enum cpu_type cpu_cluster(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (get_hmbird_rq(rq)->exclusive) { -+ return EXCLUSIVE; -+ } else { -+ if (cpumask_test_cpu(cpu, iso_masks.little)) -+ return LITTLE; -+ else if (cpumask_test_cpu(cpu, iso_masks.big)) -+ return BIG; -+ else if (cpumask_test_cpu(cpu, iso_masks.partial)) -+ return PARTIAL; -+ else if (cpumask_test_cpu(cpu, iso_masks.exclusive)) -+ return EXCLUSIVE; -+ else if (cpumask_test_cpu(cpu, iso_masks.ex_free)) -+ return EX_FREE; -+ } -+ return INVALID; -+} -+ -+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq) -+{ -+ if (!dsq) -+ return OTHER; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= GDSQS_ID_BASE) && -+ ((dsq->id & 0xff) < MAX_GLOBAL_DSQS)) -+ return GLOBAL_DSQ; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) && -+ ((dsq->id & 0xff) < max_hmbird_dsq_internal_id)) -+ return PCP_DSQ; -+ -+ return OTHER; -+} -+ -+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq) -+{ -+ enum dsq_type type; -+ -+ type = get_dsq_type(dsq); -+ switch (type) { -+ case GLOBAL_DSQ: -+ case PCP_DSQ: -+ return (dsq->id & 0xff) - GDSQS_ID_BASE; -+ default: -+ return -1; -+ } -+ return -1; -+} -+ -+static void update_cpus_idle(bool set, struct cpumask *mask) -+{ -+ int cpu; -+ struct rq *rq; -+ -+ if (set) { -+ for_each_cpu(cpu, mask) { -+ rq = cpu_rq(cpu); -+ if (is_idle_task(rq->curr)) -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ } -+ } else -+ cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask); -+} -+ -+static void set_partial_status(bool enable, bool little, bool big) -+{ -+ WRITE_ONCE(partial_enable, enable); -+ WRITE_ONCE(l_need_rescue, little); -+ WRITE_ONCE(b_need_rescue, big); -+} -+ -+static bool is_little_need_rescue(void) -+{ -+ return READ_ONCE(l_need_rescue); -+} -+ -+static bool is_big_need_rescue(void) -+{ -+ return READ_ONCE(b_need_rescue); -+} -+ -+static bool is_partial_enabled(void) -+{ -+ return READ_ONCE(partial_enable); -+} -+ -+static bool is_partial_cpu(int cpu) -+{ -+ return cpumask_test_cpu(cpu, iso_masks.partial); -+} -+ -+static void set_iso_par_free(bool enable) -+{ -+ WRITE_ONCE(isolate_ctrl, enable); -+} -+ -+static bool is_iso_par_free(void) -+{ -+ return READ_ONCE(isolate_ctrl); -+} -+ -+static bool skip_update_idle(void) -+{ -+ int cpu = smp_processor_id(); -+ enum cpu_type type = cpu_cluster(cpu); -+ -+ if ((type == EXCLUSIVE && !is_iso_par_free()) || -+ /* partial enable may changed during idle, it doesn't matter. */ -+ (!is_partial_enabled() && type == PARTIAL)) -+ return true; -+ -+ return false; -+} -+ -+static void init_isolate_cpus(void) -+{ -+ WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -+ cpumask_set_cpu(0, iso_masks.big); -+ cpumask_set_cpu(1, iso_masks.big); -+ cpumask_set_cpu(2, iso_masks.big); -+ cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(4, iso_masks.big); -+ cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(6, iso_masks.partial); -+ cpumask_set_cpu(7, iso_masks.ex_free); -+} -+ -+extern spinlock_t css_set_lock; -+ -+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp) -+{ -+ int i; -+ struct cgroup *anc; -+ -+ spin_lock_irq(&css_set_lock); -+ for (i = 0; i < cgrp->level; i++) { -+ anc = cgrp->ancestors[i]; -+ if (anc->level != CREATE_DSQ_LEVEL_WITHIN) -+ continue; -+ cgroup_get(anc); -+ spin_unlock_irq(&css_set_lock); -+ return anc; -+ } -+ spin_unlock_irq(&css_set_lock); -+ hmbird_err(NO_CGROUP_L1, ":error cgroup = %s\n", cgrp->kn->name); -+ return NULL; -+} -+ -+#define PCP_IDX_BIT (1 << 31) -+ -+static bool is_pcp_rt(struct task_struct *p) -+{ -+ return rt_prio(p->prio) && (p->nr_cpus_allowed == 1); -+} -+ -+static bool is_pcp_idx(int idx) -+{ -+ return idx >= MAX_GLOBAL_DSQS; -+} -+ -+static bool is_critical_system_task(struct task_struct *p) -+{ -+ int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK; -+ -+ return (p->prio < (MAX_RT_PRIO >> 1) && -+ (sp_dl < SCHED_PROP_DEADLINE_LEVEL3)); -+} -+ -+#define ISOLATE_TASK_TOP_BIT (1 << 17) -+#define PIPELINE_TASK_TOP_BIT (1 << 9) -+static bool is_pipeline_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT); -+} -+ -+static bool is_isolate_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT); -+} -+ -+static bool is_critical_app_task_without_isolate(struct task_struct *p) -+{ -+ return task_is_top_task(p) && !is_isolate_task(p); -+} -+ -+static int find_idx_from_task(struct task_struct *p) -+{ -+ int idx, cpu; -+ int sp_dl; -+ struct task_group *tg = p->sched_task_group; -+ -+ if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) { -+ cpu = cpumask_any(p->cpus_ptr); -+ idx = cpu + MAX_GLOBAL_DSQS; -+ return idx; -+ } -+ -+ if (is_critical_system_task(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL0; -+ goto done; -+ } -+ -+ if (is_critical_app_task_without_isolate(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL1; -+ goto done; -+ } -+ -+ sp_dl = hmbird_get_dsq_id(p); -+ if (sp_dl) { -+ idx = sp_dl; -+ goto done; -+ } -+ -+ if (rt_prio(p->prio)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL3; -+ goto done; -+ } -+ -+ if (tg && tg->css.cgroup && tg->css.cgroup->kn) { -+ if (likely(tg->css.cgroup->kn->id >= 0 && -+ tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS)) -+ idx = cgroup_ids_table[tg->css.cgroup->kn->id]; -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ -+done: -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) { -+ hmbird_err(DSQ_ID_ERR, " : idx error, idx = %d-----\n", idx); -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } -+ return idx; -+} -+ -+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p) -+{ -+ int idx; -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq; -+ -+ if (!p) -+ return NULL; -+ -+ idx = find_idx_from_task(p); -+ if (is_pcp_idx(idx)) { -+ idx -= MAX_GLOBAL_DSQS; -+ dsq = &per_cpu(pcp_ldsq, idx); -+ get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq); -+ slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx); -+ } else { -+ dsq = &gdsqs[idx]; -+ get_hmbird_ts(p)->gdsq_idx = idx; -+ slim_stats_record(GDSQ_CNT, 0, idx, 0); -+ } -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ dsq->last_consume_at = jiffies; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ return dsq; -+} -+ -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq); -+ -+static void set_partial_rescue(bool p_state, bool l_over, bool b_over) -+{ -+ set_partial_status(p_state, l_over, b_over); -+ update_cpus_idle(p_state, iso_masks.partial); -+ hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled()); -+ hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue()); -+ hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue()); -+} -+ -+static void free_isocpu(bool enable) -+{ -+ set_iso_par_free(enable); -+ update_cpus_idle(enable, iso_masks.ex_free); -+ hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free()); -+} -+ -+inline u64 get_hmbird_cpu_util(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed) -+ return 0; -+ u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed); -+ u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size); -+ -+ do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT); -+ -+ return prev_runnable_sum_fixed; -+} -+ -+static inline unsigned int get_scaling_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->max; -+} -+ -+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static u64 get_cpus_max_util(struct cpumask *mask) -+{ -+ int cpu; -+ u64 max = 0; -+ u64 util, ratio; -+ unsigned long effective_cap = 0; -+ for_each_cpu(cpu, mask) { -+ if (slim_walt_ctrl) -+ slim_get_cpu_util(cpu, &util); -+ else -+ util = get_hmbird_cpu_util(cpu); -+ -+ /* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/ -+ if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu))) -+ effective_cap = arch_scale_cpu_capacity(cpu); -+ else -+ effective_cap = arch_scale_cpu_capacity(cpu) * -+ get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu); -+ -+ ratio = util * 100 / effective_cap; -+ hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util); -+ hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n", -+ cpu, (u64)arch_scale_cpu_capacity(cpu)); -+ hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n", -+ cpu, (u64)effective_cap); -+ -+ if (ratio > 100) -+ ratio = 100; -+ -+ if (ratio > max) -+ max = ratio; -+ } -+ return max; -+} -+ -+static bool cluster_need_rescue(struct cpumask *mask, int hres) -+{ -+ return get_cpus_max_util(mask) > hres; -+} -+ -+void partial_dynamic_ctrl(void) -+{ -+ u64 lmax = 0, bmax = 0; -+ bool l_over, l_under, b_over, b_under; -+ static bool last_l_over, last_b_over; -+ static unsigned long last_check; -+ -+ /* Check partial every jiffies. need lock? */ -+ if (time_before_eq(jiffies, READ_ONCE(last_check))) -+ return; -+ -+ WRITE_ONCE(last_check, jiffies); -+ -+ lmax = get_cpus_max_util(iso_masks.little); -+ l_over = lmax > parctrl_high_ratio_l; -+ l_under = lmax < parctrl_low_ratio_l; -+ -+ bmax = get_cpus_max_util(iso_masks.big); -+ b_over = bmax > parctrl_high_ratio; -+ b_under = bmax < parctrl_low_ratio; -+ -+ if (is_partial_enabled() && (l_over || b_over)) { -+ if (last_l_over != l_over || last_b_over != b_over) -+ set_partial_rescue(true, l_over, b_over); -+ } else if (!is_partial_enabled() && (l_over || b_over)) { -+ set_partial_rescue(true, l_over, b_over); -+ } else if (is_partial_enabled() && l_under && b_under) { -+ set_partial_rescue(false, false, false); -+ } -+ last_l_over = l_over; -+ last_b_over = b_over; -+ -+ if (is_partial_enabled()) { -+ bmax = get_cpus_max_util(iso_masks.partial); -+ if (!is_iso_par_free() && bmax > isoctrl_high_ratio) { -+ hmbird_info_trace("partial max = %llu\n", bmax); -+ free_isocpu(true); -+ } else if (is_iso_par_free() && bmax < isoctrl_low_ratio) -+ free_isocpu(false); -+ } else if (is_iso_par_free()) { -+ free_isocpu(false); -+ } -+} -+ -+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx) -+{ -+ hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx); -+} -+ -+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx) -+{ -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[idx])) { -+ slim_stats_record(GDSQ_CNT, 1, idx, 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ int i; -+ -+ for (i = 0; i < UX_COMPATIBLE_IDX; i++) { -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ slim_stats_record(GDSQ_CNT, 1, i, 0); -+ return 1; -+ } -+ } -+ return 0; -+} -+ -+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) { -+ slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ goto clear_timeout; -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) -+ goto clear_timeout; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ hmbird_info_trace( -+ "dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n", -+ dsq_id_to_internal(dsq), entity->task->comm, -+ jiffies, msecs_to_jiffies(deadline), entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1); -+ return; -+ -+clear_timeout: -+ hmbird_info_trace("dsq[%d] clear timeout\n", -+ dsq_id_to_internal(dsq)); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0); -+ dsq->is_timeout = false; -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq)); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime) -+{ -+ hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime); -+} -+ -+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any) -+{ -+ bool is_timeout; -+ int cpu = cpu_of(rq); -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu); -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ is_timeout = dsq->is_timeout; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ /* -+ * dsq->is_timeout may change here, let it be. -+ * it won't cause serious problems. -+ * the same for consume_dispatch_q later. -+ */ -+ if (!is_timeout && !any) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, dsq)) { -+ if (is_timeout) { -+ hmbird_info_trace("dsq[%d] consume a pcp timeout task\n", -+ dsq_id_to_internal(dsq)); -+ update_timeout_stats(rq, dsq, pcp_dsq_deadline); -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu); -+ } -+ slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu); -+ return 1; -+ } -+ /* -+ * No pcp task, clear quota. -+ */ -+ if (any) { -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+ } -+ return 0; -+} -+ -+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf) -+{ -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ } -+ return 0; -+} -+ -+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf, -+ int tmp, int cidx, int tidx) -+{ -+ unsigned long flags; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) { -+ if (tmp != cidx) { -+ spin_lock(&sinfo.lock); -+ sinfo.curr_idx[tidx] = tmp; -+ spin_unlock(&sinfo.lock); -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]); -+ slim_stats_record(SWITCH_IDX, 0, 0, 0); -+ } -+ slim_stats_record(GDSQ_CNT, 1, tmp, 0); -+ -+ raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags); -+ gdsqs[tmp].last_consume_at = jiffies; -+ raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags); -+ return 1; -+ } -+ return 0; -+ -+} -+ -+static int get_cidx(struct cluster_ctx *ctx) -+{ -+ int cidx; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ if (cidx < ctx->lower || cidx >= ctx->upper) { -+ sinfo.curr_idx[ctx->tidx] = ctx->lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]); -+ slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ } -+ spin_unlock(&sinfo.lock); -+ -+ return cidx; -+} -+ -+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = CLUSTER_SEPARATE_IDX; -+ if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) { -+ pr_debug(" %s iso_masks.little first is %d\n", -+ __func__, cpumask_first(iso_masks.little)); -+ ctx->upper = NON_PERIOD_END; -+ } -+ ctx->tidx = 0; -+ break; -+ case LITTLE: -+ ctx->lower = CLUSTER_SEPARATE_IDX; -+ ctx->upper = NON_PERIOD_END; -+ if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) { -+ pr_debug(" %s iso_masks.big first is %d", -+ __func__, cpumask_first(iso_masks.big)); -+ ctx->lower = NON_PERIOD_START; -+ } -+ ctx->tidx = 1; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ case LITTLE: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = NON_PERIOD_END; -+ ctx->tidx = 0; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ if (cluster_separate) { -+ return gen_cluster_ctx_separate(ctx, type); -+ } else { -+ return gen_cluster_ctx_common(ctx, type); -+ } -+} -+ -+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ int i; -+ bool is_timeout; -+ unsigned long flags; -+ struct cluster_ctx ctx; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ /* Third param means only consume timeout pcp dsq. */ -+ if (consume_pcp_dsq(rq, rf, false)) -+ return 1; -+ -+ for (i = ctx.lower; i < ctx.upper; i++) { -+ raw_spin_lock_irqsave(&gdsqs[i].lock, flags); -+ is_timeout = gdsqs[i].is_timeout; -+ raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags); -+ /* gdsqs[i].is_timeout may change here, let it be... */ -+ if (is_timeout) { -+ /* -+ * consume_dispatch_q will acquire dsq-lock, -+ * So cannot keep lock here, annoy enough. -+ * may rewrite a consume_dispatch_q_locked, TODO. -+ */ -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ hmbird_info_trace("dsq[%d] consume a timeout task\n", i); -+ slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0); -+ update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]); -+ return 1; -+ } -+ } -+ } -+ return 0; -+} -+ -+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ struct cluster_ctx ctx; -+ int cidx; -+ int tmp; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ cidx = get_cidx(&ctx); -+ tmp = cidx; -+ do { -+ if (check_pcp_dsq_round(rq, rf)) -+ return 1; -+ if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx)) -+ return 1; -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[tmp] = 0; -+ systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]); -+ tmp++; -+ if (tmp >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ tmp = ctx.lower; -+ } -+ spin_unlock(&sinfo.lock); -+ } while (tmp != cidx); -+ -+ return consume_pcp_dsq(rq, rf, true); -+} -+ -+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ enum cpu_type type = cpu_cluster(cpu_of(rq)); -+ bool period_allowed = !get_hmbird_rq(rq)->period_disallow; -+ bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow; -+ -+ switch (type) { -+ case EX_FREE: -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ break; -+ case EXCLUSIVE: -+ if (!is_iso_par_free()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ } -+ /* Only non-period task can run on isolate cpus */ -+ if (!READ_ONCE(iso_free_rescue)) { -+ if (is_big_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ } else { -+ /* Free run, can run on any task. */ -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ case PARTIAL: -+ if (!is_partial_enabled()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ return 0; -+ } -+ if (is_big_need_rescue()) { -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ -+ case BIG: -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ if (is_iso_par_free() || (is_little_need_rescue() && -+ !cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) { -+ hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++); -+ return 1; -+ } -+ } -+ break; -+ -+ case LITTLE: -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ -+ if (is_iso_par_free() || (is_big_need_rescue() && -+ !cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) { -+ hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++); -+ return 1; -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return 0; -+} -+ -+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf) -+{ -+ return consume_hmbird_global_dsq(rq, rf); -+} -+ -+ -+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time) -+{ -+ int idx; -+ -+ /* which dsq belongs to while task enqueue, task will consume its running time. */ -+ idx = get_hmbird_ts(p)->gdsq_idx; -+ /* Only non-period dsq share running time between each other. */ -+ if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id) -+ return; -+ -+ if (idx >= MAX_GLOBAL_DSQS) { -+ per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } else { -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[idx] += exec_time; -+ spin_unlock(&sinfo.lock); -+ systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]); -+ } -+} -+ -+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type) -+{ -+ int cidx; -+ struct cluster_ctx ctx; -+ int cpu = cpu_of(rq); -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ if (cidx < ctx.lower || cidx >= ctx.upper) { -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ slim_stats_record(ERR_IDX, ctx.tidx, 0, 0); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ } -+ -+ while (1) { -+ if (per_cpu(pcp_info, cpu).pcp_round) { -+ if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) { -+ hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n", -+ cpu, per_cpu(pcp_info, cpu).rtime); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } -+ } -+ if (sinfo.rtime[cidx] < dsq_quota[cidx]) -+ break; -+ -+ /* clear current dsq rtime */ -+ sinfo.rtime[cidx] = 0; -+ systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]); -+ -+ sinfo.curr_idx[ctx.tidx]++; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", -+ ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ } -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ slim_stats_record(SWITCH_IDX, 1, 0, 0); -+ } -+ spin_unlock(&sinfo.lock); -+} -+ -+ -+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p) -+{ -+ enum cpu_type type; -+ -+ if (!rq || !p) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ switch (type) { -+ case PARTIAL: -+ return; -+ case EXCLUSIVE: -+ return; -+ case EX_FREE: -+ return; -+ default: -+ break; -+ } -+ update_dsq_idx(rq, p, type); -+} -+ -+ -+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ int dsq_id; -+ -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo) || dsq->is_timeout) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ hmbird_deferred_err(SCAN_ENTITY_NULL, -+ " : entity is NULL, dsq->id = %llu\n", dsq->id); -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ dsq->is_timeout = true; -+ dsq_id = dsq_id_to_internal(dsq); -+ hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n", -+ dsq_id, entity->task->comm, jiffies, entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1); -+ raw_spin_unlock(&dsq->lock); -+ -+ return true; -+} -+ -+void scan_timeout(struct rq *rq) -+{ -+ int i; -+ int cpu = cpu_of(rq); -+ struct hmbird_dispatch_q *dsq; -+ static u64 last_scan_at; -+ static DEFINE_PER_CPU(u64, pcp_last_scan_at); -+ -+ if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu))) -+ return; -+ per_cpu(pcp_last_scan_at, cpu) = jiffies; -+ -+ dsq = &per_cpu(pcp_ldsq, cpu); -+ scan_dsq_timeout(rq, dsq, pcp_dsq_deadline); -+ -+ if (time_before_eq(jiffies, (unsigned long)last_scan_at)) -+ return; -+ last_scan_at = jiffies; -+ -+ for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) { -+ dsq = &gdsqs[i]; -+ scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]); -+ } -+} -+ -+/*******************************Initialize***********************************/ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id); -+static void init_dsq_at_boot(void) -+{ -+ int i, cpu; -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i)); -+ } -+ for_each_possible_cpu(cpu) -+ init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i + cpu)); -+ -+ max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu; -+ spin_lock_init(&sinfo.lock); -+} -+ -+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgrp_name_to_idx(struct cgroup *cgrp) -+{ -+ int idx; -+ -+ if (!cgrp) -+ return -1; -+ -+ if (!strcmp(cgrp->kn->name, "display") -+ || !strcmp(cgrp->kn->name, "multimedia")) -+ idx = 5; /* 8ms */ -+ else if (!strcmp(cgrp->kn->name, "top-app") -+ || !strcmp(cgrp->kn->name, "ss-top")) -+ idx = 6; /* 16ms */ -+ else if (!strcmp(cgrp->kn->name, "ssfg") -+ || !strcmp(cgrp->kn->name, "foreground")) -+ idx = 7; /* 32ms */ -+ else if (!strcmp(cgrp->kn->name, "bg") -+ || !strcmp(cgrp->kn->name, "log") -+ || !strcmp(cgrp->kn->name, "dex2oat") -+ || !strcmp(cgrp->kn->name, "background")) -+ idx = 9; /* 128ms */ -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */ -+ -+ return idx; -+} -+ -+static void init_root_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX); -+} -+ -+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) { -+ pr_err("%s idx err!\n", __func__); -+ return; -+ } -+ -+ if (cgroup_ids_table[cgrp->kn->id] == -1) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp)); -+} -+ -+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ struct cgroup *l1cgrp; -+ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ l1cgrp = cgroup_ancestor_l1(cgrp); -+ if (l1cgrp) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp)); -+ cgroup_put(l1cgrp); -+} -+ -+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg) -+{ -+ switch (cgrp->level) { -+ case 0: -+ init_root_tg(cgrp, tg); -+ break; -+ case 1: -+ init_level1_tg(cgrp, tg); -+ break; -+ default: -+ init_child_tg(cgrp, tg); -+ break; -+ } -+} -+ -+/**************************************************************************/ -+ -+struct hmbird_task_iter { -+ struct hmbird_entity cursor; -+ struct task_struct *locked; -+ struct rq *rq; -+ struct rq_flags rf; -+}; -+ -+/** -+ * hmbird_task_iter_init - Initialize a task iterator -+ * @iter: iterator to init -+ * -+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized, -+ * @iter must eventually be exited with hmbird_task_iter_exit(). -+ * -+ * hmbird_tasks_lock may be released between this and the first next() call or -+ * between any two next() calls. If hmbird_tasks_lock is released between two -+ * next() calls, the caller is responsible for ensuring that the task being -+ * iterated remains accessible either through RCU read lock or obtaining a -+ * reference count. -+ * -+ * All tasks which existed when the iteration started are guaranteed to be -+ * visited as long as they still exist. -+ */ -+static void hmbird_task_iter_init(struct hmbird_task_iter *iter) -+{ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR }; -+ list_add(&iter->cursor.tasks_node, &hmbird_tasks); -+ iter->locked = NULL; -+} -+ -+/** -+ * hmbird_task_iter_exit - Exit a task iterator -+ * @iter: iterator to exit -+ * -+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held. -+ * If the iterator holds a task's rq lock, that rq lock is released. See -+ * hmbird_task_iter_init() for details. -+ */ -+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ if (list_empty(cursor)) -+ return; -+ -+ list_del_init(cursor); -+} -+ -+/** -+ * hmbird_task_iter_next - Next task -+ * @iter: iterator to walk -+ * -+ * Visit the next task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ struct hmbird_entity *pos; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ list_for_each_entry(pos, cursor, tasks_node) { -+ if (&pos->tasks_node == &hmbird_tasks) -+ return NULL; -+ if (!(pos->flags & HMBIRD_TASK_CURSOR)) { -+ list_move(cursor, &pos->tasks_node); -+ return pos->task; -+ } -+ } -+ -+ /* can't happen, should always terminate at hmbird_tasks above */ -+ hmbird_deferred_err(ITER_RET_NULL, " : unreachable path in scx_task_iter_next\n"); -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered - Next non-idle task -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ while ((p = hmbird_task_iter_next(iter))) { -+ if (!is_idle_task(p)) -+ return p; -+ } -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init() -+ * for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ p = hmbird_task_iter_next_filtered(iter); -+ if (!p) -+ return NULL; -+ -+ iter->rq = task_rq_lock(p, &iter->rf); -+ iter->locked = p; -+ return p; -+} -+ -+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void) -+{ -+ return atomic_read(&hmbird_ops_enable_state_var); -+} -+ -+static enum hmbird_ops_enable_state -+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to) -+{ -+ return atomic_xchg(&hmbird_ops_enable_state_var, to); -+} -+ -+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to, -+ enum hmbird_ops_enable_state from) -+{ -+ int from_v = from; -+ -+ return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to); -+} -+ -+static bool hmbird_ops_disabling(void) -+{ -+ return false; -+} -+ -+/** -+ * wait_ops_state - Busy-wait the specified ops state to end -+ * @p: target task -+ * @opss: state to wait the end of -+ * -+ * Busy-wait for @p to transition out of @opss. This can only be used when the -+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also -+ * has load_acquire semantics to ensure that the caller can see the updates made -+ * in the enqueueing and dispatching paths. -+ */ -+static void wait_ops_state(struct task_struct *p, u64 opss) -+{ -+ do { -+ cpu_relax(); -+ } while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss); -+} -+ -+ -+static void update_curr_hmbird(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ u64 now = rq_clock_task(rq); -+ u64 delta_exec; -+ -+ if (time_before_eq64(now, curr->se.exec_start)) -+ return; -+ -+ delta_exec = now - curr->se.exec_start; -+ curr->se.exec_start = now; -+ update_runningtime(rq, curr, delta_exec); -+ curr->se.sum_exec_runtime += delta_exec; -+ account_group_exec_runtime(curr, delta_exec); -+ cgroup_account_cputime(curr, delta_exec); -+ -+ if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF) -+ get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec); -+ -+ trace_sched_stat_runtime(curr, delta_exec, 0); -+} -+ -+static bool hmbird_dsq_priq_less(struct rb_node *node_a, -+ const struct rb_node *node_b) -+{ -+ const struct hmbird_entity *a = -+ container_of(node_a, struct hmbird_entity, dsq_node.priq); -+ const struct hmbird_entity *b = -+ container_of(node_b, struct hmbird_entity, dsq_node.priq); -+ -+ return time_before64(a->dsq_vtime, b->dsq_vtime); -+} -+ -+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ bool is_local = dsq->id == HMBIRD_DSQ_LOCAL; -+ unsigned long flags; -+ -+ hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq || -+ !list_empty(&get_hmbird_ts(p)->dsq_node.fifo), -+ "task = %s, dsq->id = %llu\n", p->comm, dsq->id); -+ hmbird_cond_deferred_err(ENQ_EXIST2, -+ (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq), -+ "task = %s\n", p->comm); -+ -+ if (!is_local) { -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ); -+ hmbird_ops_error(": %s\n", -+ "attempting to dispatch to a destroyed dsq"); -+ /* fall back to the global dsq */ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ dsq = &hmbird_dsq_global; -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ } -+ } -+ -+ if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) { -+ get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ; -+ rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq, -+ hmbird_dsq_priq_less); -+ } else { -+ if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT)) -+ list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ else -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ } -+ dsq->nr++; -+ get_hmbird_ts(p)->dsq = dsq; -+ -+ /* -+ * We're transitioning out of QUEUEING or DISPATCHING. store_release to -+ * match waiters' load_acquire. -+ */ -+ if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS) -+ atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE); -+ -+ if (is_local) { -+ struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq); -+ struct rq *rq = hmbird->rq; -+ bool preempt = false; -+ -+ if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr && -+ rq->curr->sched_class == &hmbird_sched_class) { -+ get_hmbird_ts(rq->curr)->slice = 0; -+ preempt = true; -+ } -+ -+ if (preempt || sched_class_above(&hmbird_sched_class, -+ rq->curr->sched_class)) -+ resched_curr(rq); -+ } else { -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ } -+} -+ -+static void task_unlink_from_dsq(struct task_struct *p, -+ struct hmbird_dispatch_q *dsq) -+{ -+ if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) { -+ rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ; -+ } else { -+ list_del_init(&get_hmbird_ts(p)->dsq_node.fifo); -+ } -+} -+ -+static bool task_linked_on_dsq(struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+} -+ -+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq; -+ bool is_local = dsq == &hmbird_rq->local_dsq; -+ -+ if (!dsq) { -+ hmbird_cond_deferred_err(TASK_LINKED1, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ /* -+ * When dispatching directly from the BPF scheduler to a local -+ * DSQ, the task isn't associated with any DSQ but -+ * @get_hmbird_ts(p)->holding_cpu may be set under the protection of -+ * %HMBIRD_OPSS_DISPATCHING. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu >= 0) -+ get_hmbird_ts(p)->holding_cpu = -1; -+ return; -+ } -+ -+ if (!is_local) -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ /* -+ * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node -+ * can't change underneath us. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu < 0) { -+ /* @p must still be on @dsq, dequeue */ -+ hmbird_cond_deferred_err(TASK_UNLINKED, -+ !task_linked_on_dsq(p), "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ } else { -+ /* -+ * We're racing against dispatch_to_local_dsq() which already -+ * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the -+ * holding_cpu which tells dispatch_to_local_dsq() that it lost -+ * the race. -+ */ -+ hmbird_cond_deferred_err(TASK_LINKED2, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ get_hmbird_ts(p)->holding_cpu = -1; -+ } -+ get_hmbird_ts(p)->dsq = NULL; -+ -+ if (!is_local) -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+ -+static bool test_rq_online(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ return rq->online; -+#else -+ return true; -+#endif -+} -+ -+static void refill_task_slice(struct task_struct *p) -+{ -+ if (is_isolate_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO; -+ else if (is_pipeline_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2; -+ else -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+} -+ -+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -+ int sticky_cpu) -+{ -+ struct hmbird_dispatch_q *d; -+ s32 cpu = -1; -+ -+ hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), -+ "task = %s\n", p->comm); -+ -+ if (is_pcp_rt(p)) { -+ /* Enqueue percpu rt task to local directly. */ -+ /* Or cause a bug when disable dispatch. */ -+ if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ } -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (cpu >= 0) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ -+ if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ /* rq migration */ -+ if (sticky_cpu == cpu_of(rq)) -+ goto local_norefill; -+ /* -+ * If !rq->online, we already told the BPF scheduler that the CPU is -+ * offline. We're just trying to on/offline the CPU. Don't bother the -+ * BPF scheduler. -+ */ -+ if (unlikely(!test_rq_online(rq))) -+ goto local; -+ -+ /* see %HMBIRD_OPS_ENQ_LAST */ -+ if (enq_flags & HMBIRD_ENQ_LAST) -+ goto local; -+ -+ if (enq_flags & HMBIRD_ENQ_LOCAL) -+ goto local; -+ else -+ goto global; -+local: -+ /* -+ * For task-ordering, slice refill must be treated as implying the end -+ * of the current slice. Otherwise, the longer @p stays on the CPU, the -+ * higher priority it becomes from hmbird_prio_less()'s POV. -+ */ -+ refill_task_slice(p); -+local_norefill: -+ dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags); -+ slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq)); -+ return; -+ -+global: -+ d = find_dsq_from_task(p); -+ if (d) { -+ refill_task_slice(p); -+ dispatch_enqueue(d, p, enq_flags); -+ return; -+ } -+ slim_stats_record(GLOBAL_STAT, 0, 0, 0); -+ refill_task_slice(p); -+ dispatch_enqueue(&hmbird_dsq_global, p, enq_flags); -+} -+ -+static bool watchdog_task_watched(const struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->watchdog_node); -+} -+ -+static void watchdog_watch_task(struct rq *rq, struct task_struct *p) -+{ -+ lockdep_assert_rq_held(rq); -+ if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ get_hmbird_ts(p)->runnable_at = jiffies; -+ clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list); -+} -+ -+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) -+{ -+ list_del_init(&get_hmbird_ts(p)->watchdog_node); -+ if (reset_timeout) -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags) -+{ -+ int sticky_cpu = get_hmbird_ts(p)->sticky_cpu; -+ -+ enq_flags |= get_hmbird_rq(rq)->extra_enq_flags; -+ -+ if (sticky_cpu >= 0) -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ -+ /* -+ * Restoring a running task will be immediately followed by -+ * set_next_task_hmbird() which expects the task to not be on the BPF -+ * scheduler as tasks can only start running through local DSQs. Force -+ * direct-dispatch into the local DSQ by setting the sticky_cpu. -+ */ -+ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -+ sticky_cpu = cpu_of(rq); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_UNWATCHED, -+ !watchdog_task_watched(p), "task = %s\n", p->comm); -+ return; -+ } -+ -+ watchdog_watch_task(rq, p); -+ set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ get_hmbird_rq(rq)->nr_running++; -+ add_nr_running(rq, 1); -+ -+ do_enqueue_task(rq, p, enq_flags, sticky_cpu); -+} -+ -+static void ops_dequeue(struct task_struct *p, u64 deq_flags) -+{ -+ u64 opss; -+ -+ watchdog_unwatch_task(p, false); -+ -+ /* acquire ensures that we see the preceding updates on QUEUED */ -+ opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state); -+ -+ switch (opss & HMBIRD_OPSS_STATE_MASK) { -+ case HMBIRD_OPSS_NONE: -+ break; -+ case HMBIRD_OPSS_QUEUEING: -+ /* -+ * QUEUEING is started and finished while holding @p's rq lock. -+ * As we're holding the rq lock now, we shouldn't see QUEUEING. -+ */ -+ hmbird_deferred_err(DEQ_DEQING, " : unreachable path in %s\n", __func__); -+ break; -+ case HMBIRD_OPSS_QUEUED: -+ if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss, -+ HMBIRD_OPSS_NONE)) -+ break; -+ fallthrough; -+ case HMBIRD_OPSS_DISPATCHING: -+ /* -+ * If @p is being dispatched from the BPF scheduler to a DSQ, -+ * wait for the transfer to complete so that @p doesn't get -+ * added to its DSQ after dequeueing is complete. -+ * -+ * As we're waiting on DISPATCHING with the rq locked, the -+ * dispatching side shouldn't try to lock the rq while -+ * DISPATCHING is set. See dispatch_to_local_dsq(). -+ * -+ * DISPATCHING shouldn't have qseq set and control can reach -+ * here with NONE @opss from the above QUEUED case block. -+ * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss. -+ */ -+ wait_ops_state(p, HMBIRD_OPSS_DISPATCHING); -+ hmbird_cond_deferred_err(HMBIRD_OPN, -+ atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE, -+ "task = %s\n", p->comm); -+ break; -+ } -+} -+ -+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ -+ if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p), -+ "task = %s\n", p->comm); -+ return; -+ } -+ -+ ops_dequeue(p, deq_flags); -+ -+ if (slim_walt_ctrl) { -+ if (task_current(rq, p)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ if (deq_flags & HMBIRD_DEQ_SLEEP) -+ set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else -+ clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), -+ (unsigned long *)&get_hmbird_ts(p)->flags); -+ -+ clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm); -+ hmbird_rq->nr_running--; -+ sub_nr_running(rq, 1); -+ dispatch_dequeue(hmbird_rq, p); -+} -+ -+static void yield_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ get_hmbird_ts(p)->slice = 0; -+} -+ -+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to) -+{ -+ return false; -+} -+ -+#ifdef CONFIG_SMP -+/** -+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -+ * @rq: rq to move the task into, currently locked -+ * @p: task to move -+ * @enq_flags: %HMBIRD_ENQ_* -+ * -+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller -+ * must: -+ * -+ * 1. Start with exclusive access to @p either through its DSQ lock or -+ * %HMBIRD_OPSS_DISPATCHING flag. -+ * -+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id(). -+ * -+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't -+ * deadlock with dequeue. -+ * -+ * 4. Lock @rq and the task_rq from #3. -+ * -+ * 5. Call this function. -+ * -+ * Returns %true if @p was successfully moved. %false after racing dequeue and -+ * losing. -+ */ -+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ struct rq *task_rq; -+ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * If dequeue got to @p while we were trying to lock both rq's, it'd -+ * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have -+ * updated it to different values afterwards, as this operation can't be -+ * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become -+ * raw_smp_processor_id() again before we're done. Thus, we can tell -+ * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is -+ * still raw_smp_processor_id(). -+ * -+ * See dispatch_dequeue() for the counterpart. -+ */ -+ if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id())) -+ return false; -+ -+ /* @p->rq couldn't have changed if we're still the holding cpu */ -+ task_rq = task_rq(p); -+ lockdep_assert_rq_held(task_rq); -+ deactivate_task(task_rq, p, 0); -+ set_task_cpu(p, cpu_of(rq)); -+ get_hmbird_ts(p)->sticky_cpu = cpu_of(rq); -+ -+ /* -+ * We want to pass hmbird-specific enq_flags but activate_task() will -+ * truncate the upper 32 bit. As we own @rq, we can pass them through -+ * @get_hmbird_rq(rq)->extra_enq_flags instead. -+ */ -+ hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags, -+ "task = %s\n", p->comm); -+ get_hmbird_rq(rq)->extra_enq_flags = enq_flags; -+ activate_task(rq, p, 0); -+ get_hmbird_rq(rq)->extra_enq_flags = 0; -+ -+ return true; -+} -+ -+#endif /* CONFIG_SMP */ -+ -+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu) -+{ -+ int fitable = 1; -+ -+ return fitable; -+} -+ -+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq, -+ struct hmbird_dispatch_q *dsq) -+{ -+ bool dsq_misfit; -+ int cpu = cpu_of(rq); -+ u64 task_util = 0; -+ struct cluster_ctx ctx; -+ int dsq_int = dsq_id_to_internal(dsq); -+ -+ if (!cpumask_test_cpu(cpu, iso_masks.little)) -+ return false; -+ -+ gen_cluster_ctx(&ctx, BIG); -+ dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 && -+ dsq_int <= SCHED_PROP_DEADLINE_LEVEL4); -+#ifdef CLUSTER_SEPARATE -+ /* In rescue mode, little will consume big cluster's dsq.*/ -+ dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper); -+#endif -+ if (!dsq_misfit) -+ return false; -+ -+ if (p) { -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &task_util); -+ else -+ task_util = get_hmbird_ts(p)->demand_scaled; -+ } -+ -+ if (task_util <= misfit_ds) -+ return false; -+ -+ hmbird_info_trace(":task %s can't run on cpu%d, util = %llu\n", -+ p->comm, cpu, task_util); -+ return true; -+} -+ -+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq) -+{ -+ if (!task_fits_cpu_hmbird(p, cpu_of(rq))) -+ return false; -+ -+ if (check_misfit_task_on_little(p, rq, dsq)) -+ return false; -+ -+ return likely(test_rq_online(rq)); -+} -+ -+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return; -+ -+ if (add) -+ skipn[idx]++; -+ else -+ skipn[idx] = 0; -+} -+ -+static bool skip_too_much(struct hmbird_dispatch_q *dsq) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return false; -+ -+ if (skip_num[idx] > 3) { -+ skip_num[idx] = 0; -+ return true; -+ } -+ -+ return false; -+} -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rb_node *rb_node; -+ struct rq *task_rq; -+ unsigned long flags; -+ bool moved = false; -+ struct task_struct *may_fit = NULL; -+ int skip = 0; -+ -+retry: -+ if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -+ return false; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) { -+ set_skip_num(dsq, skip_num, (bool)may_fit); -+ goto this_rq; -+ } -+ if (skip_too_much(dsq)) -+ goto remote_rq; -+ if (!may_fit) -+ may_fit = p; -+ if (++skip <= 3) -+ continue; -+ /* -+ * If the recent 3 tasks not fit, use the first one. -+ * and clear the skip, because the first one is consumed. -+ */ -+ set_skip_num(dsq, skip_num, false); -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ /* No more task, use the first may fit task.*/ -+ if (may_fit) { -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ -+ for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) -+ goto this_rq; -+ goto remote_rq; -+ } -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ return false; -+ -+this_rq: -+ /* @dsq is locked and @p is on this rq */ -+ hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo); -+ dsq->nr--; -+ hmbird_rq->local_dsq.nr++; -+ get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ -+remote_rq: -+#ifdef CONFIG_SMP -+ if (cpu_same_cluster_stat(p, rq, task_rq)) -+ slim_stats_record(MOVE_RQ_CNT, 0, 0, 0); -+ else -+ slim_stats_record(MOVE_RQ_CNT, 1, 0, 0); -+ /* -+ * @dsq is locked and @p is on a remote rq. @p is currently protected by -+ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -+ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -+ * rq lock or fail, do a little dancing from our side. See -+ * move_task_to_local_dsq(). -+ */ -+ hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id(); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ rq_unpin_lock(rq, rf); -+ double_lock_balance(rq, task_rq); -+ rq_repin_lock(rq, rf); -+ -+ moved = move_task_to_local_dsq(rq, p, 0); -+ -+ double_unlock_balance(rq, task_rq); -+#endif /* CONFIG_SMP */ -+ if (likely(moved)) { -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ } -+ may_fit = NULL; -+ goto retry; -+} -+ -+ -+static int balance_one(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf, bool local) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class; -+ -+ if (!hmbird_rq) -+ return 1; -+ -+ lockdep_assert_rq_held(rq); -+ -+ if (static_branch_unlikely(&hmbird_ops_cpu_preempt) && -+ unlikely(get_hmbird_rq(rq)->cpu_released)) { -+ /* -+ * If the previous sched_class for the current CPU was not HMBIRD, -+ * notify the BPF scheduler that it again has control of the -+ * core. This callback complements ->cpu_release(), which is -+ * emitted in hmbird_notify_pick_next_task(). -+ */ -+ get_hmbird_rq(rq)->cpu_released = false; -+ } -+ -+ if (prev_on_hmbird) -+ update_curr_hmbird(rq); -+ /* if there already are tasks to run, nothing to do */ -+ if (hmbird_rq->local_dsq.nr) -+ return 1; -+ -+ if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) { -+ slim_stats_record(GLOBAL_STAT, 1, 0, 0); -+ return 1; -+ } -+ -+ if (consume_dispatch_global(rq, rf)) -+ return 1; -+ -+ return 0; -+} -+ -+static int balance_hmbird(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf) -+{ -+ return balance_one(rq, prev, rf, true); -+} -+ -+/* -+ * output task util to systrace, only for debug mode. -+ * we can not output too many logs to systrace buffer even in debug mode -+ * only output debug-info while it exceed misfit_ds. -+ */ -+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p) -+{ -+ static DEFINE_PER_CPU(int, is_last_exceed); -+ int cpu = cpu_of(rq); -+ u64 util = 0; -+ -+ if (likely(!debug_enabled())) -+ return; -+ -+ if (!p) -+ return; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ util = uclamp_rq_util_with(rq, util, p); -+ -+ if (util >= misfit_ds) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util); -+ per_cpu(is_last_exceed, cpu) = true; -+ } else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0); -+ per_cpu(is_last_exceed, cpu) = false; -+ } else { -+ } -+} -+ -+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first) -+{ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ /* -+ * Core-sched might decide to execute @p before it is -+ * dispatched. Call ops_dequeue() to notify the BPF scheduler. -+ */ -+ ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC); -+ dispatch_dequeue(get_hmbird_rq(rq), p); -+ } -+ -+ p->se.exec_start = rq_clock_task(rq); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK); -+ } -+ -+ watchdog_unwatch_task(p, true); -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx); -+ systrace_output_cpu_ds(rq, p); -+ /* -+ * @p is getting newly scheduled or got kicked after someone updated its -+ * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird(). -+ */ -+ if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) != -+ (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) { -+ if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) -+ get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK; -+ else -+ get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK; -+ -+ sched_update_tick_dependency(rq); -+ } -+ -+ p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime; -+} -+ -+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p) -+{ -+ update_curr_hmbird(rq); -+ -+ update_dispatch_dsq_info(rq, p); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ watchdog_watch_task(rq, p); -+ -+ if (is_pipeline_task(p)) { -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1); -+ return; -+ } -+ -+ /* -+ * If we're in the pick_next_task path, balance_hmbird() should -+ * have already populated the local DSQ if there are any other -+ * available tasks. If empty, tell ops.enqueue() that @p is the -+ * only one available for this cpu. ops.enqueue() should put it -+ * on the local DSQ so that the subsequent pick_next_task_hmbird() -+ * can find the task unless it wants to trigger a separate -+ * follow-up scheduling event. -+ */ -+ if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1); -+ else -+ do_enqueue_task(rq, p, 0, -1); -+ } -+} -+ -+static struct task_struct *first_local_task(struct rq *rq) -+{ -+ struct rb_node *rb_node; -+ struct hmbird_entity *entity; -+ -+ if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) { -+ entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo, -+ struct hmbird_entity, dsq_node.fifo); -+ return entity->task; -+ } -+ -+ rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq); -+ if (rb_node) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ return entity->task; -+ } -+ return NULL; -+} -+ -+static struct task_struct *pick_next_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p; -+ -+ p = first_local_task(rq); -+ if (!p) -+ return NULL; -+ -+ if (unlikely(!get_hmbird_ts(p)->slice)) { -+ if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice) -+ hmbird_warned_zero_slice = true; -+ -+ refill_task_slice(p); -+ } -+ -+ set_next_task_hmbird(rq, p, true); -+ -+ return p; -+} -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task, -+ const struct sched_class *active) -+{ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * The callback is conceptually meant to convey that the CPU is no -+ * longer under the control of HMBIRD. Therefore, don't invoke the -+ * callback if the CPU is staying on HMBIRD, or going idle (in which -+ * case the HMBIRD scheduler has actively decided not to schedule any -+ * tasks on the CPU). -+ */ -+ if (likely(active >= &hmbird_sched_class)) -+ return; -+ -+ /* -+ * At this point we know that HMBIRD was preempted by a higher priority -+ * sched_class, so invoke the ->cpu_release() callback if we have not -+ * done so already. We only send the callback once between HMBIRD being -+ * preempted, and it regaining control of the CPU. -+ * -+ * ->cpu_release() complements ->cpu_acquire(), which is emitted the -+ * next time that balance_hmbird() is invoked. -+ */ -+ if (!get_hmbird_rq(rq)->cpu_released) -+ get_hmbird_rq(rq)->cpu_released = true; -+} -+ -+#ifdef CONFIG_SMP -+ -+static bool test_and_clear_cpu_idle(int cpu) -+{ -+ if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -+ if (cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ return true; -+ } else { -+ return false; -+ } -+} -+ -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) -+{ -+ int cpu; -+ -+ do { -+ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -+ if (cpu >= nr_cpu_ids) -+ return -EBUSY; -+ } while (!test_and_clear_cpu_idle(cpu)); -+ -+ return cpu; -+} -+ -+ -+static bool prev_cpu_misfit(int prev) -+{ -+ if (!is_partial_enabled() && is_partial_cpu(prev)) -+ return true; -+ -+ return false; -+} -+ -+static int heavy_rt_placement(struct task_struct *p, int prev) -+{ -+ struct cpumask tmp = {.bits = {0}}; -+ int cpu; -+ u64 util = 0; -+ -+ if (!rt_prio(p->prio)) -+ return -EFAULT; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ -+ if (util < misfit_ds) -+ return -EFAULT; -+ -+ if (is_partial_enabled()) -+ cpumask_or(&tmp, iso_masks.big, iso_masks.partial); -+ else -+ cpumask_copy(&tmp, iso_masks.big); -+ -+ cpu = hmbird_pick_idle_cpu(&tmp); -+ if (cpu >= 0) -+ return cpu; -+ -+ if (cpumask_test_cpu(prev, iso_masks.big) || -+ (is_partial_enabled() && is_partial_cpu(prev))) -+ return prev; -+ -+ return cpumask_first(&tmp); -+} -+ -+static int spec_task_before_pick_idle(struct task_struct *p, int prev) -+{ -+ int cpu; -+ -+ cpu = heavy_rt_placement(p, prev); -+ if (cpu >= 0) -+ return cpu; -+ return -EFAULT; -+} -+ -+static int cpumask_distribute_next(struct cpumask *mask, int *prev) -+{ -+ int p = *prev, n; -+ -+ n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1); -+ if (n < nr_cpu_ids) -+ WRITE_ONCE(*prev, n); -+ return n; -+} -+ -+static int repick_fallback_cpu(void) -+{ -+ /* -+ * partial cpu follow big cluster's Scheduling policy, -+ * simply return first bit cpu. -+ */ -+ return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev); -+} -+ -+/* -+ * Must return a valid cpu num, as this task's cpu. -+ */ -+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu, -+ struct cpumask *mask, int *prev_mask) -+{ -+ int cpu; -+ -+ cpu = hmbird_pick_idle_cpu(mask); -+ if (cpu >= 0) -+ return cpu; -+ return cpumask_distribute_next(mask, prev_mask); -+} -+ -+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type) -+{ -+ int idx; -+ struct cluster_ctx ctx; -+ -+ idx = find_idx_from_task(p); -+ if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS) -+ return false; -+ -+ gen_cluster_ctx(&ctx, type); -+ if (idx >= ctx.lower && idx < ctx.upper) -+ return true; -+ return false; -+} -+ -+static bool is_valid_cpu(int cpu) -+{ -+ return (cpu >= 0) && (cpu < nr_cpu_ids); -+} -+ -+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) -+{ -+ s32 cpu = -1; -+ struct cpumask mask = {.bits = {0}}; -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (is_valid_cpu(cpu)) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ partial_dynamic_ctrl(); -+ -+ if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ iso_masks.big, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ if (p->nr_cpus_allowed == 1) { -+ cpu = cpumask_any(p->cpus_ptr); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* For non-period global dsq, not contain pcp task. */ -+ if (task_only_blongs_to_cluster(p, LITTLE)) { -+ cpumask_copy(&mask, iso_masks.little); -+ if (unlikely(l_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &little_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ if (task_only_blongs_to_cluster(p, BIG)) { -+ cpumask_copy(&mask, iso_masks.big); -+ if (unlikely(b_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ cpu = spec_task_before_pick_idle(p, prev_cpu); -+ if (is_valid_cpu(cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* if the previous CPU is idle, dispatch directly to it */ -+ if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+ } -+ -+ cpu = hmbird_pick_idle_cpu(cpu_possible_mask); -+ if (is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ if (prev_cpu_misfit(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ cpu = repick_fallback_cpu(); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+} -+ -+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags) -+{ -+ return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags); -+} -+ -+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx) -+{ -+ set_cpus_allowed_common(p, ctx); -+} -+ -+static void reset_idle_masks(void) -+{ -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little); -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big); -+ if (is_partial_enabled()) -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial); -+ hmbird_has_idle_cpus = true; -+} -+ -+void __hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ int cpu = cpu_of(rq); -+ struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -+ -+ if (skip_update_idle()) -+ return; -+ -+ if (idle) { -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ if (!hmbird_has_idle_cpus) -+ hmbird_has_idle_cpus = true; -+ -+ /* -+ * idle_masks.smt handling is racy but that's fine as it's only -+ * for optimization and self-correcting. -+ */ -+ for_each_cpu(cpu, sib_mask) { -+ if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -+ return; -+ } -+ cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -+ } else { -+ cpumask_clear_cpu(cpu, idle_masks.cpu); -+ if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ -+ cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -+ } -+} -+ -+#else /* !CONFIG_SMP */ -+ -+static bool test_and_clear_cpu_idle(int cpu) { return false; } -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } -+static void reset_idle_masks(void) {} -+ -+#endif /* CONFIG_SMP */ -+ -+static bool check_rq_for_timeouts(struct rq *rq) -+{ -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rq_flags rf; -+ -+ rq_lock_irqsave(rq, &rf); -+ list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) { -+ unsigned long last_runnable; -+ -+ p = entity->task; -+ last_runnable = get_hmbird_ts(p)->runnable_at; -+ -+ if (unlikely(time_after(jiffies, -+ last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -+ -+ rq_unlock_irqrestore(rq, &rf); -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb", -+ p->comm, p->pid, -+ dur_ms / 1000, dur_ms % 1000, -+ get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0, -+ cpumask_pr_args(&p->cpus_mask)); -+ return 1; -+ } -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ return 0; -+} -+ -+static void hmbird_watchdog_workfn(struct work_struct *work) -+{ -+ int cpu; -+ bool timeout; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ -+ for_each_online_cpu(cpu) { -+ timeout = check_rq_for_timeouts(cpu_rq(cpu)); -+ if (unlikely(timeout)) -+ break; -+ -+ cond_resched(); -+ } -+ if (!timeout) -+ queue_delayed_work(system_unbound_wq, to_delayed_work(work), -+ hmbird_watchdog_timeout / 2); -+} -+ -+static void set_pcp_round(struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) { -+ per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round); -+ per_cpu(pcp_info, cpu).pcp_round = true; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+} -+ -+ -+/* -+ * Just for debug: output hmbird on/off state per 10s. -+ */ -+#define OUTPUT_INTVAL (msecs_to_jiffies(10 * 1000)) -+static void inform_hmbird_onoff_from_systrace(void) -+{ -+ static unsigned long __read_mostly next_print; -+ -+ if (time_before(jiffies, READ_ONCE(next_print))) -+ return; -+ -+ WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL); -+ hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio); -+} -+ -+void hmbird_notify_sched_tick(void) -+{ -+ unsigned long last_check; -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ hmbird_scheduler_tick(); -+ -+ if (!hmbird_enabled()) -+ return; -+ -+ last_check = hmbird_watchdog_timestamp; -+ if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -+ -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "watchdog failed to check in for %u.%03us", -+ dur_ms / 1000, dur_ms % 1000); -+ } -+ scan_timeout(rq); -+} -+ -+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued) -+{ -+ update_curr_hmbird(rq); -+ -+ set_pcp_round(rq); -+ -+ if (slim_walt_ctrl) -+ hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE); -+ /* -+ * While disabling, always resched and refresh core-sched timestamp as -+ * we can't trust the slice management or ops.core_sched_before(). -+ */ -+ if (hmbird_ops_disabling()) -+ get_hmbird_ts(curr)->slice = 0; -+ -+ if (!get_hmbird_ts(curr)->slice) -+ resched_curr(rq); -+ -+ inform_hmbird_onoff_from_systrace(); -+} -+ -+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg) -+{ -+ hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ get_hmbird_ts(p)->disallow = false; -+ -+ hmbird_sched_init_task(p); -+ -+ set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ return 0; -+} -+ -+static void hmbird_ops_enable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void hmbird_ops_disable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void set_task_hmbird_weight(struct task_struct *p) -+{ -+ u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -+ -+ get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight); -+} -+ -+/** -+ * refresh_hmbird_weight - Refresh a task's hmbird weight -+ * @p: task to refresh hmbird weight for -+ * -+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to -+ * enable easy access from the BPF scheduler. To keep it synchronized with the -+ * current task priority, this function should be called when a new task is -+ * created, priority is changed for a task on hmbird, and a task is switched -+ * to hmbird from other classes. -+ */ -+static void refresh_hmbird_weight(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ set_task_hmbird_weight(p); -+} -+ -+int hmbird_pre_fork(struct task_struct *p) -+{ -+ int ret = 0; -+ -+ p->android_oem_data1[HMBIRD_TS_IDX] = -+ (u64)(kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL)); -+ if (!get_hmbird_ts(p)) { -+ ret = -1; -+ goto lock; -+ } -+ -+ get_hmbird_ts(p)->dsq = NULL; -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node); -+ get_hmbird_ts(p)->flags = 0; -+ get_hmbird_ts(p)->weight = 0; -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ get_hmbird_ts(p)->holding_cpu = -1; -+ get_hmbird_ts(p)->kf_mask = 0; -+ atomic64_set(&get_hmbird_ts(p)->ops_state, 0); -+ get_hmbird_ts(p)->runnable_at = INITIAL_JIFFIES; -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+ get_hmbird_ts(p)->task = p; -+ hmbird_set_sched_prop(p, 0); -+ -+ get_hmbird_ts(p)->critical_affinity_cpu = -1; -+ get_hmbird_ts(p)->sched_class = &hmbird_sched_class; -+ -+ /* -+ * BPF scheduler enable/disable paths want to be able to iterate and -+ * update all tasks which can become complex when racing forks. As -+ * enable/disable are very cold paths, let's use a percpu_rwsem to -+ * exclude forks. -+ */ -+lock: -+ percpu_down_read(&hmbird_fork_rwsem); -+ -+ return ret; -+} -+ -+int hmbird_fork(struct task_struct *p) -+{ -+ percpu_rwsem_assert_held(&hmbird_fork_rwsem); -+ -+ if (hmbird_enabled()) -+ return hmbird_ops_prepare_task(p, task_group(p)); -+ else -+ return 0; -+} -+ -+void hmbird_post_fork(struct task_struct *p) -+{ -+ if (hmbird_enabled()) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ /* -+ * Set the weight manually before calling ops.enable() so that -+ * the scheduler doesn't see a stale value if they inspect the -+ * task struct. We'll invoke ops.set_weight() afterwards, as it -+ * would be odd to receive a callback on the task before we -+ * tell the scheduler that it's been fully enabled. -+ */ -+ set_task_hmbird_weight(p); -+ hmbird_ops_enable_task(p); -+ refresh_hmbird_weight(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ -+ spin_lock_irq(&hmbird_tasks_lock); -+ list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_cancel_fork(struct task_struct *p) -+{ -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ if (hmbird_enabled()) -+ hmbird_ops_disable_task(p); -+ -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_free(struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+ list_del_init(&get_hmbird_ts(p)->tasks_node); -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+ -+ /* -+ * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED -> -+ * ENABLED transitions can't race us. Disable ops for @p. -+ */ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) || -+ test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ hmbird_ops_disable_task(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+} -+ -+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio) -+{ -+} -+ -+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type) -+{ -+ return (prop >> TOP_TASK_SHIFT) & (1 << type); -+} -+ -+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p) -+{ -+ uint32_t prop = get_top_task_prop(p); -+ -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE)) -+ return HMBIRD_TASK_PROP_PIPELINE; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) || -+ !task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) { -+ return HMBIRD_TASK_PROP_COMMON; -+ } -+ return HMBIRD_TASK_PROP_DEBUG_OR_LOG; -+} -+ -+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b) -+{ -+ int type_a = hmbird_get_task_type(a); -+ int type_b = hmbird_get_task_type(b); -+ -+ return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b]; -+} -+ -+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ enum cpu_type type; -+ int sp_dl; -+ struct task_struct *curr = NULL; -+ -+ switch (hmbird_preempt_policy) { -+ case HMBIRD_PREEMPT_POLICY_PRIO_BASED: -+ curr = rq->curr; -+ if (curr && hmbird_prio_higher(p, curr)) -+ goto preempt; -+ break; -+ default: -+ break; -+ } -+ if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) || -+ (is_critical_system_task(p) && !is_critical_system_task(rq->curr))) -+ goto preempt; -+ -+ sp_dl = find_idx_from_task(p); -+ if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled())) -+ return; -+ -+ if (rq->curr->prio > p->prio) -+ goto preempt; -+ -+ return; -+ -+preempt: -+ resched_curr(rq); -+} -+ -+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {} -+ -+int hmbird_check_setscheduler(struct task_struct *p, int policy) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ /* if disallow, reject transitioning into HMBIRD */ -+ if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) && -+ p->policy != policy && policy == SCHED_HMBIRD) -+ return -EACCES; -+ -+ return 0; -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool hmbird_can_stop_tick(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ if (hmbird_ops_disabling()) -+ return false; -+ -+ if (p->sched_class != &hmbird_sched_class) -+ return true; -+ -+ return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK; -+} -+#endif -+ -+int hmbird_tg_online(struct task_group *tg) -+{ -+ struct cgroup *cgrp; -+ -+ if (!tg) -+ return 0; -+ cgrp = tg->css.cgroup; -+ if (!cgrp || !(cgrp->kn)) -+ return 0; -+ update_cgroup_ids_table(cgrp->kn->id, -1); -+ return 0; -+} -+ -+/* -+ * Omitted operations: -+ * -+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -+ * task isn't tied to the CPU at that point. Preemption is implemented by -+ * resetting the victim task's slice to 0 and triggering reschedule on the -+ * target CPU. -+ * -+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -+ * -+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -+ * their current sched_class. Call them directly from sched core instead. -+ * -+ * - task_woken, switched_from: Unnecessary. -+ */ -+DEFINE_SCHED_CLASS(hmbird) = { -+ .enqueue_task = enqueue_task_hmbird, -+ .dequeue_task = dequeue_task_hmbird, -+ .yield_task = yield_task_hmbird, -+ .yield_to_task = yield_to_task_hmbird, -+ -+ .check_preempt_curr = check_preempt_curr_hmbird, -+ -+ .pick_next_task = pick_next_task_hmbird, -+ -+ .put_prev_task = put_prev_task_hmbird, -+ .set_next_task = set_next_task_hmbird, -+ -+#ifdef CONFIG_SMP -+ .balance = balance_hmbird, -+ .select_task_rq = select_task_rq_hmbird, -+ .set_cpus_allowed = set_cpus_allowed_hmbird, -+#endif -+ .task_tick = task_tick_hmbird, -+ -+ .switched_to = switched_to_hmbird, -+ .prio_changed = prio_changed_hmbird, -+ -+ .update_curr = update_curr_hmbird, -+ -+#ifdef CONFIG_UCLAMP_TASK -+ .uclamp_enabled = 0, -+#endif -+}; -+ -+/* -+ * Must with rq lock held. -+ */ -+bool task_is_hmbird(struct task_struct *p) -+{ -+ return p->sched_class == &hmbird_sched_class; -+} -+ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id) -+{ -+ memset(dsq, 0, sizeof(*dsq)); -+ -+ raw_spin_lock_init(&dsq->lock); -+ INIT_LIST_HEAD(&dsq->fifo); -+ dsq->id = dsq_id; -+} -+ -+static int hmbird_cgroup_init(void) -+{ -+ struct cgroup_subsys_state *css; -+ -+ css_for_each_descendant_pre(css, &root_task_group.css) { -+ struct task_group *tg = css_tg(css); -+ -+ cgrp_dsq_idx_init(css->cgroup, tg); -+ } -+ return 0; -+} -+ -+/* -+ * Used by sched_fork() and __setscheduler_prio() to pick the matching -+ * sched_class. dl/rt are already handled. -+ */ -+bool task_on_hmbird(struct task_struct *p) -+{ -+ return hmbird_enabled(); -+} -+ -+static void __setscheduler_prio(struct task_struct *p, int prio) -+{ -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) { -+ p->prio = prio; -+ return; -+ } else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (on_hmbird && rt_prio(prio)) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ -+ p->prio = prio; -+} -+ -+/* -+ * Heartbeat, avoid humbird keep running while APP already exit. -+ * Check whether APP send alive-signal periodly. -+ */ -+#define HEARTBEAT_TIMEOUT (msecs_to_jiffies(2500)) -+#define HEARTBEAT_CHECK_INTERVAL (msecs_to_jiffies(1000)) -+static struct timer_list hb_timer; -+static unsigned long next_hb; -+ -+void hb_timer_handler(struct timer_list *timer) -+{ -+ if (!heartbeat_enable) -+ goto refill; -+ -+ pr_err(": enter timer.\n"); -+ if (heartbeat) { -+ heartbeat = 0; -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ } -+ -+ /* can't detect heartbeat, disable ext. */ -+ if (time_after(jiffies, READ_ONCE(next_hb))) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT, -+ "can't detect heartbeat, disable ext\n"); -+ } -+refill: -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_start(void) -+{ -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_init(void) -+{ -+ timer_setup(&hb_timer, hb_timer_handler, 0); -+} -+ -+static void hb_timer_exit(void) -+{ -+ del_timer(&hb_timer); -+} -+ -+static bool check_and_disable_cpuhp(void) -+{ -+ struct rq *rq; -+ struct rq_flags rf; -+ int cpu; -+ -+ cpu_hotplug_disable(); -+ cpus_read_lock(); -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ rq_lock_irqsave(rq, &rf); -+ if (!rq->online) { -+ rq_unlock_irqrestore(rq, &rf); -+ goto err_offline; -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ cpus_read_unlock(); -+ return true; -+ -+err_offline: -+ cpus_read_unlock(); -+ cpu_hotplug_enable(); -+ return false; -+} -+ -+static void reenable_cpuhp(void) -+{ -+ cpu_hotplug_enable(); -+} -+ -+static void scheduler_switch_done(bool final_state) -+{ -+ /* set scx_enable again, switch may fail. */ -+ scx_enable = final_state; -+ /* reset heartbeat*/ -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ -+ if (final_state) -+ hb_timer_start(); -+ else -+ hb_timer_exit(); -+} -+ -+/** -+ * ss : curr switch state -+ * finish : success or fail -+ * enable : curr operation is diabling or enabling? -+ * fail_reason : string output when failed, empty string when success. -+ */ -+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason) -+{ -+ char *s1 = finish ? "finished" : "failed"; -+ char *s2 = enable ? "enabled" : "disabled"; -+ -+ hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss); -+ if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) { -+ sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type)); -+ hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n", -+ s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason); -+ } -+ curr_ss = ss; -+} -+ -+bool get_hmbird_ops_enabled(void) -+{ -+ return atomic_read(&__hmbird_ops_enabled); -+} -+ -+bool get_non_hmbird_task(void) -+{ -+ return atomic_read(&non_hmbird_task); -+} -+ -+static void hmbird_ops_disable_workfn(struct kthread_work *work) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int cpu; -+ -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, ""); -+ cancel_delayed_work_sync(&hmbird_watchdog_work); -+ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) { -+ case HMBIRD_OPS_DISABLED: -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled"); -+ scheduler_switch_done(false); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return; -+ case HMBIRD_OPS_PREPPING: -+ fallthrough; -+ case HMBIRD_OPS_DISABLING: -+ /* shouldn't happen but handle it like ENABLING if it does */ -+ WARN_ONCE(true, "hmbird: duplicate disabling instance?"); -+ fallthrough; -+ case HMBIRD_OPS_ENABLING: -+ case HMBIRD_OPS_ENABLED: -+ break; -+ } -+ -+ /* kick all CPUs to restore ticks */ -+ for_each_possible_cpu(cpu) -+ resched_cpu(cpu); -+ -+ /* avoid racing against fork and cgroup changes */ -+ cpus_read_lock(); -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, ""); -+ spin_lock_irq(&hmbird_tasks_lock); -+ atomic_set(&__hmbird_ops_enabled, false); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ bool alive = READ_ONCE(p->__state) != TASK_DEAD; -+ -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ get_hmbird_ts(p)->slice = min_t(u64, -+ get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL); -+ -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ if (alive) -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ -+ hmbird_ops_disable_task(p); -+ } -+ hmbird_task_iter_exit(&sti); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, ""); -+ -+ atomic_set(&non_hmbird_task, true); -+ /* no task is on hmbird, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt); -+ synchronize_rcu(); -+ -+ percpu_up_write(&hmbird_fork_rwsem); -+ cpus_read_unlock(); -+ -+ if (slim_walt_ctrl) -+ slim_walt_enable(false); -+ -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 1, 0, ""); -+ scheduler_switch_done(false); -+ reenable_cpuhp(); -+} -+ -+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn); -+ -+static void schedule_hmbird_ops_disable_work(void) -+{ -+ struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper); -+ -+ /* -+ * We may be called spuriously before the first bpf_hmbird_reg(). If -+ * hmbird_ops_helper isn't set up yet, there's nothing to do. -+ */ -+ if (helper) -+ kthread_queue_work(helper, &hmbird_ops_disable_work); -+} -+ -+static void hmbird_ops_disable(void) -+{ -+ schedule_hmbird_ops_disable_work(); -+} -+ -+static void hmbird_err_exit_workfn(struct work_struct *work) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ hmbird_ctrl(false); -+ -+ for_each_present_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy) -+ continue; -+ if (cpu != policy->cpu) -+ goto put; -+ down_write(&policy->rwsem); -+ WARN_ON(store_scaling_governor(policy, -+ saved_gov[cpu], strlen(saved_gov[cpu])) <= 0); -+ up_write(&policy->rwsem); -+ hmbird_info_trace(":restore origin gov : %s\n", saved_gov[cpu]); -+put: -+ cpufreq_cpu_put(policy); -+ } -+ memset((char *)saved_gov, 0, sizeof(saved_gov)); -+} -+ -+void hmbird_ops_exit(void) -+{ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); -+} -+ -+static struct kthread_worker *hmbird_create_rt_helper(const char *name) -+{ -+ struct kthread_worker *helper; -+ -+ helper = kthread_create_worker(0, name); -+ if (helper) -+ sched_set_fifo(helper->task); -+ return helper; -+} -+ -+static inline void set_audio_thread_sched_prop(struct task_struct *p) -+{ -+ struct cgroup_subsys_state *css; -+ -+ if (likely(p->prio >= MAX_RT_PRIO)) -+ return; -+ -+ rcu_read_lock(); -+ css = task_css(p, cpuset_cgrp_id); -+ if (!css) { -+ rcu_read_unlock(); -+ return; -+ } -+ rcu_read_unlock(); -+ -+ if (!strcmp(css->cgroup->kn->name, "audio-app")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+} -+ -+static int hmbird_ops_enable(void *unused) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int ret; -+ int tcnt = 0; -+ unsigned long long start = 0; -+ -+ if (!check_and_disable_cpuhp()) { -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline"); -+ return -EBUSY; -+ } -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, ""); -+ mutex_lock(&hmbird_ops_enable_mutex); -+ -+ if (!hmbird_ops_helper) { -+ WRITE_ONCE(hmbird_ops_helper, -+ hmbird_create_rt_helper("hmbird_ops_helper")); -+ if (!hmbird_ops_helper) { -+ ret = -ENOMEM; -+ goto err_unlock; -+ } -+ } -+ -+ if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) { -+ ret = -EBUSY; -+ goto err_unlock; -+ } -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) != -+ HMBIRD_OPS_DISABLED); -+ -+ hmbird_warned_zero_slice = false; -+ -+ atomic64_set(&hmbird_nr_rejected, 0); -+ -+ /* -+ * Keep CPUs stable during enable so that the BPF scheduler can track -+ * online CPUs by watching ->on/offline_cpu() after ->init(). -+ */ -+ cpus_read_lock(); -+ -+ hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work, -+ hmbird_watchdog_timeout / 2); -+ -+ /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ */ -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ reset_idle_masks(); -+ -+ /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. -+ */ -+ ret = hmbird_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock; -+ -+ /* -+ * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem -+ * preventing new tasks from being added. No need to exclude tasks -+ * leaving as hmbird_free() can handle both prepped and enabled -+ * tasks. Prep all tasks first and then enable them with preemption -+ * disabled. -+ */ -+ spin_lock_irq(&hmbird_tasks_lock); -+ -+ atomic_set(&non_hmbird_task, false); -+ atomic_set(&__hmbird_ops_enabled, true); -+ -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered(&sti))) -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ hmbird_task_iter_exit(&sti); -+ -+ /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. -+ */ -+ preempt_disable(); -+ -+ /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ */ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) { -+ atomic_set(&non_hmbird_task, true); -+ atomic_set(&__hmbird_ops_enabled, false); -+ preempt_enable(); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ ret = -EBUSY; -+ goto err_disable_unlock; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED -+ * transitions here are synchronized against hmbird_free() through -+ * hmbird_tasks_lock. -+ */ -+ start = sched_clock(); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, ""); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ tcnt++; -+ if (READ_ONCE(p->__state) != TASK_DEAD) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ -+ set_audio_thread_sched_prop(p); -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ hmbird_ops_enable_task(p); -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ } else { -+ hmbird_ops_disable_task(p); -+ } -+ } -+ hmbird_task_iter_exit(&sti); -+ -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, ""); -+ preempt_enable(); -+ percpu_up_write(&hmbird_fork_rwsem); -+ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) { -+ ret = -EBUSY; -+ goto err_disable; -+ } -+ -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_ENABLED, 1, 1, ""); -+ scheduler_switch_done(true); -+ -+ return 0; -+ -+err_unlock: -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock"); -+ scheduler_switch_done(false); -+ return ret; -+ -+err_disable_unlock: -+ percpu_up_write(&hmbird_fork_rwsem); -+err_disable: -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ /* must be fully disabled before returning */ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable"); -+ scheduler_switch_done(false); -+ return ret; -+} -+ -+#ifdef CONFIG_SCHED_DEBUG -+static const char *hmbird_ops_enable_state_str[] = { -+ [HMBIRD_OPS_PREPPING] = "prepping", -+ [HMBIRD_OPS_ENABLING] = "enabling", -+ [HMBIRD_OPS_ENABLED] = "enabled", -+ [HMBIRD_OPS_DISABLING] = "disabling", -+ [HMBIRD_OPS_DISABLED] = "disabled", -+}; -+ -+static int hmbird_debug_show(struct seq_file *m, void *v) -+{ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled()); -+ seq_printf(m, "%-30s: %s\n", "enable_state", -+ hmbird_ops_enable_state_str[hmbird_ops_enable_state()]); -+ seq_printf(m, "%-30s: %llu\n", "nr_rejected", -+ atomic64_read(&hmbird_nr_rejected)); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return 0; -+} -+ -+static int hmbird_debug_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_debug_show, NULL); -+} -+ -+const struct file_operations sched_hmbird_fops = { -+ .open = hmbird_debug_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif -+ -+ -+static int bpf_hmbird_reg(void *kdata) -+{ -+ return hmbird_ops_enable(kdata); -+} -+ -+static int bpf_hmbird_unreg(void *kdata) -+{ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ return 0; -+} -+ -+void set_hmbird_module_loaded(int is_loaded) -+{ -+ atomic_set(&hmbird_module_loaded, is_loaded); -+} -+ -+/* -+ * MUST load hmbird module before enable hmbird scheduler -+ * load track & hmbird gover implemented in hmbird module -+ */ -+int hmbird_ctrl(bool enable) -+{ -+ if (!atomic_read(&hmbird_module_loaded)) { -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "ext module unloaded\n"); -+ return -EINVAL; -+ } -+ -+ if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED -+ || hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING -+ || hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already enabled(ing)\n"); -+ hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING || -+ hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already disabled(ing)\n"); -+ hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (enable) -+ return bpf_hmbird_reg(NULL); -+ else -+ return bpf_hmbird_unreg(NULL); -+} -+ -+void set_cpu_isomask(int cpu, cpumask_var_t *mask) -+{ -+ cpumask_clear_cpu(cpu, iso_masks.ex_free); -+ cpumask_clear_cpu(cpu, iso_masks.exclusive); -+ cpumask_clear_cpu(cpu, iso_masks.partial); -+ cpumask_clear_cpu(cpu, iso_masks.big); -+ cpumask_clear_cpu(cpu, iso_masks.little); -+ cpumask_set_cpu(cpu, *mask); -+} -+ -+void set_cpu_cluster(u64 cpu_cluster) -+{ -+ int cpu; -+ -+ for_each_present_cpu(cpu) { -+ u64 pos = 1 << cpu; -+ -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.ex_free)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.exclusive)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.partial)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.big)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) -+ set_cpu_isomask(cpu, &(iso_masks.little)); -+ } -+} -+ -+static void get_hmbird_snapshot(struct panic_snapshot_t *p) -+{ -+ struct hmbird_dispatch_q *dsq; -+ struct rq *rq; -+ int cpu, i; -+ -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ p->rq_nr[cpu] = rq->nr_running; -+ p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running; -+ } -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ struct hmbird_entity *entity; -+ -+ dsq = &gdsqs[i]; -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo)) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ p->runnable_at[i] = entity->runnable_at; -+ raw_spin_unlock(&dsq->lock); -+ -+ } -+ -+ p->snap_misc.hmbird_enabled = hmbird_enabled(); -+ p->snap_misc.curr_ss = curr_ss; -+ p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state(); -+ p->snap_misc.parctrl_high_ratio = parctrl_high_ratio; -+ p->snap_misc.parctrl_low_ratio = parctrl_low_ratio; -+ p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l; -+ p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l; -+ p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio; -+ p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio; -+ p->snap_misc.misfit_ds = misfit_ds; -+ p->snap_misc.partial_enable = partial_enable; -+ p->snap_misc.iso_free_rescue = iso_free_rescue; -+ p->snap_misc.isolate_ctrl = isolate_ctrl; -+ p->snap_misc.snap_jiffies = jiffies; -+ p->snap_misc.snap_time = local_clock(); -+} -+ -+// MTK minidump begin -+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3) -+{ -+ strscpy(m->desc_str, str, DESC_STR_LEN); -+ m->len = d1 * d2 * d3; -+ m->parse[0] = d1; -+ m->parse[1] = d2; -+ m->parse[2] = d3; -+} -+ -+static void init_desc_metas(struct md_info_t *m) -+{ -+ init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :", -+ 1, MAX_SWITCHS, SWITCH_ITEMS); -+ -+ init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1); -+ -+ init_desc_meta(&m->kern_dump.excep_rec_meta, -+ "excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS); -+ -+ init_desc_meta(&m->kern_dump.excep_idx_meta, -+ "excep idx :", 1, 1, MAX_EXCEP_ID); -+ -+ init_desc_meta(&m->kern_dump.snap.runnable_at_meta, -+ "each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS); -+ -+ init_desc_meta(&m->kern_dump.snap.rq_nr_meta, -+ "rq runnable task nr:", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta, -+ "scxrq runnable task nr :", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.snap_misc_meta, -+ "misc snap params :", 1, 1, SNAP_ITEMS); -+} -+ -+static void init_md_meta(struct md_info_t *m) -+{ -+ m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64); -+ m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64); -+ m->meta.unit_size = sizeof(u64); -+ m->meta.switches = MAX_SWITCHS; -+ m->meta.exceps = MAX_EXCEPS; -+ m->meta.global_dsqs = MAX_GLOBAL_DSQS; -+ m->meta.parse_dimens = PARSE_DIMENS; -+ m->meta.nr_cpus = num_possible_cpus(); -+ m->meta.real_cpus = nr_cpu_ids; -+ m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64); -+ m->meta.nr_meta_desc = 8; -+ m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64); -+ -+ init_desc_metas(m); -+} -+ -+#define MINIDUMP_DFL_SIZE (4 * 1024) -+ -+struct notifier_block hmbird_panic_blk; -+static int hmbird_panic_handler(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ if (!md_info) -+ return NOTIFY_DONE; -+ -+ get_hmbird_snapshot(&md_info->kern_dump.snap); -+ -+ return NOTIFY_DONE; -+} -+ -+static void panic_blk_init(void) -+{ -+ int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE); -+ -+ md_info = kzalloc(dump_size, GFP_KERNEL); -+ if (!md_info) -+ return; -+ init_md_meta(md_info); -+ -+ hmbird_panic_blk.notifier_call = hmbird_panic_handler; -+ /* make sure to execute before minidump. */ -+ hmbird_panic_blk.priority = INT_MAX; -+ atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk); -+ -+ hmbird_debug("register minidump.\n"); -+} -+ -+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size) -+{ -+ *vaddr = (unsigned long)md_info; -+ *size = sizeof(struct md_info_t); -+} -+// MTK minidump end -+ -+static inline void init_sched_prop_to_preempt_prio(void) -+{ -+ for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) { -+ switch (i) { -+ case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 5; -+ break; -+ -+ case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 4; -+ break; -+ -+ case HMBIRD_TASK_PROP_PIPELINE: -+ case HMBIRD_TASK_PROP_ISOLATE: -+ sched_prop_to_preempt_prio[i] = 3; -+ break; -+ -+ case HMBIRD_TASK_PROP_COMMON: -+ sched_prop_to_preempt_prio[i] = 1; -+ break; -+ -+ case HMBIRD_TASK_PROP_DEBUG_OR_LOG: -+ sched_prop_to_preempt_prio[i] = 0; -+ break; -+ -+ default: -+ sched_prop_to_preempt_prio[i] = 2; -+ break; -+ } -+ } -+} -+ -+void __init init_sched_hmbird_class(void) -+{ -+ int cpu; -+ u32 v; -+ struct hmbird_entity *init_hmbird; -+ -+ /* -+ * The following is to prevent the compiler from optimizing out the enum -+ * definitions so that BPF scheduler implementations can use them -+ * through the generated vmlinux.h. -+ */ -+ WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP | -+ HMBIRD_KICK_PREEMPT); -+ -+ init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL); -+ init_dsq_at_boot(); -+ init_isolate_cpus(); -+ hb_timer_init(); -+ init_sched_prop_to_preempt_prio(); -+#ifdef CONFIG_SMP -+ WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); -+#endif -+ -+ /* -+ * we can't static init init_task's hmbird struct, init here. -+ * init_task->hmbird would not use during boot. -+ */ -+ init_hmbird = kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL); -+ init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird; -+ if (init_hmbird) { -+ INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo); -+ INIT_LIST_HEAD(&init_hmbird->watchdog_node); -+ init_hmbird->sticky_cpu = -1; -+ init_hmbird->holding_cpu = -1; -+ atomic64_set(&init_hmbird->ops_state, 0); -+ init_hmbird->runnable_at = jiffies; -+ init_hmbird->slice = HMBIRD_SLICE_DFL; -+ init_hmbird->task = &init_task; -+ hmbird_set_sched_prop(&init_task, 0); -+ } else { -+ hmbird_err(INIT_TASK_FAIL, ":alloc init_task.scx failed!!!\n"); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ /* -+ * exec during boot phase, no need to care about alloc failed. -+ * lifecycle same to rq, no need to free. -+ */ -+ rq->android_oem_data1[HMBIRD_RQ_IDX] = -+ (u64)kmalloc(sizeof(struct hmbird_rq), GFP_KERNEL); -+ if (get_hmbird_rq(rq)) { -+ get_hmbird_rq(rq)->rq = rq; -+ get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window; -+ } else { -+ hmbird_err(ALLOC_RQSCX_FAIL, ":alloc rq->scx failed!!!\n"); -+ } -+ -+ rq->android_oem_data1[HMBIRD_OPS_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL); -+ if (!get_hmbird_ops(rq)) -+ pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n"); -+ init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL); -+ INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list); -+ -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL)); -+ -+ hmbird_ops_init(get_hmbird_ops(rq)); -+ } -+ -+ INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn); -+ INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn); -+ hmbird_misc_init(); -+ -+ panic_blk_init(); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_misc.c b/kernel/sched/hmbird/hmbird_misc.c -new file mode 100755 -index 000000000000..52582c22052c ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_misc.c -@@ -0,0 +1,124 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * @Description: -+ * @Version: -+ * @Author: wangrui8 -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include "hmbird_sched.h" -+#include -+#include -+#include "slim.h" -+ -+noinline int tracing_mark_write(const char *buf) -+{ -+ trace_printk(buf); -+ return 0; -+} -+ -+struct yield_opt_params yield_opt_params = { -+ .enable = 0, -+ .frame_per_sec = 120, -+ .frame_time_ns = NSEC_PER_SEC / 120, -+ .yield_headroom = 10, -+}; -+ -+DEFINE_PER_CPU(struct sched_yield_state, ystate); -+ -+static inline void hmbird_yield_state_update(struct sched_yield_state *ys) -+{ -+ if (!raw_spin_is_locked(&ys->lock)) -+ return; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ -+ if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1 -+ || ys->yield_cnt_after_sleep > yield_headroom) { -+ ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP); -+ } else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) { -+ ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP); -+ } -+ ys->yield_cnt = 0; -+ ys->sleep_times = 0; -+ ys->yield_cnt_after_sleep = 0; -+} -+ -+void hmbird_skip_yield(long *skip) -+{ -+ if (!get_hmbird_ops_enabled() || !yield_opt_params.enable) -+ return; -+ unsigned long flags, sleep_now = 0; -+ struct sched_yield_state *ys; -+ int cpu = raw_smp_processor_id(), cont_yield, new_frame; -+ int frame_time_ns = yield_opt_params.frame_time_ns; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ u64 wc; -+ -+ if (!(*skip)) { -+ wc = sched_clock(); -+ ys = &per_cpu(ystate, cpu); -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ -+ cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP; -+ new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1); -+ -+ if (!cont_yield && new_frame) { -+ hmbird_yield_state_update(ys); -+ ys->last_update_time = wc; -+ ys->sleep_end = ys->last_yield_time + frame_time_ns -+ - yield_headroom * YIELD_DURATION; -+ } -+ -+ if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) { -+ *skip = true; -+ -+ sleep_now = ys->sleep_times ? -+ max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep; -+ if (wc + sleep_now > ys->sleep_end) { -+ u64 delta = ys->sleep_end - wc; -+ -+ if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION) -+ sleep_now = delta; -+ else -+ sleep_now = 0; -+ } -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ if (sleep_now) { -+ sleep_now = div64_u64(sleep_now, 1000); -+ usleep_range_state(sleep_now, sleep_now, TASK_IDLE); -+ } -+ ys->sleep_times++; -+ ys->last_yield_time = sched_clock(); -+ return; -+ } -+ if (ys->sleep_times) -+ ys->yield_cnt_after_sleep++; -+ else -+ (ys->yield_cnt)++; -+ ys->last_yield_time = wc; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+} -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops) -+{ -+ hmbird_ops->scx_enable = get_hmbird_ops_enabled; -+ hmbird_ops->check_non_task = get_non_hmbird_task; -+ hmbird_ops->hmbird_get_md_info = hmbird_get_md_info; -+} -+ -+void hmbird_misc_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_init(&ys->lock); -+ } -+ -+ set_hmbird_module_loaded(1); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_sched.h b/kernel/sched/hmbird/hmbird_sched.h -new file mode 100755 -index 000000000000..6b5ef43cb827 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef __HMBIRD_SCHED__ -+#define __HMBIRD_SCHED__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include "../../time/tick-sched.h" -+#include "../../sched/sched.h" -+#include -+#include -+#include -+ -+#define REGISTER_TRACE_VH(vender_hook, handler) \ -+{ \ -+ ret = register_trace_##vender_hook(handler, NULL); \ -+ if (ret) { \ -+ pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \ -+ } \ -+} -+#define REGISTER_TRACE(vendor_hook, handler, data, err) \ -+do { \ -+ ret = register_trace_##vendor_hook(handler, data); \ -+ if (ret) { \ -+ pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret); \ -+ goto err; \ -+ } \ -+} while (0) -+ -+#define UNREGISTER_TRACE(vendor_hook, handler, data) \ -+ unregister_trace_##vendor_hook(handler, data) \ -+ -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+ -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int sched_ravg_window_frame_per_sec; -+extern int slim_gov_debug; -+extern int cpu7_tl; -+extern int scx_gov_ctrl; -+extern spinlock_t new_sched_ravg_window_lock; -+extern int cluster_separate; -+ -+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER BIT(31) -+#define MAX_YIELD_SLEEP (2000000ULL) -+#define MIN_YIELD_SLEEP (200000ULL) -+#define YIELD_DURATION (5000ULL) -+#define DEFAULT_YIELD_SLEEP_TH (10) -+ -+struct sched_yield_state { -+ raw_spinlock_t lock; -+ u64 last_yield_time; -+ u64 last_update_time; -+ u64 sleep_end; -+ unsigned long yield_cnt; -+ unsigned long yield_cnt_after_sleep; -+ unsigned long sleep; -+ int sleep_times; -+}; -+ -+DECLARE_PER_CPU(struct sched_yield_state, ystate); -+ -+void hmbird_window_rollover_run_once(struct rq *rq); -+void hmbird_yield_state_update_per_frame(void); -+void hmbird_misc_init(void); -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops); -+#endif /*__HMBIRD_SCHED__*/ -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.c b/kernel/sched/hmbird/hmbird_sched_proc.c -new file mode 100755 -index 000000000000..234768fc767a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.c -@@ -0,0 +1,527 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "hmbird_sched_proc.h" -+#include -+ -+#include "hmbird_util_track.h" -+#include "slim.h" -+ -+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched" -+#define SLIM_FREQ_GOV_DIR "slim_freq_gov" -+#define LOAD_TRACK_DIR "slim_walt" -+#define HMBIRD_PROC_PERMISSION 0666 -+ -+int scx_enable; -+int partial_enable; -+int cpuctrl_high_ratio = 55; -+int cpuctrl_low_ratio = 40; -+int slim_stats; -+int hmbirdcore_debug; -+int slim_for_app; -+int misfit_ds = 90; -+unsigned int highres_tick_ctrl; -+unsigned int highres_tick_ctrl_dbg; -+int cpu7_tl = 70; -+int slim_walt_ctrl; -+int slim_walt_dump; -+int slim_walt_policy; -+int slim_gov_debug; -+int scx_gov_ctrl = 1; -+int sched_ravg_window_frame_per_sec = 125; -+int parctrl_high_ratio = 55; -+int parctrl_low_ratio = 40; -+int parctrl_high_ratio_l = 65; -+int parctrl_low_ratio_l = 50; -+int isoctrl_high_ratio = 75; -+int isoctrl_low_ratio = 60; -+int isolate_ctrl; -+int iso_free_rescue; -+int heartbeat; -+int heartbeat_enable = 1; -+int watchdog_enable; -+int save_gov; -+u64 cpu_cluster_masks; -+int hmbird_preempt_policy; -+int cluster_separate; -+ -+char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= 32) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtoint(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoint\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+static int set_proc_buf_val_u64(struct file *file, const char __user *buf, -+ size_t count, u64 *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= sizeof(kbuf)) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtou64(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoul\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/* common ops begin */ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ return count; -+} -+ -+static int hmbird_common_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%d\n", *(int *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_show, pde_data(inode)); -+} -+ -+static int hmbird_common_ul_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%lu\n", *(unsigned long *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_ul_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_ul_show, pde_data(inode)); -+} -+ -+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write); -+/* common ops end */ -+ -+/* scx_enable ops begin */ -+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC); -+ if (hmbird_ctrl(*pval)) -+ return -EFAULT; -+ -+ return count; -+} -+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write); -+/* scx_enable ops end */ -+ -+/* hmbird_stats ops begin */ -+#define MAX_STATS_BUF (4096) -+static int hmbird_stats_proc_show(struct seq_file *m, void *v) -+{ -+ char *buf; -+ -+ buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ stats_print(buf, MAX_STATS_BUF); -+ -+ seq_printf(m, "%s\n", buf); -+ -+ kfree(buf); -+ return 0; -+} -+ -+static int hmbird_stats_proc_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_stats_proc_show, inode); -+} -+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL); -+/* hmbird_stats ops end */ -+ -+/* sched_ravg_window_frame_per_sec ops begin */ -+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file, -+ const char __user *buf, size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ sched_ravg_window_change(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open, -+ sched_ravg_window_frame_per_sec_proc_write); -+/* sched_ravg_window_frame_per_sec ops end */ -+ -+static ssize_t save_gov_str(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ for_each_possible_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy || (cpu != policy->cpu)) -+ continue; -+ WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0); -+ hmbird_info_systrace(":save origin gov : %s\n", saved_gov[cpu]); -+ } -+ return count; -+} -+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str); -+ -+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ u64 *pval = (u64 *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val_u64(file, buf, count, pval)) -+ return -EFAULT; -+ -+ if (scx_enable == 0) -+ set_cpu_cluster(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write); -+ -+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ int tmp_val; -+ -+ if (set_proc_buf_val(file, buf, count, &tmp_val)) -+ return -EFAULT; -+ -+ slim_walt_enable(tmp_val); -+ *pval = tmp_val; -+ -+ return count; -+} -+ -+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write); -+ -+/* yield_opt ops begin */ -+static int yield_opt_show(struct seq_file *m, void *v) -+{ -+ struct yield_opt_params *data = m->private; -+ -+ seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n", -+ data->enable, data->frame_per_sec, data->yield_headroom); -+ return 0; -+} -+ -+static int yield_opt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, yield_opt_show, pde_data(inode)); -+} -+ -+static ssize_t yield_opt_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu; -+ unsigned long flags; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if (copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp -+ != 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) || -+ (yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp; -+ yield_opt_params.frame_per_sec = frame_per_sec_tmp; -+ yield_opt_params.yield_headroom = yield_headroom_tmp; -+ yield_opt_params.enable = enable_tmp; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ ys->last_yield_time = 0; -+ ys->last_update_time = 0; -+ ys->sleep_end = 0; -+ ys->yield_cnt = 0; -+ ys->yield_cnt_after_sleep = 0; -+ ys->sleep = 0; -+ ys->sleep_times = 0; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write); -+ -+ -+static int __init hmbird_proc_init(void) -+{ -+ struct proc_dir_entry *hmbird_dir; -+ struct proc_dir_entry *load_track_dir; -+ struct proc_dir_entry *freq_gov_dir; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ -+ /* mkdir /proc/hmbird_sched */ -+ hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL); -+ if (!hmbird_dir) { -+ pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &scx_enable_proc_ops, -+ &scx_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &partial_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_stats); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbirdcore_debug); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_for_app); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &misfit_ds); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl_dbg); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpu7_tl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &cpu_cluster_masks_proc_ops, -+ &cpu_cluster_masks); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &save_gov_proc_ops, -+ &save_gov); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &watchdog_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isolate_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &iso_free_rescue); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_stats_proc_ops); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &yield_opt_proc_ops, -+ &yield_opt_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbird_preempt_policy); -+ /* /proc/hmbird_sched--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_walt */ -+ load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir); -+ if (!load_track_dir) { -+ pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_walt--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &slim_walt_ctrl_proc_ops, -+ &slim_walt_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_dump); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_policy); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &sched_ravg_window_frame_per_sec_proc_ops, -+ &sched_ravg_window_frame_per_sec); -+ /* /proc/hmbird_sched/slim_walt--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_freq_gov */ -+ freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir); -+ if (!freq_gov_dir) { -+ pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_freq_gov--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &slim_gov_debug); -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &scx_gov_ctrl); -+ /* /proc/hmbird_sched/slim_freq_gov--end */ -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cluster_separate); -+ -+ return 0; -+} -+ -+device_initcall(hmbird_proc_init); -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.h b/kernel/sched/hmbird/hmbird_sched_proc.h -new file mode 100755 -index 000000000000..e22bc75f1dd7 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SCHED_PROC_H__ -+#define __HMBIRD_SCHED_PROC_H__ -+ -+#include -+#include -+ -+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \ -+ do { \ -+ if (!proc_create(name, mode, parent, proc_ops)) { \ -+ pr_err("Error creating proc entry %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \ -+ do { \ -+ if (!proc_create_data(name, mode, parent, proc_ops, data)) { \ -+ pr_err("Error creating proc entry with data %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_PROC_OPS(name, open_func, write_func) \ -+ static const struct proc_ops name##_proc_ops = { \ -+ .proc_open = open_func, \ -+ .proc_write = write_func, \ -+ .proc_read = seq_read, \ -+ .proc_lseek = seq_lseek, \ -+ .proc_release = single_release, \ -+ } -+ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos); -+static int hmbird_common_show(struct seq_file *m, void *v); -+static int hmbird_common_open(struct inode *inode, struct file *file); -+ -+#endif -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.c b/kernel/sched/hmbird/hmbird_shadow_tick.c -new file mode 100755 -index 000000000000..21de551c5132 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.c -@@ -0,0 +1,159 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include -+#include "../../time/tick-sched.h" -+#include -+ -+#include "hmbird_shadow_tick.h" -+ -+#define HIGHRES_WATCH_CPU 0 -+ -+#include -+static bool shadow_tick_enable(void) {return highres_tick_ctrl; } -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; } -+#endif -+static bool shadow_tick_timer_init_flag; -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define shadow_tick_printk(fmt, args...) \ -+do { \ -+ int cpu = smp_processor_id(); \ -+ if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU) \ -+ trace_printk("hmbird shadow tick :"fmt, args); \ -+} while (0) -+#else -+#define shadow_tick_printk(fmt, args...) -+#endif -+ -+DEFINE_PER_CPU(struct hrtimer, stt); -+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu))) -+#define STOP_IDLE_TRIGGER (1) -+#define PERIODIC_TICK_TRIGGER (2) -+#define TICK_INTVAL (1000000ULL) -+/* -+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms, -+ * so we can't restart hrtimer in scheduler tick. -+ */ -+static DEFINE_PER_CPU(u8, trigger_event); -+ -+/* -+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick. -+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler. -+ */ -+static DEFINE_PER_CPU(u8, tick_phase); -+ -+static inline void highres_timer_ctrl(bool enable, int cpu) -+{ -+ if (enable && hmbird_enabled()) { -+ if (!hrtimer_active(shadow_tick_timer(cpu))) -+ hrtimer_start(shadow_tick_timer(cpu), -+ ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED); -+ } else { -+ if (!enable) -+ hrtimer_cancel(shadow_tick_timer(cpu)); -+ } -+} -+ -+static inline void high_res_clear_phase(int cpu) -+{ -+ per_cpu(tick_phase, cpu) = 0; -+} -+ -+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer) -+{ -+ per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3; -+ if (per_cpu(tick_phase, cpu)) { -+ hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) { -+ per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ struct rq_flags rf; -+ -+ rq_lock(rq, &rf); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ rq_unlock(rq, &rf); -+ -+ return highres_next_phase(cpu, timer); -+} -+ -+void shadow_tick_timer_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance; -+ } -+} -+ -+void start_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable()) { -+ if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER) -+ highres_timer_ctrl(false, cpu); -+ per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static void stop_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ per_cpu(trigger_event, cpu) = 0; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(false, cpu); -+} -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ stop_shadow_tick_timer(); -+} -+ -+void scheduler_tick_handler(void *unused, struct rq *rq) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ start_shadow_tick_timer(); -+} -+ -+static int __init hmbird_shadow_tick_init(void) -+{ -+ int ret = 0; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ shadow_tick_timer_init(); -+ shadow_tick_timer_init_flag = true; -+ return ret; -+} -+ -+device_initcall(hmbird_shadow_tick_init); -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.h b/kernel/sched/hmbird/hmbird_shadow_tick.h -new file mode 100755 -index 000000000000..0c26fd984f0a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.h -@@ -0,0 +1,11 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SHADOW_TICK_H__ -+#define __HMBIRD_SHADOW_TICK_H__ -+ -+#include -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+void scheduler_tick_handler(void *unused, struct rq *rq); -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state); -+#endif -diff --git a/kernel/sched/hmbird/hmbird_trace.h b/kernel/sched/hmbird/hmbird_trace.h -new file mode 100755 -index 000000000000..8468b406f347 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_trace.h -@@ -0,0 +1,92 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hmbird -+ -+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HMBIRD_H -+ -+#include -+#include -+#include -+ -+#define MAX_FATAL_INFO (64) -+ -+TRACE_EVENT(hmbird_fatal_info, -+ -+ TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info), -+ -+ TP_ARGS(type, pe, lnr, bnr, info), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, type) -+ __field(int, pe) -+ __field(int, lnr) -+ __field(int, bnr) -+ __array(char, info, MAX_FATAL_INFO)), -+ -+ TP_fast_assign( -+ __entry->type = type; -+ __entry->pe = pe; -+ __entry->lnr = lnr; -+ __entry->bnr = bnr; -+ memcpy(__entry->info, info, MAX_FATAL_INFO);), -+ -+ TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s", -+ __entry->type, __entry->pe, __entry->lnr, __entry->bnr, -+ __entry->info) -+); -+ -+TRACE_EVENT(hmbird_update_history, -+ -+ TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq, -+ struct task_struct *p, u32 runtime, int samples, int event), -+ -+ TP_ARGS(hmbird, rq, p, runtime, samples, event), -+ -+ TP_STRUCT__entry( -+ __array(char, comm, TASK_COMM_LEN) -+ __field(pid_t, pid) -+ __field(unsigned int, runtime) -+ __field(int, samples) -+ __field(int, event) -+ __field(unsigned int, demand) -+ __array(u32, hist, RAVG_HIST_SIZE) -+ __field(u16, task_util) -+ __field(int, cpu)), -+ -+ TP_fast_assign( -+ memcpy(__entry->comm, p->comm, TASK_COMM_LEN); -+ __entry->pid = p->pid; -+ __entry->runtime = runtime; -+ __entry->samples = samples; -+ __entry->event = event; -+ __entry->demand = hmbird->sts.demand; -+ memcpy(__entry->hist, hmbird->sts.sum_history, -+ RAVG_HIST_SIZE * sizeof(u32)); -+ __entry->task_util = hmbird->sts.demand_scaled; -+ __entry->cpu = rq->cpu;), -+ -+ TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d", -+ __entry->comm, __entry->pid, -+ __entry->runtime, __entry->samples, -+ __entry->event, -+ __entry->demand, -+ __entry->hist[0], __entry->hist[1], -+ __entry->hist[2], __entry->hist[3], -+ __entry->hist[4], -+ __entry->task_util, -+ __entry->cpu) -+); -+ -+#endif /*_TRACE_HMBIRD_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird -+ -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE hmbird_trace -+/* This part must be outside protection */ -+#include -diff --git a/kernel/sched/hmbird/hmbird_util_track.c b/kernel/sched/hmbird/hmbird_util_track.c -new file mode 100755 -index 000000000000..d520a8403401 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.c -@@ -0,0 +1,626 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+ -+#define CREATE_TRACE_POINTS -+#include "hmbird_trace.h" -+#undef CREATE_TRACE_POINTS -+ -+#define HMBIRD_DEBUG_PANIC (1 << 3) -+static bool init_irq_work_inited; -+ -+extern noinline int tracing_mark_write(const char *buf); -+ -+int hmbird_sched_ravg_window = 8000000; -+int new_hmbird_sched_ravg_window = 8000000; -+DEFINE_SPINLOCK(new_sched_ravg_window_lock); -+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+ -+static struct irq_work hmbird_slim_walt_irq_work; -+ -+/*Sysctl related interface*/ -+#define WINDOW_STATS_RECENT 0 -+#define WINDOW_STATS_MAX 1 -+#define WINDOW_STATS_MAX_RECENT_AVG 2 -+#define WINDOW_STATS_AVG 3 -+#define WINDOW_STATS_INVALID_POLICY 4 -+ -+ -+#define SCX_SCHED_CAPACITY_SHIFT 10 -+#define SCHED_ACCOUNT_WAIT_TIME 0 -+ -+atomic64_t hmbird_irq_work_lastq_ws; -+static u64 tick_sched_clock; -+ -+static inline u64 scale_exec_time(u64 delta, struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT; -+} -+ -+static inline u64 scale_time_to_util(u64 d) -+{ -+ do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT); -+ return d; -+} -+ -+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ delta = scale_exec_time(delta, rq); -+ sts->sum += delta; -+ if (unlikely(sts->sum > hmbird_sched_ravg_window)) -+ sts->sum = hmbird_sched_ravg_window; -+ -+ return delta; -+} -+ -+ -+static int -+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) -+{ -+ /* -+ * No need to bother updating task demand for the idle task. -+ */ -+ if (is_idle_task(p)) -+ return 0; -+ -+ /* -+ * When a task is waking up it is completing a segment of non-busy -+ * time. Likewise, if wait time is not treated as busy time, then -+ * when a task begins to run or is migrated, it is not running and -+ * is completing a segment of non-busy time. -+ */ -+ if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && -+ (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) -+ return 0; -+ -+ /* -+ * The idle exit time is not accounted for the first task _picked_ up to -+ * run on the idle CPU. -+ */ -+ if (event == PICK_NEXT_TASK && rq->curr == rq->idle) -+ return 0; -+ -+ /* -+ * TASK_UPDATE can be called on sleeping task, when its moved between -+ * related groups -+ */ -+ if (event == TASK_UPDATE) { -+ if (rq->curr == p) -+ return 1; -+ -+ return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; -+ } -+ -+ return 1; -+} -+ -+static void rollover_cpu_window(struct rq *rq, bool full_window) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 curr_sum = srq->curr_runnable_sum; -+ -+ if (unlikely(full_window)) -+ curr_sum = 0; -+ -+ srq->prev_runnable_sum = curr_sum; -+ srq->curr_runnable_sum = 0; -+} -+ -+static u64 -+update_window_start(struct rq *rq, u64 wallclock, int event) -+{ -+ s64 delta; -+ int nr_windows; -+ bool full_window; -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start = srq->window_start; -+ -+ if (wallclock < srq->latest_clock) -+ wallclock = srq->latest_clock; -+ delta = wallclock - srq->window_start; -+ if (delta < 0) { -+ delta = 0; -+ wallclock = srq->window_start; -+ } -+ srq->latest_clock = wallclock; -+ if (delta < hmbird_sched_ravg_window) -+ return old_window_start; -+ -+ nr_windows = div64_u64(delta, hmbird_sched_ravg_window); -+ srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window; -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ full_window = nr_windows > 1; -+ rollover_cpu_window(rq, full_window); -+ -+ return old_window_start; -+} -+ -+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -+ -+static inline unsigned int get_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static inline unsigned int cpu_cur_freq(int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cur; -+} -+ -+static void -+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, -+ u64 wallclock) -+{ -+ int cpu = cpu_of(rq); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * -+ arch_scale_cpu_capacity(cpu), get_max_freq(cpu)); -+} -+ -+/* -+ * Called when new window is starting for a task, to record cpu usage over -+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 -+ * when, say, a real-time task runs without preemption for several windows at a -+ * stretch. -+ */ -+static void update_history(struct rq *rq, struct task_struct *p, -+ u32 runtime, int samples, int event) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u32 *hist = &sts->sum_history[0]; -+ int i; -+ u32 max = 0, avg, demand; -+ u64 sum = 0; -+ u16 demand_scaled; -+ -+ /* Ignore windows where task had no activity */ -+ if (!runtime || is_idle_task(p) || !samples) -+ goto done; -+ -+ /* Push new 'runtime' value onto stack */ -+ for (; samples > 0; samples--) { -+ hist[sts->cidx] = runtime; -+ sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE; -+ } -+ -+ for (i = 0; i < RAVG_HIST_SIZE; i++) { -+ sum += hist[i]; -+ if (hist[i] > max) -+ max = hist[i]; -+ } -+ -+ sts->sum = 0; -+ avg = div64_u64(sum, RAVG_HIST_SIZE); -+ -+ switch (slim_walt_policy) { -+ case WINDOW_STATS_RECENT: -+ demand = runtime; -+ break; -+ case WINDOW_STATS_MAX: -+ demand = max; -+ break; -+ case WINDOW_STATS_AVG: -+ demand = avg; -+ break; -+ default: -+ demand = max(avg, runtime); -+ } -+ -+ demand_scaled = scale_time_to_util(demand); -+ -+ sts->demand = demand; -+ sts->demand_scaled = demand_scaled; -+ -+done: -+ return; -+} -+ -+ -+static u64 update_task_demand(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ -+ u64 delta, window_start = srq->window_start; -+ int new_window, nr_full_windows; -+ u32 window_size = hmbird_sched_ravg_window; -+ u64 runtime; -+ -+ new_window = mark_start < window_start; -+ if (!account_busy_for_task_demand(rq, p, event)) { -+ if (new_window) -+ /* -+ * If the time accounted isn't being accounted as -+ * busy time, and a new window started, only the -+ * previous window need be closed out with the -+ * pre-existing demand. Multiple windows may have -+ * elapsed, but since empty windows are dropped, -+ * it is not necessary to account those. -+ */ -+ update_history(rq, p, sts->sum, 1, event); -+ return 0; -+ } -+ -+ if (!new_window) { -+ /* -+ * The simple case - busy time contained within the existing -+ * window. -+ */ -+ return add_to_task_demand(rq, p, wallclock - mark_start); -+ } -+ -+ /* -+ * Busy time spans at least two windows. Temporarily rewind -+ * window_start to first window boundary after mark_start. -+ */ -+ delta = window_start - mark_start; -+ nr_full_windows = div64_u64(delta, window_size); -+ window_start -= (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (window_start - mark_start) first */ -+ runtime = add_to_task_demand(rq, p, window_start - mark_start); -+ -+ /* Push new sample(s) into task's demand history */ -+ update_history(rq, p, sts->sum, 1, event); -+ if (nr_full_windows) { -+ u64 scaled_window = scale_exec_time(window_size, rq); -+ -+ update_history(rq, p, scaled_window, nr_full_windows, event); -+ runtime += nr_full_windows * scaled_window; -+ } -+ -+ /* -+ * Roll window_start back to current to process any remainder -+ * in current window. -+ */ -+ window_start += (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (wallclock - window_start) next */ -+ mark_start = window_start; -+ runtime += add_to_task_demand(rq, p, wallclock - mark_start); -+ -+ return runtime; -+} -+ -+u16 slim_walt_cpu_util(int cpu) -+{ -+ u64 prev_runnable_sum; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ prev_runnable_sum = srq->prev_runnable_sum; -+ do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT); -+ -+ return (u16)prev_runnable_sum; -+} -+ -+static DEFINE_PER_CPU(u16, prev_cpu_util); -+static inline void cpu_util_update_systrace_c(int cpu) -+{ -+ char buf[256]; -+ u16 cpu_util = slim_walt_cpu_util(cpu); -+ -+ if (cpu_util != per_cpu(prev_cpu_util, cpu)) { -+ snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n", -+ cpu, cpu_util); -+ tracing_mark_write(buf); -+ per_cpu(prev_cpu_util, cpu) = cpu_util; -+ } -+} -+ -+ -+static inline int account_busy_for_cpu_time(struct rq *rq, -+ struct task_struct *p, -+ int event) -+{ -+ return !is_idle_task(p) && (event == PUT_PREV_TASK -+ || event == TASK_UPDATE); -+} -+ -+ -+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ int new_window, full_window = 0; -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 window_start = srq->window_start; -+ u32 window_size = srq->prev_window_size; -+ u64 delta; -+ u64 *curr_runnable_sum = &srq->curr_runnable_sum; -+ u64 *prev_runnable_sum = &srq->prev_runnable_sum; -+ -+ new_window = mark_start < window_start; -+ if (new_window) -+ full_window = (window_start - mark_start) >= window_size; -+ -+ -+ if (!account_busy_for_cpu_time(rq, p, event)) -+ goto done; -+ -+ -+ if (!new_window) { -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. No rollover -+ * since we didn't start a new window. An example of this is -+ * when a task starts execution and then sleeps within the -+ * same window. -+ */ -+ delta = wallclock - mark_start; -+ -+ delta = scale_exec_time(delta, rq); -+ *curr_runnable_sum += delta; -+ -+ goto done; -+ } -+ -+ /* -+ * situations below this need window rollover, -+ * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done -+ * in update_window_start() -+ * -+ * For task counters curr/prev_window[_cpu] are rolled over in the early part of -+ * this function. If full_window(s) have expired and time since last update needs -+ * to be accounted as busy time, set the prev to a complete window size time, else -+ * add the prev window portion. -+ * -+ * For task curr counters a new window has begun, always assign -+ */ -+ -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. A new window -+ * must have been started in udpate_window_start() -+ * If any of these three above conditions are true -+ * then this busy time can't be accounted as irqtime. -+ * -+ * Busy time for the idle task need not be accounted. -+ * -+ * An example of this would be a task that starts execution -+ * and then sleeps once a new window has begun. -+ */ -+ -+ /* -+ * A full window hasn't elapsed, account partial -+ * contribution to previous completed window. -+ */ -+ -+ -+ delta = full_window ? scale_exec_time(window_size, rq) : -+ scale_exec_time(window_start - mark_start, rq); -+ -+ *prev_runnable_sum += delta; -+ -+ /* Account piece of busy time in the current window. */ -+ delta = scale_exec_time(wallclock - window_start, rq); -+ *curr_runnable_sum += delta; -+ -+done: -+ if (slim_walt_dump && new_window) -+ cpu_util_update_systrace_c(rq->cpu); -+} -+ -+ -+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq) -+{ -+ u64 result; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 new_window_start = srq->window_start; -+ -+ if (old_window_start == new_window_start) -+ return; -+ -+ result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws, -+ old_window_start, new_window_start); -+ if (result != old_window_start) -+ return; -+ -+ if (likely(cpu_online(raw_smp_processor_id()))) -+ irq_work_queue(&hmbird_slim_walt_irq_work); -+ else -+ irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask)); -+} -+ -+/* -+ * In the core scheduler, most of the load update points update the rq_clock after -+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of -+ * obtaining the time, but to prevent the subsequent migration of the load update -+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock -+ * update here. -+ */ -+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p, -+ struct rq *rq, int event) -+{ -+ if (!(rq->clock_update_flags & RQCF_UPDATED)) -+ update_rq_clock(rq); -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock)); -+} -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start; -+ -+ if (!slim_walt_ctrl) -+ return; -+ -+ if (!srq->window_start || sts->mark_start == wallclock) -+ return; -+ -+ old_window_start = update_window_start(rq, wallclock, event); -+ -+ if (!sts->window_start) -+ sts->window_start = srq->window_start; -+ -+ if (!sts->mark_start) -+ goto done; -+ -+ update_task_rq_cpu_cycles(p, rq, event, wallclock); -+ update_task_demand(p, rq, event, wallclock); -+ update_cpu_busy_time(p, rq, event, wallclock); -+ -+ sts->window_start = srq->window_start; -+ -+done: -+ sts->mark_start = wallclock; -+ slim_walt_window_rollover_run_once(old_window_start, rq); -+} -+ -+static void slim_walt_irq_work(struct irq_work *irq_work) -+{ -+ cpumask_t lock_cpus; -+ struct hmbird_sched_rq_stats *srq; -+ struct rq *rq; -+ int cpu; -+ int level = 0; -+ u64 wc; -+ unsigned long flags; -+ -+ cpumask_copy(&lock_cpus, cpu_possible_mask); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ if (level == 0) -+ raw_spin_lock(&cpu_rq(cpu)->__lock); -+ else -+ raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); -+ level++; -+ } -+ -+ wc = sched_clock(); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ rq = cpu_rq(cpu); -+ hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc); -+ } -+ -+ cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER); -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) { -+ srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id()); -+ if (wc < srq->window_start + new_hmbird_sched_ravg_window) -+ hmbird_sched_ravg_window = new_hmbird_sched_ravg_window; -+ } -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ raw_spin_unlock(&cpu_rq(cpu)->__lock); -+ } -+} -+static void hmbird_sched_init_rq(struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ srq->task_exec_scale = 1024; -+ srq->window_start = 0; -+} -+ -+void hmbird_sched_init_task(struct task_struct *p) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ memset(sts, 0, sizeof(struct hmbird_sched_task_stats)); -+} -+ -+static void hmbird_sched_stats_init(void) -+{ -+ unsigned long flags; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ raw_spin_lock_irqsave(&rq->__lock, flags); -+ hmbird_sched_init_rq(rq); -+ raw_spin_unlock_irqrestore(&rq->__lock, flags); -+ } -+ slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG; -+ -+ if (false == init_irq_work_inited) { -+ init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work); -+ init_irq_work_inited = true; -+ } -+} -+ -+void hmbird_scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (unlikely(!tick_sched_clock)) { -+ /* -+ * Let the window begin 20us prior to the tick, -+ * that way we are guaranteed a rollover when the tick occurs. -+ * Use rq->clock directly instead of rq_clock() since -+ * we do not have the rq lock and -+ * rq->clock was updated in the tick callpath. -+ */ -+ if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000)) -+ return; -+ for_each_possible_cpu(cpu) { -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ srq->window_start = tick_sched_clock; -+ } -+ atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock); -+ } -+} -+ -+void slim_walt_enable(int enable) -+{ -+ if (1 == !!enable) { -+ hmbird_sched_stats_init(); -+ WRITE_ONCE(tick_sched_clock, 0); -+ } else -+ slim_walt_ctrl = 0; -+} -+ -+void slim_get_cpu_util(int cpu, u64 *util) -+{ -+ if (cpu < 0) -+ return; -+ -+ *util = slim_walt_cpu_util(cpu); -+} -+ -+void slim_get_task_util(struct task_struct *p, u64 *util) -+{ -+ if (p == NULL) -+ return; -+ -+ *util = get_hmbird_ts(p)->sts.demand_scaled; -+} -+ -+void sched_ravg_window_change(int frame_per_sec) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec; -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+} -diff --git a/kernel/sched/hmbird/hmbird_util_track.h b/kernel/sched/hmbird/hmbird_util_track.h -new file mode 100755 -index 000000000000..17b85df7f83b ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.h -@@ -0,0 +1,33 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef __HMBIRD_UTIL_TRACK_H__ -+#define __HMBIRD_UTIL_TRACK_H__ -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock); -+void hmbird_sched_init_task(struct task_struct *p); -+void slim_walt_enable(int enable); -+void slim_get_cpu_util(int cpu, u64 *util); -+void slim_get_task_util(struct task_struct *p, u64 *util); -+ -+extern atomic64_t hmbird_irq_work_lastq_ws; -+ -+enum task_event { -+ PUT_PREV_TASK = 0, -+ PICK_NEXT_TASK = 1, -+ TASK_WAKE = 2, -+ TASK_MIGRATE = 3, -+ TASK_UPDATE = 4, -+ IRQ_UPDATE = 5, -+}; -+ -+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+#endif /* __HMBIRD_UTIL_TRACK_H__ */ -diff --git a/kernel/sched/hmbird/slim.h b/kernel/sched/hmbird/slim.h -new file mode 100755 -index 000000000000..eb386260e950 ---- /dev/null -+++ b/kernel/sched/hmbird/slim.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+ -+#ifndef __SLIM_H -+#define __SLIM_H -+ -+extern atomic_t __hmbird_ops_enabled; -+extern atomic_t non_hmbird_task; -+extern int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+extern int heartbeat; -+extern int heartbeat_enable; -+extern int watchdog_enable; -+extern int isolate_ctrl; -+extern int parctrl_high_ratio; -+extern int parctrl_low_ratio; -+extern int isoctrl_high_ratio; -+extern int isoctrl_low_ratio; -+extern int iso_free_rescue; -+extern int yield_opt; -+ -+extern enum hmbird_switch_type sw_type; -+extern noinline int tracing_mark_write(const char *buf); -+int task_top_id(struct task_struct *p); -+void stats_print(char *buf, int len); -+void hmbird_skip_yield(long *skip); -+extern spinlock_t hmbird_tasks_lock; -+ -+struct yield_opt_params { -+ int enable; -+ int frame_per_sec; -+ u64 frame_time_ns; -+ int yield_headroom; -+}; -+ -+extern struct yield_opt_params yield_opt_params; -+ -+#define MAX_GOV_LEN (16) -+extern char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+#endif -diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c -index b33cefeb4188..487255ac6832 100644 ---- a/kernel/sched/idle.c -+++ b/kernel/sched/idle.c -@@ -408,13 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl - - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) - { -- scx_update_idle(rq, false); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, false); -+#endif - } - - static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) - { - update_idle_core(rq); -- scx_update_idle(rq, true); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, true); -+#endif - schedstat_inc(rq->sched_goidle); - } - -diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h -index cbc478992cc4..e35473a1f0ea 100644 ---- a/kernel/sched/sched.h -+++ b/kernel/sched/sched.h -@@ -187,18 +187,13 @@ static inline int idle_policy(int policy) - return policy == SCHED_IDLE; - } - --static inline int normal_policy(int policy) --{ --#ifdef CONFIG_SCHED_CLASS_EXT -- if (policy == SCHED_EXT) -- return true; --#endif -- return policy == SCHED_NORMAL; --} -- - static inline int fair_policy(int policy) - { -- return normal_policy(policy) || policy == SCHED_BATCH; -+#ifdef CONFIG_HMBIRD_SCHED -+ return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#else -+ return policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#endif - } - - static inline int rt_policy(int policy) -@@ -246,24 +241,6 @@ static inline void update_avg(u64 *avg, u64 sample) - #define shr_bound(val, shift) \ - (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) - --/* -- * cgroup weight knobs should use the common MIN, DFL and MAX values which are -- * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it -- * maps pretty well onto the shares value used by scheduler and the round-trip -- * conversions preserve the original value over the entire range. -- */ --static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) --{ -- return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); --} -- --static inline unsigned long sched_weight_to_cgroup(unsigned long weight) --{ -- return clamp_t(unsigned long, -- DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -- CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); --} -- - /* - * !! For sched_setattr_nocheck() (kernel) only !! - * -@@ -531,10 +508,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) { } - #endif /* CONFIG_SMP */ - #else /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_HMBIRD_SCHED - static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) - { - return 0; - } -+#endif - #endif /* CONFIG_FAIR_GROUP_SCHED */ - - #else /* CONFIG_CGROUP_SCHED */ -@@ -699,29 +678,6 @@ struct cfs_rq { - #endif /* CONFIG_FAIR_GROUP_SCHED */ - }; - --#ifdef CONFIG_SCHED_CLASS_EXT --/* scx_rq->flags, protected by the rq lock */ --enum scx_rq_flags { -- SCX_RQ_CAN_STOP_TICK = 1 << 0, --}; -- --struct scx_rq { -- struct scx_dispatch_q local_dsq; -- struct list_head watchdog_list; -- u64 ops_qseq; -- u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -- u32 nr_running; -- u32 flags; -- bool cpu_released; -- cpumask_var_t cpus_to_kick; -- cpumask_var_t cpus_to_preempt; -- cpumask_var_t cpus_to_wait; -- u64 pnt_seq; -- struct irq_work kick_cpus_irq_work; -- struct rq *rq; --}; --#endif /* CONFIG_SCHED_CLASS_EXT */ -- - static inline int rt_bandwidth_enabled(void) - { - return sysctl_sched_rt_runtime >= 0; -@@ -1257,11 +1213,7 @@ struct rq { - - ANDROID_OEM_DATA_ARRAY(1, 16); - --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, struct scx_rq *scx); --#else - ANDROID_KABI_RESERVE(1); --#endif - ANDROID_KABI_RESERVE(2); - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); -@@ -2339,11 +2291,6 @@ struct affinity_context { - unsigned int flags; - }; - --enum rq_onoff_reason { -- RQ_ONOFF_HOTPLUG, /* CPU is going on/offline */ -- RQ_ONOFF_TOPOLOGY, /* sched domain topology update */ --}; -- - struct sched_class { - - #ifdef CONFIG_UCLAMP_TASK -@@ -2550,7 +2497,6 @@ extern void init_sched_rt_class(void); - extern void init_sched_fair_class(void); - - extern void reweight_task(struct task_struct *p, int prio); --extern void __setscheduler_prio(struct task_struct *p, int prio); - - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); -@@ -2631,53 +2577,6 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) - extern void activate_task(struct rq *rq, struct task_struct *p, int flags); - extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); - --struct sched_change_guard { -- struct task_struct *p; -- struct rq *rq; -- bool queued; -- bool running; -- bool done; --}; -- --extern struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -- --extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags); -- --/** -- * SCHED_CHANGE_BLOCK - Nested block for task attribute updates -- * @__rq: Runqueue the target task belongs to -- * @__p: Target task -- * @__flags: DEQUEUE/ENQUEUE_* flags -- * -- * A task may need to be dequeued and put_prev_task'd for attribute updates and -- * set_next_task'd and re-enqueued afterwards. This helper defines a nested -- * block which automatically handles these preparation and cleanup operations. -- * -- * SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- * update_attribute(p); -- * ... -- * } -- * -- * If @__flags is a variable, the variable may be updated in the block body and -- * the updated value will be used when re-enqueueing @p. -- * -- * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling -- * update_rq_clock() beforehand. Otherwise, the rq clock is automatically -- * updated iff the task needs to be dequeued and re-enqueued. Only the former -- * case guarantees that the rq clock is up-to-date inside and after the block. -- */ --#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -- for (struct sched_change_guard __cg = \ -- sched_change_guard_init(__rq, __p, __flags); \ -- !__cg.done; sched_change_guard_fini(&__cg, __flags)) -- --extern void check_class_changing(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class); --extern void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio); -- - extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); - - #ifdef CONFIG_PREEMPT_RT -@@ -3709,6 +3608,8 @@ static inline bool cpu_busy_with_softirqs(int cpu) - } - #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ - --#include "ext.h" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird.h" -+#endif - - #endif /* _KERNEL_SCHED_SCHED_H */ -diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c -index 254624e4bd43..2a3f819b9e88 100644 ---- a/kernel/time/tick-sched.c -+++ b/kernel/time/tick-sched.c -@@ -34,6 +34,10 @@ - - #include - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "../sched/hmbird/hmbird_shadow_tick.h" -+#endif -+ - /* - * Per-CPU nohz control structure - */ -@@ -1112,6 +1116,9 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) - return true; - } - -+#ifdef CONFIG_HMBIRD_SCHED -+extern void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+#endif - /** - * tick_nohz_idle_stop_tick - stop the idle tick from the idle task - * -@@ -1124,7 +1131,9 @@ void tick_nohz_idle_stop_tick(void) - ktime_t expires; - - trace_android_vh_tick_nohz_idle_stop_tick(NULL); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL); -+#endif - /* - * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the - * tick timer expiration time is known already. -diff --git a/tools/sched_ext/.gitignore b/tools/sched_ext/.gitignore -deleted file mode 100644 -index a3240f9f7eba..000000000000 ---- a/tools/sched_ext/.gitignore -+++ /dev/null -@@ -1,9 +0,0 @@ --scx_example_simple --scx_example_qmap --scx_example_central --scx_example_pair --scx_example_flatcg --scx_example_userland --*.skel.h --*.subskel.h --/tools/ -diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile -deleted file mode 100644 -index 73c43782837d..000000000000 ---- a/tools/sched_ext/Makefile -+++ /dev/null -@@ -1,216 +0,0 @@ --# SPDX-License-Identifier: GPL-2.0 --# Copyright (c) 2022 Meta Platforms, Inc. and affiliates. --include ../build/Build.include --include ../scripts/Makefile.arch --include ../scripts/Makefile.include -- --ifneq ($(LLVM),) --ifneq ($(filter %/,$(LLVM)),) --LLVM_PREFIX := $(LLVM) --else ifneq ($(filter -%,$(LLVM)),) --LLVM_SUFFIX := $(LLVM) --endif -- --CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi --CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu --CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl --CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu --CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu --CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu --CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu --CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu --CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu --CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) -- --ifeq ($(CROSS_COMPILE),) --ifeq ($(CLANG_TARGET_FLAGS),) --$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk --else --CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) --endif # CLANG_TARGET_FLAGS --else --CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) --endif # CROSS_COMPILE -- --CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as --else --CC := $(CROSS_COMPILE)gcc --endif # LLVM -- --CURDIR := $(abspath .) --TOOLSDIR := $(abspath ..) --LIBDIR := $(TOOLSDIR)/lib --BPFDIR := $(LIBDIR)/bpf --TOOLSINCDIR := $(TOOLSDIR)/include --BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool --APIDIR := $(TOOLSINCDIR)/uapi --GENDIR := $(abspath ../../include/generated) --GENHDR := $(GENDIR)/autoconf.h -- --SCRATCH_DIR := $(CURDIR)/tools --BUILD_DIR := $(SCRATCH_DIR)/build --INCLUDE_DIR := $(SCRATCH_DIR)/include --BPFOBJ_DIR := $(BUILD_DIR)/libbpf --BPFOBJ := $(BPFOBJ_DIR)/libbpf.a --ifneq ($(CROSS_COMPILE),) --HOST_BUILD_DIR := $(BUILD_DIR)/host --HOST_SCRATCH_DIR := host-tools --HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include --else --HOST_BUILD_DIR := $(BUILD_DIR) --HOST_SCRATCH_DIR := $(SCRATCH_DIR) --HOST_INCLUDE_DIR := $(INCLUDE_DIR) --endif --HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a --RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids --DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -- --VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ -- $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ -- ../../vmlinux \ -- /sys/kernel/btf/vmlinux \ -- /boot/vmlinux-$(shell uname -r) --VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) --ifeq ($(VMLINUX_BTF),) --$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") --endif -- --BPFTOOL ?= $(DEFAULT_BPFTOOL) -- --ifneq ($(wildcard $(GENHDR)),) -- GENFLAGS := -DHAVE_GENHDR --endif -- --CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ -- -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -- -I$(TOOLSINCDIR) -I$(APIDIR) -- --CARGOFLAGS := --release -- --# Silence some warnings when compiled with clang --ifneq ($(LLVM),) --CFLAGS += -Wno-unused-command-line-argument --endif -- --LDFLAGS = -lelf -lz -lpthread -- --IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - &1 \ -- | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ --$(shell $(1) -dM -E - $@ --else -- $(call msg,CP,,$@) -- $(Q)cp "$(VMLINUX_H)" $@ --endif -- --%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \ -- | $(BPFOBJ) -- $(call msg,CLNG-BPF,,$@) -- $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ -- --%.skel.h: %.bpf.o $(BPFTOOL) -- $(call msg,GEN-SKEL,,$@) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) -- $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) -- $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@ -- $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h) -- --scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_userland: scx_example_userland.c scx_example_userland.skel.h \ -- scx_example_userland_common.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR) --atropos: export ATROPOS_CLANG = $(CLANG) --atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS) --atropos: $(INCLUDE_DIR)/vmlinux.h -- cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS) -- --clean: -- cargo clean --manifest-path=atropos/Cargo.toml -- rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) -- rm -f *.o *.bpf.o *.skel.h *.subskel.h -- rm -f scx_example_simple scx_example_qmap scx_example_central \ -- scx_example_pair scx_example_flatcg scx_example_userland -- --.PHONY: all atropos clean -- --# delete failed targets --.DELETE_ON_ERROR: -- --# keep intermediate (.skel.h, .bpf.o, etc) targets --.SECONDARY: -diff --git a/tools/sched_ext/atropos/.gitignore b/tools/sched_ext/atropos/.gitignore -deleted file mode 100644 -index 186dba259ec2..000000000000 ---- a/tools/sched_ext/atropos/.gitignore -+++ /dev/null -@@ -1,3 +0,0 @@ --src/bpf/.output --Cargo.lock --target -diff --git a/tools/sched_ext/atropos/Cargo.toml b/tools/sched_ext/atropos/Cargo.toml -deleted file mode 100644 -index 7462a836d53d..000000000000 ---- a/tools/sched_ext/atropos/Cargo.toml -+++ /dev/null -@@ -1,28 +0,0 @@ --[package] --name = "scx_atropos" --version = "0.5.0" --authors = ["Dan Schatzberg ", "Meta"] --edition = "2021" --description = "Userspace scheduling with BPF" --license = "GPL-2.0-only" -- --[dependencies] --anyhow = "1.0.65" --bitvec = { version = "1.0", features = ["serde"] } --clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } --ctrlc = { version = "3.1", features = ["termination"] } --fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"} --hex = "0.4.3" --libbpf-rs = "0.19.1" --libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] } --libc = "0.2.137" --log = "0.4.17" --ordered-float = "3.4.0" --simplelog = "0.12.0" -- --[build-dependencies] --bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false } --libbpf-cargo = "0.13.0" -- --[features] --enable_backtrace = [] -diff --git a/tools/sched_ext/atropos/build.rs b/tools/sched_ext/atropos/build.rs -deleted file mode 100644 -index 26e792c5e17e..000000000000 ---- a/tools/sched_ext/atropos/build.rs -+++ /dev/null -@@ -1,70 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --extern crate bindgen; -- --use std::env; --use std::fs::create_dir_all; --use std::path::Path; --use std::path::PathBuf; -- --use libbpf_cargo::SkeletonBuilder; -- --const HEADER_PATH: &str = "src/bpf/atropos.h"; -- --fn bindgen_atropos() { -- // Tell cargo to invalidate the built crate whenever the wrapper changes -- println!("cargo:rerun-if-changed={}", HEADER_PATH); -- -- // The bindgen::Builder is the main entry point -- // to bindgen, and lets you build up options for -- // the resulting bindings. -- let bindings = bindgen::Builder::default() -- // The input header we would like to generate -- // bindings for. -- .header(HEADER_PATH) -- // Tell cargo to invalidate the built crate whenever any of the -- // included header files changed. -- .parse_callbacks(Box::new(bindgen::CargoCallbacks)) -- // Finish the builder and generate the bindings. -- .generate() -- // Unwrap the Result and panic on failure. -- .expect("Unable to generate bindings"); -- -- // Write the bindings to the $OUT_DIR/bindings.rs file. -- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); -- bindings -- .write_to_file(out_path.join("atropos-sys.rs")) -- .expect("Couldn't write bindings!"); --} -- --fn gen_bpf_sched(name: &str) { -- let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap(); -- let clang = env::var("ATROPOS_CLANG").unwrap(); -- eprintln!("{}", clang); -- let outpath = format!("./src/bpf/.output/{}.skel.rs", name); -- let skel = Path::new(&outpath); -- let src = format!("./src/bpf/{}.bpf.c", name); -- SkeletonBuilder::new() -- .source(src.clone()) -- .clang(clang) -- .clang_args(bpf_cflags) -- .build_and_generate(&skel) -- .unwrap(); -- println!("cargo:rerun-if-changed={}", src); --} -- --fn main() { -- bindgen_atropos(); -- // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton. -- // Reasons are because the generated skeleton contains compiler attributes -- // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]` -- // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside -- // the path attribute either (see https://github.com/rust-lang/rust/pull/83366). -- // -- // However, there is hope! When the above feature stabilizes we can clean this -- // all up. -- create_dir_all("./src/bpf/.output").unwrap(); -- gen_bpf_sched("atropos"); --} -diff --git a/tools/sched_ext/atropos/rustfmt.toml b/tools/sched_ext/atropos/rustfmt.toml -deleted file mode 100644 -index b7258ed0a8d8..000000000000 ---- a/tools/sched_ext/atropos/rustfmt.toml -+++ /dev/null -@@ -1,8 +0,0 @@ --# Get help on options with `rustfmt --help=config` --# Please keep these in alphabetical order. --edition = "2021" --group_imports = "StdExternalCrate" --imports_granularity = "Item" --merge_derives = false --use_field_init_shorthand = true --version = "Two" -diff --git a/tools/sched_ext/atropos/src/atropos_sys.rs b/tools/sched_ext/atropos/src/atropos_sys.rs -deleted file mode 100644 -index bbeaf856d40e..000000000000 ---- a/tools/sched_ext/atropos/src/atropos_sys.rs -+++ /dev/null -@@ -1,10 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#![allow(non_upper_case_globals)] --#![allow(non_camel_case_types)] --#![allow(non_snake_case)] --#![allow(dead_code)] -- --include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs")); -diff --git a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -deleted file mode 100644 -index 3905a403e940..000000000000 ---- a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -+++ /dev/null -@@ -1,743 +0,0 @@ --/* Copyright (c) Meta Platforms, Inc. and affiliates. */ --/* -- * This software may be used and distributed according to the terms of the -- * GNU General Public License version 2. -- * -- * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF -- * part does simple round robin in each domain and the userspace part -- * calculates the load factor of each domain and tells the BPF part how to load -- * balance the domains. -- * -- * Every task has an entry in the task_data map which lists which domain the -- * task belongs to. When a task first enters the system (atropos_prep_enable), -- * they are round-robined to a domain. -- * -- * atropos_select_cpu is the primary scheduling logic, invoked when a task -- * becomes runnable. The lb_data map is populated by userspace to inform the BPF -- * scheduler that a task should be migrated to a new domain. Otherwise, the task -- * is scheduled in priority order as follows: -- * * The current core if the task was woken up synchronously and there are idle -- * cpus in the system -- * * The previous core, if idle -- * * The pinned-to core if the task is pinned to a specific core -- * * Any idle cpu in the domain -- * -- * If none of the above conditions are met, then the task is enqueued to a -- * dispatch queue corresponding to the domain (atropos_enqueue). -- * -- * atropos_dispatch will attempt to consume a task from its domain's -- * corresponding dispatch queue (this occurs after scheduling any tasks directly -- * assigned to it due to the logic in atropos_select_cpu). If no task is found, -- * then greedy load stealing will attempt to find a task on another dispatch -- * queue to run. -- * -- * Load balancing is almost entirely handled by userspace. BPF populates the -- * task weight, dom mask and current dom in the task_data map and executes the -- * load balance based on userspace populating the lb_data map. -- */ --#include "../../../scx_common.bpf.h" --#include "atropos.h" -- --#include --#include --#include --#include --#include --#include -- --char _license[] SEC("license") = "GPL"; -- --/* -- * const volatiles are set during initialization and treated as consts by the -- * jit compiler. -- */ -- --/* -- * Domains and cpus -- */ --const volatile __u32 nr_doms = 32; /* !0 for veristat, set during init */ --const volatile __u32 nr_cpus = 64; /* !0 for veristat, set during init */ --const volatile __u32 cpu_dom_id_map[MAX_CPUS]; --const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; -- --const volatile bool kthreads_local; --const volatile bool fifo_sched; --const volatile bool switch_partial; --const volatile __u32 greedy_threshold; -- --/* base slice duration */ --const volatile __u64 slice_us = 20000; -- --/* -- * Exit info -- */ --int exit_type = SCX_EXIT_NONE; --char exit_msg[SCX_EXIT_MSG_LEN]; -- --struct pcpu_ctx { -- __u32 dom_rr_cur; /* used when scanning other doms */ -- -- /* libbpf-rs does not respect the alignment, so pad out the struct explicitly */ -- __u8 _padding[CACHELINE_SIZE - sizeof(u64)]; --} __attribute__((aligned(CACHELINE_SIZE))); -- --struct pcpu_ctx pcpu_ctx[MAX_CPUS]; -- --/* -- * Domain context -- */ --struct dom_ctx { -- struct bpf_cpumask __kptr *cpumask; -- u64 vtime_now; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __type(key, u32); -- __type(value, struct dom_ctx); -- __uint(max_entries, MAX_DOMS); -- __uint(map_flags, 0); --} dom_ctx SEC(".maps"); -- --/* -- * Statistics -- */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, ATROPOS_NR_STATS); --} stats SEC(".maps"); -- --static inline void stat_add(enum stat_idx idx, u64 addend) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p) += addend; --} -- --/* Map pid -> task_ctx */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, struct task_ctx); -- __uint(max_entries, 1000000); -- __uint(map_flags, 0); --} task_data SEC(".maps"); -- --/* -- * This is populated from userspace to indicate which pids should be reassigned -- * to new doms. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, u32); -- __uint(max_entries, 1000); -- __uint(map_flags, 0); --} lb_data SEC(".maps"); -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p, -- u32 new_dom_id) --{ -- struct dom_ctx *old_domc, *new_domc; -- struct bpf_cpumask *d_cpumask, *t_cpumask; -- u32 old_dom_id = task_ctx->dom_id; -- s64 vtime_delta; -- -- old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id); -- if (!old_domc) { -- scx_bpf_error("No dom%u", old_dom_id); -- return false; -- } -- -- vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now; -- -- new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id); -- if (!new_domc) { -- scx_bpf_error("No dom%u", new_dom_id); -- return false; -- } -- -- d_cpumask = new_domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to get domain %u cpumask kptr", -- new_dom_id); -- return false; -- } -- -- t_cpumask = task_ctx->cpumask; -- if (!t_cpumask) { -- scx_bpf_error("Failed to look up task cpumask"); -- return false; -- } -- -- /* -- * set_cpumask might have happened between userspace requesting LB and -- * here and @p might not be able to run in @dom_id anymore. Verify. -- */ -- if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- p->cpus_ptr)) { -- p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta; -- task_ctx->dom_id = new_dom_id; -- bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask, -- p->cpus_ptr); -- } -- -- return task_ctx->dom_id == new_dom_id; --} -- --s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu, -- u32 wake_flags) --{ -- s32 cpu; -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- struct bpf_cpumask *p_cpumask; -- -- if (!task_ctx) -- return -ENOENT; -- -- if (kthreads_local && -- (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- cpu = prev_cpu; -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local dsq of the waker. -- */ -- if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) { -- struct task_struct *current = (void *)bpf_get_current_task(); -- -- if (!(BPF_CORE_READ(current, flags) & PF_EXITING) && -- task_ctx->dom_id < MAX_DOMS) { -- struct dom_ctx *domc; -- struct bpf_cpumask *d_cpumask; -- const struct cpumask *idle_cpumask; -- bool has_idle; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id); -- if (!domc) { -- scx_bpf_error("Failed to find dom%u", -- task_ctx->dom_id); -- return prev_cpu; -- } -- d_cpumask = domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to acquire domain %u cpumask kptr", -- task_ctx->dom_id); -- return prev_cpu; -- } -- -- idle_cpumask = scx_bpf_get_idle_cpumask(); -- -- has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- idle_cpumask); -- -- scx_bpf_put_idle_cpumask(idle_cpumask); -- -- if (has_idle) { -- cpu = bpf_get_smp_processor_id(); -- if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- stat_add(ATROPOS_STAT_WAKE_SYNC, 1); -- goto local; -- } -- } -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- stat_add(ATROPOS_STAT_PREV_IDLE, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- /* If only one core is allowed, dispatch */ -- if (p->nr_cpus_allowed == 1) { -- stat_add(ATROPOS_STAT_PINNED, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) -- return -ENOENT; -- -- /* If there is an eligible idle CPU, dispatch directly */ -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- if (cpu >= 0) { -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * @prev_cpu may be in a different domain. Returning an out-of-domain -- * CPU can lead to stalls as all in-domain CPUs may be idle by the time -- * @p gets enqueued. -- */ -- if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask)) -- cpu = prev_cpu; -- else -- cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask); -- -- return cpu; -- --local: -- task_ctx->dispatch_local = true; -- return cpu; --} -- --void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- u32 *new_dom; -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- new_dom = bpf_map_lookup_elem(&lb_data, &pid); -- if (new_dom && *new_dom != task_ctx->dom_id && -- task_set_dsq(task_ctx, p, *new_dom)) { -- struct bpf_cpumask *p_cpumask; -- s32 cpu; -- -- stat_add(ATROPOS_STAT_LOAD_BALANCE, 1); -- -- /* -- * If dispatch_local is set, We own @p's idle state but we are -- * not gonna put the task in the associated local dsq which can -- * cause the CPU to stall. Kick it. -- */ -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0); -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) { -- scx_bpf_error("Failed to get task_ctx->cpumask"); -- return; -- } -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- } -- -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags); -- return; -- } -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000, -- enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- u32 dom_id = task_ctx->dom_id; -- struct dom_ctx *domc; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, domc->vtime_now - slice_us * 1000)) -- vtime = domc->vtime_now - slice_us * 1000; -- -- scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --static u32 cpu_to_dom_id(s32 cpu) --{ -- const volatile u32 *dom_idp; -- -- if (nr_doms <= 1) -- return 0; -- -- dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]); -- if (!dom_idp) -- return MAX_DOMS; -- -- return *dom_idp; --} -- --static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) --{ -- s32 cpu; -- -- if (dom_id >= MAX_DOMS) -- return false; -- -- bpf_for(cpu, 0, nr_cpus) { -- if (bpf_cpumask_test_cpu(cpu, cpumask) && -- (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64)))) -- return true; -- } -- return false; --} -- --static u32 dom_rr_next(s32 cpu) --{ -- struct pcpu_ctx *pcpuc; -- u32 dom_id; -- -- pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]); -- if (!pcpuc) -- return 0; -- -- dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms; -- -- if (dom_id == cpu_to_dom_id(cpu)) -- dom_id = (dom_id + 1) % nr_doms; -- -- pcpuc->dom_rr_cur = dom_id; -- return dom_id; --} -- --void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 dom = cpu_to_dom_id(cpu); -- -- if (scx_bpf_consume(dom)) { -- stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1); -- return; -- } -- -- if (!greedy_threshold) -- return; -- -- bpf_repeat(nr_doms - 1) { -- u32 dom_id = dom_rr_next(cpu); -- -- if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold && -- scx_bpf_consume(dom_id)) { -- stat_add(ATROPOS_STAT_GREEDY, 1); -- break; -- } -- } --} -- --void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_at = bpf_ktime_get_ns(); --} -- --void BPF_STRUCT_OPS(atropos_running, struct task_struct *p) --{ -- struct task_ctx *taskc; -- struct dom_ctx *domc; -- pid_t pid = p->pid; -- u32 dom_id; -- -- if (fifo_sched) -- return; -- -- taskc = bpf_map_lookup_elem(&task_data, &pid); -- if (!taskc) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- dom_id = taskc->dom_id; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(domc->vtime_now, p->scx.dsq_vtime)) -- domc->vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at; -- task_ctx->runnable_at = 0; --} -- --void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->weight = weight; --} -- --struct pick_task_domain_loop_ctx { -- struct task_struct *p; -- const struct cpumask *cpumask; -- u64 dom_mask; -- u32 dom_rr_base; -- u32 dom_id; --}; -- --static int pick_task_domain_loopfn(u32 idx, void *data) --{ -- struct pick_task_domain_loop_ctx *lctx = data; -- u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms; -- -- if (dom_id >= MAX_DOMS) -- return 1; -- -- if (cpumask_intersects_domain(lctx->cpumask, dom_id)) { -- lctx->dom_mask |= 1LLU << dom_id; -- if (lctx->dom_id == MAX_DOMS) -- lctx->dom_id = dom_id; -- } -- return 0; --} -- --static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- struct pick_task_domain_loop_ctx lctx = { -- .p = p, -- .cpumask = cpumask, -- .dom_id = MAX_DOMS, -- }; -- s32 cpu = bpf_get_smp_processor_id(); -- -- if (cpu < 0 || cpu >= MAX_CPUS) -- return MAX_DOMS; -- -- lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur); -- -- bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0); -- task_ctx->dom_mask = lctx.dom_mask; -- -- return lctx.dom_id; --} -- --static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- u32 dom_id = 0; -- -- if (nr_doms > 1) -- dom_id = pick_task_domain(task_ctx, p, cpumask); -- -- if (!task_set_dsq(task_ctx, p, dom_id)) -- scx_bpf_error("Failed to set domain %d for %s[%d]", -- dom_id, p->comm, p->pid); --} -- --void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_set_domain(task_ctx, p, cpumask); --} -- --s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct bpf_cpumask *cpumask; -- struct task_ctx task_ctx, *map_value; -- long ret; -- pid_t pid; -- -- memset(&task_ctx, 0, sizeof(task_ctx)); -- -- pid = p->pid; -- ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return ret; -- } -- -- /* -- * Read the entry from the map immediately so we can add the cpumask -- * with bpf_kptr_xchg(). -- */ -- map_value = bpf_map_lookup_elem(&task_data, &pid); -- if (!map_value) -- /* Should never happen -- it was just inserted above. */ -- return -EINVAL; -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- bpf_map_delete_elem(&task_data, &pid); -- return -ENOMEM; -- } -- -- cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask); -- if (cpumask) { -- /* Should never happen as we just inserted it above. */ -- bpf_cpumask_release(cpumask); -- bpf_map_delete_elem(&task_data, &pid); -- return -EINVAL; -- } -- -- task_set_domain(map_value, p, p->cpus_ptr); -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p) --{ -- pid_t pid = p->pid; -- long ret = bpf_map_delete_elem(&task_data, &pid); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return; -- } --} -- --static int create_dom_dsq(u32 idx, void *data) --{ -- struct dom_ctx domc_init = {}, *domc; -- struct bpf_cpumask *cpumask; -- u32 cpu, dom_id = idx; -- s32 ret; -- -- ret = scx_bpf_create_dsq(dom_id, -1); -- if (ret < 0) { -- scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret); -- return 1; -- } -- -- ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0); -- if (ret) { -- scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret); -- return 1; -- } -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- /* Should never happen, we just inserted it above. */ -- scx_bpf_error("No dom%u", dom_id); -- return 1; -- } -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id); -- return 1; -- } -- -- for (cpu = 0; cpu < MAX_CPUS; cpu++) { -- const volatile __u64 *dmask; -- -- dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]); -- if (!dmask) { -- scx_bpf_error("array index error"); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- if (*dmask & (1LLU << (cpu % 64))) -- bpf_cpumask_set_cpu(cpu, cpumask); -- } -- -- cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask); -- if (cpumask) { -- scx_bpf_error("Domain %u was already present", dom_id); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(atropos_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- bpf_loop(nr_doms, create_dom_dsq, NULL, 0); -- -- for (u32 i = 0; i < nr_cpus; i++) -- pcpu_ctx[i].dom_rr_cur = i; -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg); -- exit_type = ei->type; --} -- --SEC(".struct_ops") --struct sched_ext_ops atropos = { -- .select_cpu = (void *)atropos_select_cpu, -- .enqueue = (void *)atropos_enqueue, -- .dispatch = (void *)atropos_dispatch, -- .runnable = (void *)atropos_runnable, -- .running = (void *)atropos_running, -- .stopping = (void *)atropos_stopping, -- .quiescent = (void *)atropos_quiescent, -- .set_weight = (void *)atropos_set_weight, -- .set_cpumask = (void *)atropos_set_cpumask, -- .prep_enable = (void *)atropos_prep_enable, -- .disable = (void *)atropos_disable, -- .init = (void *)atropos_init, -- .exit = (void *)atropos_exit, -- .flags = 0, -- .name = "atropos", --}; -diff --git a/tools/sched_ext/atropos/src/bpf/atropos.h b/tools/sched_ext/atropos/src/bpf/atropos.h -deleted file mode 100644 -index addf29ca104a..000000000000 ---- a/tools/sched_ext/atropos/src/bpf/atropos.h -+++ /dev/null -@@ -1,44 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#ifndef __ATROPOS_H --#define __ATROPOS_H -- --#include --#ifndef __kptr --#ifdef __KERNEL__ --#error "__kptr_ref not defined in the kernel" --#endif --#define __kptr --#endif -- --#define MAX_CPUS 512 --#define MAX_DOMS 64 /* limited to avoid complex bitmask ops */ --#define CACHELINE_SIZE 64 -- --/* Statistics */ --enum stat_idx { -- ATROPOS_STAT_TASK_GET_ERR, -- ATROPOS_STAT_WAKE_SYNC, -- ATROPOS_STAT_PREV_IDLE, -- ATROPOS_STAT_PINNED, -- ATROPOS_STAT_DIRECT_DISPATCH, -- ATROPOS_STAT_DSQ_DISPATCH, -- ATROPOS_STAT_GREEDY, -- ATROPOS_STAT_LOAD_BALANCE, -- ATROPOS_STAT_LAST_TASK, -- ATROPOS_NR_STATS, --}; -- --struct task_ctx { -- unsigned long long dom_mask; /* the domains this task can run on */ -- struct bpf_cpumask __kptr *cpumask; -- unsigned int dom_id; -- unsigned int weight; -- unsigned long long runnable_at; -- unsigned long long runnable_for; -- bool dispatch_local; --}; -- --#endif /* __ATROPOS_H */ -diff --git a/tools/sched_ext/atropos/src/main.rs b/tools/sched_ext/atropos/src/main.rs -deleted file mode 100644 -index 0d313662f713..000000000000 ---- a/tools/sched_ext/atropos/src/main.rs -+++ /dev/null -@@ -1,942 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#[path = "bpf/.output/atropos.skel.rs"] --mod atropos; --pub use atropos::*; --pub mod atropos_sys; -- --use std::cell::Cell; --use std::collections::{BTreeMap, BTreeSet}; --use std::ffi::CStr; --use std::ops::Bound::{Included, Unbounded}; --use std::sync::atomic::{AtomicBool, Ordering}; --use std::sync::Arc; --use std::time::{Duration, SystemTime}; -- --use ::fb_procfs as procfs; --use anyhow::{anyhow, bail, Context, Result}; --use bitvec::prelude::*; --use clap::Parser; --use log::{info, trace, warn}; --use ordered_float::OrderedFloat; -- --/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF --/// part does simple round robin in each domain and the userspace part --/// calculates the load factor of each domain and tells the BPF part how to load --/// balance the domains. -- --/// This scheduler demonstrates dividing scheduling logic between BPF and --/// userspace and using rust to build the userspace part. An earlier variant of --/// this scheduler was used to balance across six domains, each representing a --/// chiplet in a six-chiplet AMD processor, and could match the performance of --/// production setup using CFS. --#[derive(Debug, Parser)] --struct Opts { -- /// Scheduling slice duration in microseconds. -- #[clap(short, long, default_value = "20000")] -- slice_us: u64, -- -- /// Monitoring and load balance interval in seconds. -- #[clap(short, long, default_value = "2.0")] -- interval: f64, -- -- /// Build domains according to how CPUs are grouped at this cache level -- /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id. -- #[clap(short = 'c', long, default_value = "3")] -- cache_level: u32, -- -- /// Instead of using cache locality, set the cpumask for each domain -- /// manually, provide multiple --cpumasks, one for each domain. E.g. -- /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with -- /// the corresponding CPUs belonging to each domain. Each CPU must -- /// belong to precisely one domain. -- #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")] -- cpumasks: Vec, -- -- /// When non-zero, enable greedy task stealing. When a domain is idle, a -- /// cpu will attempt to steal tasks from a domain with at least -- /// greedy_threshold tasks enqueued. These tasks aren't permanently -- /// stolen from the domain. -- #[clap(short, long, default_value = "4")] -- greedy_threshold: u32, -- -- /// The load decay factor. Every interval, the existing load is decayed -- /// by this factor and new load is added. Must be in the range [0.0, -- /// 0.99]. The smaller the value, the more sensitive load calculation -- /// is to recent changes. When 0.0, history is ignored and the load -- /// value from the latest period is used directly. -- #[clap(short, long, default_value = "0.5")] -- load_decay_factor: f64, -- -- /// Disable load balancing. Unless disabled, periodically userspace will -- /// calculate the load factor of each domain and instruct BPF which -- /// processes to move. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- no_load_balance: bool, -- -- /// Put per-cpu kthreads directly into local dsq's. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- kthreads_local: bool, -- -- /// Use FIFO scheduling instead of weighted vtime scheduling. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- fifo_sched: bool, -- -- /// If specified, only tasks which have their scheduling policy set to -- /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all -- /// tasks are switched. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- partial: bool, -- -- /// Enable verbose output including libbpf details. Specify multiple -- /// times to increase verbosity. -- #[clap(short, long, action = clap::ArgAction::Count)] -- verbose: u8, --} -- --fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result { -- Ok(reader -- .read_stat() -- .context("Failed to read procfs")? -- .total_cpu -- .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?) --} -- --fn now_monotonic() -> u64 { -- let mut time = libc::timespec { -- tv_sec: 0, -- tv_nsec: 0, -- }; -- let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) }; -- assert!(ret == 0); -- time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 --} -- --fn clear_map(map: &mut libbpf_rs::Map) { -- // XXX: libbpf_rs has some design flaw that make it impossible to -- // delete while iterating despite it being safe so we alias it here -- let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) }; -- for key in map.keys() { -- let _ = deleter.delete(&key); -- } --} -- --#[derive(Debug)] --struct TaskLoad { -- runnable_for: u64, -- load: f64, --} -- --#[derive(Debug)] --struct TaskInfo { -- pid: i32, -- dom_mask: u64, -- migrated: Cell, --} -- --struct LoadBalancer<'a, 'b, 'c> { -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- -- tasks_by_load: Vec, TaskInfo>>, -- load_avg: f64, -- dom_loads: Vec, -- -- imbal: Vec, -- doms_to_push: BTreeMap, u32>, -- doms_to_pull: BTreeMap, u32>, -- -- nr_lb_data_errors: &'c mut u64, --} -- --impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> { -- const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10; -- const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1; -- const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50; -- -- fn new( -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- nr_lb_data_errors: &'c mut u64, -- ) -> Self { -- Self { -- maps, -- task_loads, -- nr_doms, -- load_decay_factor, -- -- tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(), -- load_avg: 0f64, -- dom_loads: vec![0.0; nr_doms], -- -- imbal: vec![0.0; nr_doms], -- doms_to_pull: BTreeMap::new(), -- doms_to_push: BTreeMap::new(), -- -- nr_lb_data_errors, -- } -- } -- -- fn read_task_loads(&mut self, period: Duration) -> Result<()> { -- let now_mono = now_monotonic(); -- let task_data = self.maps.task_data(); -- let mut this_task_loads = BTreeMap::::new(); -- let mut load_sum = 0.0f64; -- self.dom_loads = vec![0f64; self.nr_doms]; -- -- for key in task_data.keys() { -- if let Some(task_ctx_vec) = task_data -- .lookup(&key, libbpf_rs::MapFlags::ANY) -- .context("Failed to lookup task_data")? -- { -- let task_ctx = -- unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) }; -- let pid = i32::from_ne_bytes( -- key.as_slice() -- .try_into() -- .context("Invalid key length in task_data map")?, -- ); -- -- let (this_at, this_for, weight) = unsafe { -- ( -- std::ptr::read_volatile(&task_ctx.runnable_at as *const u64), -- std::ptr::read_volatile(&task_ctx.runnable_for as *const u64), -- std::ptr::read_volatile(&task_ctx.weight as *const u32), -- ) -- }; -- -- let (mut delta, prev_load) = match self.task_loads.get(&pid) { -- Some(prev) => (this_for - prev.runnable_for, Some(prev.load)), -- None => (this_for, None), -- }; -- -- // Non-zero this_at indicates that the task is currently -- // runnable. Note that we read runnable_at and runnable_for -- // without any synchronization and there is a small window -- // where we end up misaccounting. While this can cause -- // temporary error, it's unlikely to cause any noticeable -- // misbehavior especially given the load value clamping. -- if this_at > 0 && this_at < now_mono { -- delta += now_mono - this_at; -- } -- -- delta = delta.min(period.as_nanos() as u64); -- let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64) -- .clamp(0.0, weight as f64); -- -- let this_load = match prev_load { -- Some(prev_load) => { -- prev_load * self.load_decay_factor -- + this_load * (1.0 - self.load_decay_factor) -- } -- None => this_load, -- }; -- -- this_task_loads.insert( -- pid, -- TaskLoad { -- runnable_for: this_for, -- load: this_load, -- }, -- ); -- -- load_sum += this_load; -- self.dom_loads[task_ctx.dom_id as usize] += this_load; -- // Only record pids that are eligible for load balancing -- if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) { -- continue; -- } -- self.tasks_by_load[task_ctx.dom_id as usize].insert( -- OrderedFloat(this_load), -- TaskInfo { -- pid, -- dom_mask: task_ctx.dom_mask, -- migrated: Cell::new(false), -- }, -- ); -- } -- } -- -- self.load_avg = load_sum / self.nr_doms as f64; -- *self.task_loads = this_task_loads; -- Ok(()) -- } -- -- // To balance dom loads we identify doms with lower and higher load than average -- fn calculate_dom_load_balance(&mut self) -> Result<()> { -- for (dom, dom_load) in self.dom_loads.iter().enumerate() { -- let imbal = dom_load - self.load_avg; -- if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO { -- if imbal > 0f64 { -- self.doms_to_push.insert(OrderedFloat(imbal), dom as u32); -- } else { -- self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32); -- } -- self.imbal[dom] = imbal; -- } -- } -- Ok(()) -- } -- -- // Find the first candidate pid which hasn't already been migrated and -- // can run in @pull_dom. -- fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)> -- where -- I: IntoIterator, &'d TaskInfo)>, -- { -- match tasks_by_load -- .into_iter() -- .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0) -- .next() -- { -- Some((OrderedFloat(load), task)) => Some((*load, task)), -- None => None, -- } -- } -- -- fn pick_victim( -- &self, -- (push_dom, to_push): (u32, f64), -- (pull_dom, to_pull): (u32, f64), -- ) -> Option<(&TaskInfo, f64)> { -- let to_xfer = to_pull.min(to_push); -- -- trace!( -- "considering dom {}@{:.2} -> {}@{:.2}", -- push_dom, -- to_push, -- pull_dom, -- to_pull -- ); -- -- let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs(); -- -- trace!( -- "to_xfer={:.2} tasks_by_load={:?}", -- to_xfer, -- &self.tasks_by_load[push_dom as usize] -- ); -- -- // We want to pick a task to transfer from push_dom to pull_dom to -- // maximize the reduction of load imbalance between the two. IOW, -- // pick a task which has the closest load value to $to_xfer that can -- // be migrated. Find such task by locating the first migratable task -- // while scanning left from $to_xfer and the counterpart while -- // scanning right and picking the better of the two. -- let (load, task, new_imbal) = match ( -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Unbounded, Included(&OrderedFloat(to_xfer)))) -- .rev(), -- pull_dom, -- ), -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Included(&OrderedFloat(to_xfer)), Unbounded)), -- pull_dom, -- ), -- ) { -- (None, None) => return None, -- (Some((load, task)), None) | (None, Some((load, task))) => { -- (load, task, calc_new_imbal(load)) -- } -- (Some((load0, task0)), Some((load1, task1))) => { -- let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1)); -- if new_imbal0 <= new_imbal1 { -- (load0, task0, new_imbal0) -- } else { -- (load1, task1, new_imbal1) -- } -- } -- }; -- -- // If the best candidate can't reduce the imbalance, there's nothing -- // to do for this pair. -- let old_imbal = to_push + to_pull; -- if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal { -- trace!( -- "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal -- ); -- return None; -- } -- -- trace!( -- "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal, -- ); -- -- Some((task, load)) -- } -- -- // Actually execute the load balancing. Concretely this writes pid -> dom -- // entries into the lb_data map for bpf side to consume. -- fn load_balance(&mut self) -> Result<()> { -- clear_map(self.maps.lb_data()); -- -- trace!("imbal={:?}", &self.imbal); -- trace!("doms_to_push={:?}", &self.doms_to_push); -- trace!("doms_to_pull={:?}", &self.doms_to_pull); -- -- // Push from the most imbalanced to least. -- while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() { -- let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO; -- let mut pushed = 0f64; -- -- // Transfer tasks from push_dom to reduce imbalance. -- loop { -- let last_pushed = pushed; -- -- // Pull from the most imbalaned to least. -- let mut doms_to_pull = BTreeMap::<_, _>::new(); -- std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull); -- let mut pull_doms = doms_to_pull.into_iter().rev().collect::>(); -- -- for (to_pull, pull_dom) in pull_doms.iter_mut() { -- if let Some((task, load)) = -- self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull))) -- { -- // Execute migration. -- task.migrated.set(true); -- to_push -= load; -- *to_pull -= load; -- pushed += load; -- -- // Ask BPF code to execute the migration. -- let pid = task.pid; -- let cpid = (pid as libc::pid_t).to_ne_bytes(); -- if let Err(e) = self.maps.lb_data().update( -- &cpid, -- &pull_dom.to_ne_bytes(), -- libbpf_rs::MapFlags::NO_EXIST, -- ) { -- warn!( -- "Failed to update lb_data map for pid={} error={:?}", -- pid, &e -- ); -- *self.nr_lb_data_errors += 1; -- } -- -- // Always break after a successful migration so that -- // the pulling domains are always considered in the -- // descending imbalance order. -- break; -- } -- } -- -- pull_doms -- .into_iter() -- .map(|(k, v)| self.doms_to_pull.insert(k, v)) -- .count(); -- -- // Stop repeating if nothing got transferred or pushed enough. -- if pushed == last_pushed || pushed >= push_max { -- break; -- } -- } -- } -- Ok(()) -- } --} -- --struct Scheduler<'a> { -- skel: AtroposSkel<'a>, -- struct_ops: Option, -- -- nr_cpus: usize, -- nr_doms: usize, -- load_decay_factor: f64, -- balance_load: bool, -- -- proc_reader: procfs::ProcReader, -- -- prev_at: SystemTime, -- prev_total_cpu: procfs::CpuStat, -- task_loads: BTreeMap, -- -- nr_lb_data_errors: u64, --} -- --impl<'a> Scheduler<'a> { -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn parse_cpusets( -- cpumasks: &[String], -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- if cpumasks.len() > atropos_sys::MAX_DOMS as usize { -- bail!( -- "Number of requested DSQs ({}) is greater than MAX_DOMS ({})", -- cpumasks.len(), -- atropos_sys::MAX_DOMS -- ); -- } -- let mut cpus = vec![-1i32; nr_cpus]; -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()]; -- for (dq, cpumask) in cpumasks.iter().enumerate() { -- let hex_str = { -- let mut tmp_str = cpumask -- .strip_prefix("0x") -- .unwrap_or(cpumask) -- .replace('_', ""); -- if tmp_str.len() % 2 != 0 { -- tmp_str = "0".to_string() + &tmp_str; -- } -- tmp_str -- }; -- let byte_vec = hex::decode(&hex_str) -- .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?; -- -- for (index, &val) in byte_vec.iter().rev().enumerate() { -- let mut v = val; -- while v != 0 { -- let lsb = v.trailing_zeros() as usize; -- v &= !(1 << lsb); -- let cpu = index * 8 + lsb; -- if cpu > nr_cpus { -- bail!( -- concat!( -- "Found cpu ({}) in cpumask ({}) which is larger", -- " than the number of cpus on the machine ({})" -- ), -- cpu, -- cpumask, -- nr_cpus -- ); -- } -- if cpus[cpu] != -1 { -- bail!( -- "Found cpu ({}) with dq ({}) but also in cpumask ({})", -- cpu, -- cpus[cpu], -- cpumask -- ); -- } -- cpus[cpu] = dq as i32; -- cpusets[dq].set(cpu, true); -- } -- } -- cpusets[dq].set_uninitialized(false); -- } -- -- for (cpu, &dq) in cpus.iter().enumerate() { -- if dq < 0 { -- bail!( -- "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.", -- cpu -- ); -- } -- } -- -- Ok((cpusets, cpus)) -- } -- -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn cpusets_from_cache( -- level: u32, -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- let mut cpu_to_cache = vec![]; // (cpu_id, cache_id) -- let mut cache_ids = BTreeSet::::new(); -- let mut nr_not_found = 0; -- -- // Build cpu -> cache ID mapping. -- for cpu in 0..nr_cpus { -- let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level); -- let id = match std::fs::read_to_string(&path) { -- Ok(val) => val -- .trim() -- .parse::() -- .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?, -- Err(e) if e.kind() == std::io::ErrorKind::NotFound => { -- nr_not_found += 1; -- 0 -- } -- Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)), -- }; -- -- cpu_to_cache.push(id); -- cache_ids.insert(id); -- } -- -- if nr_not_found > 1 { -- warn!( -- "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0", -- level, nr_not_found, nr_cpus -- ); -- } -- -- // Cache IDs may have holes. Assign consecutive domain IDs to -- // existing cache IDs. -- let mut cache_to_dom = BTreeMap::::new(); -- let mut nr_doms = 0; -- for cache_id in cache_ids.iter() { -- cache_to_dom.insert(*cache_id, nr_doms); -- nr_doms += 1; -- } -- -- if nr_doms > atropos_sys::MAX_DOMS { -- bail!( -- "Total number of doms {} is greater than MAX_DOMS ({})", -- nr_doms, -- atropos_sys::MAX_DOMS -- ); -- } -- -- // Build and return dom -> cpumask and cpu -> dom mappings. -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize]; -- let mut cpu_to_dom = vec![]; -- -- for cpu in 0..nr_cpus { -- let dom_id = cache_to_dom[&cpu_to_cache[cpu]]; -- cpusets[dom_id as usize].set(cpu, true); -- cpu_to_dom.push(dom_id as i32); -- } -- -- Ok((cpusets, cpu_to_dom)) -- } -- -- fn init(opts: &Opts) -> Result { -- // Open the BPF prog first for verification. -- let mut skel_builder = AtroposSkelBuilder::default(); -- skel_builder.obj_builder.debug(opts.verbose > 0); -- let mut skel = skel_builder.open().context("Failed to open BPF program")?; -- -- let nr_cpus = libbpf_rs::num_possible_cpus().unwrap(); -- if nr_cpus > atropos_sys::MAX_CPUS as usize { -- bail!( -- "nr_cpus ({}) is greater than MAX_CPUS ({})", -- nr_cpus, -- atropos_sys::MAX_CPUS -- ); -- } -- -- // Initialize skel according to @opts. -- let (cpusets, cpus) = if opts.cpumasks.len() > 0 { -- Self::parse_cpusets(&opts.cpumasks, nr_cpus)? -- } else { -- Self::cpusets_from_cache(opts.cache_level, nr_cpus)? -- }; -- let nr_doms = cpusets.len(); -- skel.rodata().nr_doms = nr_doms as u32; -- skel.rodata().nr_cpus = nr_cpus as u32; -- -- for (cpu, dom) in cpus.iter().enumerate() { -- skel.rodata().cpu_dom_id_map[cpu] = *dom as u32; -- } -- -- for (dom, cpuset) in cpusets.iter().enumerate() { -- let raw_cpuset_slice = cpuset.as_raw_slice(); -- let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom]; -- let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len()); -- left.clone_from_slice(cpuset.as_raw_slice()); -- let cpumask_str = dom_cpumask_slice -- .iter() -- .take((nr_cpus + 63) / 64) -- .rev() -- .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x)); -- info!( -- "DOM[{:02}] cpumask{} ({} cpus)", -- dom, -- &cpumask_str, -- cpuset.count_ones() -- ); -- } -- -- skel.rodata().slice_us = opts.slice_us; -- skel.rodata().kthreads_local = opts.kthreads_local; -- skel.rodata().fifo_sched = opts.fifo_sched; -- skel.rodata().switch_partial = opts.partial; -- skel.rodata().greedy_threshold = opts.greedy_threshold; -- -- // Attach. -- let mut skel = skel.load().context("Failed to load BPF program")?; -- skel.attach().context("Failed to attach BPF program")?; -- let struct_ops = Some( -- skel.maps_mut() -- .atropos() -- .attach_struct_ops() -- .context("Failed to attach atropos struct ops")?, -- ); -- info!("Atropos Scheduler Attached"); -- -- // Other stuff. -- let mut proc_reader = procfs::ProcReader::new(); -- let prev_total_cpu = read_total_cpu(&mut proc_reader)?; -- -- Ok(Self { -- skel, -- struct_ops, // should be held to keep it attached -- -- nr_cpus, -- nr_doms, -- load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99), -- balance_load: !opts.no_load_balance, -- -- proc_reader, -- -- prev_at: SystemTime::now(), -- prev_total_cpu, -- task_loads: BTreeMap::new(), -- -- nr_lb_data_errors: 0, -- }) -- } -- -- fn get_cpu_busy(&mut self) -> Result { -- let total_cpu = read_total_cpu(&mut self.proc_reader)?; -- let busy = match (&self.prev_total_cpu, &total_cpu) { -- ( -- procfs::CpuStat { -- user_usec: Some(prev_user), -- nice_usec: Some(prev_nice), -- system_usec: Some(prev_system), -- idle_usec: Some(prev_idle), -- iowait_usec: Some(prev_iowait), -- irq_usec: Some(prev_irq), -- softirq_usec: Some(prev_softirq), -- stolen_usec: Some(prev_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- procfs::CpuStat { -- user_usec: Some(curr_user), -- nice_usec: Some(curr_nice), -- system_usec: Some(curr_system), -- idle_usec: Some(curr_idle), -- iowait_usec: Some(curr_iowait), -- irq_usec: Some(curr_irq), -- softirq_usec: Some(curr_softirq), -- stolen_usec: Some(curr_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- ) => { -- let idle_usec = curr_idle - prev_idle; -- let iowait_usec = curr_iowait - prev_iowait; -- let user_usec = curr_user - prev_user; -- let system_usec = curr_system - prev_system; -- let nice_usec = curr_nice - prev_nice; -- let irq_usec = curr_irq - prev_irq; -- let softirq_usec = curr_softirq - prev_softirq; -- let stolen_usec = curr_stolen - prev_stolen; -- -- let busy_usec = -- user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec; -- let total_usec = idle_usec + busy_usec + iowait_usec; -- busy_usec as f64 / total_usec as f64 -- } -- _ => { -- bail!("Some procfs stats are not populated!"); -- } -- }; -- -- self.prev_total_cpu = total_cpu; -- Ok(busy) -- } -- -- fn read_bpf_stats(&mut self) -> Result> { -- let mut maps = self.skel.maps_mut(); -- let stats_map = maps.stats(); -- let mut stats: Vec = Vec::new(); -- let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus]; -- -- for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS { -- let cpu_stat_vec = stats_map -- .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY) -- .with_context(|| format!("Failed to lookup stat {}", stat))? -- .expect("per-cpu stat should exist"); -- let sum = cpu_stat_vec -- .iter() -- .map(|val| { -- u64::from_ne_bytes( -- val.as_slice() -- .try_into() -- .expect("Invalid value length in stat map"), -- ) -- }) -- .sum(); -- stats_map -- .update_percpu( -- &(stat as u32).to_ne_bytes(), -- &zero_vec, -- libbpf_rs::MapFlags::ANY, -- ) -- .context("Failed to zero stat")?; -- stats.push(sum); -- } -- Ok(stats) -- } -- -- fn report( -- &self, -- stats: &Vec, -- cpu_busy: f64, -- processing_dur: Duration, -- load_avg: f64, -- dom_loads: &Vec, -- imbal: &Vec, -- ) { -- let stat = |idx| stats[idx as usize]; -- let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK); -- -- info!( -- "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms", -- cpu_busy * 100.0, -- load_avg, -- stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize], -- stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize], -- self.nr_lb_data_errors, -- processing_dur.as_millis(), -- ); -- -- let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0; -- -- info!( -- "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}", -- total, -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY), -- ); -- -- for i in 0..self.nr_doms { -- info!( -- "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}", -- i, -- dom_loads[i], -- if imbal[i] < 0.0 { -imbal[i] } else { 0.0 }, -- if imbal[i] > 0.0 { imbal[i] } else { 0.0 }, -- ); -- } -- } -- -- fn step(&mut self) -> Result<()> { -- let started_at = std::time::SystemTime::now(); -- let bpf_stats = self.read_bpf_stats()?; -- let cpu_busy = self.get_cpu_busy()?; -- -- let mut lb = LoadBalancer::new( -- self.skel.maps_mut(), -- &mut self.task_loads, -- self.nr_doms, -- self.load_decay_factor, -- &mut self.nr_lb_data_errors, -- ); -- -- lb.read_task_loads(started_at.duration_since(self.prev_at)?)?; -- lb.calculate_dom_load_balance()?; -- -- if self.balance_load { -- lb.load_balance()?; -- } -- -- // Extract fields needed for reporting and drop lb to release -- // mutable borrows. -- let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal); -- -- self.report( -- &bpf_stats, -- cpu_busy, -- std::time::SystemTime::now().duration_since(started_at)?, -- load_avg, -- &dom_loads, -- &imbal, -- ); -- -- self.prev_at = started_at; -- Ok(()) -- } -- -- fn read_bpf_exit_type(&mut self) -> i32 { -- unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) } -- } -- -- fn report_bpf_exit_type(&mut self) -> Result<()> { -- // Report msg if EXT_OPS_EXIT_ERROR. -- match self.read_bpf_exit_type() { -- 0 => Ok(()), -- etype if etype == 2 => { -- let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) }; -- let msg = cstr -- .to_str() -- .context("Failed to convert exit msg to string") -- .unwrap(); -- bail!("BPF exit_type={} msg={}", etype, msg); -- } -- etype => { -- info!("BPF exit_type={}", etype); -- Ok(()) -- } -- } -- } --} -- --impl<'a> Drop for Scheduler<'a> { -- fn drop(&mut self) { -- if let Some(struct_ops) = self.struct_ops.take() { -- drop(struct_ops); -- } -- } --} -- --fn main() -> Result<()> { -- let opts = Opts::parse(); -- -- let llv = match opts.verbose { -- 0 => simplelog::LevelFilter::Info, -- 1 => simplelog::LevelFilter::Debug, -- _ => simplelog::LevelFilter::Trace, -- }; -- let mut lcfg = simplelog::ConfigBuilder::new(); -- lcfg.set_time_level(simplelog::LevelFilter::Error) -- .set_location_level(simplelog::LevelFilter::Off) -- .set_target_level(simplelog::LevelFilter::Off) -- .set_thread_level(simplelog::LevelFilter::Off); -- simplelog::TermLogger::init( -- llv, -- lcfg.build(), -- simplelog::TerminalMode::Stderr, -- simplelog::ColorChoice::Auto, -- )?; -- -- let shutdown = Arc::new(AtomicBool::new(false)); -- let shutdown_clone = shutdown.clone(); -- ctrlc::set_handler(move || { -- shutdown_clone.store(true, Ordering::Relaxed); -- }) -- .context("Error setting Ctrl-C handler")?; -- -- let mut sched = Scheduler::init(&opts)?; -- -- while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 { -- std::thread::sleep(Duration::from_secs_f64(opts.interval)); -- sched.step()?; -- } -- -- sched.report_bpf_exit_type() --} -diff --git a/tools/sched_ext/gnu/stubs.h b/tools/sched_ext/gnu/stubs.h -deleted file mode 100644 -index 719225b16626..000000000000 ---- a/tools/sched_ext/gnu/stubs.h -+++ /dev/null -@@ -1 +0,0 @@ --/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ -diff --git a/tools/sched_ext/scx_common.bpf.h b/tools/sched_ext/scx_common.bpf.h -deleted file mode 100644 -index e56de9dc86f2..000000000000 ---- a/tools/sched_ext/scx_common.bpf.h -+++ /dev/null -@@ -1,288 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __SCHED_EXT_COMMON_BPF_H --#define __SCHED_EXT_COMMON_BPF_H -- --#include "vmlinux.h" --#include --#include --#include --#include "user_exit_info.h" -- --#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ --#define PF_EXITING 0x00000004 --#define CLOCK_MONOTONIC 1 -- --/* -- * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can -- * lead to really confusing misbehaviors. Let's trigger a build failure. -- */ --static inline void ___vmlinux_h_sanity_check___(void) --{ -- _Static_assert(SCX_DSQ_FLAG_BUILTIN, -- "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); --} -- --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; -- --static inline __attribute__((format(printf, 1, 2))) --void ___scx_bpf_error_format_checker(const char *fmt, ...) {} -- --/* -- * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments -- * instead of an array of u64. Note that __param[] must have at least one -- * element to keep the verifier happy. -- */ --#define scx_bpf_error(fmt, args...) \ --({ \ -- static char ___fmt[] = fmt; \ -- unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ -- \ -- _Pragma("GCC diagnostic push") \ -- _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ -- ___bpf_fill(___param, args); \ -- _Pragma("GCC diagnostic pop") \ -- \ -- scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ -- \ -- ___scx_bpf_error_format_checker(fmt, ##args); \ --}) -- --void scx_bpf_switch_all(void) __ksym; --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; --bool scx_bpf_consume(u64 dsq_id) __ksym; --u32 scx_bpf_dispatch_nr_slots(void) __ksym; --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym; --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym; --void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; --s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym; --const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; --const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; --void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; --void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; --bool scx_bpf_task_running(const struct task_struct *p) __ksym; --s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym; --u32 scx_bpf_reenqueue_local(void) __ksym; -- --#define BPF_STRUCT_OPS(name, args...) \ --SEC("struct_ops/"#name) \ --BPF_PROG(name, ##args) -- --#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ --SEC("struct_ops.s/"#name) \ --BPF_PROG(name, ##args) -- --/** -- * MEMBER_VPTR - Obtain the verified pointer to a struct or array member -- * @base: struct or array to index -- * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...) -- * -- * The verifier often gets confused by the instruction sequence the compiler -- * generates for indexing struct fields or arrays. This macro forces the -- * compiler to generate a code sequence which first calculates the byte offset, -- * checks it against the struct or array size and add that byte offset to -- * generate the pointer to the member to help the verifier. -- * -- * Ideally, we want to abort if the calculated offset is out-of-bounds. However, -- * BPF currently doesn't support abort, so evaluate to NULL instead. The caller -- * must check for NULL and take appropriate action to appease the verifier. To -- * avoid confusing the verifier, it's best to check for NULL and dereference -- * immediately. -- * -- * vptr = MEMBER_VPTR(my_array, [i][j]); -- * if (!vptr) -- * return error; -- * *vptr = new_value; -- */ --#define MEMBER_VPTR(base, member) (typeof(base member) *)({ \ -- u64 __base = (u64)base; \ -- u64 __addr = (u64)&(base member) - __base; \ -- asm volatile ( \ -- "if %0 <= %[max] goto +2\n" \ -- "%0 = 0\n" \ -- "goto +1\n" \ -- "%0 += %1\n" \ -- : "+r"(__addr) \ -- : "r"(__base), \ -- [max]"i"(sizeof(base) - sizeof(base member))); \ -- __addr; \ --}) -- --/* -- * BPF core and other generic helpers -- */ -- --/* list and rbtree */ --#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) --#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) -- --void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; --void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; -- --#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) --#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) -- --void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; --struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; --struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, -- struct bpf_rb_node *node) __ksym; --void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, -- bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym; --struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; -- --/* task */ --struct task_struct *bpf_task_from_pid(s32 pid) __ksym; --struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; --void bpf_task_release(struct task_struct *p) __ksym; -- --/* cgroup */ --struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; --void bpf_cgroup_release(struct cgroup *cgrp) __ksym; --struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; -- --/* cpumask */ --struct bpf_cpumask *bpf_cpumask_create(void) __ksym; --struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; --u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; --u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym; -- --/* rcu */ --void bpf_rcu_read_lock(void) __ksym; --void bpf_rcu_read_unlock(void) __ksym; -- --/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */ --struct bpf_iter_num; -- --extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym; --extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym; --extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym; -- --#ifndef bpf_for_each --/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for -- * using BPF open-coded iterators without having to write mundane explicit -- * low-level loop logic. Instead, it provides for()-like generic construct -- * that can be used pretty naturally. E.g., for some hypothetical cgroup -- * iterator, you'd write: -- * -- * struct cgroup *cg, *parent_cg = <...>; -- * -- * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { -- * bpf_printk("Child cgroup id = %d", cg->cgroup_id); -- * if (cg->cgroup_id == 123) -- * break; -- * } -- * -- * I.e., it looks almost like high-level for each loop in other languages, -- * supports continue/break, and is verifiable by BPF verifier. -- * -- * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) -- * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to -- * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int -- * *`, not just `int`. So for integers bpf_for() is more convenient. -- * -- * Note: this macro relies on C99 feature of allowing to declare variables -- * inside for() loop, bound to for() loop lifetime. It also utilizes GCC -- * extension: __attribute__((cleanup())), supported by both GCC and -- * Clang. -- */ --#define bpf_for_each(type, cur, args...) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ -- cleanup(bpf_iter_##type##_destroy))), \ -- /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_##type##_new(&___it, ##args), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_##type##_destroy, (void *)0); \ -- /* iteration and termination check */ \ -- (((cur) = bpf_iter_##type##_next(&___it))); \ --) --#endif /* bpf_for_each */ -- --#ifndef bpf_for --/* bpf_for(i, start, end) implements a for()-like looping construct that sets -- * provided integer variable *i* to values starting from *start* through, -- * but not including, *end*. It also proves to BPF verifier that *i* belongs -- * to range [start, end), so this can be used for accessing arrays without -- * extra checks. -- * -- * Note: *start* and *end* are assumed to be expressions with no side effects -- * and whose values do not change throughout bpf_for() loop execution. They do -- * not have to be statically known or constant, though. -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_for(i, start, end) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, (start), (end)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- ({ \ -- /* iteration step */ \ -- int *___t = bpf_iter_num_next(&___it); \ -- /* termination and bounds check */ \ -- (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ -- }); \ --) --#endif /* bpf_for */ -- --#ifndef bpf_repeat --/* bpf_repeat(N) performs N iterations without exposing iteration number -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_repeat(N) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, 0, (N)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- bpf_iter_num_next(&___it); \ -- /* nothing here */ \ --) --#endif /* bpf_repeat */ -- --#endif /* __SCHED_EXT_COMMON_BPF_H */ -diff --git a/tools/sched_ext/scx_example_central.bpf.c b/tools/sched_ext/scx_example_central.bpf.c -deleted file mode 100644 -index 4cec04b4c2ed..000000000000 ---- a/tools/sched_ext/scx_example_central.bpf.c -+++ /dev/null -@@ -1,334 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A central FIFO sched_ext scheduler which demonstrates the followings: -- * -- * a. Making all scheduling decisions from one CPU: -- * -- * The central CPU is the only one making scheduling decisions. All other -- * CPUs kick the central CPU when they run out of tasks to run. -- * -- * There is one global BPF queue and the central CPU schedules all CPUs by -- * dispatching from the global queue to each CPU's local dsq from dispatch(). -- * This isn't the most straightforward. e.g. It'd be easier to bounce -- * through per-CPU BPF queues. The current design is chosen to maximally -- * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching. -- * -- * b. Tickless operation -- * -- * All tasks are dispatched with the infinite slice which allows stopping the -- * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full -- * parameter. The tickless operation can be observed through -- * /proc/interrupts. -- * -- * Periodic switching is enforced by a periodic timer checking all CPUs and -- * preempting them as necessary. Unfortunately, BPF timer currently doesn't -- * have a way to pin to a specific CPU, so the periodic timer isn't pinned to -- * the central CPU. -- * -- * c. Preemption -- * -- * Kthreads are unconditionally queued to the head of a matching local dsq -- * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always -- * prioritized over user threads, which is required for ensuring forward -- * progress as e.g. the periodic timer may run on a ksoftirqd and if the -- * ksoftirqd gets starved by a user thread, there may not be anything else to -- * vacate that user thread. -- * -- * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the -- * next tasks. -- * -- * This scheduler is designed to maximize usage of various SCX mechanisms. A -- * more practical implementation would likely put the scheduling loop outside -- * the central CPU's dispatch() path and add some form of priority mechanism. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --enum { -- FALLBACK_DSQ_ID = 0, -- MAX_CPUS = 4096, -- MS_TO_NS = 1000LLU * 1000, -- TIMER_INTERVAL_NS = 1 * MS_TO_NS, --}; -- --const volatile bool switch_partial; --const volatile s32 central_cpu; --const volatile u32 nr_cpu_ids = 64; /* !0 for veristat, set during init */ -- --u64 nr_total, nr_locals, nr_queued, nr_lost_pids; --u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries; --u64 nr_overflows; -- --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, s32); --} central_q SEC(".maps"); -- --/* can't use percpu map due to bad lookups */ --static bool cpu_gimme_task[MAX_CPUS]; --static u64 cpu_started_at[MAX_CPUS]; -- --struct central_timer { -- struct bpf_timer timer; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, 1); -- __type(key, u32); -- __type(value, struct central_timer); --} central_timer SEC(".maps"); -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- /* -- * Steer wakeups to the central CPU as much as possible to avoid -- * disturbing other CPUs. It's safe to blindly return the central cpu as -- * select_cpu() is a hint and if @p can't be on it, the kernel will -- * automatically pick a fallback CPU. -- */ -- return central_cpu; --} -- --void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags) --{ -- s32 pid = p->pid; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- /* -- * Push per-cpu kthreads at the head of local dsq's and preempt the -- * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked -- * behind other threads which is necessary for forward progress -- * guarantee as we depend on the BPF timer which may run from ksoftirqd. -- */ -- if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- __sync_fetch_and_add(&nr_locals, 1); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF, -- enq_flags | SCX_ENQ_PREEMPT); -- return; -- } -- -- if (bpf_map_push_elem(¢ral_q, &pid, 0)) { -- __sync_fetch_and_add(&nr_overflows, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_queued, 1); -- -- if (!scx_bpf_task_running(p)) -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); --} -- --static bool dispatch_to_cpu(s32 cpu) --{ -- struct task_struct *p; -- s32 pid; -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (bpf_map_pop_elem(¢ral_q, &pid)) -- break; -- -- __sync_fetch_and_sub(&nr_queued, 1); -- -- p = bpf_task_from_pid(pid); -- if (!p) { -- __sync_fetch_and_add(&nr_lost_pids, 1); -- continue; -- } -- -- /* -- * If we can't run the task at the top, do the dumb thing and -- * bounce it to the fallback dsq. -- */ -- if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- __sync_fetch_and_add(&nr_mismatches, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0); -- bpf_task_release(p); -- continue; -- } -- -- /* dispatch to local and mark that @cpu doesn't need more */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0); -- -- if (cpu != central_cpu) -- scx_bpf_kick_cpu(cpu, 0); -- -- bpf_task_release(p); -- return true; -- } -- -- return false; --} -- --void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (cpu == central_cpu) { -- /* dispatch for all other CPUs first */ -- __sync_fetch_and_add(&nr_dispatches, 1); -- -- bpf_for(cpu, 0, nr_cpu_ids) { -- bool *gimme; -- -- if (!scx_bpf_dispatch_nr_slots()) -- break; -- -- /* central's gimme is never set */ -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme && !*gimme) -- continue; -- -- if (dispatch_to_cpu(cpu)) -- *gimme = false; -- } -- -- /* -- * Retry if we ran out of dispatch buffer slots as we might have -- * skipped some CPUs and also need to dispatch for self. The ext -- * core automatically retries if the local dsq is empty but we -- * can't rely on that as we're dispatching for other CPUs too. -- * Kick self explicitly to retry. -- */ -- if (!scx_bpf_dispatch_nr_slots()) { -- __sync_fetch_and_add(&nr_retries, 1); -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- return; -- } -- -- /* look for a task to run on the central CPU */ -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- dispatch_to_cpu(central_cpu); -- } else { -- bool *gimme; -- -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme) -- *gimme = true; -- -- /* -- * Force dispatch on the scheduling CPU so that it finds a task -- * to run for us. -- */ -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- } --} -- --void BPF_STRUCT_OPS(central_running, struct task_struct *p) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ --} -- --void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = 0; --} -- --static int central_timerfn(void *map, int *key, struct bpf_timer *timer) --{ -- u64 now = bpf_ktime_get_ns(); -- u64 nr_to_kick = nr_queued; -- s32 i; -- -- bpf_for(i, 0, nr_cpu_ids) { -- s32 cpu = (nr_timers + i) % nr_cpu_ids; -- u64 *started_at; -- -- if (cpu == central_cpu) -- continue; -- -- /* kick iff the current one exhausted its slice */ -- started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at && *started_at && -- vtime_before(now, *started_at + SCX_SLICE_DFL)) -- continue; -- -- /* and there's something pending */ -- if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) || -- scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu)) -- ; -- else if (nr_to_kick) -- nr_to_kick--; -- else -- continue; -- -- scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); -- } -- -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- -- bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- __sync_fetch_and_add(&nr_timers, 1); -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(central_init) --{ -- u32 key = 0; -- struct bpf_timer *timer; -- int ret; -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1); -- if (ret) -- return ret; -- -- timer = bpf_map_lookup_elem(¢ral_timer, &key); -- if (!timer) -- return -ESRCH; -- -- bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); -- bpf_timer_set_callback(timer, central_timerfn); -- ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- return ret; --} -- --void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops central_ops = { -- /* -- * We are offloading all scheduling decisions to the central CPU and -- * thus being the last task on a given CPU doesn't mean anything -- * special. Enqueue the last tasks like any other tasks. -- */ -- .flags = SCX_OPS_ENQ_LAST, -- -- .select_cpu = (void *)central_select_cpu, -- .enqueue = (void *)central_enqueue, -- .dispatch = (void *)central_dispatch, -- .running = (void *)central_running, -- .stopping = (void *)central_stopping, -- .init = (void *)central_init, -- .exit = (void *)central_exit, -- .name = "central", --}; -diff --git a/tools/sched_ext/scx_example_central.c b/tools/sched_ext/scx_example_central.c -deleted file mode 100644 -index 7ad591cbdc65..000000000000 ---- a/tools/sched_ext/scx_example_central.c -+++ /dev/null -@@ -1,94 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_central.skel.h" -- --const char help_fmt[] = --"A central FIFO sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-c CPU] [-p]\n" --"\n" --" -c CPU Override the central CPU (default: 0)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_central *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_central__open(); -- assert(skel); -- -- skel->rodata->central_cpu = 0; -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "c:ph")) != -1) { -- switch (opt) { -- case 'c': -- skel->rodata->central_cpu = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_central__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.central_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf("total :%10lu local:%10lu queued:%10lu lost:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_locals, -- skel->bss->nr_queued, -- skel->bss->nr_lost_pids); -- printf("timer :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n", -- skel->bss->nr_timers, -- skel->bss->nr_dispatches, -- skel->bss->nr_mismatches, -- skel->bss->nr_retries); -- printf("overflow:%10lu\n", -- skel->bss->nr_overflows); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_central__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_flatcg.bpf.c b/tools/sched_ext/scx_example_flatcg.bpf.c -deleted file mode 100644 -index f6078b9a681f..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.bpf.c -+++ /dev/null -@@ -1,872 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext flattened cgroup hierarchy scheduler. It implements -- * hierarchical weight-based cgroup CPU control by flattening the cgroup -- * hierarchy into a single layer by compounding the active weight share at each -- * level. Consider the following hierarchy with weights in parentheses: -- * -- * R + A (100) + B (100) -- * | \ C (100) -- * \ D (200) -- * -- * Ignoring the root and threaded cgroups, only B, C and D can contain tasks. -- * Let's say all three have runnable tasks. The total share that each of these -- * three cgroups is entitled to can be calculated by compounding its share at -- * each level. -- * -- * For example, B is competing against C and in that competition its share is -- * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's -- * share in that competition is 200/(200+100) == 1/3. B's eventual share in the -- * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's -- * eventual shaer is the same at 1/6. D is only competing at the top level and -- * its share is 200/(100+200) == 2/3. -- * -- * So, instead of hierarchically scheduling level-by-level, we can consider it -- * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3 -- * and keep updating the eventual shares as the cgroups' runnable states change. -- * -- * This flattening of hierarchy can bring a substantial performance gain when -- * the cgroup hierarchy is nested multiple levels. in a simple benchmark using -- * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it -- * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two -- * apache instances competing with 2:1 weight ratio nested four level deep. -- * -- * However, the gain comes at the cost of not being able to properly handle -- * thundering herd of cgroups. For example, if many cgroups which are nested -- * behind a low priority parent cgroup wake up around the same time, they may be -- * able to consume more CPU cycles than they are entitled to. In many use cases, -- * this isn't a real concern especially given the performance gain. Also, there -- * are ways to mitigate the problem further by e.g. introducing an extra -- * scheduling layer on cgroup delegation boundaries. -- * -- * The scheduler first picks the cgroup to run and then schedule the tasks -- * within by using nested weighted vtime scheduling by default. The -- * cgroup-internal scheduling can be switched to FIFO with the -f option. -- */ --#include "scx_common.bpf.h" --#include "user_exit_info.h" --#include "scx_example_flatcg.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */ --const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL; --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --u64 cvtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, u64); -- __uint(max_entries, FCG_NR_STATS); --} stats SEC(".maps"); -- --static void stat_inc(enum fcg_stat_idx idx) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p)++; --} -- --struct fcg_cpu_ctx { -- u64 cur_cgid; -- u64 cur_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, struct fcg_cpu_ctx); -- __uint(max_entries, 1); --} cpu_ctx SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_cgrp_ctx); --} cgrp_ctx SEC(".maps"); -- --struct cgv_node { -- struct bpf_rb_node rb_node; -- __u64 cvtime; -- __u64 cgid; --}; -- --private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock; --private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node); -- --struct cgv_node_stash { -- struct cgv_node __kptr *node; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, 16384); -- __type(key, __u64); -- __type(value, struct cgv_node_stash); --} cgv_node_stash SEC(".maps"); -- --struct fcg_task_ctx { -- u64 bypassed_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_task_ctx); --} task_ctx SEC(".maps"); -- --/* gets inc'd on weight tree changes to expire the cached hweights */ --unsigned long hweight_gen = 1; -- --static u64 div_round_up(u64 dividend, u64 divisor) --{ -- return (dividend + divisor - 1) / divisor; --} -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) --{ -- struct cgv_node *cgc_a, *cgc_b; -- -- cgc_a = container_of(a, struct cgv_node, rb_node); -- cgc_b = container_of(b, struct cgv_node, rb_node); -- -- return cgc_a->cvtime < cgc_b->cvtime; --} -- --static struct fcg_cpu_ctx *find_cpu_ctx(void) --{ -- struct fcg_cpu_ctx *cpuc; -- u32 idx = 0; -- -- cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx); -- if (!cpuc) { -- scx_bpf_error("cpu_ctx lookup failed"); -- return NULL; -- } -- return cpuc; --} -- --static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id); -- return NULL; -- } -- return cgc; --} -- --static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgrp = bpf_cgroup_ancestor(cgrp, level); -- if (!cgrp) { -- scx_bpf_error("ancestor cgroup lookup failed"); -- return NULL; -- } -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- scx_bpf_error("ancestor cgrp_ctx lookup failed"); -- bpf_cgroup_release(cgrp); -- return cgc; --} -- --static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- int level; -- -- if (!cgc->nr_active) { -- stat_inc(FCG_STAT_HWT_SKIP); -- return; -- } -- -- if (cgc->hweight_gen == hweight_gen) { -- stat_inc(FCG_STAT_HWT_CACHE); -- return; -- } -- -- stat_inc(FCG_STAT_HWT_UPDATES); -- bpf_for(level, 0, cgrp->level + 1) { -- struct fcg_cgrp_ctx *cgc; -- bool is_active; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- -- if (!level) { -- cgc->hweight = FCG_HWEIGHT_ONE; -- cgc->hweight_gen = hweight_gen; -- } else { -- struct fcg_cgrp_ctx *pcgc; -- -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- -- /* -- * We can be oppotunistic here and not grab the -- * cgv_tree_lock and deal with the occasional races. -- * However, hweight updates are already cached and -- * relatively low-frequency. Let's just do the -- * straightforward thing. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- is_active = cgc->nr_active; -- if (is_active) { -- cgc->hweight_gen = pcgc->hweight_gen; -- cgc->hweight = -- div_round_up(pcgc->hweight * cgc->weight, -- pcgc->child_weight_sum); -- } -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!is_active) { -- stat_inc(FCG_STAT_HWT_RACE); -- break; -- } -- } -- } --} -- --static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) --{ -- u64 delta, cvtime, max_budget; -- -- /* -- * A node which is on the rbtree can't be pointed to from elsewhere yet -- * and thus can't be updated and repositioned. Instead, we collect the -- * vtime deltas separately and apply it asynchronously here. -- */ -- delta = cgc->cvtime_delta; -- __sync_fetch_and_sub(&cgc->cvtime_delta, delta); -- cvtime = cgv_node->cvtime + delta; -- -- /* -- * Allow a cgroup to carry the maximum budget proportional to its -- * hweight such that a full-hweight cgroup can immediately take up half -- * of the CPUs at the most while staying at the front of the rbtree. -- */ -- max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / -- (2 * FCG_HWEIGHT_ONE); -- if (vtime_before(cvtime, cvtime_now - max_budget)) -- cvtime = cvtime_now - max_budget; -- -- cgv_node->cvtime = cvtime; --} -- --static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- u64 cgid = cgrp->kn->id; -- -- /* paired with cmpxchg in try_pick_next_cgroup() */ -- if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { -- stat_inc(FCG_STAT_ENQ_SKIP); -- return; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid); -- return; -- } -- -- /* NULL if the node is already on the rbtree */ -- cgv_node = bpf_kptr_xchg(&stash->node, NULL); -- if (!cgv_node) { -- stat_inc(FCG_STAT_ENQ_RACE); -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); --} -- --void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * If select_cpu_dfl() is recommending local enqueue, the target CPU is -- * idle. Follow it and charge the cgroup later in fcg_stopping() after -- * the fact. Use the same mechanism to deal with tasks with custom -- * affinities so that we don't have to worry about per-cgroup dq's -- * containing tasks that can't be executed from some CPUs. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) { -- /* -- * Tell fcg_stopping() that this bypassed the regular scheduling -- * path and should be force charged to the cgroup. 0 is used to -- * indicate that the task isn't bypassing, so if the current -- * runtime is 0, go back by one nanosecond. -- */ -- taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1; -- -- /* -- * The global dq is deprioritized as we don't want to let tasks -- * to boost themselves by constraining its cpumask. The -- * deprioritization is rather severe, so let's not apply that to -- * per-cpu kernel threads. This is ham-fisted. We probably wanna -- * implement per-cgroup fallback dq's instead so that we have -- * more control over when tasks with custom cpumask get issued. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || -- (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) { -- stat_inc(FCG_STAT_LOCAL); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- } else { -- stat_inc(FCG_STAT_GLOBAL); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } -- return; -- } -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- goto out_release; -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 tvtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) -- tvtime = cgc->tvtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, -- tvtime, enq_flags); -- } -- -- cgrp_enqueued(cgrp, cgc); --out_release: -- bpf_cgroup_release(cgrp); --} -- --/* -- * Walk the cgroup tree to update the active weight sums as tasks wake up and -- * sleep. The weight sums are used as the base when calculating the proportion a -- * given cgroup or task is entitled to at each level. -- */ --static void update_active_weight_sums(struct cgroup *cgrp, bool runnable) --{ -- struct fcg_cgrp_ctx *cgc; -- bool updated = false; -- int idx; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- /* -- * In most cases, a hot cgroup would have multiple threads going to -- * sleep and waking up while the whole cgroup stays active. In leaf -- * cgroups, ->nr_runnable which is updated with __sync operations gates -- * ->nr_active updates, so that we don't have to grab the cgv_tree_lock -- * repeatedly for a busy cgroup which is staying active. -- */ -- if (runnable) { -- if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_ACT); -- } else { -- if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_DEACT); -- } -- -- /* -- * If @cgrp is becoming runnable, its hweight should be refreshed after -- * it's added to the weight tree so that enqueue has the up-to-date -- * value. If @cgrp is becoming quiescent, the hweight should be -- * refreshed before it's removed from the weight tree so that the usage -- * charging which happens afterwards has access to the latest value. -- */ -- if (!runnable) -- cgrp_refresh_hweight(cgrp, cgc); -- -- /* propagate upwards */ -- bpf_for(idx, 0, cgrp->level) { -- int level = cgrp->level - idx; -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- bool propagate = false; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- if (level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- } -- -- /* -- * We need the propagation protected by a lock to synchronize -- * against weight changes. There's no reason to drop the lock at -- * each level but bpf_spin_lock() doesn't want any function -- * calls while locked. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- -- if (runnable) { -- if (!cgc->nr_active++) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum += cgc->weight; -- } -- } -- } else { -- if (!--cgc->nr_active) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum -= cgc->weight; -- } -- } -- } -- -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!propagate) -- break; -- } -- -- if (updated) -- __sync_fetch_and_add(&hweight_gen, 1); -- -- if (runnable) -- cgrp_refresh_hweight(cgrp, cgc); --} -- --void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, true); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) --{ -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- if (fifo_sched) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- /* -- * @cgc->tvtime_now always progresses forward as tasks start -- * executing. The test and update can be performed concurrently -- * from multiple CPUs and thus racy. Any error should be -- * contained and temporary. Let's just live with it. -- */ -- if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) -- cgc->tvtime_now = p->scx.dsq_vtime; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- /* scale the execution time by the inverse of the weight and charge */ -- if (!fifo_sched) -- p->scx.dsq_vtime += -- (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- if (!taskc->bypassed_at) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- __sync_fetch_and_add(&cgc->cvtime_delta, -- p->se.sum_exec_runtime - taskc->bypassed_at); -- taskc->bypassed_at = 0; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, false); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) --{ -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- if (cgrp->level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1); -- if (!pcgc) -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- if (pcgc && cgc->nr_active) -- pcgc->child_weight_sum += (s64)weight - cgc->weight; -- cgc->weight = weight; -- bpf_spin_unlock(&cgv_tree_lock); --} -- --static bool try_pick_next_cgroup(u64 *cgidp) --{ -- struct bpf_rb_node *rb_node; -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 cgid; -- -- /* pop the front cgroup and wind cvtime_now accordingly */ -- bpf_spin_lock(&cgv_tree_lock); -- -- rb_node = bpf_rbtree_first(&cgv_tree); -- if (!rb_node) { -- bpf_spin_unlock(&cgv_tree_lock); -- stat_inc(FCG_STAT_PNC_NO_CGRP); -- *cgidp = 0; -- return true; -- } -- -- rb_node = bpf_rbtree_remove(&cgv_tree, rb_node); -- bpf_spin_unlock(&cgv_tree_lock); -- -- cgv_node = container_of(rb_node, struct cgv_node, rb_node); -- cgid = cgv_node->cgid; -- -- if (vtime_before(cvtime_now, cgv_node->cvtime)) -- cvtime_now = cgv_node->cvtime; -- -- /* -- * If lookup fails, the cgroup's gone. Free and move on. See -- * fcg_cgroup_exit(). -- */ -- cgrp = bpf_cgroup_from_id(cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- if (!scx_bpf_consume(cgid)) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_EMPTY); -- goto out_stash; -- } -- -- /* -- * Successfully consumed from the cgroup. This will be our current -- * cgroup for the new slice. Refresh its hweight. -- */ -- cgrp_refresh_hweight(cgrp, cgc); -- -- bpf_cgroup_release(cgrp); -- -- /* -- * As the cgroup may have more tasks, add it back to the rbtree. Note -- * that here we charge the full slice upfront and then exact later -- * according to the actual consumption. This prevents lowpri thundering -- * herd from saturating the machine. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- -- *cgidp = cgid; -- stat_inc(FCG_STAT_PNC_NEXT); -- return true; -- --out_stash: -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- /* -- * Paired with cmpxchg in cgrp_enqueued(). If they see the following -- * transition, they'll enqueue the cgroup. If they are earlier, we'll -- * see their task in the dq below and requeue the cgroup. -- */ -- __sync_val_compare_and_swap(&cgc->queued, 1, 0); -- -- if (scx_bpf_dsq_nr_queued(cgid)) { -- bpf_spin_lock(&cgv_tree_lock); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- goto out_free; -- } -- } -- -- return false; -- --out_free: -- bpf_obj_drop(cgv_node); -- return false; --} -- --void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) --{ -- struct fcg_cpu_ctx *cpuc; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 now = bpf_ktime_get_ns(); -- -- cpuc = find_cpu_ctx(); -- if (!cpuc) -- return; -- -- if (!cpuc->cur_cgid) -- goto pick_next_cgroup; -- -- if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { -- if (scx_bpf_consume(cpuc->cur_cgid)) { -- stat_inc(FCG_STAT_CNS_KEEP); -- return; -- } -- stat_inc(FCG_STAT_CNS_EMPTY); -- } else { -- stat_inc(FCG_STAT_CNS_EXPIRE); -- } -- -- /* -- * The current cgroup is expiring. It was already charged a full slice. -- * Calculate the actual usage and accumulate the delta. -- */ -- cgrp = bpf_cgroup_from_id(cpuc->cur_cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_CNS_GONE); -- goto pick_next_cgroup; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (cgc) { -- /* -- * We want to update the vtime delta and then look for the next -- * cgroup to execute but the latter needs to be done in a loop -- * and we can't keep the lock held. Oh well... -- */ -- bpf_spin_lock(&cgv_tree_lock); -- __sync_fetch_and_add(&cgc->cvtime_delta, -- (cpuc->cur_at + cgrp_slice_ns - now) * -- FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- stat_inc(FCG_STAT_CNS_GONE); -- } -- -- bpf_cgroup_release(cgrp); -- --pick_next_cgroup: -- cpuc->cur_at = now; -- -- if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { -- cpuc->cur_cgid = 0; -- return; -- } -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_pick_next_cgroup(&cpuc->cur_cgid)) -- break; -- } --} -- --s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct fcg_task_ctx *taskc; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- taskc = bpf_task_storage_get(&task_ctx, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!taskc) -- return -ENOMEM; -- -- taskc->bypassed_at = 0; -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp, -- struct scx_cgroup_init_args *args) --{ -- struct fcg_cgrp_ctx *cgc; -- struct cgv_node *cgv_node; -- struct cgv_node_stash empty_stash = {}, *stash; -- u64 cgid = cgrp->kn->id; -- int ret; -- -- /* -- * Technically incorrect as cgroup ID is full 64bit while dq ID is -- * 63bit. Should not be a problem in practice and easy to spot in the -- * unlikely case that it breaks. -- */ -- ret = scx_bpf_create_dsq(cgid, -1); -- if (ret) -- return ret; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!cgc) { -- ret = -ENOMEM; -- goto err_destroy_dsq; -- } -- -- cgc->weight = args->weight; -- cgc->hweight = FCG_HWEIGHT_ONE; -- -- ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash, -- BPF_NOEXIST); -- if (ret) { -- if (ret != -ENOMEM) -- scx_bpf_error("unexpected stash creation error (%d)", -- ret); -- goto err_destroy_dsq; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("unexpected cgv_node stash lookup failure"); -- ret = -ENOENT; -- goto err_destroy_dsq; -- } -- -- cgv_node = bpf_obj_new(struct cgv_node); -- if (!cgv_node) { -- ret = -ENOMEM; -- goto err_del_cgv_node; -- } -- -- cgv_node->cgid = cgid; -- cgv_node->cvtime = cvtime_now; -- -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- ret = -EBUSY; -- goto err_drop; -- } -- -- return 0; -- --err_drop: -- bpf_obj_drop(cgv_node); --err_del_cgv_node: -- bpf_map_delete_elem(&cgv_node_stash, &cgid); --err_destroy_dsq: -- scx_bpf_destroy_dsq(cgid); -- return ret; --} -- --void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- -- /* -- * For now, there's no way find and remove the cgv_node if it's on the -- * cgv_tree. Let's drain them in the dispatch path as they get popped -- * off the front of the tree. -- */ -- bpf_map_delete_elem(&cgv_node_stash, &cgid); -- scx_bpf_destroy_dsq(cgid); --} -- --s32 BPF_STRUCT_OPS(fcg_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops flatcg_ops = { -- .enqueue = (void *)fcg_enqueue, -- .dispatch = (void *)fcg_dispatch, -- .runnable = (void *)fcg_runnable, -- .running = (void *)fcg_running, -- .stopping = (void *)fcg_stopping, -- .quiescent = (void *)fcg_quiescent, -- .prep_enable = (void *)fcg_prep_enable, -- .cgroup_set_weight = (void *)fcg_cgroup_set_weight, -- .cgroup_init = (void *)fcg_cgroup_init, -- .cgroup_exit = (void *)fcg_cgroup_exit, -- .init = (void *)fcg_init, -- .exit = (void *)fcg_exit, -- .flags = SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING, -- .name = "flatcg", --}; -diff --git a/tools/sched_ext/scx_example_flatcg.c b/tools/sched_ext/scx_example_flatcg.c -deleted file mode 100644 -index f9c8a5b84a70..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.c -+++ /dev/null -@@ -1,232 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2023 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2023 Tejun Heo -- * Copyright (c) 2023 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_flatcg.h" --#include "scx_example_flatcg.skel.h" -- --#ifndef FILEID_KERNFS --#define FILEID_KERNFS 0xfe --#endif -- --const char help_fmt[] = --"A flattened cgroup hierarchy sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -i INTERVAL Report interval\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --static float read_cpu_util(__u64 *last_sum, __u64 *last_idle) --{ -- FILE *fp; -- char buf[4096]; -- char *line, *cur = NULL, *tok; -- __u64 sum = 0, idle = 0; -- __u64 delta_sum, delta_idle; -- int idx; -- -- fp = fopen("/proc/stat", "r"); -- if (!fp) { -- perror("fopen(\"/proc/stat\")"); -- return 0.0; -- } -- -- if (!fgets(buf, sizeof(buf), fp)) { -- perror("fgets(\"/proc/stat\")"); -- fclose(fp); -- return 0.0; -- } -- fclose(fp); -- -- line = buf; -- for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) { -- char *endp = NULL; -- __u64 v; -- -- if (idx == 0) { -- line = NULL; -- continue; -- } -- v = strtoull(tok, &endp, 0); -- if (!endp || *endp != '\0') { -- fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n", -- idx, tok); -- continue; -- } -- sum += v; -- if (idx == 4) -- idle = v; -- } -- -- delta_sum = sum - *last_sum; -- delta_idle = idle - *last_idle; -- *last_sum = sum; -- *last_idle = idle; -- -- return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0; --} -- --static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats) --{ -- __u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus]; -- __u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); -- -- for (idx = 0; idx < FCG_NR_STATS; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_flatcg *skel; -- struct bpf_link *link; -- struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 }; -- bool dump_cgrps = false; -- __u64 last_cpu_sum = 0, last_cpu_idle = 0; -- __u64 last_stats[FCG_NR_STATS] = {}; -- unsigned long seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_flatcg__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open: %s\n", strerror(errno)); -- return 1; -- } -- -- skel->rodata->nr_cpus = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) { -- double v; -- -- switch (opt) { -- case 's': -- v = strtod(optarg, NULL); -- skel->rodata->cgrp_slice_ns = v * 1000; -- break; -- case 'i': -- v = strtod(optarg, NULL); -- intv_ts.tv_sec = v; -- intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000; -- break; -- case 'd': -- dump_cgrps = true; -- break; -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- case 'h': -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d", -- (double)skel->rodata->cgrp_slice_ns / 1000000.0, -- (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0, -- dump_cgrps); -- -- if (scx_example_flatcg__load(skel)) { -- fprintf(stderr, "Failed to load: %s\n", strerror(errno)); -- return 1; -- } -- -- link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops); -- if (!link) { -- fprintf(stderr, "Failed to attach_struct_ops: %s\n", -- strerror(errno)); -- return 1; -- } -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- __u64 acc_stats[FCG_NR_STATS]; -- __u64 stats[FCG_NR_STATS]; -- float cpu_util; -- int i; -- -- cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle); -- -- fcg_read_stats(skel, acc_stats); -- for (i = 0; i < FCG_NR_STATS; i++) -- stats[i] = acc_stats[i] - last_stats[i]; -- -- memcpy(last_stats, acc_stats, sizeof(acc_stats)); -- -- printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n", -- seq++, cpu_util * 100.0, skel->data->hweight_gen); -- printf(" act:%6llu deact:%6llu local:%6llu global:%6llu\n", -- stats[FCG_STAT_ACT], -- stats[FCG_STAT_DEACT], -- stats[FCG_STAT_LOCAL], -- stats[FCG_STAT_GLOBAL]); -- printf("HWT skip:%6llu race:%6llu cache:%6llu update:%6llu\n", -- stats[FCG_STAT_HWT_SKIP], -- stats[FCG_STAT_HWT_RACE], -- stats[FCG_STAT_HWT_CACHE], -- stats[FCG_STAT_HWT_UPDATES]); -- printf("ENQ skip:%6llu race:%6llu\n", -- stats[FCG_STAT_ENQ_SKIP], -- stats[FCG_STAT_ENQ_RACE]); -- printf("CNS keep:%6llu expire:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_CNS_KEEP], -- stats[FCG_STAT_CNS_EXPIRE], -- stats[FCG_STAT_CNS_EMPTY], -- stats[FCG_STAT_CNS_GONE]); -- printf("PNC nocgrp:%6llu next:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_PNC_NO_CGRP], -- stats[FCG_STAT_PNC_NEXT], -- stats[FCG_STAT_PNC_EMPTY], -- stats[FCG_STAT_PNC_GONE]); -- printf("BAD remove:%6llu\n", -- acc_stats[FCG_STAT_BAD_REMOVAL]); -- -- nanosleep(&intv_ts, NULL); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_flatcg__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_flatcg.h b/tools/sched_ext/scx_example_flatcg.h -deleted file mode 100644 -index 490758ed41f0..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.h -+++ /dev/null -@@ -1,49 +0,0 @@ --#ifndef __SCX_EXAMPLE_FLATCG_H --#define __SCX_EXAMPLE_FLATCG_H -- --enum { -- FCG_HWEIGHT_ONE = 1LLU << 16, --}; -- --enum fcg_stat_idx { -- FCG_STAT_ACT, -- FCG_STAT_DEACT, -- FCG_STAT_LOCAL, -- FCG_STAT_GLOBAL, -- -- FCG_STAT_HWT_UPDATES, -- FCG_STAT_HWT_CACHE, -- FCG_STAT_HWT_SKIP, -- FCG_STAT_HWT_RACE, -- -- FCG_STAT_ENQ_SKIP, -- FCG_STAT_ENQ_RACE, -- -- FCG_STAT_CNS_KEEP, -- FCG_STAT_CNS_EXPIRE, -- FCG_STAT_CNS_EMPTY, -- FCG_STAT_CNS_GONE, -- -- FCG_STAT_PNC_NO_CGRP, -- FCG_STAT_PNC_NEXT, -- FCG_STAT_PNC_EMPTY, -- FCG_STAT_PNC_GONE, -- -- FCG_STAT_BAD_REMOVAL, -- -- FCG_NR_STATS, --}; -- --struct fcg_cgrp_ctx { -- u32 nr_active; -- u32 nr_runnable; -- u32 queued; -- u32 weight; -- u32 hweight; -- u64 child_weight_sum; -- u64 hweight_gen; -- s64 cvtime_delta; -- u64 tvtime_now; --}; -- --#endif /* __SCX_EXAMPLE_FLATCG_H */ -diff --git a/tools/sched_ext/scx_example_pair.bpf.c b/tools/sched_ext/scx_example_pair.bpf.c -deleted file mode 100644 -index 279efe58b777..000000000000 ---- a/tools/sched_ext/scx_example_pair.bpf.c -+++ /dev/null -@@ -1,627 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext core-scheduler which always makes every sibling CPU pair -- * execute from the same CPU cgroup. -- * -- * This scheduler is a minimal implementation and would need some form of -- * priority handling both inside each cgroup and across the cgroups to be -- * practically useful. -- * -- * Each CPU in the system is paired with exactly one other CPU, according to a -- * "stride" value that can be specified when the BPF scheduler program is first -- * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee -- * that they will only ever schedule tasks that belong to the same CPU cgroup. -- * -- * Scheduler Initialization -- * ------------------------ -- * -- * The scheduler BPF program is first initialized from user space, before it is -- * enabled. During this initialization process, each CPU on the system is -- * assigned several values that are constant throughout its runtime: -- * -- * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling -- * decisions. Paired CPUs always schedule tasks from the same -- * CPU cgroup, and synchronize with each other to guarantee -- * that this constraint is not violated. -- * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access -- * a struct pair_ctx object that is shared between the pair. -- * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the -- * pair. Each struct pair_ctx has an active_mask field, -- * which is a bitmap used to indicate whether each core -- * in the pair currently has an actively running task. -- * This index specifies which entry in the bitmap corresponds -- * to each CPU in the pair. -- * -- * During this initialization, the CPUs are paired according to a "stride" that -- * may be specified when invoking the user space program that initializes and -- * loads the scheduler. By default, the stride is 1/2 the total number of CPUs. -- * -- * Tasks and cgroups -- * ----------------- -- * -- * Every cgroup in the system is registered with the scheduler using the -- * pair_cgroup_init() callback, and every task in the system is associated with -- * exactly one cgroup. At a high level, the idea with the pair scheduler is to -- * always schedule tasks from the same cgroup within a given CPU pair. When a -- * task is enqueued (i.e. passed to the pair_enqueue() callback function), its -- * cgroup ID is read from its task struct, and then a corresponding queue map -- * is used to FIFO-enqueue the task for that cgroup. -- * -- * If you look through the implementation of the scheduler, you'll notice that -- * there is quite a bit of complexity involved with looking up the per-cgroup -- * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash -- * BPF hash map that is used to map a cgroup ID to a globally unique ID that's -- * allocated in the BPF program. This is done because we use separate maps to -- * store the FIFO queue of tasks, and the length of that map, per cgroup. This -- * complexity is only present because of current deficiencies in BPF that will -- * soon be addressed. The main point to keep in mind is that newly enqueued -- * tasks are added to their cgroup's FIFO queue. -- * -- * Dispatching tasks -- * ----------------- -- * -- * This section will describe how enqueued tasks are dispatched and scheduled. -- * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is -- * as follows: -- * -- * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is -- * the structure that's used to synchronize amongst the two pair CPUs in their -- * scheduling decisions. After any of the following events have occurred: -- * -- * - The cgroup's slice run has expired, or -- * - The cgroup becomes empty, or -- * - Either CPU in the pair is preempted by a higher priority scheduling class -- * -- * The cgroup transitions to the draining state and stops executing new tasks -- * from the cgroup. -- * -- * 2. If the pair is still executing a task, mark the pair_ctx as draining, and -- * wait for the pair CPU to be preempted. -- * -- * 3. Otherwise, if the pair CPU is not running a task, we can move onto -- * scheduling new tasks. Pop the next cgroup id from the top_q queue. -- * -- * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it. -- * -- * Note again that this scheduling behavior is simple, but the implementation -- * is complex mostly because this it hits several BPF shortcomings and has to -- * work around in often awkward ways. Most of the shortcomings are expected to -- * be resolved in the near future which should allow greatly simplifying this -- * scheduler. -- * -- * Dealing with preemption -- * ----------------------- -- * -- * SCX is the lowest priority sched_class, and could be preempted by them at -- * any time. To address this, the scheduler implements pair_cpu_release() and -- * pair_cpu_acquire() callbacks which are invoked by the core scheduler when -- * the scheduler loses and gains control of the CPU respectively. -- * -- * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and -- * then invoke: -- * -- * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- * -- * This preempts the pair CPU, and waits until it has re-entered the scheduler -- * before returning. This is necessary to ensure that the higher priority -- * sched_class that preempted our scheduler does not schedule a task -- * concurrently with our pair CPU. -- * -- * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption -- * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable -- * pair scheduling. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include "scx_example_pair.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; -- --/* !0 for veristat, set during init */ --const volatile u32 nr_cpu_ids = 64; -- --/* a pair of CPUs stay on a cgroup for this duration */ --const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL; -- --/* cpu ID -> pair cpu ID */ --const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 }; -- --/* cpu ID -> pair_id */ --const volatile u32 pair_id[MAX_CPUS]; -- --/* CPU ID -> CPU # in the pair (0 or 1) */ --const volatile u32 in_pair_idx[MAX_CPUS]; -- --struct pair_ctx { -- struct bpf_spin_lock lock; -- -- /* the cgroup the pair is currently executing */ -- u64 cgid; -- -- /* the pair started executing the current cgroup at */ -- u64 started_at; -- -- /* whether the current cgroup is draining */ -- bool draining; -- -- /* the CPUs that are currently active on the cgroup */ -- u32 active_mask; -- -- /* -- * the CPUs that are currently preempted and running tasks in a -- * different scheduler. -- */ -- u32 preempted_mask; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, MAX_CPUS / 2); -- __type(key, u32); -- __type(value, struct pair_ctx); --} pair_ctx SEC(".maps"); -- --/* queue of cgrp_q's possibly with tasks on them */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- /* -- * Because it's difficult to build strong synchronization encompassing -- * multiple non-trivial operations in BPF, this queue is managed in an -- * opportunistic way so that we guarantee that a cgroup w/ active tasks -- * is always on it but possibly multiple times. Once we have more robust -- * synchronization constructs and e.g. linked list, we should be able to -- * do this in a prettier way but for now just size it big enough. -- */ -- __uint(max_entries, 4 * MAX_CGRPS); -- __type(value, u64); --} top_q SEC(".maps"); -- --/* per-cgroup q which FIFOs the tasks from the cgroup */ --struct cgrp_q { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, MAX_QUEUED); -- __type(value, u32); --}; -- --/* -- * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local -- * storage; however, a cgroup local storage can only be accessed from the BPF -- * progs attached to the cgroup. For now, work around by allocating array of -- * cgrp_q's and then allocating per-cgroup indices. -- * -- * Another caveat: It's difficult to populate a large array of maps statically -- * or from BPF. Initialize it from userland. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, MAX_CGRPS); -- __type(key, s32); -- __array(values, struct cgrp_q); --} cgrp_q_arr SEC(".maps"); -- --static u64 cgrp_q_len[MAX_CGRPS]; -- --/* -- * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be -- * useful to have as a map type. -- */ --static u32 cgrp_q_idx_cursor; --static u64 cgrp_q_idx_busy[MAX_CGRPS]; -- --/* -- * All added up, the following is what we do: -- * -- * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking -- * for a free ID. If not found, fail cgroup creation with -EBUSY. -- * -- * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following -- * cgrp_q_idx_hash. -- * -- * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from -- * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr. -- * -- * This is sadly complicated for something pretty simple. Hopefully, we should -- * be able to simplify in the future. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, MAX_CGRPS); -- __uint(key_size, sizeof(u64)); /* cgrp ID */ -- __uint(value_size, sizeof(s32)); /* cgrp_q idx */ --} cgrp_q_idx_hash SEC(".maps"); -- --/* statistics */ --u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions; --u64 nr_exps, nr_exp_waits, nr_exp_empty; --u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty; -- --struct user_exit_info uei; -- --static bool time_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- struct cgrp_q *cgq; -- s32 pid = p->pid; -- u64 cgid; -- u32 *q_idx; -- u64 *cgq_len; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- cgrp = scx_bpf_task_cgroup(p); -- cgid = cgrp->kn->id; -- bpf_cgroup_release(cgrp); -- -- /* find the cgroup's q and push @p into it */ -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!q_idx) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return; -- } -- -- cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx); -- if (!cgq) { -- scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]", -- cgid, *q_idx); -- return; -- } -- -- if (bpf_map_push_elem(cgq, &pid, 0)) { -- scx_bpf_error("cgroup[%llu] queue overflow", cgid); -- return; -- } -- -- /* bump q len, if going 0 -> 1, queue cgroup into the top_q */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len) { -- scx_bpf_error("MEMBER_VTPR malfunction"); -- return; -- } -- -- if (!__sync_fetch_and_add(cgq_len, 1) && -- bpf_map_push_elem(&top_q, &cgid, 0)) { -- scx_bpf_error("top_q overflow"); -- return; -- } --} -- --static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) --{ -- u32 *vptr; -- -- vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *pairc = bpf_map_lookup_elem(&pair_ctx, vptr); -- if (!(*pairc)) -- return -EINVAL; -- -- vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *mask = 1U << *vptr; -- -- return 0; --} -- --static int try_dispatch(s32 cpu) --{ -- struct pair_ctx *pairc; -- struct bpf_map *cgq_map; -- struct task_struct *p; -- u64 now = bpf_ktime_get_ns(); -- bool kick_pair = false; -- bool expired, pair_preempted; -- u32 *vptr, in_pair_mask; -- s32 pid, q_idx; -- u64 cgid; -- int ret; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) { -- scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]", -- cpu); -- return -ENOENT; -- } -- -- bpf_spin_lock(&pairc->lock); -- pairc->active_mask &= ~in_pair_mask; -- -- expired = time_before(pairc->started_at + pair_batch_dur_ns, now); -- if (expired || pairc->draining) { -- u64 new_cgid = 0; -- -- __sync_fetch_and_add(&nr_exps, 1); -- -- /* -- * We're done with the current cgid. An obvious optimization -- * would be not draining if the next cgroup is the current one. -- * For now, be dumb and always expire. -- */ -- pairc->draining = true; -- -- pair_preempted = pairc->preempted_mask; -- if (pairc->active_mask || pair_preempted) { -- /* -- * The other CPU is still active, or is no longer under -- * our control due to e.g. being preempted by a higher -- * priority sched_class. We want to wait until this -- * cgroup expires, or until control of our pair CPU has -- * been returned to us. -- * -- * If the pair controls its CPU, and the time already -- * expired, kick. When the other CPU arrives at -- * dispatch and clears its active mask, it'll push the -- * pair to the next cgroup and kick this CPU. -- */ -- __sync_fetch_and_add(&nr_exp_waits, 1); -- bpf_spin_unlock(&pairc->lock); -- if (expired && !pair_preempted) -- kick_pair = true; -- goto out_maybe_kick; -- } -- -- bpf_spin_unlock(&pairc->lock); -- -- /* -- * Pick the next cgroup. It'd be easier / cleaner to not drop -- * pairc->lock and use stronger synchronization here especially -- * given that we'll be switching cgroups significantly less -- * frequently than tasks. Unfortunately, bpf_spin_lock can't -- * really protect anything non-trivial. Let's do opportunistic -- * operations instead. -- */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u32 *q_idx; -- u64 *cgq_len; -- -- if (bpf_map_pop_elem(&top_q, &new_cgid)) { -- /* no active cgroup, go idle */ -- __sync_fetch_and_add(&nr_exp_empty, 1); -- return 0; -- } -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid); -- if (!q_idx) -- continue; -- -- /* -- * This is the only place where empty cgroups are taken -- * off the top_q. -- */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len || !*cgq_len) -- continue; -- -- /* -- * If it has any tasks, requeue as we may race and not -- * execute it. -- */ -- bpf_map_push_elem(&top_q, &new_cgid, 0); -- break; -- } -- -- bpf_spin_lock(&pairc->lock); -- -- /* -- * The other CPU may already have started on a new cgroup while -- * we dropped the lock. Make sure that we're still draining and -- * start on the new cgroup. -- */ -- if (pairc->draining && !pairc->active_mask) { -- __sync_fetch_and_add(&nr_cgrp_next, 1); -- pairc->cgid = new_cgid; -- pairc->started_at = now; -- pairc->draining = false; -- kick_pair = true; -- } else { -- __sync_fetch_and_add(&nr_cgrp_coll, 1); -- } -- } -- -- cgid = pairc->cgid; -- pairc->active_mask |= in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- -- /* again, it'd be better to do all these with the lock held, oh well */ -- vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!vptr) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return -ENOENT; -- } -- q_idx = *vptr; -- -- /* claim one task from cgrp_q w/ q_idx */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u64 *cgq_len, len; -- -- cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]); -- if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) { -- /* the cgroup must be empty, expire and repeat */ -- __sync_fetch_and_add(&nr_cgrp_empty, 1); -- bpf_spin_lock(&pairc->lock); -- pairc->draining = true; -- pairc->active_mask &= ~in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- return -EAGAIN; -- } -- -- if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len) -- continue; -- -- break; -- } -- -- cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx); -- if (!cgq_map) { -- scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- if (bpf_map_pop_elem(cgq_map, &pid)) { -- scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- p = bpf_task_from_pid(pid); -- if (p) { -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } else { -- /* we don't handle dequeues, retry on lost tasks */ -- __sync_fetch_and_add(&nr_missing, 1); -- return -EAGAIN; -- } -- --out_maybe_kick: -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } -- return 0; --} -- --void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) --{ -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_dispatch(cpu) != -EAGAIN) -- break; -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask &= ~in_pair_mask; -- /* Kick the pair CPU, unless it was also preempted. */ -- kick_pair = !pairc->preempted_mask; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask |= in_pair_mask; -- pairc->active_mask &= ~in_pair_mask; -- /* Kick the pair CPU if it's still running. */ -- kick_pair = pairc->active_mask; -- pairc->draining = true; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- } -- } -- __sync_fetch_and_add(&nr_preemptions, 1); --} -- --s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 i, q_idx; -- -- bpf_for(i, 0, MAX_CGRPS) { -- q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS; -- if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1)) -- break; -- } -- if (i == MAX_CGRPS) -- return -EBUSY; -- -- if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]); -- if (busy) -- *busy = 0; -- return -EBUSY; -- } -- -- return 0; --} -- --void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 *q_idx; -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (q_idx) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]); -- if (busy) -- *busy = 0; -- bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid); -- } --} -- --s32 BPF_STRUCT_OPS(pair_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops pair_ops = { -- .enqueue = (void *)pair_enqueue, -- .dispatch = (void *)pair_dispatch, -- .cpu_acquire = (void *)pair_cpu_acquire, -- .cpu_release = (void *)pair_cpu_release, -- .cgroup_init = (void *)pair_cgroup_init, -- .cgroup_exit = (void *)pair_cgroup_exit, -- .init = (void *)pair_init, -- .exit = (void *)pair_exit, -- .name = "pair", --}; -diff --git a/tools/sched_ext/scx_example_pair.c b/tools/sched_ext/scx_example_pair.c -deleted file mode 100644 -index 18e032bbc173..000000000000 ---- a/tools/sched_ext/scx_example_pair.c -+++ /dev/null -@@ -1,143 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_pair.h" --#include "scx_example_pair.skel.h" -- --const char help_fmt[] = --"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n" --"execute from the same CPU cgroup.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-S STRIDE] [-p]\n" --"\n" --" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_pair *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 stride, i, opt, outer_fd; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_pair__open(); -- assert(skel); -- -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- /* pair up the earlier half to the latter by default, override with -s */ -- stride = skel->rodata->nr_cpu_ids / 2; -- -- while ((opt = getopt(argc, argv, "S:ph")) != -1) { -- switch (opt) { -- case 'S': -- stride = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- for (i = 0; i < skel->rodata->nr_cpu_ids; i++) { -- if (skel->rodata->pair_cpu[i] < 0) { -- skel->rodata->pair_cpu[i] = i + stride; -- skel->rodata->pair_cpu[i + stride] = i; -- skel->rodata->pair_id[i] = i; -- skel->rodata->pair_id[i + stride] = i; -- skel->rodata->in_pair_idx[i] = 0; -- skel->rodata->in_pair_idx[i + stride] = 1; -- } -- } -- -- assert(!scx_example_pair__load(skel)); -- -- /* -- * Populate the cgrp_q_arr map which is an array containing per-cgroup -- * queues. It'd probably be better to do this from BPF but there are too -- * many to initialize statically and there's no way to dynamically -- * populate from BPF. -- */ -- outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr); -- assert(outer_fd >= 0); -- -- printf("Initializing"); -- for (i = 0; i < MAX_CGRPS; i++) { -- s32 inner_fd; -- -- if (exit_req) -- break; -- -- inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, -- sizeof(u32), MAX_QUEUED, NULL); -- assert(inner_fd >= 0); -- assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY)); -- close(inner_fd); -- -- if (!(i % 10)) -- printf("."); -- fflush(stdout); -- } -- printf("\n"); -- -- /* -- * Fully initialized, attach and run. -- */ -- link = bpf_map__attach_struct_ops(skel->maps.pair_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf(" total:%10lu dispatch:%10lu missing:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_dispatched, -- skel->bss->nr_missing); -- printf(" kicks:%10lu preemptions:%7lu\n", -- skel->bss->nr_kicks, -- skel->bss->nr_preemptions); -- printf(" exp:%10lu exp_wait:%10lu exp_empty:%10lu\n", -- skel->bss->nr_exps, -- skel->bss->nr_exp_waits, -- skel->bss->nr_exp_empty); -- printf("cgnext:%10lu cgcoll:%10lu cgempty:%10lu\n", -- skel->bss->nr_cgrp_next, -- skel->bss->nr_cgrp_coll, -- skel->bss->nr_cgrp_empty); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_pair__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_pair.h b/tools/sched_ext/scx_example_pair.h -deleted file mode 100644 -index f60b824272f7..000000000000 ---- a/tools/sched_ext/scx_example_pair.h -+++ /dev/null -@@ -1,10 +0,0 @@ --#ifndef __SCX_EXAMPLE_PAIR_H --#define __SCX_EXAMPLE_PAIR_H -- --enum { -- MAX_CPUS = 4096, -- MAX_QUEUED = 4096, -- MAX_CGRPS = 4096, --}; -- --#endif /* __SCX_EXAMPLE_PAIR_H */ -diff --git a/tools/sched_ext/scx_example_qmap.bpf.c b/tools/sched_ext/scx_example_qmap.bpf.c -deleted file mode 100644 -index 579ab21ae403..000000000000 ---- a/tools/sched_ext/scx_example_qmap.bpf.c -+++ /dev/null -@@ -1,401 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple five-level FIFO queue scheduler. -- * -- * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets -- * assigned to one depending on its compound weight. Each CPU round robins -- * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from -- * queue0, 2 from queue1, 4 from queue2 and so on. -- * -- * This scheduler demonstrates: -- * -- * - BPF-side queueing using PIDs. -- * - Sleepable per-task storage allocation using ops.prep_enable(). -- * - Using ops.cpu_release() to handle a higher priority scheduling class taking -- * the CPU away. -- * - Core-sched support. -- * -- * This scheduler is primarily for demonstration and testing of sched_ext -- * features and unlikely to be useful for actual workloads. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include -- --char _license[] SEC("license") = "GPL"; -- --const volatile u64 slice_ns = SCX_SLICE_DFL; --const volatile bool switch_partial; --const volatile u32 stall_user_nth; --const volatile u32 stall_kernel_nth; --const volatile u32 dsp_inf_loop_after; --const volatile s32 disallow_tgid; -- --u32 test_error_cnt; -- --struct user_exit_info uei; -- --struct qmap { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, u32); --} queue0 SEC(".maps"), -- queue1 SEC(".maps"), -- queue2 SEC(".maps"), -- queue3 SEC(".maps"), -- queue4 SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, 5); -- __type(key, int); -- __array(values, struct qmap); --} queue_arr SEC(".maps") = { -- .values = { -- [0] = &queue0, -- [1] = &queue1, -- [2] = &queue2, -- [3] = &queue3, -- [4] = &queue4, -- }, --}; -- --/* -- * Per-queue sequence numbers to implement core-sched ordering. -- * -- * Tail seq is assigned to each queued task and incremented. Head seq tracks the -- * sequence number of the latest dispatched task. The distance between the a -- * task's seq and the associated queue's head seq is called the queue distance -- * and used when comparing two tasks for ordering. See qmap_core_sched_before(). -- */ --static u64 core_sched_head_seqs[5]; --static u64 core_sched_tail_seqs[5]; -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local_dsq */ -- u64 core_sched_seq; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --/* Per-cpu dispatch index and remaining count */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(max_entries, 2); -- __type(key, u32); -- __type(value, u64); --} dispatch_idx_cnt SEC(".maps"); -- --/* Statistics */ --unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued; --unsigned long nr_core_sched_execed; -- --s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- struct task_ctx *tctx; -- s32 cpu; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- return cpu; -- -- return prev_cpu; --} -- --static int weight_to_idx(u32 weight) --{ -- /* Coarsely map the compound weight to a FIFO. */ -- if (weight <= 25) -- return 0; -- else if (weight <= 50) -- return 1; -- else if (weight < 200) -- return 2; -- else if (weight < 400) -- return 3; -- else -- return 4; --} -- --void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) --{ -- static u32 user_cnt, kernel_cnt; -- struct task_ctx *tctx; -- u32 pid = p->pid; -- int idx = weight_to_idx(p->scx.weight); -- void *ring; -- -- if (p->flags & PF_KTHREAD) { -- if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth)) -- return; -- } else { -- if (stall_user_nth && !(++user_cnt % stall_user_nth)) -- return; -- } -- -- if (test_error_cnt && !--test_error_cnt) -- scx_bpf_error("test triggering error"); -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * All enqueued tasks must have their core_sched_seq updated for correct -- * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in -- * qmap_ops.flags. -- */ -- tctx->core_sched_seq = core_sched_tail_seqs[idx]++; -- -- /* -- * If qmap_select_cpu() is telling us to or this is the last runnable -- * task on the CPU, enqueue locally. -- */ -- if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) { -- tctx->force_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); -- return; -- } -- -- /* -- * If the task was re-enqueued due to the CPU being preempted by a -- * higher priority scheduling class, just re-enqueue the task directly -- * on the global DSQ. As we want another CPU to pick it up, find and -- * kick an idle CPU. -- */ -- if (enq_flags & SCX_ENQ_REENQ) { -- s32 cpu; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags); -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- return; -- } -- -- ring = bpf_map_lookup_elem(&queue_arr, &idx); -- if (!ring) { -- scx_bpf_error("failed to find ring %d", idx); -- return; -- } -- -- /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */ -- if (bpf_map_push_elem(ring, &pid, 0)) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_enqueued, 1); --} -- --/* -- * The BPF queue map doesn't support removal and sched_ext can handle spurious -- * dispatches. qmap_dequeue() is only used to collect statistics. -- */ --void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags) --{ -- __sync_fetch_and_add(&nr_dequeued, 1); -- if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) -- __sync_fetch_and_add(&nr_core_sched_execed, 1); --} -- --static void update_core_sched_head_seq(struct task_struct *p) --{ -- struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- int idx = weight_to_idx(p->scx.weight); -- -- if (tctx) -- core_sched_head_seqs[idx] = tctx->core_sched_seq; -- else -- scx_bpf_error("task_ctx lookup failed"); --} -- --void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 zero = 0, one = 1; -- u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero); -- u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one); -- void *fifo; -- s32 pid; -- int i; -- -- if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) { -- struct task_struct *p; -- -- /* -- * PID 2 should be kthreadd which should mostly be idle and off -- * the scheduler. Let's keep dispatching it to force the kernel -- * to call this function over and over again. -- */ -- p = bpf_task_from_pid(2); -- if (p) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- if (!idx || !cnt) { -- scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt); -- return; -- } -- -- for (i = 0; i < 5; i++) { -- /* Advance the dispatch cursor and pick the fifo. */ -- if (!*cnt) { -- *idx = (*idx + 1) % 5; -- *cnt = 1 << *idx; -- } -- (*cnt)--; -- -- fifo = bpf_map_lookup_elem(&queue_arr, idx); -- if (!fifo) { -- scx_bpf_error("failed to find ring %llu", *idx); -- return; -- } -- -- /* Dispatch or advance. */ -- if (!bpf_map_pop_elem(fifo, &pid)) { -- struct task_struct *p; -- -- p = bpf_task_from_pid(pid); -- if (p) { -- update_core_sched_head_seq(p); -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- *cnt = 0; -- } --} -- --/* -- * The distance from the head of the queue scaled by the weight of the queue. -- * The lower the number, the older the task and the higher the priority. -- */ --static s64 task_qdist(struct task_struct *p) --{ -- int idx = weight_to_idx(p->scx.weight); -- struct task_ctx *tctx; -- s64 qdist; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return 0; -- } -- -- qdist = tctx->core_sched_seq - core_sched_head_seqs[idx]; -- -- /* -- * As queue index increments, the priority doubles. The queue w/ index 3 -- * is dispatched twice more frequently than 2. Reflect the difference by -- * scaling qdists accordingly. Note that the shift amount needs to be -- * flipped depending on the sign to avoid flipping priority direction. -- */ -- if (qdist >= 0) -- return qdist << (4 - idx); -- else -- return qdist << idx; --} -- --/* -- * This is called to determine the task ordering when core-sched is picking -- * tasks to execute on SMT siblings and should encode about the same ordering as -- * the regular scheduling path. Use the priority-scaled distances from the head -- * of the queues to compare the two tasks which should be consistent with the -- * dispatch path behavior. -- */ --bool BPF_STRUCT_OPS(qmap_core_sched_before, -- struct task_struct *a, struct task_struct *b) --{ -- return task_qdist(a) > task_qdist(b); --} -- --void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- u32 cnt; -- -- /* -- * Called when @cpu is taken by a higher priority scheduling class. This -- * makes @cpu no longer available for executing sched_ext tasks. As we -- * don't want the tasks in @cpu's local dsq to sit there until @cpu -- * becomes available again, re-enqueue them into the global dsq. See -- * %SCX_ENQ_REENQ handling in qmap_enqueue(). -- */ -- cnt = scx_bpf_reenqueue_local(); -- if (cnt) -- __sync_fetch_and_add(&nr_reenqueued, cnt); --} -- --s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (p->tgid == disallow_tgid) -- p->scx.disallow = true; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(qmap_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops qmap_ops = { -- .select_cpu = (void *)qmap_select_cpu, -- .enqueue = (void *)qmap_enqueue, -- .dequeue = (void *)qmap_dequeue, -- .dispatch = (void *)qmap_dispatch, -- .core_sched_before = (void *)qmap_core_sched_before, -- .cpu_release = (void *)qmap_cpu_release, -- .prep_enable = (void *)qmap_prep_enable, -- .init = (void *)qmap_init, -- .exit = (void *)qmap_exit, -- .flags = SCX_OPS_ENQ_LAST, -- .timeout_ms = 5000U, -- .name = "qmap", --}; -diff --git a/tools/sched_ext/scx_example_qmap.c b/tools/sched_ext/scx_example_qmap.c -deleted file mode 100644 -index ccb4814ee61b..000000000000 ---- a/tools/sched_ext/scx_example_qmap.c -+++ /dev/null -@@ -1,107 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_qmap.skel.h" -- --const char help_fmt[] = --"A simple five-level FIFO queue sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" --" -t COUNT Stall every COUNT'th user thread\n" --" -T COUNT Stall every COUNT'th kernel thread\n" --" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n" --" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_qmap *skel; -- struct bpf_link *link; -- int opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_qmap__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) { -- switch (opt) { -- case 's': -- skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; -- break; -- case 'e': -- skel->bss->test_error_cnt = strtoul(optarg, NULL, 0); -- break; -- case 't': -- skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0); -- break; -- case 'T': -- skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0); -- break; -- case 'l': -- skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0); -- break; -- case 'd': -- skel->rodata->disallow_tgid = strtol(optarg, NULL, 0); -- if (skel->rodata->disallow_tgid < 0) -- skel->rodata->disallow_tgid = getpid(); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_qmap__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.qmap_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- long nr_enqueued = skel->bss->nr_enqueued; -- long nr_dispatched = skel->bss->nr_dispatched; -- -- printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n", -- nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, -- skel->bss->nr_reenqueued, skel->bss->nr_dequeued, -- skel->bss->nr_core_sched_execed); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_qmap__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_simple.bpf.c b/tools/sched_ext/scx_example_simple.bpf.c -deleted file mode 100644 -index 4bccca3e2047..000000000000 ---- a/tools/sched_ext/scx_example_simple.bpf.c -+++ /dev/null -@@ -1,128 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple scheduler. -- * -- * By default, it operates as a simple global weighted vtime scheduler and can -- * be switched to FIFO scheduling. It also demonstrates the following niceties. -- * -- * - Statistics tracking how many tasks are queued to local and global dsq's. -- * - Termination notification for userspace. -- * -- * While very simple, this scheduler should work reasonably well on CPUs with a -- * uniform L3 cache topology. While preemption is not implemented, the fact that -- * the scheduling queue is shared across all CPUs means that whatever is at the -- * front of the queue is likely to be executed fairly quickly given enough -- * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads -- * but comes with the usual problems with FIFO scheduling where saturating -- * threads can easily drown out interactive ones. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --static u64 vtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, 2); /* [local, global] */ --} stats SEC(".maps"); -- --static void stat_inc(u32 idx) --{ -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); -- if (cnt_p) -- (*cnt_p)++; --} -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) --{ -- /* -- * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that -- * running @p on its CPU directly shouldn't affect fairness. Just queue -- * it on the local FIFO. -- */ -- if (enq_flags & SCX_ENQ_LOCAL) { -- stat_inc(0); /* count local queueing */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- return; -- } -- -- stat_inc(1); /* count global queueing */ -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) -- vtime = vtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --void BPF_STRUCT_OPS(simple_running, struct task_struct *p) --{ -- if (fifo_sched) -- return; -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(vtime_now, p->scx.dsq_vtime)) -- vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --s32 BPF_STRUCT_OPS(simple_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .running = (void *)simple_running, -- .stopping = (void *)simple_stopping, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", --}; -diff --git a/tools/sched_ext/scx_example_simple.c b/tools/sched_ext/scx_example_simple.c -deleted file mode 100644 -index 486b401f7c95..000000000000 ---- a/tools/sched_ext/scx_example_simple.c -+++ /dev/null -@@ -1,101 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_simple.skel.h" -- --const char help_fmt[] = --"A simple sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-f] [-p]\n" --"\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int simple) --{ -- exit_req = 1; --} -- --static void read_stats(struct scx_example_simple *skel, u64 *stats) --{ -- int nr_cpus = libbpf_num_possible_cpus(); -- u64 cnts[2][nr_cpus]; -- u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * 2); -- -- for (idx = 0; idx < 2; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_simple *skel; -- struct bpf_link *link; -- u32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_simple__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "fph")) != -1) { -- switch (opt) { -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_simple__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.simple_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- u64 stats[2]; -- -- read_stats(skel, stats); -- printf("local=%lu global=%lu\n", stats[0], stats[1]); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_simple__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_userland.bpf.c b/tools/sched_ext/scx_example_userland.bpf.c -deleted file mode 100644 -index a089bc6bbe86..000000000000 ---- a/tools/sched_ext/scx_example_userland.bpf.c -+++ /dev/null -@@ -1,269 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A minimal userland scheduler. -- * -- * In terms of scheduling, this provides two different types of behaviors: -- * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity. -- * All such tasks are direct-dispatched from the kernel, and are never -- * enqueued in user space. -- * 2. A primitive vruntime scheduler that is implemented in user space, for all -- * other tasks. -- * -- * Some parts of this example user space scheduler could be implemented more -- * efficiently using more complex and sophisticated data structures. For -- * example, rather than using BPF_MAP_TYPE_QUEUE's, -- * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between -- * user space and kernel space. Similarly, we use a simple vruntime-sorted list -- * in user space, but an rbtree could be used instead. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include --#include "scx_common.bpf.h" --#include "scx_example_userland_common.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; --const volatile s32 usersched_pid; -- --/* !0 for veristat, set during init */ --const volatile u32 num_possible_cpus = 64; -- --/* Stats that are printed by user space. */ --u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues; -- --struct user_exit_info uei; -- --/* -- * Whether the user space scheduler needs to be scheduled due to a task being -- * enqueued in user space. -- */ --static bool usersched_needed; -- --/* -- * The map containing tasks that are enqueued in user space from the kernel. -- * -- * This map is drained by the user space scheduler. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, struct scx_userland_enqueued_task); --} enqueued SEC(".maps"); -- --/* -- * The map containing tasks that are dispatched to the kernel from user space. -- * -- * Drained by the kernel in userland_dispatch(). -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, s32); --} dispatched SEC(".maps"); -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local DSQ */ --}; -- --/* Map that contains task-local storage. */ --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --static bool is_usersched_task(const struct task_struct *p) --{ -- return p->pid == usersched_pid; --} -- --static bool keep_in_kernel(const struct task_struct *p) --{ -- return p->nr_cpus_allowed < num_possible_cpus; --} -- --static struct task_struct *usersched_task(void) --{ -- struct task_struct *p; -- -- p = bpf_task_from_pid(usersched_pid); -- /* -- * Should never happen -- the usersched task should always be managed -- * by sched_ext. -- */ -- if (!p) { -- scx_bpf_error("Failed to find usersched task %d", usersched_pid); -- /* -- * We should never hit this path, and we error out of the -- * scheduler above just in case, so the scheduler will soon be -- * be evicted regardless. So as to simplify the logic in the -- * caller to not have to check for NULL, return an acquired -- * reference to the current task here rather than NULL. -- */ -- return bpf_task_acquire(bpf_get_current_task_btf()); -- } -- -- return p; --} -- --s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- if (keep_in_kernel(p)) { -- s32 cpu; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to look up task-local storage for %s", p->comm); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- tctx->force_local = true; -- return cpu; -- } -- } -- -- return prev_cpu; --} -- --static void dispatch_user_scheduler(void) --{ -- struct task_struct *p; -- -- usersched_needed = false; -- p = usersched_task(); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); --} -- --static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags) --{ -- struct scx_userland_enqueued_task task; -- -- memset(&task, 0, sizeof(task)); -- task.pid = p->pid; -- task.sum_exec_runtime = p->se.sum_exec_runtime; -- task.weight = p->scx.weight; -- -- if (bpf_map_push_elem(&enqueued, &task, 0)) { -- /* -- * If we fail to enqueue the task in user space, put it -- * directly on the global DSQ. -- */ -- __sync_fetch_and_add(&nr_failed_enqueues, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- __sync_fetch_and_add(&nr_user_enqueues, 1); -- usersched_needed = true; -- } --} -- --void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags) --{ -- if (keep_in_kernel(p)) { -- u64 dsq_id = SCX_DSQ_GLOBAL; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to lookup task ctx for %s", p->comm); -- return; -- } -- -- if (tctx->force_local) -- dsq_id = SCX_DSQ_LOCAL; -- tctx->force_local = false; -- scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags); -- __sync_fetch_and_add(&nr_kernel_enqueues, 1); -- return; -- } else if (!is_usersched_task(p)) { -- enqueue_task_in_user_space(p, enq_flags); -- } --} -- --void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (usersched_needed) -- dispatch_user_scheduler(); -- -- bpf_repeat(4096) { -- s32 pid; -- struct task_struct *p; -- -- if (bpf_map_pop_elem(&dispatched, &pid)) -- break; -- -- /* -- * The task could have exited by the time we get around to -- * dispatching it. Treat this as a normal occurrence, and simply -- * move onto the next iteration. -- */ -- p = bpf_task_from_pid(pid); -- if (!p) -- continue; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } --} -- --s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(userland_init) --{ -- if (num_possible_cpus == 0) { -- scx_bpf_error("User scheduler # CPUs uninitialized (%d)", -- num_possible_cpus); -- return -EINVAL; -- } -- -- if (usersched_pid <= 0) { -- scx_bpf_error("User scheduler pid uninitialized (%d)", -- usersched_pid); -- return -EINVAL; -- } -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops userland_ops = { -- .select_cpu = (void *)userland_select_cpu, -- .enqueue = (void *)userland_enqueue, -- .dispatch = (void *)userland_dispatch, -- .prep_enable = (void *)userland_prep_enable, -- .init = (void *)userland_init, -- .exit = (void *)userland_exit, -- .timeout_ms = 3000, -- .name = "userland", --}; -diff --git a/tools/sched_ext/scx_example_userland.c b/tools/sched_ext/scx_example_userland.c -deleted file mode 100644 -index 4152b1e65fe1..000000000000 ---- a/tools/sched_ext/scx_example_userland.c -+++ /dev/null -@@ -1,402 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext user space scheduler which provides vruntime semantics -- * using a simple ordered-list implementation. -- * -- * Each CPU in the system resides in a single, global domain. This precludes -- * the need to do any load balancing between domains. The scheduler could -- * easily be extended to support multiple domains, with load balancing -- * happening in user space. -- * -- * Any task which has any CPU affinity is scheduled entirely in BPF. This -- * program only schedules tasks which may run on any CPU. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include "user_exit_info.h" --#include "scx_example_userland_common.h" --#include "scx_example_userland.skel.h" -- --const char help_fmt[] = --"A minimal userland sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-b BATCH] [-p]\n" --"\n" --" -b BATCH The number of tasks to batch when dispatching (default: 8)\n" --" -p Don't switch all, switch only tasks on SCHED_EXT policy\n" --" -h Display this help and exit\n"; -- --/* Defined in UAPI */ --#define SCHED_EXT 7 -- --/* Number of tasks to batch when dispatching to user space. */ --static __u32 batch_size = 8; -- --static volatile int exit_req; --static int enqueued_fd, dispatched_fd; -- --static struct scx_example_userland *skel; --static struct bpf_link *ops_link; -- --/* Stats collected in user space. */ --static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches; -- --/* The data structure containing tasks that are enqueued in user space. */ --struct enqueued_task { -- LIST_ENTRY(enqueued_task) entries; -- __u64 sum_exec_runtime; -- double vruntime; --}; -- --/* -- * Use a vruntime-sorted list to store tasks. This could easily be extended to -- * a more optimal data structure, such as an rbtree as is done in CFS. We -- * currently elect to use a sorted list to simplify the example for -- * illustrative purposes. -- */ --LIST_HEAD(listhead, enqueued_task); -- --/* -- * A vruntime-sorted list of tasks. The head of the list contains the task with -- * the lowest vruntime. That is, the task that has the "highest" claim to be -- * scheduled. -- */ --static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head); -- --/* -- * The statically allocated array of tasks. We use a statically allocated list -- * here to avoid having to allocate on the enqueue path, which could cause a -- * deadlock. A more substantive user space scheduler could e.g. provide a hook -- * for newly enabled tasks that are passed to the scheduler from the -- * .prep_enable() callback to allows the scheduler to allocate on safe paths. -- */ --struct enqueued_task tasks[USERLAND_MAX_TASKS]; -- --static double min_vruntime; -- --static void sigint_handler(int userland) --{ -- exit_req = 1; --} -- --static __u32 task_pid(const struct enqueued_task *task) --{ -- return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task); --} -- --static int dispatch_task(s32 pid) --{ -- int err; -- -- err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d\n", pid); -- exit_req = 1; -- } else { -- nr_vruntime_dispatches++; -- } -- -- return err; --} -- --static struct enqueued_task *get_enqueued_task(__s32 pid) --{ -- if (pid >= USERLAND_MAX_TASKS) -- return NULL; -- -- return &tasks[pid]; --} -- --static double calc_vruntime_delta(__u64 weight, __u64 delta) --{ -- double weight_f = (double)weight / 100.0; -- double delta_f = (double)delta; -- -- return delta_f / weight_f; --} -- --static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task) --{ -- __u64 delta; -- -- delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime; -- -- enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta); -- if (min_vruntime > enqueued->vruntime) -- enqueued->vruntime = min_vruntime; -- enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime; --} -- --static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task) --{ -- struct enqueued_task *curr, *enqueued, *prev; -- -- curr = get_enqueued_task(bpf_task->pid); -- if (!curr) -- return ENOENT; -- -- update_enqueued(curr, bpf_task); -- nr_vruntime_enqueues++; -- -- /* -- * Enqueue the task in a vruntime-sorted list. A more optimal data -- * structure such as an rbtree could easily be used as well. We elect -- * to use a list here simply because it's less code, and thus the -- * example is less convoluted and better serves to illustrate what a -- * user space scheduler could look like. -- */ -- -- if (LIST_EMPTY(&vruntime_head)) { -- LIST_INSERT_HEAD(&vruntime_head, curr, entries); -- return 0; -- } -- -- LIST_FOREACH(enqueued, &vruntime_head, entries) { -- if (curr->vruntime <= enqueued->vruntime) { -- LIST_INSERT_BEFORE(enqueued, curr, entries); -- return 0; -- } -- prev = enqueued; -- } -- -- LIST_INSERT_AFTER(prev, curr, entries); -- -- return 0; --} -- --static void drain_enqueued_map(void) --{ -- while (1) { -- struct scx_userland_enqueued_task task; -- int err; -- -- if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) -- return; -- -- err = vruntime_enqueue(&task); -- if (err) { -- fprintf(stderr, "Failed to enqueue task %d: %s\n", -- task.pid, strerror(err)); -- exit_req = 1; -- return; -- } -- } --} -- --static void dispatch_batch(void) --{ -- __u32 i; -- -- for (i = 0; i < batch_size; i++) { -- struct enqueued_task *task; -- int err; -- __s32 pid; -- -- task = LIST_FIRST(&vruntime_head); -- if (!task) -- return; -- -- min_vruntime = task->vruntime; -- pid = task_pid(task); -- LIST_REMOVE(task, entries); -- err = dispatch_task(pid); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d in %u\n", -- pid, i); -- return; -- } -- } --} -- --static void *run_stats_printer(void *arg) --{ -- while (!exit_req) { -- __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; -- -- nr_failed_enqueues = skel->bss->nr_failed_enqueues; -- nr_kernel_enqueues = skel->bss->nr_kernel_enqueues; -- nr_user_enqueues = skel->bss->nr_user_enqueues; -- total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues; -- -- printf("o-----------------------o\n"); -- printf("| BPF ENQUEUES |\n"); -- printf("|-----------------------|\n"); -- printf("| kern: %10llu |\n", nr_kernel_enqueues); -- printf("| user: %10llu |\n", nr_user_enqueues); -- printf("| failed: %10llu |\n", nr_failed_enqueues); -- printf("| -------------------- |\n"); -- printf("| total: %10llu |\n", total); -- printf("| |\n"); -- printf("|-----------------------|\n"); -- printf("| VRUNTIME / USER |\n"); -- printf("|-----------------------|\n"); -- printf("| enq: %10llu |\n", nr_vruntime_enqueues); -- printf("| disp: %10llu |\n", nr_vruntime_dispatches); -- printf("o-----------------------o\n"); -- printf("\n\n"); -- sleep(1); -- } -- -- return NULL; --} -- --static int spawn_stats_thread(void) --{ -- pthread_t stats_printer; -- -- return pthread_create(&stats_printer, NULL, run_stats_printer, NULL); --} -- --static int bootstrap(int argc, char **argv) --{ -- int err; -- __u32 opt; -- struct sched_param sched_param = { -- .sched_priority = sched_get_priority_max(SCHED_EXT), -- }; -- bool switch_partial = false; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- /* -- * Enforce that the user scheduler task is managed by sched_ext. The -- * task eagerly drains the list of enqueued tasks in its main work -- * loop, and then yields the CPU. The BPF scheduler only schedules the -- * user space scheduler task when at least one other task in the system -- * needs to be scheduled. -- */ -- err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param); -- if (err) { -- fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err)); -- return err; -- } -- -- while ((opt = getopt(argc, argv, "b:ph")) != -1) { -- switch (opt) { -- case 'b': -- batch_size = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- exit(opt != 'h'); -- } -- } -- -- /* -- * It's not always safe to allocate in a user space scheduler, as an -- * enqueued task could hold a lock that we require in order to be able -- * to allocate. -- */ -- err = mlockall(MCL_CURRENT | MCL_FUTURE); -- if (err) { -- fprintf(stderr, "Failed to prefault and lock address space: %s\n", -- strerror(err)); -- return err; -- } -- -- skel = scx_example_userland__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno)); -- return errno; -- } -- skel->rodata->num_possible_cpus = libbpf_num_possible_cpus(); -- assert(skel->rodata->num_possible_cpus > 0); -- skel->rodata->usersched_pid = getpid(); -- assert(skel->rodata->usersched_pid > 0); -- skel->rodata->switch_partial = switch_partial; -- -- err = scx_example_userland__load(skel); -- if (err) { -- fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- enqueued_fd = bpf_map__fd(skel->maps.enqueued); -- dispatched_fd = bpf_map__fd(skel->maps.dispatched); -- assert(enqueued_fd > 0); -- assert(dispatched_fd > 0); -- -- err = spawn_stats_thread(); -- if (err) { -- fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops); -- if (!ops_link) { -- fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno)); -- err = errno; -- goto destroy_skel; -- } -- -- return 0; -- --destroy_skel: -- scx_example_userland__destroy(skel); -- exit_req = 1; -- return err; --} -- --static void sched_main_loop(void) --{ -- while (!exit_req) { -- /* -- * Perform the following work in the main user space scheduler -- * loop: -- * -- * 1. Drain all tasks from the enqueued map, and enqueue them -- * to the vruntime sorted list. -- * -- * 2. Dispatch a batch of tasks from the vruntime sorted list -- * down to the kernel. -- * -- * 3. Yield the CPU back to the system. The BPF scheduler will -- * reschedule the user space scheduler once another task has -- * been enqueued to user space. -- */ -- drain_enqueued_map(); -- dispatch_batch(); -- sched_yield(); -- } --} -- --int main(int argc, char **argv) --{ -- int err; -- -- err = bootstrap(argc, argv); -- if (err) { -- fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err)); -- return err; -- } -- -- sched_main_loop(); -- -- exit_req = 1; -- bpf_link__destroy(ops_link); -- uei_print(&skel->bss->uei); -- scx_example_userland__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_userland_common.h b/tools/sched_ext/scx_example_userland_common.h -deleted file mode 100644 -index 639c6809c5ff..000000000000 ---- a/tools/sched_ext/scx_example_userland_common.h -+++ /dev/null -@@ -1,19 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* Copyright (c) 2022 Meta, Inc */ -- --#ifndef __SCX_USERLAND_COMMON_H --#define __SCX_USERLAND_COMMON_H -- --#define USERLAND_MAX_TASKS 8192 -- --/* -- * An instance of a task that has been enqueued by the kernel for consumption -- * by a user space global scheduler thread. -- */ --struct scx_userland_enqueued_task { -- __s32 pid; -- u64 sum_exec_runtime; -- u64 weight; --}; -- --#endif // __SCX_USERLAND_COMMON_H -diff --git a/tools/sched_ext/user_exit_info.h b/tools/sched_ext/user_exit_info.h -deleted file mode 100644 -index e701ef0e0b86..000000000000 ---- a/tools/sched_ext/user_exit_info.h -+++ /dev/null -@@ -1,50 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Define struct user_exit_info which is shared between BPF and userspace parts -- * to communicate exit status and other information. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __USER_EXIT_INFO_H --#define __USER_EXIT_INFO_H -- --struct user_exit_info { -- int type; -- char reason[128]; -- char msg[1024]; --}; -- --#ifdef __bpf__ -- --#include "vmlinux.h" --#include -- --static inline void uei_record(struct user_exit_info *uei, -- const struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason); -- bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg); -- /* use __sync to force memory barrier */ -- __sync_val_compare_and_swap(&uei->type, uei->type, ei->type); --} -- --#else /* !__bpf__ */ -- --static inline bool uei_exited(struct user_exit_info *uei) --{ -- /* use __sync to force memory barrier */ -- return __sync_val_compare_and_swap(&uei->type, -1, -1); --} -- --static inline void uei_print(const struct user_exit_info *uei) --{ -- fprintf(stderr, "EXIT: %s", uei->reason); -- if (uei->msg[0] != '\0') -- fprintf(stderr, " (%s)", uei->msg); -- fputs("\n", stderr); --} -- --#endif /* __bpf__ */ --#endif /* __USER_EXIT_INFO_H */ -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -index ce05928392bf..f47f79079297 100755 ---- a/kernel/sched/hmbird/hmbird.c -+++ b/kernel/sched/hmbird/hmbird.c -@@ -633,14 +633,14 @@ static void init_isolate_cpus(void) - WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -- cpumask_set_cpu(0, iso_masks.big); -- cpumask_set_cpu(1, iso_masks.big); -- cpumask_set_cpu(2, iso_masks.big); -- cpumask_set_cpu(3, iso_masks.big); -- cpumask_set_cpu(4, iso_masks.big); -- cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(0, iso_masks.little); -+ cpumask_set_cpu(1, iso_masks.little); -+ cpumask_set_cpu(2, iso_masks.little); -+ cpumask_set_cpu(3, iso_masks.little); -+ cpumask_set_cpu(4, iso_masks.little); -+ cpumask_set_cpu(5, iso_masks.little); - cpumask_set_cpu(6, iso_masks.partial); -- cpumask_set_cpu(7, iso_masks.ex_free); -+ cpumask_set_cpu(7, iso_masks.big); - } - - extern spinlock_t css_set_lock; -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index 692c9aa0ffa6..c536364f4736 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -10942,7 +10942,6 @@ void sched_move_task(struct task_struct *tsk) - { - int queued, running, queue_flags = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; -- struct task_group *group; - struct rq_flags rf; - struct rq *rq; - -@@ -10958,7 +10957,7 @@ void sched_move_task(struct task_struct *tsk) - if (running) - put_prev_task(rq, tsk); - -- sched_change_group(tsk, group); -+ sched_change_group(tsk); - - if (queued) - enqueue_task(rq, tsk, queue_flags); -diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c -index 3a7bd62ef6b7..e8c98ab86084 100644 ---- a/drivers/cpufreq/cpufreq.c -+++ b/drivers/cpufreq/cpufreq.c -@@ -810,7 +810,7 @@ static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy, - /* - * show_scaling_governor - show the current policy for the specified CPU - */ --static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) -+ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) - { - if (policy->policy == CPUFREQ_POLICY_POWERSAVE) - return sprintf(buf, "powersave\n"); -@@ -825,7 +825,7 @@ static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) - /* - * store_scaling_governor - store policy for the specified CPU - */ --static ssize_t store_scaling_governor(struct cpufreq_policy *policy, -+ssize_t store_scaling_governor(struct cpufreq_policy *policy, - const char *buf, size_t count) - { - char str_governor[16]; diff --git a/oneplus/hmbird/fengchi_OP13-PJZ_A15.patch b/oneplus/hmbird/fengchi_OP13-PJZ_A15.patch deleted file mode 100644 index b62b60b2..00000000 --- a/oneplus/hmbird/fengchi_OP13-PJZ_A15.patch +++ /dev/null @@ -1,21314 +0,0 @@ -diff --git a/Documentation/scheduler/index.rst b/Documentation/scheduler/index.rst -index 0b650bb550e6..3170747226f6 100644 ---- a/Documentation/scheduler/index.rst -+++ b/Documentation/scheduler/index.rst -@@ -19,7 +19,6 @@ Scheduler - sched-nice-design - sched-rt-group - sched-stats -- sched-ext - sched-debug - - text_files -diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst -deleted file mode 100644 -index 84c30b44f104..000000000000 ---- a/Documentation/scheduler/sched-ext.rst -+++ /dev/null -@@ -1,230 +0,0 @@ --========================== --Extensible Scheduler Class --========================== -- --sched_ext is a scheduler class whose behavior can be defined by a set of BPF --programs - the BPF scheduler. -- --* sched_ext exports a full scheduling interface so that any scheduling -- algorithm can be implemented on top. -- --* The BPF scheduler can group CPUs however it sees fit and schedule them -- together, as tasks aren't tied to specific CPUs at the time of wakeup. -- --* The BPF scheduler can be turned on and off dynamically anytime. -- --* The system integrity is maintained no matter what the BPF scheduler does. -- The default scheduling behavior is restored anytime an error is detected, -- a runnable task stalls, or on invoking the SysRq key sequence -- :kbd:`SysRq-S`. -- --Switching to and from sched_ext --=============================== -- --``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and --``tools/sched_ext`` contains the example schedulers. -- --sched_ext is used only when the BPF scheduler is loaded and running. -- --If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be --treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is --loaded. On load, such tasks will be switched to and scheduled by sched_ext. -- --The BPF scheduler can choose to schedule all normal and lower class tasks by --calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this --case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and --``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers, --this mode can be selected with the ``-a`` option. -- --Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or --detection of any internal error including stalled runnable tasks aborts the --BPF scheduler and reverts all tasks back to CFS. -- --.. code-block:: none -- -- # make -j16 -C tools/sched_ext -- # tools/sched_ext/scx_example_simple -- local=0 global=3 -- local=5 global=24 -- local=9 global=44 -- local=13 global=56 -- local=17 global=72 -- ^CEXIT: BPF scheduler unregistered -- --If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler --and whether a given task is on sched_ext can be determined as follows: -- --.. code-block:: none -- -- # cat /sys/kernel/debug/sched/ext -- ops : simple -- enabled : 1 -- switching_all : 1 -- switched_all : 1 -- enable_state : enabled -- -- # grep ext /proc/self/sched -- ext.enabled : 1 -- --The Basics --========== -- --Userspace can implement an arbitrary BPF scheduler by loading a set of BPF --programs that implement ``struct sched_ext_ops``. The only mandatory field --is ``ops.name`` which must be a valid BPF object name. All operations are --optional. The following modified excerpt is from --``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO --scheduler. -- --.. code-block:: c -- -- s32 BPF_STRUCT_OPS(simple_init) -- { -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; -- } -- -- void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) -- { -- if (enq_flags & SCX_ENQ_LOCAL) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags); -- } -- -- void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) -- { -- exit_type = ei->type; -- } -- -- SEC(".struct_ops") -- struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", -- }; -- --Dispatch Queues ----------------- -- --To match the impedance between the scheduler core and the BPF scheduler, --sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a --priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``), --and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage --an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and --``scx_bpf_destroy_dsq()``. -- --A CPU always executes a task from its local DSQ. A task is "dispatched" to a --DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's --local DSQ. -- --When a CPU is looking for the next task to run, if the local DSQ is not --empty, the first task is picked. Otherwise, the CPU tries to consume the --global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()`` --is invoked. -- --Scheduling Cycle ------------------ -- --The following briefly shows how a waking task is scheduled and executed. -- --1. When a task is waking up, ``ops.select_cpu()`` is the first operation -- invoked. This serves two purposes. First, CPU selection optimization -- hint. Second, waking up the selected CPU if idle. -- -- The CPU selected by ``ops.select_cpu()`` is an optimization hint and not -- binding. The actual decision is made at the last step of scheduling. -- However, there is a small performance gain if the CPU -- ``ops.select_cpu()`` returns matches the CPU the task eventually runs on. -- -- A side-effect of selecting a CPU is waking it up from idle. While a BPF -- scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper, -- using ``ops.select_cpu()`` judiciously can be simpler and more efficient. -- -- Note that the scheduler core will ignore an invalid CPU selection, for -- example, if it's outside the allowed cpumask of the task. -- --2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can -- make one of the following decisions: -- -- * Immediately dispatch the task to either the global or local DSQ by -- calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or -- ``SCX_DSQ_LOCAL``, respectively. -- -- * Immediately dispatch the task to a custom DSQ by calling -- ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63. -- -- * Queue the task on the BPF side. -- --3. When a CPU is ready to schedule, it first looks at its local DSQ. If -- empty, it then looks at the global DSQ. If there still isn't a task to -- run, ``ops.dispatch()`` is invoked which can use the following two -- functions to populate the local DSQ. -- -- * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can -- be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``, -- ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()`` -- currently can't be called with BPF locks held, this is being worked on -- and will be supported. ``scx_bpf_dispatch()`` schedules dispatching -- rather than performing them immediately. There can be up to -- ``ops.dispatch_max_batch`` pending tasks. -- -- * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ -- to the dispatching DSQ. This function cannot be called with any BPF -- locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks -- before trying to consume the specified DSQ. -- --4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ, -- the CPU runs the first one. If empty, the following steps are taken: -- -- * Try to consume the global DSQ. If successful, run the task. -- -- * If ``ops.dispatch()`` has dispatched any tasks, retry #3. -- -- * If the previous task is an SCX task and still runnable, keep executing -- it (see ``SCX_OPS_ENQ_LAST``). -- -- * Go idle. -- --Note that the BPF scheduler can always choose to dispatch tasks immediately --in ``ops.enqueue()`` as illustrated in the above simple example. If only the --built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as --a task is never queued on the BPF scheduler and both the local and global --DSQs are consumed automatically. -- --``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use --``scx_bpf_dispatch_vtime()`` for the priority queue. See the function --documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for --more information. -- --Where to Look --============= -- --* ``include/linux/sched/ext.h`` defines the core data structures, ops table -- and constants. -- --* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. -- The functions prefixed with ``scx_bpf_`` can be called from the BPF -- scheduler. -- --* ``tools/sched_ext/`` hosts example BPF scheduler implementations. -- -- * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example -- using a custom DSQ. -- -- * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting -- five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. -- --ABI Instability --=============== -- --The APIs provided by sched_ext to BPF schedulers programs have no stability --guarantees. This includes the ops table callbacks and constants defined in --``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in --``kernel/sched/ext.c``. -- --While we will attempt to provide a relatively stable API surface when --possible, they are subject to change without warning between kernel --versions. -diff --git a/MAINTAINERS b/MAINTAINERS -index 8d13669d7816..1b46ab23da8d 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -19117,8 +19117,6 @@ R: Ben Segall (CONFIG_CFS_BANDWIDTH) - R: Mel Gorman (CONFIG_NUMA_BALANCING) - R: Daniel Bristot de Oliveira (SCHED_DEADLINE) - R: Valentin Schneider (TOPOLOGY) --R: Tejun Heo (SCHED_EXT) --R: David Vernet (SCHED_EXT) - L: linux-kernel@vger.kernel.org - S: Maintained - T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core -@@ -19127,7 +19125,6 @@ F: include/linux/sched.h - F: include/linux/wait.h - F: include/uapi/linux/sched.h - F: kernel/sched/ --F: tools/sched_ext/ - - SCSI LIBSAS SUBSYSTEM - R: John Garry -diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig -index 520a566bea97..27472095b210 100644 ---- a/arch/arm64/configs/defconfig -+++ b/arch/arm64/configs/defconfig -@@ -6,8 +6,7 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_BPF_SYSCALL=y - CONFIG_BPF_JIT=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_BSD_PROCESS_ACCT=y - CONFIG_BSD_PROCESS_ACCT_V3=y -diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig -index dbd881a37733..13780048caca 100644 ---- a/arch/arm64/configs/gki_defconfig -+++ b/arch/arm64/configs/gki_defconfig -@@ -10,8 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y - # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set - CONFIG_BPF_LSM=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_TASKSTATS=y - CONFIG_TASK_DELAY_ACCT=y -diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c -index 4bd028fa7500..5825ce9d6d7b 100644 ---- a/drivers/gpu/drm/msm/msm_drv.c -+++ b/drivers/gpu/drm/msm/msm_drv.c -@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) - } - - sched_set_fifo(ev_thread->worker->task); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3); -+#endif - } - - ret = drm_vblank_init(ddev, priv->num_crtcs); -diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c -index bd001445815e..dcf2e1ebf9c5 100644 ---- a/drivers/tty/sysrq.c -+++ b/drivers/tty/sysrq.c -@@ -524,7 +524,6 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { - NULL, /* P */ - NULL, /* Q */ - NULL, /* R */ -- /* S: May be registered by sched_ext for resetting */ - NULL, /* S */ - NULL, /* T */ - NULL, /* U */ -diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h -index b4c51e798e25..1d24e20f0e35 100644 ---- a/include/asm-generic/vmlinux.lds.h -+++ b/include/asm-generic/vmlinux.lds.h -@@ -131,7 +131,7 @@ - *(__dl_sched_class) \ - *(__rt_sched_class) \ - *(__fair_sched_class) \ -- *(__ext_sched_class) \ -+ *(__hmbird_sched_class) \ - *(__idle_sched_class) \ - __sched_class_lowest = .; - -diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h -index 90f8bd1736a2..df03e0659df0 100644 ---- a/include/linux/cpufreq.h -+++ b/include/linux/cpufreq.h -@@ -44,6 +44,10 @@ enum cpufreq_table_sorting { - CPUFREQ_TABLE_SORTED_DESCENDING - }; - -+ssize_t store_scaling_governor(struct cpufreq_policy *policy, -+ const char *buf, size_t count); -+ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf); -+ - struct cpufreq_cpuinfo { - unsigned int max_freq; - unsigned int min_freq; -diff --git a/include/linux/sched.h b/include/linux/sched.h -index 464131256ddd..39c797c5cc75 100644 ---- a/include/linux/sched.h -+++ b/include/linux/sched.h -@@ -73,7 +73,9 @@ struct task_delay_info; - struct task_group; - struct user_event_mm; - --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - - /* - * Task state bitmask. NOTE! These bits are also -@@ -298,11 +300,6 @@ enum { - - extern void scheduler_tick(void); - --#ifdef CONFIG_SLIM_SCHED --extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer); --extern void stop_shadow_tick_timer(void); --#endif -- - #define MAX_SCHEDULE_TIMEOUT LONG_MAX - - extern long schedule_timeout(long timeout); -@@ -1523,13 +1520,8 @@ struct task_struct { - */ - struct callback_head l1d_flush_kill; - #endif --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, unsigned long sched_prop); -- ANDROID_KABI_USE(2, struct sched_ext_entity *scx); --#else - ANDROID_KABI_RESERVE(1); - ANDROID_KABI_RESERVE(2); --#endif - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); - ANDROID_KABI_RESERVE(5); -@@ -1932,6 +1924,92 @@ static inline int task_nice(const struct task_struct *p) - return PRIO_TO_NICE((p)->static_prio); - } - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline bool task_is_top_task(struct task_struct *p) -+{ -+ return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ ->top_task_prop & TOP_TASK_BITS_MASK); -+} -+ -+static inline int get_top_task_prop(struct task_struct *p) -+{ -+ return get_hmbird_ts(p)->top_task_prop; -+} -+ -+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear) -+{ -+ if (set) -+ get_hmbird_ts(p)->top_task_prop |= set; -+ if (clear) -+ get_hmbird_ts(p)->top_task_prop &= ~clear; -+ return 0; -+} -+ -+static inline void reset_top_task_prop(struct task_struct *p) -+{ -+ get_hmbird_ts(p)->top_task_prop = 0; -+} -+ -+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ entity->sched_prop = sp; -+ } -+ return 0; -+} -+ -+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->sched_prop; -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq) -+{ -+ unsigned long new_dsq; -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq; -+ entity->sched_prop = new_dsq; -+ } -+} -+ -+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK); -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ entity->dsq_sync_ux = val; -+} -+ -+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->dsq_sync_ux; -+ else -+ return 0; -+} -+#endif -+ - extern int can_nice(const struct task_struct *p, const int nice); - extern int task_curr(const struct task_struct *p); - extern int idle_cpu(int cpu); -diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h -deleted file mode 100755 -index b737a00c1373..000000000000 ---- a/include/linux/sched/ext.h -+++ /dev/null -@@ -1,647 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef _LINUX_SCHED_EXT_H --#define _LINUX_SCHED_EXT_H -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --#include -- --enum scx_consts { -- SCX_OPS_NAME_LEN = 128, -- SCX_EXIT_REASON_LEN = 128, -- SCX_EXIT_BT_LEN = 64, -- SCX_EXIT_MSG_LEN = 1024, -- -- SCX_SLICE_DFL = 20 * NSEC_PER_MSEC, -- SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */ --}; -- --/* -- * DSQ (dispatch queue) IDs are 64bit of the format: -- * -- * Bits: [63] [62 .. 0] -- * [ B] [ ID ] -- * -- * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -- * ID: 63 bit ID -- * -- * Built-in IDs: -- * -- * Bits: [63] [62] [61..32] [31 .. 0] -- * [ 1] [ L] [ R ] [ V ] -- * -- * 1: 1 for built-in DSQs. -- * L: 1 for LOCAL_ON DSQ IDs, 0 for others -- * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -- */ --enum scx_dsq_id_flags { -- SCX_DSQ_FLAG_BUILTIN = 1LLU << 63, -- SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -- -- SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0, -- SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, -- SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, -- SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, -- SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, --}; -- --enum scx_exit_type { -- SCX_EXIT_NONE, -- SCX_EXIT_DONE, -- -- SCX_EXIT_UNREG = 64, /* BPF unregistration */ -- SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -- SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ -- SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ --}; -- --/* -- * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is -- * being disabled. -- */ --struct scx_exit_info { -- /* %SCX_EXIT_* - broad category of the exit reason */ -- enum scx_exit_type type; -- /* textual representation of the above */ -- char reason[SCX_EXIT_REASON_LEN]; -- /* number of entries in the backtrace */ -- u32 bt_len; -- /* backtrace if exiting due to an error */ -- unsigned long bt[SCX_EXIT_BT_LEN]; -- /* extra message */ -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --/* sched_ext_ops.flags */ --enum scx_ops_flags { -- /* -- * Keep built-in idle tracking even if ops.update_idle() is implemented. -- */ -- SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, -- -- /* -- * By default, if there are no other task to run on the CPU, ext core -- * keeps running the current task even after its slice expires. If this -- * flag is specified, such tasks are passed to ops.enqueue() with -- * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. -- */ -- SCX_OPS_ENQ_LAST = 1LLU << 1, -- -- /* -- * An exiting task may schedule after PF_EXITING is set. In such cases, -- * bpf_task_from_pid() may not be able to find the task and if the BPF -- * scheduler depends on pid lookup for dispatching, the task will be -- * lost leading to various issues including RCU grace period stalls. -- * -- * To mask this problem, by default, unhashed tasks are automatically -- * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't -- * depend on pid lookups and wants to handle these tasks directly, the -- * following flag can be used. -- */ -- SCX_OPS_ENQ_EXITING = 1LLU << 2, -- -- /* -- * CPU cgroup knob enable flags -- */ -- SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16, /* cpu.weight */ -- -- SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | -- SCX_OPS_ENQ_LAST | -- SCX_OPS_ENQ_EXITING | -- SCX_OPS_CGROUP_KNOB_WEIGHT, --}; -- --/* argument container for ops.enable() and friends */ --struct scx_enable_args { --}; -- --/* argument container for ops->cgroup_init() */ --struct scx_cgroup_init_args { -- /* the weight of the cgroup [1..10000] */ -- u32 weight; --}; -- --enum scx_cpu_preempt_reason { -- /* next task is being scheduled by &sched_class_rt */ -- SCX_CPU_PREEMPT_RT, -- /* next task is being scheduled by &sched_class_dl */ -- SCX_CPU_PREEMPT_DL, -- /* next task is being scheduled by &sched_class_stop */ -- SCX_CPU_PREEMPT_STOP, -- /* unknown reason for SCX being preempted */ -- SCX_CPU_PREEMPT_UNKNOWN, --}; -- --/* -- * Argument container for ops->cpu_acquire(). Currently empty, but may be -- * expanded in the future. -- */ --struct scx_cpu_acquire_args {}; -- --/* argument container for ops->cpu_release() */ --struct scx_cpu_release_args { -- /* the reason the CPU was preempted */ -- enum scx_cpu_preempt_reason reason; -- -- /* the task that's going to be scheduled on the CPU */ -- struct task_struct *task; --}; -- --/** -- * struct sched_ext_ops - Operation table for BPF scheduler implementation -- * -- * Userland can implement an arbitrary scheduling policy by implementing and -- * loading operations in this table. -- */ --struct sched_ext_ops { -- /** -- * select_cpu - Pick the target CPU for a task which is being woken up -- * @p: task being woken up -- * @prev_cpu: the cpu @p was on before sleeping -- * @wake_flags: SCX_WAKE_* -- * -- * Decision made here isn't final. @p may be moved to any CPU while it -- * is getting dispatched for execution later. However, as @p is not on -- * the rq at this point, getting the eventual execution CPU right here -- * saves a small bit of overhead down the line. -- * -- * If an idle CPU is returned, the CPU is kicked and will try to -- * dispatch. While an explicit custom mechanism can be added, -- * select_cpu() serves as the default way to wake up idle CPUs. -- */ -- s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); -- -- /** -- * enqueue - Enqueue a task on the BPF scheduler -- * @p: task being enqueued -- * @enq_flags: %SCX_ENQ_* -- * -- * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() -- * or enqueue on the BPF scheduler. If not directly dispatched, the bpf -- * scheduler owns @p and if it fails to dispatch @p, the task will -- * stall. -- */ -- void (*enqueue)(struct task_struct *p, u64 enq_flags); -- -- /** -- * dequeue - Remove a task from the BPF scheduler -- * @p: task being dequeued -- * @deq_flags: %SCX_DEQ_* -- * -- * Remove @p from the BPF scheduler. This is usually called to isolate -- * the task while updating its scheduling properties (e.g. priority). -- * -- * The ext core keeps track of whether the BPF side owns a given task or -- * not and can gracefully ignore spurious dispatches from BPF side, -- * which makes it safe to not implement this method. However, depending -- * on the scheduling logic, this can lead to confusing behaviors - e.g. -- * scheduling position not being updated across a priority change. -- */ -- void (*dequeue)(struct task_struct *p, u64 deq_flags); -- -- /** -- * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs -- * @cpu: CPU to dispatch tasks for -- * @prev: previous task being switched out -- * -- * Called when a CPU's local dsq is empty. The operation should dispatch -- * one or more tasks from the BPF scheduler into the DSQs using -- * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using -- * scx_bpf_consume(). -- * -- * The maximum number of times scx_bpf_dispatch() can be called without -- * an intervening scx_bpf_consume() is specified by -- * ops.dispatch_max_batch. See the comments on top of the two functions -- * for more details. -- * -- * When not %NULL, @prev is an SCX task with its slice depleted. If -- * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in -- * @prev->scx.flags, it is not enqueued yet and will be enqueued after -- * ops.dispatch() returns. To keep executing @prev, return without -- * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. -- */ -- void (*dispatch)(s32 cpu, struct task_struct *prev); -- -- /** -- * runnable - A task is becoming runnable on its associated CPU -- * @p: task becoming runnable -- * @enq_flags: %SCX_ENQ_* -- * -- * This and the following three functions can be used to track a task's -- * execution state transitions. A task becomes ->runnable() on a CPU, -- * and then goes through one or more ->running() and ->stopping() pairs -- * as it runs on the CPU, and eventually becomes ->quiescent() when it's -- * done running on the CPU. -- * -- * @p is becoming runnable on the CPU because it's -- * -- * - waking up (%SCX_ENQ_WAKEUP) -- * - being moved from another CPU -- * - being restored after temporarily taken off the queue for an -- * attribute change. -- * -- * This and ->enqueue() are related but not coupled. This operation -- * notifies @p's state transition and may not be followed by ->enqueue() -- * e.g. when @p is being dispatched to a remote CPU. Likewise, a task -- * may be ->enqueue()'d without being preceded by this operation e.g. -- * after exhausting its slice. -- */ -- void (*runnable)(struct task_struct *p, u64 enq_flags); -- -- /** -- * running - A task is starting to run on its associated CPU -- * @p: task starting to run -- * -- * See ->runnable() for explanation on the task state notifiers. -- */ -- void (*running)(struct task_struct *p); -- -- /** -- * stopping - A task is stopping execution -- * @p: task stopping to run -- * @runnable: is task @p still runnable? -- * -- * See ->runnable() for explanation on the task state notifiers. If -- * !@runnable, ->quiescent() will be invoked after this operation -- * returns. -- */ -- void (*stopping)(struct task_struct *p, bool runnable); -- -- /** -- * quiescent - A task is becoming not runnable on its associated CPU -- * @p: task becoming not runnable -- * @deq_flags: %SCX_DEQ_* -- * -- * See ->runnable() for explanation on the task state notifiers. -- * -- * @p is becoming quiescent on the CPU because it's -- * -- * - sleeping (%SCX_DEQ_SLEEP) -- * - being moved to another CPU -- * - being temporarily taken off the queue for an attribute change -- * (%SCX_DEQ_SAVE) -- * -- * This and ->dequeue() are related but not coupled. This operation -- * notifies @p's state transition and may not be preceded by ->dequeue() -- * e.g. when @p is being dispatched to a remote CPU. -- */ -- void (*quiescent)(struct task_struct *p, u64 deq_flags); -- -- /** -- * yield - Yield CPU -- * @from: yielding task -- * @to: optional yield target task -- * -- * If @to is NULL, @from is yielding the CPU to other runnable tasks. -- * The BPF scheduler should ensure that other available tasks are -- * dispatched before the yielding task. Return value is ignored in this -- * case. -- * -- * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf -- * scheduler can implement the request, return %true; otherwise, %false. -- */ -- bool (*yield)(struct task_struct *from, struct task_struct *to); -- -- /** -- * core_sched_before - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Used by core-sched to determine the ordering between two tasks. See -- * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on -- * core-sched. -- * -- * Both @a and @b are runnable and may or may not currently be queued on -- * the BPF scheduler. Should return %true if @a should run before @b. -- * %false if there's no required ordering or @b should run before @a. -- * -- * If not specified, the default is ordering them according to when they -- * became runnable. -- */ -- bool (*core_sched_before)(struct task_struct *a,struct task_struct *b); -- -- /** -- * set_weight - Set task weight -- * @p: task to set weight for -- * @weight: new eight [1..10000] -- * -- * Update @p's weight to @weight. -- */ -- void (*set_weight)(struct task_struct *p, u32 weight); -- -- /** -- * set_cpumask - Set CPU affinity -- * @p: task to set CPU affinity for -- * @cpumask: cpumask of cpus that @p can run on -- * -- * Update @p's CPU affinity to @cpumask. -- */ -- void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask); -- -- /** -- * update_idle - Update the idle state of a CPU -- * @cpu: CPU to udpate the idle state for -- * @idle: whether entering or exiting the idle state -- * -- * This operation is called when @rq's CPU goes or leaves the idle -- * state. By default, implementing this operation disables the built-in -- * idle CPU tracking and the following helpers become unavailable: -- * -- * - scx_bpf_select_cpu_dfl() -- * - scx_bpf_test_and_clear_cpu_idle() -- * - scx_bpf_pick_idle_cpu() -- * - scx_bpf_any_idle_cpu() -- * -- * The user also must implement ops.select_cpu() as the default -- * implementation relies on scx_bpf_select_cpu_dfl(). -- * -- * If you keep the built-in idle tracking, specify the -- * %SCX_OPS_KEEP_BUILTIN_IDLE flag. -- */ -- void (*update_idle)(s32 cpu, bool idle); -- -- /** -- * cpu_acquire - A CPU is becoming available to the BPF scheduler -- * @cpu: The CPU being acquired by the BPF scheduler. -- * @args: Acquire arguments, see the struct definition. -- * -- * A CPU that was previously released from the BPF scheduler is now once -- * again under its control. -- */ -- void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); -- -- /** -- * cpu_release - A CPU is taken away from the BPF scheduler -- * @cpu: The CPU being released by the BPF scheduler. -- * @args: Release arguments, see the struct definition. -- * -- * The specified CPU is no longer under the control of the BPF -- * scheduler. This could be because it was preempted by a higher -- * priority sched_class, though there may be other reasons as well. The -- * caller should consult @args->reason to determine the cause. -- */ -- void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -- -- /** -- * cpu_online - A CPU became online -- * @cpu: CPU which just came up -- * -- * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs beforehand. -- */ -- void (*cpu_online)(s32 cpu); -- -- /** -- * cpu_offline - A CPU is going offline -- * @cpu: CPU which is going offline -- * -- * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs afterwards. -- */ -- void (*cpu_offline)(s32 cpu); -- -- /** -- * prep_enable - Prepare to enable BPF scheduling for a task -- * @p: task to prepare BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Either we're loading a BPF scheduler or a new task is being forked. -- * Prepare BPF scheduling for @p. This operation may block and can be -- * used for allocations. -- * -- * Return 0 for success, -errno for failure. An error return while -- * loading will abort loading of the BPF scheduler. During a fork, will -- * abort the specific fork. -- */ -- s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * enable - Enable BPF scheduling for a task -- * @p: task to enable BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Enable @p for BPF scheduling. @p is now in the cgroup specified for -- * the preceding prep_enable() and will start running soon. -- */ -- void (*enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * cancel_enable - Cancel prep_enable() -- * @p: task being canceled -- * @args: enable arguments, see the struct definition -- * -- * @p was prep_enable()'d but failed before reaching enable(). Undo the -- * preparation. -- */ -- void (*cancel_enable)(struct task_struct *p, -- struct scx_enable_args *args); -- -- /** -- * disable - Disable BPF scheduling for a task -- * @p: task to disable BPF scheduling for -- * -- * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. -- * Disable BPF scheduling for @p. -- */ -- void (*disable)(struct task_struct *p); -- -- /* -- * All online ops must come before ops.init(). -- */ -- -- /** -- * init - Initialize the BPF scheduler -- */ -- s32 (*init)(void); -- -- /** -- * exit - Clean up after the BPF scheduler -- * @info: Exit info -- */ -- void (*exit)(struct scx_exit_info *info); -- -- /** -- * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch -- */ -- u32 dispatch_max_batch; -- -- /** -- * flags - %SCX_OPS_* flags -- */ -- u64 flags; -- -- /** -- * timeout_ms - The maximum amount of time, in milliseconds, that a -- * runnable task should be able to wait before being scheduled. The -- * maximum timeout may not exceed the default timeout of 30 seconds. -- * -- * Defaults to the maximum allowed timeout value of 30 seconds. -- */ -- u32 timeout_ms; -- -- /** -- * name - BPF scheduler's name -- * -- * Must be a non-zero valid BPF object name including only isalnum(), -- * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the -- * BPF scheduler is enabled. -- */ -- char name[SCX_OPS_NAME_LEN]; --}; -- --/* -- * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -- * scheduler core and the BPF scheduler. See the documentation for more details. -- */ --struct scx_dispatch_q { -- raw_spinlock_t lock; -- struct list_head fifo; /* processed in dispatching order */ -- struct rb_root_cached priq; /* processed in p->scx.dsq_vtime order */ -- u32 nr; -- u64 id; -- struct llist_node free_node; -- struct rcu_head rcu; --}; -- --/* scx_entity.flags */ --enum scx_ent_flags { -- SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ -- SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -- SCX_TASK_ENQ_LOCAL = 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */ -- -- SCX_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -- SCX_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -- -- SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -- SCX_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -- -- SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ --}; -- --/* scx_entity.dsq_flags */ --enum scx_ent_dsq_flags { -- SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ --}; -- --/* -- * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from -- * everywhere and the following bits track which kfunc sets are currently -- * allowed for %current. This simple per-task tracking works because SCX ops -- * nest in a limited way. BPF will likely implement a way to allow and disallow -- * kfuncs depending on the calling context which will replace this manual -- * mechanism. See scx_kf_allow(). -- */ --enum scx_kf_mask { -- SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ -- /* all non-sleepables may be nested inside INIT and SLEEPABLE */ -- SCX_KF_INIT = 1 << 0, /* running ops.init() */ -- SCX_KF_SLEEPABLE = 1 << 1, /* other sleepable init operations */ -- /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ -- SCX_KF_CPU_RELEASE = 1 << 2, /* ops.cpu_release() */ -- /* ops.dequeue (in REST) may be nested inside DISPATCH */ -- SCX_KF_DISPATCH = 1 << 3, /* ops.dispatch() */ -- SCX_KF_ENQUEUE = 1 << 4, /* ops.enqueue() */ -- SCX_KF_REST = 1 << 5, /* other rq-locked operations */ -- -- __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | -- SCX_KF_ENQUEUE | SCX_KF_REST, -- __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_REST, --}; -- --#define RAVG_HIST_SIZE 5 --struct scx_sched_task_stats { -- u64 mark_start; -- u64 window_start; -- u32 sum; -- u32 sum_history[RAVG_HIST_SIZE]; -- int cidx; -- -- u32 demand; -- u16 demand_scaled; -- void *sdsq; --}; -- -- -- --/* -- * The following is embedded in task_struct and contains all fields necessary -- * for a task to be scheduled by SCX. -- */ --struct sched_ext_entity { -- struct scx_dispatch_q *dsq; -- struct { -- struct list_head fifo; /* dispatch order */ -- struct rb_node priq; /* p->scx.dsq_vtime order */ -- } dsq_node; -- struct list_head watchdog_node; -- u32 flags; /* protected by rq lock */ -- u32 dsq_flags; /* protected by dsq lock */ -- u32 weight; -- s32 sticky_cpu; -- s32 holding_cpu; -- u32 kf_mask; /* see scx_kf_mask above */ -- struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ -- atomic64_t ops_state; -- unsigned long runnable_at; --#ifdef CONFIG_SCHED_CORE -- u64 core_sched_at; /* see scx_prio_less() */ --#endif -- -- /* BPF scheduler modifiable fields */ -- -- /* -- * Runtime budget in nsecs. This is usually set through -- * scx_bpf_dispatch() but can also be modified directly by the BPF -- * scheduler. Automatically decreased by SCX as the task executes. On -- * depletion, a scheduling event is triggered. -- * -- * This value is cleared to zero if the task is preempted by -- * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the -- * task ran. Use p->se.sum_exec_runtime instead. -- */ -- u64 slice; -- -- /* -- * Used to order tasks when dispatching to the vtime-ordered priority -- * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime() -- * but can also be modified directly by the BPF scheduler. Modifying it -- * while a task is queued on a dsq may mangle the ordering and is not -- * recommended. -- */ -- u64 dsq_vtime; -- -- /* -- * If set, reject future sched_setscheduler(2) calls updating the policy -- * to %SCHED_EXT with -%EACCES. -- * -- * If set from ops.prep_enable() and the task's policy is already -- * %SCHED_EXT, which can happen while the BPF scheduler is being loaded -- * or by inhering the parent's policy during fork, the task's policy is -- * rejected and forcefully reverted to %SCHED_NORMAL. The number of such -- * events are reported through /sys/kernel/debug/sched_ext::nr_rejected. -- */ -- bool disallow; /* reject switching into SCX */ -- -- /* cold fields */ -- struct list_head tasks_node; -- struct task_struct *task; -- struct scx_sched_task_stats sts; --}; -- --void sched_ext_free(struct task_struct *p); -- --#else /* !CONFIG_SCHED_CLASS_EXT */ -- --static inline void sched_ext_free(struct task_struct *p) {} -- --#endif /* CONFIG_SCHED_CLASS_EXT */ --#endif /* _LINUX_SCHED_EXT_H */ -diff --git a/include/linux/sched/hmbird.h b/include/linux/sched/hmbird.h -new file mode 100755 -index 000000000000..3c7861e37ade ---- /dev/null -+++ b/include/linux/sched/hmbird.h -@@ -0,0 +1,381 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class: Documentation/scheduler/hmbird.rst -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef _LINUX_SCHED_HMBIRD_H -+#define _LINUX_SCHED_HMBIRD_H -+ -+#define HMBIRD_TS_IDX 1 -+#define HMBIRD_OPS_IDX 14 -+#define HMBIRD_RQ_IDX 15 -+ -+#define get_hmbird_ts(p) \ -+ ((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ -+#define get_hmbird_rq(rq) \ -+ ((struct hmbird_rq *)(rq->android_oem_data1[HMBIRD_RQ_IDX])) -+ -+#define get_hmbird_ops(rq) \ -+ ((struct hmbird_ops *)(rq->android_oem_data1[HMBIRD_OPS_IDX])) -+ -+#define SCHED_PROP_DEADLINE_MASK (0xFF) /* deadline for ext sched class */ -+/* -+ * Every task has a DEADLINE_LEVEL which stands for -+ * max schedule latency this task can afford. LEVEL1~5 -+ * for user-aware tasks, LEVEL6~9 for other tasks. -+ */ -+#define SCHED_PROP_DEADLINE_LEVEL0 (0) -+#define SCHED_PROP_DEADLINE_LEVEL1 (1) -+#define SCHED_PROP_DEADLINE_LEVEL2 (2) -+#define SCHED_PROP_DEADLINE_LEVEL3 (3) -+#define SCHED_PROP_DEADLINE_LEVEL4 (4) -+#define SCHED_PROP_DEADLINE_LEVEL5 (5) -+#define SCHED_PROP_DEADLINE_LEVEL6 (6) -+#define SCHED_PROP_DEADLINE_LEVEL7 (7) -+#define SCHED_PROP_DEADLINE_LEVEL8 (8) -+#define SCHED_PROP_DEADLINE_LEVEL9 (9) -+/* -+ * Distinguish tasks into periodical tasks which requires -+ * low schedule latency and non-periodical tasks which are -+ * not sensitive to schedule latency. -+ */ -+#define SCHED_HMBIRD_DSQ_TYPE_PERIOD (0) /* period dsq of hmbird */ -+#define SCHED_HMBIRD_DSQ_TYPE_NON_PERIOD (1) /* non period dsq of hmbird */ -+ -+#define TOP_TASK_BITS_MASK (0xFF) -+#define TOP_TASK_BITS (8) -+#include -+ -+extern atomic_t non_hmbird_task; -+extern atomic_t __hmbird_ops_enabled; -+#define hmbird_enabled() atomic_read(&__hmbird_ops_enabled) -+#define MAX_GLOBAL_DSQS (10) -+ -+enum hmbird_consts { -+ HMBIRD_SLICE_DFL = 1 * NSEC_PER_MSEC, -+ HMBIRD_SLICE_ISO = 8 * HMBIRD_SLICE_DFL, -+ HMBIRD_SLICE_INF = U64_MAX, /* infinite, implies nohz */ -+}; -+ -+/* -+ * DSQ (dispatch queue) IDs are 64bit of the format: -+ * -+ * Bits: [63] [62 .. 0] -+ * [ B] [ ID ] -+ * -+ * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -+ * ID: 63 bit ID -+ * -+ * Built-in IDs: -+ * -+ * Bits: [63] [62] [61..32] [31 .. 0] -+ * [ 1] [ L] [ R ] [ V ] -+ * -+ * 1: 1 for built-in DSQs. -+ * L: 1 for LOCAL_ON DSQ IDs, 0 for others -+ * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -+ */ -+enum hmbird_dsq_id_flags { -+ HMBIRD_DSQ_FLAG_BUILTIN = 1LLU << 63, -+ HMBIRD_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -+ -+ HMBIRD_DSQ_INVALID = HMBIRD_DSQ_FLAG_BUILTIN | 0, -+ HMBIRD_DSQ_GLOBAL = HMBIRD_DSQ_FLAG_BUILTIN | 1, -+ HMBIRD_DSQ_LOCAL = HMBIRD_DSQ_FLAG_BUILTIN | 2, -+ HMBIRD_DSQ_LOCAL_ON = HMBIRD_DSQ_FLAG_BUILTIN | HMBIRD_DSQ_FLAG_LOCAL_ON, -+ HMBIRD_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, -+}; -+ -+enum hmbird_switch_type { -+ HMBIRD_SWITCH_PROC, -+ HMBIRD_SWITCH_ERR_WDT, -+ HMBIRD_SWITCH_ERR_HB, -+ HMBIRD_SWITCH_ERR_DSQ, -+ HMBIRD_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ -+ HMBIRD_EXIT_ERROR_HEARTBEAT, /* heart beat has stopped */ -+}; -+ -+/* -+ * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -+ * scheduler core and the BPF scheduler. See the documentation for more details. -+ */ -+struct hmbird_dispatch_q { -+ raw_spinlock_t lock; -+ struct list_head fifo; /* processed in dispatching order */ -+ struct rb_root_cached priq; -+ u32 nr; -+ u64 id; -+ struct llist_node free_node; -+ struct rcu_head rcu; -+ u64 last_consume_at; -+ bool is_timeout; -+}; -+ -+/* hmbird_entity.flags */ -+enum hmbird_ent_flags { -+ HMBIRD_TASK_QUEUED = 1 << 0, /* on hmbird runqueue */ -+ HMBIRD_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -+ HMBIRD_TASK_ENQ_LOCAL = 1 << 2, /* used by hmbird_select_cpu_dfl, set HMBIRD_ENQ_LOCAL */ -+ -+ HMBIRD_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -+ HMBIRD_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -+ -+ HMBIRD_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -+ HMBIRD_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -+ -+ HMBIRD_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ -+}; -+ -+/* hmbird_entity.dsq_flags */ -+enum hmbird_ent_dsq_flags { -+ HMBIRD_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ -+}; -+ -+#define RAVG_HIST_SIZE 5 -+struct hmbird_sched_task_stats { -+ u64 mark_start; -+ u64 window_start; -+ u32 sum; -+ u32 sum_history[RAVG_HIST_SIZE]; -+ int cidx; -+ -+ u32 demand; -+ u16 demand_scaled; -+ void *sdsq; -+}; -+ -+struct hmbird_sched_rq_stats { -+ u64 window_start; -+ u64 latest_clock; -+ u32 prev_window_size; -+ u64 task_exec_scale; -+ u64 prev_runnable_sum; -+ u64 curr_runnable_sum; -+ int *sched_ravg_window_ptr; -+}; -+ -+/* -+ * The following is embedded in task_struct and contains all fields necessary -+ * for a task to be scheduled by HMBIRD. -+ */ -+struct hmbird_entity { -+ struct hmbird_dispatch_q *dsq; -+ struct { -+ struct list_head fifo; /* dispatch order */ -+ struct rb_node priq; -+ } dsq_node; -+ struct list_head watchdog_node; -+ u32 flags; /* protected by rq lock */ -+ u32 dsq_flags; /* protected by dsq lock */ -+ u32 weight; -+ s32 sticky_cpu; -+ s32 holding_cpu; -+ u32 kf_mask; /* see hmbird_kf_mask above */ -+ struct task_struct *kf_tasks[2]; /* see HMBIRD_CALL_OP_TASK() */ -+ atomic64_t ops_state; -+ unsigned long runnable_at; -+ u64 slice; -+ u64 dsq_vtime; -+ bool disallow; /* reject switching into HMBIRD */ -+ u16 demand_scaled; -+ -+ /* cold fields */ -+ struct list_head tasks_node; -+ struct task_struct *task; -+ const struct sched_class *sched_class; -+ unsigned long sched_prop; -+ unsigned long top_task_prop; -+ struct hmbird_sched_task_stats sts; -+ unsigned long running_at; -+ int gdsq_idx; -+ -+ s32 critical_affinity_cpu; -+ int dsq_sync_ux; -+}; -+ -+ -+/* -+ * All variables use 64bits width, -+ * Avoid parsing problems caused by automatic alignment(with padding) of structures. -+ */ -+ -+/* NOTING : Must align to 64bits. */ -+#define DESC_STR_LEN (32) -+/* Supporti up to three-dimensional arrays. */ -+#define PARSE_DIMENS (3) -+struct meta_desc_t { -+ char desc_str[DESC_STR_LEN]; -+ u64 len; -+ u64 parse[PARSE_DIMENS]; -+}; -+ -+#define MAX_SWITCHS (5) -+struct hmbird_switch_t { -+ u64 switch_at; -+ u64 is_success; -+ u64 end_state; -+ u64 switch_reason; -+}; -+#define SWITCH_ITEMS (sizeof(struct hmbird_switch_t) / sizeof(u64)) -+ -+#define MAX_EXCEPS (5) -+enum excep_id { -+ NO_CGROUP_L1, -+ MODULE_UNLOAD, -+ ALREADY_ENABLED, -+ ALREADY_DISABLED, -+ INIT_TASK_FAIL, -+ ALLOC_RQSCX_FAIL, -+ DSQ_ID_ERR, -+ CPU_NO_MASK, -+ SCAN_ENTITY_NULL, -+ ITER_RET_NULL, -+ DEQ_DEQING, -+ ENQ_EXIST1, -+ ENQ_EXIST2, -+ TASK_LINKED1, -+ TASK_UNLINKED, -+ TASK_LINKED2, -+ TASK_UNQUED, -+ TASK_UNWATCHED, -+ HMBIRD_OPN, -+ TASK_WATCHED, -+ RQ_NO_RUNNING, -+ EXTRA_FLAGS, -+ HOLDING_CPU1, -+ HOLDING_CPU2, -+ TASK_OPS_PREPPED, -+ TASK_OPS_UNPREPPED, -+ HMBIRD_OPS_ERR, -+ MAX_EXCEP_ID, -+}; -+ -+struct snap_misc_t { -+ u64 hmbird_enabled; -+ u64 curr_ss; -+ u64 hmbird_ops_enable_state_var; -+ u64 non_ext_task; -+ u64 parctrl_high_ratio; -+ u64 parctrl_low_ratio; -+ u64 parctrl_high_ratio_l; -+ u64 parctrl_low_ratio_l; -+ u64 isoctrl_high_ratio; -+ u64 isoctrl_low_ratio; -+ u64 misfit_ds; -+ u64 partial_enable; -+ u64 iso_free_rescue; -+ u64 isolate_ctrl; -+ u64 snap_jiffies; -+ u64 snap_time; -+}; -+#define SNAP_ITEMS (sizeof(struct snap_misc_t) / sizeof(u64)) -+ -+struct panic_snapshot_t { -+ /* what time dose the first task of dsq turn in runnable, check starvation */ -+ struct meta_desc_t runnable_at_meta; -+ u64 runnable_at[MAX_GLOBAL_DSQS]; -+ -+ struct meta_desc_t rq_nr_meta; -+ u64 rq_nr[NR_CPUS]; -+ -+ struct meta_desc_t scxrq_nr_meta; -+ u64 scxrq_nr[NR_CPUS]; -+ -+ struct meta_desc_t snap_misc_meta; -+ struct snap_misc_t snap_misc; -+}; -+ -+/* -+ * Do not record info in hot paths unless absolutely necessary, -+ * The impact on performance should be minimized. -+ */ -+struct kernel_info_t { -+ struct meta_desc_t sw_rec_meta; -+ struct hmbird_switch_t sw_rec[MAX_SWITCHS]; -+ -+ struct meta_desc_t sw_idx_meta; -+ u64 sw_idx; -+ -+ struct meta_desc_t excep_rec_meta; -+ u64 excep_rec[MAX_EXCEP_ID][MAX_EXCEPS]; -+ -+ struct meta_desc_t excep_idx_meta; -+ u64 excep_idx[MAX_EXCEP_ID]; -+ -+ /* snapshot while panic. */ -+ struct panic_snapshot_t snap; -+}; -+ -+struct ko_info_t {}; -+ -+struct md_meta_t { -+ u64 self_len; -+ u64 unit_size; -+ u64 desc_meta_len; -+ u64 desc_str_len; -+ u64 switches; -+ u64 exceps; -+ u64 global_dsqs; -+ u64 parse_dimens; -+ u64 nr_cpus; -+ u64 real_cpus; -+ u64 nr_meta_desc; -+ u64 dump_real_size; -+}; -+ -+struct md_info_t { -+ struct md_meta_t meta; -+ struct kernel_info_t kern_dump; -+ struct ko_info_t ko_dump; -+}; -+ -+static inline void exceps_update(struct md_info_t *rec, int id, unsigned long jiffies) -+{ -+ u64 *idx; -+ -+ if (!rec) -+ return; -+ -+ idx = &rec->kern_dump.excep_idx[id]; -+ rec->kern_dump.excep_rec[id][*idx] = jiffies; -+ *idx = ++(*idx) % MAX_EXCEPS; -+} -+ -+static inline void sw_update(struct md_info_t *rec, u64 switch_at, -+ u64 is_success, u64 end_state, u64 switch_reason) -+{ -+ u64 *idx; -+ -+ if (!rec) -+ return; -+ -+ idx = &rec->kern_dump.sw_idx; -+ rec->kern_dump.sw_rec[*idx].switch_at = switch_at; -+ rec->kern_dump.sw_rec[*idx].is_success = is_success; -+ rec->kern_dump.sw_rec[*idx].end_state = end_state; -+ rec->kern_dump.sw_rec[*idx].switch_reason = switch_reason; -+ *idx = ++(*idx) % MAX_SWITCHS; -+} -+ -+struct hmbird_ops { -+ bool (*scx_enable)(void); -+ bool (*check_non_task)(void); -+ void (*do_sched_yield_before)(long *skip); -+ void (*window_rollover_run_once)(struct rq *rq); -+ void (*hmbird_get_md_info)(unsigned long *vaddr, unsigned long *size); -+}; -+ -+void hmbird_free(struct task_struct *p); -+ -+enum DSQ_SYNC_UX_FLAG { -+ DSQ_SYNC_UX_NONE = 0, -+ DSQ_SYNC_STATIC_UX = 1, -+ DSQ_SYNC_INHERIT_UX = 1 << 1, -+}; -+ -+#endif /* _LINUX_SCHED_HMBIRD_H */ -diff --git a/include/linux/sched/hmbird_proc_val.h b/include/linux/sched/hmbird_proc_val.h -new file mode 100755 -index 000000000000..e59708b16ea3 ---- /dev/null -+++ b/include/linux/sched/hmbird_proc_val.h -@@ -0,0 +1,22 @@ -+#ifndef __HMBORD_PROC_VAL_H__ -+#define __HMBORD_PROC_VAL_H__ -+ -+extern int scx_enable; -+extern int partial_enable; -+extern int cpuctrl_high_ratio; -+extern int cpuctrl_low_ratio; -+extern int slim_stats; -+extern int hmbirdcore_debug; -+extern int slim_for_app; -+extern int misfit_ds; -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+extern int cpu7_tl; -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int slim_gov_debug; -+extern int scx_gov_ctrl; -+extern int sched_ravg_window_frame_per_sec; -+ -+#endif -\ No newline at end of file -diff --git a/include/linux/sched/hmbird_version.h b/include/linux/sched/hmbird_version.h -new file mode 100755 -index 000000000000..b2843881c26f ---- /dev/null -+++ b/include/linux/sched/hmbird_version.h -@@ -0,0 +1,52 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#ifndef _OPLUS_HMBIRD_VERSION_H_ -+#define _OPLUS_HMBIRD_VERSION_H_ -+#include -+#include -+#include -+ -+enum hmbird_version { -+ HMBIRD_UNINIT, -+ HMBIRD_GKI_VERSION, -+ HMBIRD_OGKI_VERSION, -+ HMBIRD_UNKNOW_VERSION, -+}; -+ -+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT; -+ -+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type" -+ -+static inline enum hmbird_version get_hmbird_version_type(void) -+{ -+ struct device_node *np = NULL; -+ const char *hmbird_version_str = NULL; -+ if (HMBIRD_UNINIT != hmbird_version_type) -+ return hmbird_version_type; -+ np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH); -+ if (np) { -+ of_property_read_string(np, "type", &hmbird_version_str); -+ if (NULL != hmbird_version_str) { -+ if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) { -+ hmbird_version_type = HMBIRD_OGKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi"); -+ } else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) { -+ hmbird_version_type = HMBIRD_GKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi"); -+ } else { -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi"); -+ } -+ return hmbird_version_type; -+ } -+ } -+ -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION"); -+ return hmbird_version_type; -+} -+ -+#endif /*_OPLUS_HMBIRD_VERSION_H_ */ -diff --git a/include/linux/sched/sa_common.h b/include/linux/sched/sa_common.h -new file mode 100755 -index 000000000000..6f76d7cfd7bf ---- /dev/null -+++ b/include/linux/sched/sa_common.h -@@ -0,0 +1,797 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_COMMON_H_ -+#define _OPLUS_SA_COMMON_H_ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+#include -+#endif -+#include -+#include -+#include -+#include -+#include -+ -+#include "sa_oemdata.h" -+#include "sa_common_struct.h" -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) -+#define OPLUS_UX_EEVDF_COMPATIBLE 1 -+#endif -+ -+#define SA_DEBUG_ON 0 -+ -+#if (SA_DEBUG_ON >= 1) -+#define DEBUG_BUG_ON(x) BUG_ON(x) -+#define DEBUG_WARN_ON(x) WARN_ON(x) -+#else -+#define DEBUG_BUG_ON(x) -+#define DEBUG_WARN_ON(x) -+#endif -+ -+#define ux_err(fmt, ...) \ -+ pr_err("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_warn(fmt, ...) \ -+ pr_warn("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_debug(fmt, ...) \ -+ pr_info("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+ -+#define UX_MSG_LEN 64 -+#define UX_DEPTH_MAX 5 -+ -+/* define for debug */ -+#define DEBUG_SYSTRACE (1 << 0) -+#define DEBUG_FTRACE (1 << 1) -+#define DEBUG_KMSG (1 << 2) /* used for frameboost */ -+#define DEBUG_PIPELINE (1 << 3) -+#define DEBUG_DYNAMIC_HZ (1 << 4) -+#define DEBUG_DYNAMIC_PREEMPT (1 << 5) -+#define DEBUG_AMU_INSTRUCTION (1 << 6) -+#define DEBUG_VERBOSE (1 << 10) /* used for frameboost */ -+#define DEBUG_SET_DSQ_ID (1 << 11) /* used for hmbird */ -+ -+/* define for sched assist feature */ -+#define FEATURE_COMMON (1 << 0) -+#define FEATURE_SPREAD (1 << 1) -+ -+#define UX_EXEC_SLICE (4000000U) -+ -+/* define for sched assist thread type, keep same as the define in java file */ -+#define SA_OPT_CLEAR (0) -+#define SA_TYPE_LIGHT (1 << 0) -+#define SA_TYPE_HEAVY (1 << 1) -+#define SA_TYPE_ANIMATOR (1 << 2) -+/* SA_TYPE_LISTPICK for camera */ -+#define SA_TYPE_LISTPICK (1 << 3) -+#define SA_TYPE_MQ_VIP (1 << 4) -+#define SA_OPT_SET (1 << 7) -+#define SA_OPT_RESET (1 << 8) -+#define SA_OPT_SET_PRIORITY (1 << 9) -+ -+/* The following ux value only used in kernel */ -+#define SA_TYPE_SWIFT (1 << 14) -+/* clear ux type when dequeue */ -+#define SA_TYPE_ONCE (1 << 15) -+#define SA_TYPE_INHERIT (1 << 16) -+/* SA_TYPE_COMBINED isn't marked in ux_state, it only exists in return value of function */ -+#define SA_TYPE_COMBINED (1 << 17) -+#define SA_TYPE_URGENT_MASK (SA_TYPE_LIGHT|SA_TYPE_ANIMATOR|SA_TYPE_SWIFT) -+#define SCHED_ASSIST_UX_MASK (SA_TYPE_LIGHT|SA_TYPE_HEAVY|SA_TYPE_ANIMATOR|SA_TYPE_LISTPICK|SA_TYPE_SWIFT) -+ -+/* load balance operation is performed on the following ux types */ -+#define SCHED_ASSIST_LB_UX (SA_TYPE_SWIFT | SA_TYPE_ANIMATOR | SA_TYPE_LIGHT | SA_TYPE_HEAVY) -+#define POSSIBLE_UX_MASK (SA_TYPE_SWIFT | \ -+ SA_TYPE_LIGHT | \ -+ SA_TYPE_HEAVY | \ -+ SA_TYPE_ANIMATOR | \ -+ SA_TYPE_LISTPICK | \ -+ SA_TYPE_ONCE | \ -+ SA_TYPE_INHERIT) -+ -+#define SCHED_ASSIST_UX_PRIORITY_MASK (0xFF000000) -+#define SCHED_ASSIST_UX_PRIORITY_SHIFT 24 -+ -+#define UX_PRIORITY_TOP_APP 0x0A000000 -+#define UX_PRIORITY_AUDIO 0x0A000000 -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+#define UX_PRIORITY_PIPELINE_UI 0x06000000 -+#define UX_PRIORITY_PIPELINE 0x05000000 -+#endif -+ -+/* define for sched assist scene type, keep same as the define in java file */ -+#define SA_SCENE_OPT_CLEAR (0) -+#define SA_LAUNCH (1 << 0) -+#define SA_SLIDE (1 << 1) -+#define SA_CAMERA (1 << 2) -+#define SA_ANIM_START (1 << 3) /* we care about both launcher and top app */ -+#define SA_ANIM (1 << 4) /* we only care about launcher */ -+#define SA_INPUT (1 << 5) -+#define SA_LAUNCHER_SI (1 << 6) -+#define SA_SCENE_OPT_SET (1 << 7) -+ -+#define ROOT_UID 0 -+#define SYSTEM_UID 1000 -+#define FIRST_APPLICATION_UID 10000 -+#define LAST_APPLICATION_UID 19999 -+#define PER_USER_RANGE 100000 -+/* define for clear IM_FLAG -+if im_flag is 70, it should clear im_flag_audio (70 - 64 = 6) -+eg: gerrit patchset "30438485" -+ */ -+#define IM_FLAG_CLEAR 64 -+ -+extern pid_t save_audio_tgid; -+extern pid_t save_top_app_tgid; -+extern unsigned int top_app_type; -+extern int global_lowend_plat_opt; -+ -+#ifdef CONFIG_OPLUS_SCHED_HALT_MASK_PRT -+/* This must be the same as the definition of pause_type in walt_halt.c */ -+enum oplus_pause_type { -+ OPLUS_HALT, -+ OPLUS_PARTIAL_HALT, -+ -+ OPLUS_MAX_PAUSE_TYPE -+}; -+extern cpumask_t cur_cpus_halt_mask; -+extern cpumask_t cur_cpus_phalt_mask; -+DECLARE_PER_CPU(int[OPLUS_MAX_PAUSE_TYPE], oplus_cur_pause_client); -+extern void sa_corectl_systrace_c(void); -+#endif /* CONFIG_OPLUS_SCHED_HALT_MASK_PRT */ -+ -+/* define for boost threshold unit */ -+#define BOOST_THRESHOLD_UNIT (51) -+ -+ -+enum UX_STATE_TYPE { -+ UX_STATE_INVALID = 0, -+ UX_STATE_NONE, -+ UX_STATE_STATIC, -+ UX_STATE_INHERIT, -+ UX_STATE_COMBINED, -+ MAX_UX_STATE_TYPE, -+}; -+ -+enum INHERIT_UX_TYPE { -+ INHERIT_UX_BINDER = 0, -+ INHERIT_UX_RWSEM, -+ INHERIT_UX_MUTEX, -+ INHERIT_UX_FUTEX, -+ INHERIT_UX_PIFUTEX, -+ INHERIT_UX_MAX, -+}; -+ -+/* -+ * WANNING: -+ * new flag should be add before MAX_IM_FLAG_TYPE, never change -+ * the value of those existed flag type. -+ */ -+enum IM_FLAG_TYPE { -+ IM_FLAG_NONE = 0, -+ IM_FLAG_SURFACEFLINGER, -+ IM_FLAG_HWC, /* Discarded */ -+ IM_FLAG_RENDERENGINE, -+ IM_FLAG_WEBVIEW, -+ IM_FLAG_CAMERA_HAL, -+ IM_FLAG_AUDIO, -+ IM_FLAG_HWBINDER, -+ IM_FLAG_LAUNCHER, -+ IM_FLAG_LAUNCHER_NON_UX_RENDER, -+ IM_FLAG_SS_LOCK_OWNER, -+ IM_FLAG_FORBID_SET_CPU_AFFINITY = 11, /* forbid setting cpu affinity from app */ -+ IM_FLAG_SYSTEMSERVER_PID, -+ IM_FLAG_MIDASD, -+ IM_FLAG_AUDIO_CAMERA_HAL, /* audio mode disable camera hal ux */ -+ IM_FLAG_AFFINITY_THREAD, -+ IM_FLAG_TPD_SET_CPU_AFFINITY = 16, -+ IM_FLAG_COMPRESS_THREAD = 17, /* compress thread skips locking protect */ -+ IM_FLAG_RENDER_THREAD = 18, -+ MAX_IM_FLAG_TYPE, -+}; -+ -+#define MAX_IM_FLAG_PRIO MAX_IM_FLAG_TYPE -+ -+enum ots_state { -+ OTS_STATE_SET_AFFINITY, -+ OTS_STATE_DDL_ACTIVE, -+ OTS_STATE_DDL_ACTIVE_PREEMPTED, -+ OTS_STATE_MAX, -+}; -+ -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+DECLARE_PER_CPU(u64, retired_instrs); -+DECLARE_PER_CPU(u64, nvcsw); -+DECLARE_PER_CPU(u64, nivcsw); -+#endif -+ -+struct ux_sched_cluster { -+ struct cpumask cpus; -+ unsigned long capacity; -+}; -+ -+#define OPLUS_NR_CPUS (8) -+#define OPLUS_MAX_CLS (5) -+struct ux_sched_cputopo { -+ int cls_nr; -+ struct ux_sched_cluster sched_cls[OPLUS_NR_CPUS]; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ cpumask_t oplus_cpu_array[2*OPLUS_MAX_CLS][OPLUS_MAX_CLS]; -+#endif -+}; -+ -+struct oplus_rq { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_root_cached ux_list; -+ /* a tree to track minimum exec vruntime */ -+ struct rb_root_cached exec_timeline; -+ /* malloc this spinlock_t instead of built-in to shrank size of oplus_rq */ -+ spinlock_t *ux_list_lock; -+ int nr_running; -+ u64 min_vruntime; -+ u64 load_weight; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) -+ struct rb_root_cached ddl_root; -+ spinlock_t *ddl_lock; -+#endif -+ -+#ifdef CONFIG_LOCKING_PROTECT -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ struct list_head locking_thread_list; -+ spinlock_t *locking_list_lock; -+ int rq_locking_task; -+#else -+ struct sched_entity *last_entity; -+#endif -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ struct oplus_lb lb; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+ struct hrtimer *resched_timer; -+ int cpu; -+#endif -+}; -+ -+extern int global_debug_enabled; -+extern int global_sched_assist_enabled; -+extern int global_sched_assist_scene; -+extern int global_silver_perf_core; -+extern int global_sched_group_enabled; -+ -+struct rq; -+ -+#ifdef CONFIG_LOCKING_PROTECT -+struct sched_assist_locking_ops { -+ void (*check_preempt_tick)(struct task_struct *p, -+ unsigned long *ideal_runtime, bool *skip_preempt, -+ unsigned long delta_exec, struct cfs_rq *cfs_rq, -+ struct sched_entity *curr, unsigned int granularity); -+ void (*check_preempt_wakeup)(struct rq *rq, struct task_struct *p, bool *preempt, bool *nopreempt); -+ void (*state_systrace_c)(unsigned int cpu, struct task_struct *p); -+ void (*locking_tick_hit)(struct task_struct *prev, struct task_struct *next); -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ void (*replace_next_task_fair)(struct rq *rq, -+ struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*enqueue_entity)(struct rq *rq, struct task_struct *p); -+ void (*dequeue_entity)(struct rq *rq, struct task_struct *p); -+#else -+ void (*set_last_entity)(struct task_struct *prev, struct task_struct *next, struct rq *rq); -+ void (*pick_last_entity)(struct rq *rq, struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*clear_last_entity)(struct rq *rq, struct task_struct *p); -+#endif -+ void (*opt_ss_lock_contention)(struct task_struct *p, int old_im, int new_im); -+}; -+ -+extern struct sched_assist_locking_ops *locking_ops; -+ -+ -+#define LOCKING_CALL_OP(op, args...) \ -+do { \ -+ if (locking_ops && locking_ops->op) { \ -+ locking_ops->op(args); \ -+ } \ -+} while (0) -+ -+#define LOCKING_CALL_OP_RET(op, args...) \ -+({ \ -+ __typeof__(locking_ops->op(args)) __ret = 0; \ -+ if (locking_ops && locking_ops->op) \ -+ __ret = locking_ops->op(args); \ -+ __ret; \ -+}) -+ -+void register_sched_assist_locking_ops(struct sched_assist_locking_ops *ops); -+#endif -+ -+/* attention: before insert .ko, task's list->prev/next is init with 0 */ -+static inline bool oplus_list_empty(struct list_head *list) -+{ -+ return list_empty(list) || (list->prev == 0 && list->next == 0); -+} -+ -+static inline bool oplus_rbtree_empty(struct rb_root_cached *list) -+{ -+ return (rb_first_cached(list) == NULL); -+} -+ -+/** -+ * Check if there are ux tasks waiting to run on the specified cpu -+ */ -+static inline bool orq_has_ux_tasks(struct oplus_rq *orq) -+{ -+ return !oplus_rbtree_empty(&orq->ux_list); -+} -+ -+/* attention: before insert .ko, task's node->__rb_parent_color is init with 0 */ -+static inline bool oplus_rbnode_empty(struct rb_node *node) -+{ -+ return RB_EMPTY_NODE(node) || (node->__rb_parent_color == 0); -+} -+ -+static inline struct oplus_task_struct *ux_list_first_entry(struct rb_root_cached *list) -+{ -+ struct rb_node *leftmost = rb_first_cached(list); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, ux_entry); -+} -+ -+static inline struct oplus_task_struct *exec_timeline_first_entry(struct rb_root_cached *tree) -+{ -+ struct rb_node *leftmost = rb_first_cached(tree); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, exec_time_node); -+} -+ -+typedef bool (*migrate_task_callback_t)(struct task_struct *tsk, int src_cpu, int dst_cpu); -+extern migrate_task_callback_t fbg_migrate_task_callback; -+typedef void (*android_rvh_schedule_handler_t)(struct task_struct *prev, -+ struct task_struct *next, struct rq *rq); -+extern android_rvh_schedule_handler_t fbg_android_rvh_schedule_callback; -+ -+extern struct kmem_cache *oplus_task_struct_cachep; -+ -+#define ots_to_ts(ots) (ots->task) -+#define OTS_IDX 0 -+#define ORQ_IDX 0 -+ -+static inline bool test_task_is_fair(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid CFS priority is MAX_RT_PRIO..MAX_PRIO-1 */ -+ if ((task->prio >= MAX_RT_PRIO) && (task->prio <= MAX_PRIO-1)) -+ return true; -+ return false; -+} -+ -+static inline bool test_task_is_rt(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid RT priority is 0..MAX_RT_PRIO-1 */ -+ if (rt_prio(task->prio)) -+ return true; -+ -+ return false; -+} -+ -+static inline struct oplus_task_struct *get_oplus_task_struct(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = NULL; -+ -+ /* not Skip idle thread */ -+ if (!t) -+ return NULL; -+ -+ ots = (struct oplus_task_struct *) READ_ONCE(t->android_oem_data1[OTS_IDX]); -+ if (IS_ERR_OR_NULL(ots)) -+ return NULL; -+ -+ return ots; -+} -+ -+static inline unsigned long oplus_get_im_flag(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return IM_FLAG_NONE; -+ -+ return ots->im_flag; -+} -+ -+static inline bool is_optimized_audio_thread(struct task_struct *t) -+{ -+ unsigned long im_flag; -+ -+ im_flag = oplus_get_im_flag(t); -+ if (test_bit(IM_FLAG_AUDIO, &im_flag)) -+ return true; -+ -+ return false; -+} -+ -+static inline void oplus_set_im_flag(struct task_struct *t, int im_flag) -+{ -+ struct oplus_task_struct *ots = NULL; -+ ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ set_bit(im_flag, &ots->im_flag); -+} -+ -+static inline int get_ots_ux_state(struct oplus_task_struct *ots) -+{ -+ int ux_state; -+ int sub_ux_state; -+ int ux_prio; -+ int ret; -+ -+ ux_prio = ots->ux_state & SCHED_ASSIST_UX_PRIORITY_MASK; -+ ux_state = ots->ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ sub_ux_state = ots->sub_ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ -+ if (sub_ux_state) { -+ ret = ux_state | (sub_ux_state & ~SA_TYPE_INHERIT) | SA_TYPE_COMBINED; -+ } else { -+ ret = ux_state; -+ } -+ -+ if (ret & SCHED_ASSIST_UX_MASK) { -+ return ux_prio | ret; -+ } -+ -+ return 0; -+} -+ -+bool is_multiple_ux(struct oplus_task_struct *ots); -+ -+static inline int oplus_get_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return get_ots_ux_state(ots); -+} -+ -+static inline int oplus_get_static_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return 0; -+ } -+ return ots->ux_state; -+} -+ -+static inline int oplus_get_sub_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->sub_ux_state; -+} -+ -+static inline int oplus_get_inherited_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return ots->ux_state; -+ } -+ return ots->sub_ux_state; -+} -+ -+void oplus_set_ux_state_lock(struct task_struct *t, int ux_state, int inherit_type, bool need_lock_rq); -+ -+static inline s64 oplus_get_inherit_ux(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return atomic64_read(&ots->inherit_ux); -+} -+ -+static inline void oplus_set_inherit_ux(struct task_struct *t, s64 inherit_ux) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ atomic64_set(&ots->inherit_ux, inherit_ux); -+} -+ -+static inline int oplus_get_ux_depth(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->ux_depth; -+} -+ -+static inline void oplus_set_ux_depth(struct task_struct *t, int ux_depth) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->ux_depth = ux_depth; -+} -+ -+static inline u64 oplus_get_enqueue_time(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->enqueue_time; -+} -+ -+static inline void oplus_set_enqueue_time(struct task_struct *t, u64 enqueue_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->enqueue_time = enqueue_time; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* -+ * Record the number of task context switches -+ * and scheduling delays when enqueueing a task. -+ */ -+ ots->snap_pcount = t->sched_info.pcount; -+ ots->snap_run_delay = t->sched_info.run_delay; -+#endif -+} -+ -+static inline void oplus_set_inherit_ux_start(struct task_struct *t, u64 start_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->inherit_ux_start = t->se.sum_exec_runtime; -+} -+ -+static inline void init_task_ux_info(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ RB_CLEAR_NODE(&ots->ux_entry); -+ RB_CLEAR_NODE(&ots->exec_time_node); -+ ots->ux_state = 0; -+ ots->sub_ux_state = 0; -+ atomic64_set(&ots->inherit_ux, 0); -+ ots->ux_depth = 0; -+ ots->enqueue_time = 0; -+ ots->inherit_ux_start = 0; -+ ots->ux_priority = -1; -+ ots->ux_nice = -1; -+ ots->vruntime = 0; -+ ots->preset_vruntime = 0; -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG) -+ ots->abnormal_flag = 0; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_SCHED_SPREAD -+ ots->lb_state = 0; -+ ots->ld_flag = 0; -+#endif -+ ots->exec_calc_runtime = 0; -+ ots->is_update_runtime = 0; -+ ots->target_process = -1; -+ ots->wake_tid = 0; -+ ots->running_start_time = 0; -+ ots->update_running_start_time = false; -+ ots->last_wake_ts = 0; -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ memset(&ots->lkinfo, 0, sizeof(struct locking_info)); -+ INIT_LIST_HEAD(&ots->lkinfo.node); -+/*#endif*/ -+ ots->block_start_time = 0; -+#ifdef CONFIG_LOCKING_PROTECT -+ INIT_LIST_HEAD(&ots->locking_entry); -+ ots->locking_start_time = 0; -+ ots->locking_depth = 0; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ ots->snap_pcount = 0; -+ ots->snap_run_delay = 0; -+ plist_node_init(&ots->rtb, MAX_IM_FLAG_PRIO); -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+ atomic_set(&ots->pipeline_cpu, -1); -+#endif -+ -+#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO) -+ ots->amu_cycle = 0; -+ ots->amu_instruct = 0; -+#endif -+}; -+ -+static inline bool test_ux_type(struct task_struct *task, int ux_type) -+{ -+ return oplus_get_ux_state(task) & ux_type; -+} -+ -+static inline bool is_heavy_ux_task(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return false; -+ -+ return get_ots_ux_state(ots) & SA_TYPE_HEAVY; -+} -+ -+static inline bool sched_assist_scene(unsigned int scene) -+{ -+ if (unlikely(!global_sched_assist_enabled)) -+ return false; -+ -+ return global_sched_assist_scene & scene; -+} -+ -+static inline unsigned long oplus_task_util(struct task_struct *p) -+{ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+ struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; -+ -+ return wts->demand_scaled; -+#else -+ return READ_ONCE(p->se.avg.util_avg); -+#endif -+} -+ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+static inline u32 task_wts_sum(struct task_struct *tsk) -+{ -+ struct walt_task_struct *wts = (struct walt_task_struct *) tsk->android_vendor_data1; -+ -+ return wts->sum; -+} -+#endif -+ -+bool is_min_cluster(int cpu); -+bool is_max_cluster(int cpu); -+bool is_mid_cluster(int cpu); -+bool im_mali(const char *comm); -+bool is_top(struct task_struct *p); -+bool task_is_runnable(struct task_struct *task); -+struct oplus_rq *get_oplus_rq(struct rq *rq); -+ -+typedef unsigned long (*kallsyms_lookup_name_t)(const char *name); -+extern kallsyms_lookup_name_t _kallsyms_lookup_name; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+int ux_mask_to_prio(int ux_mask); -+int ux_prio_to_mask(int prio); -+#endif -+ -+noinline int tracing_mark_write(const char *buf); -+void hwbinder_systrace_c(unsigned int cpu, int flag); -+void sched_assist_init_oplus_rq(void); -+void queue_ux_thread(struct rq *rq, struct task_struct *p, int enqueue); -+ -+void inherit_ux_inc(struct task_struct *task, int type); -+void inherit_ux_sub(struct task_struct *task, int type, int value); -+void set_inherit_ux(struct task_struct *task, int type, int depth, int inherit_val); -+void reset_inherit_ux(struct task_struct *inherit_task, struct task_struct *ux_task, int reset_type); -+void unset_inherit_ux(struct task_struct *task, int type); -+void unset_inherit_ux_value(struct task_struct *task, int type, int value); -+void inc_inherit_ux_refs(struct task_struct *task, int type); -+void clear_all_inherit_type(struct task_struct *p); -+int get_max_inherit_gran(struct task_struct *p); -+ -+bool is_heavy_load_top_task(struct task_struct *p); -+bool test_task_is_fair(struct task_struct *task); -+bool test_task_is_rt(struct task_struct *task); -+ -+bool test_task_ux(struct task_struct *task); -+bool test_task_ux_depth(int ux_depth); -+bool test_inherit_ux(struct task_struct *task, int type); -+bool test_set_inherit_ux(struct task_struct *task); -+int get_ux_state_type(struct task_struct *task); -+void sched_assist_target_comm(struct task_struct *task, const char *comm); -+unsigned int ux_task_exec_limit(struct task_struct *p); -+ -+void update_ux_sched_cputopo(void); -+bool is_task_util_over(struct task_struct *tsk, int threshold); -+bool oplus_task_misfit(struct task_struct *tsk, int cpu); -+ssize_t oplus_show_cpus(const struct cpumask *mask, char *buf); -+void adjust_rt_lowest_mask(struct task_struct *p, struct cpumask *local_cpu_mask, int ret, bool force_adjust); -+bool sa_skip_rt_sync(struct rq *rq, struct task_struct *p, bool *sync); -+void ux_state_systrace_c(unsigned int cpu, struct task_struct *p); -+bool sa_rt_skip_ux_cpu(int cpu); -+int is_vip_mvp(struct task_struct *p); -+ -+/* s64 account_ux_runtime(struct rq *rq, struct task_struct *curr); */ -+void opt_ss_lock_contention(struct task_struct *p, unsigned long old_im, int new_im); -+ -+/* register vender hook in kernel/sched/topology.c */ -+void android_vh_build_sched_domains_handler(void *unused, bool has_asym); -+ -+/* register vender hook in kernel/sched/rt.c */ -+void android_rvh_select_task_rq_rt_handler(void *unused, struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags, int *new_cpu); -+void android_rvh_find_lowest_rq_handler(void *unused, struct task_struct *p, struct cpumask *local_cpu_mask, int ret, int *best_cpu); -+ -+/* register vender hook in kernel/sched/core.c */ -+void android_rvh_sched_fork_handler(void *unused, struct task_struct *p); -+void android_rvh_after_enqueue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_dequeue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_schedule_handler(void *unused, struct task_struct *prev, struct task_struct *next, struct rq *rq); -+void android_vh_scheduler_tick_handler(void *unused, struct rq *rq); -+ -+void android_vh_account_process_tick_gran_handler(void *unused, struct task_struct *p, struct rq *rq, int user_tick, int *ticks); -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+void sa_sched_switch_handler(void *unused, bool preempt, struct task_struct *prev, struct task_struct *next, unsigned int prev_state); -+#endif -+ -+void set_im_flag_with_bit(int im_flag, struct task_struct *task); -+ -+/* register vendor hook in kernel/cgroup/cgroup-v1.c */ -+void android_vh_cgroup_set_task_handler(void *unused, int ret, struct task_struct *task); -+/* register vendor hook in kernel/signal.c */ -+void android_vh_exit_signal_handler(void *unused, struct task_struct *p); -+void android_rvh_set_cpus_allowed_by_task_handler(void *unused, const struct cpumask *cpu_valid_mask, -+ const struct cpumask *new_mask, struct task_struct *task, unsigned int *dest_cpu); -+void android_rvh_setscheduler_handler(void *unused, struct task_struct *p); -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_BAN_APP_SET_AFFINITY) -+void android_vh_sched_setaffinity_early_handler(void *unused, struct task_struct *task, const struct cpumask *new_mask, bool *skip); -+#endif -+ -+#ifdef CONFIG_OPLUS_SCHED_GROUP_OPT -+void android_vh_reweight_entity_handler(void *unused, struct sched_entity *se); -+#endif -+ -+#ifdef CONFIG_BLOCKIO_UX_OPT -+int sa_blockio_init(void); -+void sa_blockio_exit(void); -+#endif -+extern struct notifier_block process_exit_notifier_block; -+#endif /* _OPLUS_SA_COMMON_H_ */ -diff --git a/include/linux/sched/sa_common_struct.h b/include/linux/sched/sa_common_struct.h -new file mode 100755 -index 000000000000..876feff48b36 ---- /dev/null -+++ b/include/linux/sched/sa_common_struct.h -@@ -0,0 +1,277 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+/* -+this file is splited from the sa_common.h to adapt the OKI, -+IS_ENABLED is not allowed in here, because the macro will not work in OKI. -+*/ -+#ifndef _OPLUS_SA_COMMON_STRUCT_H_ -+#define _OPLUS_SA_COMMON_STRUCT_H_ -+ -+#define MAX_CLUSTER (4) -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+/* hot-thread */ -+struct task_record { -+#define RECOED_WINSIZE (1 << 8) -+#define RECOED_WINIDX_MASK (RECOED_WINSIZE - 1) -+ u8 winidx; -+ u8 count; -+}; -+ -+#define MAX_TASK_COMM_LEN 256 -+struct uid_struct { -+ uid_t uid; -+ u64 uid_total_cycle; -+ u64 uid_total_inst; -+ spinlock_t lock; -+ char leader_comm[TASK_COMM_LEN]; -+ char cmdline[MAX_TASK_COMM_LEN]; -+}; -+ -+struct amu_uid_entry { -+ uid_t uid; -+ struct uid_struct *uid_struct; -+ struct hlist_node node; -+}; -+ -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+struct locking_info { -+ u64 waittime_stamp; -+ u64 holdtime_stamp; -+ /* Used in torture acquire latency statistic.*/ -+ u64 acquire_stamp; -+ /* -+ * mutex or rwsem optimistic spin start time. Because a task -+ * can't spin both on mutex and rwsem at one time, use one common -+ * threshold time is OK. -+ */ -+ u64 opt_spin_start_time; -+ struct task_struct *holder; -+ u32 waittype; -+ bool ux_contrib; -+ /* -+ * Whether task is ux when it's going to be added to mutex or -+ * rwsem waiter list. It helps us check whether there is ux -+ * task on mutex or rwsem waiter list. Also, a task can't be -+ * added to both mutex and rwsem at one time, so use one common -+ * field is OK. -+ */ -+ bool is_block_ux; -+ u32 kill_flag; -+ /* for cfs enqueue smoothly.*/ -+ struct list_head node; -+ struct task_struct *owner; -+ struct list_head lock_head; -+ u64 clear_seq; -+ atomic_t lock_depth; -+}; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+#define RAVG_HIST_SIZE 5 -+#define SCX_SLICE_DFL (1 * NSEC_PER_MSEC) -+#define SCX_SLICE_INF U64_MAX -+#define DEFAULT_CGROUP_DL_IDX (8) -+#define EXT_FLAG_RT_CHANGED (1 << 0) -+#define EXT_FLAG_CFS_CHANGED (1 << 1) -+struct scx_task_stats { -+ u64 mark_start; -+ u64 window_start; -+ u32 sum; -+ u32 sum_history[RAVG_HIST_SIZE]; -+ int cidx; -+ u32 demand; -+ u16 demand_scaled; -+ void *sdsq; -+}; -+/* -+ * The following is embedded in task_struct and contains all fields necessary -+ * for a task to be scheduled by SCX. -+ */ -+struct scx_entity { -+ struct scx_dispatch_q *dsq; -+ struct { -+ struct list_head fifo; /* dispatch order */ -+ struct rb_node priq; /* p->scx.dsq_vtime order */ -+ } dsq_node; -+ u32 flags; /* protected by rq lock */ -+ u32 dsq_flags; /* protected by dsq lock */ -+ s32 sticky_cpu; -+ unsigned long runnable_at; -+ u64 slice; -+ u64 dsq_vtime; -+ int gdsq_idx; -+ int ext_flags; -+ int prio_backup; -+ unsigned long sched_prop; -+ struct scx_task_stats sts; -+}; -+/*#endif*/ -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ -+#define MAX_CPU_FREQ_STATE 32 -+#define MAX_CPU_CNT 8 -+ -+#define STATE_IN_POOL 0 -+#define STATE_ACTIVE 1 -+ -+struct powermodel_freq_task_state { -+ u8 powermodel_enqueued:1; -+ u8 powermodel_index:7; -+}; -+ -+struct powermodel_cpu_task_state { -+ struct oplus_task_struct *ots; -+ struct powermodel_freq_task_state powermodel_freq_task_states[MAX_CPU_FREQ_STATE]; -+ u64 powermodel_last_seq; -+ u64 last_seq; -+ struct list_head node; -+ int state; -+ int pool_id; -+}; -+ -+#endif -+ -+ -+/* Please add your own members of task_struct here :) */ -+struct oplus_task_struct { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_node ux_entry; -+ struct rb_node exec_time_node; -+ struct task_struct *task; -+ atomic64_t inherit_ux; -+ u64 enqueue_time; -+ u64 inherit_ux_start; -+ /* u64 sum_exec_baseline; */ -+ u64 total_exec; -+ u64 vruntime; -+ u64 preset_vruntime; -+ /* contains ux state -+ 1. if static and inherited ux both exist, static ux stores in ux_state, inherited ux in sub_ux_state. -+ 2. if only static ux exists, static ux stores in ux_state. -+ 2. if only inherited ux exists, inherited ux stores in ux_state */ -+ int ux_state; -+ int sub_ux_state; -+ u8 ux_depth; -+ s8 ux_priority; -+ s8 ux_nice; -+ pid_t affinity_pid; -+ pid_t affinity_tgid; -+ unsigned long state; -+ unsigned long im_flag; -+ atomic_t is_vip_mvp; -+ -+/* #if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) */ -+ u64 ddl; -+ u64 ddl_active_ts; -+ u64 runnable_ts; -+ struct rb_node ddl_node; -+/* #endif */ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG)*/ -+ int abnormal_flag; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_SCHED_SPREAD */ -+ int lb_state; -+ int ld_flag:1; -+ /* CONFIG_OPLUS_FEATURE_TASK_LOAD */ -+ int is_update_runtime:1; -+ int target_process; -+ u64 wake_tid; -+ u64 running_start_time; -+ bool update_running_start_time; -+ u64 exec_calc_runtime; -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct task_record record[MAX_CLUSTER]; /* 2*u64 */ -+ u64 block_start_time; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_FRAME_BOOST */ -+ struct list_head fbg_list; -+ raw_spinlock_t fbg_list_entry_lock; -+ bool fbg_running; /* task belongs to a group, and in running */ -+ u16 fbg_state; -+ s8 preferred_cluster_id; -+ s8 fbg_depth; -+ u64 last_wake_ts; -+ int fbg_cur_group; -+/*#ifdef CONFIG_LOCKING_PROTECT*/ -+ unsigned long locking_start_time; -+ unsigned long last_jiffies; -+ struct list_head locking_entry; -+ int locking_depth; -+ int lk_tick_hit; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ struct locking_info lkinfo; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+ struct scx_entity scx; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_FDLEAK_CHECK)*/ -+ u8 fdleak_flag; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+ /* for loadbalance */ -+ struct plist_node rtb; /* rt boost task */ -+ -+ /* -+ * The following variables are used to calculate the time -+ * a task spends in the running/runnable state. -+ */ -+ u64 snap_run_delay; -+ unsigned long snap_pcount; -+/*#endif*/ -+#if IS_ENABLED(CONFIG_OPLUS_SCHED_TUNE) -+ int stune_idx; -+#endif -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE)*/ -+ atomic_t pipeline_cpu; -+/*#endif*/ -+ -+ /* for oplus secure guard */ -+ int sg_flag; -+ int sg_scno; -+ uid_t sg_uid; -+ uid_t sg_euid; -+ gid_t sg_gid; -+ gid_t sg_egid; -+/*#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct uid_struct *uid_struct; -+ u64 amu_instruct; -+ u64 amu_cycle; -+/*#endif*/ -+ /* for binder ux */ -+ int binder_async_ux_enable; -+ bool binder_async_ux_sts; -+ int binder_thread_mode; -+ struct binder_node *binder_thread_node; -+ -+/* for powermodel */ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ struct powermodel_cpu_task_state *powermodel_cpu_task_states[MAX_CPU_CNT]; -+ u64 exec_runtime; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_CFBT) -+ int cfbt_cur_group; -+ bool cfbt_running; -+#endif /* CONFIG_OPLUS_FEATURE_SCHED_CFBT */ -+} ____cacheline_aligned; -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+#define INVALID_PID (-1) -+struct oplus_lb { -+ /* used for active_balance to record the running task. */ -+ pid_t pid; -+}; -+/*#endif*/ -+ -+#endif /* _OPLUS_SA_COMMON_STRUCT_H_ */ -+ -diff --git a/include/linux/sched/sa_oemdata.h b/include/linux/sched/sa_oemdata.h -new file mode 100755 -index 000000000000..ebbbc754e391 ---- /dev/null -+++ b/include/linux/sched/sa_oemdata.h -@@ -0,0 +1,13 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_OEMDATA_H_ -+#define _OPLUS_SA_OEMDATA_H_ -+ -+int sa_oemdata_init(void); -+void sa_oemdata_deinit(void); -+ -+#endif /* _OPLUS_SA_OEMDATA_H_ */ -diff --git a/include/linux/sched/sched_ext.h b/include/linux/sched/sched_ext.h -new file mode 100755 -index 000000000000..9f1afdb8a04b ---- /dev/null -+++ b/include/linux/sched/sched_ext.h -@@ -0,0 +1,57 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef _OPLUS_SCHED_EXT_H -+#define _OPLUS_SCHED_EXT_H -+#include "sa_common.h" -+ -+#define SCHED_PROP_TOP_THREAD_SHIFT (8) -+#define SCHED_PROP_TOP_THREAD_MASK (0xf << SCHED_PROP_TOP_THREAD_SHIFT) -+#define SCHED_PROP_DEADLINE_MASK (0xFF) /* deadline for ext sched class */ -+#define SCHED_PROP_DEADLINE_LEVEL1 (1) /* 1ms for user-aware audio tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL2 (2) /* 2ms for user-aware touch tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL3 (3) /* 4ms for user aware dispaly tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL4 (4) /* 6ms */ -+#define SCHED_PROP_DEADLINE_LEVEL5 (5) /* 8ms */ -+#define SCHED_PROP_DEADLINE_LEVEL6 (6) /* 16ms */ -+#define SCHED_PROP_DEADLINE_LEVEL7 (7) /* 32ms */ -+#define SCHED_PROP_DEADLINE_LEVEL8 (8) /* 64ms */ -+#define SCHED_PROP_DEADLINE_LEVEL9 (9) /* 128ms */ -+ -+static inline int sched_prop_get_top_thread_id(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ return -EPERM; -+ } -+ -+ return ((ots->scx.sched_prop & SCHED_PROP_TOP_THREAD_MASK) >> SCHED_PROP_TOP_THREAD_SHIFT); -+} -+ -+static inline int sched_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_set_sched_prop failed! fn=%s\n", __func__); -+ return -EPERM; -+ } -+ -+ ots->scx.sched_prop = sp; -+ return 0; -+} -+ -+static inline unsigned long sched_get_sched_prop(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_get_sched_prop failed! fn=%s\n", __func__); -+ return (unsigned long)-1; -+ } -+ return ots->scx.sched_prop; -+} -+ -+#endif /*_OPLUS_SCHED_EXT_H */ -diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h -index 03d35e3eda3c..6a5f0c0d74bd 100644 ---- a/include/linux/sched/task.h -+++ b/include/linux/sched/task.h -@@ -61,8 +61,10 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); - extern void init_idle(struct task_struct *idle, int cpu); - - extern int sched_fork(unsigned long clone_flags, struct task_struct *p); --extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); --extern void sched_cancel_fork(struct task_struct *p); -+extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); -+#ifdef CONFIG_HMBIRD_SCHED -+extern void hmbird_cancel_fork(struct task_struct *p); -+#endif - extern void sched_post_fork(struct task_struct *p); - extern void sched_dead(struct task_struct *p); - -diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h -index 359a14cc76a4..969716ab4320 100644 ---- a/include/uapi/linux/sched.h -+++ b/include/uapi/linux/sched.h -@@ -118,7 +118,7 @@ struct clone_args { - /* SCHED_ISO: reserved but not implemented yet */ - #define SCHED_IDLE 5 - #define SCHED_DEADLINE 6 --#define SCHED_EXT 7 -+#define SCHED_HMBIRD 7 - - /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ - #define SCHED_RESET_ON_FORK 0x40000000 -diff --git a/init/Kconfig b/init/Kconfig -index f1a81cc2e412..b5c87221addb 100644 ---- a/init/Kconfig -+++ b/init/Kconfig -@@ -1021,10 +1021,6 @@ config RT_GROUP_SCHED - realtime bandwidth for them. - See Documentation/scheduler/sched-rt-group.rst for more information. - --config EXT_GROUP_SCHED -- bool -- depends on SCHED_CLASS_EXT && CGROUP_SCHED -- default y - endif #CGROUP_SCHED - - config SCHED_MM_CID -diff --git a/init/init_task.c b/init/init_task.c -index 69a046388995..31ceb0e469f7 100644 ---- a/init/init_task.c -+++ b/init/init_task.c -@@ -6,7 +6,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -102,9 +101,6 @@ struct task_struct init_task - #endif - #ifdef CONFIG_CGROUP_SCHED - .sched_task_group = &root_task_group, --#endif --#ifdef CONFIG_SLIM_SCHED -- .scx = NULL, - #endif - .ptraced = LIST_HEAD_INIT(init_task.ptraced), - .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), -diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt -index cf22ef6107b8..b131d5662b48 100644 ---- a/kernel/Kconfig.preempt -+++ b/kernel/Kconfig.preempt -@@ -133,12 +133,8 @@ config SCHED_CORE - which is the likely usage by Linux distributions, there should - be no measurable impact on performance. - --config SLIM_SCHED -- bool "slim sched" -- default n --config SCHED_CLASS_EXT -- bool "Extensible Scheduling Class" -- depends on BPF_SYSCALL && BPF_JIT -+config HMBIRD_SCHED -+ bool "hmbird Scheduling Class(base on ext scheduler)" - help - This option enables a new scheduler class sched_ext (SCX), which - allows scheduling policies to be implemented as BPF programs to -@@ -159,3 +155,10 @@ config SCHED_CLASS_EXT - similar to struct sched_class. - - See Documentation/scheduler/sched-ext.rst for more details. -+ -+config DYNAMIC_WALT_SUPPORT -+ bool "Dynamic WALT Support for Extensible Scheduling Class" -+ depends on HMBIRD_SCHED -+ default n -+ help -+ Enable dynamic WALT support for Extensible Scheduling Class. -diff --git a/kernel/bpf/bpf_struct_ops_types.h b/kernel/bpf/bpf_struct_ops_types.h -index 3618769d853d..5678a9ddf817 100644 ---- a/kernel/bpf/bpf_struct_ops_types.h -+++ b/kernel/bpf/bpf_struct_ops_types.h -@@ -9,8 +9,4 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) - #include - BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) - #endif --#ifdef CONFIG_SCHED_CLASS_EXT --#include --BPF_STRUCT_OPS_TYPE(sched_ext_ops) --#endif - #endif -diff --git a/kernel/fork.c b/kernel/fork.c -index d2f5ae66a322..bc97804f3411 100644 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -23,7 +23,9 @@ - #include - #include - #include --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - #include - #include - #include -@@ -998,7 +1000,9 @@ void __put_task_struct(struct task_struct *tsk) - WARN_ON(refcount_read(&tsk->usage)); - WARN_ON(tsk == current); - -- sched_ext_free(tsk); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_free(tsk); -+#endif - io_uring_free(tsk); - cgroup_free(tsk); - task_numa_free(tsk, true); -@@ -2511,7 +2515,11 @@ __latent_entropy struct task_struct *copy_process( - - retval = perf_event_init_task(p, clone_flags); - if (retval) -- goto bad_fork_sched_cancel_fork; -+#ifdef CONFIG_HMBIRD_SCHED -+ goto bad_fork_hmbird_cancel_fork; -+#else -+ goto bad_fork_cleanup_policy; -+#endif - retval = audit_alloc(p); - if (retval) - goto bad_fork_cleanup_perf; -@@ -2643,9 +2651,7 @@ __latent_entropy struct task_struct *copy_process( - * cgroup specific, it unconditionally needs to place the task on a - * runqueue. - */ -- retval = sched_cgroup_fork(p, args); -- if (retval) -- goto bad_fork_cancel_cgroup; -+ sched_cgroup_fork(p, args); - - /* - * From this point on we must avoid any synchronous user-space -@@ -2691,13 +2697,13 @@ __latent_entropy struct task_struct *copy_process( - /* Don't start children in a dying pid namespace */ - if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { - retval = -ENOMEM; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* Let kill terminate clone/fork in the middle */ - if (fatal_signal_pending(current)) { - retval = -EINTR; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* No more failure paths after this point. */ -@@ -2774,11 +2780,10 @@ __latent_entropy struct task_struct *copy_process( - - return p; - --bad_fork_core_free: -+bad_fork_cancel_cgroup: - sched_core_free(p); - spin_unlock(¤t->sighand->siglock); - write_unlock_irq(&tasklist_lock); --bad_fork_cancel_cgroup: - cgroup_cancel_fork(p, args); - bad_fork_put_pidfd: - if (clone_flags & CLONE_PIDFD) { -@@ -2817,8 +2822,10 @@ __latent_entropy struct task_struct *copy_process( - audit_free(p); - bad_fork_cleanup_perf: - perf_event_free_task(p); --bad_fork_sched_cancel_fork: -- sched_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+bad_fork_hmbird_cancel_fork: -+ hmbird_cancel_fork(p); -+#endif - bad_fork_cleanup_policy: - lockdep_free_task(p); - #ifdef CONFIG_NUMA -diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c -index 005025f55bea..c091539a0f60 100644 ---- a/kernel/sched/build_policy.c -+++ b/kernel/sched/build_policy.c -@@ -28,8 +28,10 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED - #include - #include -+#endif - - #include - -@@ -54,6 +56,10 @@ - #include "cputime.c" - #include "deadline.c" - --#ifdef CONFIG_SCHED_CLASS_EXT --# include "ext.c" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/hmbird_util_track.c" -+#include "hmbird/hmbird_sched_proc.c" -+#include "hmbird/hmbird_shadow_tick.c" -+#include "hmbird/hmbird.c" -+#include "hmbird/hmbird_misc.c" - #endif -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index 2adea38f9f5c..15133c67d0c3 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -96,6 +96,12 @@ - #include "../../io_uring/io-wq.h" - #include "../smpboot.h" - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/slim.h" -+#include "hmbird/hmbird_shadow_tick.h" -+#include "hmbird.h" -+#endif -+ - #include - #include - #include -@@ -170,7 +176,6 @@ __read_mostly int scheduler_running; - - DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); - -- - /* kernel prio, less is more */ - static inline int __task_prio(const struct task_struct *p) - { -@@ -183,8 +188,8 @@ static inline int __task_prio(const struct task_struct *p) - if (p->sched_class == &idle_sched_class) - return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - --#ifdef CONFIG_SCHED_CLASS_EXT -- if (p->sched_class == &ext_sched_class) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (p->sched_class == &hmbird_sched_class) - return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ - #endif - -@@ -217,9 +222,9 @@ static inline bool prio_less(const struct task_struct *a, - if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ - return cfs_prio_less(a, b, in_fi); - --#ifdef CONFIG_SCHED_CLASS_EXT -+#ifdef CONFIG_HMBIRD_SCHED - if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ -- return scx_prio_less(a, b, in_fi); -+ return hmbird_prio_less(a, b, in_fi); - #endif - - return false; -@@ -1277,17 +1282,26 @@ bool sched_can_stop_tick(struct rq *rq) - if (fifo_nr_running) - return true; - -+#ifdef CONFIG_HMBIRD_SCHED - /* -- * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks -+ * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks - * left. For CFS, if there's more than one we need the tick for -- * involuntary preemption. For SCX, ask. -+ * involuntary preemption. For HMBIRD, ask. - */ -- if (!scx_switched_all() && rq->nr_running > 1) -+ if (!hmbird_enabled() && rq->nr_running > 1) - return false; - -- if (scx_enabled() && !scx_can_stop_tick(rq)) -+ if (hmbird_enabled() && !hmbird_can_stop_tick(rq)) - return false; -- -+#else -+ /* -+ * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; -+ * if there's more than one we need the tick for involuntary -+ * preemption. -+ */ -+ if (rq->nr_running > 1) -+ return false; -+#endif - /* - * If there is one task and it has CFS runtime bandwidth constraints - * and it's on the cpu now we don't want to stop the tick. -@@ -2212,10 +2226,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) - } - EXPORT_SYMBOL_GPL(deactivate_task); - --struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) -+#ifdef CONFIG_HMBIRD_SCHED -+struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - { -- struct sched_change_guard cg = { -+ struct hmbird_sched_change_guard cg = { - .rq = rq, - .p = p, - .queued = task_on_rq_queued(p), -@@ -2237,7 +2252,7 @@ sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - return cg; - } - --void sched_change_guard_fini(struct sched_change_guard *cg, int flags) -+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags) - { - if (cg->queued) - enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); -@@ -2245,6 +2260,7 @@ void sched_change_guard_fini(struct sched_change_guard *cg, int flags) - set_next_task(cg->rq, cg->p); - cg->done = true; - } -+#endif - - static inline int __normal_prio(int policy, int rt_prio, int nice) - { -@@ -2310,9 +2326,9 @@ inline int task_curr(const struct task_struct *p) - * this means any call to check_class_changed() must be followed by a call to - * balance_callback(). - */ --void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio) -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) - { - if (prev_class != p->sched_class) { - if (prev_class->switched_from) -@@ -2875,6 +2891,7 @@ static void - __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - { - struct rq *rq = task_rq(p); -+ bool queued, running; - - /* - * This here violates the locking rules for affinity, since we're only -@@ -2893,9 +2910,26 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - else - lockdep_assert_held(&p->pi_lock); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->sched_class->set_cpus_allowed(p, ctx); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ -+ if (queued) { -+ /* -+ * Because __kthread_bind() calls this on blocked tasks without -+ * holding rq->lock. -+ */ -+ lockdep_assert_rq_held(rq); -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); - } -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->sched_class->set_cpus_allowed(p, ctx); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - } - - /* -@@ -3762,7 +3796,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class)) -+#else - if (unlikely(!is_cpu_allowed(p, cpu))) -+#endif - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -@@ -4880,7 +4918,9 @@ late_initcall(sched_core_sysctl_init); - */ - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { -+#ifdef CONFIG_HMBIRD_SCHED - int ret; -+#endif - - trace_android_rvh_sched_fork(p); - -@@ -4921,21 +4961,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_reset_on_fork = 0; - } - -- scx_pre_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ ret = hmbird_pre_fork(p); -+ if (ret) -+ goto out_cancel; - - if (dl_prio(p->prio)) { - ret = -EAGAIN; - goto out_cancel; --#ifdef CONFIG_SCHED_CLASS_EXT -- } else if (task_on_scx(p)) { -- p->sched_class = &ext_sched_class; --#endif -+ } else if (task_on_hmbird(p)) { -+ p->sched_class = &hmbird_sched_class; - } else if (rt_prio(p->prio)) { - p->sched_class = &rt_sched_class; - } else { - p->sched_class = &fair_sched_class; - } -- -+#else -+ if (dl_prio(p->prio)) -+ return -EAGAIN; -+ else if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - init_entity_runnable_average(&p->se); - trace_android_rvh_finish_prio_fork(p); - -@@ -4954,12 +5002,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - #endif - return 0; - -+#ifdef CONFIG_HMBIRD_SCHED - out_cancel: -- scx_cancel_fork(p); -+ hmbird_cancel_fork(p); - return ret; -+#endif - } - --int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) -+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - { - unsigned long flags; - -@@ -4986,19 +5036,17 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -- -- return scx_fork(p); --} -- --void sched_cancel_fork(struct task_struct *p) --{ -- scx_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_fork(p); -+#endif - } - - void sched_post_fork(struct task_struct *p) - { - uclamp_post_fork(p); -- scx_post_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_post_fork(p); -+#endif - } - - unsigned long to_ratio(u64 period, u64 runtime) -@@ -5860,20 +5908,28 @@ void scheduler_tick(void) - if (sched_feat(LATENCY_WARN) && resched_latency) - resched_latency_warn(cpu, resched_latency); - -- scx_notify_sched_tick(); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_notify_sched_tick(); -+#endif - perf_event_task_tick(); - - if (curr->flags & PF_WQ_WORKER) - wq_worker_tick(curr); - - #ifdef CONFIG_SMP -- if (!scx_switched_all()) { -+#ifdef CONFIG_HMBIRD_SCHED -+ if (!hmbird_enabled()) { -+#endif - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); -+#ifdef CONFIG_HMBIRD_SCHED - } - #endif -- -+#endif - trace_android_vh_scheduler_tick(rq); -+#ifdef CONFIG_HMBIRD_SCHED -+ scheduler_tick_handler(NULL, NULL); -+#endif - } - - #ifdef CONFIG_NO_HZ_FULL -@@ -6175,7 +6231,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ -+#ifdef CONFIG_HMBIRD_SCHED - for_balance_class_range(class, prev->sched_class, &idle_sched_class) { -+#else -+ for_class_range(class, prev->sched_class, &idle_sched_class) { -+#endif - if (class->balance(rq, prev, rf)) - break; - } -@@ -6193,9 +6253,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - const struct sched_class *class; - struct task_struct *p; - -- if (scx_enabled()) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (hmbird_enabled()) - goto restart; -- -+#endif - /* - * Optimization: we know that if all tasks are in the fair class we can - * call that function directly, but only if the @prev task wasn't of a -@@ -6221,13 +6282,21 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - restart: - put_prev_task_balance(rq, prev, rf); - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { - p = class->pick_next_task(rq); - if (p) { -- scx_notify_pick_next_task(rq, p, class); -+ hmbird_notify_pick_next_task(rq, p, class); - return p; - } - } -+#else -+ for_each_class(class) { -+ p = class->pick_next_task(rq); -+ if (p) -+ return p; -+ } -+#endif - - BUG(); /* The idle class should always have a runnable task. */ - } -@@ -6256,7 +6325,11 @@ static inline struct task_struct *pick_task(struct rq *rq) - const struct sched_class *class; - struct task_struct *p; - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { -+#else -+ for_each_class(class) { -+#endif - p = class->pick_task(rq); - if (p) - return p; -@@ -6898,6 +6971,9 @@ static void __sched notrace __schedule(unsigned int sched_mode) - psi_sched_switch(prev, next, !task_on_rq_queued(prev)); - - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#ifdef CONFIG_HMBIRD_SCHED -+ sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#endif - - /* Also unlocks the rq: */ - rq = context_switch(rq, prev, next, &rf); -@@ -7226,24 +7302,31 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag - } - EXPORT_SYMBOL(default_wake_function); - --void __setscheduler_prio(struct task_struct *p, int prio) -+static void __setscheduler_prio(struct task_struct *p, int prio) - { -- /* -- * After switching all rt and fair class to ext, -- * stop class is switched to ext class too. Just -- * skip it. */ -+#ifdef CONFIG_HMBIRD_SCHED -+ bool on_hmbird = task_on_hmbird(p); -+ - if (p->sched_class == &stop_sched_class) -- ;/* do nothing */ -+ ; - else if (dl_prio(prio)) - p->sched_class = &dl_sched_class; --#ifdef CONFIG_SCHED_CLASS_EXT -- else if (task_on_scx(p)) -- p->sched_class = &ext_sched_class; --#endif -+ else if (rt_prio(prio) && on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#else -+ if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; - else if (rt_prio(prio)) - p->sched_class = &rt_sched_class; - else - p->sched_class = &fair_sched_class; -+#endif - - p->prio = prio; - } -@@ -7278,7 +7361,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - */ - void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - { -- int prio, oldprio, queue_flag = -+ int prio, oldprio, queued, running, queue_flag = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *prev_class; - struct rq_flags rf; -@@ -7341,42 +7424,52 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - queue_flag &= ~DEQUEUE_MOVE; - - prev_class = p->sched_class; -- SCHED_CHANGE_BLOCK(rq, p, queue_flag) { -- /* -- * Boosting condition are: -- * 1. -rt task is running and holds mutex A -- * --> -dl task blocks on mutex A -- * -- * 2. -dl task is running and holds mutex A -- * --> -dl task blocks on mutex A and could preempt the -- * running task -- */ -- if (dl_prio(prio)) { -- if (!dl_prio(p->normal_prio) || -- (pi_task && dl_prio(pi_task->prio) && -- dl_entity_preempt(&pi_task->dl, &p->dl))) { -- p->dl.pi_se = pi_task->dl.pi_se; -- queue_flag |= ENQUEUE_REPLENISH; -- } else { -- p->dl.pi_se = &p->dl; -- } -- } else if (rt_prio(prio)) { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- if (oldprio < prio) -- queue_flag |= ENQUEUE_HEAD; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flag); -+ if (running) -+ put_prev_task(rq, p); -+ -+ /* -+ * Boosting condition are: -+ * 1. -rt task is running and holds mutex A -+ * --> -dl task blocks on mutex A -+ * -+ * 2. -dl task is running and holds mutex A -+ * --> -dl task blocks on mutex A and could preempt the -+ * running task -+ */ -+ if (dl_prio(prio)) { -+ if (!dl_prio(p->normal_prio) || -+ (pi_task && dl_prio(pi_task->prio) && -+ dl_entity_preempt(&pi_task->dl, &p->dl))) { -+ p->dl.pi_se = pi_task->dl.pi_se; -+ queue_flag |= ENQUEUE_REPLENISH; - } else { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- else if (rt_prio(oldprio)) -- p->rt.timeout = 0; -- else if (!task_has_idle_policy(p)) -- reweight_task(p, prio - MAX_RT_PRIO); -+ p->dl.pi_se = &p->dl; - } -- -- __setscheduler_prio(p, prio); -+ } else if (rt_prio(prio)) { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ if (oldprio < prio) -+ queue_flag |= ENQUEUE_HEAD; -+ } else { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ else if (rt_prio(oldprio)) -+ p->rt.timeout = 0; -+ else if (!task_has_idle_policy(p)) -+ reweight_task(p, prio - MAX_RT_PRIO); - } - -+ __setscheduler_prio(p, prio); -+ -+ if (queued) -+ enqueue_task(rq, p, queue_flag); -+ if (running) -+ set_next_task(rq, p); -+ - check_class_changed(rq, p, prev_class, oldprio); - out_unlock: - /* Avoid rq from going away on us: */ -@@ -7397,7 +7490,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - - void set_user_nice(struct task_struct *p, long nice) - { -- bool allowed = false; -+ bool queued, running, allowed = false; - int old_prio; - struct rq_flags rf; - struct rq *rq; -@@ -7426,13 +7519,22 @@ void set_user_nice(struct task_struct *p, long nice) - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); -+ if (running) -+ put_prev_task(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->static_prio = NICE_TO_PRIO(nice); -- set_load_weight(p, true); -- old_prio = p->prio; -- p->prio = effective_prio(p); -- } -+ p->static_prio = NICE_TO_PRIO(nice); -+ set_load_weight(p, true); -+ old_prio = p->prio; -+ p->prio = effective_prio(p); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - - /* - * If the task increased its priority or is running and -@@ -7835,7 +7937,7 @@ static int __sched_setscheduler(struct task_struct *p, - bool user, bool pi) - { - int oldpolicy = -1, policy = attr->sched_policy; -- int retval, oldprio, newprio; -+ int retval, oldprio, newprio, queued, running; - const struct sched_class *prev_class; - struct balance_callback *head; - struct rq_flags rf; -@@ -7843,6 +7945,9 @@ static int __sched_setscheduler(struct task_struct *p, - int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct rq *rq; - bool cpuset_locked = false; -+#ifdef CONFIG_HMBIRD_SCHED -+ unsigned long flags; -+#endif - - /* The pi code expects interrupts enabled */ - BUG_ON(pi && in_interrupt()); -@@ -7908,6 +8013,9 @@ static int __sched_setscheduler(struct task_struct *p, - * To be able to change p->policy safely, the appropriate - * runqueue lock must be held. - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+#endif - rq = task_rq_lock(p, &rf); - update_rq_clock(rq); - -@@ -7919,10 +8027,11 @@ static int __sched_setscheduler(struct task_struct *p, - goto unlock; - } - -- retval = scx_check_setscheduler(p, policy); -+#ifdef CONFIG_HMBIRD_SCHED -+ retval = hmbird_check_setscheduler(p, policy); - if (retval) - goto unlock; -- -+#endif - /* - * If not changing anything there's no need to proceed further, - * but store a possible modification of reset_on_fork. -@@ -7979,6 +8088,9 @@ static int __sched_setscheduler(struct task_struct *p, - if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { - policy = oldpolicy = -1; - task_rq_unlock(rq, p, &rf); -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - goto recheck; -@@ -8011,16 +8123,23 @@ static int __sched_setscheduler(struct task_struct *p, - queue_flags &= ~DEQUEUE_MOVE; - } - -- SCHED_CHANGE_BLOCK(rq, p, queue_flags) { -- prev_class = p->sched_class; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flags); -+ if (running) -+ put_prev_task(rq, p); - -- if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -- __setscheduler_params(p, attr); -- __setscheduler_prio(p, newprio); -- trace_android_rvh_setscheduler(p); -- } -- __setscheduler_uclamp(p, attr); -+ prev_class = p->sched_class; - -+ if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -+ __setscheduler_params(p, attr); -+ __setscheduler_prio(p, newprio); -+ trace_android_rvh_setscheduler(p); -+ } -+ __setscheduler_uclamp(p, attr); -+ -+ if (queued) { - /* - * We enqueue to tail when the priority of a task is - * increased (user space view). -@@ -8028,7 +8147,10 @@ static int __sched_setscheduler(struct task_struct *p, - if (oldprio < p->prio) - queue_flags |= ENQUEUE_HEAD; - -+ enqueue_task(rq, p, queue_flags); - } -+ if (running) -+ set_next_task(rq, p); - - check_class_changed(rq, p, prev_class, oldprio); - -@@ -8036,7 +8158,9 @@ static int __sched_setscheduler(struct task_struct *p, - preempt_disable(); - head = splice_balance_callbacks(rq); - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (pi) { - if (cpuset_locked) - cpuset_unlock(); -@@ -8051,6 +8175,9 @@ static int __sched_setscheduler(struct task_struct *p, - - unlock: - task_rq_unlock(rq, p, &rf); -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - return retval; -@@ -9272,7 +9399,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - break; - } -@@ -9300,7 +9429,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - } - return ret; -@@ -9605,15 +9736,25 @@ int migrate_task_to(struct task_struct *p, int target_cpu) - */ - void sched_setnuma(struct task_struct *p, int nid) - { -+ bool queued, running; - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) { -- p->numa_preferred_nid = nid; -- } -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE); -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->numa_preferred_nid = nid; - -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - task_rq_unlock(rq, p, &rf); - } - #endif /* CONFIG_NUMA_BALANCING */ -@@ -10153,15 +10294,22 @@ void __init sched_init(void) - int i; - - /* Make sure the linker didn't screw up */ -+#ifdef CONFIG_HMBIRD_SCHED - #ifdef CONFIG_SMP -- BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+#endif -+ WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class)); -+#else -+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 || -+ &fair_sched_class != &rt_sched_class + 1 || -+ &rt_sched_class != &dl_sched_class + 1); -+#ifdef CONFIG_SMP -+ BUG_ON(&dl_sched_class != &stop_sched_class + 1); - #endif -- BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); -- BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); -- BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); --#ifdef CONFIG_SCHED_CLASS_EXT -- BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); -- BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); - #endif - - wait_bit_init(); -@@ -10332,8 +10480,9 @@ void __init sched_init(void) - balance_push_set(smp_processor_id(), false); - #endif - init_sched_fair_class(); -- init_sched_ext_class(); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ init_sched_hmbird_class(); -+#endif - psi_init(); - - init_uclamp(); -@@ -10743,6 +10892,8 @@ static void sched_change_group(struct task_struct *tsk, struct task_group *group - */ - void sched_move_task(struct task_struct *tsk) - { -+ int queued, running, queue_flags = -+ DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct task_group *group; - struct rq_flags rf; - struct rq *rq; -@@ -10759,18 +10910,27 @@ void sched_move_task(struct task_struct *tsk) - - update_rq_clock(rq); - -- SCHED_CHANGE_BLOCK(rq, tsk, -- DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { -- sched_change_group(tsk, group); -- } -+ running = task_current(rq, tsk); -+ queued = task_on_rq_queued(tsk); - -- /* -- * After changing group, the running task may have joined a throttled -- * one but it's still the running task. Trigger a resched to make sure -- * that task can still run. -- */ -- if (task_current(rq, tsk)) -+ if (queued) -+ dequeue_task(rq, tsk, queue_flags); -+ if (running) -+ put_prev_task(rq, tsk); -+ -+ sched_change_group(tsk, group); -+ -+ if (queued) -+ enqueue_task(rq, tsk, queue_flags); -+ if (running) { -+ set_next_task(rq, tsk); -+ /* -+ * After changing group, the running task may have joined a -+ * throttled one but it's still the running task. Trigger a -+ * resched to make sure that task can still run. -+ */ - resched_curr(rq); -+ } - - unlock: - task_rq_unlock(rq, tsk, &rf); -@@ -10808,6 +10968,10 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) - struct task_group *tg = css_tg(css); - struct task_group *parent = css_tg(css->parent); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_tg_online(tg); -+#endif -+ - if (parent) - sched_online_group(tg, parent); - -@@ -10857,13 +11021,79 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) - } - #endif - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 dl) -+{ -+ int i; -+ -+ for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) { -+ if (dl < HMBIRD_BPF_DSQS_DEADLINE[i]) -+ break; -+ } -+ -+ i = max_t(int, i-1, MIN_CGROUP_DL_IDX); -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return 0; -+ update_cgroup_ids_table(css->cgroup->kn->id, i); -+ -+ return 0; -+} -+ -+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ u8 i; -+ -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX]; -+ i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1); -+ if (i < 0) { -+ pr_err(" <%s> i is %d, less than 0, name is %s\n", -+ __func__, i, css->cgroup->kn->name); -+ i = DEFAULT_CGROUP_DL_IDX; -+ } -+ -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[i]; -+} -+ -+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css, -+ struct task_struct *p, int prio) -+{ -+ if (!css || !rt_prio(prio)) -+ return; -+ -+ if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) { -+ if (!strcmp(css->cgroup->kn->name, "display")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3); -+ else if (!strcmp(css->cgroup->kn->name, "touch")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2); -+ else if (!strcmp(css->cgroup->kn->name, "multimedia")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+ } -+} -+#endif -+ - static void cpu_cgroup_attach(struct cgroup_taskset *tset) - { - struct task_struct *task; - struct cgroup_subsys_state *css; - -- cgroup_taskset_for_each(task, css, tset) -+ cgroup_taskset_for_each(task, css, tset) { -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_change_rt_sched_prop(css, task, task->prio); -+#endif - sched_move_task(task); -+ } - - trace_android_rvh_cpu_cgroup_attach(tset); - } -@@ -11542,6 +11772,13 @@ static struct cftype cpu_legacy_files[] = { - .read_u64 = cpu_uclamp_ls_read_u64, - .write_u64 = cpu_uclamp_ls_write_u64, - }, -+#endif -+#ifdef CONFIG_HMBIRD_SCHED -+ { -+ .name = "hmbird.deadline", -+ .read_u64 = cgroup_read_hmbird_deadline, -+ .write_u64 = cgroup_write_hmbird_deadline, -+ }, - #endif - { } /* Terminate */ - }; -@@ -11591,7 +11828,6 @@ static int cpu_local_stat_show(struct seq_file *sf, - } - - #ifdef CONFIG_FAIR_GROUP_SCHED -- - static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, - struct cftype *cft) - { -diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c -index 423404100c9a..2fe1a734e640 100644 ---- a/kernel/sched/debug.c -+++ b/kernel/sched/debug.c -@@ -376,8 +376,8 @@ static __init int sched_init_debug(void) - - debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); - --#ifdef CONFIG_SCHED_CLASS_EXT -- debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops); -+#ifdef CONFIG_HMBIRD_SCHED -+ debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops); - #endif - return 0; - } -@@ -1006,9 +1006,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - SEQ_printf(m, - "---------------------------------------------------------" - "----------\n"); --#ifdef CONFIG_SLIM_SCHED -- SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop); --#endif - - #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) - #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) -@@ -1102,8 +1099,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - P(dl.runtime); - P(dl.deadline); - } --#ifdef CONFIG_SCHED_CLASS_EXT -- __PS("ext.enabled", p->sched_class == &ext_sched_class); -+#ifdef CONFIG_HMBIRD_SCHED -+ __PS("hmbird.enabled", p->sched_class == &hmbird_sched_class); -+ __PS("sched_prop", get_hmbird_ts(p)->sched_prop); -+ __PS("top_task_prop", get_hmbird_ts(p)->top_task_prop); - #endif - #undef PN_SCHEDSTAT - #undef P_SCHEDSTAT -diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c -deleted file mode 100755 -index 4845d441df2b..000000000000 ---- a/kernel/sched/ext.c -+++ /dev/null -@@ -1,3978 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -- --enum scx_internal_consts { -- SCX_NR_ONLINE_OPS = SCX_OP_IDX(init), -- SCX_DSP_DFL_MAX_BATCH = 32, -- SCX_DSP_MAX_LOOPS = 32, -- SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, --}; -- --enum scx_ops_enable_state { -- SCX_OPS_PREPPING, -- SCX_OPS_ENABLING, -- SCX_OPS_ENABLED, -- SCX_OPS_DISABLING, -- SCX_OPS_DISABLED, --}; -- --static inline struct task_group *css_tg(struct cgroup_subsys_state *css) --{ -- return css ? container_of(css, struct task_group, css) : NULL; --} -- --/* -- * sched_ext_entity->ops_state -- * -- * Used to track the task ownership between the SCX core and the BPF scheduler. -- * State transitions look as follows: -- * -- * NONE -> QUEUEING -> QUEUED -> DISPATCHING -- * ^ | | -- * | v v -- * \-------------------------------/ -- * -- * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -- * sites for explanations on the conditions being waited upon and why they are -- * safe. Transitions out of them into NONE or QUEUED must store_release and the -- * waiters should load_acquire. -- * -- * Tracking scx_ops_state enables sched_ext core to reliably determine whether -- * any given task can be dispatched by the BPF scheduler at all times and thus -- * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -- * to try to dispatch any task anytime regardless of its state as the SCX core -- * can safely reject invalid dispatches. -- */ --enum scx_ops_state { -- SCX_OPSS_NONE, /* owned by the SCX core */ -- SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -- SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ -- SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ -- -- /* -- * QSEQ brands each QUEUED instance so that, when dispatch races -- * dequeue/requeue, the dispatcher can tell whether it still has a claim -- * on the task being dispatched. -- */ -- SCX_OPSS_QSEQ_SHIFT = 2, -- SCX_OPSS_STATE_MASK = (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1, -- SCX_OPSS_QSEQ_MASK = ~SCX_OPSS_STATE_MASK, --}; -- --/* -- * During exit, a task may schedule after losing its PIDs. When disabling the -- * BPF scheduler, we need to be able to iterate tasks in every state to -- * guarantee system safety. Maintain a dedicated task list which contains every -- * task between its fork and eventual free. -- */ --static DEFINE_SPINLOCK(scx_tasks_lock); --static LIST_HEAD(scx_tasks); -- --/* ops enable/disable */ --static struct kthread_worker *scx_ops_helper; --static DEFINE_MUTEX(scx_ops_enable_mutex); --DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); --DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); --static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); --static bool scx_switch_all_req; --static bool scx_switching_all; --DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -- --static struct sched_ext_ops scx_ops; --static bool scx_warned_zero_slice; -- --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); --DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); --static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); -- --struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] = -- { [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT }; -- --static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE); --static struct scx_exit_info scx_exit_info; -- --static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0); -- --/* -- * The maximum amount of time in jiffies that a task may be runnable without -- * being scheduled on a CPU. If this timeout is exceeded, it will trigger -- * scx_ops_error(). -- */ --unsigned long scx_watchdog_timeout; -- --/* -- * The last time the delayed work was run. This delayed work relies on -- * ksoftirqd being able to run to service timer interrupts, so it's possible -- * that this work itself could get wedged. To account for this, we check that -- * it's not stalled in the timer tick, and trigger an error if it is. -- */ --unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; -- --static struct delayed_work scx_watchdog_work; -- --/* idle tracking */ --#ifdef CONFIG_SMP --#ifdef CONFIG_CPUMASK_OFFSTACK --#define CL_ALIGNED_IF_ONSTACK --#else --#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp --#endif -- --static struct { -- cpumask_var_t cpu; -- cpumask_var_t smt; --} idle_masks CL_ALIGNED_IF_ONSTACK; -- --static bool __cacheline_aligned_in_smp scx_has_idle_cpus; --#endif /* CONFIG_SMP */ -- --/* for %SCX_KICK_WAIT */ --static u64 __percpu *scx_kick_cpus_pnt_seqs; -- --/* -- * Direct dispatch marker. -- * -- * Non-NULL values are used for direct dispatch from enqueue path. A valid -- * pointer points to the task currently being enqueued. An ERR_PTR value is used -- * to indicate that direct dispatch has already happened. -- */ --static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); -- --/* dispatch queues */ --static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; -- --static LLIST_HEAD(dsqs_to_free); -- --/* dispatch buf */ --struct scx_dsp_buf_ent { -- struct task_struct *task; -- u64 qseq; -- u64 dsq_id; -- u64 enq_flags; --}; -- --static u32 scx_dsp_max_batch; --static struct scx_dsp_buf_ent __percpu *scx_dsp_buf; -- --struct scx_dsp_ctx { -- struct rq *rq; -- struct rq_flags *rf; -- u32 buf_cursor; -- u32 nr_tasks; --}; -- --static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx); -- --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags); --void scx_bpf_kick_cpu(s32 cpu, u64 flags); -- --struct scx_task_iter { -- struct sched_ext_entity cursor; -- struct task_struct *locked; -- struct rq *rq; -- struct rq_flags rf; --}; -- --#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) -- --/* if the highest set bit is N, return a mask with bits [N+1, 31] set */ --static u32 higher_bits(u32 flags) --{ -- return ~((1 << fls(flags)) - 1); --} -- --/* return the mask with only the highest bit set */ --static u32 highest_bit(u32 flags) --{ -- int bit = fls(flags); -- return bit ? 1 << (bit - 1) : 0; --} -- --/* -- * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX -- * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate -- * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check -- * whether it's running from an allowed context. -- * -- * @mask is constant, always inline to cull the mask calculations. -- */ --static __always_inline void scx_kf_allow(u32 mask) --{ -- /* nesting is allowed only in increasing scx_kf_mask order */ -- WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask, -- "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n", -- current->scx->kf_mask, mask); -- current->scx->kf_mask |= mask; --} -- --static void scx_kf_disallow(u32 mask) --{ -- current->scx->kf_mask &= ~mask; --} -- --#define SCX_CALL_OP(mask, op, args...) \ --do { \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- scx_ops.op(args); \ -- } \ --} while (0) -- --#define SCX_CALL_OP_RET(mask, op, args...) \ --({ \ -- __typeof__(scx_ops.op(args)) __ret; \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- __ret = scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- __ret = scx_ops.op(args); \ -- } \ -- __ret; \ --}) -- --/* -- * Some kfuncs are allowed only on the tasks that are subjects of the -- * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such -- * restrictions, the following SCX_CALL_OP_*() variants should be used when -- * invoking scx_ops operations that take task arguments. These can only be used -- * for non-nesting operations due to the way the tasks are tracked. -- * -- * kfuncs which can only operate on such tasks can in turn use -- * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on -- * the specific task. -- */ --#define SCX_CALL_OP_TASK(mask, op, task, args...) \ --do { \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- SCX_CALL_OP(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ --} while (0) -- --#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ --({ \ -- __typeof__(scx_ops.op(task, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- __ret; \ --}) -- --#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ --({ \ -- __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task0; \ -- current->scx->kf_tasks[1] = task1; \ -- __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- current->scx->kf_tasks[1] = NULL; \ -- __ret; \ --}) -- --//#include "./slim_walt.c" -- --/* @mask is constant, always inline to cull unnecessary branches */ --static __always_inline bool scx_kf_allowed(u32 mask) --{ -- if (unlikely(!(current->scx->kf_mask & mask))) { -- scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", -- mask, current->scx->kf_mask); -- return false; -- } -- -- if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) && -- in_interrupt())) { -- scx_ops_error("sleepable kfunc called from non-sleepable context"); -- return false; -- } -- -- /* -- * Enforce nesting boundaries. e.g. A kfunc which can be called from -- * DISPATCH must not be called if we're running DEQUEUE which is nested -- * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE -- * boundary thanks to the above in_interrupt() check. -- */ -- if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && -- (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { -- scx_ops_error("cpu_release kfunc called from a nested operation"); -- return false; -- } -- -- if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && -- (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) { -- scx_ops_error("dispatch kfunc called from a nested operation"); -- return false; -- } -- -- return true; --} -- --/* see SCX_CALL_OP_TASK() */ --static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, -- struct task_struct *p) --{ -- if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED)) -- return false; -- -- if (unlikely((p != current->scx->kf_tasks[0] && -- p != current->scx->kf_tasks[1]))) { -- scx_ops_error("called on a task not being operated on"); -- return false; -- } -- -- return true; --} -- --/** -- * scx_task_iter_init - Initialize a task iterator -- * @iter: iterator to init -- * -- * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized, -- * @iter must eventually be exited with scx_task_iter_exit(). -- * -- * scx_tasks_lock may be released between this and the first next() call or -- * between any two next() calls. If scx_tasks_lock is released between two -- * next() calls, the caller is responsible for ensuring that the task being -- * iterated remains accessible either through RCU read lock or obtaining a -- * reference count. -- * -- * All tasks which existed when the iteration started are guaranteed to be -- * visited as long as they still exist. -- */ --static void scx_task_iter_init(struct scx_task_iter *iter) --{ -- lockdep_assert_held(&scx_tasks_lock); -- -- iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; -- list_add(&iter->cursor.tasks_node, &scx_tasks); -- iter->locked = NULL; --} -- --/** -- * scx_task_iter_exit - Exit a task iterator -- * @iter: iterator to exit -- * -- * Exit a previously initialized @iter. Must be called with scx_tasks_lock held. -- * If the iterator holds a task's rq lock, that rq lock is released. See -- * scx_task_iter_init() for details. -- */ --static void scx_task_iter_exit(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- if (list_empty(cursor)) -- return; -- -- list_del_init(cursor); --} -- --/** -- * scx_task_iter_next - Next task -- * @iter: iterator to walk -- * -- * Visit the next task. See scx_task_iter_init() for details. -- */ --static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- struct sched_ext_entity *pos; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- list_for_each_entry(pos, cursor, tasks_node) { -- if (&pos->tasks_node == &scx_tasks) -- return NULL; -- if (!(pos->flags & SCX_TASK_CURSOR)) { -- list_move(cursor, &pos->tasks_node); -- return pos->task; -- } -- } -- -- /* can't happen, should always terminate at scx_tasks above */ -- BUG(); --} -- --/** -- * scx_task_iter_next_filtered - Next non-idle task -- * @iter: iterator to walk -- * -- * Visit the next non-idle task. See scx_task_iter_init() for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- while ((p = scx_task_iter_next(iter))) { -- if (!is_idle_task(p)) -- return p; -- } -- return NULL; --} -- --/** -- * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked -- * @iter: iterator to walk -- * -- * Visit the next non-idle task with its rq lock held. See scx_task_iter_init() -- * for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered_locked(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- p = scx_task_iter_next_filtered(iter); -- if (!p) -- return NULL; -- -- iter->rq = task_rq_lock(p, &iter->rf); -- iter->locked = p; -- return p; --} -- --static enum scx_ops_enable_state scx_ops_enable_state(void) --{ -- return atomic_read(&scx_ops_enable_state_var); --} -- --static enum scx_ops_enable_state --scx_ops_set_enable_state(enum scx_ops_enable_state to) --{ -- return atomic_xchg(&scx_ops_enable_state_var, to); --} -- --static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, -- enum scx_ops_enable_state from) --{ -- int from_v = from; -- -- return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); --} -- --static bool scx_ops_disabling(void) --{ -- return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING); --} -- --/** -- * wait_ops_state - Busy-wait the specified ops state to end -- * @p: target task -- * @opss: state to wait the end of -- * -- * Busy-wait for @p to transition out of @opss. This can only be used when the -- * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also -- * has load_acquire semantics to ensure that the caller can see the updates made -- * in the enqueueing and dispatching paths. -- */ --static void wait_ops_state(struct task_struct *p, u64 opss) --{ -- do { -- cpu_relax(); -- } while (atomic64_read_acquire(&p->scx->ops_state) == opss); --} -- --/** -- * ops_cpu_valid - Verify a cpu number -- * @cpu: cpu number which came from a BPF ops -- * -- * @cpu is a cpu number which came from the BPF scheduler and can be any value. -- * Verify that it is in range and one of the possible cpus. -- */ --static bool ops_cpu_valid(s32 cpu) --{ -- return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); --} -- --/** -- * ops_sanitize_err - Sanitize a -errno value -- * @ops_name: operation to blame on failure -- * @err: -errno value to sanitize -- * -- * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return -- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can -- * cause misbehaviors. For an example, a large negative return from -- * ops.prep_enable() triggers an oops when passed up the call chain because the -- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is -- * handled as a pointer. -- */ --static int ops_sanitize_err(const char *ops_name, s32 err) --{ -- if (err < 0 && err >= -MAX_ERRNO) -- return err; -- -- scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); -- return -EPROTO; --} -- --/** -- * touch_core_sched - Update timestamp used for core-sched task ordering -- * @rq: rq to read clock from, must be locked -- * @p: task to update the timestamp for -- * -- * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to -- * implement global or local-DSQ FIFO ordering for core-sched. Should be called -- * when a task becomes runnable and its turn on the CPU ends (e.g. slice -- * exhaustion). -- */ --static void touch_core_sched(struct rq *rq, struct task_struct *p) --{ --#ifdef CONFIG_SCHED_CORE -- /* -- * It's okay to update the timestamp spuriously. Use -- * sched_core_disabled() which is cheaper than enabled(). -- */ -- if (!sched_core_disabled()) -- p->scx->core_sched_at = rq_clock_task(rq); --#endif --} -- --/** -- * touch_core_sched_dispatch - Update core-sched timestamp on dispatch -- * @rq: rq to read clock from, must be locked -- * @p: task being dispatched -- * -- * If the BPF scheduler implements custom core-sched ordering via -- * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO -- * ordering within each local DSQ. This function is called from dispatch paths -- * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect. -- */ --static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- assert_clock_updated(rq); -- --#ifdef CONFIG_SCHED_CORE -- if (SCX_HAS_OP(core_sched_before)) -- touch_core_sched(rq, p); --#endif --} -- --static void update_curr_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- u64 now = rq_clock_task(rq); -- u64 delta_exec; -- -- if (time_before_eq64(now, curr->se.exec_start)) -- return; -- -- delta_exec = now - curr->se.exec_start; -- curr->se.exec_start = now; -- curr->se.sum_exec_runtime += delta_exec; -- account_group_exec_runtime(curr, delta_exec); -- cgroup_account_cputime(curr, delta_exec); -- -- if (curr->scx->slice != SCX_SLICE_INF) { -- curr->scx->slice -= min(curr->scx->slice, delta_exec); -- if (!curr->scx->slice) -- touch_core_sched(rq, curr); -- } --} -- --static bool scx_dsq_priq_less(struct rb_node *node_a, -- const struct rb_node *node_b) --{ -- const struct sched_ext_entity *a = -- container_of(node_a, struct sched_ext_entity, dsq_node.priq); -- const struct sched_ext_entity *b = -- container_of(node_b, struct sched_ext_entity, dsq_node.priq); -- -- return time_before64(a->dsq_vtime, b->dsq_vtime); --} -- --static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, -- u64 enq_flags) --{ -- bool is_local = dsq->id == SCX_DSQ_LOCAL; -- -- WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo)); -- WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq)); -- -- if (!is_local) { -- raw_spin_lock(&dsq->lock); -- if (unlikely(dsq->id == SCX_DSQ_INVALID)) { -- scx_ops_error("attempting to dispatch to a destroyed dsq"); -- /* fall back to the global dsq */ -- raw_spin_unlock(&dsq->lock); -- dsq = &scx_dsq_global; -- raw_spin_lock(&dsq->lock); -- } -- } -- -- if (enq_flags & SCX_ENQ_DSQ_PRIQ) { -- p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; -- rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq, -- scx_dsq_priq_less); -- } else { -- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) -- list_add(&p->scx->dsq_node.fifo, &dsq->fifo); -- else -- list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo); -- } -- dsq->nr++; -- p->scx->dsq = dsq; -- -- /* -- * We're transitioning out of QUEUEING or DISPATCHING. store_release to -- * match waiters' load_acquire. -- */ -- if (enq_flags & SCX_ENQ_CLEAR_OPSS) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- if (is_local) { -- struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq); -- struct rq *rq = scx->rq; -- bool preempt = false; -- -- if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && -- rq->curr->sched_class == &ext_sched_class) { -- rq->curr->scx->slice = 0; -- preempt = true; -- } -- -- if (preempt || sched_class_above(&ext_sched_class, -- rq->curr->sched_class)) -- resched_curr(rq); -- } else { -- raw_spin_unlock(&dsq->lock); -- } --} -- --static void task_unlink_from_dsq(struct task_struct *p, -- struct scx_dispatch_q *dsq) --{ -- if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { -- rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; -- } else { -- list_del_init(&p->scx->dsq_node.fifo); -- } -- --} -- --static bool task_linked_on_dsq(struct task_struct *p) --{ -- return !list_empty(&p->scx->dsq_node.fifo) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq); --} -- --static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p) --{ -- struct scx_dispatch_q *dsq = p->scx->dsq; -- bool is_local = dsq == &scx_rq->local_dsq; -- -- if (!dsq) { -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- /* -- * When dispatching directly from the BPF scheduler to a local -- * DSQ, the task isn't associated with any DSQ but -- * @p->scx->holding_cpu may be set under the protection of -- * %SCX_OPSS_DISPATCHING. -- */ -- if (p->scx->holding_cpu >= 0) -- p->scx->holding_cpu = -1; -- return; -- } -- -- if (!is_local) -- raw_spin_lock(&dsq->lock); -- -- /* -- * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node -- * can't change underneath us. -- */ -- if (p->scx->holding_cpu < 0) { -- /* @p must still be on @dsq, dequeue */ -- WARN_ON_ONCE(!task_linked_on_dsq(p)); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- } else { -- /* -- * We're racing against dispatch_to_local_dsq() which already -- * removed @p from @dsq and set @p->scx->holding_cpu. Clear the -- * holding_cpu which tells dispatch_to_local_dsq() that it lost -- * the race. -- */ -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- p->scx->holding_cpu = -1; -- } -- p->scx->dsq = NULL; -- -- if (!is_local) -- raw_spin_unlock(&dsq->lock); --} -- --static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) --{ -- lockdep_assert(rcu_read_lock_any_held()); -- -- return &scx_dsq_global; --} -- --static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, -- struct task_struct *p) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id == SCX_DSQ_LOCAL) -- return &rq->scx->local_dsq; -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", -- dsq_id, p->comm, p->pid); -- return &scx_dsq_global; -- } -- -- return dsq; --} -- --static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- -- /* @p must match the task which is being enqueued */ -- if (unlikely(p != ddsp_task)) { -- if (IS_ERR(ddsp_task)) -- scx_ops_error("%s[%d] already direct-dispatched", -- p->comm, p->pid); -- else -- scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]", -- ddsp_task->comm, ddsp_task->pid, -- p->comm, p->pid); -- return; -- } -- -- /* -- * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because -- * dispatching to the local DSQ of a different CPU requires unlocking -- * the current rq which isn't allowed in the enqueue path. Use -- * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL. -- */ -- if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) { -- scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch"); -- return; -- } -- -- touch_core_sched_dispatch(task_rq(p), p); -- -- dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- -- /* -- * Mark that dispatch already happened by spoiling direct_dispatch_task -- * with a non-NULL value which can never match a valid task pointer. -- */ -- __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); --} -- --static bool test_rq_online(struct rq *rq) --{ --#ifdef CONFIG_SMP -- return rq->online; --#else -- return true; --#endif --} -- --static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- int sticky_cpu) --{ -- struct task_struct **ddsp_taskp; -- u64 qseq; -- -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- if (p->scx->flags & SCX_TASK_ENQ_LOCAL) { -- enq_flags |= SCX_ENQ_LOCAL; -- p->scx->flags &= ~SCX_TASK_ENQ_LOCAL; -- } -- -- /* rq migration */ -- if (sticky_cpu == cpu_of(rq)) -- goto local_norefill; -- -- /* -- * If !rq->online, we already told the BPF scheduler that the CPU is -- * offline. We're just trying to on/offline the CPU. Don't bother the -- * BPF scheduler. -- */ -- if (unlikely(!test_rq_online(rq))) -- goto local; -- -- /* see %SCX_OPS_ENQ_EXITING */ -- if (!static_branch_unlikely(&scx_ops_enq_exiting) && -- unlikely(p->flags & PF_EXITING)) -- goto local; -- -- /* see %SCX_OPS_ENQ_LAST */ -- if (!static_branch_unlikely(&scx_ops_enq_last) && -- (enq_flags & SCX_ENQ_LAST)) -- goto local; -- -- if (!SCX_HAS_OP(enqueue)) { -- if (enq_flags & SCX_ENQ_LOCAL) -- goto local; -- else -- goto global; -- } -- -- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ -- qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; -- -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq); -- -- ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); -- WARN_ON_ONCE(*ddsp_taskp); -- *ddsp_taskp = p; -- -- SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); -- -- /* -- * If not directly dispatched, QUEUEING isn't clear yet and dispatch or -- * dequeue may be waiting. The store_release matches their load_acquire. -- */ -- if (*ddsp_taskp == p) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq); -- *ddsp_taskp = NULL; -- return; -- --local: -- /* -- * For task-ordering, slice refill must be treated as implying the end -- * of the current slice. Otherwise, the longer @p stays on the CPU, the -- * higher priority it becomes from scx_prio_less()'s POV. -- */ -- touch_core_sched(rq, p); -- p->scx->slice = SCX_SLICE_DFL; --local_norefill: -- dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags); -- return; -- --global: -- touch_core_sched(rq, p); /* see the comment in local: */ -- p->scx->slice = SCX_SLICE_DFL; -- dispatch_enqueue(&scx_dsq_global, p, enq_flags); --} -- --static bool watchdog_task_watched(const struct task_struct *p) --{ -- return !list_empty(&p->scx->watchdog_node); --} -- --static void watchdog_watch_task(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- if (p->scx->flags & SCX_TASK_WATCHDOG_RESET) -- p->scx->runnable_at = jiffies; -- p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET; -- list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list); --} -- --static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) --{ -- list_del_init(&p->scx->watchdog_node); -- if (reset_timeout) -- p->scx->flags |= SCX_TASK_WATCHDOG_RESET; --} -- --static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) --{ -- int sticky_cpu = p->scx->sticky_cpu; -- -- enq_flags |= rq->scx->extra_enq_flags; -- -- if (sticky_cpu >= 0) -- p->scx->sticky_cpu = -1; -- -- /* -- * Restoring a running task will be immediately followed by -- * set_next_task_scx() which expects the task to not be on the BPF -- * scheduler as tasks can only start running through local DSQs. Force -- * direct-dispatch into the local DSQ by setting the sticky_cpu. -- */ -- if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -- sticky_cpu = cpu_of(rq); -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- WARN_ON_ONCE(!watchdog_task_watched(p)); -- return; -- } -- -- watchdog_watch_task(rq, p); -- p->scx->flags |= SCX_TASK_QUEUED; -- rq->scx->nr_running++; -- add_nr_running(rq, 1); -- -- if (SCX_HAS_OP(runnable)) -- SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); -- -- if (enq_flags & SCX_ENQ_WAKEUP) -- touch_core_sched(rq, p); -- -- do_enqueue_task(rq, p, enq_flags, sticky_cpu); --} -- --static void ops_dequeue(struct task_struct *p, u64 deq_flags) --{ -- u64 opss; -- -- watchdog_unwatch_task(p, false); -- -- /* acquire ensures that we see the preceding updates on QUEUED */ -- opss = atomic64_read_acquire(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_NONE: -- break; -- case SCX_OPSS_QUEUEING: -- /* -- * QUEUEING is started and finished while holding @p's rq lock. -- * As we're holding the rq lock now, we shouldn't see QUEUEING. -- */ -- BUG(); -- case SCX_OPSS_QUEUED: -- if (SCX_HAS_OP(dequeue)) -- SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); -- -- if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_NONE)) -- break; -- fallthrough; -- case SCX_OPSS_DISPATCHING: -- /* -- * If @p is being dispatched from the BPF scheduler to a DSQ, -- * wait for the transfer to complete so that @p doesn't get -- * added to its DSQ after dequeueing is complete. -- * -- * As we're waiting on DISPATCHING with the rq locked, the -- * dispatching side shouldn't try to lock the rq while -- * DISPATCHING is set. See dispatch_to_local_dsq(). -- * -- * DISPATCHING shouldn't have qseq set and control can reach -- * here with NONE @opss from the above QUEUED case block. -- * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. -- */ -- wait_ops_state(p, SCX_OPSS_DISPATCHING); -- BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- break; -- } --} -- --static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) --{ -- struct scx_rq *scx_rq = rq->scx; -- -- if (!(p->scx->flags & SCX_TASK_QUEUED)) { -- WARN_ON_ONCE(watchdog_task_watched(p)); -- return; -- } -- -- ops_dequeue(p, deq_flags); -- -- /* -- * A currently running task which is going off @rq first gets dequeued -- * and then stops running. As we want running <-> stopping transitions -- * to be contained within runnable <-> quiescent transitions, trigger -- * ->stopping() early here instead of in put_prev_task_scx(). -- * -- * @p may go through multiple stopping <-> running transitions between -- * here and put_prev_task_scx() if task attribute changes occur while -- * balance_scx() leaves @rq unlocked. However, they don't contain any -- * information meaningful to the BPF scheduler and can be suppressed by -- * skipping the callbacks if the task is !QUEUED. -- */ -- if (SCX_HAS_OP(stopping) && task_current(rq, p)) { -- update_curr_scx(rq); -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); -- } -- -- //if(task_current(rq, p)) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- if (SCX_HAS_OP(quiescent)) -- SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); -- -- if (deq_flags & SCX_DEQ_SLEEP) -- p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP; -- else -- p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP; -- -- p->scx->flags &= ~SCX_TASK_QUEUED; -- BUG_ON(!scx_rq->nr_running); -- scx_rq->nr_running--; -- sub_nr_running(rq, 1); -- -- dispatch_dequeue(scx_rq, p); --} -- --static void yield_task_scx(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); -- else -- p->scx->slice = 0; --} -- --static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) --{ -- struct task_struct *from = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); -- else -- return false; --} -- --#ifdef CONFIG_SMP --/** -- * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -- * @rq: rq to move the task into, currently locked -- * @p: task to move -- * @enq_flags: %SCX_ENQ_* -- * -- * Move @p which is currently on a different rq to @rq's local DSQ. The caller -- * must: -- * -- * 1. Start with exclusive access to @p either through its DSQ lock or -- * %SCX_OPSS_DISPATCHING flag. -- * -- * 2. Set @p->scx->holding_cpu to raw_smp_processor_id(). -- * -- * 3. Remember task_rq(@p). Release the exclusive access so that we don't -- * deadlock with dequeue. -- * -- * 4. Lock @rq and the task_rq from #3. -- * -- * 5. Call this function. -- * -- * Returns %true if @p was successfully moved. %false after racing dequeue and -- * losing. -- */ --static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -- u64 enq_flags) --{ -- struct rq *task_rq; -- -- lockdep_assert_rq_held(rq); -- -- /* -- * If dequeue got to @p while we were trying to lock both rq's, it'd -- * have cleared @p->scx->holding_cpu to -1. While other cpus may have -- * updated it to different values afterwards, as this operation can't be -- * preempted or recurse, @p->scx->holding_cpu can never become -- * raw_smp_processor_id() again before we're done. Thus, we can tell -- * whether we lost to dequeue by testing whether @p->scx->holding_cpu is -- * still raw_smp_processor_id(). -- * -- * See dispatch_dequeue() for the counterpart. -- */ -- if (unlikely(p->scx->holding_cpu != raw_smp_processor_id())) -- return false; -- -- /* @p->rq couldn't have changed if we're still the holding cpu */ -- task_rq = task_rq(p); -- lockdep_assert_rq_held(task_rq); -- -- WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)); -- deactivate_task(task_rq, p, 0); -- set_task_cpu(p, cpu_of(rq)); -- p->scx->sticky_cpu = cpu_of(rq); -- -- /* -- * We want to pass scx-specific enq_flags but activate_task() will -- * truncate the upper 32 bit. As we own @rq, we can pass them through -- * @rq->scx->extra_enq_flags instead. -- */ -- WARN_ON_ONCE(rq->scx->extra_enq_flags); -- rq->scx->extra_enq_flags = enq_flags; -- activate_task(rq, p, 0); -- rq->scx->extra_enq_flags = 0; -- -- return true; --} -- --/** -- * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * We're holding @rq lock and trying to dispatch a task from @src_rq to -- * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether -- * @rq stays locked isn't important as long as the state is restored after -- * dispatch_to_local_dsq_unlock(). -- */ --static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- rq_unpin_lock(rq, rf); -- -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(rq); -- raw_spin_rq_lock(dst_rq); -- } else if (rq == src_rq) { -- double_lock_balance(rq, dst_rq); -- rq_repin_lock(rq, rf); -- } else if (rq == dst_rq) { -- double_lock_balance(rq, src_rq); -- rq_repin_lock(rq, rf); -- } else { -- raw_spin_rq_unlock(rq); -- double_rq_lock(src_rq, dst_rq); -- } --} -- --/** -- * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock() -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return. -- */ --static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } else if (rq == src_rq) { -- double_unlock_balance(rq, dst_rq); -- } else if (rq == dst_rq) { -- double_unlock_balance(rq, src_rq); -- } else { -- double_rq_unlock(src_rq, dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } --} --#endif /* CONFIG_SMP */ -- -- --static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq) --{ -- return likely(test_rq_online(rq)) && !is_migration_disabled(p) && -- cpumask_test_cpu(cpu_of(rq), p->cpus_ptr); --} -- --static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -- struct scx_dispatch_q *dsq) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rb_node *rb_node; -- struct rq *task_rq; -- bool moved = false; --retry: -- if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -- return false; -- -- raw_spin_lock(&dsq->lock); -- -- list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- for (rb_node = rb_first_cached(&dsq->priq); rb_node; -- rb_node = rb_next(rb_node)) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- raw_spin_unlock(&dsq->lock); -- return false; -- --this_rq: -- /* @dsq is locked and @p is on this rq */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo); -- dsq->nr--; -- scx_rq->local_dsq.nr++; -- p->scx->dsq = &scx_rq->local_dsq; -- raw_spin_unlock(&dsq->lock); -- return true; -- --remote_rq: --#ifdef CONFIG_SMP -- /* -- * @dsq is locked and @p is on a remote rq. @p is currently protected by -- * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -- * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -- * rq lock or fail, do a little dancing from our side. See -- * move_task_to_local_dsq(). -- */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- p->scx->holding_cpu = raw_smp_processor_id(); -- raw_spin_unlock(&dsq->lock); -- -- rq_unpin_lock(rq, rf); -- double_lock_balance(rq, task_rq); -- rq_repin_lock(rq, rf); -- -- moved = move_task_to_local_dsq(rq, p, 0); -- -- double_unlock_balance(rq, task_rq); --#endif /* CONFIG_SMP */ -- if (likely(moved)) -- return true; -- goto retry; --} -- --enum dispatch_to_local_dsq_ret { -- DTL_DISPATCHED, /* successfully dispatched */ -- DTL_LOST, /* lost race to dequeue */ -- DTL_NOT_LOCAL, /* destination is not a local DSQ */ -- DTL_INVALID, /* invalid local dsq_id */ --}; -- --/** -- * dispatch_to_local_dsq - Dispatch a task to a local dsq -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @dsq_id: destination dsq ID -- * @p: task to dispatch -- * @enq_flags: %SCX_ENQ_* -- * -- * We're holding @rq lock and want to dispatch @p to the local DSQ identified by -- * @dsq_id. This function performs all the synchronization dancing needed -- * because local DSQs are protected with rq locks. -- * -- * The caller must have exclusive ownership of @p (e.g. through -- * %SCX_OPSS_DISPATCHING). -- */ --static enum dispatch_to_local_dsq_ret --dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id, -- struct task_struct *p, u64 enq_flags) --{ -- struct rq *src_rq = task_rq(p); -- struct rq *dst_rq; -- -- /* -- * We're synchronized against dequeue through DISPATCHING. As @p can't -- * be dequeued, its task_rq and cpus_allowed are stable too. -- */ -- if (dsq_id == SCX_DSQ_LOCAL) { -- dst_rq = rq; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]", -- cpu, p->comm, p->pid); -- return DTL_INVALID; -- } -- dst_rq = cpu_rq(cpu); -- } else { -- return DTL_NOT_LOCAL; -- } -- -- /* if dispatching to @rq that @p is already on, no lock dancing needed */ -- if (rq == src_rq && rq == dst_rq) { -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags | SCX_ENQ_CLEAR_OPSS); -- return DTL_DISPATCHED; -- } -- --#ifdef CONFIG_SMP -- if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { -- struct rq *locked_dst_rq = dst_rq; -- bool dsp; -- -- /* -- * @p is on a possibly remote @src_rq which we need to lock to -- * move the task. If dequeue is in progress, it'd be locking -- * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq -- * lock while holding DISPATCHING. -- * -- * As DISPATCHING guarantees that @p is wholly ours, we can -- * pretend that we're moving from a DSQ and use the same -- * mechanism - mark the task under transfer with holding_cpu, -- * release DISPATCHING and then follow the same protocol. -- */ -- p->scx->holding_cpu = raw_smp_processor_id(); -- -- /* store_release ensures that dequeue sees the above */ -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq); -- -- /* -- * We don't require the BPF scheduler to avoid dispatching to -- * offline CPUs mostly for convenience but also because CPUs can -- * go offline between scx_bpf_dispatch() calls and here. If @p -- * is destined to an offline CPU, queue it on its current CPU -- * instead, which should always be safe. As this is an allowed -- * behavior, don't trigger an ops error. -- */ -- if (unlikely(!test_rq_online(dst_rq))) -- dst_rq = src_rq; -- -- if (src_rq == dst_rq) { -- /* -- * As @p is staying on the same rq, there's no need to -- * go through the full deactivate/activate cycle. -- * Optimize by abbreviating the operations in -- * move_task_to_local_dsq(). -- */ -- dsp = p->scx->holding_cpu == raw_smp_processor_id(); -- if (likely(dsp)) { -- p->scx->holding_cpu = -1; -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags); -- } -- } else { -- dsp = move_task_to_local_dsq(dst_rq, p, enq_flags); -- } -- -- /* if the destination CPU is idle, wake it up */ -- if (dsp && p->sched_class > dst_rq->curr->sched_class) -- resched_curr(dst_rq); -- -- dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq); -- -- return dsp ? DTL_DISPATCHED : DTL_LOST; -- } --#endif /* CONFIG_SMP */ -- -- scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", -- cpu_of(dst_rq), p->comm, p->pid); -- return DTL_INVALID; --} -- --/** -- * finish_dispatch - Asynchronously finish dispatching a task -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @p: task to finish dispatching -- * @qseq_at_dispatch: qseq when @p started getting dispatched -- * @dsq_id: destination DSQ ID -- * @enq_flags: %SCX_ENQ_* -- * -- * Dispatching to local DSQs may need to wait for queueing to complete or -- * require rq lock dancing. As we don't wanna do either while inside -- * ops.dispatch() to avoid locking order inversion, we split dispatching into -- * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the -- * task and its qseq. Once ops.dispatch() returns, this function is called to -- * finish up. -- * -- * There is no guarantee that @p is still valid for dispatching or even that it -- * was valid in the first place. Make sure that the task is still owned by the -- * BPF scheduler and claim the ownership before dispatching. -- */ --static void finish_dispatch(struct rq *rq, struct rq_flags *rf, -- struct task_struct *p, u64 qseq_at_dispatch, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- u64 opss; -- -- touch_core_sched_dispatch(rq, p); --retry: -- /* -- * No need for _acquire here. @p is accessed only after a successful -- * try_cmpxchg to DISPATCHING. -- */ -- opss = atomic64_read(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_DISPATCHING: -- case SCX_OPSS_NONE: -- /* someone else already got to it */ -- return; -- case SCX_OPSS_QUEUED: -- /* -- * If qseq doesn't match, @p has gone through at least one -- * dispatch/dequeue and re-enqueue cycle between -- * scx_bpf_dispatch() and here and we have no claim on it. -- */ -- if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) -- return; -- -- /* -- * While we know @p is accessible, we don't yet have a claim on -- * it - the BPF scheduler is allowed to dispatch tasks -- * spuriously and there can be a racing dequeue attempt. Let's -- * claim @p by atomically transitioning it from QUEUED to -- * DISPATCHING. -- */ -- if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_DISPATCHING))) -- break; -- goto retry; -- case SCX_OPSS_QUEUEING: -- /* -- * do_enqueue_task() is in the process of transferring the task -- * to the BPF scheduler while holding @p's rq lock. As we aren't -- * holding any kernel or BPF resource that the enqueue path may -- * depend upon, it's safe to wait. -- */ -- wait_ops_state(p, opss); -- goto retry; -- } -- -- BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) { -- case DTL_DISPATCHED: -- break; -- case DTL_LOST: -- break; -- case DTL_INVALID: -- dsq_id = SCX_DSQ_GLOBAL; -- fallthrough; -- case DTL_NOT_LOCAL: -- dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()), -- dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- break; -- } --} -- --static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- u32 u; -- -- for (u = 0; u < dspc->buf_cursor; u++) { -- struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u]; -- -- finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id, -- ent->enq_flags); -- } -- -- dspc->nr_tasks += dspc->buf_cursor; -- dspc->buf_cursor = 0; --} -- --static int balance_one(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf, bool local) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- bool prev_on_scx = prev->sched_class == &ext_sched_class; -- int nr_loops = SCX_DSP_MAX_LOOPS; -- -- lockdep_assert_rq_held(rq); -- -- if (static_branch_unlikely(&scx_ops_cpu_preempt) && -- unlikely(rq->scx->cpu_released)) { -- /* -- * If the previous sched_class for the current CPU was not SCX, -- * notify the BPF scheduler that it again has control of the -- * core. This callback complements ->cpu_release(), which is -- * emitted in scx_notify_pick_next_task(). -- */ -- if (SCX_HAS_OP(cpu_acquire)) -- SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq), -- NULL); -- rq->scx->cpu_released = false; -- } -- -- if (prev_on_scx) { -- WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP)); -- update_curr_scx(rq); -- -- /* -- * If @prev is runnable & has slice left, it has priority and -- * fetching more just increases latency for the fetched tasks. -- * Tell put_prev_task_scx() to put @prev on local_dsq. If the -- * BPF scheduler wants to handle this explicitly, it should -- * implement ->cpu_released(). -- * -- * See scx_ops_disable_workfn() for the explanation on the -- * disabling() test. -- * -- * When balancing a remote CPU for core-sched, there won't be a -- * following put_prev_task_scx() call and we don't own -- * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the -- * same conditions later and pick @rq->curr accordingly. -- */ -- if ((prev->scx->flags & SCX_TASK_QUEUED) && -- prev->scx->slice && !scx_ops_disabling()) { -- if (local) -- prev->scx->flags |= SCX_TASK_BAL_KEEP; -- return 1; -- } -- } -- -- /* if there already are tasks to run, nothing to do */ -- if (scx_rq->local_dsq.nr) -- return 1; -- -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- if (!SCX_HAS_OP(dispatch)) -- return 0; -- -- dspc->rq = rq; -- dspc->rf = rf; -- -- /* -- * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, -- * the local DSQ might still end up empty after a successful -- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() -- * produced some tasks, retry. The BPF scheduler may depend on this -- * looping behavior to simplify its implementation. -- */ -- do { -- dspc->nr_tasks = 0; -- -- SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), -- prev_on_scx ? prev : NULL); -- -- flush_dispatch_buf(rq, rf); -- -- if (scx_rq->local_dsq.nr) -- return 1; -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- /* -- * ops.dispatch() can trap us in this loop by repeatedly -- * dispatching ineligible tasks. Break out once in a while to -- * allow the watchdog to run. As IRQ can't be enabled in -- * balance(), we want to complete this scheduling cycle and then -- * start a new one. IOW, we want to call resched_curr() on the -- * next, most likely idle, task, not the current one. Use -- * scx_bpf_kick_cpu() for deferred kicking. -- */ -- if (unlikely(!--nr_loops)) { -- scx_bpf_kick_cpu(cpu_of(rq), 0); -- break; -- } -- } while (dspc->nr_tasks); -- -- return 0; --} -- --static int balance_scx(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf) --{ -- int ret; -- -- ret = balance_one(rq, prev, rf, true); --#ifdef CONFIG_SCHED_SMT -- /* -- * When core-sched is enabled, this ops.balance() call will be followed -- * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx() -- * on the SMT siblings. Balance the siblings too. -- */ -- if (sched_core_enabled(rq)) { -- const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); -- int scpu; -- -- for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { -- struct rq *srq = cpu_rq(scpu); -- struct rq_flags srf; -- struct task_struct *sprev = srq->curr; -- -- /* -- * While core-scheduling, rq lock is shared among -- * siblings but the debug annotations and rq clock -- * aren't. Do pinning dance to transfer the ownership. -- */ -- WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); -- rq_unpin_lock(rq, rf); -- rq_pin_lock(srq, &srf); -- -- update_rq_clock(srq); -- balance_one(srq, sprev, &srf, false); -- -- rq_unpin_lock(srq, &srf); -- rq_repin_lock(rq, rf); -- } -- } --#endif -- return ret; --} -- --static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) --{ -- if (p->scx->flags & SCX_TASK_QUEUED) { -- /* -- * Core-sched might decide to execute @p before it is -- * dispatched. Call ops_dequeue() to notify the BPF scheduler. -- */ -- ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); -- dispatch_dequeue(rq->scx, p); -- } -- -- p->se.exec_start = rq_clock_task(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, running, p); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock); -- -- watchdog_unwatch_task(p, true); -- -- /* -- * @p is getting newly scheduled or got kicked after someone updated its -- * slice. Refresh whether tick can be stopped. See can_stop_tick_scx(). -- */ -- if ((p->scx->slice == SCX_SLICE_INF) != -- (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) { -- if (p->scx->slice == SCX_SLICE_INF) -- rq->scx->flags |= SCX_RQ_CAN_STOP_TICK; -- else -- rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK; -- -- sched_update_tick_dependency(rq); -- } --} -- --static void put_prev_task_scx(struct rq *rq, struct task_struct *p) --{ --#ifndef CONFIG_SMP -- /* -- * UP workaround. -- * -- * Because SCX may transfer tasks across CPUs during dispatch, dispatch -- * is performed from its balance operation which isn't called in UP. -- * Let's work around by calling it from the operations which come right -- * after. -- * -- * 1. If the prev task is on SCX, pick_next_task() calls -- * .put_prev_task() right after. As .put_prev_task() is also called -- * from other places, we need to distinguish the calls which can be -- * done by looking at the previous task's state - if still queued or -- * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task(). -- * This case is handled here. -- * -- * 2. If the prev task is not on SCX, the first following call into SCX -- * will be .pick_next_task(), which is covered by calling -- * balance_scx() from pick_next_task_scx(). -- * -- * Note that we can't merge the first case into the second as -- * balance_scx() must be called before the previous SCX task goes -- * through put_prev_task_scx(). -- * -- * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf. -- * Pass in %NULL. -- */ -- if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP)) -- balance_scx(rq, p, NULL); --#endif -- -- update_curr_scx(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- -- /* -- * If we're being called from put_prev_task_balance(), balance_scx() may -- * have decided that @p should keep running. -- */ -- if (p->scx->flags & SCX_TASK_BAL_KEEP) { -- p->scx->flags &= ~SCX_TASK_BAL_KEEP; -- watchdog_watch_task(rq, p); -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- watchdog_watch_task(rq, p); -- -- /* -- * If @p has slice left and balance_scx() didn't tag it for -- * keeping, @p is getting preempted by a higher priority -- * scheduler class or core-sched forcing a different task. Leave -- * it at the head of the local DSQ. -- */ -- if (p->scx->slice && !scx_ops_disabling()) { -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- /* -- * If we're in the pick_next_task path, balance_scx() should -- * have already populated the local DSQ if there are any other -- * available tasks. If empty, tell ops.enqueue() that @p is the -- * only one available for this cpu. ops.enqueue() should put it -- * on the local DSQ so that the subsequent pick_next_task_scx() -- * can find the task unless it wants to trigger a separate -- * follow-up scheduling event. -- */ -- if (list_empty(&rq->scx->local_dsq.fifo)) -- do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1); -- else -- do_enqueue_task(rq, p, 0, -1); -- } --} -- --static struct task_struct *first_local_task(struct rq *rq) --{ -- struct rb_node *rb_node; -- struct sched_ext_entity *entity; -- -- if (!list_empty(&rq->scx->local_dsq.fifo)) { -- entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo); -- return entity->task; -- } -- -- rb_node = rb_first_cached(&rq->scx->local_dsq.priq); -- if (rb_node) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- return entity->task; -- } -- -- return NULL; --} -- --static struct task_struct *pick_next_task_scx(struct rq *rq) --{ -- struct task_struct *p; -- --#ifndef CONFIG_SMP -- /* UP workaround - see the comment at the head of put_prev_task_scx() */ -- if (unlikely(rq->curr->sched_class != &ext_sched_class)) -- balance_scx(rq, rq->curr, NULL); --#endif -- -- p = first_local_task(rq); -- if (!p) -- return NULL; -- -- if (unlikely(!p->scx->slice)) { -- if (!scx_ops_disabling() && !scx_warned_zero_slice) { -- printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n", -- p->comm, p->pid); -- scx_warned_zero_slice = true; -- } -- p->scx->slice = SCX_SLICE_DFL; -- } -- -- set_next_task_scx(rq, p, true); -- -- return p; --} -- --#ifdef CONFIG_SCHED_CORE --/** -- * scx_prio_less - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Core-sched is implemented as an additional scheduling layer on top of the -- * usual sched_class'es and needs to find out the expected task ordering. For -- * SCX, core-sched calls this function to interrogate the task ordering. -- * -- * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used -- * to implement the default task ordering. The older the timestamp, the higher -- * prority the task - the global FIFO ordering matching the default scheduling -- * behavior. -- * -- * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to -- * implement FIFO ordering within each local DSQ. See pick_task_scx(). -- */ --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi) --{ -- /* -- * The const qualifiers are dropped from task_struct pointers when -- * calling ops.core_sched_before(). Accesses are controlled by the -- * verifier. -- */ -- if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling()) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, -- (struct task_struct *)a, -- (struct task_struct *)b); -- else -- return time_after64(a->scx->core_sched_at, b->scx->core_sched_at); --} -- --/** -- * pick_task_scx - Pick a candidate task for core-sched -- * @rq: rq to pick the candidate task from -- * -- * Core-sched calls this function on each SMT sibling to determine the next -- * tasks to run on the SMT siblings. balance_one() has been called on all -- * siblings and put_prev_task_scx() has been called only for the current CPU. -- * -- * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look -- * at the first task in the local dsq. @rq->curr has to be considered explicitly -- * to mimic %SCX_TASK_BAL_KEEP. -- */ --static struct task_struct *pick_task_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- struct task_struct *first = first_local_task(rq); -- -- if (curr->scx->flags & SCX_TASK_QUEUED) { -- /* is curr the only runnable task? */ -- if (!first) -- return curr; -- -- /* -- * Does curr trump first? We can always go by core_sched_at for -- * this comparison as it represents global FIFO ordering when -- * the default core-sched ordering is used and local-DSQ FIFO -- * ordering otherwise. -- * -- * We can have a task with an earlier timestamp on the DSQ. For -- * example, when a current task is preempted by a sibling -- * picking a different cookie, the task would be requeued at the -- * head of the local DSQ with an earlier timestamp than the -- * core-sched picked next task. Besides, the BPF scheduler may -- * dispatch any tasks to the local DSQ anytime. -- */ -- if (curr->scx->slice && time_before64(curr->scx->core_sched_at, -- first->scx->core_sched_at)) -- return curr; -- } -- -- return first; /* this may be %NULL */ --} --#endif /* CONFIG_SCHED_CORE */ -- --static enum scx_cpu_preempt_reason --preempt_reason_from_class(const struct sched_class *class) --{ --#ifdef CONFIG_SMP -- if (class == &stop_sched_class) -- return SCX_CPU_PREEMPT_STOP; --#endif -- if (class == &dl_sched_class) -- return SCX_CPU_PREEMPT_DL; -- if (class == &rt_sched_class) -- return SCX_CPU_PREEMPT_RT; -- return SCX_CPU_PREEMPT_UNKNOWN; --} -- --void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task, -- const struct sched_class *active) --{ -- lockdep_assert_rq_held(rq); -- -- /* -- * The callback is conceptually meant to convey that the CPU is no -- * longer under the control of SCX. Therefore, don't invoke the -- * callback if the CPU is is staying on SCX, or going idle (in which -- * case the SCX scheduler has actively decided not to schedule any -- * tasks on the CPU). -- */ -- if (likely(active >= &ext_sched_class)) -- return; -- -- /* -- * At this point we know that SCX was preempted by a higher priority -- * sched_class, so invoke the ->cpu_release() callback if we have not -- * done so already. We only send the callback once between SCX being -- * preempted, and it regaining control of the CPU. -- * -- * ->cpu_release() complements ->cpu_acquire(), which is emitted the -- * next time that balance_scx() is invoked. -- */ -- if (!rq->scx->cpu_released) { -- if (SCX_HAS_OP(cpu_release)) { -- struct scx_cpu_release_args args = { -- .reason = preempt_reason_from_class(active), -- .task = task, -- }; -- -- SCX_CALL_OP(SCX_KF_CPU_RELEASE, -- cpu_release, cpu_of(rq), &args); -- } -- rq->scx->cpu_released = true; -- } --} -- --#ifdef CONFIG_SMP -- --static bool test_and_clear_cpu_idle(int cpu) --{ -- if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -- if (cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- return true; -- } else { -- return false; -- } --} -- --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- int cpu; -- -- do { -- cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); -- if (cpu < nr_cpu_ids) { -- const struct cpumask *sbm = topology_sibling_cpumask(cpu); -- -- /* -- * If offline, @cpu is not its own sibling and we can -- * get caught in an infinite loop as @cpu is never -- * cleared from idle_masks.smt. Clear @cpu directly in -- * such cases. -- */ -- if (likely(cpumask_test_cpu(cpu, sbm))) -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm); -- else -- cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu)); -- } else { -- cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -- if (cpu >= nr_cpu_ids) -- return -EBUSY; -- } -- } while (!test_and_clear_cpu_idle(cpu)); -- -- return cpu; --} -- --static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) --{ -- s32 cpu; -- -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return prev_cpu; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local DSQ of the waker. -- */ -- if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 && -- scx_has_idle_cpus && !(current->flags & PF_EXITING)) { -- cpu = smp_processor_id(); -- if (cpumask_test_cpu(cpu, p->cpus_ptr)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (test_and_clear_cpu_idle(prev_cpu)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return prev_cpu; -- } -- -- if (p->nr_cpus_allowed == 1) -- return prev_cpu; -- -- cpu = scx_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- -- return prev_cpu; --} -- --static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) --{ -- if (SCX_HAS_OP(select_cpu)) { -- s32 cpu; -- -- cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu, -- wake_flags); -- if (ops_cpu_valid(cpu)) { -- return cpu; -- } else { -- scx_ops_error("select_cpu returned invalid cpu %d", cpu); -- return prev_cpu; -- } -- } else { -- return scx_select_cpu_dfl(p, prev_cpu, wake_flags); -- } --} -- --static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx) --{ -- set_cpus_allowed_common(p, ctx); -- -- /* -- * The effective cpumask is stored in @p->cpus_ptr which may temporarily -- * differ from the configured one in @p->cpus_mask. Always tell the bpf -- * scheduler the effective one. -- * -- * Fine-grained memory write control is enforced by BPF making the const -- * designation pointless. Cast it away when calling the operation. -- */ -- if (SCX_HAS_OP(set_cpumask)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, -- (struct cpumask *)p->cpus_ptr); --} -- --static void reset_idle_masks(void) --{ -- /* consider all cpus idle, should converge to the actual state quickly */ -- cpumask_setall(idle_masks.cpu); -- cpumask_setall(idle_masks.smt); -- scx_has_idle_cpus = true; --} -- --void __scx_update_idle(struct rq *rq, bool idle) --{ -- int cpu = cpu_of(rq); -- struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -- -- if (SCX_HAS_OP(update_idle)) { -- SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); -- if (!static_branch_unlikely(&scx_builtin_idle_enabled)) -- return; -- } -- -- if (idle) { -- cpumask_set_cpu(cpu, idle_masks.cpu); -- if (!scx_has_idle_cpus) -- scx_has_idle_cpus = true; -- -- /* -- * idle_masks.smt handling is racy but that's fine as it's only -- * for optimization and self-correcting. -- */ -- for_each_cpu(cpu, sib_mask) { -- if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -- return; -- } -- cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -- } else { -- cpumask_clear_cpu(cpu, idle_masks.cpu); -- if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -- } --} -- --#else /* !CONFIG_SMP */ -- --static bool test_and_clear_cpu_idle(int cpu) { return false; } --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } --static void reset_idle_masks(void) {} -- --#endif /* CONFIG_SMP */ -- --static bool check_rq_for_timeouts(struct rq *rq) --{ -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rq_flags rf; -- bool timed_out = false; -- -- rq_lock_irqsave(rq, &rf); -- list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) { -- unsigned long last_runnable; -- -- p = entity->task; -- last_runnable = p->scx->runnable_at; -- -- if (unlikely(time_after(jiffies, -- last_runnable + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "%s[%d] failed to run for %u.%03us", -- p->comm, p->pid, -- dur_ms / 1000, dur_ms % 1000); -- timed_out = true; -- break; -- } -- } -- rq_unlock_irqrestore(rq, &rf); -- -- return timed_out; --} -- --static void scx_watchdog_workfn(struct work_struct *work) --{ -- int cpu; -- -- scx_watchdog_timestamp = jiffies; -- -- for_each_online_cpu(cpu) { -- if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) -- break; -- -- cond_resched(); -- } -- queue_delayed_work(system_unbound_wq, to_delayed_work(work), -- scx_watchdog_timeout / 2); --} -- --static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) --{ -- update_curr_scx(rq); -- //scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock); -- /* -- * While disabling, always resched and refresh core-sched timestamp as -- * we can't trust the slice management or ops.core_sched_before(). -- */ -- if (scx_ops_disabling()) { -- curr->scx->slice = 0; -- touch_core_sched(rq, curr); -- } -- -- if (!curr->scx->slice) -- resched_curr(rq); --} -- --#define SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- --static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg) --{ -- int ret; -- -- WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED); -- -- p->scx->disallow = false; -- -- if (SCX_HAS_OP(prep_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- }; -- -- ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args); -- if (unlikely(ret)) { -- ret = ops_sanitize_err("prep_enable", ret); -- return ret; -- } -- } -- //scx_sched_init_task(p); -- -- if (p->scx->disallow) { -- struct rq *rq; -- struct rq_flags rf; -- -- rq = task_rq_lock(p, &rf); -- -- /* -- * We're either in fork or load path and @p->policy will be -- * applied right after. Reverting @p->policy here and rejecting -- * %SCHED_EXT transitions from scx_check_setscheduler() -- * guarantees that if ops.prep_enable() sets @p->disallow, @p -- * can never be in SCX. -- */ -- if (p->policy == SCHED_EXT) { -- p->policy = SCHED_NORMAL; -- atomic64_inc(&scx_nr_rejected); -- } -- -- task_rq_unlock(rq, p, &rf); -- } -- -- p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET); -- return 0; --} -- --static void scx_ops_enable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED)); -- -- if (SCX_HAS_OP(enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- p->scx->flags |= SCX_TASK_OPS_ENABLED; --} -- --static void scx_ops_disable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- if (p->scx->flags & SCX_TASK_OPS_PREPPED) { -- if (SCX_HAS_OP(cancel_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- } else if (p->scx->flags & SCX_TASK_OPS_ENABLED) { -- if (SCX_HAS_OP(disable)) -- SCX_CALL_OP(SCX_KF_REST, disable, p); -- p->scx->flags &= ~SCX_TASK_OPS_ENABLED; -- } --} -- --static void set_task_scx_weight(struct task_struct *p) --{ -- u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -- -- p->scx->weight = sched_weight_to_cgroup(weight); --} -- --/** -- * refresh_scx_weight - Refresh a task's ext weight -- * @p: task to refresh ext weight for -- * -- * @p->scx->weight carries the task's static priority in cgroup weight scale to -- * enable easy access from the BPF scheduler. To keep it synchronized with the -- * current task priority, this function should be called when a new task is -- * created, priority is changed for a task on sched_ext, and a task is switched -- * to sched_ext from other classes. -- */ --static void refresh_scx_weight(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- set_task_scx_weight(p); -- if (SCX_HAS_OP(set_weight)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight); --} -- --void scx_pre_fork(struct task_struct *p) --{ -- p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL); -- if (!p->scx) { -- goto lock; -- } -- -- p->scx->dsq = NULL; -- INIT_LIST_HEAD(&p->scx->dsq_node.fifo); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- INIT_LIST_HEAD(&p->scx->watchdog_node); -- p->scx->flags = 0; -- p->scx->weight = 0; -- p->scx->sticky_cpu = -1; -- p->scx->holding_cpu = -1; -- p->scx->kf_mask = 0; -- atomic64_set(&p->scx->ops_state, 0); -- p->scx->runnable_at = INITIAL_JIFFIES; -- p->scx->slice = SCX_SLICE_DFL; -- p->scx->task = p; -- p->sched_prop = 0; -- -- /* -- * BPF scheduler enable/disable paths want to be able to iterate and -- * update all tasks which can become complex when racing forks. As -- * enable/disable are very cold paths, let's use a percpu_rwsem to -- * exclude forks. -- */ --lock: -- percpu_down_read(&scx_fork_rwsem); --} -- --int scx_fork(struct task_struct *p) --{ -- percpu_rwsem_assert_held(&scx_fork_rwsem); -- -- if (scx_enabled()) -- return scx_ops_prepare_task(p, task_group(p)); -- else -- return 0; --} -- --void scx_post_fork(struct task_struct *p) --{ -- if (scx_enabled()) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- /* -- * Set the weight manually before calling ops.enable() so that -- * the scheduler doesn't see a stale value if they inspect the -- * task struct. We'll invoke ops.set_weight() afterwards, as it -- * would be odd to receive a callback on the task before we -- * tell the scheduler that it's been fully enabled. -- */ -- set_task_scx_weight(p); -- scx_ops_enable_task(p); -- refresh_scx_weight(p); -- task_rq_unlock(rq, p, &rf); -- } -- -- spin_lock_irq(&scx_tasks_lock); -- list_add_tail(&p->scx->tasks_node, &scx_tasks); -- spin_unlock_irq(&scx_tasks_lock); -- -- percpu_up_read(&scx_fork_rwsem); --} -- --void scx_cancel_fork(struct task_struct *p) --{ -- if (scx_enabled()) -- scx_ops_disable_task(p); -- percpu_up_read(&scx_fork_rwsem); --} -- --void sched_ext_free(struct task_struct *p) --{ -- unsigned long flags; -- -- spin_lock_irqsave(&scx_tasks_lock, flags); -- list_del_init(&p->scx->tasks_node); -- spin_unlock_irqrestore(&scx_tasks_lock, flags); -- -- /* -- * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED -> -- * ENABLED transitions can't race us. Disable ops for @p. -- */ -- if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- scx_ops_disable_task(p); -- task_rq_unlock(rq, p, &rf); -- } --} -- --static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) --{ --} -- --static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} --static void switched_to_scx(struct rq *rq, struct task_struct *p) {} -- --int scx_check_setscheduler(struct task_struct *p, int policy) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- /* if disallow, reject transitioning into SCX */ -- if (scx_enabled() && READ_ONCE(p->scx->disallow) && -- p->policy != policy && policy == SCHED_EXT) -- return -EACCES; -- -- return 0; --} -- --#ifdef CONFIG_NO_HZ_FULL --bool scx_can_stop_tick(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (scx_ops_disabling()) -- return false; -- -- if (p->sched_class != &ext_sched_class) -- return true; -- -- /* -- * @rq can dispatch from different DSQs, so we can't tell whether it -- * needs the tick or not by looking at nr_running. Allow stopping ticks -- * iff the BPF scheduler indicated so. See set_next_task_scx(). -- */ -- return rq->scx->flags & SCX_RQ_CAN_STOP_TICK; --} --#endif -- --static inline void scx_cgroup_lock(void) {} --static inline void scx_cgroup_unlock(void) {} -- --/* -- * Omitted operations: -- * -- * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -- * task isn't tied to the CPU at that point. Preemption is implemented by -- * resetting the victim task's slice to 0 and triggering reschedule on the -- * target CPU. -- * -- * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -- * -- * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -- * their current sched_class. Call them directly from sched core instead. -- * -- * - task_woken, switched_from: Unnecessary. -- */ --DEFINE_SCHED_CLASS(ext) = { -- .enqueue_task = enqueue_task_scx, -- .dequeue_task = dequeue_task_scx, -- .yield_task = yield_task_scx, -- .yield_to_task = yield_to_task_scx, -- -- .check_preempt_curr = check_preempt_curr_scx, -- -- .pick_next_task = pick_next_task_scx, -- -- .put_prev_task = put_prev_task_scx, -- .set_next_task = set_next_task_scx, -- --#ifdef CONFIG_SMP -- .balance = balance_scx, -- .select_task_rq = select_task_rq_scx, -- .set_cpus_allowed = set_cpus_allowed_scx, --#endif -- --#ifdef CONFIG_SCHED_CORE -- .pick_task = pick_task_scx, --#endif -- -- .task_tick = task_tick_scx, -- -- .switched_to = switched_to_scx, -- .prio_changed = prio_changed_scx, -- -- .update_curr = update_curr_scx, -- --#ifdef CONFIG_UCLAMP_TASK -- .uclamp_enabled = 0, --#endif --}; -- --static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) --{ -- memset(dsq, 0, sizeof(*dsq)); -- -- raw_spin_lock_init(&dsq->lock); -- INIT_LIST_HEAD(&dsq->fifo); -- dsq->id = dsq_id; --} -- --static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id & SCX_DSQ_FLAG_BUILTIN) -- return ERR_PTR(-EINVAL); -- -- dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node); -- if (!dsq) -- return ERR_PTR(-ENOMEM); -- -- init_dsq(dsq, dsq_id); -- -- return dsq; --} -- --static void free_dsq_irq_workfn(struct irq_work *irq_work) --{ -- struct llist_node *to_free = llist_del_all(&dsqs_to_free); -- struct scx_dispatch_q *dsq, *tmp_dsq; -- -- llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) -- kfree_rcu(dsq, rcu); --} -- --static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); -- --static void destroy_dsq(u64 dsq_id) --{ --} -- --static void scx_cgroup_exit(void) {} --static int scx_cgroup_init(void) { return 0; } --static void scx_cgroup_config_knobs(void) {} -- --/* -- * Used by sched_fork() and __setscheduler_prio() to pick the matching -- * sched_class. dl/rt are already handled. -- */ --bool task_on_scx(struct task_struct *p) --{ -- if (!scx_enabled() || scx_ops_disabling()) -- return false; -- if (READ_ONCE(scx_switching_all)) -- return true; -- return p->policy == SCHED_EXT; --} -- --static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags) --{ -- if (enq_flags & SCX_ENQ_LAST) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); --} -- --static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {} -- --static void scx_ops_disable_workfn(struct kthread_work *work) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- struct scx_task_iter sti; -- struct task_struct *p; -- const char *reason; -- int i, cpu, type; -- -- type = atomic_read(&scx_exit_type); -- while (true) { -- /* -- * NONE indicates that a new scx_ops has been registered since -- * disable was scheduled - don't kill the new ops. DONE -- * indicates that the ops has already been disabled. -- */ -- if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE) -- return; -- if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE)) -- break; -- } -- -- cancel_delayed_work_sync(&scx_watchdog_work); -- -- switch (type) { -- case SCX_EXIT_UNREG: -- reason = "BPF scheduler unregistered"; -- break; -- case SCX_EXIT_SYSRQ: -- reason = "disabled by sysrq-S"; -- break; -- case SCX_EXIT_ERROR: -- reason = "runtime error"; -- break; -- case SCX_EXIT_ERROR_BPF: -- reason = "scx_bpf_error"; -- break; -- case SCX_EXIT_ERROR_STALL: -- reason = "runnable task stall"; -- break; -- default: -- reason = ""; -- } -- -- ei->type = type; -- strlcpy(ei->reason, reason, sizeof(ei->reason)); -- -- switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { -- case SCX_OPS_DISABLED: -- pr_warn("sched_ext: ops error detected without ops (%s)\n", -- scx_exit_info.msg); -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- return; -- case SCX_OPS_PREPPING: -- goto forward_progress_guaranteed; -- case SCX_OPS_DISABLING: -- /* shouldn't happen but handle it like ENABLING if it does */ -- WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); -- fallthrough; -- case SCX_OPS_ENABLING: -- case SCX_OPS_ENABLED: -- break; -- } -- -- /* -- * DISABLING is set and ops was either ENABLING or ENABLED indicating -- * that the ops and static branches are set. -- * -- * We must guarantee that all runnable tasks make forward progress -- * without trusting the BPF scheduler. We can't grab any mutexes or -- * rwsems as they might be held by tasks that the BPF scheduler is -- * forgetting to run, which unfortunately also excludes toggling the -- * static branches. -- * -- * Let's work around by overriding a couple ops and modifying behaviors -- * based on the DISABLING state and then cycling the tasks through -- * dequeue/enqueue to force global FIFO scheduling. -- * -- * a. ops.enqueue() and .dispatch() are overridden for simple global -- * FIFO scheduling. -- * -- * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value -- * can't be trusted. Whenever a tick triggers, the running task is -- * rotated to the tail of the queue with core_sched_at touched. -- * -- * c. pick_next_task() suppresses zero slice warning. -- * -- * d. scx_prio_less() reverts to the default core_sched_at order. -- */ -- scx_ops.enqueue = scx_ops_fallback_enqueue; -- scx_ops.dispatch = scx_ops_fallback_dispatch; -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- SCHED_CHANGE_BLOCK(task_rq(p), p, -- DEQUEUE_SAVE | DEQUEUE_MOVE) { -- /* cycling deq/enq is enough, see above */ -- } -- } -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* kick all CPUs to restore ticks */ -- for_each_possible_cpu(cpu) -- resched_cpu(cpu); -- --forward_progress_guaranteed: -- /* -- * Here, every runnable task is guaranteed to make forward progress and -- * we can safely use blocking synchronization constructs. Actually -- * disable ops. -- */ -- mutex_lock(&scx_ops_enable_mutex); -- -- static_branch_disable(&__scx_switched_all); -- WRITE_ONCE(scx_switching_all, false); -- -- /* avoid racing against fork and cgroup changes */ -- cpus_read_lock(); -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- bool alive = READ_ONCE(p->__state) != TASK_DEAD; -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL); -- -- __setscheduler_prio(p, p->prio); -- } -- -- if (alive) -- check_class_changed(task_rq(p), p, old_class, p->prio); -- -- scx_ops_disable_task(p); -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* no task is on scx, turn off all the switches and flush in-progress calls */ -- static_branch_disable_cpuslocked(&__scx_ops_enabled); -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- static_branch_disable_cpuslocked(&scx_has_op[i]); -- static_branch_disable_cpuslocked(&scx_ops_enq_last); -- static_branch_disable_cpuslocked(&scx_ops_enq_exiting); -- static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- synchronize_rcu(); -- -- scx_cgroup_exit(); -- -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- cpus_read_unlock(); -- -- if (ei->type >= SCX_EXIT_ERROR) { -- printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name); -- -- if (ei->msg[0] == '\0') -- printk(KERN_ERR "sched_ext: %s\n", ei->reason); -- else -- printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg); -- -- stack_trace_print(ei->bt, ei->bt_len, 2); -- } -- -- if (scx_ops.exit) -- SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); -- -- memset(&scx_ops, 0, sizeof(scx_ops)); -- -- free_percpu(scx_dsp_buf); -- scx_dsp_buf = NULL; -- scx_dsp_max_batch = 0; -- -- mutex_unlock(&scx_ops_enable_mutex); -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- -- scx_cgroup_config_knobs(); --} -- --static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); -- --static void schedule_scx_ops_disable_work(void) --{ -- struct kthread_worker *helper = READ_ONCE(scx_ops_helper); -- -- /* -- * We may be called spuriously before the first bpf_sched_ext_reg(). If -- * scx_ops_helper isn't set up yet, there's nothing to do. -- */ -- if (helper) -- kthread_queue_work(helper, &scx_ops_disable_work); --} -- --static void scx_ops_disable(enum scx_exit_type type) --{ -- int none = SCX_EXIT_NONE; -- -- if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)) -- type = SCX_EXIT_ERROR; -- -- atomic_try_cmpxchg(&scx_exit_type, &none, type); -- -- schedule_scx_ops_disable_work(); --} -- --static void scx_ops_error_irq_workfn(struct irq_work *irq_work) --{ -- schedule_scx_ops_disable_work(); --} -- --static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- int none = SCX_EXIT_NONE; -- va_list args; -- -- if (!atomic_try_cmpxchg(&scx_exit_type, &none, type)) -- return; -- -- ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1); -- -- va_start(args, fmt); -- vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args); -- va_end(args); -- -- irq_work_queue(&scx_ops_error_irq_work); --} -- --static struct kthread_worker *scx_create_rt_helper(const char *name) --{ -- struct kthread_worker *helper; -- -- helper = kthread_create_worker(0, name); -- if (helper) -- sched_set_fifo(helper->task); -- return helper; --} -- --static int scx_ops_enable(struct sched_ext_ops *ops) --{ -- struct scx_task_iter sti; -- struct task_struct *p; -- int i, ret; -- int tcnt = 0; -- unsigned long long start = 0; -- unsigned long long total_start = sched_clock(); -- -- mutex_lock(&scx_ops_enable_mutex); -- -- if (!scx_ops_helper) { -- WRITE_ONCE(scx_ops_helper, -- scx_create_rt_helper("sched_ext_ops_helper")); -- if (!scx_ops_helper) { -- ret = -ENOMEM; -- goto err_unlock; -- } -- } -- -- if (scx_ops_enable_state() != SCX_OPS_DISABLED) { -- ret = -EBUSY; -- goto err_unlock; -- } -- -- //slim_walt_enable(true); -- -- /* -- * Set scx_ops, transition to PREPPING and clear exit info to arm the -- * disable path. Failure triggers full disabling from here on. -- */ -- scx_ops = *ops; -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != -- SCX_OPS_DISABLED); -- -- memset(&scx_exit_info, 0, sizeof(scx_exit_info)); -- atomic_set(&scx_exit_type, SCX_EXIT_NONE); -- scx_warned_zero_slice = false; -- -- atomic64_set(&scx_nr_rejected, 0); -- -- /* -- * Keep CPUs stable during enable so that the BPF scheduler can track -- * online CPUs by watching ->on/offline_cpu() after ->init(). -- */ -- cpus_read_lock(); -- -- scx_switch_all_req = true; -- if (scx_ops.init) { -- ret = SCX_CALL_OP_RET(SCX_KF_INIT, init); -- if (ret) { -- ret = ops_sanitize_err("init", ret); -- goto err_disable; -- } -- -- /* -- * Exit early if ops.init() triggered scx_bpf_error(). Not -- * strictly necessary as we'll fail transitioning into ENABLING -- * later but that'd be after calling ops.prep_enable() on all -- * tasks and with -EBUSY which isn't very intuitive. Let's exit -- * early with success so that the condition is notified through -- * ops.exit() like other scx_bpf_error() invocations. -- */ -- if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE) -- goto err_disable; -- } -- -- WARN_ON_ONCE(scx_dsp_buf); -- scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; -- scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch, -- __alignof__(scx_dsp_buf[0])); -- if (!scx_dsp_buf) { -- ret = -ENOMEM; -- goto err_disable; -- } -- -- scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; -- if (ops->timeout_ms) -- scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); -- -- scx_watchdog_timestamp = jiffies; -- queue_delayed_work(system_unbound_wq, &scx_watchdog_work, -- scx_watchdog_timeout / 2); -- -- /* -- * Lock out forks, cgroup on/offlining and moves before opening the -- * floodgate so that they don't wander into the operations prematurely. -- */ -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- if (((void (**)(void))ops)[i]) -- static_branch_enable_cpuslocked(&scx_has_op[i]); -- -- if (ops->flags & SCX_OPS_ENQ_LAST) -- static_branch_enable_cpuslocked(&scx_ops_enq_last); -- -- if (ops->flags & SCX_OPS_ENQ_EXITING) -- static_branch_enable_cpuslocked(&scx_ops_enq_exiting); -- if (scx_ops.cpu_acquire || scx_ops.cpu_release) -- static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); -- -- if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { -- reset_idle_masks(); -- static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); -- } else { -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- } -- -- /* -- * All cgroups should be initialized before letting in tasks. cgroup -- * on/offlining and task migrations are already locked out. -- */ -- ret = scx_cgroup_init(); -- if (ret) -- goto err_disable_unlock; -- -- static_branch_enable_cpuslocked(&__scx_ops_enabled); -- -- /* -- * Enable ops for every task. Fork is excluded by scx_fork_rwsem -- * preventing new tasks from being added. No need to exclude tasks -- * leaving as sched_ext_free() can handle both prepped and enabled -- * tasks. Prep all tasks first and then enable them with preemption -- * disabled. -- */ -- spin_lock_irq(&scx_tasks_lock); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered(&sti))) { -- get_task_struct(p); -- spin_unlock_irq(&scx_tasks_lock); -- -- ret = scx_ops_prepare_task(p, task_group(p)); -- if (ret) { -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n", -- ret, p->comm, p->pid); -- goto err_disable_unlock; -- } -- -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- } -- scx_task_iter_exit(&sti); -- -- /* -- * All tasks are prepped but are still ops-disabled. Ensure that -- * %current can't be scheduled out and switch everyone. -- * preempt_disable() is necessary because we can't guarantee that -- * %current won't be starved if scheduled out while switching. -- */ -- preempt_disable(); -- -- /* -- * From here on, the disable path must assume that tasks have ops -- * enabled and need to be recovered. -- */ -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { -- preempt_enable(); -- spin_unlock_irq(&scx_tasks_lock); -- ret = -EBUSY; -- goto err_disable_unlock; -- } -- -- /* -- * We're fully committed and can't fail. The PREPPED -> ENABLED -- * transitions here are synchronized against sched_ext_free() through -- * scx_tasks_lock. -- */ -- WRITE_ONCE(scx_switching_all, scx_switch_all_req); -- start = sched_clock(); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- tcnt++; -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- scx_ops_enable_task(p); -- __setscheduler_prio(p, p->prio); -- } -- -- check_class_changed(task_rq(p), p, old_class, p->prio); -- } else { -- scx_ops_disable_task(p); -- } -- } -- printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id()); -- scx_task_iter_exit(&sti); -- -- spin_unlock_irq(&scx_tasks_lock); -- preempt_enable(); -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { -- ret = -EBUSY; -- goto err_disable; -- } -- -- if (scx_switch_all_req) -- static_branch_enable_cpuslocked(&__scx_switched_all); -- -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- -- scx_cgroup_config_knobs(); -- printk("\n total duration = %llu , current cpu = %d\n", sched_clock() - total_start, smp_processor_id()); -- -- return 0; -- --err_unlock: -- mutex_unlock(&scx_ops_enable_mutex); -- return ret; -- --err_disable_unlock: -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); --err_disable: -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- /* must be fully disabled before returning */ -- scx_ops_disable(SCX_EXIT_ERROR); -- kthread_flush_work(&scx_ops_disable_work); -- return ret; --} -- --#ifdef CONFIG_SCHED_DEBUG --static const char *scx_ops_enable_state_str[] = { -- [SCX_OPS_PREPPING] = "prepping", -- [SCX_OPS_ENABLING] = "enabling", -- [SCX_OPS_ENABLED] = "enabled", -- [SCX_OPS_DISABLING] = "disabling", -- [SCX_OPS_DISABLED] = "disabled", --}; -- --static int scx_debug_show(struct seq_file *m, void *v) --{ -- mutex_lock(&scx_ops_enable_mutex); -- seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name); -- seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled()); -- seq_printf(m, "%-30s: %d\n", "switching_all", -- READ_ONCE(scx_switching_all)); -- seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all()); -- seq_printf(m, "%-30s: %s\n", "enable_state", -- scx_ops_enable_state_str[scx_ops_enable_state()]); -- seq_printf(m, "%-30s: %llu\n", "nr_rejected", -- atomic64_read(&scx_nr_rejected)); -- mutex_unlock(&scx_ops_enable_mutex); -- return 0; --} -- --static int scx_debug_open(struct inode *inode, struct file *file) --{ -- return single_open(file, scx_debug_show, NULL); --} -- --const struct file_operations sched_ext_fops = { -- .open = scx_debug_open, -- .read = seq_read, -- .llseek = seq_lseek, -- .release = single_release, --}; --#endif -- --/******************************************************************************** -- * bpf_struct_ops plumbing. -- */ --#include --#include --#include -- --extern struct btf *btf_vmlinux; --static const struct btf_type *task_struct_type; -- --static bool bpf_scx_is_valid_access(int off, int size, -- enum bpf_access_type type, -- const struct bpf_prog *prog, -- struct bpf_insn_access_aux *info) --{ -- if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) -- return false; -- if (type != BPF_READ) -- return false; -- if (off % size != 0) -- return false; -- -- return btf_ctx_access(off, size, type, prog, info); --} -- --static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, -- const struct bpf_reg_state *reg, -- int off, int size) --{ -- return 0; --} -- --static const struct bpf_func_proto * --bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) --{ -- switch (func_id) { -- case BPF_FUNC_task_storage_get: -- return &bpf_task_storage_get_proto; -- case BPF_FUNC_task_storage_delete: -- return &bpf_task_storage_delete_proto; -- default: -- return bpf_base_func_proto(func_id); -- } --} -- --const struct bpf_verifier_ops bpf_scx_verifier_ops = { -- .get_func_proto = bpf_scx_get_func_proto, -- .is_valid_access = bpf_scx_is_valid_access, -- .btf_struct_access = bpf_scx_btf_struct_access, --}; -- --static int bpf_scx_init_member(const struct btf_type *t, -- const struct btf_member *member, -- void *kdata, const void *udata) --{ -- const struct sched_ext_ops *uops = udata; -- struct sched_ext_ops *ops = kdata; -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- int ret; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, dispatch_max_batch): -- if (*(u32 *)(udata + moff) > INT_MAX) -- return -E2BIG; -- ops->dispatch_max_batch = *(u32 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, flags): -- if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) -- return -EINVAL; -- ops->flags = *(u64 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, name): -- ret = bpf_obj_name_cpy(ops->name, uops->name, -- sizeof(ops->name)); -- if (ret < 0) -- return ret; -- if (ret == 0) -- return -EINVAL; -- return 1; -- case offsetof(struct sched_ext_ops, timeout_ms): -- if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT) -- return -E2BIG; -- ops->timeout_ms = *(u32 *)(udata + moff); -- return 1; -- } -- -- return 0; --} -- --static int bpf_scx_check_member(const struct btf_type *t, -- const struct btf_member *member, -- const struct bpf_prog *prog) --{ -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, prep_enable): -- case offsetof(struct sched_ext_ops, init): -- case offsetof(struct sched_ext_ops, exit): -- break; -- default: -- /* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */ -- /* if (prog->aux->sleepable) -- return -EINVAL; */ -- break; -- } -- -- return 0; --} -- --static int bpf_scx_reg(void *kdata) --{ -- return scx_ops_enable(kdata); --} -- --static void bpf_scx_unreg(void *kdata) --{ -- scx_ops_disable(SCX_EXIT_UNREG); -- kthread_flush_work(&scx_ops_disable_work); --} -- --static int bpf_scx_init(struct btf *btf) --{ -- u32 type_id; -- -- type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); -- if (type_id < 0) -- return -EINVAL; -- task_struct_type = btf_type_by_id(btf, type_id); -- -- return 0; --} -- --/* "extern" to avoid sparse warning, only used in this file */ --extern struct bpf_struct_ops bpf_sched_ext_ops; -- --struct bpf_struct_ops bpf_sched_ext_ops = { -- .verifier_ops = &bpf_scx_verifier_ops, -- .reg = bpf_scx_reg, -- .unreg = bpf_scx_unreg, -- .check_member = bpf_scx_check_member, -- .init_member = bpf_scx_init_member, -- .init = bpf_scx_init, -- .name = "sched_ext_ops", --}; -- --static void sysrq_handle_sched_ext_reset(u8 key) --{ -- if (scx_ops_helper) -- scx_ops_disable(SCX_EXIT_SYSRQ); -- else -- pr_info("sched_ext: BPF scheduler not yet used\n"); --} -- --static const struct sysrq_key_op sysrq_sched_ext_reset_op = { -- .handler = sysrq_handle_sched_ext_reset, -- .help_msg = "reset-sched-ext(S)", -- .action_msg = "Disable sched_ext and revert all tasks to CFS", -- .enable_mask = SYSRQ_ENABLE_RTNICE, --}; -- --static void kick_cpus_irq_workfn(struct irq_work *irq_work) --{ -- struct rq *this_rq = this_rq(); -- u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); -- int this_cpu = cpu_of(this_rq); -- int cpu; -- -- for_each_cpu(cpu, this_rq->scx->cpus_to_kick) { -- struct rq *rq = cpu_rq(cpu); -- unsigned long flags; -- -- raw_spin_rq_lock_irqsave(rq, flags); -- -- if (cpu_online(cpu) || cpu == this_cpu) { -- if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) && -- rq->curr->sched_class == &ext_sched_class) -- rq->curr->scx->slice = 0; -- pseqs[cpu] = rq->scx->pnt_seq; -- resched_curr(rq); -- } else { -- cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait); -- } -- -- raw_spin_rq_unlock_irqrestore(rq, flags); -- } -- -- for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait, -- cpumask_of(this_cpu)) { -- /* -- * Pairs with smp_store_release() issued by this CPU in -- * scx_notify_pick_next_task() on the resched path. -- * -- * We busy-wait here to guarantee that no other task can be -- * scheduled on our core before the target CPU has entered the -- * resched path. -- */ -- while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu]) -- cpu_relax(); -- } -- -- cpumask_clear(this_rq->scx->cpus_to_kick); -- cpumask_clear(this_rq->scx->cpus_to_preempt); -- cpumask_clear(this_rq->scx->cpus_to_wait); --} -- --void __init init_sched_ext_class(void) --{ -- int cpu; -- u32 v; -- -- /* -- * The following is to prevent the compiler from optimizing out the enum -- * definitions so that BPF scheduler implementations can use them -- * through the generated vmlinux.h. -- */ -- WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | -- SCX_TG_ONLINE | SCX_KICK_PREEMPT); -- -- init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); --#ifdef CONFIG_SMP -- BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -- BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); --#endif -- scx_kick_cpus_pnt_seqs = -- __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * -- num_possible_cpus(), -- __alignof__(scx_kick_cpus_pnt_seqs[0])); -- BUG_ON(!scx_kick_cpus_pnt_seqs); -- -- for_each_possible_cpu(cpu) { -- struct rq *rq = cpu_rq(cpu); -- -- rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL); -- if (rq->scx) -- rq->scx->rq = rq; -- else -- pr_err("fatal error : alloc rq->scx failed!!!\n"); -- -- init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL); -- INIT_LIST_HEAD(&rq->scx->watchdog_list); -- -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL)); -- init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn); -- } -- -- register_sysrq_key('S', &sysrq_sched_ext_reset_op); -- INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); -- scx_cgroup_config_knobs(); --} -- -- --/******************************************************************************** -- * Helpers that can be called from the BPF scheduler. -- */ --#include -- --/* Disables missing prototype warnings for kfuncs */ --__diag_push(); --__diag_ignore_all("-Wmissing-prototypes", -- "Global functions as their definitions will be in vmlinux BTF"); -- --/** -- * scx_bpf_switch_all - Switch all tasks into SCX -- * @into_scx: switch direction -- * -- * If @into_scx is %true, all existing and future non-dl/rt tasks are switched -- * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on -- * SCX. The actual switching is asynchronous. Can be called from ops.init(). -- */ --void scx_bpf_switch_all(void) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return; -- -- scx_switch_all_req = true; --} -- --BTF_SET8_START(scx_kfunc_ids_init) --BTF_ID_FLAGS(func, scx_bpf_switch_all) --BTF_SET8_END(scx_kfunc_ids_init) -- --static const struct btf_kfunc_id_set scx_kfunc_set_init = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_init, --}; -- --/** -- * scx_bpf_create_dsq - Create a custom DSQ -- * @dsq_id: DSQ to create -- * @node: NUMA node to allocate from -- * -- * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(), -- * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move(). -- */ --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return -EINVAL; -- -- if (unlikely(node >= (int)nr_node_ids || -- (node < 0 && node != NUMA_NO_NODE))) -- return -EINVAL; -- return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); --} -- --BTF_SET8_START(scx_kfunc_ids_sleepable) --BTF_ID_FLAGS(func, scx_bpf_create_dsq) --BTF_SET8_END(scx_kfunc_ids_sleepable) -- --static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_sleepable, --}; -- --static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) --{ -- if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) -- return false; -- -- lockdep_assert_irqs_disabled(); -- -- if (unlikely(!p)) { -- scx_ops_error("called with NULL task"); -- return false; -- } -- -- if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { -- scx_ops_error("invalid enq_flags 0x%llx", enq_flags); -- return false; -- } -- -- return true; --} -- --static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) --{ -- struct task_struct *ddsp_task; -- int idx; -- -- ddsp_task = __this_cpu_read(direct_dispatch_task); -- if (ddsp_task) { -- direct_dispatch(ddsp_task, p, dsq_id, enq_flags); -- return; -- } -- -- idx = __this_cpu_read(scx_dsp_ctx.buf_cursor); -- if (unlikely(idx >= scx_dsp_max_batch)) { -- scx_ops_error("dispatch buffer overflow"); -- return; -- } -- -- this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){ -- .task = p, -- .qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK, -- .dsq_id = dsq_id, -- .enq_flags = enq_flags, -- }; -- __this_cpu_inc(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe -- * to call this function spuriously. Can be called from ops.enqueue() and -- * ops.dispatch(). -- * -- * When called from ops.enqueue(), it's for direct dispatch and @p must match -- * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the -- * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be -- * on the target CPU in the first place. -- * -- * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id -- * and this function can be called upto ops.dispatch_max_batch times to dispatch -- * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the -- * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. -- * -- * This function doesn't have any locking restrictions and may be called under -- * BPF locks (in the future when BPF introduces more flexible locking). -- * -- * @p is allowed to run for @slice. The scheduling path is triggered on slice -- * exhaustion. If zero, the current residual slice is maintained. If -- * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with -- * scx_bpf_kick_cpu() to trigger scheduling. -- */ --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- scx_dispatch_commit(p, dsq_id, enq_flags); --} -- --/** -- * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. -- * Tasks queued into the priority queue are ordered by @vtime and always -- * consumed after the tasks in the FIFO queue. All other aspects are identical -- * to scx_bpf_dispatch(). -- * -- * @vtime ordering is according to time_before64() which considers wrapping. A -- * numerically larger vtime may indicate an earlier position in the ordering and -- * vice-versa. -- */ --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 vtime, u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- p->scx->dsq_vtime = vtime; -- -- scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); --} -- --BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime) --BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_enqueue_dispatch, --}; -- --/** -- * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots -- * -- * Can only be called from ops.dispatch(). -- */ --u32 scx_bpf_dispatch_nr_slots(void) --{ -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return 0; -- -- return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ -- * @dsq_id: DSQ to consume -- * -- * Consume a task from the non-local DSQ identified by @dsq_id and transfer it -- * to the current CPU's local DSQ for execution. Can only be called from -- * ops.dispatch(). -- * -- * This function flushes the in-flight dispatches from scx_bpf_dispatch() before -- * trying to consume the specified DSQ. It may also grab rq locks and thus can't -- * be called under any BPF locks. -- * -- * Returns %true if a task has been consumed, %false if there isn't any task to -- * consume. -- */ --bool scx_bpf_consume(u64 dsq_id) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- struct scx_dispatch_q *dsq; -- -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return false; -- -- flush_dispatch_buf(dspc->rq, dspc->rf); -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); -- return false; -- } -- -- if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) { -- /* -- * A successfully consumed task can be dequeued before it starts -- * running while the CPU is trying to migrate other dispatched -- * tasks. Bump nr_tasks to tell balance_scx() to retry on empty -- * local DSQ. -- */ -- dspc->nr_tasks++; -- return true; -- } else { -- return false; -- } --} -- --BTF_SET8_START(scx_kfunc_ids_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) --BTF_ID_FLAGS(func, scx_bpf_consume) --BTF_SET8_END(scx_kfunc_ids_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_dispatch, --}; -- --/** -- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ -- * -- * Iterate over all of the tasks currently enqueued on the local DSQ of the -- * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of -- * processed tasks. Can only be called from ops.cpu_release(). -- */ --u32 scx_bpf_reenqueue_local(void) --{ -- u32 nr_enqueued, i; -- struct rq *rq; -- struct scx_rq *scx_rq; -- -- if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) -- return 0; -- -- rq = cpu_rq(smp_processor_id()); -- lockdep_assert_rq_held(rq); -- scx_rq = rq->scx; -- -- /* -- * Get the number of tasks on the local DSQ before iterating over it to -- * pull off tasks. The enqueue callback below can signal that it wants -- * the task to stay on the local DSQ, and we want to prevent the BPF -- * scheduler from causing us to loop indefinitely. -- */ -- nr_enqueued = scx_rq->local_dsq.nr; -- for (i = 0; i < nr_enqueued; i++) { -- struct task_struct *p; -- -- p = first_local_task(rq); -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- WARN_ON_ONCE(p->scx->holding_cpu != -1); -- dispatch_dequeue(scx_rq, p); -- do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); -- } -- -- return nr_enqueued; --} -- --BTF_SET8_START(scx_kfunc_ids_cpu_release) --BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) --BTF_SET8_END(scx_kfunc_ids_cpu_release) -- --static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_cpu_release, --}; -- --/** -- * scx_bpf_kick_cpu - Trigger reschedule on a CPU -- * @cpu: cpu to kick -- * @flags: SCX_KICK_* flags -- * -- * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or -- * trigger rescheduling on a busy CPU. This can be called from any online -- * scx_ops operation and the actual kicking is performed asynchronously through -- * an irq work. -- */ --void scx_bpf_kick_cpu(s32 cpu, u64 flags) --{ -- struct rq *rq; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d", cpu); -- return; -- } -- -- preempt_disable(); -- rq = this_rq(); -- -- /* -- * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting -- * rq locks. We can probably be smarter and avoid bouncing if called -- * from ops which don't hold a rq lock. -- */ -- cpumask_set_cpu(cpu, rq->scx->cpus_to_kick); -- if (flags & SCX_KICK_PREEMPT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt); -- if (flags & SCX_KICK_WAIT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_wait); -- -- irq_work_queue(&rq->scx->kick_cpus_irq_work); -- preempt_enable(); --} -- --/** -- * scx_bpf_dsq_nr_queued - Return the number of queued tasks -- * @dsq_id: id of the DSQ -- * -- * Return the number of tasks in the DSQ matching @dsq_id. If not found, -- * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops -- * operations. -- */ --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) --{ -- struct scx_dispatch_q *dsq; -- -- lockdep_assert(rcu_read_lock_any_held()); -- -- if (dsq_id == SCX_DSQ_LOCAL) { -- return this_rq()->scx->local_dsq.nr; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (ops_cpu_valid(cpu)) -- return cpu_rq(cpu)->scx->local_dsq.nr; -- } else { -- dsq = find_non_local_dsq(dsq_id); -- if (dsq) -- return dsq->nr; -- } -- return -ENOENT; --} -- --/** -- * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state -- * @cpu: cpu to test and clear idle for -- * -- * Returns %true if @cpu was idle and its idle state was successfully cleared. -- * %false otherwise. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return false; -- } -- -- if (ops_cpu_valid(cpu)) -- return test_and_clear_cpu_idle(cpu); -- else -- return false; --} -- --/** -- * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu -- * @cpus_allowed: Allowed cpumask -- * -- * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked -- * idle cpu number on success. -%EBUSY if no matching cpu was found. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return -EBUSY; -- } -- -- return scx_pick_idle_cpu(cpus_allowed); --} -- --/** -- * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking -- * per-CPU cpumask. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_cpumask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.cpu; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, -- * per-physical-core cpumask. Can be used to determine if an entire physical -- * core is free. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_smtmask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.smt; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to -- * either the percpu, or SMT idle-tracking cpumask. -- */ --void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) --{ -- /* -- * Empty function body because we aren't actually acquiring or -- * releasing a reference to a global idle cpumask, which is read-only -- * in the caller and is never released. The acquire / release semantics -- * here are just used to make the cpumask is a trusted pointer in the -- * caller. -- */ --} -- --struct scx_bpf_error_bstr_bufs { -- u64 data[MAX_BPRINTF_VARARGS]; -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs); -- --/** -- * scx_bpf_error_bstr - Indicate fatal error -- * @fmt: error message format string -- * @data: format string parameters packaged using ___bpf_fill() macro -- * @data__sz: @data len, must end in '__sz' for the verifier -- * -- * Indicate that the BPF scheduler encountered a fatal error and initiate ops -- * disabling. -- */ --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) --{ -- struct scx_bpf_error_bstr_bufs *bufs; -- unsigned long flags; -- struct bpf_bprintf_data args = {}; -- int ret; -- -- local_irq_save(flags); -- bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs); -- -- if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || -- (data__sz && !data)) { -- scx_ops_error("invalid data=%p and data__sz=%u", -- (void *)data, data__sz); -- goto out_restore; -- } -- -- ret = copy_from_kernel_nofault(bufs->data, data, data__sz); -- if (ret) { -- scx_ops_error("failed to read data fields (%d)", ret); -- goto out_restore; -- } -- -- ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args); -- if (ret < 0) { -- scx_ops_error("failed to format prepration (%d)", ret); -- goto out_restore; -- } -- -- ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt, -- args.bin_args); -- bpf_bprintf_cleanup(&args); -- if (ret < 0) { -- scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format", -- fmt, data, data__sz); -- goto out_restore; -- } -- -- scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg); --out_restore: -- local_irq_restore(flags); --} -- --/** -- * scx_bpf_destroy_dsq - Destroy a custom DSQ -- * @dsq_id: DSQ to destroy -- * -- * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with -- * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is -- * empty and no further tasks are dispatched to it. Ignored if called on a DSQ -- * which doesn't exist. Can be called from any online scx_ops operations. -- */ --void scx_bpf_destroy_dsq(u64 dsq_id) --{ -- destroy_dsq(dsq_id); --} -- --/** -- * scx_bpf_task_running - Is task currently running? -- * @p: task of interest -- */ --bool scx_bpf_task_running(const struct task_struct *p) --{ -- return task_rq(p)->curr == p; --} -- --/** -- * scx_bpf_task_cpu - CPU a task is currently associated with -- * @p: task of interest -- */ --s32 scx_bpf_task_cpu(const struct task_struct *p) --{ -- return task_cpu(p); --} -- --/** -- * scx_bpf_task_cgroup - Return the sched cgroup of a task -- * @p: task of interest -- * -- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with -- * from the scheduler's POV. SCX operations should use this function to -- * determine @p's current cgroup as, unlike following @p->cgroups, -- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all -- * rq-locked operations. Can be called on the parameter tasks of rq-locked -- * operations. The restriction guarantees that @p's rq is locked by the caller. -- */ --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) --{ -- struct task_group *tg = p->sched_task_group; -- struct cgroup *cgrp = &cgrp_dfl_root.cgrp; -- -- if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) -- goto out; -- -- /* -- * A task_group may either be a cgroup or an autogroup. In the latter -- * case, @tg->css.cgroup is %NULL. A task_group can't become the other -- * kind once created. -- */ -- if (tg && tg->css.cgroup) -- cgrp = tg->css.cgroup; -- else -- cgrp = &cgrp_dfl_root.cgrp; --out: -- cgroup_get(cgrp); -- return cgrp; --} -- --BTF_SET8_START(scx_kfunc_ids_any) --BTF_ID_FLAGS(func, scx_bpf_kick_cpu) --BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) --BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) --BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu) --BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) --BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) --BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) --BTF_ID_FLAGS(func, scx_bpf_task_running) --BTF_ID_FLAGS(func, scx_bpf_task_cpu) --BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE) --BTF_SET8_END(scx_kfunc_ids_any) -- --static const struct btf_kfunc_id_set scx_kfunc_set_any = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_any, --}; -- --__diag_pop(); -- --/* -- * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set() -- * needs most of the system to be up. -- */ --static int __init register_ext_kfuncs(void) --{ -- int ret; -- -- /* -- * Some kfuncs are context-sensitive and can only be called from -- * specific SCX ops. They are grouped into BTF sets accordingly. -- * Unfortunately, BPF currently doesn't have a way of enforcing such -- * restrictions. Eventually, the verifier should be able to enforce -- * them. For now, register them the same and make each kfunc explicitly -- * check using scx_kf_allowed(). -- */ -- if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_init)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_sleepable)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_enqueue_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_cpu_release)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_any))) { -- pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret); -- return ret; -- } -- -- return 0; --} --__initcall(register_ext_kfuncs); -diff --git a/kernel/sched/ext.h b/kernel/sched/ext.h -deleted file mode 100755 -index fba8ed845f99..000000000000 ---- a/kernel/sched/ext.h -+++ /dev/null -@@ -1,257 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ -- -- --/* -- * Tag marking a kernel function as a kfunc. This is meant to minimize the -- * amount of copy-paste that kfunc authors have to include for correctness so -- * as to avoid issues such as the compiler inlining or eliding either a static -- * kfunc, or a global kfunc in an LTO build. -- */ --#define __bpf_kfunc __used noinline -- --enum scx_wake_flags { -- /* expose select WF_* flags as enums */ -- SCX_WAKE_EXEC = WF_EXEC, -- SCX_WAKE_FORK = WF_FORK, -- SCX_WAKE_TTWU = WF_TTWU, -- SCX_WAKE_SYNC = WF_SYNC, --}; -- --enum scx_enq_flags { -- /* expose select ENQUEUE_* flags as enums */ -- SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, -- SCX_ENQ_HEAD = ENQUEUE_HEAD, -- -- /* high 32bits are SCX specific */ -- -- /* -- * Set the following to trigger preemption when calling -- * scx_bpf_dispatch() with a local dsq as the target. The slice of the -- * current task is cleared to zero and the CPU is kicked into the -- * scheduling path. Implies %SCX_ENQ_HEAD. -- */ -- SCX_ENQ_PREEMPT = 1LLU << 32, -- -- /* -- * The task being enqueued was previously enqueued on the current CPU's -- * %SCX_DSQ_LOCAL, but was removed from it in a call to the -- * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was -- * invoked in a ->cpu_release() callback, and the task is again -- * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the -- * task will not be scheduled on the CPU until at least the next invocation -- * of the ->cpu_acquire() callback. -- */ -- SCX_ENQ_REENQ = 1LLU << 40, -- -- /* -- * The task being enqueued is the only task available for the cpu. By -- * default, ext core keeps executing such tasks but when -- * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with -- * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set. -- * -- * If the BPF scheduler wants to continue executing the task, -- * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately. -- * If the task gets queued on a different dsq or the BPF side, the BPF -- * scheduler is responsible for triggering a follow-up scheduling event. -- * Otherwise, Execution may stall. -- */ -- SCX_ENQ_LAST = 1LLU << 41, -- -- /* -- * A hint indicating that it's advisable to enqueue the task on the -- * local dsq of the currently selected CPU. Currently used by -- * select_cpu_dfl() and together with %SCX_ENQ_LAST. -- */ -- SCX_ENQ_LOCAL = 1LLU << 42, -- -- /* high 8 bits are internal */ -- __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, -- -- SCX_ENQ_CLEAR_OPSS = 1LLU << 56, -- SCX_ENQ_DSQ_PRIQ = 1LLU << 57, --}; -- --enum scx_deq_flags { -- /* expose select DEQUEUE_* flags as enums */ -- SCX_DEQ_SLEEP = DEQUEUE_SLEEP, -- -- /* high 32bits are SCX specific */ -- -- /* -- * The generic core-sched layer decided to execute the task even though -- * it hasn't been dispatched yet. Dequeue from the BPF side. -- */ -- SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, --}; -- --enum scx_tg_flags { -- SCX_TG_ONLINE = 1U << 0, -- SCX_TG_INITED = 1U << 1, --}; -- --enum scx_kick_flags { -- SCX_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -- SCX_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ --}; -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --extern const struct sched_class ext_sched_class; --extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops; --extern const struct file_operations sched_ext_fops; --extern unsigned long scx_watchdog_timeout; --extern unsigned long scx_watchdog_timestamp; -- --DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); --DECLARE_STATIC_KEY_FALSE(__scx_switched_all); --#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) --#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) -- --DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); -- --bool task_on_scx(struct task_struct *p); --void scx_pre_fork(struct task_struct *p); --int scx_fork(struct task_struct *p); --void scx_post_fork(struct task_struct *p); --void scx_cancel_fork(struct task_struct *p); --int scx_check_setscheduler(struct task_struct *p, int policy); --bool scx_can_stop_tick(struct rq *rq); --void init_sched_ext_class(void); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...); --#define scx_ops_error(fmt, args...) \ -- scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args) -- --void __scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active); -- --static inline void scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active) --{ -- if (!scx_enabled()) -- return; --#ifdef CONFIG_SMP -- /* -- * Pairs with the smp_load_acquire() issued by a CPU in -- * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -- * resched. -- */ -- smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1); --#endif -- if (!static_branch_unlikely(&scx_ops_cpu_preempt)) -- return; -- __scx_notify_pick_next_task(rq, p, active); --} -- --//extern void scx_scheduler_tick(void); --static inline void scx_notify_sched_tick(void) --{ -- unsigned long last_check; -- -- if (!scx_enabled()) -- return; -- //scx_scheduler_tick(); -- -- last_check = scx_watchdog_timestamp; -- if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "watchdog failed to check in for %u.%03us", -- dur_ms / 1000, dur_ms % 1000); -- } --} -- --static inline const struct sched_class *next_active_class(const struct sched_class *class) --{ -- class++; -- if (scx_switched_all() && class == &fair_sched_class) -- class++; -- if (!scx_enabled() && class == &ext_sched_class) -- class++; -- return class; --} -- --#define for_active_class_range(class, _from, _to) \ -- for (class = (_from); class != (_to); class = next_active_class(class)) -- --#define for_each_active_class(class) \ -- for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -- --/* -- * SCX requires a balance() call before every pick_next_task() call including -- * when waking up from idle. -- */ --#define for_balance_class_range(class, prev_class, end_class) \ -- for_active_class_range(class, (prev_class) > &ext_sched_class ? \ -- &ext_sched_class : (prev_class), (end_class)) -- --#ifdef CONFIG_SCHED_CORE --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi); --#endif -- --#else /* CONFIG_SCHED_CLASS_EXT */ -- --#define scx_enabled() false --#define scx_switched_all() false -- --static inline void scx_pre_fork(struct task_struct *p) {} --static inline int scx_fork(struct task_struct *p) { return 0; } --static inline void scx_post_fork(struct task_struct *p) {} --static inline void scx_cancel_fork(struct task_struct *p) {} --static inline int scx_check_setscheduler(struct task_struct *p, -- int policy) { return 0; } --static inline bool scx_can_stop_tick(struct rq *rq) { return true; } --static inline void init_sched_ext_class(void) {} --static inline void scx_notify_pick_next_task(struct rq *rq, -- const struct task_struct *p, -- const struct sched_class *active) {} --static inline void scx_notify_sched_tick(void) {} -- --#define for_each_active_class for_each_class --#define for_balance_class_range for_class_range -- --#endif /* CONFIG_SCHED_CLASS_EXT */ -- --#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) --void __scx_update_idle(struct rq *rq, bool idle); -- --static inline void scx_update_idle(struct rq *rq, bool idle) --{ -- if (scx_enabled()) -- __scx_update_idle(rq, idle); --} --#else --static inline void scx_update_idle(struct rq *rq, bool idle) {} --#endif -- --#ifdef CONFIG_CGROUP_SCHED --#ifdef CONFIG_EXT_GROUP_SCHED --int scx_tg_online(struct task_group *tg); --void scx_tg_offline(struct task_group *tg); --int scx_cgroup_can_attach(struct cgroup_taskset *tset); --void scx_move_task(struct task_struct *p); --void scx_cgroup_finish_attach(void); --void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); --void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); --#else /* CONFIG_EXT_GROUP_SCHED */ --static inline int scx_tg_online(struct task_group *tg) { return 0; } --static inline void scx_tg_offline(struct task_group *tg) {} --static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } --static inline void scx_move_task(struct task_struct *p) {} --static inline void scx_cgroup_finish_attach(void) {} --static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} --static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} --#endif /* CONFIG_EXT_GROUP_SCHED */ --#endif /* CONFIG_CGROUP_SCHED */ -diff --git a/kernel/sched/hmbird.h b/kernel/sched/hmbird.h -new file mode 100755 -index 000000000000..d0d84ddee13d ---- /dev/null -+++ b/kernel/sched/hmbird.h -@@ -0,0 +1,343 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#ifndef _HMBIRD_H_ -+#define _HMBIRD_H_ -+ -+/* -+ * Tag marking a kernel function as a kfunc. This is meant to minimize the -+ * amount of copy-paste that kfunc authors have to include for correctness so -+ * as to avoid issues such as the compiler inlining or eliding either a static -+ * kfunc, or a global kfunc in an LTO build. -+ */ -+ -+#define DEBUG_INTERNAL (1 << 0) -+#define DEBUG_INFO_TRACE (1 << 1) -+#define DEBUG_INFO_SYSTRACE (1 << 2) -+ -+#define NUMS_CGROUP_KINDS (512) -+#define SLIM_FOR_SGAME (1 << 0) -+#define SLIM_FOR_GENSHIN (1 << 1) -+ -+#ifdef MAX_TASK_NR -+#define MAX_KEY_THREAD_RECORD ((MAX_TASK_NR + 1) >> 1) -+#else -+#define MAX_KEY_THREAD_RECORD (1 << 3) -+#endif /* MAX_TASK_NR */ -+ -+#define TOP_TASK_SHIFT (8) -+#define TOP_TASK_MAX (1 << TOP_TASK_SHIFT) -+#ifndef TOP_TASK_BITS_MASK -+#define TOP_TASK_BITS_MASK (TOP_TASK_MAX - 1) -+#endif /* TOP_TASK_BITS_MASK */ -+ -+extern struct md_info_t *md_info; -+ -+#define debug_enabled() \ -+ (unlikely(hmbirdcore_debug)) -+ -+#define hmbird_debug(fmt, ...) \ -+ pr_info(":"fmt, ##__VA_ARGS__) -+ -+#define hmbird_err(id, fmt, ...) \ -+do { \ -+ pr_err(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_deferred_err(id, fmt, ...) \ -+do { \ -+ printk_deferred(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \ -+do { \ -+ if (unlikely(cond)) { \ -+ hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+ } \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_info_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_info_trace(fmt, ...) -+#endif -+ -+#define hmbird_info_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+ } \ -+} while (0) -+ -+#define hmbird_output_systrace(fmt, ...) \ -+do { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_internal_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_internal_trace(fmt, ...) -+#endif -+ -+#define hmbird_internal_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) { \ -+ hmbird_output_systrace(fmt, ##__VA_ARGS__); \ -+ } \ -+} while (0) -+ -+enum hmbird_wake_flags { -+ /* expose select WF_* flags as enums */ -+ HMBIRD_WAKE_EXEC = WF_EXEC, -+ HMBIRD_WAKE_FORK = WF_FORK, -+ HMBIRD_WAKE_TTWU = WF_TTWU, -+ HMBIRD_WAKE_SYNC = WF_SYNC, -+}; -+ -+enum hmbird_enq_flags { -+ /* expose select ENQUEUE_* flags as enums */ -+ HMBIRD_ENQ_WAKEUP = ENQUEUE_WAKEUP, -+ HMBIRD_ENQ_HEAD = ENQUEUE_HEAD, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * Set the following to trigger preemption when calling -+ * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the -+ * current task is cleared to zero and the CPU is kicked into the -+ * scheduling path. Implies %HMBIRD_ENQ_HEAD. -+ */ -+ HMBIRD_ENQ_PREEMPT = 1LLU << 32, -+ HMBIRD_ENQ_REENQ = 1LLU << 40, -+ HMBIRD_ENQ_LAST = 1LLU << 41, -+ -+ /* -+ * A hint indicating that it's advisable to enqueue the task on the -+ * local dsq of the currently selected CPU. Currently used by -+ * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST. -+ */ -+ HMBIRD_ENQ_LOCAL = 1LLU << 42, -+ -+ /* high 8 bits are internal */ -+ __HMBIRD_ENQ_INTERNAL_MASK = 0xffLLU << 56, -+ -+ HMBIRD_ENQ_CLEAR_OPSS = 1LLU << 56, -+ HMBIRD_ENQ_DSQ_PRIQ = 1LLU << 57, -+}; -+ -+enum hmbird_deq_flags { -+ /* expose select DEQUEUE_* flags as enums */ -+ HMBIRD_DEQ_SLEEP = DEQUEUE_SLEEP, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * The generic core-sched layer decided to execute the task even though -+ * it hasn't been dispatched yet. Dequeue from the BPF side. -+ */ -+ HMBIRD_DEQ_CORE_SCHED_EXEC = 1LLU << 32, -+}; -+ -+enum hmbird_kick_flags { -+ HMBIRD_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -+ HMBIRD_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ -+}; -+ -+enum hmbird_task_prop_type { -+ HMBIRD_TASK_PROP_COMMON, -+ HMBIRD_TASK_PROP_PIPELINE, -+ HMBIRD_TASK_PROP_DEBUG_OR_LOG, -+ HMBIRD_TASK_PROP_HIGH_LOAD, -+ HMBIRD_TASK_PROP_IO, -+ HMBIRD_TASK_PROP_NETWORK, -+ HMBIRD_TASK_PROP_PERIODIC, -+ HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL, -+ HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL, -+ HMBIRD_TASK_PROP_ISOLATE, -+ HMBIRD_TASK_PROP_MAX, -+}; -+ -+enum hmbird_preempt_policy_id { -+ HMBIRD_PREEMPT_POLICY_NONE, -+ HMBIRD_PREEMPT_POLICY_PRIO_BASED, -+}; -+ -+extern const struct sched_class hmbird_sched_class; -+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops; -+extern const struct file_operations sched_hmbird_fops; -+extern unsigned long hmbird_watchdog_timeout; -+extern unsigned long hmbird_watchdog_timestamp; -+ -+enum scx_rq_flags { -+ HMBIRD_RQ_CAN_STOP_TICK = 1 << 0, -+}; -+ -+struct hmbird_rq { -+ struct hmbird_dispatch_q local_dsq; -+ struct list_head watchdog_list; -+ u64 ops_qseq; -+ u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -+ u32 nr_running; -+ u32 flags; -+ bool cpu_released; -+ cpumask_var_t cpus_to_kick; -+ cpumask_var_t cpus_to_preempt; -+ cpumask_var_t cpus_to_wait; -+ u64 pnt_seq; -+ u64 *prev_runnable_sum_fixed; -+ u32 *prev_window_size; -+ struct irq_work kick_cpus_irq_work; -+ bool pipeline; -+ bool exclusive; -+ bool period_disallow; -+ bool nonperiod_disallow; -+ struct rq *rq; -+ struct hmbird_sched_rq_stats *srq; -+}; -+ -+struct hmbird_sched_change_guard { -+ struct task_struct *p; -+ struct rq *rq; -+ bool queued; -+ bool running; -+ bool done; -+}; -+ -+extern struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -+ -+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags); -+ -+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -+ for (struct hmbird_sched_change_guard __cg = \ -+ hmbird_sched_change_guard_init(__rq, __p, __flags); \ -+ !__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags)) -+ -+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+bool task_on_hmbird(struct task_struct *p); -+int hmbird_pre_fork(struct task_struct *p); -+int hmbird_fork(struct task_struct *p); -+void hmbird_post_fork(struct task_struct *p); -+void hmbird_cancel_fork(struct task_struct *p); -+int hmbird_check_setscheduler(struct task_struct *p, int policy); -+bool hmbird_can_stop_tick(struct rq *rq); -+void init_sched_hmbird_class(void); -+ -+void hmbird_ops_exit(void); -+#define hmbird_ops_error(fmt, ...) \ -+do { \ -+ hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__); \ -+ hmbird_ops_exit(); \ -+} while (0) -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active); -+ -+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight) -+{ -+ return clamp_t(unsigned long, -+ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -+ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); -+} -+ -+static inline void hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active) -+{ -+ if (!hmbird_enabled()) -+ return; -+#ifdef CONFIG_SMP -+ /* -+ * Pairs with the smp_load_acquire() issued by a CPU in -+ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -+ * resched. -+ */ -+ smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1); -+#endif -+ if (!static_branch_unlikely(&hmbird_ops_cpu_preempt)) -+ return; -+ __hmbird_notify_pick_next_task(rq, p, active); -+} -+ -+extern void hmbird_scheduler_tick(void); -+void scan_timeout(struct rq *rq); -+void hmbird_notify_sched_tick(void); -+ -+static inline const struct sched_class *next_active_class(const struct sched_class *class) -+{ -+ class++; -+ -+ if (hmbird_enabled() && class == &fair_sched_class) -+ class++; -+ if (!hmbird_enabled() && class == &hmbird_sched_class) -+ class++; -+ return class; -+} -+ -+#define for_active_class_range(class, _from, _to) \ -+ for (class = (_from); class != (_to); class = next_active_class(class)) -+ -+#define for_each_active_class(class) \ -+ for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -+ -+/* -+ * HMBIRD requires a balance() call before every pick_next_task() call including -+ * when waking up from idle. -+ */ -+#define for_balance_class_range(class, prev_class, end_class) \ -+ for_active_class_range(class, (prev_class) > &hmbird_sched_class ? \ -+ &hmbird_sched_class : (prev_class), (end_class)) -+ -+#define MIN_CGROUP_DL_IDX (5) /* 8ms */ -+#define DEFAULT_CGROUP_DL_IDX (8) /* 64ms */ -+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS]; -+ -+ -+void __hmbird_update_idle(struct rq *rq, bool idle); -+ -+static inline void hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ if (hmbird_enabled()) -+ __hmbird_update_idle(rq, idle); -+} -+ -+int hmbird_ctrl(bool enable); -+ -+int hmbird_tg_online(struct task_group *tg); -+ -+ -+static inline u16 hmbird_task_util(struct task_struct *p) -+{ -+ return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0; -+} -+u16 hmbird_cpu_util(int cpu); -+ -+bool get_hmbird_ops_enabled(void); -+bool get_non_hmbird_task(void); -+void set_cpu_cluster(u64 cpu_cluster); -+#endif /*_EXT_H_*/ -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -new file mode 100755 -index 000000000000..ce05928392bf ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird.c -@@ -0,0 +1,4313 @@ -+// SPDX-License-Identifier: GPL-2.0 -+ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#include -+#include -+ -+#include "slim.h" -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+#include -+ -+#define CLUSTER_SEPARATE -+ -+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0); -+atomic_t non_hmbird_task; -+atomic_t hmbird_module_loaded = ATOMIC_INIT(0); -+int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0}; -+ -+enum hmbird_internal_consts { -+ HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ, -+}; -+ -+enum hmbird_ops_enable_state { -+ HMBIRD_OPS_PREPPING, -+ HMBIRD_OPS_ENABLING, -+ HMBIRD_OPS_ENABLED, -+ HMBIRD_OPS_DISABLING, -+ HMBIRD_OPS_DISABLED, -+}; -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+/* -+ * hmbird_entity->ops_state -+ * -+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler. -+ * State transitions look as follows: -+ * -+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING -+ * ^ | | -+ * | v v -+ * \-------------------------------/ -+ * -+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -+ * sites for explanations on the conditions being waited upon and why they are -+ * safe. Transitions out of them into NONE or QUEUED must store_release and the -+ * waiters should load_acquire. -+ * -+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether -+ * any given task can be dispatched by the BPF scheduler at all times and thus -+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core -+ * can safely reject invalid dispatches. -+ */ -+enum hmbird_ops_state { -+ HMBIRD_OPSS_NONE, /* owned by the HMBIRD core */ -+ HMBIRD_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -+ HMBIRD_OPSS_QUEUED, /* owned by the BPF scheduler */ -+ HMBIRD_OPSS_DISPATCHING, /* in transit back to the HMBIRD core */ -+ -+ /* -+ * QSEQ brands each QUEUED instance so that, when dispatch races -+ * dequeue/requeue, the dispatcher can tell whether it still has a claim -+ * on the task being dispatched. -+ */ -+ HMBIRD_OPSS_QSEQ_SHIFT = 2, -+ HMBIRD_OPSS_STATE_MASK = (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1, -+ HMBIRD_OPSS_QSEQ_MASK = ~HMBIRD_OPSS_STATE_MASK, -+}; -+ -+enum switch_stat { -+ HMBIRD_DISABLED, -+ HMBIRD_SWITCH_PREP, -+ HMBIRD_RQ_SWITCH_BEGIN, -+ HMBIRD_RQ_SWITCH_DONE, -+ HMBIRD_ENABLED, -+}; -+enum switch_stat curr_ss; -+ -+/* -+ * During exit, a task may schedule after losing its PIDs. When disabling the -+ * BPF scheduler, we need to be able to iterate tasks in every state to -+ * guarantee system safety. Maintain a dedicated task list which contains every -+ * task between its fork and eventual free. -+ */ -+DEFINE_SPINLOCK(hmbird_tasks_lock); -+static LIST_HEAD(hmbird_tasks); -+ -+/* ops enable/disable */ -+static struct kthread_worker *hmbird_ops_helper; -+static DEFINE_MUTEX(hmbird_ops_enable_mutex); -+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem); -+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED); -+ -+static bool hmbird_warned_zero_slice; -+enum hmbird_switch_type sw_type; -+static int skip_num[MAX_GLOBAL_DSQS]; -+static int big_distribute_mask_prev; -+static int little_distribute_mask_prev; -+ -+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0); -+ -+/* -+ * The maximum amount of time in jiffies that a task may be runnable without -+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger -+ * hmbird_ops_error(). -+ */ -+unsigned long hmbird_watchdog_timeout; -+ -+/* -+ * The last time the delayed work was run. This delayed work relies on -+ * ksoftirqd being able to run to service timer interrupts, so it's possible -+ * that this work itself could get wedged. To account for this, we check that -+ * it's not stalled in the timer tick, and trigger an error if it is. -+ */ -+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES; -+ -+static struct delayed_work hmbird_watchdog_work; -+static struct work_struct hmbird_err_exit_work; -+ -+/* idle tracking */ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_CPUMASK_OFFSTACK -+#define CL_ALIGNED_IF_ONSTACK -+#else -+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp -+#endif -+ -+static struct { -+ cpumask_var_t cpu; -+ cpumask_var_t smt; -+} idle_masks CL_ALIGNED_IF_ONSTACK; -+ -+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus; -+#endif /* CONFIG_SMP */ -+ -+/* dispatch queues */ -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global; -+ -+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128}; -+u32 pcp_dsq_deadline = 20; -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS]; -+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq); -+ -+static u64 max_hmbird_dsq_internal_id; -+ -+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/ -+#define CLUSTER_SEPARATE_IDX (8) -+#define GDSQS_ID_BASE (3) -+#define UX_COMPATIBLE_IDX (4) -+#define NON_PERIOD_START (5) -+#define NON_PERIOD_END (MAX_GLOBAL_DSQS) -+#define CREATE_DSQ_LEVEL_WITHIN (1) -+ -+struct hmbird_sched_info { -+ spinlock_t lock; -+ int curr_idx[2]; -+ int rtime[MAX_GLOBAL_DSQS]; -+}; -+ -+struct pcp_sched_info { -+ s64 pcp_seq; -+ int rtime; -+ bool pcp_round; -+}; -+ -+/* -+ * pcp_info may rw by another cpu. -+ * protected by rq->lock. -+ */ -+atomic64_t pcp_dsq_round; -+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info); -+ -+struct md_info_t *md_info; -+ -+static int b_rescue_l, l_rescue_b; -+static struct hmbird_sched_info sinfo; -+ -+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC; -+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = { -+ 0, 0, 0, 0, 0, -+ 32 * NSEC_PER_MSEC, -+ 20 * NSEC_PER_MSEC, -+ 14 * NSEC_PER_MSEC, -+ 8 * NSEC_PER_MSEC, -+ 6 * NSEC_PER_MSEC -+}; -+ -+struct cluster_ctx { -+ /* cpu-dsq map must within [lower, upper) */ -+ int upper; -+ int lower; -+ int tidx; -+}; -+ -+enum stat_items { -+ GLOBAL_STAT, -+ CPU_ALLOW_FAIL, -+ RT_CNT, -+ KEY_TASK_CNT, -+ SWITCH_IDX, -+ TIMEOUT_CNT, -+ -+ TOTAL_DSP_CNT, -+ MOVE_RQ_CNT, -+ SELECT_CPU, -+ -+ DWORD_STAT_END = SELECT_CPU, -+ -+ GDSQ_CNT, -+ ERR_IDX, -+ PCP_TIMEOUT_CNT, -+ PCP_LDSQ_CNT, -+ PCP_ENQL_CNT, -+ -+ MAX_ITEMS, -+}; -+static DEFINE_SPINLOCK(stats_lock); -+static char *stats_str[MAX_ITEMS] = { -+ "global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt", -+ "switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt", -+ "select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt", -+ "pcp_ldsq_cnt", "pcp_enql_cnt" -+}; -+ -+ -+struct stats_struct { -+ u64 global_stat[2]; -+ u64 cpu_allow_fail[2]; -+ u64 rt_cnt[2]; -+ u64 key_task_cnt[2]; -+ u64 switch_idx[2]; -+ u64 timeout_cnt[2]; -+ -+ u64 total_dsp_cnt[2]; -+ u64 move_rq_cnt[2]; -+ u64 select_cpu[2]; -+ -+ u64 gdsq_count[MAX_GLOBAL_DSQS][2]; -+ u64 err_idx[5]; -+ u64 pcp_timeout_cnt[NR_CPUS]; -+ u64 pcp_ldsq_count[NR_CPUS][2]; -+ u64 pcp_enql_cnt[NR_CPUS]; -+} stats_data; -+ -+static struct { -+ cpumask_var_t ex_free; -+ cpumask_var_t exclusive; -+ cpumask_var_t partial; -+ cpumask_var_t big; -+ cpumask_var_t little; -+} iso_masks __read_mostly; -+ -+/* -+ * Need more synchronization for these two variables? -+ * I choose not to. -+ */ -+static int l_need_rescue, b_need_rescue; -+ -+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...) \ -+{ \ -+ char buf[MAX_FATAL_INFO]; \ -+ \ -+ scnprintf(buf, MAX_FATAL_INFO, fmt, ##args); \ -+ hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf); \ -+ trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable), \ -+ READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf); \ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); \ -+} \ -+ -+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2) -+{ -+ int c1, c2; -+ struct cpumask mask = {.bits = {0}}; -+ struct cpumask tmp = {.bits = {0}}; -+ -+ if (!slim_stats) -+ return false; -+ -+ if (!rq1 || !rq2) -+ return false; -+ -+ c1 = cpu_of(rq1); -+ c2 = cpu_of(rq2); -+ cpumask_set_cpu(c1, &mask); -+ cpumask_set_cpu(c2, &mask); -+ -+ if (cpumask_and(&tmp, iso_masks.little, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.big, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.partial, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ return false; -+} -+ -+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu) -+{ -+ unsigned long flags; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ if (!slim_stats) -+ return; -+ -+ switch (item) { -+ case GLOBAL_STAT: -+ fallthrough; -+ case CPU_ALLOW_FAIL: -+ fallthrough; -+ case RT_CNT: -+ fallthrough; -+ case KEY_TASK_CNT: -+ fallthrough; -+ case SWITCH_IDX: -+ fallthrough; -+ case TIMEOUT_CNT: -+ fallthrough; -+ case TOTAL_DSP_CNT: -+ fallthrough; -+ case MOVE_RQ_CNT: -+ fallthrough; -+ case SELECT_CPU: -+ pval = pbase + item * 2 + idx; -+ break; -+ case GDSQ_CNT: -+ pval = &stats_data.gdsq_count[dsq_id][idx]; -+ break; -+ case ERR_IDX: -+ pval = &stats_data.err_idx[idx]; -+ break; -+ case PCP_TIMEOUT_CNT: -+ pval = &stats_data.pcp_timeout_cnt[cpu]; -+ break; -+ case PCP_LDSQ_CNT: -+ pval = &stats_data.pcp_ldsq_count[cpu][idx]; -+ break; -+ case PCP_ENQL_CNT: -+ pval = &stats_data.pcp_enql_cnt[cpu]; -+ break; -+ default: -+ return; -+ } -+ -+ spin_lock_irqsave(&stats_lock, flags); -+ *pval += 1; -+ spin_unlock_irqrestore(&stats_lock, flags); -+} -+ -+static inline bool handle_ret(int ret, int *idx, int len) -+{ -+ if (ret < 0 || ret >= len - *idx) -+ return true; -+ *idx += ret; -+ return false; -+} -+ -+#define PRINT_INTV (5 * HZ) -+void stats_print(char *buf, int len) -+{ -+ int idx = 0, i, j, ret; -+ int item = 0; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ for (item = 0; item < MAX_ITEMS; item++) { -+ if (item <= DWORD_STAT_END) { -+ pval = pbase + item * 2; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n", -+ stats_str[item], pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == GDSQ_CNT) { -+ for (j = 0; j < MAX_GLOBAL_DSQS; j++) { -+ pval = (u64 *)&stats_data.gdsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == ERR_IDX) { -+ pval = (u64 *)&stats_data.err_idx; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n", -+ stats_str[item], pval[0], -+ pval[1], pval[2], pval[3], pval[4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == PCP_TIMEOUT_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_timeout_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_LDSQ_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_ldsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_ENQL_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_enql_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } -+ } -+ -+ if (!md_info) { -+ buf[idx] = '\0'; -+ return; -+ } -+ -+ ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_SWITCHS; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "sw_rec[%d] = %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.sw_rec[i].switch_at, -+ md_info->kern_dump.sw_rec[i].is_success, -+ md_info->kern_dump.sw_rec[i].end_state, -+ md_info->kern_dump.sw_rec[i].switch_reason); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_EXCEP_ID; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "excep[%d] = %llu %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.excep_rec[i][0], -+ md_info->kern_dump.excep_rec[i][1], -+ md_info->kern_dump.excep_rec[i][2], -+ md_info->kern_dump.excep_rec[i][3], -+ md_info->kern_dump.excep_rec[i][4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n", -+ i, md_info->kern_dump.excep_idx[i]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ -+ buf[idx] = '\0'; -+} -+ -+enum cpu_type { -+ LITTLE, -+ BIG, -+ PARTIAL, -+ EXCLUSIVE, -+ EX_FREE, -+ INVALID -+}; -+ -+enum dsq_type { -+ GLOBAL_DSQ, -+ PCP_DSQ, -+ OTHER, -+ MAX_DSQ_TYPE, -+}; -+ -+static enum cpu_type cpu_cluster(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (get_hmbird_rq(rq)->exclusive) { -+ return EXCLUSIVE; -+ } else { -+ if (cpumask_test_cpu(cpu, iso_masks.little)) -+ return LITTLE; -+ else if (cpumask_test_cpu(cpu, iso_masks.big)) -+ return BIG; -+ else if (cpumask_test_cpu(cpu, iso_masks.partial)) -+ return PARTIAL; -+ else if (cpumask_test_cpu(cpu, iso_masks.exclusive)) -+ return EXCLUSIVE; -+ else if (cpumask_test_cpu(cpu, iso_masks.ex_free)) -+ return EX_FREE; -+ } -+ return INVALID; -+} -+ -+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq) -+{ -+ if (!dsq) -+ return OTHER; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= GDSQS_ID_BASE) && -+ ((dsq->id & 0xff) < MAX_GLOBAL_DSQS)) -+ return GLOBAL_DSQ; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) && -+ ((dsq->id & 0xff) < max_hmbird_dsq_internal_id)) -+ return PCP_DSQ; -+ -+ return OTHER; -+} -+ -+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq) -+{ -+ enum dsq_type type; -+ -+ type = get_dsq_type(dsq); -+ switch (type) { -+ case GLOBAL_DSQ: -+ case PCP_DSQ: -+ return (dsq->id & 0xff) - GDSQS_ID_BASE; -+ default: -+ return -1; -+ } -+ return -1; -+} -+ -+static void update_cpus_idle(bool set, struct cpumask *mask) -+{ -+ int cpu; -+ struct rq *rq; -+ -+ if (set) { -+ for_each_cpu(cpu, mask) { -+ rq = cpu_rq(cpu); -+ if (is_idle_task(rq->curr)) -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ } -+ } else -+ cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask); -+} -+ -+static void set_partial_status(bool enable, bool little, bool big) -+{ -+ WRITE_ONCE(partial_enable, enable); -+ WRITE_ONCE(l_need_rescue, little); -+ WRITE_ONCE(b_need_rescue, big); -+} -+ -+static bool is_little_need_rescue(void) -+{ -+ return READ_ONCE(l_need_rescue); -+} -+ -+static bool is_big_need_rescue(void) -+{ -+ return READ_ONCE(b_need_rescue); -+} -+ -+static bool is_partial_enabled(void) -+{ -+ return READ_ONCE(partial_enable); -+} -+ -+static bool is_partial_cpu(int cpu) -+{ -+ return cpumask_test_cpu(cpu, iso_masks.partial); -+} -+ -+static void set_iso_par_free(bool enable) -+{ -+ WRITE_ONCE(isolate_ctrl, enable); -+} -+ -+static bool is_iso_par_free(void) -+{ -+ return READ_ONCE(isolate_ctrl); -+} -+ -+static bool skip_update_idle(void) -+{ -+ int cpu = smp_processor_id(); -+ enum cpu_type type = cpu_cluster(cpu); -+ -+ if ((type == EXCLUSIVE && !is_iso_par_free()) || -+ /* partial enable may changed during idle, it doesn't matter. */ -+ (!is_partial_enabled() && type == PARTIAL)) -+ return true; -+ -+ return false; -+} -+ -+static void init_isolate_cpus(void) -+{ -+ WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -+ cpumask_set_cpu(0, iso_masks.big); -+ cpumask_set_cpu(1, iso_masks.big); -+ cpumask_set_cpu(2, iso_masks.big); -+ cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(4, iso_masks.big); -+ cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(6, iso_masks.partial); -+ cpumask_set_cpu(7, iso_masks.ex_free); -+} -+ -+extern spinlock_t css_set_lock; -+ -+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp) -+{ -+ int i; -+ struct cgroup *anc; -+ -+ spin_lock_irq(&css_set_lock); -+ for (i = 0; i < cgrp->level; i++) { -+ anc = cgrp->ancestors[i]; -+ if (anc->level != CREATE_DSQ_LEVEL_WITHIN) -+ continue; -+ cgroup_get(anc); -+ spin_unlock_irq(&css_set_lock); -+ return anc; -+ } -+ spin_unlock_irq(&css_set_lock); -+ hmbird_err(NO_CGROUP_L1, ":error cgroup = %s\n", cgrp->kn->name); -+ return NULL; -+} -+ -+#define PCP_IDX_BIT (1 << 31) -+ -+static bool is_pcp_rt(struct task_struct *p) -+{ -+ return rt_prio(p->prio) && (p->nr_cpus_allowed == 1); -+} -+ -+static bool is_pcp_idx(int idx) -+{ -+ return idx >= MAX_GLOBAL_DSQS; -+} -+ -+static bool is_critical_system_task(struct task_struct *p) -+{ -+ int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK; -+ -+ return (p->prio < (MAX_RT_PRIO >> 1) && -+ (sp_dl < SCHED_PROP_DEADLINE_LEVEL3)); -+} -+ -+#define ISOLATE_TASK_TOP_BIT (1 << 17) -+#define PIPELINE_TASK_TOP_BIT (1 << 9) -+static bool is_pipeline_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT); -+} -+ -+static bool is_isolate_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT); -+} -+ -+static bool is_critical_app_task_without_isolate(struct task_struct *p) -+{ -+ return task_is_top_task(p) && !is_isolate_task(p); -+} -+ -+static int find_idx_from_task(struct task_struct *p) -+{ -+ int idx, cpu; -+ int sp_dl; -+ struct task_group *tg = p->sched_task_group; -+ -+ if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) { -+ cpu = cpumask_any(p->cpus_ptr); -+ idx = cpu + MAX_GLOBAL_DSQS; -+ return idx; -+ } -+ -+ if (is_critical_system_task(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL0; -+ goto done; -+ } -+ -+ if (is_critical_app_task_without_isolate(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL1; -+ goto done; -+ } -+ -+ sp_dl = hmbird_get_dsq_id(p); -+ if (sp_dl) { -+ idx = sp_dl; -+ goto done; -+ } -+ -+ if (rt_prio(p->prio)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL3; -+ goto done; -+ } -+ -+ if (tg && tg->css.cgroup && tg->css.cgroup->kn) { -+ if (likely(tg->css.cgroup->kn->id >= 0 && -+ tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS)) -+ idx = cgroup_ids_table[tg->css.cgroup->kn->id]; -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ -+done: -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) { -+ hmbird_err(DSQ_ID_ERR, " : idx error, idx = %d-----\n", idx); -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } -+ return idx; -+} -+ -+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p) -+{ -+ int idx; -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq; -+ -+ if (!p) -+ return NULL; -+ -+ idx = find_idx_from_task(p); -+ if (is_pcp_idx(idx)) { -+ idx -= MAX_GLOBAL_DSQS; -+ dsq = &per_cpu(pcp_ldsq, idx); -+ get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq); -+ slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx); -+ } else { -+ dsq = &gdsqs[idx]; -+ get_hmbird_ts(p)->gdsq_idx = idx; -+ slim_stats_record(GDSQ_CNT, 0, idx, 0); -+ } -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ dsq->last_consume_at = jiffies; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ return dsq; -+} -+ -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq); -+ -+static void set_partial_rescue(bool p_state, bool l_over, bool b_over) -+{ -+ set_partial_status(p_state, l_over, b_over); -+ update_cpus_idle(p_state, iso_masks.partial); -+ hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled()); -+ hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue()); -+ hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue()); -+} -+ -+static void free_isocpu(bool enable) -+{ -+ set_iso_par_free(enable); -+ update_cpus_idle(enable, iso_masks.ex_free); -+ hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free()); -+} -+ -+inline u64 get_hmbird_cpu_util(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed) -+ return 0; -+ u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed); -+ u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size); -+ -+ do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT); -+ -+ return prev_runnable_sum_fixed; -+} -+ -+static inline unsigned int get_scaling_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->max; -+} -+ -+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static u64 get_cpus_max_util(struct cpumask *mask) -+{ -+ int cpu; -+ u64 max = 0; -+ u64 util, ratio; -+ unsigned long effective_cap = 0; -+ for_each_cpu(cpu, mask) { -+ if (slim_walt_ctrl) -+ slim_get_cpu_util(cpu, &util); -+ else -+ util = get_hmbird_cpu_util(cpu); -+ -+ /* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/ -+ if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu))) -+ effective_cap = arch_scale_cpu_capacity(cpu); -+ else -+ effective_cap = arch_scale_cpu_capacity(cpu) * -+ get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu); -+ -+ ratio = util * 100 / effective_cap; -+ hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util); -+ hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n", -+ cpu, (u64)arch_scale_cpu_capacity(cpu)); -+ hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n", -+ cpu, (u64)effective_cap); -+ -+ if (ratio > 100) -+ ratio = 100; -+ -+ if (ratio > max) -+ max = ratio; -+ } -+ return max; -+} -+ -+static bool cluster_need_rescue(struct cpumask *mask, int hres) -+{ -+ return get_cpus_max_util(mask) > hres; -+} -+ -+void partial_dynamic_ctrl(void) -+{ -+ u64 lmax = 0, bmax = 0; -+ bool l_over, l_under, b_over, b_under; -+ static bool last_l_over, last_b_over; -+ static unsigned long last_check; -+ -+ /* Check partial every jiffies. need lock? */ -+ if (time_before_eq(jiffies, READ_ONCE(last_check))) -+ return; -+ -+ WRITE_ONCE(last_check, jiffies); -+ -+ lmax = get_cpus_max_util(iso_masks.little); -+ l_over = lmax > parctrl_high_ratio_l; -+ l_under = lmax < parctrl_low_ratio_l; -+ -+ bmax = get_cpus_max_util(iso_masks.big); -+ b_over = bmax > parctrl_high_ratio; -+ b_under = bmax < parctrl_low_ratio; -+ -+ if (is_partial_enabled() && (l_over || b_over)) { -+ if (last_l_over != l_over || last_b_over != b_over) -+ set_partial_rescue(true, l_over, b_over); -+ } else if (!is_partial_enabled() && (l_over || b_over)) { -+ set_partial_rescue(true, l_over, b_over); -+ } else if (is_partial_enabled() && l_under && b_under) { -+ set_partial_rescue(false, false, false); -+ } -+ last_l_over = l_over; -+ last_b_over = b_over; -+ -+ if (is_partial_enabled()) { -+ bmax = get_cpus_max_util(iso_masks.partial); -+ if (!is_iso_par_free() && bmax > isoctrl_high_ratio) { -+ hmbird_info_trace("partial max = %llu\n", bmax); -+ free_isocpu(true); -+ } else if (is_iso_par_free() && bmax < isoctrl_low_ratio) -+ free_isocpu(false); -+ } else if (is_iso_par_free()) { -+ free_isocpu(false); -+ } -+} -+ -+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx) -+{ -+ hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx); -+} -+ -+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx) -+{ -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[idx])) { -+ slim_stats_record(GDSQ_CNT, 1, idx, 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ int i; -+ -+ for (i = 0; i < UX_COMPATIBLE_IDX; i++) { -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ slim_stats_record(GDSQ_CNT, 1, i, 0); -+ return 1; -+ } -+ } -+ return 0; -+} -+ -+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) { -+ slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ goto clear_timeout; -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) -+ goto clear_timeout; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ hmbird_info_trace( -+ "dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n", -+ dsq_id_to_internal(dsq), entity->task->comm, -+ jiffies, msecs_to_jiffies(deadline), entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1); -+ return; -+ -+clear_timeout: -+ hmbird_info_trace("dsq[%d] clear timeout\n", -+ dsq_id_to_internal(dsq)); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0); -+ dsq->is_timeout = false; -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq)); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime) -+{ -+ hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime); -+} -+ -+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any) -+{ -+ bool is_timeout; -+ int cpu = cpu_of(rq); -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu); -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ is_timeout = dsq->is_timeout; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ /* -+ * dsq->is_timeout may change here, let it be. -+ * it won't cause serious problems. -+ * the same for consume_dispatch_q later. -+ */ -+ if (!is_timeout && !any) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, dsq)) { -+ if (is_timeout) { -+ hmbird_info_trace("dsq[%d] consume a pcp timeout task\n", -+ dsq_id_to_internal(dsq)); -+ update_timeout_stats(rq, dsq, pcp_dsq_deadline); -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu); -+ } -+ slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu); -+ return 1; -+ } -+ /* -+ * No pcp task, clear quota. -+ */ -+ if (any) { -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+ } -+ return 0; -+} -+ -+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf) -+{ -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ } -+ return 0; -+} -+ -+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf, -+ int tmp, int cidx, int tidx) -+{ -+ unsigned long flags; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) { -+ if (tmp != cidx) { -+ spin_lock(&sinfo.lock); -+ sinfo.curr_idx[tidx] = tmp; -+ spin_unlock(&sinfo.lock); -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]); -+ slim_stats_record(SWITCH_IDX, 0, 0, 0); -+ } -+ slim_stats_record(GDSQ_CNT, 1, tmp, 0); -+ -+ raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags); -+ gdsqs[tmp].last_consume_at = jiffies; -+ raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags); -+ return 1; -+ } -+ return 0; -+ -+} -+ -+static int get_cidx(struct cluster_ctx *ctx) -+{ -+ int cidx; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ if (cidx < ctx->lower || cidx >= ctx->upper) { -+ sinfo.curr_idx[ctx->tidx] = ctx->lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]); -+ slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ } -+ spin_unlock(&sinfo.lock); -+ -+ return cidx; -+} -+ -+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = CLUSTER_SEPARATE_IDX; -+ if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) { -+ pr_debug(" %s iso_masks.little first is %d\n", -+ __func__, cpumask_first(iso_masks.little)); -+ ctx->upper = NON_PERIOD_END; -+ } -+ ctx->tidx = 0; -+ break; -+ case LITTLE: -+ ctx->lower = CLUSTER_SEPARATE_IDX; -+ ctx->upper = NON_PERIOD_END; -+ if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) { -+ pr_debug(" %s iso_masks.big first is %d", -+ __func__, cpumask_first(iso_masks.big)); -+ ctx->lower = NON_PERIOD_START; -+ } -+ ctx->tidx = 1; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ case LITTLE: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = NON_PERIOD_END; -+ ctx->tidx = 0; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ if (cluster_separate) { -+ return gen_cluster_ctx_separate(ctx, type); -+ } else { -+ return gen_cluster_ctx_common(ctx, type); -+ } -+} -+ -+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ int i; -+ bool is_timeout; -+ unsigned long flags; -+ struct cluster_ctx ctx; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ /* Third param means only consume timeout pcp dsq. */ -+ if (consume_pcp_dsq(rq, rf, false)) -+ return 1; -+ -+ for (i = ctx.lower; i < ctx.upper; i++) { -+ raw_spin_lock_irqsave(&gdsqs[i].lock, flags); -+ is_timeout = gdsqs[i].is_timeout; -+ raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags); -+ /* gdsqs[i].is_timeout may change here, let it be... */ -+ if (is_timeout) { -+ /* -+ * consume_dispatch_q will acquire dsq-lock, -+ * So cannot keep lock here, annoy enough. -+ * may rewrite a consume_dispatch_q_locked, TODO. -+ */ -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ hmbird_info_trace("dsq[%d] consume a timeout task\n", i); -+ slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0); -+ update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]); -+ return 1; -+ } -+ } -+ } -+ return 0; -+} -+ -+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ struct cluster_ctx ctx; -+ int cidx; -+ int tmp; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ cidx = get_cidx(&ctx); -+ tmp = cidx; -+ do { -+ if (check_pcp_dsq_round(rq, rf)) -+ return 1; -+ if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx)) -+ return 1; -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[tmp] = 0; -+ systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]); -+ tmp++; -+ if (tmp >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ tmp = ctx.lower; -+ } -+ spin_unlock(&sinfo.lock); -+ } while (tmp != cidx); -+ -+ return consume_pcp_dsq(rq, rf, true); -+} -+ -+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ enum cpu_type type = cpu_cluster(cpu_of(rq)); -+ bool period_allowed = !get_hmbird_rq(rq)->period_disallow; -+ bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow; -+ -+ switch (type) { -+ case EX_FREE: -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ break; -+ case EXCLUSIVE: -+ if (!is_iso_par_free()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ } -+ /* Only non-period task can run on isolate cpus */ -+ if (!READ_ONCE(iso_free_rescue)) { -+ if (is_big_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ } else { -+ /* Free run, can run on any task. */ -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ case PARTIAL: -+ if (!is_partial_enabled()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ return 0; -+ } -+ if (is_big_need_rescue()) { -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ -+ case BIG: -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ if (is_iso_par_free() || (is_little_need_rescue() && -+ !cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) { -+ hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++); -+ return 1; -+ } -+ } -+ break; -+ -+ case LITTLE: -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ -+ if (is_iso_par_free() || (is_big_need_rescue() && -+ !cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) { -+ hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++); -+ return 1; -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return 0; -+} -+ -+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf) -+{ -+ return consume_hmbird_global_dsq(rq, rf); -+} -+ -+ -+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time) -+{ -+ int idx; -+ -+ /* which dsq belongs to while task enqueue, task will consume its running time. */ -+ idx = get_hmbird_ts(p)->gdsq_idx; -+ /* Only non-period dsq share running time between each other. */ -+ if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id) -+ return; -+ -+ if (idx >= MAX_GLOBAL_DSQS) { -+ per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } else { -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[idx] += exec_time; -+ spin_unlock(&sinfo.lock); -+ systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]); -+ } -+} -+ -+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type) -+{ -+ int cidx; -+ struct cluster_ctx ctx; -+ int cpu = cpu_of(rq); -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ if (cidx < ctx.lower || cidx >= ctx.upper) { -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ slim_stats_record(ERR_IDX, ctx.tidx, 0, 0); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ } -+ -+ while (1) { -+ if (per_cpu(pcp_info, cpu).pcp_round) { -+ if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) { -+ hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n", -+ cpu, per_cpu(pcp_info, cpu).rtime); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } -+ } -+ if (sinfo.rtime[cidx] < dsq_quota[cidx]) -+ break; -+ -+ /* clear current dsq rtime */ -+ sinfo.rtime[cidx] = 0; -+ systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]); -+ -+ sinfo.curr_idx[ctx.tidx]++; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", -+ ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ } -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ slim_stats_record(SWITCH_IDX, 1, 0, 0); -+ } -+ spin_unlock(&sinfo.lock); -+} -+ -+ -+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p) -+{ -+ enum cpu_type type; -+ -+ if (!rq || !p) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ switch (type) { -+ case PARTIAL: -+ return; -+ case EXCLUSIVE: -+ return; -+ case EX_FREE: -+ return; -+ default: -+ break; -+ } -+ update_dsq_idx(rq, p, type); -+} -+ -+ -+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ int dsq_id; -+ -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo) || dsq->is_timeout) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ hmbird_deferred_err(SCAN_ENTITY_NULL, -+ " : entity is NULL, dsq->id = %llu\n", dsq->id); -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ dsq->is_timeout = true; -+ dsq_id = dsq_id_to_internal(dsq); -+ hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n", -+ dsq_id, entity->task->comm, jiffies, entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1); -+ raw_spin_unlock(&dsq->lock); -+ -+ return true; -+} -+ -+void scan_timeout(struct rq *rq) -+{ -+ int i; -+ int cpu = cpu_of(rq); -+ struct hmbird_dispatch_q *dsq; -+ static u64 last_scan_at; -+ static DEFINE_PER_CPU(u64, pcp_last_scan_at); -+ -+ if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu))) -+ return; -+ per_cpu(pcp_last_scan_at, cpu) = jiffies; -+ -+ dsq = &per_cpu(pcp_ldsq, cpu); -+ scan_dsq_timeout(rq, dsq, pcp_dsq_deadline); -+ -+ if (time_before_eq(jiffies, (unsigned long)last_scan_at)) -+ return; -+ last_scan_at = jiffies; -+ -+ for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) { -+ dsq = &gdsqs[i]; -+ scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]); -+ } -+} -+ -+/*******************************Initialize***********************************/ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id); -+static void init_dsq_at_boot(void) -+{ -+ int i, cpu; -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i)); -+ } -+ for_each_possible_cpu(cpu) -+ init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i + cpu)); -+ -+ max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu; -+ spin_lock_init(&sinfo.lock); -+} -+ -+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgrp_name_to_idx(struct cgroup *cgrp) -+{ -+ int idx; -+ -+ if (!cgrp) -+ return -1; -+ -+ if (!strcmp(cgrp->kn->name, "display") -+ || !strcmp(cgrp->kn->name, "multimedia")) -+ idx = 5; /* 8ms */ -+ else if (!strcmp(cgrp->kn->name, "top-app") -+ || !strcmp(cgrp->kn->name, "ss-top")) -+ idx = 6; /* 16ms */ -+ else if (!strcmp(cgrp->kn->name, "ssfg") -+ || !strcmp(cgrp->kn->name, "foreground")) -+ idx = 7; /* 32ms */ -+ else if (!strcmp(cgrp->kn->name, "bg") -+ || !strcmp(cgrp->kn->name, "log") -+ || !strcmp(cgrp->kn->name, "dex2oat") -+ || !strcmp(cgrp->kn->name, "background")) -+ idx = 9; /* 128ms */ -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */ -+ -+ return idx; -+} -+ -+static void init_root_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX); -+} -+ -+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) { -+ pr_err("%s idx err!\n", __func__); -+ return; -+ } -+ -+ if (cgroup_ids_table[cgrp->kn->id] == -1) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp)); -+} -+ -+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ struct cgroup *l1cgrp; -+ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ l1cgrp = cgroup_ancestor_l1(cgrp); -+ if (l1cgrp) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp)); -+ cgroup_put(l1cgrp); -+} -+ -+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg) -+{ -+ switch (cgrp->level) { -+ case 0: -+ init_root_tg(cgrp, tg); -+ break; -+ case 1: -+ init_level1_tg(cgrp, tg); -+ break; -+ default: -+ init_child_tg(cgrp, tg); -+ break; -+ } -+} -+ -+/**************************************************************************/ -+ -+struct hmbird_task_iter { -+ struct hmbird_entity cursor; -+ struct task_struct *locked; -+ struct rq *rq; -+ struct rq_flags rf; -+}; -+ -+/** -+ * hmbird_task_iter_init - Initialize a task iterator -+ * @iter: iterator to init -+ * -+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized, -+ * @iter must eventually be exited with hmbird_task_iter_exit(). -+ * -+ * hmbird_tasks_lock may be released between this and the first next() call or -+ * between any two next() calls. If hmbird_tasks_lock is released between two -+ * next() calls, the caller is responsible for ensuring that the task being -+ * iterated remains accessible either through RCU read lock or obtaining a -+ * reference count. -+ * -+ * All tasks which existed when the iteration started are guaranteed to be -+ * visited as long as they still exist. -+ */ -+static void hmbird_task_iter_init(struct hmbird_task_iter *iter) -+{ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR }; -+ list_add(&iter->cursor.tasks_node, &hmbird_tasks); -+ iter->locked = NULL; -+} -+ -+/** -+ * hmbird_task_iter_exit - Exit a task iterator -+ * @iter: iterator to exit -+ * -+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held. -+ * If the iterator holds a task's rq lock, that rq lock is released. See -+ * hmbird_task_iter_init() for details. -+ */ -+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ if (list_empty(cursor)) -+ return; -+ -+ list_del_init(cursor); -+} -+ -+/** -+ * hmbird_task_iter_next - Next task -+ * @iter: iterator to walk -+ * -+ * Visit the next task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ struct hmbird_entity *pos; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ list_for_each_entry(pos, cursor, tasks_node) { -+ if (&pos->tasks_node == &hmbird_tasks) -+ return NULL; -+ if (!(pos->flags & HMBIRD_TASK_CURSOR)) { -+ list_move(cursor, &pos->tasks_node); -+ return pos->task; -+ } -+ } -+ -+ /* can't happen, should always terminate at hmbird_tasks above */ -+ hmbird_deferred_err(ITER_RET_NULL, " : unreachable path in scx_task_iter_next\n"); -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered - Next non-idle task -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ while ((p = hmbird_task_iter_next(iter))) { -+ if (!is_idle_task(p)) -+ return p; -+ } -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init() -+ * for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ p = hmbird_task_iter_next_filtered(iter); -+ if (!p) -+ return NULL; -+ -+ iter->rq = task_rq_lock(p, &iter->rf); -+ iter->locked = p; -+ return p; -+} -+ -+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void) -+{ -+ return atomic_read(&hmbird_ops_enable_state_var); -+} -+ -+static enum hmbird_ops_enable_state -+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to) -+{ -+ return atomic_xchg(&hmbird_ops_enable_state_var, to); -+} -+ -+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to, -+ enum hmbird_ops_enable_state from) -+{ -+ int from_v = from; -+ -+ return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to); -+} -+ -+static bool hmbird_ops_disabling(void) -+{ -+ return false; -+} -+ -+/** -+ * wait_ops_state - Busy-wait the specified ops state to end -+ * @p: target task -+ * @opss: state to wait the end of -+ * -+ * Busy-wait for @p to transition out of @opss. This can only be used when the -+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also -+ * has load_acquire semantics to ensure that the caller can see the updates made -+ * in the enqueueing and dispatching paths. -+ */ -+static void wait_ops_state(struct task_struct *p, u64 opss) -+{ -+ do { -+ cpu_relax(); -+ } while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss); -+} -+ -+ -+static void update_curr_hmbird(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ u64 now = rq_clock_task(rq); -+ u64 delta_exec; -+ -+ if (time_before_eq64(now, curr->se.exec_start)) -+ return; -+ -+ delta_exec = now - curr->se.exec_start; -+ curr->se.exec_start = now; -+ update_runningtime(rq, curr, delta_exec); -+ curr->se.sum_exec_runtime += delta_exec; -+ account_group_exec_runtime(curr, delta_exec); -+ cgroup_account_cputime(curr, delta_exec); -+ -+ if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF) -+ get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec); -+ -+ trace_sched_stat_runtime(curr, delta_exec, 0); -+} -+ -+static bool hmbird_dsq_priq_less(struct rb_node *node_a, -+ const struct rb_node *node_b) -+{ -+ const struct hmbird_entity *a = -+ container_of(node_a, struct hmbird_entity, dsq_node.priq); -+ const struct hmbird_entity *b = -+ container_of(node_b, struct hmbird_entity, dsq_node.priq); -+ -+ return time_before64(a->dsq_vtime, b->dsq_vtime); -+} -+ -+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ bool is_local = dsq->id == HMBIRD_DSQ_LOCAL; -+ unsigned long flags; -+ -+ hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq || -+ !list_empty(&get_hmbird_ts(p)->dsq_node.fifo), -+ "task = %s, dsq->id = %llu\n", p->comm, dsq->id); -+ hmbird_cond_deferred_err(ENQ_EXIST2, -+ (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq), -+ "task = %s\n", p->comm); -+ -+ if (!is_local) { -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ); -+ hmbird_ops_error(": %s\n", -+ "attempting to dispatch to a destroyed dsq"); -+ /* fall back to the global dsq */ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ dsq = &hmbird_dsq_global; -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ } -+ } -+ -+ if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) { -+ get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ; -+ rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq, -+ hmbird_dsq_priq_less); -+ } else { -+ if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT)) -+ list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ else -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ } -+ dsq->nr++; -+ get_hmbird_ts(p)->dsq = dsq; -+ -+ /* -+ * We're transitioning out of QUEUEING or DISPATCHING. store_release to -+ * match waiters' load_acquire. -+ */ -+ if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS) -+ atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE); -+ -+ if (is_local) { -+ struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq); -+ struct rq *rq = hmbird->rq; -+ bool preempt = false; -+ -+ if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr && -+ rq->curr->sched_class == &hmbird_sched_class) { -+ get_hmbird_ts(rq->curr)->slice = 0; -+ preempt = true; -+ } -+ -+ if (preempt || sched_class_above(&hmbird_sched_class, -+ rq->curr->sched_class)) -+ resched_curr(rq); -+ } else { -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ } -+} -+ -+static void task_unlink_from_dsq(struct task_struct *p, -+ struct hmbird_dispatch_q *dsq) -+{ -+ if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) { -+ rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ; -+ } else { -+ list_del_init(&get_hmbird_ts(p)->dsq_node.fifo); -+ } -+} -+ -+static bool task_linked_on_dsq(struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+} -+ -+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq; -+ bool is_local = dsq == &hmbird_rq->local_dsq; -+ -+ if (!dsq) { -+ hmbird_cond_deferred_err(TASK_LINKED1, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ /* -+ * When dispatching directly from the BPF scheduler to a local -+ * DSQ, the task isn't associated with any DSQ but -+ * @get_hmbird_ts(p)->holding_cpu may be set under the protection of -+ * %HMBIRD_OPSS_DISPATCHING. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu >= 0) -+ get_hmbird_ts(p)->holding_cpu = -1; -+ return; -+ } -+ -+ if (!is_local) -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ /* -+ * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node -+ * can't change underneath us. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu < 0) { -+ /* @p must still be on @dsq, dequeue */ -+ hmbird_cond_deferred_err(TASK_UNLINKED, -+ !task_linked_on_dsq(p), "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ } else { -+ /* -+ * We're racing against dispatch_to_local_dsq() which already -+ * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the -+ * holding_cpu which tells dispatch_to_local_dsq() that it lost -+ * the race. -+ */ -+ hmbird_cond_deferred_err(TASK_LINKED2, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ get_hmbird_ts(p)->holding_cpu = -1; -+ } -+ get_hmbird_ts(p)->dsq = NULL; -+ -+ if (!is_local) -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+ -+static bool test_rq_online(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ return rq->online; -+#else -+ return true; -+#endif -+} -+ -+static void refill_task_slice(struct task_struct *p) -+{ -+ if (is_isolate_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO; -+ else if (is_pipeline_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2; -+ else -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+} -+ -+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -+ int sticky_cpu) -+{ -+ struct hmbird_dispatch_q *d; -+ s32 cpu = -1; -+ -+ hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), -+ "task = %s\n", p->comm); -+ -+ if (is_pcp_rt(p)) { -+ /* Enqueue percpu rt task to local directly. */ -+ /* Or cause a bug when disable dispatch. */ -+ if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ } -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (cpu >= 0) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ -+ if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ /* rq migration */ -+ if (sticky_cpu == cpu_of(rq)) -+ goto local_norefill; -+ /* -+ * If !rq->online, we already told the BPF scheduler that the CPU is -+ * offline. We're just trying to on/offline the CPU. Don't bother the -+ * BPF scheduler. -+ */ -+ if (unlikely(!test_rq_online(rq))) -+ goto local; -+ -+ /* see %HMBIRD_OPS_ENQ_LAST */ -+ if (enq_flags & HMBIRD_ENQ_LAST) -+ goto local; -+ -+ if (enq_flags & HMBIRD_ENQ_LOCAL) -+ goto local; -+ else -+ goto global; -+local: -+ /* -+ * For task-ordering, slice refill must be treated as implying the end -+ * of the current slice. Otherwise, the longer @p stays on the CPU, the -+ * higher priority it becomes from hmbird_prio_less()'s POV. -+ */ -+ refill_task_slice(p); -+local_norefill: -+ dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags); -+ slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq)); -+ return; -+ -+global: -+ d = find_dsq_from_task(p); -+ if (d) { -+ refill_task_slice(p); -+ dispatch_enqueue(d, p, enq_flags); -+ return; -+ } -+ slim_stats_record(GLOBAL_STAT, 0, 0, 0); -+ refill_task_slice(p); -+ dispatch_enqueue(&hmbird_dsq_global, p, enq_flags); -+} -+ -+static bool watchdog_task_watched(const struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->watchdog_node); -+} -+ -+static void watchdog_watch_task(struct rq *rq, struct task_struct *p) -+{ -+ lockdep_assert_rq_held(rq); -+ if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ get_hmbird_ts(p)->runnable_at = jiffies; -+ clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list); -+} -+ -+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) -+{ -+ list_del_init(&get_hmbird_ts(p)->watchdog_node); -+ if (reset_timeout) -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags) -+{ -+ int sticky_cpu = get_hmbird_ts(p)->sticky_cpu; -+ -+ enq_flags |= get_hmbird_rq(rq)->extra_enq_flags; -+ -+ if (sticky_cpu >= 0) -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ -+ /* -+ * Restoring a running task will be immediately followed by -+ * set_next_task_hmbird() which expects the task to not be on the BPF -+ * scheduler as tasks can only start running through local DSQs. Force -+ * direct-dispatch into the local DSQ by setting the sticky_cpu. -+ */ -+ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -+ sticky_cpu = cpu_of(rq); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_UNWATCHED, -+ !watchdog_task_watched(p), "task = %s\n", p->comm); -+ return; -+ } -+ -+ watchdog_watch_task(rq, p); -+ set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ get_hmbird_rq(rq)->nr_running++; -+ add_nr_running(rq, 1); -+ -+ do_enqueue_task(rq, p, enq_flags, sticky_cpu); -+} -+ -+static void ops_dequeue(struct task_struct *p, u64 deq_flags) -+{ -+ u64 opss; -+ -+ watchdog_unwatch_task(p, false); -+ -+ /* acquire ensures that we see the preceding updates on QUEUED */ -+ opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state); -+ -+ switch (opss & HMBIRD_OPSS_STATE_MASK) { -+ case HMBIRD_OPSS_NONE: -+ break; -+ case HMBIRD_OPSS_QUEUEING: -+ /* -+ * QUEUEING is started and finished while holding @p's rq lock. -+ * As we're holding the rq lock now, we shouldn't see QUEUEING. -+ */ -+ hmbird_deferred_err(DEQ_DEQING, " : unreachable path in %s\n", __func__); -+ break; -+ case HMBIRD_OPSS_QUEUED: -+ if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss, -+ HMBIRD_OPSS_NONE)) -+ break; -+ fallthrough; -+ case HMBIRD_OPSS_DISPATCHING: -+ /* -+ * If @p is being dispatched from the BPF scheduler to a DSQ, -+ * wait for the transfer to complete so that @p doesn't get -+ * added to its DSQ after dequeueing is complete. -+ * -+ * As we're waiting on DISPATCHING with the rq locked, the -+ * dispatching side shouldn't try to lock the rq while -+ * DISPATCHING is set. See dispatch_to_local_dsq(). -+ * -+ * DISPATCHING shouldn't have qseq set and control can reach -+ * here with NONE @opss from the above QUEUED case block. -+ * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss. -+ */ -+ wait_ops_state(p, HMBIRD_OPSS_DISPATCHING); -+ hmbird_cond_deferred_err(HMBIRD_OPN, -+ atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE, -+ "task = %s\n", p->comm); -+ break; -+ } -+} -+ -+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ -+ if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p), -+ "task = %s\n", p->comm); -+ return; -+ } -+ -+ ops_dequeue(p, deq_flags); -+ -+ if (slim_walt_ctrl) { -+ if (task_current(rq, p)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ if (deq_flags & HMBIRD_DEQ_SLEEP) -+ set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else -+ clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), -+ (unsigned long *)&get_hmbird_ts(p)->flags); -+ -+ clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm); -+ hmbird_rq->nr_running--; -+ sub_nr_running(rq, 1); -+ dispatch_dequeue(hmbird_rq, p); -+} -+ -+static void yield_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ get_hmbird_ts(p)->slice = 0; -+} -+ -+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to) -+{ -+ return false; -+} -+ -+#ifdef CONFIG_SMP -+/** -+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -+ * @rq: rq to move the task into, currently locked -+ * @p: task to move -+ * @enq_flags: %HMBIRD_ENQ_* -+ * -+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller -+ * must: -+ * -+ * 1. Start with exclusive access to @p either through its DSQ lock or -+ * %HMBIRD_OPSS_DISPATCHING flag. -+ * -+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id(). -+ * -+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't -+ * deadlock with dequeue. -+ * -+ * 4. Lock @rq and the task_rq from #3. -+ * -+ * 5. Call this function. -+ * -+ * Returns %true if @p was successfully moved. %false after racing dequeue and -+ * losing. -+ */ -+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ struct rq *task_rq; -+ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * If dequeue got to @p while we were trying to lock both rq's, it'd -+ * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have -+ * updated it to different values afterwards, as this operation can't be -+ * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become -+ * raw_smp_processor_id() again before we're done. Thus, we can tell -+ * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is -+ * still raw_smp_processor_id(). -+ * -+ * See dispatch_dequeue() for the counterpart. -+ */ -+ if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id())) -+ return false; -+ -+ /* @p->rq couldn't have changed if we're still the holding cpu */ -+ task_rq = task_rq(p); -+ lockdep_assert_rq_held(task_rq); -+ deactivate_task(task_rq, p, 0); -+ set_task_cpu(p, cpu_of(rq)); -+ get_hmbird_ts(p)->sticky_cpu = cpu_of(rq); -+ -+ /* -+ * We want to pass hmbird-specific enq_flags but activate_task() will -+ * truncate the upper 32 bit. As we own @rq, we can pass them through -+ * @get_hmbird_rq(rq)->extra_enq_flags instead. -+ */ -+ hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags, -+ "task = %s\n", p->comm); -+ get_hmbird_rq(rq)->extra_enq_flags = enq_flags; -+ activate_task(rq, p, 0); -+ get_hmbird_rq(rq)->extra_enq_flags = 0; -+ -+ return true; -+} -+ -+#endif /* CONFIG_SMP */ -+ -+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu) -+{ -+ int fitable = 1; -+ -+ return fitable; -+} -+ -+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq, -+ struct hmbird_dispatch_q *dsq) -+{ -+ bool dsq_misfit; -+ int cpu = cpu_of(rq); -+ u64 task_util = 0; -+ struct cluster_ctx ctx; -+ int dsq_int = dsq_id_to_internal(dsq); -+ -+ if (!cpumask_test_cpu(cpu, iso_masks.little)) -+ return false; -+ -+ gen_cluster_ctx(&ctx, BIG); -+ dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 && -+ dsq_int <= SCHED_PROP_DEADLINE_LEVEL4); -+#ifdef CLUSTER_SEPARATE -+ /* In rescue mode, little will consume big cluster's dsq.*/ -+ dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper); -+#endif -+ if (!dsq_misfit) -+ return false; -+ -+ if (p) { -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &task_util); -+ else -+ task_util = get_hmbird_ts(p)->demand_scaled; -+ } -+ -+ if (task_util <= misfit_ds) -+ return false; -+ -+ hmbird_info_trace(":task %s can't run on cpu%d, util = %llu\n", -+ p->comm, cpu, task_util); -+ return true; -+} -+ -+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq) -+{ -+ if (!task_fits_cpu_hmbird(p, cpu_of(rq))) -+ return false; -+ -+ if (check_misfit_task_on_little(p, rq, dsq)) -+ return false; -+ -+ return likely(test_rq_online(rq)); -+} -+ -+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return; -+ -+ if (add) -+ skipn[idx]++; -+ else -+ skipn[idx] = 0; -+} -+ -+static bool skip_too_much(struct hmbird_dispatch_q *dsq) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return false; -+ -+ if (skip_num[idx] > 3) { -+ skip_num[idx] = 0; -+ return true; -+ } -+ -+ return false; -+} -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rb_node *rb_node; -+ struct rq *task_rq; -+ unsigned long flags; -+ bool moved = false; -+ struct task_struct *may_fit = NULL; -+ int skip = 0; -+ -+retry: -+ if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -+ return false; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) { -+ set_skip_num(dsq, skip_num, (bool)may_fit); -+ goto this_rq; -+ } -+ if (skip_too_much(dsq)) -+ goto remote_rq; -+ if (!may_fit) -+ may_fit = p; -+ if (++skip <= 3) -+ continue; -+ /* -+ * If the recent 3 tasks not fit, use the first one. -+ * and clear the skip, because the first one is consumed. -+ */ -+ set_skip_num(dsq, skip_num, false); -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ /* No more task, use the first may fit task.*/ -+ if (may_fit) { -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ -+ for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) -+ goto this_rq; -+ goto remote_rq; -+ } -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ return false; -+ -+this_rq: -+ /* @dsq is locked and @p is on this rq */ -+ hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo); -+ dsq->nr--; -+ hmbird_rq->local_dsq.nr++; -+ get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ -+remote_rq: -+#ifdef CONFIG_SMP -+ if (cpu_same_cluster_stat(p, rq, task_rq)) -+ slim_stats_record(MOVE_RQ_CNT, 0, 0, 0); -+ else -+ slim_stats_record(MOVE_RQ_CNT, 1, 0, 0); -+ /* -+ * @dsq is locked and @p is on a remote rq. @p is currently protected by -+ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -+ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -+ * rq lock or fail, do a little dancing from our side. See -+ * move_task_to_local_dsq(). -+ */ -+ hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id(); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ rq_unpin_lock(rq, rf); -+ double_lock_balance(rq, task_rq); -+ rq_repin_lock(rq, rf); -+ -+ moved = move_task_to_local_dsq(rq, p, 0); -+ -+ double_unlock_balance(rq, task_rq); -+#endif /* CONFIG_SMP */ -+ if (likely(moved)) { -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ } -+ may_fit = NULL; -+ goto retry; -+} -+ -+ -+static int balance_one(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf, bool local) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class; -+ -+ if (!hmbird_rq) -+ return 1; -+ -+ lockdep_assert_rq_held(rq); -+ -+ if (static_branch_unlikely(&hmbird_ops_cpu_preempt) && -+ unlikely(get_hmbird_rq(rq)->cpu_released)) { -+ /* -+ * If the previous sched_class for the current CPU was not HMBIRD, -+ * notify the BPF scheduler that it again has control of the -+ * core. This callback complements ->cpu_release(), which is -+ * emitted in hmbird_notify_pick_next_task(). -+ */ -+ get_hmbird_rq(rq)->cpu_released = false; -+ } -+ -+ if (prev_on_hmbird) -+ update_curr_hmbird(rq); -+ /* if there already are tasks to run, nothing to do */ -+ if (hmbird_rq->local_dsq.nr) -+ return 1; -+ -+ if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) { -+ slim_stats_record(GLOBAL_STAT, 1, 0, 0); -+ return 1; -+ } -+ -+ if (consume_dispatch_global(rq, rf)) -+ return 1; -+ -+ return 0; -+} -+ -+static int balance_hmbird(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf) -+{ -+ return balance_one(rq, prev, rf, true); -+} -+ -+/* -+ * output task util to systrace, only for debug mode. -+ * we can not output too many logs to systrace buffer even in debug mode -+ * only output debug-info while it exceed misfit_ds. -+ */ -+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p) -+{ -+ static DEFINE_PER_CPU(int, is_last_exceed); -+ int cpu = cpu_of(rq); -+ u64 util = 0; -+ -+ if (likely(!debug_enabled())) -+ return; -+ -+ if (!p) -+ return; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ util = uclamp_rq_util_with(rq, util, p); -+ -+ if (util >= misfit_ds) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util); -+ per_cpu(is_last_exceed, cpu) = true; -+ } else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0); -+ per_cpu(is_last_exceed, cpu) = false; -+ } else { -+ } -+} -+ -+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first) -+{ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ /* -+ * Core-sched might decide to execute @p before it is -+ * dispatched. Call ops_dequeue() to notify the BPF scheduler. -+ */ -+ ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC); -+ dispatch_dequeue(get_hmbird_rq(rq), p); -+ } -+ -+ p->se.exec_start = rq_clock_task(rq); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK); -+ } -+ -+ watchdog_unwatch_task(p, true); -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx); -+ systrace_output_cpu_ds(rq, p); -+ /* -+ * @p is getting newly scheduled or got kicked after someone updated its -+ * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird(). -+ */ -+ if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) != -+ (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) { -+ if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) -+ get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK; -+ else -+ get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK; -+ -+ sched_update_tick_dependency(rq); -+ } -+ -+ p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime; -+} -+ -+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p) -+{ -+ update_curr_hmbird(rq); -+ -+ update_dispatch_dsq_info(rq, p); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ watchdog_watch_task(rq, p); -+ -+ if (is_pipeline_task(p)) { -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1); -+ return; -+ } -+ -+ /* -+ * If we're in the pick_next_task path, balance_hmbird() should -+ * have already populated the local DSQ if there are any other -+ * available tasks. If empty, tell ops.enqueue() that @p is the -+ * only one available for this cpu. ops.enqueue() should put it -+ * on the local DSQ so that the subsequent pick_next_task_hmbird() -+ * can find the task unless it wants to trigger a separate -+ * follow-up scheduling event. -+ */ -+ if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1); -+ else -+ do_enqueue_task(rq, p, 0, -1); -+ } -+} -+ -+static struct task_struct *first_local_task(struct rq *rq) -+{ -+ struct rb_node *rb_node; -+ struct hmbird_entity *entity; -+ -+ if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) { -+ entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo, -+ struct hmbird_entity, dsq_node.fifo); -+ return entity->task; -+ } -+ -+ rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq); -+ if (rb_node) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ return entity->task; -+ } -+ return NULL; -+} -+ -+static struct task_struct *pick_next_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p; -+ -+ p = first_local_task(rq); -+ if (!p) -+ return NULL; -+ -+ if (unlikely(!get_hmbird_ts(p)->slice)) { -+ if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice) -+ hmbird_warned_zero_slice = true; -+ -+ refill_task_slice(p); -+ } -+ -+ set_next_task_hmbird(rq, p, true); -+ -+ return p; -+} -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task, -+ const struct sched_class *active) -+{ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * The callback is conceptually meant to convey that the CPU is no -+ * longer under the control of HMBIRD. Therefore, don't invoke the -+ * callback if the CPU is staying on HMBIRD, or going idle (in which -+ * case the HMBIRD scheduler has actively decided not to schedule any -+ * tasks on the CPU). -+ */ -+ if (likely(active >= &hmbird_sched_class)) -+ return; -+ -+ /* -+ * At this point we know that HMBIRD was preempted by a higher priority -+ * sched_class, so invoke the ->cpu_release() callback if we have not -+ * done so already. We only send the callback once between HMBIRD being -+ * preempted, and it regaining control of the CPU. -+ * -+ * ->cpu_release() complements ->cpu_acquire(), which is emitted the -+ * next time that balance_hmbird() is invoked. -+ */ -+ if (!get_hmbird_rq(rq)->cpu_released) -+ get_hmbird_rq(rq)->cpu_released = true; -+} -+ -+#ifdef CONFIG_SMP -+ -+static bool test_and_clear_cpu_idle(int cpu) -+{ -+ if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -+ if (cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ return true; -+ } else { -+ return false; -+ } -+} -+ -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) -+{ -+ int cpu; -+ -+ do { -+ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -+ if (cpu >= nr_cpu_ids) -+ return -EBUSY; -+ } while (!test_and_clear_cpu_idle(cpu)); -+ -+ return cpu; -+} -+ -+ -+static bool prev_cpu_misfit(int prev) -+{ -+ if (!is_partial_enabled() && is_partial_cpu(prev)) -+ return true; -+ -+ return false; -+} -+ -+static int heavy_rt_placement(struct task_struct *p, int prev) -+{ -+ struct cpumask tmp = {.bits = {0}}; -+ int cpu; -+ u64 util = 0; -+ -+ if (!rt_prio(p->prio)) -+ return -EFAULT; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ -+ if (util < misfit_ds) -+ return -EFAULT; -+ -+ if (is_partial_enabled()) -+ cpumask_or(&tmp, iso_masks.big, iso_masks.partial); -+ else -+ cpumask_copy(&tmp, iso_masks.big); -+ -+ cpu = hmbird_pick_idle_cpu(&tmp); -+ if (cpu >= 0) -+ return cpu; -+ -+ if (cpumask_test_cpu(prev, iso_masks.big) || -+ (is_partial_enabled() && is_partial_cpu(prev))) -+ return prev; -+ -+ return cpumask_first(&tmp); -+} -+ -+static int spec_task_before_pick_idle(struct task_struct *p, int prev) -+{ -+ int cpu; -+ -+ cpu = heavy_rt_placement(p, prev); -+ if (cpu >= 0) -+ return cpu; -+ return -EFAULT; -+} -+ -+static int cpumask_distribute_next(struct cpumask *mask, int *prev) -+{ -+ int p = *prev, n; -+ -+ n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1); -+ if (n < nr_cpu_ids) -+ WRITE_ONCE(*prev, n); -+ return n; -+} -+ -+static int repick_fallback_cpu(void) -+{ -+ /* -+ * partial cpu follow big cluster's Scheduling policy, -+ * simply return first bit cpu. -+ */ -+ return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev); -+} -+ -+/* -+ * Must return a valid cpu num, as this task's cpu. -+ */ -+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu, -+ struct cpumask *mask, int *prev_mask) -+{ -+ int cpu; -+ -+ cpu = hmbird_pick_idle_cpu(mask); -+ if (cpu >= 0) -+ return cpu; -+ return cpumask_distribute_next(mask, prev_mask); -+} -+ -+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type) -+{ -+ int idx; -+ struct cluster_ctx ctx; -+ -+ idx = find_idx_from_task(p); -+ if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS) -+ return false; -+ -+ gen_cluster_ctx(&ctx, type); -+ if (idx >= ctx.lower && idx < ctx.upper) -+ return true; -+ return false; -+} -+ -+static bool is_valid_cpu(int cpu) -+{ -+ return (cpu >= 0) && (cpu < nr_cpu_ids); -+} -+ -+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) -+{ -+ s32 cpu = -1; -+ struct cpumask mask = {.bits = {0}}; -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (is_valid_cpu(cpu)) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ partial_dynamic_ctrl(); -+ -+ if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ iso_masks.big, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ if (p->nr_cpus_allowed == 1) { -+ cpu = cpumask_any(p->cpus_ptr); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* For non-period global dsq, not contain pcp task. */ -+ if (task_only_blongs_to_cluster(p, LITTLE)) { -+ cpumask_copy(&mask, iso_masks.little); -+ if (unlikely(l_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &little_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ if (task_only_blongs_to_cluster(p, BIG)) { -+ cpumask_copy(&mask, iso_masks.big); -+ if (unlikely(b_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ cpu = spec_task_before_pick_idle(p, prev_cpu); -+ if (is_valid_cpu(cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* if the previous CPU is idle, dispatch directly to it */ -+ if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+ } -+ -+ cpu = hmbird_pick_idle_cpu(cpu_possible_mask); -+ if (is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ if (prev_cpu_misfit(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ cpu = repick_fallback_cpu(); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+} -+ -+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags) -+{ -+ return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags); -+} -+ -+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx) -+{ -+ set_cpus_allowed_common(p, ctx); -+} -+ -+static void reset_idle_masks(void) -+{ -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little); -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big); -+ if (is_partial_enabled()) -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial); -+ hmbird_has_idle_cpus = true; -+} -+ -+void __hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ int cpu = cpu_of(rq); -+ struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -+ -+ if (skip_update_idle()) -+ return; -+ -+ if (idle) { -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ if (!hmbird_has_idle_cpus) -+ hmbird_has_idle_cpus = true; -+ -+ /* -+ * idle_masks.smt handling is racy but that's fine as it's only -+ * for optimization and self-correcting. -+ */ -+ for_each_cpu(cpu, sib_mask) { -+ if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -+ return; -+ } -+ cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -+ } else { -+ cpumask_clear_cpu(cpu, idle_masks.cpu); -+ if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ -+ cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -+ } -+} -+ -+#else /* !CONFIG_SMP */ -+ -+static bool test_and_clear_cpu_idle(int cpu) { return false; } -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } -+static void reset_idle_masks(void) {} -+ -+#endif /* CONFIG_SMP */ -+ -+static bool check_rq_for_timeouts(struct rq *rq) -+{ -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rq_flags rf; -+ -+ rq_lock_irqsave(rq, &rf); -+ list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) { -+ unsigned long last_runnable; -+ -+ p = entity->task; -+ last_runnable = get_hmbird_ts(p)->runnable_at; -+ -+ if (unlikely(time_after(jiffies, -+ last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -+ -+ rq_unlock_irqrestore(rq, &rf); -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb", -+ p->comm, p->pid, -+ dur_ms / 1000, dur_ms % 1000, -+ get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0, -+ cpumask_pr_args(&p->cpus_mask)); -+ return 1; -+ } -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ return 0; -+} -+ -+static void hmbird_watchdog_workfn(struct work_struct *work) -+{ -+ int cpu; -+ bool timeout; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ -+ for_each_online_cpu(cpu) { -+ timeout = check_rq_for_timeouts(cpu_rq(cpu)); -+ if (unlikely(timeout)) -+ break; -+ -+ cond_resched(); -+ } -+ if (!timeout) -+ queue_delayed_work(system_unbound_wq, to_delayed_work(work), -+ hmbird_watchdog_timeout / 2); -+} -+ -+static void set_pcp_round(struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) { -+ per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round); -+ per_cpu(pcp_info, cpu).pcp_round = true; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+} -+ -+ -+/* -+ * Just for debug: output hmbird on/off state per 10s. -+ */ -+#define OUTPUT_INTVAL (msecs_to_jiffies(10 * 1000)) -+static void inform_hmbird_onoff_from_systrace(void) -+{ -+ static unsigned long __read_mostly next_print; -+ -+ if (time_before(jiffies, READ_ONCE(next_print))) -+ return; -+ -+ WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL); -+ hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio); -+} -+ -+void hmbird_notify_sched_tick(void) -+{ -+ unsigned long last_check; -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ hmbird_scheduler_tick(); -+ -+ if (!hmbird_enabled()) -+ return; -+ -+ last_check = hmbird_watchdog_timestamp; -+ if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -+ -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "watchdog failed to check in for %u.%03us", -+ dur_ms / 1000, dur_ms % 1000); -+ } -+ scan_timeout(rq); -+} -+ -+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued) -+{ -+ update_curr_hmbird(rq); -+ -+ set_pcp_round(rq); -+ -+ if (slim_walt_ctrl) -+ hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE); -+ /* -+ * While disabling, always resched and refresh core-sched timestamp as -+ * we can't trust the slice management or ops.core_sched_before(). -+ */ -+ if (hmbird_ops_disabling()) -+ get_hmbird_ts(curr)->slice = 0; -+ -+ if (!get_hmbird_ts(curr)->slice) -+ resched_curr(rq); -+ -+ inform_hmbird_onoff_from_systrace(); -+} -+ -+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg) -+{ -+ hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ get_hmbird_ts(p)->disallow = false; -+ -+ hmbird_sched_init_task(p); -+ -+ set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ return 0; -+} -+ -+static void hmbird_ops_enable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void hmbird_ops_disable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void set_task_hmbird_weight(struct task_struct *p) -+{ -+ u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -+ -+ get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight); -+} -+ -+/** -+ * refresh_hmbird_weight - Refresh a task's hmbird weight -+ * @p: task to refresh hmbird weight for -+ * -+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to -+ * enable easy access from the BPF scheduler. To keep it synchronized with the -+ * current task priority, this function should be called when a new task is -+ * created, priority is changed for a task on hmbird, and a task is switched -+ * to hmbird from other classes. -+ */ -+static void refresh_hmbird_weight(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ set_task_hmbird_weight(p); -+} -+ -+int hmbird_pre_fork(struct task_struct *p) -+{ -+ int ret = 0; -+ -+ p->android_oem_data1[HMBIRD_TS_IDX] = -+ (u64)(kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL)); -+ if (!get_hmbird_ts(p)) { -+ ret = -1; -+ goto lock; -+ } -+ -+ get_hmbird_ts(p)->dsq = NULL; -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node); -+ get_hmbird_ts(p)->flags = 0; -+ get_hmbird_ts(p)->weight = 0; -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ get_hmbird_ts(p)->holding_cpu = -1; -+ get_hmbird_ts(p)->kf_mask = 0; -+ atomic64_set(&get_hmbird_ts(p)->ops_state, 0); -+ get_hmbird_ts(p)->runnable_at = INITIAL_JIFFIES; -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+ get_hmbird_ts(p)->task = p; -+ hmbird_set_sched_prop(p, 0); -+ -+ get_hmbird_ts(p)->critical_affinity_cpu = -1; -+ get_hmbird_ts(p)->sched_class = &hmbird_sched_class; -+ -+ /* -+ * BPF scheduler enable/disable paths want to be able to iterate and -+ * update all tasks which can become complex when racing forks. As -+ * enable/disable are very cold paths, let's use a percpu_rwsem to -+ * exclude forks. -+ */ -+lock: -+ percpu_down_read(&hmbird_fork_rwsem); -+ -+ return ret; -+} -+ -+int hmbird_fork(struct task_struct *p) -+{ -+ percpu_rwsem_assert_held(&hmbird_fork_rwsem); -+ -+ if (hmbird_enabled()) -+ return hmbird_ops_prepare_task(p, task_group(p)); -+ else -+ return 0; -+} -+ -+void hmbird_post_fork(struct task_struct *p) -+{ -+ if (hmbird_enabled()) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ /* -+ * Set the weight manually before calling ops.enable() so that -+ * the scheduler doesn't see a stale value if they inspect the -+ * task struct. We'll invoke ops.set_weight() afterwards, as it -+ * would be odd to receive a callback on the task before we -+ * tell the scheduler that it's been fully enabled. -+ */ -+ set_task_hmbird_weight(p); -+ hmbird_ops_enable_task(p); -+ refresh_hmbird_weight(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ -+ spin_lock_irq(&hmbird_tasks_lock); -+ list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_cancel_fork(struct task_struct *p) -+{ -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ if (hmbird_enabled()) -+ hmbird_ops_disable_task(p); -+ -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_free(struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+ list_del_init(&get_hmbird_ts(p)->tasks_node); -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+ -+ /* -+ * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED -> -+ * ENABLED transitions can't race us. Disable ops for @p. -+ */ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) || -+ test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ hmbird_ops_disable_task(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+} -+ -+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio) -+{ -+} -+ -+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type) -+{ -+ return (prop >> TOP_TASK_SHIFT) & (1 << type); -+} -+ -+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p) -+{ -+ uint32_t prop = get_top_task_prop(p); -+ -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE)) -+ return HMBIRD_TASK_PROP_PIPELINE; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) || -+ !task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) { -+ return HMBIRD_TASK_PROP_COMMON; -+ } -+ return HMBIRD_TASK_PROP_DEBUG_OR_LOG; -+} -+ -+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b) -+{ -+ int type_a = hmbird_get_task_type(a); -+ int type_b = hmbird_get_task_type(b); -+ -+ return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b]; -+} -+ -+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ enum cpu_type type; -+ int sp_dl; -+ struct task_struct *curr = NULL; -+ -+ switch (hmbird_preempt_policy) { -+ case HMBIRD_PREEMPT_POLICY_PRIO_BASED: -+ curr = rq->curr; -+ if (curr && hmbird_prio_higher(p, curr)) -+ goto preempt; -+ break; -+ default: -+ break; -+ } -+ if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) || -+ (is_critical_system_task(p) && !is_critical_system_task(rq->curr))) -+ goto preempt; -+ -+ sp_dl = find_idx_from_task(p); -+ if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled())) -+ return; -+ -+ if (rq->curr->prio > p->prio) -+ goto preempt; -+ -+ return; -+ -+preempt: -+ resched_curr(rq); -+} -+ -+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {} -+ -+int hmbird_check_setscheduler(struct task_struct *p, int policy) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ /* if disallow, reject transitioning into HMBIRD */ -+ if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) && -+ p->policy != policy && policy == SCHED_HMBIRD) -+ return -EACCES; -+ -+ return 0; -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool hmbird_can_stop_tick(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ if (hmbird_ops_disabling()) -+ return false; -+ -+ if (p->sched_class != &hmbird_sched_class) -+ return true; -+ -+ return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK; -+} -+#endif -+ -+int hmbird_tg_online(struct task_group *tg) -+{ -+ struct cgroup *cgrp; -+ -+ if (!tg) -+ return 0; -+ cgrp = tg->css.cgroup; -+ if (!cgrp || !(cgrp->kn)) -+ return 0; -+ update_cgroup_ids_table(cgrp->kn->id, -1); -+ return 0; -+} -+ -+/* -+ * Omitted operations: -+ * -+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -+ * task isn't tied to the CPU at that point. Preemption is implemented by -+ * resetting the victim task's slice to 0 and triggering reschedule on the -+ * target CPU. -+ * -+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -+ * -+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -+ * their current sched_class. Call them directly from sched core instead. -+ * -+ * - task_woken, switched_from: Unnecessary. -+ */ -+DEFINE_SCHED_CLASS(hmbird) = { -+ .enqueue_task = enqueue_task_hmbird, -+ .dequeue_task = dequeue_task_hmbird, -+ .yield_task = yield_task_hmbird, -+ .yield_to_task = yield_to_task_hmbird, -+ -+ .check_preempt_curr = check_preempt_curr_hmbird, -+ -+ .pick_next_task = pick_next_task_hmbird, -+ -+ .put_prev_task = put_prev_task_hmbird, -+ .set_next_task = set_next_task_hmbird, -+ -+#ifdef CONFIG_SMP -+ .balance = balance_hmbird, -+ .select_task_rq = select_task_rq_hmbird, -+ .set_cpus_allowed = set_cpus_allowed_hmbird, -+#endif -+ .task_tick = task_tick_hmbird, -+ -+ .switched_to = switched_to_hmbird, -+ .prio_changed = prio_changed_hmbird, -+ -+ .update_curr = update_curr_hmbird, -+ -+#ifdef CONFIG_UCLAMP_TASK -+ .uclamp_enabled = 0, -+#endif -+}; -+ -+/* -+ * Must with rq lock held. -+ */ -+bool task_is_hmbird(struct task_struct *p) -+{ -+ return p->sched_class == &hmbird_sched_class; -+} -+ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id) -+{ -+ memset(dsq, 0, sizeof(*dsq)); -+ -+ raw_spin_lock_init(&dsq->lock); -+ INIT_LIST_HEAD(&dsq->fifo); -+ dsq->id = dsq_id; -+} -+ -+static int hmbird_cgroup_init(void) -+{ -+ struct cgroup_subsys_state *css; -+ -+ css_for_each_descendant_pre(css, &root_task_group.css) { -+ struct task_group *tg = css_tg(css); -+ -+ cgrp_dsq_idx_init(css->cgroup, tg); -+ } -+ return 0; -+} -+ -+/* -+ * Used by sched_fork() and __setscheduler_prio() to pick the matching -+ * sched_class. dl/rt are already handled. -+ */ -+bool task_on_hmbird(struct task_struct *p) -+{ -+ return hmbird_enabled(); -+} -+ -+static void __setscheduler_prio(struct task_struct *p, int prio) -+{ -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) { -+ p->prio = prio; -+ return; -+ } else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (on_hmbird && rt_prio(prio)) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ -+ p->prio = prio; -+} -+ -+/* -+ * Heartbeat, avoid humbird keep running while APP already exit. -+ * Check whether APP send alive-signal periodly. -+ */ -+#define HEARTBEAT_TIMEOUT (msecs_to_jiffies(2500)) -+#define HEARTBEAT_CHECK_INTERVAL (msecs_to_jiffies(1000)) -+static struct timer_list hb_timer; -+static unsigned long next_hb; -+ -+void hb_timer_handler(struct timer_list *timer) -+{ -+ if (!heartbeat_enable) -+ goto refill; -+ -+ pr_err(": enter timer.\n"); -+ if (heartbeat) { -+ heartbeat = 0; -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ } -+ -+ /* can't detect heartbeat, disable ext. */ -+ if (time_after(jiffies, READ_ONCE(next_hb))) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT, -+ "can't detect heartbeat, disable ext\n"); -+ } -+refill: -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_start(void) -+{ -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_init(void) -+{ -+ timer_setup(&hb_timer, hb_timer_handler, 0); -+} -+ -+static void hb_timer_exit(void) -+{ -+ del_timer(&hb_timer); -+} -+ -+static bool check_and_disable_cpuhp(void) -+{ -+ struct rq *rq; -+ struct rq_flags rf; -+ int cpu; -+ -+ cpu_hotplug_disable(); -+ cpus_read_lock(); -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ rq_lock_irqsave(rq, &rf); -+ if (!rq->online) { -+ rq_unlock_irqrestore(rq, &rf); -+ goto err_offline; -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ cpus_read_unlock(); -+ return true; -+ -+err_offline: -+ cpus_read_unlock(); -+ cpu_hotplug_enable(); -+ return false; -+} -+ -+static void reenable_cpuhp(void) -+{ -+ cpu_hotplug_enable(); -+} -+ -+static void scheduler_switch_done(bool final_state) -+{ -+ /* set scx_enable again, switch may fail. */ -+ scx_enable = final_state; -+ /* reset heartbeat*/ -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ -+ if (final_state) -+ hb_timer_start(); -+ else -+ hb_timer_exit(); -+} -+ -+/** -+ * ss : curr switch state -+ * finish : success or fail -+ * enable : curr operation is diabling or enabling? -+ * fail_reason : string output when failed, empty string when success. -+ */ -+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason) -+{ -+ char *s1 = finish ? "finished" : "failed"; -+ char *s2 = enable ? "enabled" : "disabled"; -+ -+ hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss); -+ if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) { -+ sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type)); -+ hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n", -+ s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason); -+ } -+ curr_ss = ss; -+} -+ -+bool get_hmbird_ops_enabled(void) -+{ -+ return atomic_read(&__hmbird_ops_enabled); -+} -+ -+bool get_non_hmbird_task(void) -+{ -+ return atomic_read(&non_hmbird_task); -+} -+ -+static void hmbird_ops_disable_workfn(struct kthread_work *work) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int cpu; -+ -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, ""); -+ cancel_delayed_work_sync(&hmbird_watchdog_work); -+ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) { -+ case HMBIRD_OPS_DISABLED: -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled"); -+ scheduler_switch_done(false); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return; -+ case HMBIRD_OPS_PREPPING: -+ fallthrough; -+ case HMBIRD_OPS_DISABLING: -+ /* shouldn't happen but handle it like ENABLING if it does */ -+ WARN_ONCE(true, "hmbird: duplicate disabling instance?"); -+ fallthrough; -+ case HMBIRD_OPS_ENABLING: -+ case HMBIRD_OPS_ENABLED: -+ break; -+ } -+ -+ /* kick all CPUs to restore ticks */ -+ for_each_possible_cpu(cpu) -+ resched_cpu(cpu); -+ -+ /* avoid racing against fork and cgroup changes */ -+ cpus_read_lock(); -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, ""); -+ spin_lock_irq(&hmbird_tasks_lock); -+ atomic_set(&__hmbird_ops_enabled, false); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ bool alive = READ_ONCE(p->__state) != TASK_DEAD; -+ -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ get_hmbird_ts(p)->slice = min_t(u64, -+ get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL); -+ -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ if (alive) -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ -+ hmbird_ops_disable_task(p); -+ } -+ hmbird_task_iter_exit(&sti); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, ""); -+ -+ atomic_set(&non_hmbird_task, true); -+ /* no task is on hmbird, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt); -+ synchronize_rcu(); -+ -+ percpu_up_write(&hmbird_fork_rwsem); -+ cpus_read_unlock(); -+ -+ if (slim_walt_ctrl) -+ slim_walt_enable(false); -+ -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 1, 0, ""); -+ scheduler_switch_done(false); -+ reenable_cpuhp(); -+} -+ -+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn); -+ -+static void schedule_hmbird_ops_disable_work(void) -+{ -+ struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper); -+ -+ /* -+ * We may be called spuriously before the first bpf_hmbird_reg(). If -+ * hmbird_ops_helper isn't set up yet, there's nothing to do. -+ */ -+ if (helper) -+ kthread_queue_work(helper, &hmbird_ops_disable_work); -+} -+ -+static void hmbird_ops_disable(void) -+{ -+ schedule_hmbird_ops_disable_work(); -+} -+ -+static void hmbird_err_exit_workfn(struct work_struct *work) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ hmbird_ctrl(false); -+ -+ for_each_present_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy) -+ continue; -+ if (cpu != policy->cpu) -+ goto put; -+ down_write(&policy->rwsem); -+ WARN_ON(store_scaling_governor(policy, -+ saved_gov[cpu], strlen(saved_gov[cpu])) <= 0); -+ up_write(&policy->rwsem); -+ hmbird_info_trace(":restore origin gov : %s\n", saved_gov[cpu]); -+put: -+ cpufreq_cpu_put(policy); -+ } -+ memset((char *)saved_gov, 0, sizeof(saved_gov)); -+} -+ -+void hmbird_ops_exit(void) -+{ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); -+} -+ -+static struct kthread_worker *hmbird_create_rt_helper(const char *name) -+{ -+ struct kthread_worker *helper; -+ -+ helper = kthread_create_worker(0, name); -+ if (helper) -+ sched_set_fifo(helper->task); -+ return helper; -+} -+ -+static inline void set_audio_thread_sched_prop(struct task_struct *p) -+{ -+ struct cgroup_subsys_state *css; -+ -+ if (likely(p->prio >= MAX_RT_PRIO)) -+ return; -+ -+ rcu_read_lock(); -+ css = task_css(p, cpuset_cgrp_id); -+ if (!css) { -+ rcu_read_unlock(); -+ return; -+ } -+ rcu_read_unlock(); -+ -+ if (!strcmp(css->cgroup->kn->name, "audio-app")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+} -+ -+static int hmbird_ops_enable(void *unused) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int ret; -+ int tcnt = 0; -+ unsigned long long start = 0; -+ -+ if (!check_and_disable_cpuhp()) { -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline"); -+ return -EBUSY; -+ } -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, ""); -+ mutex_lock(&hmbird_ops_enable_mutex); -+ -+ if (!hmbird_ops_helper) { -+ WRITE_ONCE(hmbird_ops_helper, -+ hmbird_create_rt_helper("hmbird_ops_helper")); -+ if (!hmbird_ops_helper) { -+ ret = -ENOMEM; -+ goto err_unlock; -+ } -+ } -+ -+ if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) { -+ ret = -EBUSY; -+ goto err_unlock; -+ } -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) != -+ HMBIRD_OPS_DISABLED); -+ -+ hmbird_warned_zero_slice = false; -+ -+ atomic64_set(&hmbird_nr_rejected, 0); -+ -+ /* -+ * Keep CPUs stable during enable so that the BPF scheduler can track -+ * online CPUs by watching ->on/offline_cpu() after ->init(). -+ */ -+ cpus_read_lock(); -+ -+ hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work, -+ hmbird_watchdog_timeout / 2); -+ -+ /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ */ -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ reset_idle_masks(); -+ -+ /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. -+ */ -+ ret = hmbird_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock; -+ -+ /* -+ * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem -+ * preventing new tasks from being added. No need to exclude tasks -+ * leaving as hmbird_free() can handle both prepped and enabled -+ * tasks. Prep all tasks first and then enable them with preemption -+ * disabled. -+ */ -+ spin_lock_irq(&hmbird_tasks_lock); -+ -+ atomic_set(&non_hmbird_task, false); -+ atomic_set(&__hmbird_ops_enabled, true); -+ -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered(&sti))) -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ hmbird_task_iter_exit(&sti); -+ -+ /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. -+ */ -+ preempt_disable(); -+ -+ /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ */ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) { -+ atomic_set(&non_hmbird_task, true); -+ atomic_set(&__hmbird_ops_enabled, false); -+ preempt_enable(); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ ret = -EBUSY; -+ goto err_disable_unlock; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED -+ * transitions here are synchronized against hmbird_free() through -+ * hmbird_tasks_lock. -+ */ -+ start = sched_clock(); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, ""); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ tcnt++; -+ if (READ_ONCE(p->__state) != TASK_DEAD) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ -+ set_audio_thread_sched_prop(p); -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ hmbird_ops_enable_task(p); -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ } else { -+ hmbird_ops_disable_task(p); -+ } -+ } -+ hmbird_task_iter_exit(&sti); -+ -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, ""); -+ preempt_enable(); -+ percpu_up_write(&hmbird_fork_rwsem); -+ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) { -+ ret = -EBUSY; -+ goto err_disable; -+ } -+ -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_ENABLED, 1, 1, ""); -+ scheduler_switch_done(true); -+ -+ return 0; -+ -+err_unlock: -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock"); -+ scheduler_switch_done(false); -+ return ret; -+ -+err_disable_unlock: -+ percpu_up_write(&hmbird_fork_rwsem); -+err_disable: -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ /* must be fully disabled before returning */ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable"); -+ scheduler_switch_done(false); -+ return ret; -+} -+ -+#ifdef CONFIG_SCHED_DEBUG -+static const char *hmbird_ops_enable_state_str[] = { -+ [HMBIRD_OPS_PREPPING] = "prepping", -+ [HMBIRD_OPS_ENABLING] = "enabling", -+ [HMBIRD_OPS_ENABLED] = "enabled", -+ [HMBIRD_OPS_DISABLING] = "disabling", -+ [HMBIRD_OPS_DISABLED] = "disabled", -+}; -+ -+static int hmbird_debug_show(struct seq_file *m, void *v) -+{ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled()); -+ seq_printf(m, "%-30s: %s\n", "enable_state", -+ hmbird_ops_enable_state_str[hmbird_ops_enable_state()]); -+ seq_printf(m, "%-30s: %llu\n", "nr_rejected", -+ atomic64_read(&hmbird_nr_rejected)); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return 0; -+} -+ -+static int hmbird_debug_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_debug_show, NULL); -+} -+ -+const struct file_operations sched_hmbird_fops = { -+ .open = hmbird_debug_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif -+ -+ -+static int bpf_hmbird_reg(void *kdata) -+{ -+ return hmbird_ops_enable(kdata); -+} -+ -+static int bpf_hmbird_unreg(void *kdata) -+{ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ return 0; -+} -+ -+void set_hmbird_module_loaded(int is_loaded) -+{ -+ atomic_set(&hmbird_module_loaded, is_loaded); -+} -+ -+/* -+ * MUST load hmbird module before enable hmbird scheduler -+ * load track & hmbird gover implemented in hmbird module -+ */ -+int hmbird_ctrl(bool enable) -+{ -+ if (!atomic_read(&hmbird_module_loaded)) { -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "ext module unloaded\n"); -+ return -EINVAL; -+ } -+ -+ if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED -+ || hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING -+ || hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already enabled(ing)\n"); -+ hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING || -+ hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already disabled(ing)\n"); -+ hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (enable) -+ return bpf_hmbird_reg(NULL); -+ else -+ return bpf_hmbird_unreg(NULL); -+} -+ -+void set_cpu_isomask(int cpu, cpumask_var_t *mask) -+{ -+ cpumask_clear_cpu(cpu, iso_masks.ex_free); -+ cpumask_clear_cpu(cpu, iso_masks.exclusive); -+ cpumask_clear_cpu(cpu, iso_masks.partial); -+ cpumask_clear_cpu(cpu, iso_masks.big); -+ cpumask_clear_cpu(cpu, iso_masks.little); -+ cpumask_set_cpu(cpu, *mask); -+} -+ -+void set_cpu_cluster(u64 cpu_cluster) -+{ -+ int cpu; -+ -+ for_each_present_cpu(cpu) { -+ u64 pos = 1 << cpu; -+ -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.ex_free)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.exclusive)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.partial)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.big)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) -+ set_cpu_isomask(cpu, &(iso_masks.little)); -+ } -+} -+ -+static void get_hmbird_snapshot(struct panic_snapshot_t *p) -+{ -+ struct hmbird_dispatch_q *dsq; -+ struct rq *rq; -+ int cpu, i; -+ -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ p->rq_nr[cpu] = rq->nr_running; -+ p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running; -+ } -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ struct hmbird_entity *entity; -+ -+ dsq = &gdsqs[i]; -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo)) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ p->runnable_at[i] = entity->runnable_at; -+ raw_spin_unlock(&dsq->lock); -+ -+ } -+ -+ p->snap_misc.hmbird_enabled = hmbird_enabled(); -+ p->snap_misc.curr_ss = curr_ss; -+ p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state(); -+ p->snap_misc.parctrl_high_ratio = parctrl_high_ratio; -+ p->snap_misc.parctrl_low_ratio = parctrl_low_ratio; -+ p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l; -+ p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l; -+ p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio; -+ p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio; -+ p->snap_misc.misfit_ds = misfit_ds; -+ p->snap_misc.partial_enable = partial_enable; -+ p->snap_misc.iso_free_rescue = iso_free_rescue; -+ p->snap_misc.isolate_ctrl = isolate_ctrl; -+ p->snap_misc.snap_jiffies = jiffies; -+ p->snap_misc.snap_time = local_clock(); -+} -+ -+// MTK minidump begin -+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3) -+{ -+ strscpy(m->desc_str, str, DESC_STR_LEN); -+ m->len = d1 * d2 * d3; -+ m->parse[0] = d1; -+ m->parse[1] = d2; -+ m->parse[2] = d3; -+} -+ -+static void init_desc_metas(struct md_info_t *m) -+{ -+ init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :", -+ 1, MAX_SWITCHS, SWITCH_ITEMS); -+ -+ init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1); -+ -+ init_desc_meta(&m->kern_dump.excep_rec_meta, -+ "excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS); -+ -+ init_desc_meta(&m->kern_dump.excep_idx_meta, -+ "excep idx :", 1, 1, MAX_EXCEP_ID); -+ -+ init_desc_meta(&m->kern_dump.snap.runnable_at_meta, -+ "each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS); -+ -+ init_desc_meta(&m->kern_dump.snap.rq_nr_meta, -+ "rq runnable task nr:", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta, -+ "scxrq runnable task nr :", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.snap_misc_meta, -+ "misc snap params :", 1, 1, SNAP_ITEMS); -+} -+ -+static void init_md_meta(struct md_info_t *m) -+{ -+ m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64); -+ m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64); -+ m->meta.unit_size = sizeof(u64); -+ m->meta.switches = MAX_SWITCHS; -+ m->meta.exceps = MAX_EXCEPS; -+ m->meta.global_dsqs = MAX_GLOBAL_DSQS; -+ m->meta.parse_dimens = PARSE_DIMENS; -+ m->meta.nr_cpus = num_possible_cpus(); -+ m->meta.real_cpus = nr_cpu_ids; -+ m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64); -+ m->meta.nr_meta_desc = 8; -+ m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64); -+ -+ init_desc_metas(m); -+} -+ -+#define MINIDUMP_DFL_SIZE (4 * 1024) -+ -+struct notifier_block hmbird_panic_blk; -+static int hmbird_panic_handler(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ if (!md_info) -+ return NOTIFY_DONE; -+ -+ get_hmbird_snapshot(&md_info->kern_dump.snap); -+ -+ return NOTIFY_DONE; -+} -+ -+static void panic_blk_init(void) -+{ -+ int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE); -+ -+ md_info = kzalloc(dump_size, GFP_KERNEL); -+ if (!md_info) -+ return; -+ init_md_meta(md_info); -+ -+ hmbird_panic_blk.notifier_call = hmbird_panic_handler; -+ /* make sure to execute before minidump. */ -+ hmbird_panic_blk.priority = INT_MAX; -+ atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk); -+ -+ hmbird_debug("register minidump.\n"); -+} -+ -+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size) -+{ -+ *vaddr = (unsigned long)md_info; -+ *size = sizeof(struct md_info_t); -+} -+// MTK minidump end -+ -+static inline void init_sched_prop_to_preempt_prio(void) -+{ -+ for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) { -+ switch (i) { -+ case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 5; -+ break; -+ -+ case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 4; -+ break; -+ -+ case HMBIRD_TASK_PROP_PIPELINE: -+ case HMBIRD_TASK_PROP_ISOLATE: -+ sched_prop_to_preempt_prio[i] = 3; -+ break; -+ -+ case HMBIRD_TASK_PROP_COMMON: -+ sched_prop_to_preempt_prio[i] = 1; -+ break; -+ -+ case HMBIRD_TASK_PROP_DEBUG_OR_LOG: -+ sched_prop_to_preempt_prio[i] = 0; -+ break; -+ -+ default: -+ sched_prop_to_preempt_prio[i] = 2; -+ break; -+ } -+ } -+} -+ -+void __init init_sched_hmbird_class(void) -+{ -+ int cpu; -+ u32 v; -+ struct hmbird_entity *init_hmbird; -+ -+ /* -+ * The following is to prevent the compiler from optimizing out the enum -+ * definitions so that BPF scheduler implementations can use them -+ * through the generated vmlinux.h. -+ */ -+ WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP | -+ HMBIRD_KICK_PREEMPT); -+ -+ init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL); -+ init_dsq_at_boot(); -+ init_isolate_cpus(); -+ hb_timer_init(); -+ init_sched_prop_to_preempt_prio(); -+#ifdef CONFIG_SMP -+ WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); -+#endif -+ -+ /* -+ * we can't static init init_task's hmbird struct, init here. -+ * init_task->hmbird would not use during boot. -+ */ -+ init_hmbird = kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL); -+ init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird; -+ if (init_hmbird) { -+ INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo); -+ INIT_LIST_HEAD(&init_hmbird->watchdog_node); -+ init_hmbird->sticky_cpu = -1; -+ init_hmbird->holding_cpu = -1; -+ atomic64_set(&init_hmbird->ops_state, 0); -+ init_hmbird->runnable_at = jiffies; -+ init_hmbird->slice = HMBIRD_SLICE_DFL; -+ init_hmbird->task = &init_task; -+ hmbird_set_sched_prop(&init_task, 0); -+ } else { -+ hmbird_err(INIT_TASK_FAIL, ":alloc init_task.scx failed!!!\n"); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ /* -+ * exec during boot phase, no need to care about alloc failed. -+ * lifecycle same to rq, no need to free. -+ */ -+ rq->android_oem_data1[HMBIRD_RQ_IDX] = -+ (u64)kmalloc(sizeof(struct hmbird_rq), GFP_KERNEL); -+ if (get_hmbird_rq(rq)) { -+ get_hmbird_rq(rq)->rq = rq; -+ get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window; -+ } else { -+ hmbird_err(ALLOC_RQSCX_FAIL, ":alloc rq->scx failed!!!\n"); -+ } -+ -+ rq->android_oem_data1[HMBIRD_OPS_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL); -+ if (!get_hmbird_ops(rq)) -+ pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n"); -+ init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL); -+ INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list); -+ -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL)); -+ -+ hmbird_ops_init(get_hmbird_ops(rq)); -+ } -+ -+ INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn); -+ INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn); -+ hmbird_misc_init(); -+ -+ panic_blk_init(); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_misc.c b/kernel/sched/hmbird/hmbird_misc.c -new file mode 100755 -index 000000000000..52582c22052c ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_misc.c -@@ -0,0 +1,124 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * @Description: -+ * @Version: -+ * @Author: wangrui8 -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include "hmbird_sched.h" -+#include -+#include -+#include "slim.h" -+ -+noinline int tracing_mark_write(const char *buf) -+{ -+ trace_printk(buf); -+ return 0; -+} -+ -+struct yield_opt_params yield_opt_params = { -+ .enable = 0, -+ .frame_per_sec = 120, -+ .frame_time_ns = NSEC_PER_SEC / 120, -+ .yield_headroom = 10, -+}; -+ -+DEFINE_PER_CPU(struct sched_yield_state, ystate); -+ -+static inline void hmbird_yield_state_update(struct sched_yield_state *ys) -+{ -+ if (!raw_spin_is_locked(&ys->lock)) -+ return; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ -+ if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1 -+ || ys->yield_cnt_after_sleep > yield_headroom) { -+ ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP); -+ } else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) { -+ ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP); -+ } -+ ys->yield_cnt = 0; -+ ys->sleep_times = 0; -+ ys->yield_cnt_after_sleep = 0; -+} -+ -+void hmbird_skip_yield(long *skip) -+{ -+ if (!get_hmbird_ops_enabled() || !yield_opt_params.enable) -+ return; -+ unsigned long flags, sleep_now = 0; -+ struct sched_yield_state *ys; -+ int cpu = raw_smp_processor_id(), cont_yield, new_frame; -+ int frame_time_ns = yield_opt_params.frame_time_ns; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ u64 wc; -+ -+ if (!(*skip)) { -+ wc = sched_clock(); -+ ys = &per_cpu(ystate, cpu); -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ -+ cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP; -+ new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1); -+ -+ if (!cont_yield && new_frame) { -+ hmbird_yield_state_update(ys); -+ ys->last_update_time = wc; -+ ys->sleep_end = ys->last_yield_time + frame_time_ns -+ - yield_headroom * YIELD_DURATION; -+ } -+ -+ if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) { -+ *skip = true; -+ -+ sleep_now = ys->sleep_times ? -+ max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep; -+ if (wc + sleep_now > ys->sleep_end) { -+ u64 delta = ys->sleep_end - wc; -+ -+ if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION) -+ sleep_now = delta; -+ else -+ sleep_now = 0; -+ } -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ if (sleep_now) { -+ sleep_now = div64_u64(sleep_now, 1000); -+ usleep_range_state(sleep_now, sleep_now, TASK_IDLE); -+ } -+ ys->sleep_times++; -+ ys->last_yield_time = sched_clock(); -+ return; -+ } -+ if (ys->sleep_times) -+ ys->yield_cnt_after_sleep++; -+ else -+ (ys->yield_cnt)++; -+ ys->last_yield_time = wc; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+} -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops) -+{ -+ hmbird_ops->scx_enable = get_hmbird_ops_enabled; -+ hmbird_ops->check_non_task = get_non_hmbird_task; -+ hmbird_ops->hmbird_get_md_info = hmbird_get_md_info; -+} -+ -+void hmbird_misc_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_init(&ys->lock); -+ } -+ -+ set_hmbird_module_loaded(1); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_sched.h b/kernel/sched/hmbird/hmbird_sched.h -new file mode 100755 -index 000000000000..6b5ef43cb827 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef __HMBIRD_SCHED__ -+#define __HMBIRD_SCHED__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include "../../time/tick-sched.h" -+#include "../../sched/sched.h" -+#include -+#include -+#include -+ -+#define REGISTER_TRACE_VH(vender_hook, handler) \ -+{ \ -+ ret = register_trace_##vender_hook(handler, NULL); \ -+ if (ret) { \ -+ pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \ -+ } \ -+} -+#define REGISTER_TRACE(vendor_hook, handler, data, err) \ -+do { \ -+ ret = register_trace_##vendor_hook(handler, data); \ -+ if (ret) { \ -+ pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret); \ -+ goto err; \ -+ } \ -+} while (0) -+ -+#define UNREGISTER_TRACE(vendor_hook, handler, data) \ -+ unregister_trace_##vendor_hook(handler, data) \ -+ -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+ -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int sched_ravg_window_frame_per_sec; -+extern int slim_gov_debug; -+extern int cpu7_tl; -+extern int scx_gov_ctrl; -+extern spinlock_t new_sched_ravg_window_lock; -+extern int cluster_separate; -+ -+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER BIT(31) -+#define MAX_YIELD_SLEEP (2000000ULL) -+#define MIN_YIELD_SLEEP (200000ULL) -+#define YIELD_DURATION (5000ULL) -+#define DEFAULT_YIELD_SLEEP_TH (10) -+ -+struct sched_yield_state { -+ raw_spinlock_t lock; -+ u64 last_yield_time; -+ u64 last_update_time; -+ u64 sleep_end; -+ unsigned long yield_cnt; -+ unsigned long yield_cnt_after_sleep; -+ unsigned long sleep; -+ int sleep_times; -+}; -+ -+DECLARE_PER_CPU(struct sched_yield_state, ystate); -+ -+void hmbird_window_rollover_run_once(struct rq *rq); -+void hmbird_yield_state_update_per_frame(void); -+void hmbird_misc_init(void); -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops); -+#endif /*__HMBIRD_SCHED__*/ -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.c b/kernel/sched/hmbird/hmbird_sched_proc.c -new file mode 100755 -index 000000000000..234768fc767a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.c -@@ -0,0 +1,527 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "hmbird_sched_proc.h" -+#include -+ -+#include "hmbird_util_track.h" -+#include "slim.h" -+ -+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched" -+#define SLIM_FREQ_GOV_DIR "slim_freq_gov" -+#define LOAD_TRACK_DIR "slim_walt" -+#define HMBIRD_PROC_PERMISSION 0666 -+ -+int scx_enable; -+int partial_enable; -+int cpuctrl_high_ratio = 55; -+int cpuctrl_low_ratio = 40; -+int slim_stats; -+int hmbirdcore_debug; -+int slim_for_app; -+int misfit_ds = 90; -+unsigned int highres_tick_ctrl; -+unsigned int highres_tick_ctrl_dbg; -+int cpu7_tl = 70; -+int slim_walt_ctrl; -+int slim_walt_dump; -+int slim_walt_policy; -+int slim_gov_debug; -+int scx_gov_ctrl = 1; -+int sched_ravg_window_frame_per_sec = 125; -+int parctrl_high_ratio = 55; -+int parctrl_low_ratio = 40; -+int parctrl_high_ratio_l = 65; -+int parctrl_low_ratio_l = 50; -+int isoctrl_high_ratio = 75; -+int isoctrl_low_ratio = 60; -+int isolate_ctrl; -+int iso_free_rescue; -+int heartbeat; -+int heartbeat_enable = 1; -+int watchdog_enable; -+int save_gov; -+u64 cpu_cluster_masks; -+int hmbird_preempt_policy; -+int cluster_separate; -+ -+char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= 32) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtoint(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoint\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+static int set_proc_buf_val_u64(struct file *file, const char __user *buf, -+ size_t count, u64 *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= sizeof(kbuf)) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtou64(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoul\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/* common ops begin */ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ return count; -+} -+ -+static int hmbird_common_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%d\n", *(int *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_show, pde_data(inode)); -+} -+ -+static int hmbird_common_ul_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%lu\n", *(unsigned long *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_ul_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_ul_show, pde_data(inode)); -+} -+ -+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write); -+/* common ops end */ -+ -+/* scx_enable ops begin */ -+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC); -+ if (hmbird_ctrl(*pval)) -+ return -EFAULT; -+ -+ return count; -+} -+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write); -+/* scx_enable ops end */ -+ -+/* hmbird_stats ops begin */ -+#define MAX_STATS_BUF (4096) -+static int hmbird_stats_proc_show(struct seq_file *m, void *v) -+{ -+ char *buf; -+ -+ buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ stats_print(buf, MAX_STATS_BUF); -+ -+ seq_printf(m, "%s\n", buf); -+ -+ kfree(buf); -+ return 0; -+} -+ -+static int hmbird_stats_proc_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_stats_proc_show, inode); -+} -+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL); -+/* hmbird_stats ops end */ -+ -+/* sched_ravg_window_frame_per_sec ops begin */ -+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file, -+ const char __user *buf, size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ sched_ravg_window_change(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open, -+ sched_ravg_window_frame_per_sec_proc_write); -+/* sched_ravg_window_frame_per_sec ops end */ -+ -+static ssize_t save_gov_str(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ for_each_possible_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy || (cpu != policy->cpu)) -+ continue; -+ WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0); -+ hmbird_info_systrace(":save origin gov : %s\n", saved_gov[cpu]); -+ } -+ return count; -+} -+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str); -+ -+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ u64 *pval = (u64 *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val_u64(file, buf, count, pval)) -+ return -EFAULT; -+ -+ if (scx_enable == 0) -+ set_cpu_cluster(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write); -+ -+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ int tmp_val; -+ -+ if (set_proc_buf_val(file, buf, count, &tmp_val)) -+ return -EFAULT; -+ -+ slim_walt_enable(tmp_val); -+ *pval = tmp_val; -+ -+ return count; -+} -+ -+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write); -+ -+/* yield_opt ops begin */ -+static int yield_opt_show(struct seq_file *m, void *v) -+{ -+ struct yield_opt_params *data = m->private; -+ -+ seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n", -+ data->enable, data->frame_per_sec, data->yield_headroom); -+ return 0; -+} -+ -+static int yield_opt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, yield_opt_show, pde_data(inode)); -+} -+ -+static ssize_t yield_opt_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu; -+ unsigned long flags; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if (copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp -+ != 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) || -+ (yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp; -+ yield_opt_params.frame_per_sec = frame_per_sec_tmp; -+ yield_opt_params.yield_headroom = yield_headroom_tmp; -+ yield_opt_params.enable = enable_tmp; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ ys->last_yield_time = 0; -+ ys->last_update_time = 0; -+ ys->sleep_end = 0; -+ ys->yield_cnt = 0; -+ ys->yield_cnt_after_sleep = 0; -+ ys->sleep = 0; -+ ys->sleep_times = 0; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write); -+ -+ -+static int __init hmbird_proc_init(void) -+{ -+ struct proc_dir_entry *hmbird_dir; -+ struct proc_dir_entry *load_track_dir; -+ struct proc_dir_entry *freq_gov_dir; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ -+ /* mkdir /proc/hmbird_sched */ -+ hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL); -+ if (!hmbird_dir) { -+ pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &scx_enable_proc_ops, -+ &scx_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &partial_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_stats); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbirdcore_debug); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_for_app); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &misfit_ds); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl_dbg); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpu7_tl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &cpu_cluster_masks_proc_ops, -+ &cpu_cluster_masks); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &save_gov_proc_ops, -+ &save_gov); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &watchdog_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isolate_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &iso_free_rescue); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_stats_proc_ops); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &yield_opt_proc_ops, -+ &yield_opt_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbird_preempt_policy); -+ /* /proc/hmbird_sched--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_walt */ -+ load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir); -+ if (!load_track_dir) { -+ pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_walt--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &slim_walt_ctrl_proc_ops, -+ &slim_walt_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_dump); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_policy); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &sched_ravg_window_frame_per_sec_proc_ops, -+ &sched_ravg_window_frame_per_sec); -+ /* /proc/hmbird_sched/slim_walt--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_freq_gov */ -+ freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir); -+ if (!freq_gov_dir) { -+ pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_freq_gov--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &slim_gov_debug); -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &scx_gov_ctrl); -+ /* /proc/hmbird_sched/slim_freq_gov--end */ -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cluster_separate); -+ -+ return 0; -+} -+ -+device_initcall(hmbird_proc_init); -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.h b/kernel/sched/hmbird/hmbird_sched_proc.h -new file mode 100755 -index 000000000000..e22bc75f1dd7 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SCHED_PROC_H__ -+#define __HMBIRD_SCHED_PROC_H__ -+ -+#include -+#include -+ -+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \ -+ do { \ -+ if (!proc_create(name, mode, parent, proc_ops)) { \ -+ pr_err("Error creating proc entry %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \ -+ do { \ -+ if (!proc_create_data(name, mode, parent, proc_ops, data)) { \ -+ pr_err("Error creating proc entry with data %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_PROC_OPS(name, open_func, write_func) \ -+ static const struct proc_ops name##_proc_ops = { \ -+ .proc_open = open_func, \ -+ .proc_write = write_func, \ -+ .proc_read = seq_read, \ -+ .proc_lseek = seq_lseek, \ -+ .proc_release = single_release, \ -+ } -+ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos); -+static int hmbird_common_show(struct seq_file *m, void *v); -+static int hmbird_common_open(struct inode *inode, struct file *file); -+ -+#endif -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.c b/kernel/sched/hmbird/hmbird_shadow_tick.c -new file mode 100755 -index 000000000000..21de551c5132 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.c -@@ -0,0 +1,159 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include -+#include "../../time/tick-sched.h" -+#include -+ -+#include "hmbird_shadow_tick.h" -+ -+#define HIGHRES_WATCH_CPU 0 -+ -+#include -+static bool shadow_tick_enable(void) {return highres_tick_ctrl; } -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; } -+#endif -+static bool shadow_tick_timer_init_flag; -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define shadow_tick_printk(fmt, args...) \ -+do { \ -+ int cpu = smp_processor_id(); \ -+ if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU) \ -+ trace_printk("hmbird shadow tick :"fmt, args); \ -+} while (0) -+#else -+#define shadow_tick_printk(fmt, args...) -+#endif -+ -+DEFINE_PER_CPU(struct hrtimer, stt); -+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu))) -+#define STOP_IDLE_TRIGGER (1) -+#define PERIODIC_TICK_TRIGGER (2) -+#define TICK_INTVAL (1000000ULL) -+/* -+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms, -+ * so we can't restart hrtimer in scheduler tick. -+ */ -+static DEFINE_PER_CPU(u8, trigger_event); -+ -+/* -+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick. -+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler. -+ */ -+static DEFINE_PER_CPU(u8, tick_phase); -+ -+static inline void highres_timer_ctrl(bool enable, int cpu) -+{ -+ if (enable && hmbird_enabled()) { -+ if (!hrtimer_active(shadow_tick_timer(cpu))) -+ hrtimer_start(shadow_tick_timer(cpu), -+ ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED); -+ } else { -+ if (!enable) -+ hrtimer_cancel(shadow_tick_timer(cpu)); -+ } -+} -+ -+static inline void high_res_clear_phase(int cpu) -+{ -+ per_cpu(tick_phase, cpu) = 0; -+} -+ -+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer) -+{ -+ per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3; -+ if (per_cpu(tick_phase, cpu)) { -+ hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) { -+ per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ struct rq_flags rf; -+ -+ rq_lock(rq, &rf); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ rq_unlock(rq, &rf); -+ -+ return highres_next_phase(cpu, timer); -+} -+ -+void shadow_tick_timer_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance; -+ } -+} -+ -+void start_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable()) { -+ if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER) -+ highres_timer_ctrl(false, cpu); -+ per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static void stop_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ per_cpu(trigger_event, cpu) = 0; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(false, cpu); -+} -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ stop_shadow_tick_timer(); -+} -+ -+void scheduler_tick_handler(void *unused, struct rq *rq) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ start_shadow_tick_timer(); -+} -+ -+static int __init hmbird_shadow_tick_init(void) -+{ -+ int ret = 0; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ shadow_tick_timer_init(); -+ shadow_tick_timer_init_flag = true; -+ return ret; -+} -+ -+device_initcall(hmbird_shadow_tick_init); -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.h b/kernel/sched/hmbird/hmbird_shadow_tick.h -new file mode 100755 -index 000000000000..0c26fd984f0a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.h -@@ -0,0 +1,11 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SHADOW_TICK_H__ -+#define __HMBIRD_SHADOW_TICK_H__ -+ -+#include -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+void scheduler_tick_handler(void *unused, struct rq *rq); -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state); -+#endif -diff --git a/kernel/sched/hmbird/hmbird_trace.h b/kernel/sched/hmbird/hmbird_trace.h -new file mode 100755 -index 000000000000..8468b406f347 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_trace.h -@@ -0,0 +1,92 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hmbird -+ -+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HMBIRD_H -+ -+#include -+#include -+#include -+ -+#define MAX_FATAL_INFO (64) -+ -+TRACE_EVENT(hmbird_fatal_info, -+ -+ TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info), -+ -+ TP_ARGS(type, pe, lnr, bnr, info), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, type) -+ __field(int, pe) -+ __field(int, lnr) -+ __field(int, bnr) -+ __array(char, info, MAX_FATAL_INFO)), -+ -+ TP_fast_assign( -+ __entry->type = type; -+ __entry->pe = pe; -+ __entry->lnr = lnr; -+ __entry->bnr = bnr; -+ memcpy(__entry->info, info, MAX_FATAL_INFO);), -+ -+ TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s", -+ __entry->type, __entry->pe, __entry->lnr, __entry->bnr, -+ __entry->info) -+); -+ -+TRACE_EVENT(hmbird_update_history, -+ -+ TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq, -+ struct task_struct *p, u32 runtime, int samples, int event), -+ -+ TP_ARGS(hmbird, rq, p, runtime, samples, event), -+ -+ TP_STRUCT__entry( -+ __array(char, comm, TASK_COMM_LEN) -+ __field(pid_t, pid) -+ __field(unsigned int, runtime) -+ __field(int, samples) -+ __field(int, event) -+ __field(unsigned int, demand) -+ __array(u32, hist, RAVG_HIST_SIZE) -+ __field(u16, task_util) -+ __field(int, cpu)), -+ -+ TP_fast_assign( -+ memcpy(__entry->comm, p->comm, TASK_COMM_LEN); -+ __entry->pid = p->pid; -+ __entry->runtime = runtime; -+ __entry->samples = samples; -+ __entry->event = event; -+ __entry->demand = hmbird->sts.demand; -+ memcpy(__entry->hist, hmbird->sts.sum_history, -+ RAVG_HIST_SIZE * sizeof(u32)); -+ __entry->task_util = hmbird->sts.demand_scaled; -+ __entry->cpu = rq->cpu;), -+ -+ TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d", -+ __entry->comm, __entry->pid, -+ __entry->runtime, __entry->samples, -+ __entry->event, -+ __entry->demand, -+ __entry->hist[0], __entry->hist[1], -+ __entry->hist[2], __entry->hist[3], -+ __entry->hist[4], -+ __entry->task_util, -+ __entry->cpu) -+); -+ -+#endif /*_TRACE_HMBIRD_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird -+ -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE hmbird_trace -+/* This part must be outside protection */ -+#include -diff --git a/kernel/sched/hmbird/hmbird_util_track.c b/kernel/sched/hmbird/hmbird_util_track.c -new file mode 100755 -index 000000000000..d520a8403401 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.c -@@ -0,0 +1,626 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+ -+#define CREATE_TRACE_POINTS -+#include "hmbird_trace.h" -+#undef CREATE_TRACE_POINTS -+ -+#define HMBIRD_DEBUG_PANIC (1 << 3) -+static bool init_irq_work_inited; -+ -+extern noinline int tracing_mark_write(const char *buf); -+ -+int hmbird_sched_ravg_window = 8000000; -+int new_hmbird_sched_ravg_window = 8000000; -+DEFINE_SPINLOCK(new_sched_ravg_window_lock); -+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+ -+static struct irq_work hmbird_slim_walt_irq_work; -+ -+/*Sysctl related interface*/ -+#define WINDOW_STATS_RECENT 0 -+#define WINDOW_STATS_MAX 1 -+#define WINDOW_STATS_MAX_RECENT_AVG 2 -+#define WINDOW_STATS_AVG 3 -+#define WINDOW_STATS_INVALID_POLICY 4 -+ -+ -+#define SCX_SCHED_CAPACITY_SHIFT 10 -+#define SCHED_ACCOUNT_WAIT_TIME 0 -+ -+atomic64_t hmbird_irq_work_lastq_ws; -+static u64 tick_sched_clock; -+ -+static inline u64 scale_exec_time(u64 delta, struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT; -+} -+ -+static inline u64 scale_time_to_util(u64 d) -+{ -+ do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT); -+ return d; -+} -+ -+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ delta = scale_exec_time(delta, rq); -+ sts->sum += delta; -+ if (unlikely(sts->sum > hmbird_sched_ravg_window)) -+ sts->sum = hmbird_sched_ravg_window; -+ -+ return delta; -+} -+ -+ -+static int -+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) -+{ -+ /* -+ * No need to bother updating task demand for the idle task. -+ */ -+ if (is_idle_task(p)) -+ return 0; -+ -+ /* -+ * When a task is waking up it is completing a segment of non-busy -+ * time. Likewise, if wait time is not treated as busy time, then -+ * when a task begins to run or is migrated, it is not running and -+ * is completing a segment of non-busy time. -+ */ -+ if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && -+ (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) -+ return 0; -+ -+ /* -+ * The idle exit time is not accounted for the first task _picked_ up to -+ * run on the idle CPU. -+ */ -+ if (event == PICK_NEXT_TASK && rq->curr == rq->idle) -+ return 0; -+ -+ /* -+ * TASK_UPDATE can be called on sleeping task, when its moved between -+ * related groups -+ */ -+ if (event == TASK_UPDATE) { -+ if (rq->curr == p) -+ return 1; -+ -+ return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; -+ } -+ -+ return 1; -+} -+ -+static void rollover_cpu_window(struct rq *rq, bool full_window) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 curr_sum = srq->curr_runnable_sum; -+ -+ if (unlikely(full_window)) -+ curr_sum = 0; -+ -+ srq->prev_runnable_sum = curr_sum; -+ srq->curr_runnable_sum = 0; -+} -+ -+static u64 -+update_window_start(struct rq *rq, u64 wallclock, int event) -+{ -+ s64 delta; -+ int nr_windows; -+ bool full_window; -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start = srq->window_start; -+ -+ if (wallclock < srq->latest_clock) -+ wallclock = srq->latest_clock; -+ delta = wallclock - srq->window_start; -+ if (delta < 0) { -+ delta = 0; -+ wallclock = srq->window_start; -+ } -+ srq->latest_clock = wallclock; -+ if (delta < hmbird_sched_ravg_window) -+ return old_window_start; -+ -+ nr_windows = div64_u64(delta, hmbird_sched_ravg_window); -+ srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window; -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ full_window = nr_windows > 1; -+ rollover_cpu_window(rq, full_window); -+ -+ return old_window_start; -+} -+ -+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -+ -+static inline unsigned int get_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static inline unsigned int cpu_cur_freq(int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cur; -+} -+ -+static void -+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, -+ u64 wallclock) -+{ -+ int cpu = cpu_of(rq); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * -+ arch_scale_cpu_capacity(cpu), get_max_freq(cpu)); -+} -+ -+/* -+ * Called when new window is starting for a task, to record cpu usage over -+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 -+ * when, say, a real-time task runs without preemption for several windows at a -+ * stretch. -+ */ -+static void update_history(struct rq *rq, struct task_struct *p, -+ u32 runtime, int samples, int event) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u32 *hist = &sts->sum_history[0]; -+ int i; -+ u32 max = 0, avg, demand; -+ u64 sum = 0; -+ u16 demand_scaled; -+ -+ /* Ignore windows where task had no activity */ -+ if (!runtime || is_idle_task(p) || !samples) -+ goto done; -+ -+ /* Push new 'runtime' value onto stack */ -+ for (; samples > 0; samples--) { -+ hist[sts->cidx] = runtime; -+ sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE; -+ } -+ -+ for (i = 0; i < RAVG_HIST_SIZE; i++) { -+ sum += hist[i]; -+ if (hist[i] > max) -+ max = hist[i]; -+ } -+ -+ sts->sum = 0; -+ avg = div64_u64(sum, RAVG_HIST_SIZE); -+ -+ switch (slim_walt_policy) { -+ case WINDOW_STATS_RECENT: -+ demand = runtime; -+ break; -+ case WINDOW_STATS_MAX: -+ demand = max; -+ break; -+ case WINDOW_STATS_AVG: -+ demand = avg; -+ break; -+ default: -+ demand = max(avg, runtime); -+ } -+ -+ demand_scaled = scale_time_to_util(demand); -+ -+ sts->demand = demand; -+ sts->demand_scaled = demand_scaled; -+ -+done: -+ return; -+} -+ -+ -+static u64 update_task_demand(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ -+ u64 delta, window_start = srq->window_start; -+ int new_window, nr_full_windows; -+ u32 window_size = hmbird_sched_ravg_window; -+ u64 runtime; -+ -+ new_window = mark_start < window_start; -+ if (!account_busy_for_task_demand(rq, p, event)) { -+ if (new_window) -+ /* -+ * If the time accounted isn't being accounted as -+ * busy time, and a new window started, only the -+ * previous window need be closed out with the -+ * pre-existing demand. Multiple windows may have -+ * elapsed, but since empty windows are dropped, -+ * it is not necessary to account those. -+ */ -+ update_history(rq, p, sts->sum, 1, event); -+ return 0; -+ } -+ -+ if (!new_window) { -+ /* -+ * The simple case - busy time contained within the existing -+ * window. -+ */ -+ return add_to_task_demand(rq, p, wallclock - mark_start); -+ } -+ -+ /* -+ * Busy time spans at least two windows. Temporarily rewind -+ * window_start to first window boundary after mark_start. -+ */ -+ delta = window_start - mark_start; -+ nr_full_windows = div64_u64(delta, window_size); -+ window_start -= (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (window_start - mark_start) first */ -+ runtime = add_to_task_demand(rq, p, window_start - mark_start); -+ -+ /* Push new sample(s) into task's demand history */ -+ update_history(rq, p, sts->sum, 1, event); -+ if (nr_full_windows) { -+ u64 scaled_window = scale_exec_time(window_size, rq); -+ -+ update_history(rq, p, scaled_window, nr_full_windows, event); -+ runtime += nr_full_windows * scaled_window; -+ } -+ -+ /* -+ * Roll window_start back to current to process any remainder -+ * in current window. -+ */ -+ window_start += (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (wallclock - window_start) next */ -+ mark_start = window_start; -+ runtime += add_to_task_demand(rq, p, wallclock - mark_start); -+ -+ return runtime; -+} -+ -+u16 slim_walt_cpu_util(int cpu) -+{ -+ u64 prev_runnable_sum; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ prev_runnable_sum = srq->prev_runnable_sum; -+ do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT); -+ -+ return (u16)prev_runnable_sum; -+} -+ -+static DEFINE_PER_CPU(u16, prev_cpu_util); -+static inline void cpu_util_update_systrace_c(int cpu) -+{ -+ char buf[256]; -+ u16 cpu_util = slim_walt_cpu_util(cpu); -+ -+ if (cpu_util != per_cpu(prev_cpu_util, cpu)) { -+ snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n", -+ cpu, cpu_util); -+ tracing_mark_write(buf); -+ per_cpu(prev_cpu_util, cpu) = cpu_util; -+ } -+} -+ -+ -+static inline int account_busy_for_cpu_time(struct rq *rq, -+ struct task_struct *p, -+ int event) -+{ -+ return !is_idle_task(p) && (event == PUT_PREV_TASK -+ || event == TASK_UPDATE); -+} -+ -+ -+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ int new_window, full_window = 0; -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 window_start = srq->window_start; -+ u32 window_size = srq->prev_window_size; -+ u64 delta; -+ u64 *curr_runnable_sum = &srq->curr_runnable_sum; -+ u64 *prev_runnable_sum = &srq->prev_runnable_sum; -+ -+ new_window = mark_start < window_start; -+ if (new_window) -+ full_window = (window_start - mark_start) >= window_size; -+ -+ -+ if (!account_busy_for_cpu_time(rq, p, event)) -+ goto done; -+ -+ -+ if (!new_window) { -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. No rollover -+ * since we didn't start a new window. An example of this is -+ * when a task starts execution and then sleeps within the -+ * same window. -+ */ -+ delta = wallclock - mark_start; -+ -+ delta = scale_exec_time(delta, rq); -+ *curr_runnable_sum += delta; -+ -+ goto done; -+ } -+ -+ /* -+ * situations below this need window rollover, -+ * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done -+ * in update_window_start() -+ * -+ * For task counters curr/prev_window[_cpu] are rolled over in the early part of -+ * this function. If full_window(s) have expired and time since last update needs -+ * to be accounted as busy time, set the prev to a complete window size time, else -+ * add the prev window portion. -+ * -+ * For task curr counters a new window has begun, always assign -+ */ -+ -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. A new window -+ * must have been started in udpate_window_start() -+ * If any of these three above conditions are true -+ * then this busy time can't be accounted as irqtime. -+ * -+ * Busy time for the idle task need not be accounted. -+ * -+ * An example of this would be a task that starts execution -+ * and then sleeps once a new window has begun. -+ */ -+ -+ /* -+ * A full window hasn't elapsed, account partial -+ * contribution to previous completed window. -+ */ -+ -+ -+ delta = full_window ? scale_exec_time(window_size, rq) : -+ scale_exec_time(window_start - mark_start, rq); -+ -+ *prev_runnable_sum += delta; -+ -+ /* Account piece of busy time in the current window. */ -+ delta = scale_exec_time(wallclock - window_start, rq); -+ *curr_runnable_sum += delta; -+ -+done: -+ if (slim_walt_dump && new_window) -+ cpu_util_update_systrace_c(rq->cpu); -+} -+ -+ -+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq) -+{ -+ u64 result; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 new_window_start = srq->window_start; -+ -+ if (old_window_start == new_window_start) -+ return; -+ -+ result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws, -+ old_window_start, new_window_start); -+ if (result != old_window_start) -+ return; -+ -+ if (likely(cpu_online(raw_smp_processor_id()))) -+ irq_work_queue(&hmbird_slim_walt_irq_work); -+ else -+ irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask)); -+} -+ -+/* -+ * In the core scheduler, most of the load update points update the rq_clock after -+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of -+ * obtaining the time, but to prevent the subsequent migration of the load update -+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock -+ * update here. -+ */ -+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p, -+ struct rq *rq, int event) -+{ -+ if (!(rq->clock_update_flags & RQCF_UPDATED)) -+ update_rq_clock(rq); -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock)); -+} -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start; -+ -+ if (!slim_walt_ctrl) -+ return; -+ -+ if (!srq->window_start || sts->mark_start == wallclock) -+ return; -+ -+ old_window_start = update_window_start(rq, wallclock, event); -+ -+ if (!sts->window_start) -+ sts->window_start = srq->window_start; -+ -+ if (!sts->mark_start) -+ goto done; -+ -+ update_task_rq_cpu_cycles(p, rq, event, wallclock); -+ update_task_demand(p, rq, event, wallclock); -+ update_cpu_busy_time(p, rq, event, wallclock); -+ -+ sts->window_start = srq->window_start; -+ -+done: -+ sts->mark_start = wallclock; -+ slim_walt_window_rollover_run_once(old_window_start, rq); -+} -+ -+static void slim_walt_irq_work(struct irq_work *irq_work) -+{ -+ cpumask_t lock_cpus; -+ struct hmbird_sched_rq_stats *srq; -+ struct rq *rq; -+ int cpu; -+ int level = 0; -+ u64 wc; -+ unsigned long flags; -+ -+ cpumask_copy(&lock_cpus, cpu_possible_mask); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ if (level == 0) -+ raw_spin_lock(&cpu_rq(cpu)->__lock); -+ else -+ raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); -+ level++; -+ } -+ -+ wc = sched_clock(); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ rq = cpu_rq(cpu); -+ hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc); -+ } -+ -+ cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER); -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) { -+ srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id()); -+ if (wc < srq->window_start + new_hmbird_sched_ravg_window) -+ hmbird_sched_ravg_window = new_hmbird_sched_ravg_window; -+ } -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ raw_spin_unlock(&cpu_rq(cpu)->__lock); -+ } -+} -+static void hmbird_sched_init_rq(struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ srq->task_exec_scale = 1024; -+ srq->window_start = 0; -+} -+ -+void hmbird_sched_init_task(struct task_struct *p) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ memset(sts, 0, sizeof(struct hmbird_sched_task_stats)); -+} -+ -+static void hmbird_sched_stats_init(void) -+{ -+ unsigned long flags; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ raw_spin_lock_irqsave(&rq->__lock, flags); -+ hmbird_sched_init_rq(rq); -+ raw_spin_unlock_irqrestore(&rq->__lock, flags); -+ } -+ slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG; -+ -+ if (false == init_irq_work_inited) { -+ init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work); -+ init_irq_work_inited = true; -+ } -+} -+ -+void hmbird_scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (unlikely(!tick_sched_clock)) { -+ /* -+ * Let the window begin 20us prior to the tick, -+ * that way we are guaranteed a rollover when the tick occurs. -+ * Use rq->clock directly instead of rq_clock() since -+ * we do not have the rq lock and -+ * rq->clock was updated in the tick callpath. -+ */ -+ if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000)) -+ return; -+ for_each_possible_cpu(cpu) { -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ srq->window_start = tick_sched_clock; -+ } -+ atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock); -+ } -+} -+ -+void slim_walt_enable(int enable) -+{ -+ if (1 == !!enable) { -+ hmbird_sched_stats_init(); -+ WRITE_ONCE(tick_sched_clock, 0); -+ } else -+ slim_walt_ctrl = 0; -+} -+ -+void slim_get_cpu_util(int cpu, u64 *util) -+{ -+ if (cpu < 0) -+ return; -+ -+ *util = slim_walt_cpu_util(cpu); -+} -+ -+void slim_get_task_util(struct task_struct *p, u64 *util) -+{ -+ if (p == NULL) -+ return; -+ -+ *util = get_hmbird_ts(p)->sts.demand_scaled; -+} -+ -+void sched_ravg_window_change(int frame_per_sec) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec; -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+} -diff --git a/kernel/sched/hmbird/hmbird_util_track.h b/kernel/sched/hmbird/hmbird_util_track.h -new file mode 100755 -index 000000000000..17b85df7f83b ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.h -@@ -0,0 +1,33 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef __HMBIRD_UTIL_TRACK_H__ -+#define __HMBIRD_UTIL_TRACK_H__ -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock); -+void hmbird_sched_init_task(struct task_struct *p); -+void slim_walt_enable(int enable); -+void slim_get_cpu_util(int cpu, u64 *util); -+void slim_get_task_util(struct task_struct *p, u64 *util); -+ -+extern atomic64_t hmbird_irq_work_lastq_ws; -+ -+enum task_event { -+ PUT_PREV_TASK = 0, -+ PICK_NEXT_TASK = 1, -+ TASK_WAKE = 2, -+ TASK_MIGRATE = 3, -+ TASK_UPDATE = 4, -+ IRQ_UPDATE = 5, -+}; -+ -+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+#endif /* __HMBIRD_UTIL_TRACK_H__ */ -diff --git a/kernel/sched/hmbird/slim.h b/kernel/sched/hmbird/slim.h -new file mode 100755 -index 000000000000..eb386260e950 ---- /dev/null -+++ b/kernel/sched/hmbird/slim.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+ -+#ifndef __SLIM_H -+#define __SLIM_H -+ -+extern atomic_t __hmbird_ops_enabled; -+extern atomic_t non_hmbird_task; -+extern int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+extern int heartbeat; -+extern int heartbeat_enable; -+extern int watchdog_enable; -+extern int isolate_ctrl; -+extern int parctrl_high_ratio; -+extern int parctrl_low_ratio; -+extern int isoctrl_high_ratio; -+extern int isoctrl_low_ratio; -+extern int iso_free_rescue; -+extern int yield_opt; -+ -+extern enum hmbird_switch_type sw_type; -+extern noinline int tracing_mark_write(const char *buf); -+int task_top_id(struct task_struct *p); -+void stats_print(char *buf, int len); -+void hmbird_skip_yield(long *skip); -+extern spinlock_t hmbird_tasks_lock; -+ -+struct yield_opt_params { -+ int enable; -+ int frame_per_sec; -+ u64 frame_time_ns; -+ int yield_headroom; -+}; -+ -+extern struct yield_opt_params yield_opt_params; -+ -+#define MAX_GOV_LEN (16) -+extern char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+#endif -diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c -index b33cefeb4188..487255ac6832 100644 ---- a/kernel/sched/idle.c -+++ b/kernel/sched/idle.c -@@ -408,13 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl - - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) - { -- scx_update_idle(rq, false); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, false); -+#endif - } - - static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) - { - update_idle_core(rq); -- scx_update_idle(rq, true); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, true); -+#endif - schedstat_inc(rq->sched_goidle); - } - -diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h -index cbc478992cc4..e35473a1f0ea 100644 ---- a/kernel/sched/sched.h -+++ b/kernel/sched/sched.h -@@ -187,18 +187,13 @@ static inline int idle_policy(int policy) - return policy == SCHED_IDLE; - } - --static inline int normal_policy(int policy) --{ --#ifdef CONFIG_SCHED_CLASS_EXT -- if (policy == SCHED_EXT) -- return true; --#endif -- return policy == SCHED_NORMAL; --} -- - static inline int fair_policy(int policy) - { -- return normal_policy(policy) || policy == SCHED_BATCH; -+#ifdef CONFIG_HMBIRD_SCHED -+ return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#else -+ return policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#endif - } - - static inline int rt_policy(int policy) -@@ -246,24 +241,6 @@ static inline void update_avg(u64 *avg, u64 sample) - #define shr_bound(val, shift) \ - (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) - --/* -- * cgroup weight knobs should use the common MIN, DFL and MAX values which are -- * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it -- * maps pretty well onto the shares value used by scheduler and the round-trip -- * conversions preserve the original value over the entire range. -- */ --static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) --{ -- return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); --} -- --static inline unsigned long sched_weight_to_cgroup(unsigned long weight) --{ -- return clamp_t(unsigned long, -- DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -- CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); --} -- - /* - * !! For sched_setattr_nocheck() (kernel) only !! - * -@@ -531,10 +508,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) { } - #endif /* CONFIG_SMP */ - #else /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_HMBIRD_SCHED - static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) - { - return 0; - } -+#endif - #endif /* CONFIG_FAIR_GROUP_SCHED */ - - #else /* CONFIG_CGROUP_SCHED */ -@@ -699,29 +678,6 @@ struct cfs_rq { - #endif /* CONFIG_FAIR_GROUP_SCHED */ - }; - --#ifdef CONFIG_SCHED_CLASS_EXT --/* scx_rq->flags, protected by the rq lock */ --enum scx_rq_flags { -- SCX_RQ_CAN_STOP_TICK = 1 << 0, --}; -- --struct scx_rq { -- struct scx_dispatch_q local_dsq; -- struct list_head watchdog_list; -- u64 ops_qseq; -- u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -- u32 nr_running; -- u32 flags; -- bool cpu_released; -- cpumask_var_t cpus_to_kick; -- cpumask_var_t cpus_to_preempt; -- cpumask_var_t cpus_to_wait; -- u64 pnt_seq; -- struct irq_work kick_cpus_irq_work; -- struct rq *rq; --}; --#endif /* CONFIG_SCHED_CLASS_EXT */ -- - static inline int rt_bandwidth_enabled(void) - { - return sysctl_sched_rt_runtime >= 0; -@@ -1257,11 +1213,7 @@ struct rq { - - ANDROID_OEM_DATA_ARRAY(1, 16); - --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, struct scx_rq *scx); --#else - ANDROID_KABI_RESERVE(1); --#endif - ANDROID_KABI_RESERVE(2); - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); -@@ -2339,11 +2291,6 @@ struct affinity_context { - unsigned int flags; - }; - --enum rq_onoff_reason { -- RQ_ONOFF_HOTPLUG, /* CPU is going on/offline */ -- RQ_ONOFF_TOPOLOGY, /* sched domain topology update */ --}; -- - struct sched_class { - - #ifdef CONFIG_UCLAMP_TASK -@@ -2550,7 +2497,6 @@ extern void init_sched_rt_class(void); - extern void init_sched_fair_class(void); - - extern void reweight_task(struct task_struct *p, int prio); --extern void __setscheduler_prio(struct task_struct *p, int prio); - - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); -@@ -2631,53 +2577,6 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) - extern void activate_task(struct rq *rq, struct task_struct *p, int flags); - extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); - --struct sched_change_guard { -- struct task_struct *p; -- struct rq *rq; -- bool queued; -- bool running; -- bool done; --}; -- --extern struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -- --extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags); -- --/** -- * SCHED_CHANGE_BLOCK - Nested block for task attribute updates -- * @__rq: Runqueue the target task belongs to -- * @__p: Target task -- * @__flags: DEQUEUE/ENQUEUE_* flags -- * -- * A task may need to be dequeued and put_prev_task'd for attribute updates and -- * set_next_task'd and re-enqueued afterwards. This helper defines a nested -- * block which automatically handles these preparation and cleanup operations. -- * -- * SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- * update_attribute(p); -- * ... -- * } -- * -- * If @__flags is a variable, the variable may be updated in the block body and -- * the updated value will be used when re-enqueueing @p. -- * -- * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling -- * update_rq_clock() beforehand. Otherwise, the rq clock is automatically -- * updated iff the task needs to be dequeued and re-enqueued. Only the former -- * case guarantees that the rq clock is up-to-date inside and after the block. -- */ --#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -- for (struct sched_change_guard __cg = \ -- sched_change_guard_init(__rq, __p, __flags); \ -- !__cg.done; sched_change_guard_fini(&__cg, __flags)) -- --extern void check_class_changing(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class); --extern void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio); -- - extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); - - #ifdef CONFIG_PREEMPT_RT -@@ -3709,6 +3608,8 @@ static inline bool cpu_busy_with_softirqs(int cpu) - } - #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ - --#include "ext.h" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird.h" -+#endif - - #endif /* _KERNEL_SCHED_SCHED_H */ -diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c -index 9c556c716382..7a26c09182e8 100644 ---- a/kernel/time/tick-sched.c -+++ b/kernel/time/tick-sched.c -@@ -34,6 +34,10 @@ - - #include - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "../sched/hmbird/hmbird_shadow_tick.h" -+#endif -+ - /* - * Per-CPU nohz control structure - */ -@@ -1103,6 +1107,9 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) - return true; - } - -+#ifdef CONFIG_HMBIRD_SCHED -+extern void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+#endif - /** - * tick_nohz_idle_stop_tick - stop the idle tick from the idle task - * -@@ -1115,7 +1122,9 @@ void tick_nohz_idle_stop_tick(void) - ktime_t expires; - - trace_android_vh_tick_nohz_idle_stop_tick(NULL); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL); -+#endif - /* - * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the - * tick timer expiration time is known already. -diff --git a/tools/sched_ext/.gitignore b/tools/sched_ext/.gitignore -deleted file mode 100644 -index a3240f9f7eba..000000000000 ---- a/tools/sched_ext/.gitignore -+++ /dev/null -@@ -1,9 +0,0 @@ --scx_example_simple --scx_example_qmap --scx_example_central --scx_example_pair --scx_example_flatcg --scx_example_userland --*.skel.h --*.subskel.h --/tools/ -diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile -deleted file mode 100644 -index 73c43782837d..000000000000 ---- a/tools/sched_ext/Makefile -+++ /dev/null -@@ -1,216 +0,0 @@ --# SPDX-License-Identifier: GPL-2.0 --# Copyright (c) 2022 Meta Platforms, Inc. and affiliates. --include ../build/Build.include --include ../scripts/Makefile.arch --include ../scripts/Makefile.include -- --ifneq ($(LLVM),) --ifneq ($(filter %/,$(LLVM)),) --LLVM_PREFIX := $(LLVM) --else ifneq ($(filter -%,$(LLVM)),) --LLVM_SUFFIX := $(LLVM) --endif -- --CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi --CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu --CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl --CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu --CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu --CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu --CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu --CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu --CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu --CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) -- --ifeq ($(CROSS_COMPILE),) --ifeq ($(CLANG_TARGET_FLAGS),) --$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk --else --CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) --endif # CLANG_TARGET_FLAGS --else --CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) --endif # CROSS_COMPILE -- --CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as --else --CC := $(CROSS_COMPILE)gcc --endif # LLVM -- --CURDIR := $(abspath .) --TOOLSDIR := $(abspath ..) --LIBDIR := $(TOOLSDIR)/lib --BPFDIR := $(LIBDIR)/bpf --TOOLSINCDIR := $(TOOLSDIR)/include --BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool --APIDIR := $(TOOLSINCDIR)/uapi --GENDIR := $(abspath ../../include/generated) --GENHDR := $(GENDIR)/autoconf.h -- --SCRATCH_DIR := $(CURDIR)/tools --BUILD_DIR := $(SCRATCH_DIR)/build --INCLUDE_DIR := $(SCRATCH_DIR)/include --BPFOBJ_DIR := $(BUILD_DIR)/libbpf --BPFOBJ := $(BPFOBJ_DIR)/libbpf.a --ifneq ($(CROSS_COMPILE),) --HOST_BUILD_DIR := $(BUILD_DIR)/host --HOST_SCRATCH_DIR := host-tools --HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include --else --HOST_BUILD_DIR := $(BUILD_DIR) --HOST_SCRATCH_DIR := $(SCRATCH_DIR) --HOST_INCLUDE_DIR := $(INCLUDE_DIR) --endif --HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a --RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids --DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -- --VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ -- $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ -- ../../vmlinux \ -- /sys/kernel/btf/vmlinux \ -- /boot/vmlinux-$(shell uname -r) --VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) --ifeq ($(VMLINUX_BTF),) --$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") --endif -- --BPFTOOL ?= $(DEFAULT_BPFTOOL) -- --ifneq ($(wildcard $(GENHDR)),) -- GENFLAGS := -DHAVE_GENHDR --endif -- --CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ -- -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -- -I$(TOOLSINCDIR) -I$(APIDIR) -- --CARGOFLAGS := --release -- --# Silence some warnings when compiled with clang --ifneq ($(LLVM),) --CFLAGS += -Wno-unused-command-line-argument --endif -- --LDFLAGS = -lelf -lz -lpthread -- --IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - &1 \ -- | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ --$(shell $(1) -dM -E - $@ --else -- $(call msg,CP,,$@) -- $(Q)cp "$(VMLINUX_H)" $@ --endif -- --%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \ -- | $(BPFOBJ) -- $(call msg,CLNG-BPF,,$@) -- $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ -- --%.skel.h: %.bpf.o $(BPFTOOL) -- $(call msg,GEN-SKEL,,$@) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) -- $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) -- $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@ -- $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h) -- --scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_userland: scx_example_userland.c scx_example_userland.skel.h \ -- scx_example_userland_common.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR) --atropos: export ATROPOS_CLANG = $(CLANG) --atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS) --atropos: $(INCLUDE_DIR)/vmlinux.h -- cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS) -- --clean: -- cargo clean --manifest-path=atropos/Cargo.toml -- rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) -- rm -f *.o *.bpf.o *.skel.h *.subskel.h -- rm -f scx_example_simple scx_example_qmap scx_example_central \ -- scx_example_pair scx_example_flatcg scx_example_userland -- --.PHONY: all atropos clean -- --# delete failed targets --.DELETE_ON_ERROR: -- --# keep intermediate (.skel.h, .bpf.o, etc) targets --.SECONDARY: -diff --git a/tools/sched_ext/atropos/.gitignore b/tools/sched_ext/atropos/.gitignore -deleted file mode 100644 -index 186dba259ec2..000000000000 ---- a/tools/sched_ext/atropos/.gitignore -+++ /dev/null -@@ -1,3 +0,0 @@ --src/bpf/.output --Cargo.lock --target -diff --git a/tools/sched_ext/atropos/Cargo.toml b/tools/sched_ext/atropos/Cargo.toml -deleted file mode 100644 -index 7462a836d53d..000000000000 ---- a/tools/sched_ext/atropos/Cargo.toml -+++ /dev/null -@@ -1,28 +0,0 @@ --[package] --name = "scx_atropos" --version = "0.5.0" --authors = ["Dan Schatzberg ", "Meta"] --edition = "2021" --description = "Userspace scheduling with BPF" --license = "GPL-2.0-only" -- --[dependencies] --anyhow = "1.0.65" --bitvec = { version = "1.0", features = ["serde"] } --clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } --ctrlc = { version = "3.1", features = ["termination"] } --fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"} --hex = "0.4.3" --libbpf-rs = "0.19.1" --libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] } --libc = "0.2.137" --log = "0.4.17" --ordered-float = "3.4.0" --simplelog = "0.12.0" -- --[build-dependencies] --bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false } --libbpf-cargo = "0.13.0" -- --[features] --enable_backtrace = [] -diff --git a/tools/sched_ext/atropos/build.rs b/tools/sched_ext/atropos/build.rs -deleted file mode 100644 -index 26e792c5e17e..000000000000 ---- a/tools/sched_ext/atropos/build.rs -+++ /dev/null -@@ -1,70 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --extern crate bindgen; -- --use std::env; --use std::fs::create_dir_all; --use std::path::Path; --use std::path::PathBuf; -- --use libbpf_cargo::SkeletonBuilder; -- --const HEADER_PATH: &str = "src/bpf/atropos.h"; -- --fn bindgen_atropos() { -- // Tell cargo to invalidate the built crate whenever the wrapper changes -- println!("cargo:rerun-if-changed={}", HEADER_PATH); -- -- // The bindgen::Builder is the main entry point -- // to bindgen, and lets you build up options for -- // the resulting bindings. -- let bindings = bindgen::Builder::default() -- // The input header we would like to generate -- // bindings for. -- .header(HEADER_PATH) -- // Tell cargo to invalidate the built crate whenever any of the -- // included header files changed. -- .parse_callbacks(Box::new(bindgen::CargoCallbacks)) -- // Finish the builder and generate the bindings. -- .generate() -- // Unwrap the Result and panic on failure. -- .expect("Unable to generate bindings"); -- -- // Write the bindings to the $OUT_DIR/bindings.rs file. -- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); -- bindings -- .write_to_file(out_path.join("atropos-sys.rs")) -- .expect("Couldn't write bindings!"); --} -- --fn gen_bpf_sched(name: &str) { -- let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap(); -- let clang = env::var("ATROPOS_CLANG").unwrap(); -- eprintln!("{}", clang); -- let outpath = format!("./src/bpf/.output/{}.skel.rs", name); -- let skel = Path::new(&outpath); -- let src = format!("./src/bpf/{}.bpf.c", name); -- SkeletonBuilder::new() -- .source(src.clone()) -- .clang(clang) -- .clang_args(bpf_cflags) -- .build_and_generate(&skel) -- .unwrap(); -- println!("cargo:rerun-if-changed={}", src); --} -- --fn main() { -- bindgen_atropos(); -- // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton. -- // Reasons are because the generated skeleton contains compiler attributes -- // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]` -- // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside -- // the path attribute either (see https://github.com/rust-lang/rust/pull/83366). -- // -- // However, there is hope! When the above feature stabilizes we can clean this -- // all up. -- create_dir_all("./src/bpf/.output").unwrap(); -- gen_bpf_sched("atropos"); --} -diff --git a/tools/sched_ext/atropos/rustfmt.toml b/tools/sched_ext/atropos/rustfmt.toml -deleted file mode 100644 -index b7258ed0a8d8..000000000000 ---- a/tools/sched_ext/atropos/rustfmt.toml -+++ /dev/null -@@ -1,8 +0,0 @@ --# Get help on options with `rustfmt --help=config` --# Please keep these in alphabetical order. --edition = "2021" --group_imports = "StdExternalCrate" --imports_granularity = "Item" --merge_derives = false --use_field_init_shorthand = true --version = "Two" -diff --git a/tools/sched_ext/atropos/src/atropos_sys.rs b/tools/sched_ext/atropos/src/atropos_sys.rs -deleted file mode 100644 -index bbeaf856d40e..000000000000 ---- a/tools/sched_ext/atropos/src/atropos_sys.rs -+++ /dev/null -@@ -1,10 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#![allow(non_upper_case_globals)] --#![allow(non_camel_case_types)] --#![allow(non_snake_case)] --#![allow(dead_code)] -- --include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs")); -diff --git a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -deleted file mode 100644 -index 3905a403e940..000000000000 ---- a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -+++ /dev/null -@@ -1,743 +0,0 @@ --/* Copyright (c) Meta Platforms, Inc. and affiliates. */ --/* -- * This software may be used and distributed according to the terms of the -- * GNU General Public License version 2. -- * -- * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF -- * part does simple round robin in each domain and the userspace part -- * calculates the load factor of each domain and tells the BPF part how to load -- * balance the domains. -- * -- * Every task has an entry in the task_data map which lists which domain the -- * task belongs to. When a task first enters the system (atropos_prep_enable), -- * they are round-robined to a domain. -- * -- * atropos_select_cpu is the primary scheduling logic, invoked when a task -- * becomes runnable. The lb_data map is populated by userspace to inform the BPF -- * scheduler that a task should be migrated to a new domain. Otherwise, the task -- * is scheduled in priority order as follows: -- * * The current core if the task was woken up synchronously and there are idle -- * cpus in the system -- * * The previous core, if idle -- * * The pinned-to core if the task is pinned to a specific core -- * * Any idle cpu in the domain -- * -- * If none of the above conditions are met, then the task is enqueued to a -- * dispatch queue corresponding to the domain (atropos_enqueue). -- * -- * atropos_dispatch will attempt to consume a task from its domain's -- * corresponding dispatch queue (this occurs after scheduling any tasks directly -- * assigned to it due to the logic in atropos_select_cpu). If no task is found, -- * then greedy load stealing will attempt to find a task on another dispatch -- * queue to run. -- * -- * Load balancing is almost entirely handled by userspace. BPF populates the -- * task weight, dom mask and current dom in the task_data map and executes the -- * load balance based on userspace populating the lb_data map. -- */ --#include "../../../scx_common.bpf.h" --#include "atropos.h" -- --#include --#include --#include --#include --#include --#include -- --char _license[] SEC("license") = "GPL"; -- --/* -- * const volatiles are set during initialization and treated as consts by the -- * jit compiler. -- */ -- --/* -- * Domains and cpus -- */ --const volatile __u32 nr_doms = 32; /* !0 for veristat, set during init */ --const volatile __u32 nr_cpus = 64; /* !0 for veristat, set during init */ --const volatile __u32 cpu_dom_id_map[MAX_CPUS]; --const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; -- --const volatile bool kthreads_local; --const volatile bool fifo_sched; --const volatile bool switch_partial; --const volatile __u32 greedy_threshold; -- --/* base slice duration */ --const volatile __u64 slice_us = 20000; -- --/* -- * Exit info -- */ --int exit_type = SCX_EXIT_NONE; --char exit_msg[SCX_EXIT_MSG_LEN]; -- --struct pcpu_ctx { -- __u32 dom_rr_cur; /* used when scanning other doms */ -- -- /* libbpf-rs does not respect the alignment, so pad out the struct explicitly */ -- __u8 _padding[CACHELINE_SIZE - sizeof(u64)]; --} __attribute__((aligned(CACHELINE_SIZE))); -- --struct pcpu_ctx pcpu_ctx[MAX_CPUS]; -- --/* -- * Domain context -- */ --struct dom_ctx { -- struct bpf_cpumask __kptr *cpumask; -- u64 vtime_now; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __type(key, u32); -- __type(value, struct dom_ctx); -- __uint(max_entries, MAX_DOMS); -- __uint(map_flags, 0); --} dom_ctx SEC(".maps"); -- --/* -- * Statistics -- */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, ATROPOS_NR_STATS); --} stats SEC(".maps"); -- --static inline void stat_add(enum stat_idx idx, u64 addend) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p) += addend; --} -- --/* Map pid -> task_ctx */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, struct task_ctx); -- __uint(max_entries, 1000000); -- __uint(map_flags, 0); --} task_data SEC(".maps"); -- --/* -- * This is populated from userspace to indicate which pids should be reassigned -- * to new doms. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, u32); -- __uint(max_entries, 1000); -- __uint(map_flags, 0); --} lb_data SEC(".maps"); -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p, -- u32 new_dom_id) --{ -- struct dom_ctx *old_domc, *new_domc; -- struct bpf_cpumask *d_cpumask, *t_cpumask; -- u32 old_dom_id = task_ctx->dom_id; -- s64 vtime_delta; -- -- old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id); -- if (!old_domc) { -- scx_bpf_error("No dom%u", old_dom_id); -- return false; -- } -- -- vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now; -- -- new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id); -- if (!new_domc) { -- scx_bpf_error("No dom%u", new_dom_id); -- return false; -- } -- -- d_cpumask = new_domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to get domain %u cpumask kptr", -- new_dom_id); -- return false; -- } -- -- t_cpumask = task_ctx->cpumask; -- if (!t_cpumask) { -- scx_bpf_error("Failed to look up task cpumask"); -- return false; -- } -- -- /* -- * set_cpumask might have happened between userspace requesting LB and -- * here and @p might not be able to run in @dom_id anymore. Verify. -- */ -- if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- p->cpus_ptr)) { -- p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta; -- task_ctx->dom_id = new_dom_id; -- bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask, -- p->cpus_ptr); -- } -- -- return task_ctx->dom_id == new_dom_id; --} -- --s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu, -- u32 wake_flags) --{ -- s32 cpu; -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- struct bpf_cpumask *p_cpumask; -- -- if (!task_ctx) -- return -ENOENT; -- -- if (kthreads_local && -- (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- cpu = prev_cpu; -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local dsq of the waker. -- */ -- if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) { -- struct task_struct *current = (void *)bpf_get_current_task(); -- -- if (!(BPF_CORE_READ(current, flags) & PF_EXITING) && -- task_ctx->dom_id < MAX_DOMS) { -- struct dom_ctx *domc; -- struct bpf_cpumask *d_cpumask; -- const struct cpumask *idle_cpumask; -- bool has_idle; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id); -- if (!domc) { -- scx_bpf_error("Failed to find dom%u", -- task_ctx->dom_id); -- return prev_cpu; -- } -- d_cpumask = domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to acquire domain %u cpumask kptr", -- task_ctx->dom_id); -- return prev_cpu; -- } -- -- idle_cpumask = scx_bpf_get_idle_cpumask(); -- -- has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- idle_cpumask); -- -- scx_bpf_put_idle_cpumask(idle_cpumask); -- -- if (has_idle) { -- cpu = bpf_get_smp_processor_id(); -- if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- stat_add(ATROPOS_STAT_WAKE_SYNC, 1); -- goto local; -- } -- } -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- stat_add(ATROPOS_STAT_PREV_IDLE, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- /* If only one core is allowed, dispatch */ -- if (p->nr_cpus_allowed == 1) { -- stat_add(ATROPOS_STAT_PINNED, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) -- return -ENOENT; -- -- /* If there is an eligible idle CPU, dispatch directly */ -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- if (cpu >= 0) { -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * @prev_cpu may be in a different domain. Returning an out-of-domain -- * CPU can lead to stalls as all in-domain CPUs may be idle by the time -- * @p gets enqueued. -- */ -- if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask)) -- cpu = prev_cpu; -- else -- cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask); -- -- return cpu; -- --local: -- task_ctx->dispatch_local = true; -- return cpu; --} -- --void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- u32 *new_dom; -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- new_dom = bpf_map_lookup_elem(&lb_data, &pid); -- if (new_dom && *new_dom != task_ctx->dom_id && -- task_set_dsq(task_ctx, p, *new_dom)) { -- struct bpf_cpumask *p_cpumask; -- s32 cpu; -- -- stat_add(ATROPOS_STAT_LOAD_BALANCE, 1); -- -- /* -- * If dispatch_local is set, We own @p's idle state but we are -- * not gonna put the task in the associated local dsq which can -- * cause the CPU to stall. Kick it. -- */ -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0); -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) { -- scx_bpf_error("Failed to get task_ctx->cpumask"); -- return; -- } -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- } -- -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags); -- return; -- } -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000, -- enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- u32 dom_id = task_ctx->dom_id; -- struct dom_ctx *domc; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, domc->vtime_now - slice_us * 1000)) -- vtime = domc->vtime_now - slice_us * 1000; -- -- scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --static u32 cpu_to_dom_id(s32 cpu) --{ -- const volatile u32 *dom_idp; -- -- if (nr_doms <= 1) -- return 0; -- -- dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]); -- if (!dom_idp) -- return MAX_DOMS; -- -- return *dom_idp; --} -- --static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) --{ -- s32 cpu; -- -- if (dom_id >= MAX_DOMS) -- return false; -- -- bpf_for(cpu, 0, nr_cpus) { -- if (bpf_cpumask_test_cpu(cpu, cpumask) && -- (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64)))) -- return true; -- } -- return false; --} -- --static u32 dom_rr_next(s32 cpu) --{ -- struct pcpu_ctx *pcpuc; -- u32 dom_id; -- -- pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]); -- if (!pcpuc) -- return 0; -- -- dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms; -- -- if (dom_id == cpu_to_dom_id(cpu)) -- dom_id = (dom_id + 1) % nr_doms; -- -- pcpuc->dom_rr_cur = dom_id; -- return dom_id; --} -- --void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 dom = cpu_to_dom_id(cpu); -- -- if (scx_bpf_consume(dom)) { -- stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1); -- return; -- } -- -- if (!greedy_threshold) -- return; -- -- bpf_repeat(nr_doms - 1) { -- u32 dom_id = dom_rr_next(cpu); -- -- if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold && -- scx_bpf_consume(dom_id)) { -- stat_add(ATROPOS_STAT_GREEDY, 1); -- break; -- } -- } --} -- --void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_at = bpf_ktime_get_ns(); --} -- --void BPF_STRUCT_OPS(atropos_running, struct task_struct *p) --{ -- struct task_ctx *taskc; -- struct dom_ctx *domc; -- pid_t pid = p->pid; -- u32 dom_id; -- -- if (fifo_sched) -- return; -- -- taskc = bpf_map_lookup_elem(&task_data, &pid); -- if (!taskc) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- dom_id = taskc->dom_id; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(domc->vtime_now, p->scx.dsq_vtime)) -- domc->vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at; -- task_ctx->runnable_at = 0; --} -- --void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->weight = weight; --} -- --struct pick_task_domain_loop_ctx { -- struct task_struct *p; -- const struct cpumask *cpumask; -- u64 dom_mask; -- u32 dom_rr_base; -- u32 dom_id; --}; -- --static int pick_task_domain_loopfn(u32 idx, void *data) --{ -- struct pick_task_domain_loop_ctx *lctx = data; -- u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms; -- -- if (dom_id >= MAX_DOMS) -- return 1; -- -- if (cpumask_intersects_domain(lctx->cpumask, dom_id)) { -- lctx->dom_mask |= 1LLU << dom_id; -- if (lctx->dom_id == MAX_DOMS) -- lctx->dom_id = dom_id; -- } -- return 0; --} -- --static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- struct pick_task_domain_loop_ctx lctx = { -- .p = p, -- .cpumask = cpumask, -- .dom_id = MAX_DOMS, -- }; -- s32 cpu = bpf_get_smp_processor_id(); -- -- if (cpu < 0 || cpu >= MAX_CPUS) -- return MAX_DOMS; -- -- lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur); -- -- bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0); -- task_ctx->dom_mask = lctx.dom_mask; -- -- return lctx.dom_id; --} -- --static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- u32 dom_id = 0; -- -- if (nr_doms > 1) -- dom_id = pick_task_domain(task_ctx, p, cpumask); -- -- if (!task_set_dsq(task_ctx, p, dom_id)) -- scx_bpf_error("Failed to set domain %d for %s[%d]", -- dom_id, p->comm, p->pid); --} -- --void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_set_domain(task_ctx, p, cpumask); --} -- --s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct bpf_cpumask *cpumask; -- struct task_ctx task_ctx, *map_value; -- long ret; -- pid_t pid; -- -- memset(&task_ctx, 0, sizeof(task_ctx)); -- -- pid = p->pid; -- ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return ret; -- } -- -- /* -- * Read the entry from the map immediately so we can add the cpumask -- * with bpf_kptr_xchg(). -- */ -- map_value = bpf_map_lookup_elem(&task_data, &pid); -- if (!map_value) -- /* Should never happen -- it was just inserted above. */ -- return -EINVAL; -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- bpf_map_delete_elem(&task_data, &pid); -- return -ENOMEM; -- } -- -- cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask); -- if (cpumask) { -- /* Should never happen as we just inserted it above. */ -- bpf_cpumask_release(cpumask); -- bpf_map_delete_elem(&task_data, &pid); -- return -EINVAL; -- } -- -- task_set_domain(map_value, p, p->cpus_ptr); -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p) --{ -- pid_t pid = p->pid; -- long ret = bpf_map_delete_elem(&task_data, &pid); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return; -- } --} -- --static int create_dom_dsq(u32 idx, void *data) --{ -- struct dom_ctx domc_init = {}, *domc; -- struct bpf_cpumask *cpumask; -- u32 cpu, dom_id = idx; -- s32 ret; -- -- ret = scx_bpf_create_dsq(dom_id, -1); -- if (ret < 0) { -- scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret); -- return 1; -- } -- -- ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0); -- if (ret) { -- scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret); -- return 1; -- } -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- /* Should never happen, we just inserted it above. */ -- scx_bpf_error("No dom%u", dom_id); -- return 1; -- } -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id); -- return 1; -- } -- -- for (cpu = 0; cpu < MAX_CPUS; cpu++) { -- const volatile __u64 *dmask; -- -- dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]); -- if (!dmask) { -- scx_bpf_error("array index error"); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- if (*dmask & (1LLU << (cpu % 64))) -- bpf_cpumask_set_cpu(cpu, cpumask); -- } -- -- cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask); -- if (cpumask) { -- scx_bpf_error("Domain %u was already present", dom_id); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(atropos_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- bpf_loop(nr_doms, create_dom_dsq, NULL, 0); -- -- for (u32 i = 0; i < nr_cpus; i++) -- pcpu_ctx[i].dom_rr_cur = i; -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg); -- exit_type = ei->type; --} -- --SEC(".struct_ops") --struct sched_ext_ops atropos = { -- .select_cpu = (void *)atropos_select_cpu, -- .enqueue = (void *)atropos_enqueue, -- .dispatch = (void *)atropos_dispatch, -- .runnable = (void *)atropos_runnable, -- .running = (void *)atropos_running, -- .stopping = (void *)atropos_stopping, -- .quiescent = (void *)atropos_quiescent, -- .set_weight = (void *)atropos_set_weight, -- .set_cpumask = (void *)atropos_set_cpumask, -- .prep_enable = (void *)atropos_prep_enable, -- .disable = (void *)atropos_disable, -- .init = (void *)atropos_init, -- .exit = (void *)atropos_exit, -- .flags = 0, -- .name = "atropos", --}; -diff --git a/tools/sched_ext/atropos/src/bpf/atropos.h b/tools/sched_ext/atropos/src/bpf/atropos.h -deleted file mode 100644 -index addf29ca104a..000000000000 ---- a/tools/sched_ext/atropos/src/bpf/atropos.h -+++ /dev/null -@@ -1,44 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#ifndef __ATROPOS_H --#define __ATROPOS_H -- --#include --#ifndef __kptr --#ifdef __KERNEL__ --#error "__kptr_ref not defined in the kernel" --#endif --#define __kptr --#endif -- --#define MAX_CPUS 512 --#define MAX_DOMS 64 /* limited to avoid complex bitmask ops */ --#define CACHELINE_SIZE 64 -- --/* Statistics */ --enum stat_idx { -- ATROPOS_STAT_TASK_GET_ERR, -- ATROPOS_STAT_WAKE_SYNC, -- ATROPOS_STAT_PREV_IDLE, -- ATROPOS_STAT_PINNED, -- ATROPOS_STAT_DIRECT_DISPATCH, -- ATROPOS_STAT_DSQ_DISPATCH, -- ATROPOS_STAT_GREEDY, -- ATROPOS_STAT_LOAD_BALANCE, -- ATROPOS_STAT_LAST_TASK, -- ATROPOS_NR_STATS, --}; -- --struct task_ctx { -- unsigned long long dom_mask; /* the domains this task can run on */ -- struct bpf_cpumask __kptr *cpumask; -- unsigned int dom_id; -- unsigned int weight; -- unsigned long long runnable_at; -- unsigned long long runnable_for; -- bool dispatch_local; --}; -- --#endif /* __ATROPOS_H */ -diff --git a/tools/sched_ext/atropos/src/main.rs b/tools/sched_ext/atropos/src/main.rs -deleted file mode 100644 -index 0d313662f713..000000000000 ---- a/tools/sched_ext/atropos/src/main.rs -+++ /dev/null -@@ -1,942 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#[path = "bpf/.output/atropos.skel.rs"] --mod atropos; --pub use atropos::*; --pub mod atropos_sys; -- --use std::cell::Cell; --use std::collections::{BTreeMap, BTreeSet}; --use std::ffi::CStr; --use std::ops::Bound::{Included, Unbounded}; --use std::sync::atomic::{AtomicBool, Ordering}; --use std::sync::Arc; --use std::time::{Duration, SystemTime}; -- --use ::fb_procfs as procfs; --use anyhow::{anyhow, bail, Context, Result}; --use bitvec::prelude::*; --use clap::Parser; --use log::{info, trace, warn}; --use ordered_float::OrderedFloat; -- --/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF --/// part does simple round robin in each domain and the userspace part --/// calculates the load factor of each domain and tells the BPF part how to load --/// balance the domains. -- --/// This scheduler demonstrates dividing scheduling logic between BPF and --/// userspace and using rust to build the userspace part. An earlier variant of --/// this scheduler was used to balance across six domains, each representing a --/// chiplet in a six-chiplet AMD processor, and could match the performance of --/// production setup using CFS. --#[derive(Debug, Parser)] --struct Opts { -- /// Scheduling slice duration in microseconds. -- #[clap(short, long, default_value = "20000")] -- slice_us: u64, -- -- /// Monitoring and load balance interval in seconds. -- #[clap(short, long, default_value = "2.0")] -- interval: f64, -- -- /// Build domains according to how CPUs are grouped at this cache level -- /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id. -- #[clap(short = 'c', long, default_value = "3")] -- cache_level: u32, -- -- /// Instead of using cache locality, set the cpumask for each domain -- /// manually, provide multiple --cpumasks, one for each domain. E.g. -- /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with -- /// the corresponding CPUs belonging to each domain. Each CPU must -- /// belong to precisely one domain. -- #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")] -- cpumasks: Vec, -- -- /// When non-zero, enable greedy task stealing. When a domain is idle, a -- /// cpu will attempt to steal tasks from a domain with at least -- /// greedy_threshold tasks enqueued. These tasks aren't permanently -- /// stolen from the domain. -- #[clap(short, long, default_value = "4")] -- greedy_threshold: u32, -- -- /// The load decay factor. Every interval, the existing load is decayed -- /// by this factor and new load is added. Must be in the range [0.0, -- /// 0.99]. The smaller the value, the more sensitive load calculation -- /// is to recent changes. When 0.0, history is ignored and the load -- /// value from the latest period is used directly. -- #[clap(short, long, default_value = "0.5")] -- load_decay_factor: f64, -- -- /// Disable load balancing. Unless disabled, periodically userspace will -- /// calculate the load factor of each domain and instruct BPF which -- /// processes to move. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- no_load_balance: bool, -- -- /// Put per-cpu kthreads directly into local dsq's. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- kthreads_local: bool, -- -- /// Use FIFO scheduling instead of weighted vtime scheduling. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- fifo_sched: bool, -- -- /// If specified, only tasks which have their scheduling policy set to -- /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all -- /// tasks are switched. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- partial: bool, -- -- /// Enable verbose output including libbpf details. Specify multiple -- /// times to increase verbosity. -- #[clap(short, long, action = clap::ArgAction::Count)] -- verbose: u8, --} -- --fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result { -- Ok(reader -- .read_stat() -- .context("Failed to read procfs")? -- .total_cpu -- .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?) --} -- --fn now_monotonic() -> u64 { -- let mut time = libc::timespec { -- tv_sec: 0, -- tv_nsec: 0, -- }; -- let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) }; -- assert!(ret == 0); -- time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 --} -- --fn clear_map(map: &mut libbpf_rs::Map) { -- // XXX: libbpf_rs has some design flaw that make it impossible to -- // delete while iterating despite it being safe so we alias it here -- let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) }; -- for key in map.keys() { -- let _ = deleter.delete(&key); -- } --} -- --#[derive(Debug)] --struct TaskLoad { -- runnable_for: u64, -- load: f64, --} -- --#[derive(Debug)] --struct TaskInfo { -- pid: i32, -- dom_mask: u64, -- migrated: Cell, --} -- --struct LoadBalancer<'a, 'b, 'c> { -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- -- tasks_by_load: Vec, TaskInfo>>, -- load_avg: f64, -- dom_loads: Vec, -- -- imbal: Vec, -- doms_to_push: BTreeMap, u32>, -- doms_to_pull: BTreeMap, u32>, -- -- nr_lb_data_errors: &'c mut u64, --} -- --impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> { -- const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10; -- const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1; -- const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50; -- -- fn new( -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- nr_lb_data_errors: &'c mut u64, -- ) -> Self { -- Self { -- maps, -- task_loads, -- nr_doms, -- load_decay_factor, -- -- tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(), -- load_avg: 0f64, -- dom_loads: vec![0.0; nr_doms], -- -- imbal: vec![0.0; nr_doms], -- doms_to_pull: BTreeMap::new(), -- doms_to_push: BTreeMap::new(), -- -- nr_lb_data_errors, -- } -- } -- -- fn read_task_loads(&mut self, period: Duration) -> Result<()> { -- let now_mono = now_monotonic(); -- let task_data = self.maps.task_data(); -- let mut this_task_loads = BTreeMap::::new(); -- let mut load_sum = 0.0f64; -- self.dom_loads = vec![0f64; self.nr_doms]; -- -- for key in task_data.keys() { -- if let Some(task_ctx_vec) = task_data -- .lookup(&key, libbpf_rs::MapFlags::ANY) -- .context("Failed to lookup task_data")? -- { -- let task_ctx = -- unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) }; -- let pid = i32::from_ne_bytes( -- key.as_slice() -- .try_into() -- .context("Invalid key length in task_data map")?, -- ); -- -- let (this_at, this_for, weight) = unsafe { -- ( -- std::ptr::read_volatile(&task_ctx.runnable_at as *const u64), -- std::ptr::read_volatile(&task_ctx.runnable_for as *const u64), -- std::ptr::read_volatile(&task_ctx.weight as *const u32), -- ) -- }; -- -- let (mut delta, prev_load) = match self.task_loads.get(&pid) { -- Some(prev) => (this_for - prev.runnable_for, Some(prev.load)), -- None => (this_for, None), -- }; -- -- // Non-zero this_at indicates that the task is currently -- // runnable. Note that we read runnable_at and runnable_for -- // without any synchronization and there is a small window -- // where we end up misaccounting. While this can cause -- // temporary error, it's unlikely to cause any noticeable -- // misbehavior especially given the load value clamping. -- if this_at > 0 && this_at < now_mono { -- delta += now_mono - this_at; -- } -- -- delta = delta.min(period.as_nanos() as u64); -- let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64) -- .clamp(0.0, weight as f64); -- -- let this_load = match prev_load { -- Some(prev_load) => { -- prev_load * self.load_decay_factor -- + this_load * (1.0 - self.load_decay_factor) -- } -- None => this_load, -- }; -- -- this_task_loads.insert( -- pid, -- TaskLoad { -- runnable_for: this_for, -- load: this_load, -- }, -- ); -- -- load_sum += this_load; -- self.dom_loads[task_ctx.dom_id as usize] += this_load; -- // Only record pids that are eligible for load balancing -- if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) { -- continue; -- } -- self.tasks_by_load[task_ctx.dom_id as usize].insert( -- OrderedFloat(this_load), -- TaskInfo { -- pid, -- dom_mask: task_ctx.dom_mask, -- migrated: Cell::new(false), -- }, -- ); -- } -- } -- -- self.load_avg = load_sum / self.nr_doms as f64; -- *self.task_loads = this_task_loads; -- Ok(()) -- } -- -- // To balance dom loads we identify doms with lower and higher load than average -- fn calculate_dom_load_balance(&mut self) -> Result<()> { -- for (dom, dom_load) in self.dom_loads.iter().enumerate() { -- let imbal = dom_load - self.load_avg; -- if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO { -- if imbal > 0f64 { -- self.doms_to_push.insert(OrderedFloat(imbal), dom as u32); -- } else { -- self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32); -- } -- self.imbal[dom] = imbal; -- } -- } -- Ok(()) -- } -- -- // Find the first candidate pid which hasn't already been migrated and -- // can run in @pull_dom. -- fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)> -- where -- I: IntoIterator, &'d TaskInfo)>, -- { -- match tasks_by_load -- .into_iter() -- .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0) -- .next() -- { -- Some((OrderedFloat(load), task)) => Some((*load, task)), -- None => None, -- } -- } -- -- fn pick_victim( -- &self, -- (push_dom, to_push): (u32, f64), -- (pull_dom, to_pull): (u32, f64), -- ) -> Option<(&TaskInfo, f64)> { -- let to_xfer = to_pull.min(to_push); -- -- trace!( -- "considering dom {}@{:.2} -> {}@{:.2}", -- push_dom, -- to_push, -- pull_dom, -- to_pull -- ); -- -- let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs(); -- -- trace!( -- "to_xfer={:.2} tasks_by_load={:?}", -- to_xfer, -- &self.tasks_by_load[push_dom as usize] -- ); -- -- // We want to pick a task to transfer from push_dom to pull_dom to -- // maximize the reduction of load imbalance between the two. IOW, -- // pick a task which has the closest load value to $to_xfer that can -- // be migrated. Find such task by locating the first migratable task -- // while scanning left from $to_xfer and the counterpart while -- // scanning right and picking the better of the two. -- let (load, task, new_imbal) = match ( -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Unbounded, Included(&OrderedFloat(to_xfer)))) -- .rev(), -- pull_dom, -- ), -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Included(&OrderedFloat(to_xfer)), Unbounded)), -- pull_dom, -- ), -- ) { -- (None, None) => return None, -- (Some((load, task)), None) | (None, Some((load, task))) => { -- (load, task, calc_new_imbal(load)) -- } -- (Some((load0, task0)), Some((load1, task1))) => { -- let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1)); -- if new_imbal0 <= new_imbal1 { -- (load0, task0, new_imbal0) -- } else { -- (load1, task1, new_imbal1) -- } -- } -- }; -- -- // If the best candidate can't reduce the imbalance, there's nothing -- // to do for this pair. -- let old_imbal = to_push + to_pull; -- if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal { -- trace!( -- "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal -- ); -- return None; -- } -- -- trace!( -- "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal, -- ); -- -- Some((task, load)) -- } -- -- // Actually execute the load balancing. Concretely this writes pid -> dom -- // entries into the lb_data map for bpf side to consume. -- fn load_balance(&mut self) -> Result<()> { -- clear_map(self.maps.lb_data()); -- -- trace!("imbal={:?}", &self.imbal); -- trace!("doms_to_push={:?}", &self.doms_to_push); -- trace!("doms_to_pull={:?}", &self.doms_to_pull); -- -- // Push from the most imbalanced to least. -- while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() { -- let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO; -- let mut pushed = 0f64; -- -- // Transfer tasks from push_dom to reduce imbalance. -- loop { -- let last_pushed = pushed; -- -- // Pull from the most imbalaned to least. -- let mut doms_to_pull = BTreeMap::<_, _>::new(); -- std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull); -- let mut pull_doms = doms_to_pull.into_iter().rev().collect::>(); -- -- for (to_pull, pull_dom) in pull_doms.iter_mut() { -- if let Some((task, load)) = -- self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull))) -- { -- // Execute migration. -- task.migrated.set(true); -- to_push -= load; -- *to_pull -= load; -- pushed += load; -- -- // Ask BPF code to execute the migration. -- let pid = task.pid; -- let cpid = (pid as libc::pid_t).to_ne_bytes(); -- if let Err(e) = self.maps.lb_data().update( -- &cpid, -- &pull_dom.to_ne_bytes(), -- libbpf_rs::MapFlags::NO_EXIST, -- ) { -- warn!( -- "Failed to update lb_data map for pid={} error={:?}", -- pid, &e -- ); -- *self.nr_lb_data_errors += 1; -- } -- -- // Always break after a successful migration so that -- // the pulling domains are always considered in the -- // descending imbalance order. -- break; -- } -- } -- -- pull_doms -- .into_iter() -- .map(|(k, v)| self.doms_to_pull.insert(k, v)) -- .count(); -- -- // Stop repeating if nothing got transferred or pushed enough. -- if pushed == last_pushed || pushed >= push_max { -- break; -- } -- } -- } -- Ok(()) -- } --} -- --struct Scheduler<'a> { -- skel: AtroposSkel<'a>, -- struct_ops: Option, -- -- nr_cpus: usize, -- nr_doms: usize, -- load_decay_factor: f64, -- balance_load: bool, -- -- proc_reader: procfs::ProcReader, -- -- prev_at: SystemTime, -- prev_total_cpu: procfs::CpuStat, -- task_loads: BTreeMap, -- -- nr_lb_data_errors: u64, --} -- --impl<'a> Scheduler<'a> { -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn parse_cpusets( -- cpumasks: &[String], -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- if cpumasks.len() > atropos_sys::MAX_DOMS as usize { -- bail!( -- "Number of requested DSQs ({}) is greater than MAX_DOMS ({})", -- cpumasks.len(), -- atropos_sys::MAX_DOMS -- ); -- } -- let mut cpus = vec![-1i32; nr_cpus]; -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()]; -- for (dq, cpumask) in cpumasks.iter().enumerate() { -- let hex_str = { -- let mut tmp_str = cpumask -- .strip_prefix("0x") -- .unwrap_or(cpumask) -- .replace('_', ""); -- if tmp_str.len() % 2 != 0 { -- tmp_str = "0".to_string() + &tmp_str; -- } -- tmp_str -- }; -- let byte_vec = hex::decode(&hex_str) -- .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?; -- -- for (index, &val) in byte_vec.iter().rev().enumerate() { -- let mut v = val; -- while v != 0 { -- let lsb = v.trailing_zeros() as usize; -- v &= !(1 << lsb); -- let cpu = index * 8 + lsb; -- if cpu > nr_cpus { -- bail!( -- concat!( -- "Found cpu ({}) in cpumask ({}) which is larger", -- " than the number of cpus on the machine ({})" -- ), -- cpu, -- cpumask, -- nr_cpus -- ); -- } -- if cpus[cpu] != -1 { -- bail!( -- "Found cpu ({}) with dq ({}) but also in cpumask ({})", -- cpu, -- cpus[cpu], -- cpumask -- ); -- } -- cpus[cpu] = dq as i32; -- cpusets[dq].set(cpu, true); -- } -- } -- cpusets[dq].set_uninitialized(false); -- } -- -- for (cpu, &dq) in cpus.iter().enumerate() { -- if dq < 0 { -- bail!( -- "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.", -- cpu -- ); -- } -- } -- -- Ok((cpusets, cpus)) -- } -- -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn cpusets_from_cache( -- level: u32, -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- let mut cpu_to_cache = vec![]; // (cpu_id, cache_id) -- let mut cache_ids = BTreeSet::::new(); -- let mut nr_not_found = 0; -- -- // Build cpu -> cache ID mapping. -- for cpu in 0..nr_cpus { -- let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level); -- let id = match std::fs::read_to_string(&path) { -- Ok(val) => val -- .trim() -- .parse::() -- .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?, -- Err(e) if e.kind() == std::io::ErrorKind::NotFound => { -- nr_not_found += 1; -- 0 -- } -- Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)), -- }; -- -- cpu_to_cache.push(id); -- cache_ids.insert(id); -- } -- -- if nr_not_found > 1 { -- warn!( -- "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0", -- level, nr_not_found, nr_cpus -- ); -- } -- -- // Cache IDs may have holes. Assign consecutive domain IDs to -- // existing cache IDs. -- let mut cache_to_dom = BTreeMap::::new(); -- let mut nr_doms = 0; -- for cache_id in cache_ids.iter() { -- cache_to_dom.insert(*cache_id, nr_doms); -- nr_doms += 1; -- } -- -- if nr_doms > atropos_sys::MAX_DOMS { -- bail!( -- "Total number of doms {} is greater than MAX_DOMS ({})", -- nr_doms, -- atropos_sys::MAX_DOMS -- ); -- } -- -- // Build and return dom -> cpumask and cpu -> dom mappings. -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize]; -- let mut cpu_to_dom = vec![]; -- -- for cpu in 0..nr_cpus { -- let dom_id = cache_to_dom[&cpu_to_cache[cpu]]; -- cpusets[dom_id as usize].set(cpu, true); -- cpu_to_dom.push(dom_id as i32); -- } -- -- Ok((cpusets, cpu_to_dom)) -- } -- -- fn init(opts: &Opts) -> Result { -- // Open the BPF prog first for verification. -- let mut skel_builder = AtroposSkelBuilder::default(); -- skel_builder.obj_builder.debug(opts.verbose > 0); -- let mut skel = skel_builder.open().context("Failed to open BPF program")?; -- -- let nr_cpus = libbpf_rs::num_possible_cpus().unwrap(); -- if nr_cpus > atropos_sys::MAX_CPUS as usize { -- bail!( -- "nr_cpus ({}) is greater than MAX_CPUS ({})", -- nr_cpus, -- atropos_sys::MAX_CPUS -- ); -- } -- -- // Initialize skel according to @opts. -- let (cpusets, cpus) = if opts.cpumasks.len() > 0 { -- Self::parse_cpusets(&opts.cpumasks, nr_cpus)? -- } else { -- Self::cpusets_from_cache(opts.cache_level, nr_cpus)? -- }; -- let nr_doms = cpusets.len(); -- skel.rodata().nr_doms = nr_doms as u32; -- skel.rodata().nr_cpus = nr_cpus as u32; -- -- for (cpu, dom) in cpus.iter().enumerate() { -- skel.rodata().cpu_dom_id_map[cpu] = *dom as u32; -- } -- -- for (dom, cpuset) in cpusets.iter().enumerate() { -- let raw_cpuset_slice = cpuset.as_raw_slice(); -- let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom]; -- let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len()); -- left.clone_from_slice(cpuset.as_raw_slice()); -- let cpumask_str = dom_cpumask_slice -- .iter() -- .take((nr_cpus + 63) / 64) -- .rev() -- .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x)); -- info!( -- "DOM[{:02}] cpumask{} ({} cpus)", -- dom, -- &cpumask_str, -- cpuset.count_ones() -- ); -- } -- -- skel.rodata().slice_us = opts.slice_us; -- skel.rodata().kthreads_local = opts.kthreads_local; -- skel.rodata().fifo_sched = opts.fifo_sched; -- skel.rodata().switch_partial = opts.partial; -- skel.rodata().greedy_threshold = opts.greedy_threshold; -- -- // Attach. -- let mut skel = skel.load().context("Failed to load BPF program")?; -- skel.attach().context("Failed to attach BPF program")?; -- let struct_ops = Some( -- skel.maps_mut() -- .atropos() -- .attach_struct_ops() -- .context("Failed to attach atropos struct ops")?, -- ); -- info!("Atropos Scheduler Attached"); -- -- // Other stuff. -- let mut proc_reader = procfs::ProcReader::new(); -- let prev_total_cpu = read_total_cpu(&mut proc_reader)?; -- -- Ok(Self { -- skel, -- struct_ops, // should be held to keep it attached -- -- nr_cpus, -- nr_doms, -- load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99), -- balance_load: !opts.no_load_balance, -- -- proc_reader, -- -- prev_at: SystemTime::now(), -- prev_total_cpu, -- task_loads: BTreeMap::new(), -- -- nr_lb_data_errors: 0, -- }) -- } -- -- fn get_cpu_busy(&mut self) -> Result { -- let total_cpu = read_total_cpu(&mut self.proc_reader)?; -- let busy = match (&self.prev_total_cpu, &total_cpu) { -- ( -- procfs::CpuStat { -- user_usec: Some(prev_user), -- nice_usec: Some(prev_nice), -- system_usec: Some(prev_system), -- idle_usec: Some(prev_idle), -- iowait_usec: Some(prev_iowait), -- irq_usec: Some(prev_irq), -- softirq_usec: Some(prev_softirq), -- stolen_usec: Some(prev_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- procfs::CpuStat { -- user_usec: Some(curr_user), -- nice_usec: Some(curr_nice), -- system_usec: Some(curr_system), -- idle_usec: Some(curr_idle), -- iowait_usec: Some(curr_iowait), -- irq_usec: Some(curr_irq), -- softirq_usec: Some(curr_softirq), -- stolen_usec: Some(curr_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- ) => { -- let idle_usec = curr_idle - prev_idle; -- let iowait_usec = curr_iowait - prev_iowait; -- let user_usec = curr_user - prev_user; -- let system_usec = curr_system - prev_system; -- let nice_usec = curr_nice - prev_nice; -- let irq_usec = curr_irq - prev_irq; -- let softirq_usec = curr_softirq - prev_softirq; -- let stolen_usec = curr_stolen - prev_stolen; -- -- let busy_usec = -- user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec; -- let total_usec = idle_usec + busy_usec + iowait_usec; -- busy_usec as f64 / total_usec as f64 -- } -- _ => { -- bail!("Some procfs stats are not populated!"); -- } -- }; -- -- self.prev_total_cpu = total_cpu; -- Ok(busy) -- } -- -- fn read_bpf_stats(&mut self) -> Result> { -- let mut maps = self.skel.maps_mut(); -- let stats_map = maps.stats(); -- let mut stats: Vec = Vec::new(); -- let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus]; -- -- for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS { -- let cpu_stat_vec = stats_map -- .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY) -- .with_context(|| format!("Failed to lookup stat {}", stat))? -- .expect("per-cpu stat should exist"); -- let sum = cpu_stat_vec -- .iter() -- .map(|val| { -- u64::from_ne_bytes( -- val.as_slice() -- .try_into() -- .expect("Invalid value length in stat map"), -- ) -- }) -- .sum(); -- stats_map -- .update_percpu( -- &(stat as u32).to_ne_bytes(), -- &zero_vec, -- libbpf_rs::MapFlags::ANY, -- ) -- .context("Failed to zero stat")?; -- stats.push(sum); -- } -- Ok(stats) -- } -- -- fn report( -- &self, -- stats: &Vec, -- cpu_busy: f64, -- processing_dur: Duration, -- load_avg: f64, -- dom_loads: &Vec, -- imbal: &Vec, -- ) { -- let stat = |idx| stats[idx as usize]; -- let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK); -- -- info!( -- "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms", -- cpu_busy * 100.0, -- load_avg, -- stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize], -- stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize], -- self.nr_lb_data_errors, -- processing_dur.as_millis(), -- ); -- -- let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0; -- -- info!( -- "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}", -- total, -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY), -- ); -- -- for i in 0..self.nr_doms { -- info!( -- "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}", -- i, -- dom_loads[i], -- if imbal[i] < 0.0 { -imbal[i] } else { 0.0 }, -- if imbal[i] > 0.0 { imbal[i] } else { 0.0 }, -- ); -- } -- } -- -- fn step(&mut self) -> Result<()> { -- let started_at = std::time::SystemTime::now(); -- let bpf_stats = self.read_bpf_stats()?; -- let cpu_busy = self.get_cpu_busy()?; -- -- let mut lb = LoadBalancer::new( -- self.skel.maps_mut(), -- &mut self.task_loads, -- self.nr_doms, -- self.load_decay_factor, -- &mut self.nr_lb_data_errors, -- ); -- -- lb.read_task_loads(started_at.duration_since(self.prev_at)?)?; -- lb.calculate_dom_load_balance()?; -- -- if self.balance_load { -- lb.load_balance()?; -- } -- -- // Extract fields needed for reporting and drop lb to release -- // mutable borrows. -- let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal); -- -- self.report( -- &bpf_stats, -- cpu_busy, -- std::time::SystemTime::now().duration_since(started_at)?, -- load_avg, -- &dom_loads, -- &imbal, -- ); -- -- self.prev_at = started_at; -- Ok(()) -- } -- -- fn read_bpf_exit_type(&mut self) -> i32 { -- unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) } -- } -- -- fn report_bpf_exit_type(&mut self) -> Result<()> { -- // Report msg if EXT_OPS_EXIT_ERROR. -- match self.read_bpf_exit_type() { -- 0 => Ok(()), -- etype if etype == 2 => { -- let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) }; -- let msg = cstr -- .to_str() -- .context("Failed to convert exit msg to string") -- .unwrap(); -- bail!("BPF exit_type={} msg={}", etype, msg); -- } -- etype => { -- info!("BPF exit_type={}", etype); -- Ok(()) -- } -- } -- } --} -- --impl<'a> Drop for Scheduler<'a> { -- fn drop(&mut self) { -- if let Some(struct_ops) = self.struct_ops.take() { -- drop(struct_ops); -- } -- } --} -- --fn main() -> Result<()> { -- let opts = Opts::parse(); -- -- let llv = match opts.verbose { -- 0 => simplelog::LevelFilter::Info, -- 1 => simplelog::LevelFilter::Debug, -- _ => simplelog::LevelFilter::Trace, -- }; -- let mut lcfg = simplelog::ConfigBuilder::new(); -- lcfg.set_time_level(simplelog::LevelFilter::Error) -- .set_location_level(simplelog::LevelFilter::Off) -- .set_target_level(simplelog::LevelFilter::Off) -- .set_thread_level(simplelog::LevelFilter::Off); -- simplelog::TermLogger::init( -- llv, -- lcfg.build(), -- simplelog::TerminalMode::Stderr, -- simplelog::ColorChoice::Auto, -- )?; -- -- let shutdown = Arc::new(AtomicBool::new(false)); -- let shutdown_clone = shutdown.clone(); -- ctrlc::set_handler(move || { -- shutdown_clone.store(true, Ordering::Relaxed); -- }) -- .context("Error setting Ctrl-C handler")?; -- -- let mut sched = Scheduler::init(&opts)?; -- -- while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 { -- std::thread::sleep(Duration::from_secs_f64(opts.interval)); -- sched.step()?; -- } -- -- sched.report_bpf_exit_type() --} -diff --git a/tools/sched_ext/gnu/stubs.h b/tools/sched_ext/gnu/stubs.h -deleted file mode 100644 -index 719225b16626..000000000000 ---- a/tools/sched_ext/gnu/stubs.h -+++ /dev/null -@@ -1 +0,0 @@ --/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ -diff --git a/tools/sched_ext/scx_common.bpf.h b/tools/sched_ext/scx_common.bpf.h -deleted file mode 100644 -index e56de9dc86f2..000000000000 ---- a/tools/sched_ext/scx_common.bpf.h -+++ /dev/null -@@ -1,288 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __SCHED_EXT_COMMON_BPF_H --#define __SCHED_EXT_COMMON_BPF_H -- --#include "vmlinux.h" --#include --#include --#include --#include "user_exit_info.h" -- --#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ --#define PF_EXITING 0x00000004 --#define CLOCK_MONOTONIC 1 -- --/* -- * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can -- * lead to really confusing misbehaviors. Let's trigger a build failure. -- */ --static inline void ___vmlinux_h_sanity_check___(void) --{ -- _Static_assert(SCX_DSQ_FLAG_BUILTIN, -- "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); --} -- --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; -- --static inline __attribute__((format(printf, 1, 2))) --void ___scx_bpf_error_format_checker(const char *fmt, ...) {} -- --/* -- * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments -- * instead of an array of u64. Note that __param[] must have at least one -- * element to keep the verifier happy. -- */ --#define scx_bpf_error(fmt, args...) \ --({ \ -- static char ___fmt[] = fmt; \ -- unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ -- \ -- _Pragma("GCC diagnostic push") \ -- _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ -- ___bpf_fill(___param, args); \ -- _Pragma("GCC diagnostic pop") \ -- \ -- scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ -- \ -- ___scx_bpf_error_format_checker(fmt, ##args); \ --}) -- --void scx_bpf_switch_all(void) __ksym; --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; --bool scx_bpf_consume(u64 dsq_id) __ksym; --u32 scx_bpf_dispatch_nr_slots(void) __ksym; --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym; --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym; --void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; --s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym; --const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; --const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; --void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; --void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; --bool scx_bpf_task_running(const struct task_struct *p) __ksym; --s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym; --u32 scx_bpf_reenqueue_local(void) __ksym; -- --#define BPF_STRUCT_OPS(name, args...) \ --SEC("struct_ops/"#name) \ --BPF_PROG(name, ##args) -- --#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ --SEC("struct_ops.s/"#name) \ --BPF_PROG(name, ##args) -- --/** -- * MEMBER_VPTR - Obtain the verified pointer to a struct or array member -- * @base: struct or array to index -- * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...) -- * -- * The verifier often gets confused by the instruction sequence the compiler -- * generates for indexing struct fields or arrays. This macro forces the -- * compiler to generate a code sequence which first calculates the byte offset, -- * checks it against the struct or array size and add that byte offset to -- * generate the pointer to the member to help the verifier. -- * -- * Ideally, we want to abort if the calculated offset is out-of-bounds. However, -- * BPF currently doesn't support abort, so evaluate to NULL instead. The caller -- * must check for NULL and take appropriate action to appease the verifier. To -- * avoid confusing the verifier, it's best to check for NULL and dereference -- * immediately. -- * -- * vptr = MEMBER_VPTR(my_array, [i][j]); -- * if (!vptr) -- * return error; -- * *vptr = new_value; -- */ --#define MEMBER_VPTR(base, member) (typeof(base member) *)({ \ -- u64 __base = (u64)base; \ -- u64 __addr = (u64)&(base member) - __base; \ -- asm volatile ( \ -- "if %0 <= %[max] goto +2\n" \ -- "%0 = 0\n" \ -- "goto +1\n" \ -- "%0 += %1\n" \ -- : "+r"(__addr) \ -- : "r"(__base), \ -- [max]"i"(sizeof(base) - sizeof(base member))); \ -- __addr; \ --}) -- --/* -- * BPF core and other generic helpers -- */ -- --/* list and rbtree */ --#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) --#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) -- --void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; --void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; -- --#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) --#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) -- --void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; --struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; --struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, -- struct bpf_rb_node *node) __ksym; --void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, -- bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym; --struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; -- --/* task */ --struct task_struct *bpf_task_from_pid(s32 pid) __ksym; --struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; --void bpf_task_release(struct task_struct *p) __ksym; -- --/* cgroup */ --struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; --void bpf_cgroup_release(struct cgroup *cgrp) __ksym; --struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; -- --/* cpumask */ --struct bpf_cpumask *bpf_cpumask_create(void) __ksym; --struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; --u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; --u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym; -- --/* rcu */ --void bpf_rcu_read_lock(void) __ksym; --void bpf_rcu_read_unlock(void) __ksym; -- --/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */ --struct bpf_iter_num; -- --extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym; --extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym; --extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym; -- --#ifndef bpf_for_each --/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for -- * using BPF open-coded iterators without having to write mundane explicit -- * low-level loop logic. Instead, it provides for()-like generic construct -- * that can be used pretty naturally. E.g., for some hypothetical cgroup -- * iterator, you'd write: -- * -- * struct cgroup *cg, *parent_cg = <...>; -- * -- * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { -- * bpf_printk("Child cgroup id = %d", cg->cgroup_id); -- * if (cg->cgroup_id == 123) -- * break; -- * } -- * -- * I.e., it looks almost like high-level for each loop in other languages, -- * supports continue/break, and is verifiable by BPF verifier. -- * -- * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) -- * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to -- * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int -- * *`, not just `int`. So for integers bpf_for() is more convenient. -- * -- * Note: this macro relies on C99 feature of allowing to declare variables -- * inside for() loop, bound to for() loop lifetime. It also utilizes GCC -- * extension: __attribute__((cleanup())), supported by both GCC and -- * Clang. -- */ --#define bpf_for_each(type, cur, args...) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ -- cleanup(bpf_iter_##type##_destroy))), \ -- /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_##type##_new(&___it, ##args), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_##type##_destroy, (void *)0); \ -- /* iteration and termination check */ \ -- (((cur) = bpf_iter_##type##_next(&___it))); \ --) --#endif /* bpf_for_each */ -- --#ifndef bpf_for --/* bpf_for(i, start, end) implements a for()-like looping construct that sets -- * provided integer variable *i* to values starting from *start* through, -- * but not including, *end*. It also proves to BPF verifier that *i* belongs -- * to range [start, end), so this can be used for accessing arrays without -- * extra checks. -- * -- * Note: *start* and *end* are assumed to be expressions with no side effects -- * and whose values do not change throughout bpf_for() loop execution. They do -- * not have to be statically known or constant, though. -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_for(i, start, end) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, (start), (end)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- ({ \ -- /* iteration step */ \ -- int *___t = bpf_iter_num_next(&___it); \ -- /* termination and bounds check */ \ -- (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ -- }); \ --) --#endif /* bpf_for */ -- --#ifndef bpf_repeat --/* bpf_repeat(N) performs N iterations without exposing iteration number -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_repeat(N) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, 0, (N)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- bpf_iter_num_next(&___it); \ -- /* nothing here */ \ --) --#endif /* bpf_repeat */ -- --#endif /* __SCHED_EXT_COMMON_BPF_H */ -diff --git a/tools/sched_ext/scx_example_central.bpf.c b/tools/sched_ext/scx_example_central.bpf.c -deleted file mode 100644 -index 4cec04b4c2ed..000000000000 ---- a/tools/sched_ext/scx_example_central.bpf.c -+++ /dev/null -@@ -1,334 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A central FIFO sched_ext scheduler which demonstrates the followings: -- * -- * a. Making all scheduling decisions from one CPU: -- * -- * The central CPU is the only one making scheduling decisions. All other -- * CPUs kick the central CPU when they run out of tasks to run. -- * -- * There is one global BPF queue and the central CPU schedules all CPUs by -- * dispatching from the global queue to each CPU's local dsq from dispatch(). -- * This isn't the most straightforward. e.g. It'd be easier to bounce -- * through per-CPU BPF queues. The current design is chosen to maximally -- * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching. -- * -- * b. Tickless operation -- * -- * All tasks are dispatched with the infinite slice which allows stopping the -- * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full -- * parameter. The tickless operation can be observed through -- * /proc/interrupts. -- * -- * Periodic switching is enforced by a periodic timer checking all CPUs and -- * preempting them as necessary. Unfortunately, BPF timer currently doesn't -- * have a way to pin to a specific CPU, so the periodic timer isn't pinned to -- * the central CPU. -- * -- * c. Preemption -- * -- * Kthreads are unconditionally queued to the head of a matching local dsq -- * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always -- * prioritized over user threads, which is required for ensuring forward -- * progress as e.g. the periodic timer may run on a ksoftirqd and if the -- * ksoftirqd gets starved by a user thread, there may not be anything else to -- * vacate that user thread. -- * -- * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the -- * next tasks. -- * -- * This scheduler is designed to maximize usage of various SCX mechanisms. A -- * more practical implementation would likely put the scheduling loop outside -- * the central CPU's dispatch() path and add some form of priority mechanism. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --enum { -- FALLBACK_DSQ_ID = 0, -- MAX_CPUS = 4096, -- MS_TO_NS = 1000LLU * 1000, -- TIMER_INTERVAL_NS = 1 * MS_TO_NS, --}; -- --const volatile bool switch_partial; --const volatile s32 central_cpu; --const volatile u32 nr_cpu_ids = 64; /* !0 for veristat, set during init */ -- --u64 nr_total, nr_locals, nr_queued, nr_lost_pids; --u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries; --u64 nr_overflows; -- --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, s32); --} central_q SEC(".maps"); -- --/* can't use percpu map due to bad lookups */ --static bool cpu_gimme_task[MAX_CPUS]; --static u64 cpu_started_at[MAX_CPUS]; -- --struct central_timer { -- struct bpf_timer timer; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, 1); -- __type(key, u32); -- __type(value, struct central_timer); --} central_timer SEC(".maps"); -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- /* -- * Steer wakeups to the central CPU as much as possible to avoid -- * disturbing other CPUs. It's safe to blindly return the central cpu as -- * select_cpu() is a hint and if @p can't be on it, the kernel will -- * automatically pick a fallback CPU. -- */ -- return central_cpu; --} -- --void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags) --{ -- s32 pid = p->pid; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- /* -- * Push per-cpu kthreads at the head of local dsq's and preempt the -- * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked -- * behind other threads which is necessary for forward progress -- * guarantee as we depend on the BPF timer which may run from ksoftirqd. -- */ -- if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- __sync_fetch_and_add(&nr_locals, 1); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF, -- enq_flags | SCX_ENQ_PREEMPT); -- return; -- } -- -- if (bpf_map_push_elem(¢ral_q, &pid, 0)) { -- __sync_fetch_and_add(&nr_overflows, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_queued, 1); -- -- if (!scx_bpf_task_running(p)) -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); --} -- --static bool dispatch_to_cpu(s32 cpu) --{ -- struct task_struct *p; -- s32 pid; -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (bpf_map_pop_elem(¢ral_q, &pid)) -- break; -- -- __sync_fetch_and_sub(&nr_queued, 1); -- -- p = bpf_task_from_pid(pid); -- if (!p) { -- __sync_fetch_and_add(&nr_lost_pids, 1); -- continue; -- } -- -- /* -- * If we can't run the task at the top, do the dumb thing and -- * bounce it to the fallback dsq. -- */ -- if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- __sync_fetch_and_add(&nr_mismatches, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0); -- bpf_task_release(p); -- continue; -- } -- -- /* dispatch to local and mark that @cpu doesn't need more */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0); -- -- if (cpu != central_cpu) -- scx_bpf_kick_cpu(cpu, 0); -- -- bpf_task_release(p); -- return true; -- } -- -- return false; --} -- --void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (cpu == central_cpu) { -- /* dispatch for all other CPUs first */ -- __sync_fetch_and_add(&nr_dispatches, 1); -- -- bpf_for(cpu, 0, nr_cpu_ids) { -- bool *gimme; -- -- if (!scx_bpf_dispatch_nr_slots()) -- break; -- -- /* central's gimme is never set */ -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme && !*gimme) -- continue; -- -- if (dispatch_to_cpu(cpu)) -- *gimme = false; -- } -- -- /* -- * Retry if we ran out of dispatch buffer slots as we might have -- * skipped some CPUs and also need to dispatch for self. The ext -- * core automatically retries if the local dsq is empty but we -- * can't rely on that as we're dispatching for other CPUs too. -- * Kick self explicitly to retry. -- */ -- if (!scx_bpf_dispatch_nr_slots()) { -- __sync_fetch_and_add(&nr_retries, 1); -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- return; -- } -- -- /* look for a task to run on the central CPU */ -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- dispatch_to_cpu(central_cpu); -- } else { -- bool *gimme; -- -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme) -- *gimme = true; -- -- /* -- * Force dispatch on the scheduling CPU so that it finds a task -- * to run for us. -- */ -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- } --} -- --void BPF_STRUCT_OPS(central_running, struct task_struct *p) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ --} -- --void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = 0; --} -- --static int central_timerfn(void *map, int *key, struct bpf_timer *timer) --{ -- u64 now = bpf_ktime_get_ns(); -- u64 nr_to_kick = nr_queued; -- s32 i; -- -- bpf_for(i, 0, nr_cpu_ids) { -- s32 cpu = (nr_timers + i) % nr_cpu_ids; -- u64 *started_at; -- -- if (cpu == central_cpu) -- continue; -- -- /* kick iff the current one exhausted its slice */ -- started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at && *started_at && -- vtime_before(now, *started_at + SCX_SLICE_DFL)) -- continue; -- -- /* and there's something pending */ -- if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) || -- scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu)) -- ; -- else if (nr_to_kick) -- nr_to_kick--; -- else -- continue; -- -- scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); -- } -- -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- -- bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- __sync_fetch_and_add(&nr_timers, 1); -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(central_init) --{ -- u32 key = 0; -- struct bpf_timer *timer; -- int ret; -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1); -- if (ret) -- return ret; -- -- timer = bpf_map_lookup_elem(¢ral_timer, &key); -- if (!timer) -- return -ESRCH; -- -- bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); -- bpf_timer_set_callback(timer, central_timerfn); -- ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- return ret; --} -- --void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops central_ops = { -- /* -- * We are offloading all scheduling decisions to the central CPU and -- * thus being the last task on a given CPU doesn't mean anything -- * special. Enqueue the last tasks like any other tasks. -- */ -- .flags = SCX_OPS_ENQ_LAST, -- -- .select_cpu = (void *)central_select_cpu, -- .enqueue = (void *)central_enqueue, -- .dispatch = (void *)central_dispatch, -- .running = (void *)central_running, -- .stopping = (void *)central_stopping, -- .init = (void *)central_init, -- .exit = (void *)central_exit, -- .name = "central", --}; -diff --git a/tools/sched_ext/scx_example_central.c b/tools/sched_ext/scx_example_central.c -deleted file mode 100644 -index 7ad591cbdc65..000000000000 ---- a/tools/sched_ext/scx_example_central.c -+++ /dev/null -@@ -1,94 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_central.skel.h" -- --const char help_fmt[] = --"A central FIFO sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-c CPU] [-p]\n" --"\n" --" -c CPU Override the central CPU (default: 0)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_central *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_central__open(); -- assert(skel); -- -- skel->rodata->central_cpu = 0; -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "c:ph")) != -1) { -- switch (opt) { -- case 'c': -- skel->rodata->central_cpu = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_central__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.central_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf("total :%10lu local:%10lu queued:%10lu lost:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_locals, -- skel->bss->nr_queued, -- skel->bss->nr_lost_pids); -- printf("timer :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n", -- skel->bss->nr_timers, -- skel->bss->nr_dispatches, -- skel->bss->nr_mismatches, -- skel->bss->nr_retries); -- printf("overflow:%10lu\n", -- skel->bss->nr_overflows); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_central__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_flatcg.bpf.c b/tools/sched_ext/scx_example_flatcg.bpf.c -deleted file mode 100644 -index f6078b9a681f..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.bpf.c -+++ /dev/null -@@ -1,872 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext flattened cgroup hierarchy scheduler. It implements -- * hierarchical weight-based cgroup CPU control by flattening the cgroup -- * hierarchy into a single layer by compounding the active weight share at each -- * level. Consider the following hierarchy with weights in parentheses: -- * -- * R + A (100) + B (100) -- * | \ C (100) -- * \ D (200) -- * -- * Ignoring the root and threaded cgroups, only B, C and D can contain tasks. -- * Let's say all three have runnable tasks. The total share that each of these -- * three cgroups is entitled to can be calculated by compounding its share at -- * each level. -- * -- * For example, B is competing against C and in that competition its share is -- * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's -- * share in that competition is 200/(200+100) == 1/3. B's eventual share in the -- * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's -- * eventual shaer is the same at 1/6. D is only competing at the top level and -- * its share is 200/(100+200) == 2/3. -- * -- * So, instead of hierarchically scheduling level-by-level, we can consider it -- * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3 -- * and keep updating the eventual shares as the cgroups' runnable states change. -- * -- * This flattening of hierarchy can bring a substantial performance gain when -- * the cgroup hierarchy is nested multiple levels. in a simple benchmark using -- * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it -- * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two -- * apache instances competing with 2:1 weight ratio nested four level deep. -- * -- * However, the gain comes at the cost of not being able to properly handle -- * thundering herd of cgroups. For example, if many cgroups which are nested -- * behind a low priority parent cgroup wake up around the same time, they may be -- * able to consume more CPU cycles than they are entitled to. In many use cases, -- * this isn't a real concern especially given the performance gain. Also, there -- * are ways to mitigate the problem further by e.g. introducing an extra -- * scheduling layer on cgroup delegation boundaries. -- * -- * The scheduler first picks the cgroup to run and then schedule the tasks -- * within by using nested weighted vtime scheduling by default. The -- * cgroup-internal scheduling can be switched to FIFO with the -f option. -- */ --#include "scx_common.bpf.h" --#include "user_exit_info.h" --#include "scx_example_flatcg.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */ --const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL; --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --u64 cvtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, u64); -- __uint(max_entries, FCG_NR_STATS); --} stats SEC(".maps"); -- --static void stat_inc(enum fcg_stat_idx idx) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p)++; --} -- --struct fcg_cpu_ctx { -- u64 cur_cgid; -- u64 cur_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, struct fcg_cpu_ctx); -- __uint(max_entries, 1); --} cpu_ctx SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_cgrp_ctx); --} cgrp_ctx SEC(".maps"); -- --struct cgv_node { -- struct bpf_rb_node rb_node; -- __u64 cvtime; -- __u64 cgid; --}; -- --private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock; --private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node); -- --struct cgv_node_stash { -- struct cgv_node __kptr *node; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, 16384); -- __type(key, __u64); -- __type(value, struct cgv_node_stash); --} cgv_node_stash SEC(".maps"); -- --struct fcg_task_ctx { -- u64 bypassed_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_task_ctx); --} task_ctx SEC(".maps"); -- --/* gets inc'd on weight tree changes to expire the cached hweights */ --unsigned long hweight_gen = 1; -- --static u64 div_round_up(u64 dividend, u64 divisor) --{ -- return (dividend + divisor - 1) / divisor; --} -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) --{ -- struct cgv_node *cgc_a, *cgc_b; -- -- cgc_a = container_of(a, struct cgv_node, rb_node); -- cgc_b = container_of(b, struct cgv_node, rb_node); -- -- return cgc_a->cvtime < cgc_b->cvtime; --} -- --static struct fcg_cpu_ctx *find_cpu_ctx(void) --{ -- struct fcg_cpu_ctx *cpuc; -- u32 idx = 0; -- -- cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx); -- if (!cpuc) { -- scx_bpf_error("cpu_ctx lookup failed"); -- return NULL; -- } -- return cpuc; --} -- --static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id); -- return NULL; -- } -- return cgc; --} -- --static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgrp = bpf_cgroup_ancestor(cgrp, level); -- if (!cgrp) { -- scx_bpf_error("ancestor cgroup lookup failed"); -- return NULL; -- } -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- scx_bpf_error("ancestor cgrp_ctx lookup failed"); -- bpf_cgroup_release(cgrp); -- return cgc; --} -- --static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- int level; -- -- if (!cgc->nr_active) { -- stat_inc(FCG_STAT_HWT_SKIP); -- return; -- } -- -- if (cgc->hweight_gen == hweight_gen) { -- stat_inc(FCG_STAT_HWT_CACHE); -- return; -- } -- -- stat_inc(FCG_STAT_HWT_UPDATES); -- bpf_for(level, 0, cgrp->level + 1) { -- struct fcg_cgrp_ctx *cgc; -- bool is_active; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- -- if (!level) { -- cgc->hweight = FCG_HWEIGHT_ONE; -- cgc->hweight_gen = hweight_gen; -- } else { -- struct fcg_cgrp_ctx *pcgc; -- -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- -- /* -- * We can be oppotunistic here and not grab the -- * cgv_tree_lock and deal with the occasional races. -- * However, hweight updates are already cached and -- * relatively low-frequency. Let's just do the -- * straightforward thing. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- is_active = cgc->nr_active; -- if (is_active) { -- cgc->hweight_gen = pcgc->hweight_gen; -- cgc->hweight = -- div_round_up(pcgc->hweight * cgc->weight, -- pcgc->child_weight_sum); -- } -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!is_active) { -- stat_inc(FCG_STAT_HWT_RACE); -- break; -- } -- } -- } --} -- --static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) --{ -- u64 delta, cvtime, max_budget; -- -- /* -- * A node which is on the rbtree can't be pointed to from elsewhere yet -- * and thus can't be updated and repositioned. Instead, we collect the -- * vtime deltas separately and apply it asynchronously here. -- */ -- delta = cgc->cvtime_delta; -- __sync_fetch_and_sub(&cgc->cvtime_delta, delta); -- cvtime = cgv_node->cvtime + delta; -- -- /* -- * Allow a cgroup to carry the maximum budget proportional to its -- * hweight such that a full-hweight cgroup can immediately take up half -- * of the CPUs at the most while staying at the front of the rbtree. -- */ -- max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / -- (2 * FCG_HWEIGHT_ONE); -- if (vtime_before(cvtime, cvtime_now - max_budget)) -- cvtime = cvtime_now - max_budget; -- -- cgv_node->cvtime = cvtime; --} -- --static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- u64 cgid = cgrp->kn->id; -- -- /* paired with cmpxchg in try_pick_next_cgroup() */ -- if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { -- stat_inc(FCG_STAT_ENQ_SKIP); -- return; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid); -- return; -- } -- -- /* NULL if the node is already on the rbtree */ -- cgv_node = bpf_kptr_xchg(&stash->node, NULL); -- if (!cgv_node) { -- stat_inc(FCG_STAT_ENQ_RACE); -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); --} -- --void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * If select_cpu_dfl() is recommending local enqueue, the target CPU is -- * idle. Follow it and charge the cgroup later in fcg_stopping() after -- * the fact. Use the same mechanism to deal with tasks with custom -- * affinities so that we don't have to worry about per-cgroup dq's -- * containing tasks that can't be executed from some CPUs. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) { -- /* -- * Tell fcg_stopping() that this bypassed the regular scheduling -- * path and should be force charged to the cgroup. 0 is used to -- * indicate that the task isn't bypassing, so if the current -- * runtime is 0, go back by one nanosecond. -- */ -- taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1; -- -- /* -- * The global dq is deprioritized as we don't want to let tasks -- * to boost themselves by constraining its cpumask. The -- * deprioritization is rather severe, so let's not apply that to -- * per-cpu kernel threads. This is ham-fisted. We probably wanna -- * implement per-cgroup fallback dq's instead so that we have -- * more control over when tasks with custom cpumask get issued. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || -- (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) { -- stat_inc(FCG_STAT_LOCAL); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- } else { -- stat_inc(FCG_STAT_GLOBAL); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } -- return; -- } -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- goto out_release; -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 tvtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) -- tvtime = cgc->tvtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, -- tvtime, enq_flags); -- } -- -- cgrp_enqueued(cgrp, cgc); --out_release: -- bpf_cgroup_release(cgrp); --} -- --/* -- * Walk the cgroup tree to update the active weight sums as tasks wake up and -- * sleep. The weight sums are used as the base when calculating the proportion a -- * given cgroup or task is entitled to at each level. -- */ --static void update_active_weight_sums(struct cgroup *cgrp, bool runnable) --{ -- struct fcg_cgrp_ctx *cgc; -- bool updated = false; -- int idx; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- /* -- * In most cases, a hot cgroup would have multiple threads going to -- * sleep and waking up while the whole cgroup stays active. In leaf -- * cgroups, ->nr_runnable which is updated with __sync operations gates -- * ->nr_active updates, so that we don't have to grab the cgv_tree_lock -- * repeatedly for a busy cgroup which is staying active. -- */ -- if (runnable) { -- if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_ACT); -- } else { -- if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_DEACT); -- } -- -- /* -- * If @cgrp is becoming runnable, its hweight should be refreshed after -- * it's added to the weight tree so that enqueue has the up-to-date -- * value. If @cgrp is becoming quiescent, the hweight should be -- * refreshed before it's removed from the weight tree so that the usage -- * charging which happens afterwards has access to the latest value. -- */ -- if (!runnable) -- cgrp_refresh_hweight(cgrp, cgc); -- -- /* propagate upwards */ -- bpf_for(idx, 0, cgrp->level) { -- int level = cgrp->level - idx; -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- bool propagate = false; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- if (level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- } -- -- /* -- * We need the propagation protected by a lock to synchronize -- * against weight changes. There's no reason to drop the lock at -- * each level but bpf_spin_lock() doesn't want any function -- * calls while locked. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- -- if (runnable) { -- if (!cgc->nr_active++) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum += cgc->weight; -- } -- } -- } else { -- if (!--cgc->nr_active) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum -= cgc->weight; -- } -- } -- } -- -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!propagate) -- break; -- } -- -- if (updated) -- __sync_fetch_and_add(&hweight_gen, 1); -- -- if (runnable) -- cgrp_refresh_hweight(cgrp, cgc); --} -- --void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, true); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) --{ -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- if (fifo_sched) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- /* -- * @cgc->tvtime_now always progresses forward as tasks start -- * executing. The test and update can be performed concurrently -- * from multiple CPUs and thus racy. Any error should be -- * contained and temporary. Let's just live with it. -- */ -- if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) -- cgc->tvtime_now = p->scx.dsq_vtime; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- /* scale the execution time by the inverse of the weight and charge */ -- if (!fifo_sched) -- p->scx.dsq_vtime += -- (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- if (!taskc->bypassed_at) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- __sync_fetch_and_add(&cgc->cvtime_delta, -- p->se.sum_exec_runtime - taskc->bypassed_at); -- taskc->bypassed_at = 0; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, false); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) --{ -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- if (cgrp->level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1); -- if (!pcgc) -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- if (pcgc && cgc->nr_active) -- pcgc->child_weight_sum += (s64)weight - cgc->weight; -- cgc->weight = weight; -- bpf_spin_unlock(&cgv_tree_lock); --} -- --static bool try_pick_next_cgroup(u64 *cgidp) --{ -- struct bpf_rb_node *rb_node; -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 cgid; -- -- /* pop the front cgroup and wind cvtime_now accordingly */ -- bpf_spin_lock(&cgv_tree_lock); -- -- rb_node = bpf_rbtree_first(&cgv_tree); -- if (!rb_node) { -- bpf_spin_unlock(&cgv_tree_lock); -- stat_inc(FCG_STAT_PNC_NO_CGRP); -- *cgidp = 0; -- return true; -- } -- -- rb_node = bpf_rbtree_remove(&cgv_tree, rb_node); -- bpf_spin_unlock(&cgv_tree_lock); -- -- cgv_node = container_of(rb_node, struct cgv_node, rb_node); -- cgid = cgv_node->cgid; -- -- if (vtime_before(cvtime_now, cgv_node->cvtime)) -- cvtime_now = cgv_node->cvtime; -- -- /* -- * If lookup fails, the cgroup's gone. Free and move on. See -- * fcg_cgroup_exit(). -- */ -- cgrp = bpf_cgroup_from_id(cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- if (!scx_bpf_consume(cgid)) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_EMPTY); -- goto out_stash; -- } -- -- /* -- * Successfully consumed from the cgroup. This will be our current -- * cgroup for the new slice. Refresh its hweight. -- */ -- cgrp_refresh_hweight(cgrp, cgc); -- -- bpf_cgroup_release(cgrp); -- -- /* -- * As the cgroup may have more tasks, add it back to the rbtree. Note -- * that here we charge the full slice upfront and then exact later -- * according to the actual consumption. This prevents lowpri thundering -- * herd from saturating the machine. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- -- *cgidp = cgid; -- stat_inc(FCG_STAT_PNC_NEXT); -- return true; -- --out_stash: -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- /* -- * Paired with cmpxchg in cgrp_enqueued(). If they see the following -- * transition, they'll enqueue the cgroup. If they are earlier, we'll -- * see their task in the dq below and requeue the cgroup. -- */ -- __sync_val_compare_and_swap(&cgc->queued, 1, 0); -- -- if (scx_bpf_dsq_nr_queued(cgid)) { -- bpf_spin_lock(&cgv_tree_lock); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- goto out_free; -- } -- } -- -- return false; -- --out_free: -- bpf_obj_drop(cgv_node); -- return false; --} -- --void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) --{ -- struct fcg_cpu_ctx *cpuc; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 now = bpf_ktime_get_ns(); -- -- cpuc = find_cpu_ctx(); -- if (!cpuc) -- return; -- -- if (!cpuc->cur_cgid) -- goto pick_next_cgroup; -- -- if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { -- if (scx_bpf_consume(cpuc->cur_cgid)) { -- stat_inc(FCG_STAT_CNS_KEEP); -- return; -- } -- stat_inc(FCG_STAT_CNS_EMPTY); -- } else { -- stat_inc(FCG_STAT_CNS_EXPIRE); -- } -- -- /* -- * The current cgroup is expiring. It was already charged a full slice. -- * Calculate the actual usage and accumulate the delta. -- */ -- cgrp = bpf_cgroup_from_id(cpuc->cur_cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_CNS_GONE); -- goto pick_next_cgroup; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (cgc) { -- /* -- * We want to update the vtime delta and then look for the next -- * cgroup to execute but the latter needs to be done in a loop -- * and we can't keep the lock held. Oh well... -- */ -- bpf_spin_lock(&cgv_tree_lock); -- __sync_fetch_and_add(&cgc->cvtime_delta, -- (cpuc->cur_at + cgrp_slice_ns - now) * -- FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- stat_inc(FCG_STAT_CNS_GONE); -- } -- -- bpf_cgroup_release(cgrp); -- --pick_next_cgroup: -- cpuc->cur_at = now; -- -- if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { -- cpuc->cur_cgid = 0; -- return; -- } -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_pick_next_cgroup(&cpuc->cur_cgid)) -- break; -- } --} -- --s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct fcg_task_ctx *taskc; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- taskc = bpf_task_storage_get(&task_ctx, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!taskc) -- return -ENOMEM; -- -- taskc->bypassed_at = 0; -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp, -- struct scx_cgroup_init_args *args) --{ -- struct fcg_cgrp_ctx *cgc; -- struct cgv_node *cgv_node; -- struct cgv_node_stash empty_stash = {}, *stash; -- u64 cgid = cgrp->kn->id; -- int ret; -- -- /* -- * Technically incorrect as cgroup ID is full 64bit while dq ID is -- * 63bit. Should not be a problem in practice and easy to spot in the -- * unlikely case that it breaks. -- */ -- ret = scx_bpf_create_dsq(cgid, -1); -- if (ret) -- return ret; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!cgc) { -- ret = -ENOMEM; -- goto err_destroy_dsq; -- } -- -- cgc->weight = args->weight; -- cgc->hweight = FCG_HWEIGHT_ONE; -- -- ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash, -- BPF_NOEXIST); -- if (ret) { -- if (ret != -ENOMEM) -- scx_bpf_error("unexpected stash creation error (%d)", -- ret); -- goto err_destroy_dsq; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("unexpected cgv_node stash lookup failure"); -- ret = -ENOENT; -- goto err_destroy_dsq; -- } -- -- cgv_node = bpf_obj_new(struct cgv_node); -- if (!cgv_node) { -- ret = -ENOMEM; -- goto err_del_cgv_node; -- } -- -- cgv_node->cgid = cgid; -- cgv_node->cvtime = cvtime_now; -- -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- ret = -EBUSY; -- goto err_drop; -- } -- -- return 0; -- --err_drop: -- bpf_obj_drop(cgv_node); --err_del_cgv_node: -- bpf_map_delete_elem(&cgv_node_stash, &cgid); --err_destroy_dsq: -- scx_bpf_destroy_dsq(cgid); -- return ret; --} -- --void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- -- /* -- * For now, there's no way find and remove the cgv_node if it's on the -- * cgv_tree. Let's drain them in the dispatch path as they get popped -- * off the front of the tree. -- */ -- bpf_map_delete_elem(&cgv_node_stash, &cgid); -- scx_bpf_destroy_dsq(cgid); --} -- --s32 BPF_STRUCT_OPS(fcg_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops flatcg_ops = { -- .enqueue = (void *)fcg_enqueue, -- .dispatch = (void *)fcg_dispatch, -- .runnable = (void *)fcg_runnable, -- .running = (void *)fcg_running, -- .stopping = (void *)fcg_stopping, -- .quiescent = (void *)fcg_quiescent, -- .prep_enable = (void *)fcg_prep_enable, -- .cgroup_set_weight = (void *)fcg_cgroup_set_weight, -- .cgroup_init = (void *)fcg_cgroup_init, -- .cgroup_exit = (void *)fcg_cgroup_exit, -- .init = (void *)fcg_init, -- .exit = (void *)fcg_exit, -- .flags = SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING, -- .name = "flatcg", --}; -diff --git a/tools/sched_ext/scx_example_flatcg.c b/tools/sched_ext/scx_example_flatcg.c -deleted file mode 100644 -index f9c8a5b84a70..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.c -+++ /dev/null -@@ -1,232 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2023 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2023 Tejun Heo -- * Copyright (c) 2023 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_flatcg.h" --#include "scx_example_flatcg.skel.h" -- --#ifndef FILEID_KERNFS --#define FILEID_KERNFS 0xfe --#endif -- --const char help_fmt[] = --"A flattened cgroup hierarchy sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -i INTERVAL Report interval\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --static float read_cpu_util(__u64 *last_sum, __u64 *last_idle) --{ -- FILE *fp; -- char buf[4096]; -- char *line, *cur = NULL, *tok; -- __u64 sum = 0, idle = 0; -- __u64 delta_sum, delta_idle; -- int idx; -- -- fp = fopen("/proc/stat", "r"); -- if (!fp) { -- perror("fopen(\"/proc/stat\")"); -- return 0.0; -- } -- -- if (!fgets(buf, sizeof(buf), fp)) { -- perror("fgets(\"/proc/stat\")"); -- fclose(fp); -- return 0.0; -- } -- fclose(fp); -- -- line = buf; -- for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) { -- char *endp = NULL; -- __u64 v; -- -- if (idx == 0) { -- line = NULL; -- continue; -- } -- v = strtoull(tok, &endp, 0); -- if (!endp || *endp != '\0') { -- fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n", -- idx, tok); -- continue; -- } -- sum += v; -- if (idx == 4) -- idle = v; -- } -- -- delta_sum = sum - *last_sum; -- delta_idle = idle - *last_idle; -- *last_sum = sum; -- *last_idle = idle; -- -- return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0; --} -- --static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats) --{ -- __u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus]; -- __u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); -- -- for (idx = 0; idx < FCG_NR_STATS; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_flatcg *skel; -- struct bpf_link *link; -- struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 }; -- bool dump_cgrps = false; -- __u64 last_cpu_sum = 0, last_cpu_idle = 0; -- __u64 last_stats[FCG_NR_STATS] = {}; -- unsigned long seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_flatcg__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open: %s\n", strerror(errno)); -- return 1; -- } -- -- skel->rodata->nr_cpus = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) { -- double v; -- -- switch (opt) { -- case 's': -- v = strtod(optarg, NULL); -- skel->rodata->cgrp_slice_ns = v * 1000; -- break; -- case 'i': -- v = strtod(optarg, NULL); -- intv_ts.tv_sec = v; -- intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000; -- break; -- case 'd': -- dump_cgrps = true; -- break; -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- case 'h': -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d", -- (double)skel->rodata->cgrp_slice_ns / 1000000.0, -- (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0, -- dump_cgrps); -- -- if (scx_example_flatcg__load(skel)) { -- fprintf(stderr, "Failed to load: %s\n", strerror(errno)); -- return 1; -- } -- -- link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops); -- if (!link) { -- fprintf(stderr, "Failed to attach_struct_ops: %s\n", -- strerror(errno)); -- return 1; -- } -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- __u64 acc_stats[FCG_NR_STATS]; -- __u64 stats[FCG_NR_STATS]; -- float cpu_util; -- int i; -- -- cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle); -- -- fcg_read_stats(skel, acc_stats); -- for (i = 0; i < FCG_NR_STATS; i++) -- stats[i] = acc_stats[i] - last_stats[i]; -- -- memcpy(last_stats, acc_stats, sizeof(acc_stats)); -- -- printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n", -- seq++, cpu_util * 100.0, skel->data->hweight_gen); -- printf(" act:%6llu deact:%6llu local:%6llu global:%6llu\n", -- stats[FCG_STAT_ACT], -- stats[FCG_STAT_DEACT], -- stats[FCG_STAT_LOCAL], -- stats[FCG_STAT_GLOBAL]); -- printf("HWT skip:%6llu race:%6llu cache:%6llu update:%6llu\n", -- stats[FCG_STAT_HWT_SKIP], -- stats[FCG_STAT_HWT_RACE], -- stats[FCG_STAT_HWT_CACHE], -- stats[FCG_STAT_HWT_UPDATES]); -- printf("ENQ skip:%6llu race:%6llu\n", -- stats[FCG_STAT_ENQ_SKIP], -- stats[FCG_STAT_ENQ_RACE]); -- printf("CNS keep:%6llu expire:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_CNS_KEEP], -- stats[FCG_STAT_CNS_EXPIRE], -- stats[FCG_STAT_CNS_EMPTY], -- stats[FCG_STAT_CNS_GONE]); -- printf("PNC nocgrp:%6llu next:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_PNC_NO_CGRP], -- stats[FCG_STAT_PNC_NEXT], -- stats[FCG_STAT_PNC_EMPTY], -- stats[FCG_STAT_PNC_GONE]); -- printf("BAD remove:%6llu\n", -- acc_stats[FCG_STAT_BAD_REMOVAL]); -- -- nanosleep(&intv_ts, NULL); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_flatcg__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_flatcg.h b/tools/sched_ext/scx_example_flatcg.h -deleted file mode 100644 -index 490758ed41f0..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.h -+++ /dev/null -@@ -1,49 +0,0 @@ --#ifndef __SCX_EXAMPLE_FLATCG_H --#define __SCX_EXAMPLE_FLATCG_H -- --enum { -- FCG_HWEIGHT_ONE = 1LLU << 16, --}; -- --enum fcg_stat_idx { -- FCG_STAT_ACT, -- FCG_STAT_DEACT, -- FCG_STAT_LOCAL, -- FCG_STAT_GLOBAL, -- -- FCG_STAT_HWT_UPDATES, -- FCG_STAT_HWT_CACHE, -- FCG_STAT_HWT_SKIP, -- FCG_STAT_HWT_RACE, -- -- FCG_STAT_ENQ_SKIP, -- FCG_STAT_ENQ_RACE, -- -- FCG_STAT_CNS_KEEP, -- FCG_STAT_CNS_EXPIRE, -- FCG_STAT_CNS_EMPTY, -- FCG_STAT_CNS_GONE, -- -- FCG_STAT_PNC_NO_CGRP, -- FCG_STAT_PNC_NEXT, -- FCG_STAT_PNC_EMPTY, -- FCG_STAT_PNC_GONE, -- -- FCG_STAT_BAD_REMOVAL, -- -- FCG_NR_STATS, --}; -- --struct fcg_cgrp_ctx { -- u32 nr_active; -- u32 nr_runnable; -- u32 queued; -- u32 weight; -- u32 hweight; -- u64 child_weight_sum; -- u64 hweight_gen; -- s64 cvtime_delta; -- u64 tvtime_now; --}; -- --#endif /* __SCX_EXAMPLE_FLATCG_H */ -diff --git a/tools/sched_ext/scx_example_pair.bpf.c b/tools/sched_ext/scx_example_pair.bpf.c -deleted file mode 100644 -index 279efe58b777..000000000000 ---- a/tools/sched_ext/scx_example_pair.bpf.c -+++ /dev/null -@@ -1,627 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext core-scheduler which always makes every sibling CPU pair -- * execute from the same CPU cgroup. -- * -- * This scheduler is a minimal implementation and would need some form of -- * priority handling both inside each cgroup and across the cgroups to be -- * practically useful. -- * -- * Each CPU in the system is paired with exactly one other CPU, according to a -- * "stride" value that can be specified when the BPF scheduler program is first -- * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee -- * that they will only ever schedule tasks that belong to the same CPU cgroup. -- * -- * Scheduler Initialization -- * ------------------------ -- * -- * The scheduler BPF program is first initialized from user space, before it is -- * enabled. During this initialization process, each CPU on the system is -- * assigned several values that are constant throughout its runtime: -- * -- * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling -- * decisions. Paired CPUs always schedule tasks from the same -- * CPU cgroup, and synchronize with each other to guarantee -- * that this constraint is not violated. -- * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access -- * a struct pair_ctx object that is shared between the pair. -- * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the -- * pair. Each struct pair_ctx has an active_mask field, -- * which is a bitmap used to indicate whether each core -- * in the pair currently has an actively running task. -- * This index specifies which entry in the bitmap corresponds -- * to each CPU in the pair. -- * -- * During this initialization, the CPUs are paired according to a "stride" that -- * may be specified when invoking the user space program that initializes and -- * loads the scheduler. By default, the stride is 1/2 the total number of CPUs. -- * -- * Tasks and cgroups -- * ----------------- -- * -- * Every cgroup in the system is registered with the scheduler using the -- * pair_cgroup_init() callback, and every task in the system is associated with -- * exactly one cgroup. At a high level, the idea with the pair scheduler is to -- * always schedule tasks from the same cgroup within a given CPU pair. When a -- * task is enqueued (i.e. passed to the pair_enqueue() callback function), its -- * cgroup ID is read from its task struct, and then a corresponding queue map -- * is used to FIFO-enqueue the task for that cgroup. -- * -- * If you look through the implementation of the scheduler, you'll notice that -- * there is quite a bit of complexity involved with looking up the per-cgroup -- * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash -- * BPF hash map that is used to map a cgroup ID to a globally unique ID that's -- * allocated in the BPF program. This is done because we use separate maps to -- * store the FIFO queue of tasks, and the length of that map, per cgroup. This -- * complexity is only present because of current deficiencies in BPF that will -- * soon be addressed. The main point to keep in mind is that newly enqueued -- * tasks are added to their cgroup's FIFO queue. -- * -- * Dispatching tasks -- * ----------------- -- * -- * This section will describe how enqueued tasks are dispatched and scheduled. -- * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is -- * as follows: -- * -- * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is -- * the structure that's used to synchronize amongst the two pair CPUs in their -- * scheduling decisions. After any of the following events have occurred: -- * -- * - The cgroup's slice run has expired, or -- * - The cgroup becomes empty, or -- * - Either CPU in the pair is preempted by a higher priority scheduling class -- * -- * The cgroup transitions to the draining state and stops executing new tasks -- * from the cgroup. -- * -- * 2. If the pair is still executing a task, mark the pair_ctx as draining, and -- * wait for the pair CPU to be preempted. -- * -- * 3. Otherwise, if the pair CPU is not running a task, we can move onto -- * scheduling new tasks. Pop the next cgroup id from the top_q queue. -- * -- * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it. -- * -- * Note again that this scheduling behavior is simple, but the implementation -- * is complex mostly because this it hits several BPF shortcomings and has to -- * work around in often awkward ways. Most of the shortcomings are expected to -- * be resolved in the near future which should allow greatly simplifying this -- * scheduler. -- * -- * Dealing with preemption -- * ----------------------- -- * -- * SCX is the lowest priority sched_class, and could be preempted by them at -- * any time. To address this, the scheduler implements pair_cpu_release() and -- * pair_cpu_acquire() callbacks which are invoked by the core scheduler when -- * the scheduler loses and gains control of the CPU respectively. -- * -- * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and -- * then invoke: -- * -- * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- * -- * This preempts the pair CPU, and waits until it has re-entered the scheduler -- * before returning. This is necessary to ensure that the higher priority -- * sched_class that preempted our scheduler does not schedule a task -- * concurrently with our pair CPU. -- * -- * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption -- * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable -- * pair scheduling. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include "scx_example_pair.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; -- --/* !0 for veristat, set during init */ --const volatile u32 nr_cpu_ids = 64; -- --/* a pair of CPUs stay on a cgroup for this duration */ --const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL; -- --/* cpu ID -> pair cpu ID */ --const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 }; -- --/* cpu ID -> pair_id */ --const volatile u32 pair_id[MAX_CPUS]; -- --/* CPU ID -> CPU # in the pair (0 or 1) */ --const volatile u32 in_pair_idx[MAX_CPUS]; -- --struct pair_ctx { -- struct bpf_spin_lock lock; -- -- /* the cgroup the pair is currently executing */ -- u64 cgid; -- -- /* the pair started executing the current cgroup at */ -- u64 started_at; -- -- /* whether the current cgroup is draining */ -- bool draining; -- -- /* the CPUs that are currently active on the cgroup */ -- u32 active_mask; -- -- /* -- * the CPUs that are currently preempted and running tasks in a -- * different scheduler. -- */ -- u32 preempted_mask; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, MAX_CPUS / 2); -- __type(key, u32); -- __type(value, struct pair_ctx); --} pair_ctx SEC(".maps"); -- --/* queue of cgrp_q's possibly with tasks on them */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- /* -- * Because it's difficult to build strong synchronization encompassing -- * multiple non-trivial operations in BPF, this queue is managed in an -- * opportunistic way so that we guarantee that a cgroup w/ active tasks -- * is always on it but possibly multiple times. Once we have more robust -- * synchronization constructs and e.g. linked list, we should be able to -- * do this in a prettier way but for now just size it big enough. -- */ -- __uint(max_entries, 4 * MAX_CGRPS); -- __type(value, u64); --} top_q SEC(".maps"); -- --/* per-cgroup q which FIFOs the tasks from the cgroup */ --struct cgrp_q { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, MAX_QUEUED); -- __type(value, u32); --}; -- --/* -- * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local -- * storage; however, a cgroup local storage can only be accessed from the BPF -- * progs attached to the cgroup. For now, work around by allocating array of -- * cgrp_q's and then allocating per-cgroup indices. -- * -- * Another caveat: It's difficult to populate a large array of maps statically -- * or from BPF. Initialize it from userland. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, MAX_CGRPS); -- __type(key, s32); -- __array(values, struct cgrp_q); --} cgrp_q_arr SEC(".maps"); -- --static u64 cgrp_q_len[MAX_CGRPS]; -- --/* -- * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be -- * useful to have as a map type. -- */ --static u32 cgrp_q_idx_cursor; --static u64 cgrp_q_idx_busy[MAX_CGRPS]; -- --/* -- * All added up, the following is what we do: -- * -- * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking -- * for a free ID. If not found, fail cgroup creation with -EBUSY. -- * -- * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following -- * cgrp_q_idx_hash. -- * -- * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from -- * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr. -- * -- * This is sadly complicated for something pretty simple. Hopefully, we should -- * be able to simplify in the future. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, MAX_CGRPS); -- __uint(key_size, sizeof(u64)); /* cgrp ID */ -- __uint(value_size, sizeof(s32)); /* cgrp_q idx */ --} cgrp_q_idx_hash SEC(".maps"); -- --/* statistics */ --u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions; --u64 nr_exps, nr_exp_waits, nr_exp_empty; --u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty; -- --struct user_exit_info uei; -- --static bool time_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- struct cgrp_q *cgq; -- s32 pid = p->pid; -- u64 cgid; -- u32 *q_idx; -- u64 *cgq_len; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- cgrp = scx_bpf_task_cgroup(p); -- cgid = cgrp->kn->id; -- bpf_cgroup_release(cgrp); -- -- /* find the cgroup's q and push @p into it */ -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!q_idx) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return; -- } -- -- cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx); -- if (!cgq) { -- scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]", -- cgid, *q_idx); -- return; -- } -- -- if (bpf_map_push_elem(cgq, &pid, 0)) { -- scx_bpf_error("cgroup[%llu] queue overflow", cgid); -- return; -- } -- -- /* bump q len, if going 0 -> 1, queue cgroup into the top_q */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len) { -- scx_bpf_error("MEMBER_VTPR malfunction"); -- return; -- } -- -- if (!__sync_fetch_and_add(cgq_len, 1) && -- bpf_map_push_elem(&top_q, &cgid, 0)) { -- scx_bpf_error("top_q overflow"); -- return; -- } --} -- --static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) --{ -- u32 *vptr; -- -- vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *pairc = bpf_map_lookup_elem(&pair_ctx, vptr); -- if (!(*pairc)) -- return -EINVAL; -- -- vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *mask = 1U << *vptr; -- -- return 0; --} -- --static int try_dispatch(s32 cpu) --{ -- struct pair_ctx *pairc; -- struct bpf_map *cgq_map; -- struct task_struct *p; -- u64 now = bpf_ktime_get_ns(); -- bool kick_pair = false; -- bool expired, pair_preempted; -- u32 *vptr, in_pair_mask; -- s32 pid, q_idx; -- u64 cgid; -- int ret; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) { -- scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]", -- cpu); -- return -ENOENT; -- } -- -- bpf_spin_lock(&pairc->lock); -- pairc->active_mask &= ~in_pair_mask; -- -- expired = time_before(pairc->started_at + pair_batch_dur_ns, now); -- if (expired || pairc->draining) { -- u64 new_cgid = 0; -- -- __sync_fetch_and_add(&nr_exps, 1); -- -- /* -- * We're done with the current cgid. An obvious optimization -- * would be not draining if the next cgroup is the current one. -- * For now, be dumb and always expire. -- */ -- pairc->draining = true; -- -- pair_preempted = pairc->preempted_mask; -- if (pairc->active_mask || pair_preempted) { -- /* -- * The other CPU is still active, or is no longer under -- * our control due to e.g. being preempted by a higher -- * priority sched_class. We want to wait until this -- * cgroup expires, or until control of our pair CPU has -- * been returned to us. -- * -- * If the pair controls its CPU, and the time already -- * expired, kick. When the other CPU arrives at -- * dispatch and clears its active mask, it'll push the -- * pair to the next cgroup and kick this CPU. -- */ -- __sync_fetch_and_add(&nr_exp_waits, 1); -- bpf_spin_unlock(&pairc->lock); -- if (expired && !pair_preempted) -- kick_pair = true; -- goto out_maybe_kick; -- } -- -- bpf_spin_unlock(&pairc->lock); -- -- /* -- * Pick the next cgroup. It'd be easier / cleaner to not drop -- * pairc->lock and use stronger synchronization here especially -- * given that we'll be switching cgroups significantly less -- * frequently than tasks. Unfortunately, bpf_spin_lock can't -- * really protect anything non-trivial. Let's do opportunistic -- * operations instead. -- */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u32 *q_idx; -- u64 *cgq_len; -- -- if (bpf_map_pop_elem(&top_q, &new_cgid)) { -- /* no active cgroup, go idle */ -- __sync_fetch_and_add(&nr_exp_empty, 1); -- return 0; -- } -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid); -- if (!q_idx) -- continue; -- -- /* -- * This is the only place where empty cgroups are taken -- * off the top_q. -- */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len || !*cgq_len) -- continue; -- -- /* -- * If it has any tasks, requeue as we may race and not -- * execute it. -- */ -- bpf_map_push_elem(&top_q, &new_cgid, 0); -- break; -- } -- -- bpf_spin_lock(&pairc->lock); -- -- /* -- * The other CPU may already have started on a new cgroup while -- * we dropped the lock. Make sure that we're still draining and -- * start on the new cgroup. -- */ -- if (pairc->draining && !pairc->active_mask) { -- __sync_fetch_and_add(&nr_cgrp_next, 1); -- pairc->cgid = new_cgid; -- pairc->started_at = now; -- pairc->draining = false; -- kick_pair = true; -- } else { -- __sync_fetch_and_add(&nr_cgrp_coll, 1); -- } -- } -- -- cgid = pairc->cgid; -- pairc->active_mask |= in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- -- /* again, it'd be better to do all these with the lock held, oh well */ -- vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!vptr) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return -ENOENT; -- } -- q_idx = *vptr; -- -- /* claim one task from cgrp_q w/ q_idx */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u64 *cgq_len, len; -- -- cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]); -- if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) { -- /* the cgroup must be empty, expire and repeat */ -- __sync_fetch_and_add(&nr_cgrp_empty, 1); -- bpf_spin_lock(&pairc->lock); -- pairc->draining = true; -- pairc->active_mask &= ~in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- return -EAGAIN; -- } -- -- if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len) -- continue; -- -- break; -- } -- -- cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx); -- if (!cgq_map) { -- scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- if (bpf_map_pop_elem(cgq_map, &pid)) { -- scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- p = bpf_task_from_pid(pid); -- if (p) { -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } else { -- /* we don't handle dequeues, retry on lost tasks */ -- __sync_fetch_and_add(&nr_missing, 1); -- return -EAGAIN; -- } -- --out_maybe_kick: -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } -- return 0; --} -- --void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) --{ -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_dispatch(cpu) != -EAGAIN) -- break; -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask &= ~in_pair_mask; -- /* Kick the pair CPU, unless it was also preempted. */ -- kick_pair = !pairc->preempted_mask; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask |= in_pair_mask; -- pairc->active_mask &= ~in_pair_mask; -- /* Kick the pair CPU if it's still running. */ -- kick_pair = pairc->active_mask; -- pairc->draining = true; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- } -- } -- __sync_fetch_and_add(&nr_preemptions, 1); --} -- --s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 i, q_idx; -- -- bpf_for(i, 0, MAX_CGRPS) { -- q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS; -- if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1)) -- break; -- } -- if (i == MAX_CGRPS) -- return -EBUSY; -- -- if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]); -- if (busy) -- *busy = 0; -- return -EBUSY; -- } -- -- return 0; --} -- --void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 *q_idx; -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (q_idx) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]); -- if (busy) -- *busy = 0; -- bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid); -- } --} -- --s32 BPF_STRUCT_OPS(pair_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops pair_ops = { -- .enqueue = (void *)pair_enqueue, -- .dispatch = (void *)pair_dispatch, -- .cpu_acquire = (void *)pair_cpu_acquire, -- .cpu_release = (void *)pair_cpu_release, -- .cgroup_init = (void *)pair_cgroup_init, -- .cgroup_exit = (void *)pair_cgroup_exit, -- .init = (void *)pair_init, -- .exit = (void *)pair_exit, -- .name = "pair", --}; -diff --git a/tools/sched_ext/scx_example_pair.c b/tools/sched_ext/scx_example_pair.c -deleted file mode 100644 -index 18e032bbc173..000000000000 ---- a/tools/sched_ext/scx_example_pair.c -+++ /dev/null -@@ -1,143 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_pair.h" --#include "scx_example_pair.skel.h" -- --const char help_fmt[] = --"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n" --"execute from the same CPU cgroup.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-S STRIDE] [-p]\n" --"\n" --" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_pair *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 stride, i, opt, outer_fd; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_pair__open(); -- assert(skel); -- -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- /* pair up the earlier half to the latter by default, override with -s */ -- stride = skel->rodata->nr_cpu_ids / 2; -- -- while ((opt = getopt(argc, argv, "S:ph")) != -1) { -- switch (opt) { -- case 'S': -- stride = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- for (i = 0; i < skel->rodata->nr_cpu_ids; i++) { -- if (skel->rodata->pair_cpu[i] < 0) { -- skel->rodata->pair_cpu[i] = i + stride; -- skel->rodata->pair_cpu[i + stride] = i; -- skel->rodata->pair_id[i] = i; -- skel->rodata->pair_id[i + stride] = i; -- skel->rodata->in_pair_idx[i] = 0; -- skel->rodata->in_pair_idx[i + stride] = 1; -- } -- } -- -- assert(!scx_example_pair__load(skel)); -- -- /* -- * Populate the cgrp_q_arr map which is an array containing per-cgroup -- * queues. It'd probably be better to do this from BPF but there are too -- * many to initialize statically and there's no way to dynamically -- * populate from BPF. -- */ -- outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr); -- assert(outer_fd >= 0); -- -- printf("Initializing"); -- for (i = 0; i < MAX_CGRPS; i++) { -- s32 inner_fd; -- -- if (exit_req) -- break; -- -- inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, -- sizeof(u32), MAX_QUEUED, NULL); -- assert(inner_fd >= 0); -- assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY)); -- close(inner_fd); -- -- if (!(i % 10)) -- printf("."); -- fflush(stdout); -- } -- printf("\n"); -- -- /* -- * Fully initialized, attach and run. -- */ -- link = bpf_map__attach_struct_ops(skel->maps.pair_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf(" total:%10lu dispatch:%10lu missing:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_dispatched, -- skel->bss->nr_missing); -- printf(" kicks:%10lu preemptions:%7lu\n", -- skel->bss->nr_kicks, -- skel->bss->nr_preemptions); -- printf(" exp:%10lu exp_wait:%10lu exp_empty:%10lu\n", -- skel->bss->nr_exps, -- skel->bss->nr_exp_waits, -- skel->bss->nr_exp_empty); -- printf("cgnext:%10lu cgcoll:%10lu cgempty:%10lu\n", -- skel->bss->nr_cgrp_next, -- skel->bss->nr_cgrp_coll, -- skel->bss->nr_cgrp_empty); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_pair__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_pair.h b/tools/sched_ext/scx_example_pair.h -deleted file mode 100644 -index f60b824272f7..000000000000 ---- a/tools/sched_ext/scx_example_pair.h -+++ /dev/null -@@ -1,10 +0,0 @@ --#ifndef __SCX_EXAMPLE_PAIR_H --#define __SCX_EXAMPLE_PAIR_H -- --enum { -- MAX_CPUS = 4096, -- MAX_QUEUED = 4096, -- MAX_CGRPS = 4096, --}; -- --#endif /* __SCX_EXAMPLE_PAIR_H */ -diff --git a/tools/sched_ext/scx_example_qmap.bpf.c b/tools/sched_ext/scx_example_qmap.bpf.c -deleted file mode 100644 -index 579ab21ae403..000000000000 ---- a/tools/sched_ext/scx_example_qmap.bpf.c -+++ /dev/null -@@ -1,401 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple five-level FIFO queue scheduler. -- * -- * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets -- * assigned to one depending on its compound weight. Each CPU round robins -- * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from -- * queue0, 2 from queue1, 4 from queue2 and so on. -- * -- * This scheduler demonstrates: -- * -- * - BPF-side queueing using PIDs. -- * - Sleepable per-task storage allocation using ops.prep_enable(). -- * - Using ops.cpu_release() to handle a higher priority scheduling class taking -- * the CPU away. -- * - Core-sched support. -- * -- * This scheduler is primarily for demonstration and testing of sched_ext -- * features and unlikely to be useful for actual workloads. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include -- --char _license[] SEC("license") = "GPL"; -- --const volatile u64 slice_ns = SCX_SLICE_DFL; --const volatile bool switch_partial; --const volatile u32 stall_user_nth; --const volatile u32 stall_kernel_nth; --const volatile u32 dsp_inf_loop_after; --const volatile s32 disallow_tgid; -- --u32 test_error_cnt; -- --struct user_exit_info uei; -- --struct qmap { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, u32); --} queue0 SEC(".maps"), -- queue1 SEC(".maps"), -- queue2 SEC(".maps"), -- queue3 SEC(".maps"), -- queue4 SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, 5); -- __type(key, int); -- __array(values, struct qmap); --} queue_arr SEC(".maps") = { -- .values = { -- [0] = &queue0, -- [1] = &queue1, -- [2] = &queue2, -- [3] = &queue3, -- [4] = &queue4, -- }, --}; -- --/* -- * Per-queue sequence numbers to implement core-sched ordering. -- * -- * Tail seq is assigned to each queued task and incremented. Head seq tracks the -- * sequence number of the latest dispatched task. The distance between the a -- * task's seq and the associated queue's head seq is called the queue distance -- * and used when comparing two tasks for ordering. See qmap_core_sched_before(). -- */ --static u64 core_sched_head_seqs[5]; --static u64 core_sched_tail_seqs[5]; -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local_dsq */ -- u64 core_sched_seq; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --/* Per-cpu dispatch index and remaining count */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(max_entries, 2); -- __type(key, u32); -- __type(value, u64); --} dispatch_idx_cnt SEC(".maps"); -- --/* Statistics */ --unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued; --unsigned long nr_core_sched_execed; -- --s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- struct task_ctx *tctx; -- s32 cpu; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- return cpu; -- -- return prev_cpu; --} -- --static int weight_to_idx(u32 weight) --{ -- /* Coarsely map the compound weight to a FIFO. */ -- if (weight <= 25) -- return 0; -- else if (weight <= 50) -- return 1; -- else if (weight < 200) -- return 2; -- else if (weight < 400) -- return 3; -- else -- return 4; --} -- --void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) --{ -- static u32 user_cnt, kernel_cnt; -- struct task_ctx *tctx; -- u32 pid = p->pid; -- int idx = weight_to_idx(p->scx.weight); -- void *ring; -- -- if (p->flags & PF_KTHREAD) { -- if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth)) -- return; -- } else { -- if (stall_user_nth && !(++user_cnt % stall_user_nth)) -- return; -- } -- -- if (test_error_cnt && !--test_error_cnt) -- scx_bpf_error("test triggering error"); -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * All enqueued tasks must have their core_sched_seq updated for correct -- * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in -- * qmap_ops.flags. -- */ -- tctx->core_sched_seq = core_sched_tail_seqs[idx]++; -- -- /* -- * If qmap_select_cpu() is telling us to or this is the last runnable -- * task on the CPU, enqueue locally. -- */ -- if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) { -- tctx->force_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); -- return; -- } -- -- /* -- * If the task was re-enqueued due to the CPU being preempted by a -- * higher priority scheduling class, just re-enqueue the task directly -- * on the global DSQ. As we want another CPU to pick it up, find and -- * kick an idle CPU. -- */ -- if (enq_flags & SCX_ENQ_REENQ) { -- s32 cpu; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags); -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- return; -- } -- -- ring = bpf_map_lookup_elem(&queue_arr, &idx); -- if (!ring) { -- scx_bpf_error("failed to find ring %d", idx); -- return; -- } -- -- /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */ -- if (bpf_map_push_elem(ring, &pid, 0)) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_enqueued, 1); --} -- --/* -- * The BPF queue map doesn't support removal and sched_ext can handle spurious -- * dispatches. qmap_dequeue() is only used to collect statistics. -- */ --void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags) --{ -- __sync_fetch_and_add(&nr_dequeued, 1); -- if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) -- __sync_fetch_and_add(&nr_core_sched_execed, 1); --} -- --static void update_core_sched_head_seq(struct task_struct *p) --{ -- struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- int idx = weight_to_idx(p->scx.weight); -- -- if (tctx) -- core_sched_head_seqs[idx] = tctx->core_sched_seq; -- else -- scx_bpf_error("task_ctx lookup failed"); --} -- --void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 zero = 0, one = 1; -- u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero); -- u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one); -- void *fifo; -- s32 pid; -- int i; -- -- if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) { -- struct task_struct *p; -- -- /* -- * PID 2 should be kthreadd which should mostly be idle and off -- * the scheduler. Let's keep dispatching it to force the kernel -- * to call this function over and over again. -- */ -- p = bpf_task_from_pid(2); -- if (p) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- if (!idx || !cnt) { -- scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt); -- return; -- } -- -- for (i = 0; i < 5; i++) { -- /* Advance the dispatch cursor and pick the fifo. */ -- if (!*cnt) { -- *idx = (*idx + 1) % 5; -- *cnt = 1 << *idx; -- } -- (*cnt)--; -- -- fifo = bpf_map_lookup_elem(&queue_arr, idx); -- if (!fifo) { -- scx_bpf_error("failed to find ring %llu", *idx); -- return; -- } -- -- /* Dispatch or advance. */ -- if (!bpf_map_pop_elem(fifo, &pid)) { -- struct task_struct *p; -- -- p = bpf_task_from_pid(pid); -- if (p) { -- update_core_sched_head_seq(p); -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- *cnt = 0; -- } --} -- --/* -- * The distance from the head of the queue scaled by the weight of the queue. -- * The lower the number, the older the task and the higher the priority. -- */ --static s64 task_qdist(struct task_struct *p) --{ -- int idx = weight_to_idx(p->scx.weight); -- struct task_ctx *tctx; -- s64 qdist; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return 0; -- } -- -- qdist = tctx->core_sched_seq - core_sched_head_seqs[idx]; -- -- /* -- * As queue index increments, the priority doubles. The queue w/ index 3 -- * is dispatched twice more frequently than 2. Reflect the difference by -- * scaling qdists accordingly. Note that the shift amount needs to be -- * flipped depending on the sign to avoid flipping priority direction. -- */ -- if (qdist >= 0) -- return qdist << (4 - idx); -- else -- return qdist << idx; --} -- --/* -- * This is called to determine the task ordering when core-sched is picking -- * tasks to execute on SMT siblings and should encode about the same ordering as -- * the regular scheduling path. Use the priority-scaled distances from the head -- * of the queues to compare the two tasks which should be consistent with the -- * dispatch path behavior. -- */ --bool BPF_STRUCT_OPS(qmap_core_sched_before, -- struct task_struct *a, struct task_struct *b) --{ -- return task_qdist(a) > task_qdist(b); --} -- --void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- u32 cnt; -- -- /* -- * Called when @cpu is taken by a higher priority scheduling class. This -- * makes @cpu no longer available for executing sched_ext tasks. As we -- * don't want the tasks in @cpu's local dsq to sit there until @cpu -- * becomes available again, re-enqueue them into the global dsq. See -- * %SCX_ENQ_REENQ handling in qmap_enqueue(). -- */ -- cnt = scx_bpf_reenqueue_local(); -- if (cnt) -- __sync_fetch_and_add(&nr_reenqueued, cnt); --} -- --s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (p->tgid == disallow_tgid) -- p->scx.disallow = true; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(qmap_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops qmap_ops = { -- .select_cpu = (void *)qmap_select_cpu, -- .enqueue = (void *)qmap_enqueue, -- .dequeue = (void *)qmap_dequeue, -- .dispatch = (void *)qmap_dispatch, -- .core_sched_before = (void *)qmap_core_sched_before, -- .cpu_release = (void *)qmap_cpu_release, -- .prep_enable = (void *)qmap_prep_enable, -- .init = (void *)qmap_init, -- .exit = (void *)qmap_exit, -- .flags = SCX_OPS_ENQ_LAST, -- .timeout_ms = 5000U, -- .name = "qmap", --}; -diff --git a/tools/sched_ext/scx_example_qmap.c b/tools/sched_ext/scx_example_qmap.c -deleted file mode 100644 -index ccb4814ee61b..000000000000 ---- a/tools/sched_ext/scx_example_qmap.c -+++ /dev/null -@@ -1,107 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_qmap.skel.h" -- --const char help_fmt[] = --"A simple five-level FIFO queue sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" --" -t COUNT Stall every COUNT'th user thread\n" --" -T COUNT Stall every COUNT'th kernel thread\n" --" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n" --" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_qmap *skel; -- struct bpf_link *link; -- int opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_qmap__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) { -- switch (opt) { -- case 's': -- skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; -- break; -- case 'e': -- skel->bss->test_error_cnt = strtoul(optarg, NULL, 0); -- break; -- case 't': -- skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0); -- break; -- case 'T': -- skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0); -- break; -- case 'l': -- skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0); -- break; -- case 'd': -- skel->rodata->disallow_tgid = strtol(optarg, NULL, 0); -- if (skel->rodata->disallow_tgid < 0) -- skel->rodata->disallow_tgid = getpid(); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_qmap__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.qmap_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- long nr_enqueued = skel->bss->nr_enqueued; -- long nr_dispatched = skel->bss->nr_dispatched; -- -- printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n", -- nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, -- skel->bss->nr_reenqueued, skel->bss->nr_dequeued, -- skel->bss->nr_core_sched_execed); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_qmap__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_simple.bpf.c b/tools/sched_ext/scx_example_simple.bpf.c -deleted file mode 100644 -index 4bccca3e2047..000000000000 ---- a/tools/sched_ext/scx_example_simple.bpf.c -+++ /dev/null -@@ -1,128 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple scheduler. -- * -- * By default, it operates as a simple global weighted vtime scheduler and can -- * be switched to FIFO scheduling. It also demonstrates the following niceties. -- * -- * - Statistics tracking how many tasks are queued to local and global dsq's. -- * - Termination notification for userspace. -- * -- * While very simple, this scheduler should work reasonably well on CPUs with a -- * uniform L3 cache topology. While preemption is not implemented, the fact that -- * the scheduling queue is shared across all CPUs means that whatever is at the -- * front of the queue is likely to be executed fairly quickly given enough -- * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads -- * but comes with the usual problems with FIFO scheduling where saturating -- * threads can easily drown out interactive ones. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --static u64 vtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, 2); /* [local, global] */ --} stats SEC(".maps"); -- --static void stat_inc(u32 idx) --{ -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); -- if (cnt_p) -- (*cnt_p)++; --} -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) --{ -- /* -- * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that -- * running @p on its CPU directly shouldn't affect fairness. Just queue -- * it on the local FIFO. -- */ -- if (enq_flags & SCX_ENQ_LOCAL) { -- stat_inc(0); /* count local queueing */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- return; -- } -- -- stat_inc(1); /* count global queueing */ -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) -- vtime = vtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --void BPF_STRUCT_OPS(simple_running, struct task_struct *p) --{ -- if (fifo_sched) -- return; -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(vtime_now, p->scx.dsq_vtime)) -- vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --s32 BPF_STRUCT_OPS(simple_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .running = (void *)simple_running, -- .stopping = (void *)simple_stopping, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", --}; -diff --git a/tools/sched_ext/scx_example_simple.c b/tools/sched_ext/scx_example_simple.c -deleted file mode 100644 -index 486b401f7c95..000000000000 ---- a/tools/sched_ext/scx_example_simple.c -+++ /dev/null -@@ -1,101 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_simple.skel.h" -- --const char help_fmt[] = --"A simple sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-f] [-p]\n" --"\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int simple) --{ -- exit_req = 1; --} -- --static void read_stats(struct scx_example_simple *skel, u64 *stats) --{ -- int nr_cpus = libbpf_num_possible_cpus(); -- u64 cnts[2][nr_cpus]; -- u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * 2); -- -- for (idx = 0; idx < 2; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_simple *skel; -- struct bpf_link *link; -- u32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_simple__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "fph")) != -1) { -- switch (opt) { -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_simple__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.simple_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- u64 stats[2]; -- -- read_stats(skel, stats); -- printf("local=%lu global=%lu\n", stats[0], stats[1]); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_simple__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_userland.bpf.c b/tools/sched_ext/scx_example_userland.bpf.c -deleted file mode 100644 -index a089bc6bbe86..000000000000 ---- a/tools/sched_ext/scx_example_userland.bpf.c -+++ /dev/null -@@ -1,269 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A minimal userland scheduler. -- * -- * In terms of scheduling, this provides two different types of behaviors: -- * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity. -- * All such tasks are direct-dispatched from the kernel, and are never -- * enqueued in user space. -- * 2. A primitive vruntime scheduler that is implemented in user space, for all -- * other tasks. -- * -- * Some parts of this example user space scheduler could be implemented more -- * efficiently using more complex and sophisticated data structures. For -- * example, rather than using BPF_MAP_TYPE_QUEUE's, -- * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between -- * user space and kernel space. Similarly, we use a simple vruntime-sorted list -- * in user space, but an rbtree could be used instead. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include --#include "scx_common.bpf.h" --#include "scx_example_userland_common.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; --const volatile s32 usersched_pid; -- --/* !0 for veristat, set during init */ --const volatile u32 num_possible_cpus = 64; -- --/* Stats that are printed by user space. */ --u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues; -- --struct user_exit_info uei; -- --/* -- * Whether the user space scheduler needs to be scheduled due to a task being -- * enqueued in user space. -- */ --static bool usersched_needed; -- --/* -- * The map containing tasks that are enqueued in user space from the kernel. -- * -- * This map is drained by the user space scheduler. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, struct scx_userland_enqueued_task); --} enqueued SEC(".maps"); -- --/* -- * The map containing tasks that are dispatched to the kernel from user space. -- * -- * Drained by the kernel in userland_dispatch(). -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, s32); --} dispatched SEC(".maps"); -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local DSQ */ --}; -- --/* Map that contains task-local storage. */ --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --static bool is_usersched_task(const struct task_struct *p) --{ -- return p->pid == usersched_pid; --} -- --static bool keep_in_kernel(const struct task_struct *p) --{ -- return p->nr_cpus_allowed < num_possible_cpus; --} -- --static struct task_struct *usersched_task(void) --{ -- struct task_struct *p; -- -- p = bpf_task_from_pid(usersched_pid); -- /* -- * Should never happen -- the usersched task should always be managed -- * by sched_ext. -- */ -- if (!p) { -- scx_bpf_error("Failed to find usersched task %d", usersched_pid); -- /* -- * We should never hit this path, and we error out of the -- * scheduler above just in case, so the scheduler will soon be -- * be evicted regardless. So as to simplify the logic in the -- * caller to not have to check for NULL, return an acquired -- * reference to the current task here rather than NULL. -- */ -- return bpf_task_acquire(bpf_get_current_task_btf()); -- } -- -- return p; --} -- --s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- if (keep_in_kernel(p)) { -- s32 cpu; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to look up task-local storage for %s", p->comm); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- tctx->force_local = true; -- return cpu; -- } -- } -- -- return prev_cpu; --} -- --static void dispatch_user_scheduler(void) --{ -- struct task_struct *p; -- -- usersched_needed = false; -- p = usersched_task(); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); --} -- --static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags) --{ -- struct scx_userland_enqueued_task task; -- -- memset(&task, 0, sizeof(task)); -- task.pid = p->pid; -- task.sum_exec_runtime = p->se.sum_exec_runtime; -- task.weight = p->scx.weight; -- -- if (bpf_map_push_elem(&enqueued, &task, 0)) { -- /* -- * If we fail to enqueue the task in user space, put it -- * directly on the global DSQ. -- */ -- __sync_fetch_and_add(&nr_failed_enqueues, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- __sync_fetch_and_add(&nr_user_enqueues, 1); -- usersched_needed = true; -- } --} -- --void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags) --{ -- if (keep_in_kernel(p)) { -- u64 dsq_id = SCX_DSQ_GLOBAL; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to lookup task ctx for %s", p->comm); -- return; -- } -- -- if (tctx->force_local) -- dsq_id = SCX_DSQ_LOCAL; -- tctx->force_local = false; -- scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags); -- __sync_fetch_and_add(&nr_kernel_enqueues, 1); -- return; -- } else if (!is_usersched_task(p)) { -- enqueue_task_in_user_space(p, enq_flags); -- } --} -- --void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (usersched_needed) -- dispatch_user_scheduler(); -- -- bpf_repeat(4096) { -- s32 pid; -- struct task_struct *p; -- -- if (bpf_map_pop_elem(&dispatched, &pid)) -- break; -- -- /* -- * The task could have exited by the time we get around to -- * dispatching it. Treat this as a normal occurrence, and simply -- * move onto the next iteration. -- */ -- p = bpf_task_from_pid(pid); -- if (!p) -- continue; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } --} -- --s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(userland_init) --{ -- if (num_possible_cpus == 0) { -- scx_bpf_error("User scheduler # CPUs uninitialized (%d)", -- num_possible_cpus); -- return -EINVAL; -- } -- -- if (usersched_pid <= 0) { -- scx_bpf_error("User scheduler pid uninitialized (%d)", -- usersched_pid); -- return -EINVAL; -- } -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops userland_ops = { -- .select_cpu = (void *)userland_select_cpu, -- .enqueue = (void *)userland_enqueue, -- .dispatch = (void *)userland_dispatch, -- .prep_enable = (void *)userland_prep_enable, -- .init = (void *)userland_init, -- .exit = (void *)userland_exit, -- .timeout_ms = 3000, -- .name = "userland", --}; -diff --git a/tools/sched_ext/scx_example_userland.c b/tools/sched_ext/scx_example_userland.c -deleted file mode 100644 -index 4152b1e65fe1..000000000000 ---- a/tools/sched_ext/scx_example_userland.c -+++ /dev/null -@@ -1,402 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext user space scheduler which provides vruntime semantics -- * using a simple ordered-list implementation. -- * -- * Each CPU in the system resides in a single, global domain. This precludes -- * the need to do any load balancing between domains. The scheduler could -- * easily be extended to support multiple domains, with load balancing -- * happening in user space. -- * -- * Any task which has any CPU affinity is scheduled entirely in BPF. This -- * program only schedules tasks which may run on any CPU. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include "user_exit_info.h" --#include "scx_example_userland_common.h" --#include "scx_example_userland.skel.h" -- --const char help_fmt[] = --"A minimal userland sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-b BATCH] [-p]\n" --"\n" --" -b BATCH The number of tasks to batch when dispatching (default: 8)\n" --" -p Don't switch all, switch only tasks on SCHED_EXT policy\n" --" -h Display this help and exit\n"; -- --/* Defined in UAPI */ --#define SCHED_EXT 7 -- --/* Number of tasks to batch when dispatching to user space. */ --static __u32 batch_size = 8; -- --static volatile int exit_req; --static int enqueued_fd, dispatched_fd; -- --static struct scx_example_userland *skel; --static struct bpf_link *ops_link; -- --/* Stats collected in user space. */ --static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches; -- --/* The data structure containing tasks that are enqueued in user space. */ --struct enqueued_task { -- LIST_ENTRY(enqueued_task) entries; -- __u64 sum_exec_runtime; -- double vruntime; --}; -- --/* -- * Use a vruntime-sorted list to store tasks. This could easily be extended to -- * a more optimal data structure, such as an rbtree as is done in CFS. We -- * currently elect to use a sorted list to simplify the example for -- * illustrative purposes. -- */ --LIST_HEAD(listhead, enqueued_task); -- --/* -- * A vruntime-sorted list of tasks. The head of the list contains the task with -- * the lowest vruntime. That is, the task that has the "highest" claim to be -- * scheduled. -- */ --static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head); -- --/* -- * The statically allocated array of tasks. We use a statically allocated list -- * here to avoid having to allocate on the enqueue path, which could cause a -- * deadlock. A more substantive user space scheduler could e.g. provide a hook -- * for newly enabled tasks that are passed to the scheduler from the -- * .prep_enable() callback to allows the scheduler to allocate on safe paths. -- */ --struct enqueued_task tasks[USERLAND_MAX_TASKS]; -- --static double min_vruntime; -- --static void sigint_handler(int userland) --{ -- exit_req = 1; --} -- --static __u32 task_pid(const struct enqueued_task *task) --{ -- return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task); --} -- --static int dispatch_task(s32 pid) --{ -- int err; -- -- err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d\n", pid); -- exit_req = 1; -- } else { -- nr_vruntime_dispatches++; -- } -- -- return err; --} -- --static struct enqueued_task *get_enqueued_task(__s32 pid) --{ -- if (pid >= USERLAND_MAX_TASKS) -- return NULL; -- -- return &tasks[pid]; --} -- --static double calc_vruntime_delta(__u64 weight, __u64 delta) --{ -- double weight_f = (double)weight / 100.0; -- double delta_f = (double)delta; -- -- return delta_f / weight_f; --} -- --static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task) --{ -- __u64 delta; -- -- delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime; -- -- enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta); -- if (min_vruntime > enqueued->vruntime) -- enqueued->vruntime = min_vruntime; -- enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime; --} -- --static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task) --{ -- struct enqueued_task *curr, *enqueued, *prev; -- -- curr = get_enqueued_task(bpf_task->pid); -- if (!curr) -- return ENOENT; -- -- update_enqueued(curr, bpf_task); -- nr_vruntime_enqueues++; -- -- /* -- * Enqueue the task in a vruntime-sorted list. A more optimal data -- * structure such as an rbtree could easily be used as well. We elect -- * to use a list here simply because it's less code, and thus the -- * example is less convoluted and better serves to illustrate what a -- * user space scheduler could look like. -- */ -- -- if (LIST_EMPTY(&vruntime_head)) { -- LIST_INSERT_HEAD(&vruntime_head, curr, entries); -- return 0; -- } -- -- LIST_FOREACH(enqueued, &vruntime_head, entries) { -- if (curr->vruntime <= enqueued->vruntime) { -- LIST_INSERT_BEFORE(enqueued, curr, entries); -- return 0; -- } -- prev = enqueued; -- } -- -- LIST_INSERT_AFTER(prev, curr, entries); -- -- return 0; --} -- --static void drain_enqueued_map(void) --{ -- while (1) { -- struct scx_userland_enqueued_task task; -- int err; -- -- if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) -- return; -- -- err = vruntime_enqueue(&task); -- if (err) { -- fprintf(stderr, "Failed to enqueue task %d: %s\n", -- task.pid, strerror(err)); -- exit_req = 1; -- return; -- } -- } --} -- --static void dispatch_batch(void) --{ -- __u32 i; -- -- for (i = 0; i < batch_size; i++) { -- struct enqueued_task *task; -- int err; -- __s32 pid; -- -- task = LIST_FIRST(&vruntime_head); -- if (!task) -- return; -- -- min_vruntime = task->vruntime; -- pid = task_pid(task); -- LIST_REMOVE(task, entries); -- err = dispatch_task(pid); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d in %u\n", -- pid, i); -- return; -- } -- } --} -- --static void *run_stats_printer(void *arg) --{ -- while (!exit_req) { -- __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; -- -- nr_failed_enqueues = skel->bss->nr_failed_enqueues; -- nr_kernel_enqueues = skel->bss->nr_kernel_enqueues; -- nr_user_enqueues = skel->bss->nr_user_enqueues; -- total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues; -- -- printf("o-----------------------o\n"); -- printf("| BPF ENQUEUES |\n"); -- printf("|-----------------------|\n"); -- printf("| kern: %10llu |\n", nr_kernel_enqueues); -- printf("| user: %10llu |\n", nr_user_enqueues); -- printf("| failed: %10llu |\n", nr_failed_enqueues); -- printf("| -------------------- |\n"); -- printf("| total: %10llu |\n", total); -- printf("| |\n"); -- printf("|-----------------------|\n"); -- printf("| VRUNTIME / USER |\n"); -- printf("|-----------------------|\n"); -- printf("| enq: %10llu |\n", nr_vruntime_enqueues); -- printf("| disp: %10llu |\n", nr_vruntime_dispatches); -- printf("o-----------------------o\n"); -- printf("\n\n"); -- sleep(1); -- } -- -- return NULL; --} -- --static int spawn_stats_thread(void) --{ -- pthread_t stats_printer; -- -- return pthread_create(&stats_printer, NULL, run_stats_printer, NULL); --} -- --static int bootstrap(int argc, char **argv) --{ -- int err; -- __u32 opt; -- struct sched_param sched_param = { -- .sched_priority = sched_get_priority_max(SCHED_EXT), -- }; -- bool switch_partial = false; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- /* -- * Enforce that the user scheduler task is managed by sched_ext. The -- * task eagerly drains the list of enqueued tasks in its main work -- * loop, and then yields the CPU. The BPF scheduler only schedules the -- * user space scheduler task when at least one other task in the system -- * needs to be scheduled. -- */ -- err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param); -- if (err) { -- fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err)); -- return err; -- } -- -- while ((opt = getopt(argc, argv, "b:ph")) != -1) { -- switch (opt) { -- case 'b': -- batch_size = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- exit(opt != 'h'); -- } -- } -- -- /* -- * It's not always safe to allocate in a user space scheduler, as an -- * enqueued task could hold a lock that we require in order to be able -- * to allocate. -- */ -- err = mlockall(MCL_CURRENT | MCL_FUTURE); -- if (err) { -- fprintf(stderr, "Failed to prefault and lock address space: %s\n", -- strerror(err)); -- return err; -- } -- -- skel = scx_example_userland__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno)); -- return errno; -- } -- skel->rodata->num_possible_cpus = libbpf_num_possible_cpus(); -- assert(skel->rodata->num_possible_cpus > 0); -- skel->rodata->usersched_pid = getpid(); -- assert(skel->rodata->usersched_pid > 0); -- skel->rodata->switch_partial = switch_partial; -- -- err = scx_example_userland__load(skel); -- if (err) { -- fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- enqueued_fd = bpf_map__fd(skel->maps.enqueued); -- dispatched_fd = bpf_map__fd(skel->maps.dispatched); -- assert(enqueued_fd > 0); -- assert(dispatched_fd > 0); -- -- err = spawn_stats_thread(); -- if (err) { -- fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops); -- if (!ops_link) { -- fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno)); -- err = errno; -- goto destroy_skel; -- } -- -- return 0; -- --destroy_skel: -- scx_example_userland__destroy(skel); -- exit_req = 1; -- return err; --} -- --static void sched_main_loop(void) --{ -- while (!exit_req) { -- /* -- * Perform the following work in the main user space scheduler -- * loop: -- * -- * 1. Drain all tasks from the enqueued map, and enqueue them -- * to the vruntime sorted list. -- * -- * 2. Dispatch a batch of tasks from the vruntime sorted list -- * down to the kernel. -- * -- * 3. Yield the CPU back to the system. The BPF scheduler will -- * reschedule the user space scheduler once another task has -- * been enqueued to user space. -- */ -- drain_enqueued_map(); -- dispatch_batch(); -- sched_yield(); -- } --} -- --int main(int argc, char **argv) --{ -- int err; -- -- err = bootstrap(argc, argv); -- if (err) { -- fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err)); -- return err; -- } -- -- sched_main_loop(); -- -- exit_req = 1; -- bpf_link__destroy(ops_link); -- uei_print(&skel->bss->uei); -- scx_example_userland__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_userland_common.h b/tools/sched_ext/scx_example_userland_common.h -deleted file mode 100644 -index 639c6809c5ff..000000000000 ---- a/tools/sched_ext/scx_example_userland_common.h -+++ /dev/null -@@ -1,19 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* Copyright (c) 2022 Meta, Inc */ -- --#ifndef __SCX_USERLAND_COMMON_H --#define __SCX_USERLAND_COMMON_H -- --#define USERLAND_MAX_TASKS 8192 -- --/* -- * An instance of a task that has been enqueued by the kernel for consumption -- * by a user space global scheduler thread. -- */ --struct scx_userland_enqueued_task { -- __s32 pid; -- u64 sum_exec_runtime; -- u64 weight; --}; -- --#endif // __SCX_USERLAND_COMMON_H -diff --git a/tools/sched_ext/user_exit_info.h b/tools/sched_ext/user_exit_info.h -deleted file mode 100644 -index e701ef0e0b86..000000000000 ---- a/tools/sched_ext/user_exit_info.h -+++ /dev/null -@@ -1,50 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Define struct user_exit_info which is shared between BPF and userspace parts -- * to communicate exit status and other information. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __USER_EXIT_INFO_H --#define __USER_EXIT_INFO_H -- --struct user_exit_info { -- int type; -- char reason[128]; -- char msg[1024]; --}; -- --#ifdef __bpf__ -- --#include "vmlinux.h" --#include -- --static inline void uei_record(struct user_exit_info *uei, -- const struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason); -- bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg); -- /* use __sync to force memory barrier */ -- __sync_val_compare_and_swap(&uei->type, uei->type, ei->type); --} -- --#else /* !__bpf__ */ -- --static inline bool uei_exited(struct user_exit_info *uei) --{ -- /* use __sync to force memory barrier */ -- return __sync_val_compare_and_swap(&uei->type, -1, -1); --} -- --static inline void uei_print(const struct user_exit_info *uei) --{ -- fprintf(stderr, "EXIT: %s", uei->reason); -- if (uei->msg[0] != '\0') -- fprintf(stderr, " (%s)", uei->msg); -- fputs("\n", stderr); --} -- --#endif /* __bpf__ */ --#endif /* __USER_EXIT_INFO_H */ -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -index ce05928392bf..f47f79079297 100755 ---- a/kernel/sched/hmbird/hmbird.c -+++ b/kernel/sched/hmbird/hmbird.c -@@ -633,14 +633,14 @@ static void init_isolate_cpus(void) - WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -- cpumask_set_cpu(0, iso_masks.big); -- cpumask_set_cpu(1, iso_masks.big); -- cpumask_set_cpu(2, iso_masks.big); -- cpumask_set_cpu(3, iso_masks.big); -- cpumask_set_cpu(4, iso_masks.big); -- cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(0, iso_masks.little); -+ cpumask_set_cpu(1, iso_masks.little); -+ cpumask_set_cpu(2, iso_masks.little); -+ cpumask_set_cpu(3, iso_masks.little); -+ cpumask_set_cpu(4, iso_masks.little); -+ cpumask_set_cpu(5, iso_masks.little); - cpumask_set_cpu(6, iso_masks.partial); -- cpumask_set_cpu(7, iso_masks.ex_free); -+ cpumask_set_cpu(7, iso_masks.big); - } - - extern spinlock_t css_set_lock; diff --git a/oneplus/hmbird/fengchi_OP13S_A15.patch b/oneplus/hmbird/fengchi_OP13S_A15.patch deleted file mode 100644 index e5d21faa..00000000 --- a/oneplus/hmbird/fengchi_OP13S_A15.patch +++ /dev/null @@ -1,21125 +0,0 @@ -diff --git a/Documentation/scheduler/index.rst b/Documentation/scheduler/index.rst -index 0b650bb550e6..3170747226f6 100644 ---- a/Documentation/scheduler/index.rst -+++ b/Documentation/scheduler/index.rst -@@ -19,7 +19,6 @@ Scheduler - sched-nice-design - sched-rt-group - sched-stats -- sched-ext - sched-debug - - text_files -diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst -deleted file mode 100644 -index 84c30b44f104..000000000000 ---- a/Documentation/scheduler/sched-ext.rst -+++ /dev/null -@@ -1,230 +0,0 @@ --========================== --Extensible Scheduler Class --========================== -- --sched_ext is a scheduler class whose behavior can be defined by a set of BPF --programs - the BPF scheduler. -- --* sched_ext exports a full scheduling interface so that any scheduling -- algorithm can be implemented on top. -- --* The BPF scheduler can group CPUs however it sees fit and schedule them -- together, as tasks aren't tied to specific CPUs at the time of wakeup. -- --* The BPF scheduler can be turned on and off dynamically anytime. -- --* The system integrity is maintained no matter what the BPF scheduler does. -- The default scheduling behavior is restored anytime an error is detected, -- a runnable task stalls, or on invoking the SysRq key sequence -- :kbd:`SysRq-S`. -- --Switching to and from sched_ext --=============================== -- --``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and --``tools/sched_ext`` contains the example schedulers. -- --sched_ext is used only when the BPF scheduler is loaded and running. -- --If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be --treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is --loaded. On load, such tasks will be switched to and scheduled by sched_ext. -- --The BPF scheduler can choose to schedule all normal and lower class tasks by --calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this --case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and --``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers, --this mode can be selected with the ``-a`` option. -- --Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or --detection of any internal error including stalled runnable tasks aborts the --BPF scheduler and reverts all tasks back to CFS. -- --.. code-block:: none -- -- # make -j16 -C tools/sched_ext -- # tools/sched_ext/scx_example_simple -- local=0 global=3 -- local=5 global=24 -- local=9 global=44 -- local=13 global=56 -- local=17 global=72 -- ^CEXIT: BPF scheduler unregistered -- --If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler --and whether a given task is on sched_ext can be determined as follows: -- --.. code-block:: none -- -- # cat /sys/kernel/debug/sched/ext -- ops : simple -- enabled : 1 -- switching_all : 1 -- switched_all : 1 -- enable_state : enabled -- -- # grep ext /proc/self/sched -- ext.enabled : 1 -- --The Basics --========== -- --Userspace can implement an arbitrary BPF scheduler by loading a set of BPF --programs that implement ``struct sched_ext_ops``. The only mandatory field --is ``ops.name`` which must be a valid BPF object name. All operations are --optional. The following modified excerpt is from --``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO --scheduler. -- --.. code-block:: c -- -- s32 BPF_STRUCT_OPS(simple_init) -- { -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; -- } -- -- void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) -- { -- if (enq_flags & SCX_ENQ_LOCAL) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags); -- } -- -- void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) -- { -- exit_type = ei->type; -- } -- -- SEC(".struct_ops") -- struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", -- }; -- --Dispatch Queues ----------------- -- --To match the impedance between the scheduler core and the BPF scheduler, --sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a --priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``), --and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage --an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and --``scx_bpf_destroy_dsq()``. -- --A CPU always executes a task from its local DSQ. A task is "dispatched" to a --DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's --local DSQ. -- --When a CPU is looking for the next task to run, if the local DSQ is not --empty, the first task is picked. Otherwise, the CPU tries to consume the --global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()`` --is invoked. -- --Scheduling Cycle ------------------ -- --The following briefly shows how a waking task is scheduled and executed. -- --1. When a task is waking up, ``ops.select_cpu()`` is the first operation -- invoked. This serves two purposes. First, CPU selection optimization -- hint. Second, waking up the selected CPU if idle. -- -- The CPU selected by ``ops.select_cpu()`` is an optimization hint and not -- binding. The actual decision is made at the last step of scheduling. -- However, there is a small performance gain if the CPU -- ``ops.select_cpu()`` returns matches the CPU the task eventually runs on. -- -- A side-effect of selecting a CPU is waking it up from idle. While a BPF -- scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper, -- using ``ops.select_cpu()`` judiciously can be simpler and more efficient. -- -- Note that the scheduler core will ignore an invalid CPU selection, for -- example, if it's outside the allowed cpumask of the task. -- --2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can -- make one of the following decisions: -- -- * Immediately dispatch the task to either the global or local DSQ by -- calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or -- ``SCX_DSQ_LOCAL``, respectively. -- -- * Immediately dispatch the task to a custom DSQ by calling -- ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63. -- -- * Queue the task on the BPF side. -- --3. When a CPU is ready to schedule, it first looks at its local DSQ. If -- empty, it then looks at the global DSQ. If there still isn't a task to -- run, ``ops.dispatch()`` is invoked which can use the following two -- functions to populate the local DSQ. -- -- * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can -- be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``, -- ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()`` -- currently can't be called with BPF locks held, this is being worked on -- and will be supported. ``scx_bpf_dispatch()`` schedules dispatching -- rather than performing them immediately. There can be up to -- ``ops.dispatch_max_batch`` pending tasks. -- -- * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ -- to the dispatching DSQ. This function cannot be called with any BPF -- locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks -- before trying to consume the specified DSQ. -- --4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ, -- the CPU runs the first one. If empty, the following steps are taken: -- -- * Try to consume the global DSQ. If successful, run the task. -- -- * If ``ops.dispatch()`` has dispatched any tasks, retry #3. -- -- * If the previous task is an SCX task and still runnable, keep executing -- it (see ``SCX_OPS_ENQ_LAST``). -- -- * Go idle. -- --Note that the BPF scheduler can always choose to dispatch tasks immediately --in ``ops.enqueue()`` as illustrated in the above simple example. If only the --built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as --a task is never queued on the BPF scheduler and both the local and global --DSQs are consumed automatically. -- --``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use --``scx_bpf_dispatch_vtime()`` for the priority queue. See the function --documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for --more information. -- --Where to Look --============= -- --* ``include/linux/sched/ext.h`` defines the core data structures, ops table -- and constants. -- --* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. -- The functions prefixed with ``scx_bpf_`` can be called from the BPF -- scheduler. -- --* ``tools/sched_ext/`` hosts example BPF scheduler implementations. -- -- * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example -- using a custom DSQ. -- -- * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting -- five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. -- --ABI Instability --=============== -- --The APIs provided by sched_ext to BPF schedulers programs have no stability --guarantees. This includes the ops table callbacks and constants defined in --``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in --``kernel/sched/ext.c``. -- --While we will attempt to provide a relatively stable API surface when --possible, they are subject to change without warning between kernel --versions. -diff --git a/MAINTAINERS b/MAINTAINERS -index ff6b9ea9b788..46ca82b0b506 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -19125,8 +19125,6 @@ R: Ben Segall (CONFIG_CFS_BANDWIDTH) - R: Mel Gorman (CONFIG_NUMA_BALANCING) - R: Daniel Bristot de Oliveira (SCHED_DEADLINE) - R: Valentin Schneider (TOPOLOGY) --R: Tejun Heo (SCHED_EXT) --R: David Vernet (SCHED_EXT) - L: linux-kernel@vger.kernel.org - S: Maintained - T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core -@@ -19135,7 +19133,6 @@ F: include/linux/sched.h - F: include/linux/wait.h - F: include/uapi/linux/sched.h - F: kernel/sched/ --F: tools/sched_ext/ - - SCSI LIBSAS SUBSYSTEM - R: John Garry -diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig -index f93980c09d7f..5ba590235a8c 100644 ---- a/arch/arm64/configs/defconfig -+++ b/arch/arm64/configs/defconfig -@@ -6,8 +6,7 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_BPF_SYSCALL=y - CONFIG_BPF_JIT=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_BSD_PROCESS_ACCT=y - CONFIG_BSD_PROCESS_ACCT_V3=y -diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig -index 42c199371c79..ef087e70fba4 100644 ---- a/arch/arm64/configs/gki_defconfig -+++ b/arch/arm64/configs/gki_defconfig -@@ -10,8 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y - # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set - CONFIG_BPF_LSM=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_TASKSTATS=y - CONFIG_TASK_DELAY_ACCT=y -diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c -index 4bd028fa7500..5825ce9d6d7b 100755 ---- a/drivers/gpu/drm/msm/msm_drv.c -+++ b/drivers/gpu/drm/msm/msm_drv.c -@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) - } - - sched_set_fifo(ev_thread->worker->task); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3); -+#endif - } - - ret = drm_vblank_init(ddev, priv->num_crtcs); -diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c -index bd001445815e..dcf2e1ebf9c5 100644 ---- a/drivers/tty/sysrq.c -+++ b/drivers/tty/sysrq.c -@@ -524,7 +524,6 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { - NULL, /* P */ - NULL, /* Q */ - NULL, /* R */ -- /* S: May be registered by sched_ext for resetting */ - NULL, /* S */ - NULL, /* T */ - NULL, /* U */ -diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h -index 3b9dde0c15c0..c0fee44f5417 100755 ---- a/include/asm-generic/vmlinux.lds.h -+++ b/include/asm-generic/vmlinux.lds.h -@@ -131,7 +131,7 @@ - *(__dl_sched_class) \ - *(__rt_sched_class) \ - *(__fair_sched_class) \ -- *(__ext_sched_class) \ -+ *(__hmbird_sched_class) \ - *(__idle_sched_class) \ - __sched_class_lowest = .; - -diff --git a/include/linux/sched.h b/include/linux/sched.h -index 6991b190b8dc..39c797c5cc75 100755 ---- a/include/linux/sched.h -+++ b/include/linux/sched.h -@@ -73,7 +73,9 @@ struct task_delay_info; - struct task_group; - struct user_event_mm; - --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - - /* - * Task state bitmask. NOTE! These bits are also -@@ -298,11 +300,6 @@ enum { - - extern void scheduler_tick(void); - --#ifdef CONFIG_SLIM_SCHED --extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer); --extern void stop_shadow_tick_timer(void); --#endif -- - #define MAX_SCHEDULE_TIMEOUT LONG_MAX - - extern long schedule_timeout(long timeout); -@@ -1523,13 +1520,8 @@ struct task_struct { - */ - struct callback_head l1d_flush_kill; - #endif --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, unsigned long sched_prop); -- ANDROID_KABI_USE(2, struct sched_ext_entity *scx); --#else - ANDROID_KABI_RESERVE(1); - ANDROID_KABI_RESERVE(2); --#endif - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); - ANDROID_KABI_RESERVE(5); -@@ -1932,6 +1924,91 @@ static inline int task_nice(const struct task_struct *p) - return PRIO_TO_NICE((p)->static_prio); - } - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline bool task_is_top_task(struct task_struct *p) -+{ -+ return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ ->top_task_prop & TOP_TASK_BITS_MASK); -+} -+ -+static inline int get_top_task_prop(struct task_struct *p) -+{ -+ return get_hmbird_ts(p)->top_task_prop; -+} -+ -+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear) -+{ -+ if (set) -+ get_hmbird_ts(p)->top_task_prop |= set; -+ if (clear) -+ get_hmbird_ts(p)->top_task_prop &= ~clear; -+ return 0; -+} -+ -+static inline void reset_top_task_prop(struct task_struct *p) -+{ -+ get_hmbird_ts(p)->top_task_prop = 0; -+} -+ -+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ entity->sched_prop = sp; -+ } -+ return 0; -+} -+ -+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->sched_prop; -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq) -+{ -+ unsigned long new_dsq; -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq; -+ entity->sched_prop = new_dsq; -+ } -+} -+ -+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK); -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ entity->dsq_sync_ux = val; -+} -+ -+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->dsq_sync_ux; -+ else -+ return 0; -+} -+#endif - - extern int can_nice(const struct task_struct *p, const int nice); - extern int task_curr(const struct task_struct *p); -diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h -deleted file mode 100755 -index b737a00c1373..000000000000 ---- a/include/linux/sched/ext.h -+++ /dev/null -@@ -1,647 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef _LINUX_SCHED_EXT_H --#define _LINUX_SCHED_EXT_H -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --#include -- --enum scx_consts { -- SCX_OPS_NAME_LEN = 128, -- SCX_EXIT_REASON_LEN = 128, -- SCX_EXIT_BT_LEN = 64, -- SCX_EXIT_MSG_LEN = 1024, -- -- SCX_SLICE_DFL = 20 * NSEC_PER_MSEC, -- SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */ --}; -- --/* -- * DSQ (dispatch queue) IDs are 64bit of the format: -- * -- * Bits: [63] [62 .. 0] -- * [ B] [ ID ] -- * -- * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -- * ID: 63 bit ID -- * -- * Built-in IDs: -- * -- * Bits: [63] [62] [61..32] [31 .. 0] -- * [ 1] [ L] [ R ] [ V ] -- * -- * 1: 1 for built-in DSQs. -- * L: 1 for LOCAL_ON DSQ IDs, 0 for others -- * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -- */ --enum scx_dsq_id_flags { -- SCX_DSQ_FLAG_BUILTIN = 1LLU << 63, -- SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -- -- SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0, -- SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, -- SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, -- SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, -- SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, --}; -- --enum scx_exit_type { -- SCX_EXIT_NONE, -- SCX_EXIT_DONE, -- -- SCX_EXIT_UNREG = 64, /* BPF unregistration */ -- SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -- SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ -- SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ --}; -- --/* -- * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is -- * being disabled. -- */ --struct scx_exit_info { -- /* %SCX_EXIT_* - broad category of the exit reason */ -- enum scx_exit_type type; -- /* textual representation of the above */ -- char reason[SCX_EXIT_REASON_LEN]; -- /* number of entries in the backtrace */ -- u32 bt_len; -- /* backtrace if exiting due to an error */ -- unsigned long bt[SCX_EXIT_BT_LEN]; -- /* extra message */ -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --/* sched_ext_ops.flags */ --enum scx_ops_flags { -- /* -- * Keep built-in idle tracking even if ops.update_idle() is implemented. -- */ -- SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, -- -- /* -- * By default, if there are no other task to run on the CPU, ext core -- * keeps running the current task even after its slice expires. If this -- * flag is specified, such tasks are passed to ops.enqueue() with -- * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. -- */ -- SCX_OPS_ENQ_LAST = 1LLU << 1, -- -- /* -- * An exiting task may schedule after PF_EXITING is set. In such cases, -- * bpf_task_from_pid() may not be able to find the task and if the BPF -- * scheduler depends on pid lookup for dispatching, the task will be -- * lost leading to various issues including RCU grace period stalls. -- * -- * To mask this problem, by default, unhashed tasks are automatically -- * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't -- * depend on pid lookups and wants to handle these tasks directly, the -- * following flag can be used. -- */ -- SCX_OPS_ENQ_EXITING = 1LLU << 2, -- -- /* -- * CPU cgroup knob enable flags -- */ -- SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16, /* cpu.weight */ -- -- SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | -- SCX_OPS_ENQ_LAST | -- SCX_OPS_ENQ_EXITING | -- SCX_OPS_CGROUP_KNOB_WEIGHT, --}; -- --/* argument container for ops.enable() and friends */ --struct scx_enable_args { --}; -- --/* argument container for ops->cgroup_init() */ --struct scx_cgroup_init_args { -- /* the weight of the cgroup [1..10000] */ -- u32 weight; --}; -- --enum scx_cpu_preempt_reason { -- /* next task is being scheduled by &sched_class_rt */ -- SCX_CPU_PREEMPT_RT, -- /* next task is being scheduled by &sched_class_dl */ -- SCX_CPU_PREEMPT_DL, -- /* next task is being scheduled by &sched_class_stop */ -- SCX_CPU_PREEMPT_STOP, -- /* unknown reason for SCX being preempted */ -- SCX_CPU_PREEMPT_UNKNOWN, --}; -- --/* -- * Argument container for ops->cpu_acquire(). Currently empty, but may be -- * expanded in the future. -- */ --struct scx_cpu_acquire_args {}; -- --/* argument container for ops->cpu_release() */ --struct scx_cpu_release_args { -- /* the reason the CPU was preempted */ -- enum scx_cpu_preempt_reason reason; -- -- /* the task that's going to be scheduled on the CPU */ -- struct task_struct *task; --}; -- --/** -- * struct sched_ext_ops - Operation table for BPF scheduler implementation -- * -- * Userland can implement an arbitrary scheduling policy by implementing and -- * loading operations in this table. -- */ --struct sched_ext_ops { -- /** -- * select_cpu - Pick the target CPU for a task which is being woken up -- * @p: task being woken up -- * @prev_cpu: the cpu @p was on before sleeping -- * @wake_flags: SCX_WAKE_* -- * -- * Decision made here isn't final. @p may be moved to any CPU while it -- * is getting dispatched for execution later. However, as @p is not on -- * the rq at this point, getting the eventual execution CPU right here -- * saves a small bit of overhead down the line. -- * -- * If an idle CPU is returned, the CPU is kicked and will try to -- * dispatch. While an explicit custom mechanism can be added, -- * select_cpu() serves as the default way to wake up idle CPUs. -- */ -- s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); -- -- /** -- * enqueue - Enqueue a task on the BPF scheduler -- * @p: task being enqueued -- * @enq_flags: %SCX_ENQ_* -- * -- * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() -- * or enqueue on the BPF scheduler. If not directly dispatched, the bpf -- * scheduler owns @p and if it fails to dispatch @p, the task will -- * stall. -- */ -- void (*enqueue)(struct task_struct *p, u64 enq_flags); -- -- /** -- * dequeue - Remove a task from the BPF scheduler -- * @p: task being dequeued -- * @deq_flags: %SCX_DEQ_* -- * -- * Remove @p from the BPF scheduler. This is usually called to isolate -- * the task while updating its scheduling properties (e.g. priority). -- * -- * The ext core keeps track of whether the BPF side owns a given task or -- * not and can gracefully ignore spurious dispatches from BPF side, -- * which makes it safe to not implement this method. However, depending -- * on the scheduling logic, this can lead to confusing behaviors - e.g. -- * scheduling position not being updated across a priority change. -- */ -- void (*dequeue)(struct task_struct *p, u64 deq_flags); -- -- /** -- * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs -- * @cpu: CPU to dispatch tasks for -- * @prev: previous task being switched out -- * -- * Called when a CPU's local dsq is empty. The operation should dispatch -- * one or more tasks from the BPF scheduler into the DSQs using -- * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using -- * scx_bpf_consume(). -- * -- * The maximum number of times scx_bpf_dispatch() can be called without -- * an intervening scx_bpf_consume() is specified by -- * ops.dispatch_max_batch. See the comments on top of the two functions -- * for more details. -- * -- * When not %NULL, @prev is an SCX task with its slice depleted. If -- * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in -- * @prev->scx.flags, it is not enqueued yet and will be enqueued after -- * ops.dispatch() returns. To keep executing @prev, return without -- * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. -- */ -- void (*dispatch)(s32 cpu, struct task_struct *prev); -- -- /** -- * runnable - A task is becoming runnable on its associated CPU -- * @p: task becoming runnable -- * @enq_flags: %SCX_ENQ_* -- * -- * This and the following three functions can be used to track a task's -- * execution state transitions. A task becomes ->runnable() on a CPU, -- * and then goes through one or more ->running() and ->stopping() pairs -- * as it runs on the CPU, and eventually becomes ->quiescent() when it's -- * done running on the CPU. -- * -- * @p is becoming runnable on the CPU because it's -- * -- * - waking up (%SCX_ENQ_WAKEUP) -- * - being moved from another CPU -- * - being restored after temporarily taken off the queue for an -- * attribute change. -- * -- * This and ->enqueue() are related but not coupled. This operation -- * notifies @p's state transition and may not be followed by ->enqueue() -- * e.g. when @p is being dispatched to a remote CPU. Likewise, a task -- * may be ->enqueue()'d without being preceded by this operation e.g. -- * after exhausting its slice. -- */ -- void (*runnable)(struct task_struct *p, u64 enq_flags); -- -- /** -- * running - A task is starting to run on its associated CPU -- * @p: task starting to run -- * -- * See ->runnable() for explanation on the task state notifiers. -- */ -- void (*running)(struct task_struct *p); -- -- /** -- * stopping - A task is stopping execution -- * @p: task stopping to run -- * @runnable: is task @p still runnable? -- * -- * See ->runnable() for explanation on the task state notifiers. If -- * !@runnable, ->quiescent() will be invoked after this operation -- * returns. -- */ -- void (*stopping)(struct task_struct *p, bool runnable); -- -- /** -- * quiescent - A task is becoming not runnable on its associated CPU -- * @p: task becoming not runnable -- * @deq_flags: %SCX_DEQ_* -- * -- * See ->runnable() for explanation on the task state notifiers. -- * -- * @p is becoming quiescent on the CPU because it's -- * -- * - sleeping (%SCX_DEQ_SLEEP) -- * - being moved to another CPU -- * - being temporarily taken off the queue for an attribute change -- * (%SCX_DEQ_SAVE) -- * -- * This and ->dequeue() are related but not coupled. This operation -- * notifies @p's state transition and may not be preceded by ->dequeue() -- * e.g. when @p is being dispatched to a remote CPU. -- */ -- void (*quiescent)(struct task_struct *p, u64 deq_flags); -- -- /** -- * yield - Yield CPU -- * @from: yielding task -- * @to: optional yield target task -- * -- * If @to is NULL, @from is yielding the CPU to other runnable tasks. -- * The BPF scheduler should ensure that other available tasks are -- * dispatched before the yielding task. Return value is ignored in this -- * case. -- * -- * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf -- * scheduler can implement the request, return %true; otherwise, %false. -- */ -- bool (*yield)(struct task_struct *from, struct task_struct *to); -- -- /** -- * core_sched_before - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Used by core-sched to determine the ordering between two tasks. See -- * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on -- * core-sched. -- * -- * Both @a and @b are runnable and may or may not currently be queued on -- * the BPF scheduler. Should return %true if @a should run before @b. -- * %false if there's no required ordering or @b should run before @a. -- * -- * If not specified, the default is ordering them according to when they -- * became runnable. -- */ -- bool (*core_sched_before)(struct task_struct *a,struct task_struct *b); -- -- /** -- * set_weight - Set task weight -- * @p: task to set weight for -- * @weight: new eight [1..10000] -- * -- * Update @p's weight to @weight. -- */ -- void (*set_weight)(struct task_struct *p, u32 weight); -- -- /** -- * set_cpumask - Set CPU affinity -- * @p: task to set CPU affinity for -- * @cpumask: cpumask of cpus that @p can run on -- * -- * Update @p's CPU affinity to @cpumask. -- */ -- void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask); -- -- /** -- * update_idle - Update the idle state of a CPU -- * @cpu: CPU to udpate the idle state for -- * @idle: whether entering or exiting the idle state -- * -- * This operation is called when @rq's CPU goes or leaves the idle -- * state. By default, implementing this operation disables the built-in -- * idle CPU tracking and the following helpers become unavailable: -- * -- * - scx_bpf_select_cpu_dfl() -- * - scx_bpf_test_and_clear_cpu_idle() -- * - scx_bpf_pick_idle_cpu() -- * - scx_bpf_any_idle_cpu() -- * -- * The user also must implement ops.select_cpu() as the default -- * implementation relies on scx_bpf_select_cpu_dfl(). -- * -- * If you keep the built-in idle tracking, specify the -- * %SCX_OPS_KEEP_BUILTIN_IDLE flag. -- */ -- void (*update_idle)(s32 cpu, bool idle); -- -- /** -- * cpu_acquire - A CPU is becoming available to the BPF scheduler -- * @cpu: The CPU being acquired by the BPF scheduler. -- * @args: Acquire arguments, see the struct definition. -- * -- * A CPU that was previously released from the BPF scheduler is now once -- * again under its control. -- */ -- void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); -- -- /** -- * cpu_release - A CPU is taken away from the BPF scheduler -- * @cpu: The CPU being released by the BPF scheduler. -- * @args: Release arguments, see the struct definition. -- * -- * The specified CPU is no longer under the control of the BPF -- * scheduler. This could be because it was preempted by a higher -- * priority sched_class, though there may be other reasons as well. The -- * caller should consult @args->reason to determine the cause. -- */ -- void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -- -- /** -- * cpu_online - A CPU became online -- * @cpu: CPU which just came up -- * -- * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs beforehand. -- */ -- void (*cpu_online)(s32 cpu); -- -- /** -- * cpu_offline - A CPU is going offline -- * @cpu: CPU which is going offline -- * -- * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs afterwards. -- */ -- void (*cpu_offline)(s32 cpu); -- -- /** -- * prep_enable - Prepare to enable BPF scheduling for a task -- * @p: task to prepare BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Either we're loading a BPF scheduler or a new task is being forked. -- * Prepare BPF scheduling for @p. This operation may block and can be -- * used for allocations. -- * -- * Return 0 for success, -errno for failure. An error return while -- * loading will abort loading of the BPF scheduler. During a fork, will -- * abort the specific fork. -- */ -- s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * enable - Enable BPF scheduling for a task -- * @p: task to enable BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Enable @p for BPF scheduling. @p is now in the cgroup specified for -- * the preceding prep_enable() and will start running soon. -- */ -- void (*enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * cancel_enable - Cancel prep_enable() -- * @p: task being canceled -- * @args: enable arguments, see the struct definition -- * -- * @p was prep_enable()'d but failed before reaching enable(). Undo the -- * preparation. -- */ -- void (*cancel_enable)(struct task_struct *p, -- struct scx_enable_args *args); -- -- /** -- * disable - Disable BPF scheduling for a task -- * @p: task to disable BPF scheduling for -- * -- * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. -- * Disable BPF scheduling for @p. -- */ -- void (*disable)(struct task_struct *p); -- -- /* -- * All online ops must come before ops.init(). -- */ -- -- /** -- * init - Initialize the BPF scheduler -- */ -- s32 (*init)(void); -- -- /** -- * exit - Clean up after the BPF scheduler -- * @info: Exit info -- */ -- void (*exit)(struct scx_exit_info *info); -- -- /** -- * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch -- */ -- u32 dispatch_max_batch; -- -- /** -- * flags - %SCX_OPS_* flags -- */ -- u64 flags; -- -- /** -- * timeout_ms - The maximum amount of time, in milliseconds, that a -- * runnable task should be able to wait before being scheduled. The -- * maximum timeout may not exceed the default timeout of 30 seconds. -- * -- * Defaults to the maximum allowed timeout value of 30 seconds. -- */ -- u32 timeout_ms; -- -- /** -- * name - BPF scheduler's name -- * -- * Must be a non-zero valid BPF object name including only isalnum(), -- * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the -- * BPF scheduler is enabled. -- */ -- char name[SCX_OPS_NAME_LEN]; --}; -- --/* -- * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -- * scheduler core and the BPF scheduler. See the documentation for more details. -- */ --struct scx_dispatch_q { -- raw_spinlock_t lock; -- struct list_head fifo; /* processed in dispatching order */ -- struct rb_root_cached priq; /* processed in p->scx.dsq_vtime order */ -- u32 nr; -- u64 id; -- struct llist_node free_node; -- struct rcu_head rcu; --}; -- --/* scx_entity.flags */ --enum scx_ent_flags { -- SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ -- SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -- SCX_TASK_ENQ_LOCAL = 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */ -- -- SCX_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -- SCX_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -- -- SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -- SCX_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -- -- SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ --}; -- --/* scx_entity.dsq_flags */ --enum scx_ent_dsq_flags { -- SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ --}; -- --/* -- * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from -- * everywhere and the following bits track which kfunc sets are currently -- * allowed for %current. This simple per-task tracking works because SCX ops -- * nest in a limited way. BPF will likely implement a way to allow and disallow -- * kfuncs depending on the calling context which will replace this manual -- * mechanism. See scx_kf_allow(). -- */ --enum scx_kf_mask { -- SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ -- /* all non-sleepables may be nested inside INIT and SLEEPABLE */ -- SCX_KF_INIT = 1 << 0, /* running ops.init() */ -- SCX_KF_SLEEPABLE = 1 << 1, /* other sleepable init operations */ -- /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ -- SCX_KF_CPU_RELEASE = 1 << 2, /* ops.cpu_release() */ -- /* ops.dequeue (in REST) may be nested inside DISPATCH */ -- SCX_KF_DISPATCH = 1 << 3, /* ops.dispatch() */ -- SCX_KF_ENQUEUE = 1 << 4, /* ops.enqueue() */ -- SCX_KF_REST = 1 << 5, /* other rq-locked operations */ -- -- __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | -- SCX_KF_ENQUEUE | SCX_KF_REST, -- __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_REST, --}; -- --#define RAVG_HIST_SIZE 5 --struct scx_sched_task_stats { -- u64 mark_start; -- u64 window_start; -- u32 sum; -- u32 sum_history[RAVG_HIST_SIZE]; -- int cidx; -- -- u32 demand; -- u16 demand_scaled; -- void *sdsq; --}; -- -- -- --/* -- * The following is embedded in task_struct and contains all fields necessary -- * for a task to be scheduled by SCX. -- */ --struct sched_ext_entity { -- struct scx_dispatch_q *dsq; -- struct { -- struct list_head fifo; /* dispatch order */ -- struct rb_node priq; /* p->scx.dsq_vtime order */ -- } dsq_node; -- struct list_head watchdog_node; -- u32 flags; /* protected by rq lock */ -- u32 dsq_flags; /* protected by dsq lock */ -- u32 weight; -- s32 sticky_cpu; -- s32 holding_cpu; -- u32 kf_mask; /* see scx_kf_mask above */ -- struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ -- atomic64_t ops_state; -- unsigned long runnable_at; --#ifdef CONFIG_SCHED_CORE -- u64 core_sched_at; /* see scx_prio_less() */ --#endif -- -- /* BPF scheduler modifiable fields */ -- -- /* -- * Runtime budget in nsecs. This is usually set through -- * scx_bpf_dispatch() but can also be modified directly by the BPF -- * scheduler. Automatically decreased by SCX as the task executes. On -- * depletion, a scheduling event is triggered. -- * -- * This value is cleared to zero if the task is preempted by -- * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the -- * task ran. Use p->se.sum_exec_runtime instead. -- */ -- u64 slice; -- -- /* -- * Used to order tasks when dispatching to the vtime-ordered priority -- * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime() -- * but can also be modified directly by the BPF scheduler. Modifying it -- * while a task is queued on a dsq may mangle the ordering and is not -- * recommended. -- */ -- u64 dsq_vtime; -- -- /* -- * If set, reject future sched_setscheduler(2) calls updating the policy -- * to %SCHED_EXT with -%EACCES. -- * -- * If set from ops.prep_enable() and the task's policy is already -- * %SCHED_EXT, which can happen while the BPF scheduler is being loaded -- * or by inhering the parent's policy during fork, the task's policy is -- * rejected and forcefully reverted to %SCHED_NORMAL. The number of such -- * events are reported through /sys/kernel/debug/sched_ext::nr_rejected. -- */ -- bool disallow; /* reject switching into SCX */ -- -- /* cold fields */ -- struct list_head tasks_node; -- struct task_struct *task; -- struct scx_sched_task_stats sts; --}; -- --void sched_ext_free(struct task_struct *p); -- --#else /* !CONFIG_SCHED_CLASS_EXT */ -- --static inline void sched_ext_free(struct task_struct *p) {} -- --#endif /* CONFIG_SCHED_CLASS_EXT */ --#endif /* _LINUX_SCHED_EXT_H */ -diff --git a/include/linux/sched/hmbird.h b/include/linux/sched/hmbird.h -index 780a545667e9..3c7861e37ade 100755 ---- a/include/linux/sched/hmbird.h -+++ b/include/linux/sched/hmbird.h -@@ -54,6 +54,7 @@ - extern atomic_t non_hmbird_task; - extern atomic_t __hmbird_ops_enabled; - #define hmbird_enabled() atomic_read(&__hmbird_ops_enabled) -+#define MAX_GLOBAL_DSQS (10) - - enum hmbird_consts { - HMBIRD_SLICE_DFL = 1 * NSEC_PER_MSEC, -@@ -90,15 +91,13 @@ enum hmbird_dsq_id_flags { - HMBIRD_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, - }; - --enum hmbird_exit_type { -- HMBIRD_EXIT_NONE, -- HMBIRD_EXIT_DONE, -- -- HMBIRD_EXIT_UNREG = 64, /* BPF unregistration */ -- HMBIRD_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- HMBIRD_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -+enum hmbird_switch_type { -+ HMBIRD_SWITCH_PROC, -+ HMBIRD_SWITCH_ERR_WDT, -+ HMBIRD_SWITCH_ERR_HB, -+ HMBIRD_SWITCH_ERR_DSQ, - HMBIRD_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ -+ HMBIRD_EXIT_ERROR_HEARTBEAT, /* heart beat has stopped */ - }; - - /* -@@ -199,11 +198,176 @@ struct hmbird_entity { - int dsq_sync_ux; - }; - -+ -+/* -+ * All variables use 64bits width, -+ * Avoid parsing problems caused by automatic alignment(with padding) of structures. -+ */ -+ -+/* NOTING : Must align to 64bits. */ -+#define DESC_STR_LEN (32) -+/* Supporti up to three-dimensional arrays. */ -+#define PARSE_DIMENS (3) -+struct meta_desc_t { -+ char desc_str[DESC_STR_LEN]; -+ u64 len; -+ u64 parse[PARSE_DIMENS]; -+}; -+ -+#define MAX_SWITCHS (5) -+struct hmbird_switch_t { -+ u64 switch_at; -+ u64 is_success; -+ u64 end_state; -+ u64 switch_reason; -+}; -+#define SWITCH_ITEMS (sizeof(struct hmbird_switch_t) / sizeof(u64)) -+ -+#define MAX_EXCEPS (5) -+enum excep_id { -+ NO_CGROUP_L1, -+ MODULE_UNLOAD, -+ ALREADY_ENABLED, -+ ALREADY_DISABLED, -+ INIT_TASK_FAIL, -+ ALLOC_RQSCX_FAIL, -+ DSQ_ID_ERR, -+ CPU_NO_MASK, -+ SCAN_ENTITY_NULL, -+ ITER_RET_NULL, -+ DEQ_DEQING, -+ ENQ_EXIST1, -+ ENQ_EXIST2, -+ TASK_LINKED1, -+ TASK_UNLINKED, -+ TASK_LINKED2, -+ TASK_UNQUED, -+ TASK_UNWATCHED, -+ HMBIRD_OPN, -+ TASK_WATCHED, -+ RQ_NO_RUNNING, -+ EXTRA_FLAGS, -+ HOLDING_CPU1, -+ HOLDING_CPU2, -+ TASK_OPS_PREPPED, -+ TASK_OPS_UNPREPPED, -+ HMBIRD_OPS_ERR, -+ MAX_EXCEP_ID, -+}; -+ -+struct snap_misc_t { -+ u64 hmbird_enabled; -+ u64 curr_ss; -+ u64 hmbird_ops_enable_state_var; -+ u64 non_ext_task; -+ u64 parctrl_high_ratio; -+ u64 parctrl_low_ratio; -+ u64 parctrl_high_ratio_l; -+ u64 parctrl_low_ratio_l; -+ u64 isoctrl_high_ratio; -+ u64 isoctrl_low_ratio; -+ u64 misfit_ds; -+ u64 partial_enable; -+ u64 iso_free_rescue; -+ u64 isolate_ctrl; -+ u64 snap_jiffies; -+ u64 snap_time; -+}; -+#define SNAP_ITEMS (sizeof(struct snap_misc_t) / sizeof(u64)) -+ -+struct panic_snapshot_t { -+ /* what time dose the first task of dsq turn in runnable, check starvation */ -+ struct meta_desc_t runnable_at_meta; -+ u64 runnable_at[MAX_GLOBAL_DSQS]; -+ -+ struct meta_desc_t rq_nr_meta; -+ u64 rq_nr[NR_CPUS]; -+ -+ struct meta_desc_t scxrq_nr_meta; -+ u64 scxrq_nr[NR_CPUS]; -+ -+ struct meta_desc_t snap_misc_meta; -+ struct snap_misc_t snap_misc; -+}; -+ -+/* -+ * Do not record info in hot paths unless absolutely necessary, -+ * The impact on performance should be minimized. -+ */ -+struct kernel_info_t { -+ struct meta_desc_t sw_rec_meta; -+ struct hmbird_switch_t sw_rec[MAX_SWITCHS]; -+ -+ struct meta_desc_t sw_idx_meta; -+ u64 sw_idx; -+ -+ struct meta_desc_t excep_rec_meta; -+ u64 excep_rec[MAX_EXCEP_ID][MAX_EXCEPS]; -+ -+ struct meta_desc_t excep_idx_meta; -+ u64 excep_idx[MAX_EXCEP_ID]; -+ -+ /* snapshot while panic. */ -+ struct panic_snapshot_t snap; -+}; -+ -+struct ko_info_t {}; -+ -+struct md_meta_t { -+ u64 self_len; -+ u64 unit_size; -+ u64 desc_meta_len; -+ u64 desc_str_len; -+ u64 switches; -+ u64 exceps; -+ u64 global_dsqs; -+ u64 parse_dimens; -+ u64 nr_cpus; -+ u64 real_cpus; -+ u64 nr_meta_desc; -+ u64 dump_real_size; -+}; -+ -+struct md_info_t { -+ struct md_meta_t meta; -+ struct kernel_info_t kern_dump; -+ struct ko_info_t ko_dump; -+}; -+ -+static inline void exceps_update(struct md_info_t *rec, int id, unsigned long jiffies) -+{ -+ u64 *idx; -+ -+ if (!rec) -+ return; -+ -+ idx = &rec->kern_dump.excep_idx[id]; -+ rec->kern_dump.excep_rec[id][*idx] = jiffies; -+ *idx = ++(*idx) % MAX_EXCEPS; -+} -+ -+static inline void sw_update(struct md_info_t *rec, u64 switch_at, -+ u64 is_success, u64 end_state, u64 switch_reason) -+{ -+ u64 *idx; -+ -+ if (!rec) -+ return; -+ -+ idx = &rec->kern_dump.sw_idx; -+ rec->kern_dump.sw_rec[*idx].switch_at = switch_at; -+ rec->kern_dump.sw_rec[*idx].is_success = is_success; -+ rec->kern_dump.sw_rec[*idx].end_state = end_state; -+ rec->kern_dump.sw_rec[*idx].switch_reason = switch_reason; -+ *idx = ++(*idx) % MAX_SWITCHS; -+} -+ - struct hmbird_ops { - bool (*scx_enable)(void); - bool (*check_non_task)(void); - void (*do_sched_yield_before)(long *skip); - void (*window_rollover_run_once)(struct rq *rq); -+ void (*hmbird_get_md_info)(unsigned long *vaddr, unsigned long *size); - }; - - void hmbird_free(struct task_struct *p); -diff --git a/include/linux/sched/hmbird_proc_val.h b/include/linux/sched/hmbird_proc_val.h -new file mode 100755 -index 000000000000..e59708b16ea3 ---- /dev/null -+++ b/include/linux/sched/hmbird_proc_val.h -@@ -0,0 +1,22 @@ -+#ifndef __HMBORD_PROC_VAL_H__ -+#define __HMBORD_PROC_VAL_H__ -+ -+extern int scx_enable; -+extern int partial_enable; -+extern int cpuctrl_high_ratio; -+extern int cpuctrl_low_ratio; -+extern int slim_stats; -+extern int hmbirdcore_debug; -+extern int slim_for_app; -+extern int misfit_ds; -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+extern int cpu7_tl; -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int slim_gov_debug; -+extern int scx_gov_ctrl; -+extern int sched_ravg_window_frame_per_sec; -+ -+#endif -\ No newline at end of file -diff --git a/include/linux/sched/hmbird_version.h b/include/linux/sched/hmbird_version.h -new file mode 100755 -index 000000000000..b2843881c26f ---- /dev/null -+++ b/include/linux/sched/hmbird_version.h -@@ -0,0 +1,52 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#ifndef _OPLUS_HMBIRD_VERSION_H_ -+#define _OPLUS_HMBIRD_VERSION_H_ -+#include -+#include -+#include -+ -+enum hmbird_version { -+ HMBIRD_UNINIT, -+ HMBIRD_GKI_VERSION, -+ HMBIRD_OGKI_VERSION, -+ HMBIRD_UNKNOW_VERSION, -+}; -+ -+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT; -+ -+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type" -+ -+static inline enum hmbird_version get_hmbird_version_type(void) -+{ -+ struct device_node *np = NULL; -+ const char *hmbird_version_str = NULL; -+ if (HMBIRD_UNINIT != hmbird_version_type) -+ return hmbird_version_type; -+ np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH); -+ if (np) { -+ of_property_read_string(np, "type", &hmbird_version_str); -+ if (NULL != hmbird_version_str) { -+ if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) { -+ hmbird_version_type = HMBIRD_OGKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi"); -+ } else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) { -+ hmbird_version_type = HMBIRD_GKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi"); -+ } else { -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi"); -+ } -+ return hmbird_version_type; -+ } -+ } -+ -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION"); -+ return hmbird_version_type; -+} -+ -+#endif /*_OPLUS_HMBIRD_VERSION_H_ */ -diff --git a/include/linux/sched/sa_common.h b/include/linux/sched/sa_common.h -new file mode 100755 -index 000000000000..6f76d7cfd7bf ---- /dev/null -+++ b/include/linux/sched/sa_common.h -@@ -0,0 +1,797 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_COMMON_H_ -+#define _OPLUS_SA_COMMON_H_ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+#include -+#endif -+#include -+#include -+#include -+#include -+#include -+ -+#include "sa_oemdata.h" -+#include "sa_common_struct.h" -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) -+#define OPLUS_UX_EEVDF_COMPATIBLE 1 -+#endif -+ -+#define SA_DEBUG_ON 0 -+ -+#if (SA_DEBUG_ON >= 1) -+#define DEBUG_BUG_ON(x) BUG_ON(x) -+#define DEBUG_WARN_ON(x) WARN_ON(x) -+#else -+#define DEBUG_BUG_ON(x) -+#define DEBUG_WARN_ON(x) -+#endif -+ -+#define ux_err(fmt, ...) \ -+ pr_err("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_warn(fmt, ...) \ -+ pr_warn("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_debug(fmt, ...) \ -+ pr_info("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+ -+#define UX_MSG_LEN 64 -+#define UX_DEPTH_MAX 5 -+ -+/* define for debug */ -+#define DEBUG_SYSTRACE (1 << 0) -+#define DEBUG_FTRACE (1 << 1) -+#define DEBUG_KMSG (1 << 2) /* used for frameboost */ -+#define DEBUG_PIPELINE (1 << 3) -+#define DEBUG_DYNAMIC_HZ (1 << 4) -+#define DEBUG_DYNAMIC_PREEMPT (1 << 5) -+#define DEBUG_AMU_INSTRUCTION (1 << 6) -+#define DEBUG_VERBOSE (1 << 10) /* used for frameboost */ -+#define DEBUG_SET_DSQ_ID (1 << 11) /* used for hmbird */ -+ -+/* define for sched assist feature */ -+#define FEATURE_COMMON (1 << 0) -+#define FEATURE_SPREAD (1 << 1) -+ -+#define UX_EXEC_SLICE (4000000U) -+ -+/* define for sched assist thread type, keep same as the define in java file */ -+#define SA_OPT_CLEAR (0) -+#define SA_TYPE_LIGHT (1 << 0) -+#define SA_TYPE_HEAVY (1 << 1) -+#define SA_TYPE_ANIMATOR (1 << 2) -+/* SA_TYPE_LISTPICK for camera */ -+#define SA_TYPE_LISTPICK (1 << 3) -+#define SA_TYPE_MQ_VIP (1 << 4) -+#define SA_OPT_SET (1 << 7) -+#define SA_OPT_RESET (1 << 8) -+#define SA_OPT_SET_PRIORITY (1 << 9) -+ -+/* The following ux value only used in kernel */ -+#define SA_TYPE_SWIFT (1 << 14) -+/* clear ux type when dequeue */ -+#define SA_TYPE_ONCE (1 << 15) -+#define SA_TYPE_INHERIT (1 << 16) -+/* SA_TYPE_COMBINED isn't marked in ux_state, it only exists in return value of function */ -+#define SA_TYPE_COMBINED (1 << 17) -+#define SA_TYPE_URGENT_MASK (SA_TYPE_LIGHT|SA_TYPE_ANIMATOR|SA_TYPE_SWIFT) -+#define SCHED_ASSIST_UX_MASK (SA_TYPE_LIGHT|SA_TYPE_HEAVY|SA_TYPE_ANIMATOR|SA_TYPE_LISTPICK|SA_TYPE_SWIFT) -+ -+/* load balance operation is performed on the following ux types */ -+#define SCHED_ASSIST_LB_UX (SA_TYPE_SWIFT | SA_TYPE_ANIMATOR | SA_TYPE_LIGHT | SA_TYPE_HEAVY) -+#define POSSIBLE_UX_MASK (SA_TYPE_SWIFT | \ -+ SA_TYPE_LIGHT | \ -+ SA_TYPE_HEAVY | \ -+ SA_TYPE_ANIMATOR | \ -+ SA_TYPE_LISTPICK | \ -+ SA_TYPE_ONCE | \ -+ SA_TYPE_INHERIT) -+ -+#define SCHED_ASSIST_UX_PRIORITY_MASK (0xFF000000) -+#define SCHED_ASSIST_UX_PRIORITY_SHIFT 24 -+ -+#define UX_PRIORITY_TOP_APP 0x0A000000 -+#define UX_PRIORITY_AUDIO 0x0A000000 -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+#define UX_PRIORITY_PIPELINE_UI 0x06000000 -+#define UX_PRIORITY_PIPELINE 0x05000000 -+#endif -+ -+/* define for sched assist scene type, keep same as the define in java file */ -+#define SA_SCENE_OPT_CLEAR (0) -+#define SA_LAUNCH (1 << 0) -+#define SA_SLIDE (1 << 1) -+#define SA_CAMERA (1 << 2) -+#define SA_ANIM_START (1 << 3) /* we care about both launcher and top app */ -+#define SA_ANIM (1 << 4) /* we only care about launcher */ -+#define SA_INPUT (1 << 5) -+#define SA_LAUNCHER_SI (1 << 6) -+#define SA_SCENE_OPT_SET (1 << 7) -+ -+#define ROOT_UID 0 -+#define SYSTEM_UID 1000 -+#define FIRST_APPLICATION_UID 10000 -+#define LAST_APPLICATION_UID 19999 -+#define PER_USER_RANGE 100000 -+/* define for clear IM_FLAG -+if im_flag is 70, it should clear im_flag_audio (70 - 64 = 6) -+eg: gerrit patchset "30438485" -+ */ -+#define IM_FLAG_CLEAR 64 -+ -+extern pid_t save_audio_tgid; -+extern pid_t save_top_app_tgid; -+extern unsigned int top_app_type; -+extern int global_lowend_plat_opt; -+ -+#ifdef CONFIG_OPLUS_SCHED_HALT_MASK_PRT -+/* This must be the same as the definition of pause_type in walt_halt.c */ -+enum oplus_pause_type { -+ OPLUS_HALT, -+ OPLUS_PARTIAL_HALT, -+ -+ OPLUS_MAX_PAUSE_TYPE -+}; -+extern cpumask_t cur_cpus_halt_mask; -+extern cpumask_t cur_cpus_phalt_mask; -+DECLARE_PER_CPU(int[OPLUS_MAX_PAUSE_TYPE], oplus_cur_pause_client); -+extern void sa_corectl_systrace_c(void); -+#endif /* CONFIG_OPLUS_SCHED_HALT_MASK_PRT */ -+ -+/* define for boost threshold unit */ -+#define BOOST_THRESHOLD_UNIT (51) -+ -+ -+enum UX_STATE_TYPE { -+ UX_STATE_INVALID = 0, -+ UX_STATE_NONE, -+ UX_STATE_STATIC, -+ UX_STATE_INHERIT, -+ UX_STATE_COMBINED, -+ MAX_UX_STATE_TYPE, -+}; -+ -+enum INHERIT_UX_TYPE { -+ INHERIT_UX_BINDER = 0, -+ INHERIT_UX_RWSEM, -+ INHERIT_UX_MUTEX, -+ INHERIT_UX_FUTEX, -+ INHERIT_UX_PIFUTEX, -+ INHERIT_UX_MAX, -+}; -+ -+/* -+ * WANNING: -+ * new flag should be add before MAX_IM_FLAG_TYPE, never change -+ * the value of those existed flag type. -+ */ -+enum IM_FLAG_TYPE { -+ IM_FLAG_NONE = 0, -+ IM_FLAG_SURFACEFLINGER, -+ IM_FLAG_HWC, /* Discarded */ -+ IM_FLAG_RENDERENGINE, -+ IM_FLAG_WEBVIEW, -+ IM_FLAG_CAMERA_HAL, -+ IM_FLAG_AUDIO, -+ IM_FLAG_HWBINDER, -+ IM_FLAG_LAUNCHER, -+ IM_FLAG_LAUNCHER_NON_UX_RENDER, -+ IM_FLAG_SS_LOCK_OWNER, -+ IM_FLAG_FORBID_SET_CPU_AFFINITY = 11, /* forbid setting cpu affinity from app */ -+ IM_FLAG_SYSTEMSERVER_PID, -+ IM_FLAG_MIDASD, -+ IM_FLAG_AUDIO_CAMERA_HAL, /* audio mode disable camera hal ux */ -+ IM_FLAG_AFFINITY_THREAD, -+ IM_FLAG_TPD_SET_CPU_AFFINITY = 16, -+ IM_FLAG_COMPRESS_THREAD = 17, /* compress thread skips locking protect */ -+ IM_FLAG_RENDER_THREAD = 18, -+ MAX_IM_FLAG_TYPE, -+}; -+ -+#define MAX_IM_FLAG_PRIO MAX_IM_FLAG_TYPE -+ -+enum ots_state { -+ OTS_STATE_SET_AFFINITY, -+ OTS_STATE_DDL_ACTIVE, -+ OTS_STATE_DDL_ACTIVE_PREEMPTED, -+ OTS_STATE_MAX, -+}; -+ -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+DECLARE_PER_CPU(u64, retired_instrs); -+DECLARE_PER_CPU(u64, nvcsw); -+DECLARE_PER_CPU(u64, nivcsw); -+#endif -+ -+struct ux_sched_cluster { -+ struct cpumask cpus; -+ unsigned long capacity; -+}; -+ -+#define OPLUS_NR_CPUS (8) -+#define OPLUS_MAX_CLS (5) -+struct ux_sched_cputopo { -+ int cls_nr; -+ struct ux_sched_cluster sched_cls[OPLUS_NR_CPUS]; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ cpumask_t oplus_cpu_array[2*OPLUS_MAX_CLS][OPLUS_MAX_CLS]; -+#endif -+}; -+ -+struct oplus_rq { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_root_cached ux_list; -+ /* a tree to track minimum exec vruntime */ -+ struct rb_root_cached exec_timeline; -+ /* malloc this spinlock_t instead of built-in to shrank size of oplus_rq */ -+ spinlock_t *ux_list_lock; -+ int nr_running; -+ u64 min_vruntime; -+ u64 load_weight; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) -+ struct rb_root_cached ddl_root; -+ spinlock_t *ddl_lock; -+#endif -+ -+#ifdef CONFIG_LOCKING_PROTECT -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ struct list_head locking_thread_list; -+ spinlock_t *locking_list_lock; -+ int rq_locking_task; -+#else -+ struct sched_entity *last_entity; -+#endif -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ struct oplus_lb lb; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+ struct hrtimer *resched_timer; -+ int cpu; -+#endif -+}; -+ -+extern int global_debug_enabled; -+extern int global_sched_assist_enabled; -+extern int global_sched_assist_scene; -+extern int global_silver_perf_core; -+extern int global_sched_group_enabled; -+ -+struct rq; -+ -+#ifdef CONFIG_LOCKING_PROTECT -+struct sched_assist_locking_ops { -+ void (*check_preempt_tick)(struct task_struct *p, -+ unsigned long *ideal_runtime, bool *skip_preempt, -+ unsigned long delta_exec, struct cfs_rq *cfs_rq, -+ struct sched_entity *curr, unsigned int granularity); -+ void (*check_preempt_wakeup)(struct rq *rq, struct task_struct *p, bool *preempt, bool *nopreempt); -+ void (*state_systrace_c)(unsigned int cpu, struct task_struct *p); -+ void (*locking_tick_hit)(struct task_struct *prev, struct task_struct *next); -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ void (*replace_next_task_fair)(struct rq *rq, -+ struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*enqueue_entity)(struct rq *rq, struct task_struct *p); -+ void (*dequeue_entity)(struct rq *rq, struct task_struct *p); -+#else -+ void (*set_last_entity)(struct task_struct *prev, struct task_struct *next, struct rq *rq); -+ void (*pick_last_entity)(struct rq *rq, struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*clear_last_entity)(struct rq *rq, struct task_struct *p); -+#endif -+ void (*opt_ss_lock_contention)(struct task_struct *p, int old_im, int new_im); -+}; -+ -+extern struct sched_assist_locking_ops *locking_ops; -+ -+ -+#define LOCKING_CALL_OP(op, args...) \ -+do { \ -+ if (locking_ops && locking_ops->op) { \ -+ locking_ops->op(args); \ -+ } \ -+} while (0) -+ -+#define LOCKING_CALL_OP_RET(op, args...) \ -+({ \ -+ __typeof__(locking_ops->op(args)) __ret = 0; \ -+ if (locking_ops && locking_ops->op) \ -+ __ret = locking_ops->op(args); \ -+ __ret; \ -+}) -+ -+void register_sched_assist_locking_ops(struct sched_assist_locking_ops *ops); -+#endif -+ -+/* attention: before insert .ko, task's list->prev/next is init with 0 */ -+static inline bool oplus_list_empty(struct list_head *list) -+{ -+ return list_empty(list) || (list->prev == 0 && list->next == 0); -+} -+ -+static inline bool oplus_rbtree_empty(struct rb_root_cached *list) -+{ -+ return (rb_first_cached(list) == NULL); -+} -+ -+/** -+ * Check if there are ux tasks waiting to run on the specified cpu -+ */ -+static inline bool orq_has_ux_tasks(struct oplus_rq *orq) -+{ -+ return !oplus_rbtree_empty(&orq->ux_list); -+} -+ -+/* attention: before insert .ko, task's node->__rb_parent_color is init with 0 */ -+static inline bool oplus_rbnode_empty(struct rb_node *node) -+{ -+ return RB_EMPTY_NODE(node) || (node->__rb_parent_color == 0); -+} -+ -+static inline struct oplus_task_struct *ux_list_first_entry(struct rb_root_cached *list) -+{ -+ struct rb_node *leftmost = rb_first_cached(list); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, ux_entry); -+} -+ -+static inline struct oplus_task_struct *exec_timeline_first_entry(struct rb_root_cached *tree) -+{ -+ struct rb_node *leftmost = rb_first_cached(tree); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, exec_time_node); -+} -+ -+typedef bool (*migrate_task_callback_t)(struct task_struct *tsk, int src_cpu, int dst_cpu); -+extern migrate_task_callback_t fbg_migrate_task_callback; -+typedef void (*android_rvh_schedule_handler_t)(struct task_struct *prev, -+ struct task_struct *next, struct rq *rq); -+extern android_rvh_schedule_handler_t fbg_android_rvh_schedule_callback; -+ -+extern struct kmem_cache *oplus_task_struct_cachep; -+ -+#define ots_to_ts(ots) (ots->task) -+#define OTS_IDX 0 -+#define ORQ_IDX 0 -+ -+static inline bool test_task_is_fair(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid CFS priority is MAX_RT_PRIO..MAX_PRIO-1 */ -+ if ((task->prio >= MAX_RT_PRIO) && (task->prio <= MAX_PRIO-1)) -+ return true; -+ return false; -+} -+ -+static inline bool test_task_is_rt(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid RT priority is 0..MAX_RT_PRIO-1 */ -+ if (rt_prio(task->prio)) -+ return true; -+ -+ return false; -+} -+ -+static inline struct oplus_task_struct *get_oplus_task_struct(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = NULL; -+ -+ /* not Skip idle thread */ -+ if (!t) -+ return NULL; -+ -+ ots = (struct oplus_task_struct *) READ_ONCE(t->android_oem_data1[OTS_IDX]); -+ if (IS_ERR_OR_NULL(ots)) -+ return NULL; -+ -+ return ots; -+} -+ -+static inline unsigned long oplus_get_im_flag(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return IM_FLAG_NONE; -+ -+ return ots->im_flag; -+} -+ -+static inline bool is_optimized_audio_thread(struct task_struct *t) -+{ -+ unsigned long im_flag; -+ -+ im_flag = oplus_get_im_flag(t); -+ if (test_bit(IM_FLAG_AUDIO, &im_flag)) -+ return true; -+ -+ return false; -+} -+ -+static inline void oplus_set_im_flag(struct task_struct *t, int im_flag) -+{ -+ struct oplus_task_struct *ots = NULL; -+ ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ set_bit(im_flag, &ots->im_flag); -+} -+ -+static inline int get_ots_ux_state(struct oplus_task_struct *ots) -+{ -+ int ux_state; -+ int sub_ux_state; -+ int ux_prio; -+ int ret; -+ -+ ux_prio = ots->ux_state & SCHED_ASSIST_UX_PRIORITY_MASK; -+ ux_state = ots->ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ sub_ux_state = ots->sub_ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ -+ if (sub_ux_state) { -+ ret = ux_state | (sub_ux_state & ~SA_TYPE_INHERIT) | SA_TYPE_COMBINED; -+ } else { -+ ret = ux_state; -+ } -+ -+ if (ret & SCHED_ASSIST_UX_MASK) { -+ return ux_prio | ret; -+ } -+ -+ return 0; -+} -+ -+bool is_multiple_ux(struct oplus_task_struct *ots); -+ -+static inline int oplus_get_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return get_ots_ux_state(ots); -+} -+ -+static inline int oplus_get_static_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return 0; -+ } -+ return ots->ux_state; -+} -+ -+static inline int oplus_get_sub_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->sub_ux_state; -+} -+ -+static inline int oplus_get_inherited_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return ots->ux_state; -+ } -+ return ots->sub_ux_state; -+} -+ -+void oplus_set_ux_state_lock(struct task_struct *t, int ux_state, int inherit_type, bool need_lock_rq); -+ -+static inline s64 oplus_get_inherit_ux(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return atomic64_read(&ots->inherit_ux); -+} -+ -+static inline void oplus_set_inherit_ux(struct task_struct *t, s64 inherit_ux) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ atomic64_set(&ots->inherit_ux, inherit_ux); -+} -+ -+static inline int oplus_get_ux_depth(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->ux_depth; -+} -+ -+static inline void oplus_set_ux_depth(struct task_struct *t, int ux_depth) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->ux_depth = ux_depth; -+} -+ -+static inline u64 oplus_get_enqueue_time(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->enqueue_time; -+} -+ -+static inline void oplus_set_enqueue_time(struct task_struct *t, u64 enqueue_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->enqueue_time = enqueue_time; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* -+ * Record the number of task context switches -+ * and scheduling delays when enqueueing a task. -+ */ -+ ots->snap_pcount = t->sched_info.pcount; -+ ots->snap_run_delay = t->sched_info.run_delay; -+#endif -+} -+ -+static inline void oplus_set_inherit_ux_start(struct task_struct *t, u64 start_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->inherit_ux_start = t->se.sum_exec_runtime; -+} -+ -+static inline void init_task_ux_info(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ RB_CLEAR_NODE(&ots->ux_entry); -+ RB_CLEAR_NODE(&ots->exec_time_node); -+ ots->ux_state = 0; -+ ots->sub_ux_state = 0; -+ atomic64_set(&ots->inherit_ux, 0); -+ ots->ux_depth = 0; -+ ots->enqueue_time = 0; -+ ots->inherit_ux_start = 0; -+ ots->ux_priority = -1; -+ ots->ux_nice = -1; -+ ots->vruntime = 0; -+ ots->preset_vruntime = 0; -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG) -+ ots->abnormal_flag = 0; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_SCHED_SPREAD -+ ots->lb_state = 0; -+ ots->ld_flag = 0; -+#endif -+ ots->exec_calc_runtime = 0; -+ ots->is_update_runtime = 0; -+ ots->target_process = -1; -+ ots->wake_tid = 0; -+ ots->running_start_time = 0; -+ ots->update_running_start_time = false; -+ ots->last_wake_ts = 0; -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ memset(&ots->lkinfo, 0, sizeof(struct locking_info)); -+ INIT_LIST_HEAD(&ots->lkinfo.node); -+/*#endif*/ -+ ots->block_start_time = 0; -+#ifdef CONFIG_LOCKING_PROTECT -+ INIT_LIST_HEAD(&ots->locking_entry); -+ ots->locking_start_time = 0; -+ ots->locking_depth = 0; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ ots->snap_pcount = 0; -+ ots->snap_run_delay = 0; -+ plist_node_init(&ots->rtb, MAX_IM_FLAG_PRIO); -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+ atomic_set(&ots->pipeline_cpu, -1); -+#endif -+ -+#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO) -+ ots->amu_cycle = 0; -+ ots->amu_instruct = 0; -+#endif -+}; -+ -+static inline bool test_ux_type(struct task_struct *task, int ux_type) -+{ -+ return oplus_get_ux_state(task) & ux_type; -+} -+ -+static inline bool is_heavy_ux_task(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return false; -+ -+ return get_ots_ux_state(ots) & SA_TYPE_HEAVY; -+} -+ -+static inline bool sched_assist_scene(unsigned int scene) -+{ -+ if (unlikely(!global_sched_assist_enabled)) -+ return false; -+ -+ return global_sched_assist_scene & scene; -+} -+ -+static inline unsigned long oplus_task_util(struct task_struct *p) -+{ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+ struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; -+ -+ return wts->demand_scaled; -+#else -+ return READ_ONCE(p->se.avg.util_avg); -+#endif -+} -+ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+static inline u32 task_wts_sum(struct task_struct *tsk) -+{ -+ struct walt_task_struct *wts = (struct walt_task_struct *) tsk->android_vendor_data1; -+ -+ return wts->sum; -+} -+#endif -+ -+bool is_min_cluster(int cpu); -+bool is_max_cluster(int cpu); -+bool is_mid_cluster(int cpu); -+bool im_mali(const char *comm); -+bool is_top(struct task_struct *p); -+bool task_is_runnable(struct task_struct *task); -+struct oplus_rq *get_oplus_rq(struct rq *rq); -+ -+typedef unsigned long (*kallsyms_lookup_name_t)(const char *name); -+extern kallsyms_lookup_name_t _kallsyms_lookup_name; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+int ux_mask_to_prio(int ux_mask); -+int ux_prio_to_mask(int prio); -+#endif -+ -+noinline int tracing_mark_write(const char *buf); -+void hwbinder_systrace_c(unsigned int cpu, int flag); -+void sched_assist_init_oplus_rq(void); -+void queue_ux_thread(struct rq *rq, struct task_struct *p, int enqueue); -+ -+void inherit_ux_inc(struct task_struct *task, int type); -+void inherit_ux_sub(struct task_struct *task, int type, int value); -+void set_inherit_ux(struct task_struct *task, int type, int depth, int inherit_val); -+void reset_inherit_ux(struct task_struct *inherit_task, struct task_struct *ux_task, int reset_type); -+void unset_inherit_ux(struct task_struct *task, int type); -+void unset_inherit_ux_value(struct task_struct *task, int type, int value); -+void inc_inherit_ux_refs(struct task_struct *task, int type); -+void clear_all_inherit_type(struct task_struct *p); -+int get_max_inherit_gran(struct task_struct *p); -+ -+bool is_heavy_load_top_task(struct task_struct *p); -+bool test_task_is_fair(struct task_struct *task); -+bool test_task_is_rt(struct task_struct *task); -+ -+bool test_task_ux(struct task_struct *task); -+bool test_task_ux_depth(int ux_depth); -+bool test_inherit_ux(struct task_struct *task, int type); -+bool test_set_inherit_ux(struct task_struct *task); -+int get_ux_state_type(struct task_struct *task); -+void sched_assist_target_comm(struct task_struct *task, const char *comm); -+unsigned int ux_task_exec_limit(struct task_struct *p); -+ -+void update_ux_sched_cputopo(void); -+bool is_task_util_over(struct task_struct *tsk, int threshold); -+bool oplus_task_misfit(struct task_struct *tsk, int cpu); -+ssize_t oplus_show_cpus(const struct cpumask *mask, char *buf); -+void adjust_rt_lowest_mask(struct task_struct *p, struct cpumask *local_cpu_mask, int ret, bool force_adjust); -+bool sa_skip_rt_sync(struct rq *rq, struct task_struct *p, bool *sync); -+void ux_state_systrace_c(unsigned int cpu, struct task_struct *p); -+bool sa_rt_skip_ux_cpu(int cpu); -+int is_vip_mvp(struct task_struct *p); -+ -+/* s64 account_ux_runtime(struct rq *rq, struct task_struct *curr); */ -+void opt_ss_lock_contention(struct task_struct *p, unsigned long old_im, int new_im); -+ -+/* register vender hook in kernel/sched/topology.c */ -+void android_vh_build_sched_domains_handler(void *unused, bool has_asym); -+ -+/* register vender hook in kernel/sched/rt.c */ -+void android_rvh_select_task_rq_rt_handler(void *unused, struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags, int *new_cpu); -+void android_rvh_find_lowest_rq_handler(void *unused, struct task_struct *p, struct cpumask *local_cpu_mask, int ret, int *best_cpu); -+ -+/* register vender hook in kernel/sched/core.c */ -+void android_rvh_sched_fork_handler(void *unused, struct task_struct *p); -+void android_rvh_after_enqueue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_dequeue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_schedule_handler(void *unused, struct task_struct *prev, struct task_struct *next, struct rq *rq); -+void android_vh_scheduler_tick_handler(void *unused, struct rq *rq); -+ -+void android_vh_account_process_tick_gran_handler(void *unused, struct task_struct *p, struct rq *rq, int user_tick, int *ticks); -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+void sa_sched_switch_handler(void *unused, bool preempt, struct task_struct *prev, struct task_struct *next, unsigned int prev_state); -+#endif -+ -+void set_im_flag_with_bit(int im_flag, struct task_struct *task); -+ -+/* register vendor hook in kernel/cgroup/cgroup-v1.c */ -+void android_vh_cgroup_set_task_handler(void *unused, int ret, struct task_struct *task); -+/* register vendor hook in kernel/signal.c */ -+void android_vh_exit_signal_handler(void *unused, struct task_struct *p); -+void android_rvh_set_cpus_allowed_by_task_handler(void *unused, const struct cpumask *cpu_valid_mask, -+ const struct cpumask *new_mask, struct task_struct *task, unsigned int *dest_cpu); -+void android_rvh_setscheduler_handler(void *unused, struct task_struct *p); -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_BAN_APP_SET_AFFINITY) -+void android_vh_sched_setaffinity_early_handler(void *unused, struct task_struct *task, const struct cpumask *new_mask, bool *skip); -+#endif -+ -+#ifdef CONFIG_OPLUS_SCHED_GROUP_OPT -+void android_vh_reweight_entity_handler(void *unused, struct sched_entity *se); -+#endif -+ -+#ifdef CONFIG_BLOCKIO_UX_OPT -+int sa_blockio_init(void); -+void sa_blockio_exit(void); -+#endif -+extern struct notifier_block process_exit_notifier_block; -+#endif /* _OPLUS_SA_COMMON_H_ */ -diff --git a/include/linux/sched/sa_common_struct.h b/include/linux/sched/sa_common_struct.h -new file mode 100755 -index 000000000000..876feff48b36 ---- /dev/null -+++ b/include/linux/sched/sa_common_struct.h -@@ -0,0 +1,277 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+/* -+this file is splited from the sa_common.h to adapt the OKI, -+IS_ENABLED is not allowed in here, because the macro will not work in OKI. -+*/ -+#ifndef _OPLUS_SA_COMMON_STRUCT_H_ -+#define _OPLUS_SA_COMMON_STRUCT_H_ -+ -+#define MAX_CLUSTER (4) -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+/* hot-thread */ -+struct task_record { -+#define RECOED_WINSIZE (1 << 8) -+#define RECOED_WINIDX_MASK (RECOED_WINSIZE - 1) -+ u8 winidx; -+ u8 count; -+}; -+ -+#define MAX_TASK_COMM_LEN 256 -+struct uid_struct { -+ uid_t uid; -+ u64 uid_total_cycle; -+ u64 uid_total_inst; -+ spinlock_t lock; -+ char leader_comm[TASK_COMM_LEN]; -+ char cmdline[MAX_TASK_COMM_LEN]; -+}; -+ -+struct amu_uid_entry { -+ uid_t uid; -+ struct uid_struct *uid_struct; -+ struct hlist_node node; -+}; -+ -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+struct locking_info { -+ u64 waittime_stamp; -+ u64 holdtime_stamp; -+ /* Used in torture acquire latency statistic.*/ -+ u64 acquire_stamp; -+ /* -+ * mutex or rwsem optimistic spin start time. Because a task -+ * can't spin both on mutex and rwsem at one time, use one common -+ * threshold time is OK. -+ */ -+ u64 opt_spin_start_time; -+ struct task_struct *holder; -+ u32 waittype; -+ bool ux_contrib; -+ /* -+ * Whether task is ux when it's going to be added to mutex or -+ * rwsem waiter list. It helps us check whether there is ux -+ * task on mutex or rwsem waiter list. Also, a task can't be -+ * added to both mutex and rwsem at one time, so use one common -+ * field is OK. -+ */ -+ bool is_block_ux; -+ u32 kill_flag; -+ /* for cfs enqueue smoothly.*/ -+ struct list_head node; -+ struct task_struct *owner; -+ struct list_head lock_head; -+ u64 clear_seq; -+ atomic_t lock_depth; -+}; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+#define RAVG_HIST_SIZE 5 -+#define SCX_SLICE_DFL (1 * NSEC_PER_MSEC) -+#define SCX_SLICE_INF U64_MAX -+#define DEFAULT_CGROUP_DL_IDX (8) -+#define EXT_FLAG_RT_CHANGED (1 << 0) -+#define EXT_FLAG_CFS_CHANGED (1 << 1) -+struct scx_task_stats { -+ u64 mark_start; -+ u64 window_start; -+ u32 sum; -+ u32 sum_history[RAVG_HIST_SIZE]; -+ int cidx; -+ u32 demand; -+ u16 demand_scaled; -+ void *sdsq; -+}; -+/* -+ * The following is embedded in task_struct and contains all fields necessary -+ * for a task to be scheduled by SCX. -+ */ -+struct scx_entity { -+ struct scx_dispatch_q *dsq; -+ struct { -+ struct list_head fifo; /* dispatch order */ -+ struct rb_node priq; /* p->scx.dsq_vtime order */ -+ } dsq_node; -+ u32 flags; /* protected by rq lock */ -+ u32 dsq_flags; /* protected by dsq lock */ -+ s32 sticky_cpu; -+ unsigned long runnable_at; -+ u64 slice; -+ u64 dsq_vtime; -+ int gdsq_idx; -+ int ext_flags; -+ int prio_backup; -+ unsigned long sched_prop; -+ struct scx_task_stats sts; -+}; -+/*#endif*/ -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ -+#define MAX_CPU_FREQ_STATE 32 -+#define MAX_CPU_CNT 8 -+ -+#define STATE_IN_POOL 0 -+#define STATE_ACTIVE 1 -+ -+struct powermodel_freq_task_state { -+ u8 powermodel_enqueued:1; -+ u8 powermodel_index:7; -+}; -+ -+struct powermodel_cpu_task_state { -+ struct oplus_task_struct *ots; -+ struct powermodel_freq_task_state powermodel_freq_task_states[MAX_CPU_FREQ_STATE]; -+ u64 powermodel_last_seq; -+ u64 last_seq; -+ struct list_head node; -+ int state; -+ int pool_id; -+}; -+ -+#endif -+ -+ -+/* Please add your own members of task_struct here :) */ -+struct oplus_task_struct { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_node ux_entry; -+ struct rb_node exec_time_node; -+ struct task_struct *task; -+ atomic64_t inherit_ux; -+ u64 enqueue_time; -+ u64 inherit_ux_start; -+ /* u64 sum_exec_baseline; */ -+ u64 total_exec; -+ u64 vruntime; -+ u64 preset_vruntime; -+ /* contains ux state -+ 1. if static and inherited ux both exist, static ux stores in ux_state, inherited ux in sub_ux_state. -+ 2. if only static ux exists, static ux stores in ux_state. -+ 2. if only inherited ux exists, inherited ux stores in ux_state */ -+ int ux_state; -+ int sub_ux_state; -+ u8 ux_depth; -+ s8 ux_priority; -+ s8 ux_nice; -+ pid_t affinity_pid; -+ pid_t affinity_tgid; -+ unsigned long state; -+ unsigned long im_flag; -+ atomic_t is_vip_mvp; -+ -+/* #if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) */ -+ u64 ddl; -+ u64 ddl_active_ts; -+ u64 runnable_ts; -+ struct rb_node ddl_node; -+/* #endif */ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG)*/ -+ int abnormal_flag; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_SCHED_SPREAD */ -+ int lb_state; -+ int ld_flag:1; -+ /* CONFIG_OPLUS_FEATURE_TASK_LOAD */ -+ int is_update_runtime:1; -+ int target_process; -+ u64 wake_tid; -+ u64 running_start_time; -+ bool update_running_start_time; -+ u64 exec_calc_runtime; -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct task_record record[MAX_CLUSTER]; /* 2*u64 */ -+ u64 block_start_time; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_FRAME_BOOST */ -+ struct list_head fbg_list; -+ raw_spinlock_t fbg_list_entry_lock; -+ bool fbg_running; /* task belongs to a group, and in running */ -+ u16 fbg_state; -+ s8 preferred_cluster_id; -+ s8 fbg_depth; -+ u64 last_wake_ts; -+ int fbg_cur_group; -+/*#ifdef CONFIG_LOCKING_PROTECT*/ -+ unsigned long locking_start_time; -+ unsigned long last_jiffies; -+ struct list_head locking_entry; -+ int locking_depth; -+ int lk_tick_hit; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ struct locking_info lkinfo; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+ struct scx_entity scx; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_FDLEAK_CHECK)*/ -+ u8 fdleak_flag; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+ /* for loadbalance */ -+ struct plist_node rtb; /* rt boost task */ -+ -+ /* -+ * The following variables are used to calculate the time -+ * a task spends in the running/runnable state. -+ */ -+ u64 snap_run_delay; -+ unsigned long snap_pcount; -+/*#endif*/ -+#if IS_ENABLED(CONFIG_OPLUS_SCHED_TUNE) -+ int stune_idx; -+#endif -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE)*/ -+ atomic_t pipeline_cpu; -+/*#endif*/ -+ -+ /* for oplus secure guard */ -+ int sg_flag; -+ int sg_scno; -+ uid_t sg_uid; -+ uid_t sg_euid; -+ gid_t sg_gid; -+ gid_t sg_egid; -+/*#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct uid_struct *uid_struct; -+ u64 amu_instruct; -+ u64 amu_cycle; -+/*#endif*/ -+ /* for binder ux */ -+ int binder_async_ux_enable; -+ bool binder_async_ux_sts; -+ int binder_thread_mode; -+ struct binder_node *binder_thread_node; -+ -+/* for powermodel */ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ struct powermodel_cpu_task_state *powermodel_cpu_task_states[MAX_CPU_CNT]; -+ u64 exec_runtime; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_CFBT) -+ int cfbt_cur_group; -+ bool cfbt_running; -+#endif /* CONFIG_OPLUS_FEATURE_SCHED_CFBT */ -+} ____cacheline_aligned; -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+#define INVALID_PID (-1) -+struct oplus_lb { -+ /* used for active_balance to record the running task. */ -+ pid_t pid; -+}; -+/*#endif*/ -+ -+#endif /* _OPLUS_SA_COMMON_STRUCT_H_ */ -+ -diff --git a/include/linux/sched/sa_oemdata.h b/include/linux/sched/sa_oemdata.h -new file mode 100755 -index 000000000000..ebbbc754e391 ---- /dev/null -+++ b/include/linux/sched/sa_oemdata.h -@@ -0,0 +1,13 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_OEMDATA_H_ -+#define _OPLUS_SA_OEMDATA_H_ -+ -+int sa_oemdata_init(void); -+void sa_oemdata_deinit(void); -+ -+#endif /* _OPLUS_SA_OEMDATA_H_ */ -diff --git a/include/linux/sched/sched_ext.h b/include/linux/sched/sched_ext.h -new file mode 100755 -index 000000000000..9f1afdb8a04b ---- /dev/null -+++ b/include/linux/sched/sched_ext.h -@@ -0,0 +1,57 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef _OPLUS_SCHED_EXT_H -+#define _OPLUS_SCHED_EXT_H -+#include "sa_common.h" -+ -+#define SCHED_PROP_TOP_THREAD_SHIFT (8) -+#define SCHED_PROP_TOP_THREAD_MASK (0xf << SCHED_PROP_TOP_THREAD_SHIFT) -+#define SCHED_PROP_DEADLINE_MASK (0xFF) /* deadline for ext sched class */ -+#define SCHED_PROP_DEADLINE_LEVEL1 (1) /* 1ms for user-aware audio tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL2 (2) /* 2ms for user-aware touch tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL3 (3) /* 4ms for user aware dispaly tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL4 (4) /* 6ms */ -+#define SCHED_PROP_DEADLINE_LEVEL5 (5) /* 8ms */ -+#define SCHED_PROP_DEADLINE_LEVEL6 (6) /* 16ms */ -+#define SCHED_PROP_DEADLINE_LEVEL7 (7) /* 32ms */ -+#define SCHED_PROP_DEADLINE_LEVEL8 (8) /* 64ms */ -+#define SCHED_PROP_DEADLINE_LEVEL9 (9) /* 128ms */ -+ -+static inline int sched_prop_get_top_thread_id(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ return -EPERM; -+ } -+ -+ return ((ots->scx.sched_prop & SCHED_PROP_TOP_THREAD_MASK) >> SCHED_PROP_TOP_THREAD_SHIFT); -+} -+ -+static inline int sched_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_set_sched_prop failed! fn=%s\n", __func__); -+ return -EPERM; -+ } -+ -+ ots->scx.sched_prop = sp; -+ return 0; -+} -+ -+static inline unsigned long sched_get_sched_prop(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_get_sched_prop failed! fn=%s\n", __func__); -+ return (unsigned long)-1; -+ } -+ return ots->scx.sched_prop; -+} -+ -+#endif /*_OPLUS_SCHED_EXT_H */ -diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h -index 03d35e3eda3c..6a5f0c0d74bd 100644 ---- a/include/linux/sched/task.h -+++ b/include/linux/sched/task.h -@@ -61,8 +61,10 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); - extern void init_idle(struct task_struct *idle, int cpu); - - extern int sched_fork(unsigned long clone_flags, struct task_struct *p); --extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); --extern void sched_cancel_fork(struct task_struct *p); -+extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); -+#ifdef CONFIG_HMBIRD_SCHED -+extern void hmbird_cancel_fork(struct task_struct *p); -+#endif - extern void sched_post_fork(struct task_struct *p); - extern void sched_dead(struct task_struct *p); - -diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h -index 359a14cc76a4..969716ab4320 100755 ---- a/include/uapi/linux/sched.h -+++ b/include/uapi/linux/sched.h -@@ -118,7 +118,7 @@ struct clone_args { - /* SCHED_ISO: reserved but not implemented yet */ - #define SCHED_IDLE 5 - #define SCHED_DEADLINE 6 --#define SCHED_EXT 7 -+#define SCHED_HMBIRD 7 - - /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ - #define SCHED_RESET_ON_FORK 0x40000000 -diff --git a/init/Kconfig b/init/Kconfig -index 48c86ee02f26..f0e97bc54978 100644 ---- a/init/Kconfig -+++ b/init/Kconfig -@@ -1021,10 +1021,6 @@ config RT_GROUP_SCHED - realtime bandwidth for them. - See Documentation/scheduler/sched-rt-group.rst for more information. - --config EXT_GROUP_SCHED -- bool -- depends on SCHED_CLASS_EXT && CGROUP_SCHED -- default y - endif #CGROUP_SCHED - - config SCHED_MM_CID -diff --git a/init/init_task.c b/init/init_task.c -index 69a046388995..31ceb0e469f7 100755 ---- a/init/init_task.c -+++ b/init/init_task.c -@@ -6,7 +6,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -102,9 +101,6 @@ struct task_struct init_task - #endif - #ifdef CONFIG_CGROUP_SCHED - .sched_task_group = &root_task_group, --#endif --#ifdef CONFIG_SLIM_SCHED -- .scx = NULL, - #endif - .ptraced = LIST_HEAD_INIT(init_task.ptraced), - .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), -diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt -index cf22ef6107b8..b131d5662b48 100755 ---- a/kernel/Kconfig.preempt -+++ b/kernel/Kconfig.preempt -@@ -133,12 +133,8 @@ config SCHED_CORE - which is the likely usage by Linux distributions, there should - be no measurable impact on performance. - --config SLIM_SCHED -- bool "slim sched" -- default n --config SCHED_CLASS_EXT -- bool "Extensible Scheduling Class" -- depends on BPF_SYSCALL && BPF_JIT -+config HMBIRD_SCHED -+ bool "hmbird Scheduling Class(base on ext scheduler)" - help - This option enables a new scheduler class sched_ext (SCX), which - allows scheduling policies to be implemented as BPF programs to -@@ -159,3 +155,10 @@ config SCHED_CLASS_EXT - similar to struct sched_class. - - See Documentation/scheduler/sched-ext.rst for more details. -+ -+config DYNAMIC_WALT_SUPPORT -+ bool "Dynamic WALT Support for Extensible Scheduling Class" -+ depends on HMBIRD_SCHED -+ default n -+ help -+ Enable dynamic WALT support for Extensible Scheduling Class. -diff --git a/kernel/bpf/bpf_struct_ops_types.h b/kernel/bpf/bpf_struct_ops_types.h -index 3618769d853d..5678a9ddf817 100644 ---- a/kernel/bpf/bpf_struct_ops_types.h -+++ b/kernel/bpf/bpf_struct_ops_types.h -@@ -9,8 +9,4 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) - #include - BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) - #endif --#ifdef CONFIG_SCHED_CLASS_EXT --#include --BPF_STRUCT_OPS_TYPE(sched_ext_ops) --#endif - #endif -diff --git a/kernel/fork.c b/kernel/fork.c -index f202fd6d20a6..6a393a0d638e 100755 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -23,7 +23,9 @@ - #include - #include - #include --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - #include - #include - #include -@@ -997,7 +999,9 @@ void __put_task_struct(struct task_struct *tsk) - WARN_ON(refcount_read(&tsk->usage)); - WARN_ON(tsk == current); - -- sched_ext_free(tsk); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_free(tsk); -+#endif - io_uring_free(tsk); - cgroup_free(tsk); - task_numa_free(tsk, true); -@@ -2508,7 +2512,11 @@ __latent_entropy struct task_struct *copy_process( - - retval = perf_event_init_task(p, clone_flags); - if (retval) -- goto bad_fork_sched_cancel_fork; -+#ifdef CONFIG_HMBIRD_SCHED -+ goto bad_fork_hmbird_cancel_fork; -+#else -+ goto bad_fork_cleanup_policy; -+#endif - retval = audit_alloc(p); - if (retval) - goto bad_fork_cleanup_perf; -@@ -2640,9 +2648,7 @@ __latent_entropy struct task_struct *copy_process( - * cgroup specific, it unconditionally needs to place the task on a - * runqueue. - */ -- retval = sched_cgroup_fork(p, args); -- if (retval) -- goto bad_fork_cancel_cgroup; -+ sched_cgroup_fork(p, args); - - /* - * From this point on we must avoid any synchronous user-space -@@ -2688,13 +2694,13 @@ __latent_entropy struct task_struct *copy_process( - /* Don't start children in a dying pid namespace */ - if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { - retval = -ENOMEM; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* Let kill terminate clone/fork in the middle */ - if (fatal_signal_pending(current)) { - retval = -EINTR; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* No more failure paths after this point. */ -@@ -2771,11 +2777,10 @@ __latent_entropy struct task_struct *copy_process( - - return p; - --bad_fork_core_free: -+bad_fork_cancel_cgroup: - sched_core_free(p); - spin_unlock(¤t->sighand->siglock); - write_unlock_irq(&tasklist_lock); --bad_fork_cancel_cgroup: - cgroup_cancel_fork(p, args); - bad_fork_put_pidfd: - if (clone_flags & CLONE_PIDFD) { -@@ -2814,8 +2819,10 @@ __latent_entropy struct task_struct *copy_process( - audit_free(p); - bad_fork_cleanup_perf: - perf_event_free_task(p); --bad_fork_sched_cancel_fork: -- sched_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+bad_fork_hmbird_cancel_fork: -+ hmbird_cancel_fork(p); -+#endif - bad_fork_cleanup_policy: - lockdep_free_task(p); - #ifdef CONFIG_NUMA -diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c -index 005025f55bea..c091539a0f60 100755 ---- a/kernel/sched/build_policy.c -+++ b/kernel/sched/build_policy.c -@@ -28,8 +28,10 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED - #include - #include -+#endif - - #include - -@@ -54,6 +56,10 @@ - #include "cputime.c" - #include "deadline.c" - --#ifdef CONFIG_SCHED_CLASS_EXT --# include "ext.c" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/hmbird_util_track.c" -+#include "hmbird/hmbird_sched_proc.c" -+#include "hmbird/hmbird_shadow_tick.c" -+#include "hmbird/hmbird.c" -+#include "hmbird/hmbird_misc.c" - #endif -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index 85ca05ced366..a258adcea40c 100755 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -96,6 +96,12 @@ - #include "../../io_uring/io-wq.h" - #include "../smpboot.h" - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/slim.h" -+#include "hmbird/hmbird_shadow_tick.h" -+#include "hmbird.h" -+#endif -+ - #include - #include - #include -@@ -170,7 +176,6 @@ __read_mostly int scheduler_running; - - DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); - -- - /* kernel prio, less is more */ - static inline int __task_prio(const struct task_struct *p) - { -@@ -183,8 +188,8 @@ static inline int __task_prio(const struct task_struct *p) - if (p->sched_class == &idle_sched_class) - return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - --#ifdef CONFIG_SCHED_CLASS_EXT -- if (p->sched_class == &ext_sched_class) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (p->sched_class == &hmbird_sched_class) - return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ - #endif - -@@ -217,9 +222,9 @@ static inline bool prio_less(const struct task_struct *a, - if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ - return cfs_prio_less(a, b, in_fi); - --#ifdef CONFIG_SCHED_CLASS_EXT -+#ifdef CONFIG_HMBIRD_SCHED - if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ -- return scx_prio_less(a, b, in_fi); -+ return hmbird_prio_less(a, b, in_fi); - #endif - - return false; -@@ -1276,17 +1281,26 @@ bool sched_can_stop_tick(struct rq *rq) - if (fifo_nr_running) - return true; - -+#ifdef CONFIG_HMBIRD_SCHED - /* -- * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks -+ * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks - * left. For CFS, if there's more than one we need the tick for -- * involuntary preemption. For SCX, ask. -+ * involuntary preemption. For HMBIRD, ask. - */ -- if (!scx_switched_all() && rq->nr_running > 1) -+ if (!hmbird_enabled() && rq->nr_running > 1) - return false; - -- if (scx_enabled() && !scx_can_stop_tick(rq)) -+ if (hmbird_enabled() && !hmbird_can_stop_tick(rq)) - return false; -- -+#else -+ /* -+ * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; -+ * if there's more than one we need the tick for involuntary -+ * preemption. -+ */ -+ if (rq->nr_running > 1) -+ return false; -+#endif - /* - * If there is one task and it has CFS runtime bandwidth constraints - * and it's on the cpu now we don't want to stop the tick. -@@ -2211,10 +2225,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) - } - EXPORT_SYMBOL_GPL(deactivate_task); - --struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) -+#ifdef CONFIG_HMBIRD_SCHED -+struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - { -- struct sched_change_guard cg = { -+ struct hmbird_sched_change_guard cg = { - .rq = rq, - .p = p, - .queued = task_on_rq_queued(p), -@@ -2236,7 +2251,7 @@ sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - return cg; - } - --void sched_change_guard_fini(struct sched_change_guard *cg, int flags) -+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags) - { - if (cg->queued) - enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); -@@ -2244,6 +2259,7 @@ void sched_change_guard_fini(struct sched_change_guard *cg, int flags) - set_next_task(cg->rq, cg->p); - cg->done = true; - } -+#endif - - static inline int __normal_prio(int policy, int rt_prio, int nice) - { -@@ -2309,9 +2325,9 @@ inline int task_curr(const struct task_struct *p) - * this means any call to check_class_changed() must be followed by a call to - * balance_callback(). - */ --void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio) -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) - { - if (prev_class != p->sched_class) { - if (prev_class->switched_from) -@@ -2874,6 +2890,7 @@ static void - __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - { - struct rq *rq = task_rq(p); -+ bool queued, running; - - /* - * This here violates the locking rules for affinity, since we're only -@@ -2892,9 +2909,26 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - else - lockdep_assert_held(&p->pi_lock); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->sched_class->set_cpus_allowed(p, ctx); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ -+ if (queued) { -+ /* -+ * Because __kthread_bind() calls this on blocked tasks without -+ * holding rq->lock. -+ */ -+ lockdep_assert_rq_held(rq); -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); - } -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->sched_class->set_cpus_allowed(p, ctx); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - } - - /* -@@ -3761,7 +3795,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class)) -+#else - if (unlikely(!is_cpu_allowed(p, cpu))) -+#endif - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -@@ -4879,8 +4917,9 @@ late_initcall(sched_core_sysctl_init); - */ - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { -- -+#ifdef CONFIG_HMBIRD_SCHED - int ret; -+#endif - - trace_android_rvh_sched_fork(p); - -@@ -4921,21 +4960,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_reset_on_fork = 0; - } - -- scx_pre_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ ret = hmbird_pre_fork(p); -+ if (ret) -+ goto out_cancel; - - if (dl_prio(p->prio)) { - ret = -EAGAIN; - goto out_cancel; --#ifdef CONFIG_SCHED_CLASS_EXT -- } else if (task_on_scx(p)) { -- p->sched_class = &ext_sched_class; --#endif -+ } else if (task_on_hmbird(p)) { -+ p->sched_class = &hmbird_sched_class; - } else if (rt_prio(p->prio)) { - p->sched_class = &rt_sched_class; - } else { - p->sched_class = &fair_sched_class; - } -- -+#else -+ if (dl_prio(p->prio)) -+ return -EAGAIN; -+ else if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - init_entity_runnable_average(&p->se); - trace_android_rvh_finish_prio_fork(p); - -@@ -4954,12 +5001,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - #endif - return 0; - -+#ifdef CONFIG_HMBIRD_SCHED - out_cancel: -- scx_cancel_fork(p); -+ hmbird_cancel_fork(p); - return ret; -+#endif - } - --int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) -+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - { - unsigned long flags; - -@@ -4986,20 +5035,17 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -- -- return scx_fork(p); --} -- --void sched_cancel_fork(struct task_struct *p) --{ -- scx_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_fork(p); -+#endif - } - - void sched_post_fork(struct task_struct *p) - { - uclamp_post_fork(p); -- -- scx_post_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_post_fork(p); -+#endif - } - - unsigned long to_ratio(u64 period, u64 runtime) -@@ -5864,20 +5910,28 @@ void scheduler_tick(void) - if (sched_feat(LATENCY_WARN) && resched_latency) - resched_latency_warn(cpu, resched_latency); - -- scx_notify_sched_tick(); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_notify_sched_tick(); -+#endif - perf_event_task_tick(); - - if (curr->flags & PF_WQ_WORKER) - wq_worker_tick(curr); - - #ifdef CONFIG_SMP -- if (!scx_switched_all()) { -+#ifdef CONFIG_HMBIRD_SCHED -+ if (!hmbird_enabled()) { -+#endif - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); -+#ifdef CONFIG_HMBIRD_SCHED - } -+#endif - #endif - trace_android_vh_scheduler_tick(rq); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ scheduler_tick_handler(NULL, NULL); -+#endif - } - - #ifdef CONFIG_NO_HZ_FULL -@@ -6179,7 +6233,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ -+#ifdef CONFIG_HMBIRD_SCHED - for_balance_class_range(class, prev->sched_class, &idle_sched_class) { -+#else -+ for_class_range(class, prev->sched_class, &idle_sched_class) { -+#endif - if (class->balance(rq, prev, rf)) - break; - } -@@ -6197,9 +6255,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - const struct sched_class *class; - struct task_struct *p; - -- if (scx_enabled()) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (hmbird_enabled()) - goto restart; -- -+#endif - /* - * Optimization: we know that if all tasks are in the fair class we can - * call that function directly, but only if the @prev task wasn't of a -@@ -6225,13 +6284,21 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - restart: - put_prev_task_balance(rq, prev, rf); - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { - p = class->pick_next_task(rq); - if (p) { -- scx_notify_pick_next_task(rq, p, class); -+ hmbird_notify_pick_next_task(rq, p, class); - return p; - } - } -+#else -+ for_each_class(class) { -+ p = class->pick_next_task(rq); -+ if (p) -+ return p; -+ } -+#endif - - BUG(); /* The idle class should always have a runnable task. */ - } -@@ -6260,7 +6327,11 @@ static inline struct task_struct *pick_task(struct rq *rq) - const struct sched_class *class; - struct task_struct *p; - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { -+#else -+ for_each_class(class) { -+#endif - p = class->pick_task(rq); - if (p) - return p; -@@ -6904,6 +6975,9 @@ static void __sched notrace __schedule(unsigned int sched_mode) - psi_sched_switch(prev, next, !task_on_rq_queued(prev)); - - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#ifdef CONFIG_HMBIRD_SCHED -+ sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#endif - - /* Also unlocks the rq: */ - rq = context_switch(rq, prev, next, &rf); -@@ -7232,24 +7306,31 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag - } - EXPORT_SYMBOL(default_wake_function); - --void __setscheduler_prio(struct task_struct *p, int prio) -+static void __setscheduler_prio(struct task_struct *p, int prio) - { -- /* -- * After switching all rt and fair class to ext, -- * stop class is switched to ext class too. Just -- * skip it. */ -+#ifdef CONFIG_HMBIRD_SCHED -+ bool on_hmbird = task_on_hmbird(p); -+ - if (p->sched_class == &stop_sched_class) -- ;/* do nothing */ -+ ; - else if (dl_prio(prio)) - p->sched_class = &dl_sched_class; --#ifdef CONFIG_SCHED_CLASS_EXT -- else if (task_on_scx(p)) -- p->sched_class = &ext_sched_class; --#endif -+ else if (rt_prio(prio) && on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#else -+ if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; - else if (rt_prio(prio)) - p->sched_class = &rt_sched_class; - else - p->sched_class = &fair_sched_class; -+#endif - - p->prio = prio; - } -@@ -7284,7 +7365,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - */ - void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - { -- int prio, oldprio, queue_flag = -+ int prio, oldprio, queued, running, queue_flag = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *prev_class; - struct rq_flags rf; -@@ -7347,42 +7428,52 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - queue_flag &= ~DEQUEUE_MOVE; - - prev_class = p->sched_class; -- SCHED_CHANGE_BLOCK(rq, p, queue_flag) { -- /* -- * Boosting condition are: -- * 1. -rt task is running and holds mutex A -- * --> -dl task blocks on mutex A -- * -- * 2. -dl task is running and holds mutex A -- * --> -dl task blocks on mutex A and could preempt the -- * running task -- */ -- if (dl_prio(prio)) { -- if (!dl_prio(p->normal_prio) || -- (pi_task && dl_prio(pi_task->prio) && -- dl_entity_preempt(&pi_task->dl, &p->dl))) { -- p->dl.pi_se = pi_task->dl.pi_se; -- queue_flag |= ENQUEUE_REPLENISH; -- } else { -- p->dl.pi_se = &p->dl; -- } -- } else if (rt_prio(prio)) { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- if (oldprio < prio) -- queue_flag |= ENQUEUE_HEAD; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flag); -+ if (running) -+ put_prev_task(rq, p); -+ -+ /* -+ * Boosting condition are: -+ * 1. -rt task is running and holds mutex A -+ * --> -dl task blocks on mutex A -+ * -+ * 2. -dl task is running and holds mutex A -+ * --> -dl task blocks on mutex A and could preempt the -+ * running task -+ */ -+ if (dl_prio(prio)) { -+ if (!dl_prio(p->normal_prio) || -+ (pi_task && dl_prio(pi_task->prio) && -+ dl_entity_preempt(&pi_task->dl, &p->dl))) { -+ p->dl.pi_se = pi_task->dl.pi_se; -+ queue_flag |= ENQUEUE_REPLENISH; - } else { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- else if (rt_prio(oldprio)) -- p->rt.timeout = 0; -- else if (!task_has_idle_policy(p)) -- reweight_task(p, prio - MAX_RT_PRIO); -+ p->dl.pi_se = &p->dl; - } -- -- __setscheduler_prio(p, prio); -+ } else if (rt_prio(prio)) { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ if (oldprio < prio) -+ queue_flag |= ENQUEUE_HEAD; -+ } else { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ else if (rt_prio(oldprio)) -+ p->rt.timeout = 0; -+ else if (!task_has_idle_policy(p)) -+ reweight_task(p, prio - MAX_RT_PRIO); - } - -+ __setscheduler_prio(p, prio); -+ -+ if (queued) -+ enqueue_task(rq, p, queue_flag); -+ if (running) -+ set_next_task(rq, p); -+ - check_class_changed(rq, p, prev_class, oldprio); - out_unlock: - /* Avoid rq from going away on us: */ -@@ -7403,7 +7494,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - - void set_user_nice(struct task_struct *p, long nice) - { -- bool allowed = false; -+ bool queued, running, allowed = false; - int old_prio; - struct rq_flags rf; - struct rq *rq; -@@ -7432,13 +7523,22 @@ void set_user_nice(struct task_struct *p, long nice) - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); -+ if (running) -+ put_prev_task(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->static_prio = NICE_TO_PRIO(nice); -- set_load_weight(p, true); -- old_prio = p->prio; -- p->prio = effective_prio(p); -- } -+ p->static_prio = NICE_TO_PRIO(nice); -+ set_load_weight(p, true); -+ old_prio = p->prio; -+ p->prio = effective_prio(p); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - - /* - * If the task increased its priority or is running and -@@ -7847,7 +7947,7 @@ static int __sched_setscheduler(struct task_struct *p, - bool user, bool pi) - { - int oldpolicy = -1, policy = attr->sched_policy; -- int retval, oldprio, newprio; -+ int retval, oldprio, newprio, queued, running; - const struct sched_class *prev_class; - struct balance_callback *head; - struct rq_flags rf; -@@ -7855,6 +7955,9 @@ static int __sched_setscheduler(struct task_struct *p, - int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct rq *rq; - bool cpuset_locked = false; -+#ifdef CONFIG_HMBIRD_SCHED -+ unsigned long flags; -+#endif - - - /* The pi code expects interrupts enabled */ -@@ -7921,7 +8024,9 @@ static int __sched_setscheduler(struct task_struct *p, - * To be able to change p->policy safely, the appropriate - * runqueue lock must be held. - */ -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+#endif - rq = task_rq_lock(p, &rf); - update_rq_clock(rq); - -@@ -7933,10 +8038,11 @@ static int __sched_setscheduler(struct task_struct *p, - goto unlock; - } - -- retval = scx_check_setscheduler(p, policy); -+#ifdef CONFIG_HMBIRD_SCHED -+ retval = hmbird_check_setscheduler(p, policy); - if (retval) - goto unlock; -- -+#endif - /* - * If not changing anything there's no need to proceed further, - * but store a possible modification of reset_on_fork. -@@ -7993,7 +8099,9 @@ static int __sched_setscheduler(struct task_struct *p, - if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { - policy = oldpolicy = -1; - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - goto recheck; -@@ -8026,16 +8134,23 @@ static int __sched_setscheduler(struct task_struct *p, - queue_flags &= ~DEQUEUE_MOVE; - } - -- SCHED_CHANGE_BLOCK(rq, p, queue_flags) { -- prev_class = p->sched_class; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flags); -+ if (running) -+ put_prev_task(rq, p); -+ -+ prev_class = p->sched_class; - -- if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -- __setscheduler_params(p, attr); -- __setscheduler_prio(p, newprio); -- trace_android_rvh_setscheduler(p); -- } -- __setscheduler_uclamp(p, attr); -+ if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -+ __setscheduler_params(p, attr); -+ __setscheduler_prio(p, newprio); -+ trace_android_rvh_setscheduler(p); -+ } -+ __setscheduler_uclamp(p, attr); - -+ if (queued) { - /* - * We enqueue to tail when the priority of a task is - * increased (user space view). -@@ -8043,7 +8158,10 @@ static int __sched_setscheduler(struct task_struct *p, - if (oldprio < p->prio) - queue_flags |= ENQUEUE_HEAD; - -+ enqueue_task(rq, p, queue_flags); - } -+ if (running) -+ set_next_task(rq, p); - - check_class_changed(rq, p, prev_class, oldprio); - -@@ -8051,7 +8169,9 @@ static int __sched_setscheduler(struct task_struct *p, - preempt_disable(); - head = splice_balance_callbacks(rq); - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (pi) { - if (cpuset_locked) - cpuset_unlock(); -@@ -8066,7 +8186,9 @@ static int __sched_setscheduler(struct task_struct *p, - - unlock: - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - return retval; -@@ -9288,7 +9410,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - break; - } -@@ -9316,7 +9440,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - } - return ret; -@@ -9621,15 +9747,25 @@ int migrate_task_to(struct task_struct *p, int target_cpu) - */ - void sched_setnuma(struct task_struct *p, int nid) - { -+ bool queued, running; - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) { -- p->numa_preferred_nid = nid; -- } -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE); -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->numa_preferred_nid = nid; - -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - task_rq_unlock(rq, p, &rf); - } - #endif /* CONFIG_NUMA_BALANCING */ -@@ -10195,15 +10331,22 @@ void __init sched_init(void) - int i; - - /* Make sure the linker didn't screw up */ -+#ifdef CONFIG_HMBIRD_SCHED -+#ifdef CONFIG_SMP -+ WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+#endif -+ WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class)); -+#else -+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 || -+ &fair_sched_class != &rt_sched_class + 1 || -+ &rt_sched_class != &dl_sched_class + 1); - #ifdef CONFIG_SMP -- BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+ BUG_ON(&dl_sched_class != &stop_sched_class + 1); - #endif -- BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); -- BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); -- BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); --#ifdef CONFIG_SCHED_CLASS_EXT -- BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); -- BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); - #endif - - wait_bit_init(); -@@ -10374,8 +10517,9 @@ void __init sched_init(void) - balance_push_set(smp_processor_id(), false); - #endif - init_sched_fair_class(); -- init_sched_ext_class(); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ init_sched_hmbird_class(); -+#endif - psi_init(); - - init_uclamp(); -@@ -10785,6 +10929,8 @@ static void sched_change_group(struct task_struct *tsk, struct task_group *group - */ - void sched_move_task(struct task_struct *tsk) - { -+ int queued, running, queue_flags = -+ DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct task_group *group; - struct rq_flags rf; - struct rq *rq; -@@ -10801,18 +10947,27 @@ void sched_move_task(struct task_struct *tsk) - - update_rq_clock(rq); - -- SCHED_CHANGE_BLOCK(rq, tsk, -- DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { -- sched_change_group(tsk, group); -- } -+ running = task_current(rq, tsk); -+ queued = task_on_rq_queued(tsk); - -- /* -- * After changing group, the running task may have joined a throttled -- * one but it's still the running task. Trigger a resched to make sure -- * that task can still run. -- */ -- if (task_current(rq, tsk)) -+ if (queued) -+ dequeue_task(rq, tsk, queue_flags); -+ if (running) -+ put_prev_task(rq, tsk); -+ -+ sched_change_group(tsk, group); -+ -+ if (queued) -+ enqueue_task(rq, tsk, queue_flags); -+ if (running) { -+ set_next_task(rq, tsk); -+ /* -+ * After changing group, the running task may have joined a -+ * throttled one but it's still the running task. Trigger a -+ * resched to make sure that task can still run. -+ */ - resched_curr(rq); -+ } - - unlock: - task_rq_unlock(rq, tsk, &rf); -@@ -10850,6 +11005,10 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) - struct task_group *tg = css_tg(css); - struct task_group *parent = css_tg(css->parent); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_tg_online(tg); -+#endif -+ - if (parent) - sched_online_group(tg, parent); - -@@ -10899,13 +11058,77 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) - } - #endif - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 dl) -+{ -+ int i; -+ -+ for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) { -+ if (dl < HMBIRD_BPF_DSQS_DEADLINE[i]) -+ break; -+ } -+ -+ i = max_t(int, i-1, MIN_CGROUP_DL_IDX); -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return 0; -+ update_cgroup_ids_table(css->cgroup->kn->id, i); -+ -+ return 0; -+} -+ -+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ u8 i; -+ -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX]; -+ i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1); -+ if (i < 0) { -+ pr_err(" <%s> i is %d, less than 0, name is %s\n", -+ __func__, i, css->cgroup->kn->name); -+ i = DEFAULT_CGROUP_DL_IDX; -+ } -+ -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[i]; -+} -+ -+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css, -+ struct task_struct *p, int prio) -+{ -+ if (!css || !rt_prio(prio)) -+ return; -+ -+ if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) { -+ if (!strcmp(css->cgroup->kn->name, "display")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3); -+ else if (!strcmp(css->cgroup->kn->name, "touch")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2); -+ else if (!strcmp(css->cgroup->kn->name, "multimedia")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+ } -+} -+#endif -+ - static void cpu_cgroup_attach(struct cgroup_taskset *tset) - { - struct task_struct *task; - struct cgroup_subsys_state *css; - - cgroup_taskset_for_each(task, css, tset) { -- -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_change_rt_sched_prop(css, task, task->prio); -+#endif - sched_move_task(task); - } - -@@ -11587,7 +11810,13 @@ static struct cftype cpu_legacy_files[] = { - .write_u64 = cpu_uclamp_ls_write_u64, - }, - #endif -- -+#ifdef CONFIG_HMBIRD_SCHED -+ { -+ .name = "hmbird.deadline", -+ .read_u64 = cgroup_read_hmbird_deadline, -+ .write_u64 = cgroup_write_hmbird_deadline, -+ }, -+#endif - { } /* Terminate */ - }; - -@@ -11636,7 +11865,6 @@ static int cpu_local_stat_show(struct seq_file *sf, - } - - #ifdef CONFIG_FAIR_GROUP_SCHED -- - static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, - struct cftype *cft) - { -diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c -index 423404100c9a..2fe1a734e640 100755 ---- a/kernel/sched/debug.c -+++ b/kernel/sched/debug.c -@@ -376,8 +376,8 @@ static __init int sched_init_debug(void) - - debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); - --#ifdef CONFIG_SCHED_CLASS_EXT -- debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops); -+#ifdef CONFIG_HMBIRD_SCHED -+ debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops); - #endif - return 0; - } -@@ -1006,9 +1006,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - SEQ_printf(m, - "---------------------------------------------------------" - "----------\n"); --#ifdef CONFIG_SLIM_SCHED -- SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop); --#endif - - #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) - #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) -@@ -1102,8 +1099,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - P(dl.runtime); - P(dl.deadline); - } --#ifdef CONFIG_SCHED_CLASS_EXT -- __PS("ext.enabled", p->sched_class == &ext_sched_class); -+#ifdef CONFIG_HMBIRD_SCHED -+ __PS("hmbird.enabled", p->sched_class == &hmbird_sched_class); -+ __PS("sched_prop", get_hmbird_ts(p)->sched_prop); -+ __PS("top_task_prop", get_hmbird_ts(p)->top_task_prop); - #endif - #undef PN_SCHEDSTAT - #undef P_SCHEDSTAT -diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c -deleted file mode 100755 -index 4845d441df2b..000000000000 ---- a/kernel/sched/ext.c -+++ /dev/null -@@ -1,3978 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -- --enum scx_internal_consts { -- SCX_NR_ONLINE_OPS = SCX_OP_IDX(init), -- SCX_DSP_DFL_MAX_BATCH = 32, -- SCX_DSP_MAX_LOOPS = 32, -- SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, --}; -- --enum scx_ops_enable_state { -- SCX_OPS_PREPPING, -- SCX_OPS_ENABLING, -- SCX_OPS_ENABLED, -- SCX_OPS_DISABLING, -- SCX_OPS_DISABLED, --}; -- --static inline struct task_group *css_tg(struct cgroup_subsys_state *css) --{ -- return css ? container_of(css, struct task_group, css) : NULL; --} -- --/* -- * sched_ext_entity->ops_state -- * -- * Used to track the task ownership between the SCX core and the BPF scheduler. -- * State transitions look as follows: -- * -- * NONE -> QUEUEING -> QUEUED -> DISPATCHING -- * ^ | | -- * | v v -- * \-------------------------------/ -- * -- * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -- * sites for explanations on the conditions being waited upon and why they are -- * safe. Transitions out of them into NONE or QUEUED must store_release and the -- * waiters should load_acquire. -- * -- * Tracking scx_ops_state enables sched_ext core to reliably determine whether -- * any given task can be dispatched by the BPF scheduler at all times and thus -- * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -- * to try to dispatch any task anytime regardless of its state as the SCX core -- * can safely reject invalid dispatches. -- */ --enum scx_ops_state { -- SCX_OPSS_NONE, /* owned by the SCX core */ -- SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -- SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ -- SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ -- -- /* -- * QSEQ brands each QUEUED instance so that, when dispatch races -- * dequeue/requeue, the dispatcher can tell whether it still has a claim -- * on the task being dispatched. -- */ -- SCX_OPSS_QSEQ_SHIFT = 2, -- SCX_OPSS_STATE_MASK = (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1, -- SCX_OPSS_QSEQ_MASK = ~SCX_OPSS_STATE_MASK, --}; -- --/* -- * During exit, a task may schedule after losing its PIDs. When disabling the -- * BPF scheduler, we need to be able to iterate tasks in every state to -- * guarantee system safety. Maintain a dedicated task list which contains every -- * task between its fork and eventual free. -- */ --static DEFINE_SPINLOCK(scx_tasks_lock); --static LIST_HEAD(scx_tasks); -- --/* ops enable/disable */ --static struct kthread_worker *scx_ops_helper; --static DEFINE_MUTEX(scx_ops_enable_mutex); --DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); --DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); --static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); --static bool scx_switch_all_req; --static bool scx_switching_all; --DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -- --static struct sched_ext_ops scx_ops; --static bool scx_warned_zero_slice; -- --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); --DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); --static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); -- --struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] = -- { [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT }; -- --static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE); --static struct scx_exit_info scx_exit_info; -- --static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0); -- --/* -- * The maximum amount of time in jiffies that a task may be runnable without -- * being scheduled on a CPU. If this timeout is exceeded, it will trigger -- * scx_ops_error(). -- */ --unsigned long scx_watchdog_timeout; -- --/* -- * The last time the delayed work was run. This delayed work relies on -- * ksoftirqd being able to run to service timer interrupts, so it's possible -- * that this work itself could get wedged. To account for this, we check that -- * it's not stalled in the timer tick, and trigger an error if it is. -- */ --unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; -- --static struct delayed_work scx_watchdog_work; -- --/* idle tracking */ --#ifdef CONFIG_SMP --#ifdef CONFIG_CPUMASK_OFFSTACK --#define CL_ALIGNED_IF_ONSTACK --#else --#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp --#endif -- --static struct { -- cpumask_var_t cpu; -- cpumask_var_t smt; --} idle_masks CL_ALIGNED_IF_ONSTACK; -- --static bool __cacheline_aligned_in_smp scx_has_idle_cpus; --#endif /* CONFIG_SMP */ -- --/* for %SCX_KICK_WAIT */ --static u64 __percpu *scx_kick_cpus_pnt_seqs; -- --/* -- * Direct dispatch marker. -- * -- * Non-NULL values are used for direct dispatch from enqueue path. A valid -- * pointer points to the task currently being enqueued. An ERR_PTR value is used -- * to indicate that direct dispatch has already happened. -- */ --static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); -- --/* dispatch queues */ --static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; -- --static LLIST_HEAD(dsqs_to_free); -- --/* dispatch buf */ --struct scx_dsp_buf_ent { -- struct task_struct *task; -- u64 qseq; -- u64 dsq_id; -- u64 enq_flags; --}; -- --static u32 scx_dsp_max_batch; --static struct scx_dsp_buf_ent __percpu *scx_dsp_buf; -- --struct scx_dsp_ctx { -- struct rq *rq; -- struct rq_flags *rf; -- u32 buf_cursor; -- u32 nr_tasks; --}; -- --static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx); -- --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags); --void scx_bpf_kick_cpu(s32 cpu, u64 flags); -- --struct scx_task_iter { -- struct sched_ext_entity cursor; -- struct task_struct *locked; -- struct rq *rq; -- struct rq_flags rf; --}; -- --#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) -- --/* if the highest set bit is N, return a mask with bits [N+1, 31] set */ --static u32 higher_bits(u32 flags) --{ -- return ~((1 << fls(flags)) - 1); --} -- --/* return the mask with only the highest bit set */ --static u32 highest_bit(u32 flags) --{ -- int bit = fls(flags); -- return bit ? 1 << (bit - 1) : 0; --} -- --/* -- * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX -- * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate -- * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check -- * whether it's running from an allowed context. -- * -- * @mask is constant, always inline to cull the mask calculations. -- */ --static __always_inline void scx_kf_allow(u32 mask) --{ -- /* nesting is allowed only in increasing scx_kf_mask order */ -- WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask, -- "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n", -- current->scx->kf_mask, mask); -- current->scx->kf_mask |= mask; --} -- --static void scx_kf_disallow(u32 mask) --{ -- current->scx->kf_mask &= ~mask; --} -- --#define SCX_CALL_OP(mask, op, args...) \ --do { \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- scx_ops.op(args); \ -- } \ --} while (0) -- --#define SCX_CALL_OP_RET(mask, op, args...) \ --({ \ -- __typeof__(scx_ops.op(args)) __ret; \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- __ret = scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- __ret = scx_ops.op(args); \ -- } \ -- __ret; \ --}) -- --/* -- * Some kfuncs are allowed only on the tasks that are subjects of the -- * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such -- * restrictions, the following SCX_CALL_OP_*() variants should be used when -- * invoking scx_ops operations that take task arguments. These can only be used -- * for non-nesting operations due to the way the tasks are tracked. -- * -- * kfuncs which can only operate on such tasks can in turn use -- * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on -- * the specific task. -- */ --#define SCX_CALL_OP_TASK(mask, op, task, args...) \ --do { \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- SCX_CALL_OP(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ --} while (0) -- --#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ --({ \ -- __typeof__(scx_ops.op(task, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- __ret; \ --}) -- --#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ --({ \ -- __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task0; \ -- current->scx->kf_tasks[1] = task1; \ -- __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- current->scx->kf_tasks[1] = NULL; \ -- __ret; \ --}) -- --//#include "./slim_walt.c" -- --/* @mask is constant, always inline to cull unnecessary branches */ --static __always_inline bool scx_kf_allowed(u32 mask) --{ -- if (unlikely(!(current->scx->kf_mask & mask))) { -- scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", -- mask, current->scx->kf_mask); -- return false; -- } -- -- if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) && -- in_interrupt())) { -- scx_ops_error("sleepable kfunc called from non-sleepable context"); -- return false; -- } -- -- /* -- * Enforce nesting boundaries. e.g. A kfunc which can be called from -- * DISPATCH must not be called if we're running DEQUEUE which is nested -- * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE -- * boundary thanks to the above in_interrupt() check. -- */ -- if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && -- (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { -- scx_ops_error("cpu_release kfunc called from a nested operation"); -- return false; -- } -- -- if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && -- (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) { -- scx_ops_error("dispatch kfunc called from a nested operation"); -- return false; -- } -- -- return true; --} -- --/* see SCX_CALL_OP_TASK() */ --static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, -- struct task_struct *p) --{ -- if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED)) -- return false; -- -- if (unlikely((p != current->scx->kf_tasks[0] && -- p != current->scx->kf_tasks[1]))) { -- scx_ops_error("called on a task not being operated on"); -- return false; -- } -- -- return true; --} -- --/** -- * scx_task_iter_init - Initialize a task iterator -- * @iter: iterator to init -- * -- * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized, -- * @iter must eventually be exited with scx_task_iter_exit(). -- * -- * scx_tasks_lock may be released between this and the first next() call or -- * between any two next() calls. If scx_tasks_lock is released between two -- * next() calls, the caller is responsible for ensuring that the task being -- * iterated remains accessible either through RCU read lock or obtaining a -- * reference count. -- * -- * All tasks which existed when the iteration started are guaranteed to be -- * visited as long as they still exist. -- */ --static void scx_task_iter_init(struct scx_task_iter *iter) --{ -- lockdep_assert_held(&scx_tasks_lock); -- -- iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; -- list_add(&iter->cursor.tasks_node, &scx_tasks); -- iter->locked = NULL; --} -- --/** -- * scx_task_iter_exit - Exit a task iterator -- * @iter: iterator to exit -- * -- * Exit a previously initialized @iter. Must be called with scx_tasks_lock held. -- * If the iterator holds a task's rq lock, that rq lock is released. See -- * scx_task_iter_init() for details. -- */ --static void scx_task_iter_exit(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- if (list_empty(cursor)) -- return; -- -- list_del_init(cursor); --} -- --/** -- * scx_task_iter_next - Next task -- * @iter: iterator to walk -- * -- * Visit the next task. See scx_task_iter_init() for details. -- */ --static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- struct sched_ext_entity *pos; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- list_for_each_entry(pos, cursor, tasks_node) { -- if (&pos->tasks_node == &scx_tasks) -- return NULL; -- if (!(pos->flags & SCX_TASK_CURSOR)) { -- list_move(cursor, &pos->tasks_node); -- return pos->task; -- } -- } -- -- /* can't happen, should always terminate at scx_tasks above */ -- BUG(); --} -- --/** -- * scx_task_iter_next_filtered - Next non-idle task -- * @iter: iterator to walk -- * -- * Visit the next non-idle task. See scx_task_iter_init() for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- while ((p = scx_task_iter_next(iter))) { -- if (!is_idle_task(p)) -- return p; -- } -- return NULL; --} -- --/** -- * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked -- * @iter: iterator to walk -- * -- * Visit the next non-idle task with its rq lock held. See scx_task_iter_init() -- * for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered_locked(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- p = scx_task_iter_next_filtered(iter); -- if (!p) -- return NULL; -- -- iter->rq = task_rq_lock(p, &iter->rf); -- iter->locked = p; -- return p; --} -- --static enum scx_ops_enable_state scx_ops_enable_state(void) --{ -- return atomic_read(&scx_ops_enable_state_var); --} -- --static enum scx_ops_enable_state --scx_ops_set_enable_state(enum scx_ops_enable_state to) --{ -- return atomic_xchg(&scx_ops_enable_state_var, to); --} -- --static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, -- enum scx_ops_enable_state from) --{ -- int from_v = from; -- -- return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); --} -- --static bool scx_ops_disabling(void) --{ -- return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING); --} -- --/** -- * wait_ops_state - Busy-wait the specified ops state to end -- * @p: target task -- * @opss: state to wait the end of -- * -- * Busy-wait for @p to transition out of @opss. This can only be used when the -- * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also -- * has load_acquire semantics to ensure that the caller can see the updates made -- * in the enqueueing and dispatching paths. -- */ --static void wait_ops_state(struct task_struct *p, u64 opss) --{ -- do { -- cpu_relax(); -- } while (atomic64_read_acquire(&p->scx->ops_state) == opss); --} -- --/** -- * ops_cpu_valid - Verify a cpu number -- * @cpu: cpu number which came from a BPF ops -- * -- * @cpu is a cpu number which came from the BPF scheduler and can be any value. -- * Verify that it is in range and one of the possible cpus. -- */ --static bool ops_cpu_valid(s32 cpu) --{ -- return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); --} -- --/** -- * ops_sanitize_err - Sanitize a -errno value -- * @ops_name: operation to blame on failure -- * @err: -errno value to sanitize -- * -- * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return -- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can -- * cause misbehaviors. For an example, a large negative return from -- * ops.prep_enable() triggers an oops when passed up the call chain because the -- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is -- * handled as a pointer. -- */ --static int ops_sanitize_err(const char *ops_name, s32 err) --{ -- if (err < 0 && err >= -MAX_ERRNO) -- return err; -- -- scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); -- return -EPROTO; --} -- --/** -- * touch_core_sched - Update timestamp used for core-sched task ordering -- * @rq: rq to read clock from, must be locked -- * @p: task to update the timestamp for -- * -- * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to -- * implement global or local-DSQ FIFO ordering for core-sched. Should be called -- * when a task becomes runnable and its turn on the CPU ends (e.g. slice -- * exhaustion). -- */ --static void touch_core_sched(struct rq *rq, struct task_struct *p) --{ --#ifdef CONFIG_SCHED_CORE -- /* -- * It's okay to update the timestamp spuriously. Use -- * sched_core_disabled() which is cheaper than enabled(). -- */ -- if (!sched_core_disabled()) -- p->scx->core_sched_at = rq_clock_task(rq); --#endif --} -- --/** -- * touch_core_sched_dispatch - Update core-sched timestamp on dispatch -- * @rq: rq to read clock from, must be locked -- * @p: task being dispatched -- * -- * If the BPF scheduler implements custom core-sched ordering via -- * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO -- * ordering within each local DSQ. This function is called from dispatch paths -- * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect. -- */ --static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- assert_clock_updated(rq); -- --#ifdef CONFIG_SCHED_CORE -- if (SCX_HAS_OP(core_sched_before)) -- touch_core_sched(rq, p); --#endif --} -- --static void update_curr_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- u64 now = rq_clock_task(rq); -- u64 delta_exec; -- -- if (time_before_eq64(now, curr->se.exec_start)) -- return; -- -- delta_exec = now - curr->se.exec_start; -- curr->se.exec_start = now; -- curr->se.sum_exec_runtime += delta_exec; -- account_group_exec_runtime(curr, delta_exec); -- cgroup_account_cputime(curr, delta_exec); -- -- if (curr->scx->slice != SCX_SLICE_INF) { -- curr->scx->slice -= min(curr->scx->slice, delta_exec); -- if (!curr->scx->slice) -- touch_core_sched(rq, curr); -- } --} -- --static bool scx_dsq_priq_less(struct rb_node *node_a, -- const struct rb_node *node_b) --{ -- const struct sched_ext_entity *a = -- container_of(node_a, struct sched_ext_entity, dsq_node.priq); -- const struct sched_ext_entity *b = -- container_of(node_b, struct sched_ext_entity, dsq_node.priq); -- -- return time_before64(a->dsq_vtime, b->dsq_vtime); --} -- --static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, -- u64 enq_flags) --{ -- bool is_local = dsq->id == SCX_DSQ_LOCAL; -- -- WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo)); -- WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq)); -- -- if (!is_local) { -- raw_spin_lock(&dsq->lock); -- if (unlikely(dsq->id == SCX_DSQ_INVALID)) { -- scx_ops_error("attempting to dispatch to a destroyed dsq"); -- /* fall back to the global dsq */ -- raw_spin_unlock(&dsq->lock); -- dsq = &scx_dsq_global; -- raw_spin_lock(&dsq->lock); -- } -- } -- -- if (enq_flags & SCX_ENQ_DSQ_PRIQ) { -- p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; -- rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq, -- scx_dsq_priq_less); -- } else { -- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) -- list_add(&p->scx->dsq_node.fifo, &dsq->fifo); -- else -- list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo); -- } -- dsq->nr++; -- p->scx->dsq = dsq; -- -- /* -- * We're transitioning out of QUEUEING or DISPATCHING. store_release to -- * match waiters' load_acquire. -- */ -- if (enq_flags & SCX_ENQ_CLEAR_OPSS) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- if (is_local) { -- struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq); -- struct rq *rq = scx->rq; -- bool preempt = false; -- -- if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && -- rq->curr->sched_class == &ext_sched_class) { -- rq->curr->scx->slice = 0; -- preempt = true; -- } -- -- if (preempt || sched_class_above(&ext_sched_class, -- rq->curr->sched_class)) -- resched_curr(rq); -- } else { -- raw_spin_unlock(&dsq->lock); -- } --} -- --static void task_unlink_from_dsq(struct task_struct *p, -- struct scx_dispatch_q *dsq) --{ -- if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { -- rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; -- } else { -- list_del_init(&p->scx->dsq_node.fifo); -- } -- --} -- --static bool task_linked_on_dsq(struct task_struct *p) --{ -- return !list_empty(&p->scx->dsq_node.fifo) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq); --} -- --static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p) --{ -- struct scx_dispatch_q *dsq = p->scx->dsq; -- bool is_local = dsq == &scx_rq->local_dsq; -- -- if (!dsq) { -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- /* -- * When dispatching directly from the BPF scheduler to a local -- * DSQ, the task isn't associated with any DSQ but -- * @p->scx->holding_cpu may be set under the protection of -- * %SCX_OPSS_DISPATCHING. -- */ -- if (p->scx->holding_cpu >= 0) -- p->scx->holding_cpu = -1; -- return; -- } -- -- if (!is_local) -- raw_spin_lock(&dsq->lock); -- -- /* -- * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node -- * can't change underneath us. -- */ -- if (p->scx->holding_cpu < 0) { -- /* @p must still be on @dsq, dequeue */ -- WARN_ON_ONCE(!task_linked_on_dsq(p)); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- } else { -- /* -- * We're racing against dispatch_to_local_dsq() which already -- * removed @p from @dsq and set @p->scx->holding_cpu. Clear the -- * holding_cpu which tells dispatch_to_local_dsq() that it lost -- * the race. -- */ -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- p->scx->holding_cpu = -1; -- } -- p->scx->dsq = NULL; -- -- if (!is_local) -- raw_spin_unlock(&dsq->lock); --} -- --static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) --{ -- lockdep_assert(rcu_read_lock_any_held()); -- -- return &scx_dsq_global; --} -- --static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, -- struct task_struct *p) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id == SCX_DSQ_LOCAL) -- return &rq->scx->local_dsq; -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", -- dsq_id, p->comm, p->pid); -- return &scx_dsq_global; -- } -- -- return dsq; --} -- --static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- -- /* @p must match the task which is being enqueued */ -- if (unlikely(p != ddsp_task)) { -- if (IS_ERR(ddsp_task)) -- scx_ops_error("%s[%d] already direct-dispatched", -- p->comm, p->pid); -- else -- scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]", -- ddsp_task->comm, ddsp_task->pid, -- p->comm, p->pid); -- return; -- } -- -- /* -- * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because -- * dispatching to the local DSQ of a different CPU requires unlocking -- * the current rq which isn't allowed in the enqueue path. Use -- * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL. -- */ -- if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) { -- scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch"); -- return; -- } -- -- touch_core_sched_dispatch(task_rq(p), p); -- -- dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- -- /* -- * Mark that dispatch already happened by spoiling direct_dispatch_task -- * with a non-NULL value which can never match a valid task pointer. -- */ -- __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); --} -- --static bool test_rq_online(struct rq *rq) --{ --#ifdef CONFIG_SMP -- return rq->online; --#else -- return true; --#endif --} -- --static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- int sticky_cpu) --{ -- struct task_struct **ddsp_taskp; -- u64 qseq; -- -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- if (p->scx->flags & SCX_TASK_ENQ_LOCAL) { -- enq_flags |= SCX_ENQ_LOCAL; -- p->scx->flags &= ~SCX_TASK_ENQ_LOCAL; -- } -- -- /* rq migration */ -- if (sticky_cpu == cpu_of(rq)) -- goto local_norefill; -- -- /* -- * If !rq->online, we already told the BPF scheduler that the CPU is -- * offline. We're just trying to on/offline the CPU. Don't bother the -- * BPF scheduler. -- */ -- if (unlikely(!test_rq_online(rq))) -- goto local; -- -- /* see %SCX_OPS_ENQ_EXITING */ -- if (!static_branch_unlikely(&scx_ops_enq_exiting) && -- unlikely(p->flags & PF_EXITING)) -- goto local; -- -- /* see %SCX_OPS_ENQ_LAST */ -- if (!static_branch_unlikely(&scx_ops_enq_last) && -- (enq_flags & SCX_ENQ_LAST)) -- goto local; -- -- if (!SCX_HAS_OP(enqueue)) { -- if (enq_flags & SCX_ENQ_LOCAL) -- goto local; -- else -- goto global; -- } -- -- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ -- qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; -- -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq); -- -- ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); -- WARN_ON_ONCE(*ddsp_taskp); -- *ddsp_taskp = p; -- -- SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); -- -- /* -- * If not directly dispatched, QUEUEING isn't clear yet and dispatch or -- * dequeue may be waiting. The store_release matches their load_acquire. -- */ -- if (*ddsp_taskp == p) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq); -- *ddsp_taskp = NULL; -- return; -- --local: -- /* -- * For task-ordering, slice refill must be treated as implying the end -- * of the current slice. Otherwise, the longer @p stays on the CPU, the -- * higher priority it becomes from scx_prio_less()'s POV. -- */ -- touch_core_sched(rq, p); -- p->scx->slice = SCX_SLICE_DFL; --local_norefill: -- dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags); -- return; -- --global: -- touch_core_sched(rq, p); /* see the comment in local: */ -- p->scx->slice = SCX_SLICE_DFL; -- dispatch_enqueue(&scx_dsq_global, p, enq_flags); --} -- --static bool watchdog_task_watched(const struct task_struct *p) --{ -- return !list_empty(&p->scx->watchdog_node); --} -- --static void watchdog_watch_task(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- if (p->scx->flags & SCX_TASK_WATCHDOG_RESET) -- p->scx->runnable_at = jiffies; -- p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET; -- list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list); --} -- --static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) --{ -- list_del_init(&p->scx->watchdog_node); -- if (reset_timeout) -- p->scx->flags |= SCX_TASK_WATCHDOG_RESET; --} -- --static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) --{ -- int sticky_cpu = p->scx->sticky_cpu; -- -- enq_flags |= rq->scx->extra_enq_flags; -- -- if (sticky_cpu >= 0) -- p->scx->sticky_cpu = -1; -- -- /* -- * Restoring a running task will be immediately followed by -- * set_next_task_scx() which expects the task to not be on the BPF -- * scheduler as tasks can only start running through local DSQs. Force -- * direct-dispatch into the local DSQ by setting the sticky_cpu. -- */ -- if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -- sticky_cpu = cpu_of(rq); -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- WARN_ON_ONCE(!watchdog_task_watched(p)); -- return; -- } -- -- watchdog_watch_task(rq, p); -- p->scx->flags |= SCX_TASK_QUEUED; -- rq->scx->nr_running++; -- add_nr_running(rq, 1); -- -- if (SCX_HAS_OP(runnable)) -- SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); -- -- if (enq_flags & SCX_ENQ_WAKEUP) -- touch_core_sched(rq, p); -- -- do_enqueue_task(rq, p, enq_flags, sticky_cpu); --} -- --static void ops_dequeue(struct task_struct *p, u64 deq_flags) --{ -- u64 opss; -- -- watchdog_unwatch_task(p, false); -- -- /* acquire ensures that we see the preceding updates on QUEUED */ -- opss = atomic64_read_acquire(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_NONE: -- break; -- case SCX_OPSS_QUEUEING: -- /* -- * QUEUEING is started and finished while holding @p's rq lock. -- * As we're holding the rq lock now, we shouldn't see QUEUEING. -- */ -- BUG(); -- case SCX_OPSS_QUEUED: -- if (SCX_HAS_OP(dequeue)) -- SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); -- -- if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_NONE)) -- break; -- fallthrough; -- case SCX_OPSS_DISPATCHING: -- /* -- * If @p is being dispatched from the BPF scheduler to a DSQ, -- * wait for the transfer to complete so that @p doesn't get -- * added to its DSQ after dequeueing is complete. -- * -- * As we're waiting on DISPATCHING with the rq locked, the -- * dispatching side shouldn't try to lock the rq while -- * DISPATCHING is set. See dispatch_to_local_dsq(). -- * -- * DISPATCHING shouldn't have qseq set and control can reach -- * here with NONE @opss from the above QUEUED case block. -- * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. -- */ -- wait_ops_state(p, SCX_OPSS_DISPATCHING); -- BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- break; -- } --} -- --static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) --{ -- struct scx_rq *scx_rq = rq->scx; -- -- if (!(p->scx->flags & SCX_TASK_QUEUED)) { -- WARN_ON_ONCE(watchdog_task_watched(p)); -- return; -- } -- -- ops_dequeue(p, deq_flags); -- -- /* -- * A currently running task which is going off @rq first gets dequeued -- * and then stops running. As we want running <-> stopping transitions -- * to be contained within runnable <-> quiescent transitions, trigger -- * ->stopping() early here instead of in put_prev_task_scx(). -- * -- * @p may go through multiple stopping <-> running transitions between -- * here and put_prev_task_scx() if task attribute changes occur while -- * balance_scx() leaves @rq unlocked. However, they don't contain any -- * information meaningful to the BPF scheduler and can be suppressed by -- * skipping the callbacks if the task is !QUEUED. -- */ -- if (SCX_HAS_OP(stopping) && task_current(rq, p)) { -- update_curr_scx(rq); -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); -- } -- -- //if(task_current(rq, p)) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- if (SCX_HAS_OP(quiescent)) -- SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); -- -- if (deq_flags & SCX_DEQ_SLEEP) -- p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP; -- else -- p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP; -- -- p->scx->flags &= ~SCX_TASK_QUEUED; -- BUG_ON(!scx_rq->nr_running); -- scx_rq->nr_running--; -- sub_nr_running(rq, 1); -- -- dispatch_dequeue(scx_rq, p); --} -- --static void yield_task_scx(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); -- else -- p->scx->slice = 0; --} -- --static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) --{ -- struct task_struct *from = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); -- else -- return false; --} -- --#ifdef CONFIG_SMP --/** -- * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -- * @rq: rq to move the task into, currently locked -- * @p: task to move -- * @enq_flags: %SCX_ENQ_* -- * -- * Move @p which is currently on a different rq to @rq's local DSQ. The caller -- * must: -- * -- * 1. Start with exclusive access to @p either through its DSQ lock or -- * %SCX_OPSS_DISPATCHING flag. -- * -- * 2. Set @p->scx->holding_cpu to raw_smp_processor_id(). -- * -- * 3. Remember task_rq(@p). Release the exclusive access so that we don't -- * deadlock with dequeue. -- * -- * 4. Lock @rq and the task_rq from #3. -- * -- * 5. Call this function. -- * -- * Returns %true if @p was successfully moved. %false after racing dequeue and -- * losing. -- */ --static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -- u64 enq_flags) --{ -- struct rq *task_rq; -- -- lockdep_assert_rq_held(rq); -- -- /* -- * If dequeue got to @p while we were trying to lock both rq's, it'd -- * have cleared @p->scx->holding_cpu to -1. While other cpus may have -- * updated it to different values afterwards, as this operation can't be -- * preempted or recurse, @p->scx->holding_cpu can never become -- * raw_smp_processor_id() again before we're done. Thus, we can tell -- * whether we lost to dequeue by testing whether @p->scx->holding_cpu is -- * still raw_smp_processor_id(). -- * -- * See dispatch_dequeue() for the counterpart. -- */ -- if (unlikely(p->scx->holding_cpu != raw_smp_processor_id())) -- return false; -- -- /* @p->rq couldn't have changed if we're still the holding cpu */ -- task_rq = task_rq(p); -- lockdep_assert_rq_held(task_rq); -- -- WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)); -- deactivate_task(task_rq, p, 0); -- set_task_cpu(p, cpu_of(rq)); -- p->scx->sticky_cpu = cpu_of(rq); -- -- /* -- * We want to pass scx-specific enq_flags but activate_task() will -- * truncate the upper 32 bit. As we own @rq, we can pass them through -- * @rq->scx->extra_enq_flags instead. -- */ -- WARN_ON_ONCE(rq->scx->extra_enq_flags); -- rq->scx->extra_enq_flags = enq_flags; -- activate_task(rq, p, 0); -- rq->scx->extra_enq_flags = 0; -- -- return true; --} -- --/** -- * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * We're holding @rq lock and trying to dispatch a task from @src_rq to -- * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether -- * @rq stays locked isn't important as long as the state is restored after -- * dispatch_to_local_dsq_unlock(). -- */ --static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- rq_unpin_lock(rq, rf); -- -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(rq); -- raw_spin_rq_lock(dst_rq); -- } else if (rq == src_rq) { -- double_lock_balance(rq, dst_rq); -- rq_repin_lock(rq, rf); -- } else if (rq == dst_rq) { -- double_lock_balance(rq, src_rq); -- rq_repin_lock(rq, rf); -- } else { -- raw_spin_rq_unlock(rq); -- double_rq_lock(src_rq, dst_rq); -- } --} -- --/** -- * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock() -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return. -- */ --static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } else if (rq == src_rq) { -- double_unlock_balance(rq, dst_rq); -- } else if (rq == dst_rq) { -- double_unlock_balance(rq, src_rq); -- } else { -- double_rq_unlock(src_rq, dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } --} --#endif /* CONFIG_SMP */ -- -- --static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq) --{ -- return likely(test_rq_online(rq)) && !is_migration_disabled(p) && -- cpumask_test_cpu(cpu_of(rq), p->cpus_ptr); --} -- --static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -- struct scx_dispatch_q *dsq) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rb_node *rb_node; -- struct rq *task_rq; -- bool moved = false; --retry: -- if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -- return false; -- -- raw_spin_lock(&dsq->lock); -- -- list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- for (rb_node = rb_first_cached(&dsq->priq); rb_node; -- rb_node = rb_next(rb_node)) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- raw_spin_unlock(&dsq->lock); -- return false; -- --this_rq: -- /* @dsq is locked and @p is on this rq */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo); -- dsq->nr--; -- scx_rq->local_dsq.nr++; -- p->scx->dsq = &scx_rq->local_dsq; -- raw_spin_unlock(&dsq->lock); -- return true; -- --remote_rq: --#ifdef CONFIG_SMP -- /* -- * @dsq is locked and @p is on a remote rq. @p is currently protected by -- * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -- * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -- * rq lock or fail, do a little dancing from our side. See -- * move_task_to_local_dsq(). -- */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- p->scx->holding_cpu = raw_smp_processor_id(); -- raw_spin_unlock(&dsq->lock); -- -- rq_unpin_lock(rq, rf); -- double_lock_balance(rq, task_rq); -- rq_repin_lock(rq, rf); -- -- moved = move_task_to_local_dsq(rq, p, 0); -- -- double_unlock_balance(rq, task_rq); --#endif /* CONFIG_SMP */ -- if (likely(moved)) -- return true; -- goto retry; --} -- --enum dispatch_to_local_dsq_ret { -- DTL_DISPATCHED, /* successfully dispatched */ -- DTL_LOST, /* lost race to dequeue */ -- DTL_NOT_LOCAL, /* destination is not a local DSQ */ -- DTL_INVALID, /* invalid local dsq_id */ --}; -- --/** -- * dispatch_to_local_dsq - Dispatch a task to a local dsq -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @dsq_id: destination dsq ID -- * @p: task to dispatch -- * @enq_flags: %SCX_ENQ_* -- * -- * We're holding @rq lock and want to dispatch @p to the local DSQ identified by -- * @dsq_id. This function performs all the synchronization dancing needed -- * because local DSQs are protected with rq locks. -- * -- * The caller must have exclusive ownership of @p (e.g. through -- * %SCX_OPSS_DISPATCHING). -- */ --static enum dispatch_to_local_dsq_ret --dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id, -- struct task_struct *p, u64 enq_flags) --{ -- struct rq *src_rq = task_rq(p); -- struct rq *dst_rq; -- -- /* -- * We're synchronized against dequeue through DISPATCHING. As @p can't -- * be dequeued, its task_rq and cpus_allowed are stable too. -- */ -- if (dsq_id == SCX_DSQ_LOCAL) { -- dst_rq = rq; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]", -- cpu, p->comm, p->pid); -- return DTL_INVALID; -- } -- dst_rq = cpu_rq(cpu); -- } else { -- return DTL_NOT_LOCAL; -- } -- -- /* if dispatching to @rq that @p is already on, no lock dancing needed */ -- if (rq == src_rq && rq == dst_rq) { -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags | SCX_ENQ_CLEAR_OPSS); -- return DTL_DISPATCHED; -- } -- --#ifdef CONFIG_SMP -- if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { -- struct rq *locked_dst_rq = dst_rq; -- bool dsp; -- -- /* -- * @p is on a possibly remote @src_rq which we need to lock to -- * move the task. If dequeue is in progress, it'd be locking -- * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq -- * lock while holding DISPATCHING. -- * -- * As DISPATCHING guarantees that @p is wholly ours, we can -- * pretend that we're moving from a DSQ and use the same -- * mechanism - mark the task under transfer with holding_cpu, -- * release DISPATCHING and then follow the same protocol. -- */ -- p->scx->holding_cpu = raw_smp_processor_id(); -- -- /* store_release ensures that dequeue sees the above */ -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq); -- -- /* -- * We don't require the BPF scheduler to avoid dispatching to -- * offline CPUs mostly for convenience but also because CPUs can -- * go offline between scx_bpf_dispatch() calls and here. If @p -- * is destined to an offline CPU, queue it on its current CPU -- * instead, which should always be safe. As this is an allowed -- * behavior, don't trigger an ops error. -- */ -- if (unlikely(!test_rq_online(dst_rq))) -- dst_rq = src_rq; -- -- if (src_rq == dst_rq) { -- /* -- * As @p is staying on the same rq, there's no need to -- * go through the full deactivate/activate cycle. -- * Optimize by abbreviating the operations in -- * move_task_to_local_dsq(). -- */ -- dsp = p->scx->holding_cpu == raw_smp_processor_id(); -- if (likely(dsp)) { -- p->scx->holding_cpu = -1; -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags); -- } -- } else { -- dsp = move_task_to_local_dsq(dst_rq, p, enq_flags); -- } -- -- /* if the destination CPU is idle, wake it up */ -- if (dsp && p->sched_class > dst_rq->curr->sched_class) -- resched_curr(dst_rq); -- -- dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq); -- -- return dsp ? DTL_DISPATCHED : DTL_LOST; -- } --#endif /* CONFIG_SMP */ -- -- scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", -- cpu_of(dst_rq), p->comm, p->pid); -- return DTL_INVALID; --} -- --/** -- * finish_dispatch - Asynchronously finish dispatching a task -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @p: task to finish dispatching -- * @qseq_at_dispatch: qseq when @p started getting dispatched -- * @dsq_id: destination DSQ ID -- * @enq_flags: %SCX_ENQ_* -- * -- * Dispatching to local DSQs may need to wait for queueing to complete or -- * require rq lock dancing. As we don't wanna do either while inside -- * ops.dispatch() to avoid locking order inversion, we split dispatching into -- * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the -- * task and its qseq. Once ops.dispatch() returns, this function is called to -- * finish up. -- * -- * There is no guarantee that @p is still valid for dispatching or even that it -- * was valid in the first place. Make sure that the task is still owned by the -- * BPF scheduler and claim the ownership before dispatching. -- */ --static void finish_dispatch(struct rq *rq, struct rq_flags *rf, -- struct task_struct *p, u64 qseq_at_dispatch, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- u64 opss; -- -- touch_core_sched_dispatch(rq, p); --retry: -- /* -- * No need for _acquire here. @p is accessed only after a successful -- * try_cmpxchg to DISPATCHING. -- */ -- opss = atomic64_read(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_DISPATCHING: -- case SCX_OPSS_NONE: -- /* someone else already got to it */ -- return; -- case SCX_OPSS_QUEUED: -- /* -- * If qseq doesn't match, @p has gone through at least one -- * dispatch/dequeue and re-enqueue cycle between -- * scx_bpf_dispatch() and here and we have no claim on it. -- */ -- if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) -- return; -- -- /* -- * While we know @p is accessible, we don't yet have a claim on -- * it - the BPF scheduler is allowed to dispatch tasks -- * spuriously and there can be a racing dequeue attempt. Let's -- * claim @p by atomically transitioning it from QUEUED to -- * DISPATCHING. -- */ -- if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_DISPATCHING))) -- break; -- goto retry; -- case SCX_OPSS_QUEUEING: -- /* -- * do_enqueue_task() is in the process of transferring the task -- * to the BPF scheduler while holding @p's rq lock. As we aren't -- * holding any kernel or BPF resource that the enqueue path may -- * depend upon, it's safe to wait. -- */ -- wait_ops_state(p, opss); -- goto retry; -- } -- -- BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) { -- case DTL_DISPATCHED: -- break; -- case DTL_LOST: -- break; -- case DTL_INVALID: -- dsq_id = SCX_DSQ_GLOBAL; -- fallthrough; -- case DTL_NOT_LOCAL: -- dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()), -- dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- break; -- } --} -- --static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- u32 u; -- -- for (u = 0; u < dspc->buf_cursor; u++) { -- struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u]; -- -- finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id, -- ent->enq_flags); -- } -- -- dspc->nr_tasks += dspc->buf_cursor; -- dspc->buf_cursor = 0; --} -- --static int balance_one(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf, bool local) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- bool prev_on_scx = prev->sched_class == &ext_sched_class; -- int nr_loops = SCX_DSP_MAX_LOOPS; -- -- lockdep_assert_rq_held(rq); -- -- if (static_branch_unlikely(&scx_ops_cpu_preempt) && -- unlikely(rq->scx->cpu_released)) { -- /* -- * If the previous sched_class for the current CPU was not SCX, -- * notify the BPF scheduler that it again has control of the -- * core. This callback complements ->cpu_release(), which is -- * emitted in scx_notify_pick_next_task(). -- */ -- if (SCX_HAS_OP(cpu_acquire)) -- SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq), -- NULL); -- rq->scx->cpu_released = false; -- } -- -- if (prev_on_scx) { -- WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP)); -- update_curr_scx(rq); -- -- /* -- * If @prev is runnable & has slice left, it has priority and -- * fetching more just increases latency for the fetched tasks. -- * Tell put_prev_task_scx() to put @prev on local_dsq. If the -- * BPF scheduler wants to handle this explicitly, it should -- * implement ->cpu_released(). -- * -- * See scx_ops_disable_workfn() for the explanation on the -- * disabling() test. -- * -- * When balancing a remote CPU for core-sched, there won't be a -- * following put_prev_task_scx() call and we don't own -- * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the -- * same conditions later and pick @rq->curr accordingly. -- */ -- if ((prev->scx->flags & SCX_TASK_QUEUED) && -- prev->scx->slice && !scx_ops_disabling()) { -- if (local) -- prev->scx->flags |= SCX_TASK_BAL_KEEP; -- return 1; -- } -- } -- -- /* if there already are tasks to run, nothing to do */ -- if (scx_rq->local_dsq.nr) -- return 1; -- -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- if (!SCX_HAS_OP(dispatch)) -- return 0; -- -- dspc->rq = rq; -- dspc->rf = rf; -- -- /* -- * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, -- * the local DSQ might still end up empty after a successful -- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() -- * produced some tasks, retry. The BPF scheduler may depend on this -- * looping behavior to simplify its implementation. -- */ -- do { -- dspc->nr_tasks = 0; -- -- SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), -- prev_on_scx ? prev : NULL); -- -- flush_dispatch_buf(rq, rf); -- -- if (scx_rq->local_dsq.nr) -- return 1; -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- /* -- * ops.dispatch() can trap us in this loop by repeatedly -- * dispatching ineligible tasks. Break out once in a while to -- * allow the watchdog to run. As IRQ can't be enabled in -- * balance(), we want to complete this scheduling cycle and then -- * start a new one. IOW, we want to call resched_curr() on the -- * next, most likely idle, task, not the current one. Use -- * scx_bpf_kick_cpu() for deferred kicking. -- */ -- if (unlikely(!--nr_loops)) { -- scx_bpf_kick_cpu(cpu_of(rq), 0); -- break; -- } -- } while (dspc->nr_tasks); -- -- return 0; --} -- --static int balance_scx(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf) --{ -- int ret; -- -- ret = balance_one(rq, prev, rf, true); --#ifdef CONFIG_SCHED_SMT -- /* -- * When core-sched is enabled, this ops.balance() call will be followed -- * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx() -- * on the SMT siblings. Balance the siblings too. -- */ -- if (sched_core_enabled(rq)) { -- const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); -- int scpu; -- -- for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { -- struct rq *srq = cpu_rq(scpu); -- struct rq_flags srf; -- struct task_struct *sprev = srq->curr; -- -- /* -- * While core-scheduling, rq lock is shared among -- * siblings but the debug annotations and rq clock -- * aren't. Do pinning dance to transfer the ownership. -- */ -- WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); -- rq_unpin_lock(rq, rf); -- rq_pin_lock(srq, &srf); -- -- update_rq_clock(srq); -- balance_one(srq, sprev, &srf, false); -- -- rq_unpin_lock(srq, &srf); -- rq_repin_lock(rq, rf); -- } -- } --#endif -- return ret; --} -- --static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) --{ -- if (p->scx->flags & SCX_TASK_QUEUED) { -- /* -- * Core-sched might decide to execute @p before it is -- * dispatched. Call ops_dequeue() to notify the BPF scheduler. -- */ -- ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); -- dispatch_dequeue(rq->scx, p); -- } -- -- p->se.exec_start = rq_clock_task(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, running, p); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock); -- -- watchdog_unwatch_task(p, true); -- -- /* -- * @p is getting newly scheduled or got kicked after someone updated its -- * slice. Refresh whether tick can be stopped. See can_stop_tick_scx(). -- */ -- if ((p->scx->slice == SCX_SLICE_INF) != -- (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) { -- if (p->scx->slice == SCX_SLICE_INF) -- rq->scx->flags |= SCX_RQ_CAN_STOP_TICK; -- else -- rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK; -- -- sched_update_tick_dependency(rq); -- } --} -- --static void put_prev_task_scx(struct rq *rq, struct task_struct *p) --{ --#ifndef CONFIG_SMP -- /* -- * UP workaround. -- * -- * Because SCX may transfer tasks across CPUs during dispatch, dispatch -- * is performed from its balance operation which isn't called in UP. -- * Let's work around by calling it from the operations which come right -- * after. -- * -- * 1. If the prev task is on SCX, pick_next_task() calls -- * .put_prev_task() right after. As .put_prev_task() is also called -- * from other places, we need to distinguish the calls which can be -- * done by looking at the previous task's state - if still queued or -- * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task(). -- * This case is handled here. -- * -- * 2. If the prev task is not on SCX, the first following call into SCX -- * will be .pick_next_task(), which is covered by calling -- * balance_scx() from pick_next_task_scx(). -- * -- * Note that we can't merge the first case into the second as -- * balance_scx() must be called before the previous SCX task goes -- * through put_prev_task_scx(). -- * -- * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf. -- * Pass in %NULL. -- */ -- if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP)) -- balance_scx(rq, p, NULL); --#endif -- -- update_curr_scx(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- -- /* -- * If we're being called from put_prev_task_balance(), balance_scx() may -- * have decided that @p should keep running. -- */ -- if (p->scx->flags & SCX_TASK_BAL_KEEP) { -- p->scx->flags &= ~SCX_TASK_BAL_KEEP; -- watchdog_watch_task(rq, p); -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- watchdog_watch_task(rq, p); -- -- /* -- * If @p has slice left and balance_scx() didn't tag it for -- * keeping, @p is getting preempted by a higher priority -- * scheduler class or core-sched forcing a different task. Leave -- * it at the head of the local DSQ. -- */ -- if (p->scx->slice && !scx_ops_disabling()) { -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- /* -- * If we're in the pick_next_task path, balance_scx() should -- * have already populated the local DSQ if there are any other -- * available tasks. If empty, tell ops.enqueue() that @p is the -- * only one available for this cpu. ops.enqueue() should put it -- * on the local DSQ so that the subsequent pick_next_task_scx() -- * can find the task unless it wants to trigger a separate -- * follow-up scheduling event. -- */ -- if (list_empty(&rq->scx->local_dsq.fifo)) -- do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1); -- else -- do_enqueue_task(rq, p, 0, -1); -- } --} -- --static struct task_struct *first_local_task(struct rq *rq) --{ -- struct rb_node *rb_node; -- struct sched_ext_entity *entity; -- -- if (!list_empty(&rq->scx->local_dsq.fifo)) { -- entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo); -- return entity->task; -- } -- -- rb_node = rb_first_cached(&rq->scx->local_dsq.priq); -- if (rb_node) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- return entity->task; -- } -- -- return NULL; --} -- --static struct task_struct *pick_next_task_scx(struct rq *rq) --{ -- struct task_struct *p; -- --#ifndef CONFIG_SMP -- /* UP workaround - see the comment at the head of put_prev_task_scx() */ -- if (unlikely(rq->curr->sched_class != &ext_sched_class)) -- balance_scx(rq, rq->curr, NULL); --#endif -- -- p = first_local_task(rq); -- if (!p) -- return NULL; -- -- if (unlikely(!p->scx->slice)) { -- if (!scx_ops_disabling() && !scx_warned_zero_slice) { -- printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n", -- p->comm, p->pid); -- scx_warned_zero_slice = true; -- } -- p->scx->slice = SCX_SLICE_DFL; -- } -- -- set_next_task_scx(rq, p, true); -- -- return p; --} -- --#ifdef CONFIG_SCHED_CORE --/** -- * scx_prio_less - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Core-sched is implemented as an additional scheduling layer on top of the -- * usual sched_class'es and needs to find out the expected task ordering. For -- * SCX, core-sched calls this function to interrogate the task ordering. -- * -- * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used -- * to implement the default task ordering. The older the timestamp, the higher -- * prority the task - the global FIFO ordering matching the default scheduling -- * behavior. -- * -- * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to -- * implement FIFO ordering within each local DSQ. See pick_task_scx(). -- */ --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi) --{ -- /* -- * The const qualifiers are dropped from task_struct pointers when -- * calling ops.core_sched_before(). Accesses are controlled by the -- * verifier. -- */ -- if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling()) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, -- (struct task_struct *)a, -- (struct task_struct *)b); -- else -- return time_after64(a->scx->core_sched_at, b->scx->core_sched_at); --} -- --/** -- * pick_task_scx - Pick a candidate task for core-sched -- * @rq: rq to pick the candidate task from -- * -- * Core-sched calls this function on each SMT sibling to determine the next -- * tasks to run on the SMT siblings. balance_one() has been called on all -- * siblings and put_prev_task_scx() has been called only for the current CPU. -- * -- * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look -- * at the first task in the local dsq. @rq->curr has to be considered explicitly -- * to mimic %SCX_TASK_BAL_KEEP. -- */ --static struct task_struct *pick_task_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- struct task_struct *first = first_local_task(rq); -- -- if (curr->scx->flags & SCX_TASK_QUEUED) { -- /* is curr the only runnable task? */ -- if (!first) -- return curr; -- -- /* -- * Does curr trump first? We can always go by core_sched_at for -- * this comparison as it represents global FIFO ordering when -- * the default core-sched ordering is used and local-DSQ FIFO -- * ordering otherwise. -- * -- * We can have a task with an earlier timestamp on the DSQ. For -- * example, when a current task is preempted by a sibling -- * picking a different cookie, the task would be requeued at the -- * head of the local DSQ with an earlier timestamp than the -- * core-sched picked next task. Besides, the BPF scheduler may -- * dispatch any tasks to the local DSQ anytime. -- */ -- if (curr->scx->slice && time_before64(curr->scx->core_sched_at, -- first->scx->core_sched_at)) -- return curr; -- } -- -- return first; /* this may be %NULL */ --} --#endif /* CONFIG_SCHED_CORE */ -- --static enum scx_cpu_preempt_reason --preempt_reason_from_class(const struct sched_class *class) --{ --#ifdef CONFIG_SMP -- if (class == &stop_sched_class) -- return SCX_CPU_PREEMPT_STOP; --#endif -- if (class == &dl_sched_class) -- return SCX_CPU_PREEMPT_DL; -- if (class == &rt_sched_class) -- return SCX_CPU_PREEMPT_RT; -- return SCX_CPU_PREEMPT_UNKNOWN; --} -- --void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task, -- const struct sched_class *active) --{ -- lockdep_assert_rq_held(rq); -- -- /* -- * The callback is conceptually meant to convey that the CPU is no -- * longer under the control of SCX. Therefore, don't invoke the -- * callback if the CPU is is staying on SCX, or going idle (in which -- * case the SCX scheduler has actively decided not to schedule any -- * tasks on the CPU). -- */ -- if (likely(active >= &ext_sched_class)) -- return; -- -- /* -- * At this point we know that SCX was preempted by a higher priority -- * sched_class, so invoke the ->cpu_release() callback if we have not -- * done so already. We only send the callback once between SCX being -- * preempted, and it regaining control of the CPU. -- * -- * ->cpu_release() complements ->cpu_acquire(), which is emitted the -- * next time that balance_scx() is invoked. -- */ -- if (!rq->scx->cpu_released) { -- if (SCX_HAS_OP(cpu_release)) { -- struct scx_cpu_release_args args = { -- .reason = preempt_reason_from_class(active), -- .task = task, -- }; -- -- SCX_CALL_OP(SCX_KF_CPU_RELEASE, -- cpu_release, cpu_of(rq), &args); -- } -- rq->scx->cpu_released = true; -- } --} -- --#ifdef CONFIG_SMP -- --static bool test_and_clear_cpu_idle(int cpu) --{ -- if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -- if (cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- return true; -- } else { -- return false; -- } --} -- --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- int cpu; -- -- do { -- cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); -- if (cpu < nr_cpu_ids) { -- const struct cpumask *sbm = topology_sibling_cpumask(cpu); -- -- /* -- * If offline, @cpu is not its own sibling and we can -- * get caught in an infinite loop as @cpu is never -- * cleared from idle_masks.smt. Clear @cpu directly in -- * such cases. -- */ -- if (likely(cpumask_test_cpu(cpu, sbm))) -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm); -- else -- cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu)); -- } else { -- cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -- if (cpu >= nr_cpu_ids) -- return -EBUSY; -- } -- } while (!test_and_clear_cpu_idle(cpu)); -- -- return cpu; --} -- --static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) --{ -- s32 cpu; -- -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return prev_cpu; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local DSQ of the waker. -- */ -- if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 && -- scx_has_idle_cpus && !(current->flags & PF_EXITING)) { -- cpu = smp_processor_id(); -- if (cpumask_test_cpu(cpu, p->cpus_ptr)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (test_and_clear_cpu_idle(prev_cpu)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return prev_cpu; -- } -- -- if (p->nr_cpus_allowed == 1) -- return prev_cpu; -- -- cpu = scx_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- -- return prev_cpu; --} -- --static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) --{ -- if (SCX_HAS_OP(select_cpu)) { -- s32 cpu; -- -- cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu, -- wake_flags); -- if (ops_cpu_valid(cpu)) { -- return cpu; -- } else { -- scx_ops_error("select_cpu returned invalid cpu %d", cpu); -- return prev_cpu; -- } -- } else { -- return scx_select_cpu_dfl(p, prev_cpu, wake_flags); -- } --} -- --static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx) --{ -- set_cpus_allowed_common(p, ctx); -- -- /* -- * The effective cpumask is stored in @p->cpus_ptr which may temporarily -- * differ from the configured one in @p->cpus_mask. Always tell the bpf -- * scheduler the effective one. -- * -- * Fine-grained memory write control is enforced by BPF making the const -- * designation pointless. Cast it away when calling the operation. -- */ -- if (SCX_HAS_OP(set_cpumask)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, -- (struct cpumask *)p->cpus_ptr); --} -- --static void reset_idle_masks(void) --{ -- /* consider all cpus idle, should converge to the actual state quickly */ -- cpumask_setall(idle_masks.cpu); -- cpumask_setall(idle_masks.smt); -- scx_has_idle_cpus = true; --} -- --void __scx_update_idle(struct rq *rq, bool idle) --{ -- int cpu = cpu_of(rq); -- struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -- -- if (SCX_HAS_OP(update_idle)) { -- SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); -- if (!static_branch_unlikely(&scx_builtin_idle_enabled)) -- return; -- } -- -- if (idle) { -- cpumask_set_cpu(cpu, idle_masks.cpu); -- if (!scx_has_idle_cpus) -- scx_has_idle_cpus = true; -- -- /* -- * idle_masks.smt handling is racy but that's fine as it's only -- * for optimization and self-correcting. -- */ -- for_each_cpu(cpu, sib_mask) { -- if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -- return; -- } -- cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -- } else { -- cpumask_clear_cpu(cpu, idle_masks.cpu); -- if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -- } --} -- --#else /* !CONFIG_SMP */ -- --static bool test_and_clear_cpu_idle(int cpu) { return false; } --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } --static void reset_idle_masks(void) {} -- --#endif /* CONFIG_SMP */ -- --static bool check_rq_for_timeouts(struct rq *rq) --{ -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rq_flags rf; -- bool timed_out = false; -- -- rq_lock_irqsave(rq, &rf); -- list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) { -- unsigned long last_runnable; -- -- p = entity->task; -- last_runnable = p->scx->runnable_at; -- -- if (unlikely(time_after(jiffies, -- last_runnable + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "%s[%d] failed to run for %u.%03us", -- p->comm, p->pid, -- dur_ms / 1000, dur_ms % 1000); -- timed_out = true; -- break; -- } -- } -- rq_unlock_irqrestore(rq, &rf); -- -- return timed_out; --} -- --static void scx_watchdog_workfn(struct work_struct *work) --{ -- int cpu; -- -- scx_watchdog_timestamp = jiffies; -- -- for_each_online_cpu(cpu) { -- if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) -- break; -- -- cond_resched(); -- } -- queue_delayed_work(system_unbound_wq, to_delayed_work(work), -- scx_watchdog_timeout / 2); --} -- --static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) --{ -- update_curr_scx(rq); -- //scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock); -- /* -- * While disabling, always resched and refresh core-sched timestamp as -- * we can't trust the slice management or ops.core_sched_before(). -- */ -- if (scx_ops_disabling()) { -- curr->scx->slice = 0; -- touch_core_sched(rq, curr); -- } -- -- if (!curr->scx->slice) -- resched_curr(rq); --} -- --#define SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- --static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg) --{ -- int ret; -- -- WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED); -- -- p->scx->disallow = false; -- -- if (SCX_HAS_OP(prep_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- }; -- -- ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args); -- if (unlikely(ret)) { -- ret = ops_sanitize_err("prep_enable", ret); -- return ret; -- } -- } -- //scx_sched_init_task(p); -- -- if (p->scx->disallow) { -- struct rq *rq; -- struct rq_flags rf; -- -- rq = task_rq_lock(p, &rf); -- -- /* -- * We're either in fork or load path and @p->policy will be -- * applied right after. Reverting @p->policy here and rejecting -- * %SCHED_EXT transitions from scx_check_setscheduler() -- * guarantees that if ops.prep_enable() sets @p->disallow, @p -- * can never be in SCX. -- */ -- if (p->policy == SCHED_EXT) { -- p->policy = SCHED_NORMAL; -- atomic64_inc(&scx_nr_rejected); -- } -- -- task_rq_unlock(rq, p, &rf); -- } -- -- p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET); -- return 0; --} -- --static void scx_ops_enable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED)); -- -- if (SCX_HAS_OP(enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- p->scx->flags |= SCX_TASK_OPS_ENABLED; --} -- --static void scx_ops_disable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- if (p->scx->flags & SCX_TASK_OPS_PREPPED) { -- if (SCX_HAS_OP(cancel_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- } else if (p->scx->flags & SCX_TASK_OPS_ENABLED) { -- if (SCX_HAS_OP(disable)) -- SCX_CALL_OP(SCX_KF_REST, disable, p); -- p->scx->flags &= ~SCX_TASK_OPS_ENABLED; -- } --} -- --static void set_task_scx_weight(struct task_struct *p) --{ -- u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -- -- p->scx->weight = sched_weight_to_cgroup(weight); --} -- --/** -- * refresh_scx_weight - Refresh a task's ext weight -- * @p: task to refresh ext weight for -- * -- * @p->scx->weight carries the task's static priority in cgroup weight scale to -- * enable easy access from the BPF scheduler. To keep it synchronized with the -- * current task priority, this function should be called when a new task is -- * created, priority is changed for a task on sched_ext, and a task is switched -- * to sched_ext from other classes. -- */ --static void refresh_scx_weight(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- set_task_scx_weight(p); -- if (SCX_HAS_OP(set_weight)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight); --} -- --void scx_pre_fork(struct task_struct *p) --{ -- p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL); -- if (!p->scx) { -- goto lock; -- } -- -- p->scx->dsq = NULL; -- INIT_LIST_HEAD(&p->scx->dsq_node.fifo); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- INIT_LIST_HEAD(&p->scx->watchdog_node); -- p->scx->flags = 0; -- p->scx->weight = 0; -- p->scx->sticky_cpu = -1; -- p->scx->holding_cpu = -1; -- p->scx->kf_mask = 0; -- atomic64_set(&p->scx->ops_state, 0); -- p->scx->runnable_at = INITIAL_JIFFIES; -- p->scx->slice = SCX_SLICE_DFL; -- p->scx->task = p; -- p->sched_prop = 0; -- -- /* -- * BPF scheduler enable/disable paths want to be able to iterate and -- * update all tasks which can become complex when racing forks. As -- * enable/disable are very cold paths, let's use a percpu_rwsem to -- * exclude forks. -- */ --lock: -- percpu_down_read(&scx_fork_rwsem); --} -- --int scx_fork(struct task_struct *p) --{ -- percpu_rwsem_assert_held(&scx_fork_rwsem); -- -- if (scx_enabled()) -- return scx_ops_prepare_task(p, task_group(p)); -- else -- return 0; --} -- --void scx_post_fork(struct task_struct *p) --{ -- if (scx_enabled()) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- /* -- * Set the weight manually before calling ops.enable() so that -- * the scheduler doesn't see a stale value if they inspect the -- * task struct. We'll invoke ops.set_weight() afterwards, as it -- * would be odd to receive a callback on the task before we -- * tell the scheduler that it's been fully enabled. -- */ -- set_task_scx_weight(p); -- scx_ops_enable_task(p); -- refresh_scx_weight(p); -- task_rq_unlock(rq, p, &rf); -- } -- -- spin_lock_irq(&scx_tasks_lock); -- list_add_tail(&p->scx->tasks_node, &scx_tasks); -- spin_unlock_irq(&scx_tasks_lock); -- -- percpu_up_read(&scx_fork_rwsem); --} -- --void scx_cancel_fork(struct task_struct *p) --{ -- if (scx_enabled()) -- scx_ops_disable_task(p); -- percpu_up_read(&scx_fork_rwsem); --} -- --void sched_ext_free(struct task_struct *p) --{ -- unsigned long flags; -- -- spin_lock_irqsave(&scx_tasks_lock, flags); -- list_del_init(&p->scx->tasks_node); -- spin_unlock_irqrestore(&scx_tasks_lock, flags); -- -- /* -- * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED -> -- * ENABLED transitions can't race us. Disable ops for @p. -- */ -- if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- scx_ops_disable_task(p); -- task_rq_unlock(rq, p, &rf); -- } --} -- --static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) --{ --} -- --static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} --static void switched_to_scx(struct rq *rq, struct task_struct *p) {} -- --int scx_check_setscheduler(struct task_struct *p, int policy) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- /* if disallow, reject transitioning into SCX */ -- if (scx_enabled() && READ_ONCE(p->scx->disallow) && -- p->policy != policy && policy == SCHED_EXT) -- return -EACCES; -- -- return 0; --} -- --#ifdef CONFIG_NO_HZ_FULL --bool scx_can_stop_tick(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (scx_ops_disabling()) -- return false; -- -- if (p->sched_class != &ext_sched_class) -- return true; -- -- /* -- * @rq can dispatch from different DSQs, so we can't tell whether it -- * needs the tick or not by looking at nr_running. Allow stopping ticks -- * iff the BPF scheduler indicated so. See set_next_task_scx(). -- */ -- return rq->scx->flags & SCX_RQ_CAN_STOP_TICK; --} --#endif -- --static inline void scx_cgroup_lock(void) {} --static inline void scx_cgroup_unlock(void) {} -- --/* -- * Omitted operations: -- * -- * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -- * task isn't tied to the CPU at that point. Preemption is implemented by -- * resetting the victim task's slice to 0 and triggering reschedule on the -- * target CPU. -- * -- * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -- * -- * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -- * their current sched_class. Call them directly from sched core instead. -- * -- * - task_woken, switched_from: Unnecessary. -- */ --DEFINE_SCHED_CLASS(ext) = { -- .enqueue_task = enqueue_task_scx, -- .dequeue_task = dequeue_task_scx, -- .yield_task = yield_task_scx, -- .yield_to_task = yield_to_task_scx, -- -- .check_preempt_curr = check_preempt_curr_scx, -- -- .pick_next_task = pick_next_task_scx, -- -- .put_prev_task = put_prev_task_scx, -- .set_next_task = set_next_task_scx, -- --#ifdef CONFIG_SMP -- .balance = balance_scx, -- .select_task_rq = select_task_rq_scx, -- .set_cpus_allowed = set_cpus_allowed_scx, --#endif -- --#ifdef CONFIG_SCHED_CORE -- .pick_task = pick_task_scx, --#endif -- -- .task_tick = task_tick_scx, -- -- .switched_to = switched_to_scx, -- .prio_changed = prio_changed_scx, -- -- .update_curr = update_curr_scx, -- --#ifdef CONFIG_UCLAMP_TASK -- .uclamp_enabled = 0, --#endif --}; -- --static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) --{ -- memset(dsq, 0, sizeof(*dsq)); -- -- raw_spin_lock_init(&dsq->lock); -- INIT_LIST_HEAD(&dsq->fifo); -- dsq->id = dsq_id; --} -- --static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id & SCX_DSQ_FLAG_BUILTIN) -- return ERR_PTR(-EINVAL); -- -- dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node); -- if (!dsq) -- return ERR_PTR(-ENOMEM); -- -- init_dsq(dsq, dsq_id); -- -- return dsq; --} -- --static void free_dsq_irq_workfn(struct irq_work *irq_work) --{ -- struct llist_node *to_free = llist_del_all(&dsqs_to_free); -- struct scx_dispatch_q *dsq, *tmp_dsq; -- -- llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) -- kfree_rcu(dsq, rcu); --} -- --static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); -- --static void destroy_dsq(u64 dsq_id) --{ --} -- --static void scx_cgroup_exit(void) {} --static int scx_cgroup_init(void) { return 0; } --static void scx_cgroup_config_knobs(void) {} -- --/* -- * Used by sched_fork() and __setscheduler_prio() to pick the matching -- * sched_class. dl/rt are already handled. -- */ --bool task_on_scx(struct task_struct *p) --{ -- if (!scx_enabled() || scx_ops_disabling()) -- return false; -- if (READ_ONCE(scx_switching_all)) -- return true; -- return p->policy == SCHED_EXT; --} -- --static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags) --{ -- if (enq_flags & SCX_ENQ_LAST) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); --} -- --static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {} -- --static void scx_ops_disable_workfn(struct kthread_work *work) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- struct scx_task_iter sti; -- struct task_struct *p; -- const char *reason; -- int i, cpu, type; -- -- type = atomic_read(&scx_exit_type); -- while (true) { -- /* -- * NONE indicates that a new scx_ops has been registered since -- * disable was scheduled - don't kill the new ops. DONE -- * indicates that the ops has already been disabled. -- */ -- if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE) -- return; -- if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE)) -- break; -- } -- -- cancel_delayed_work_sync(&scx_watchdog_work); -- -- switch (type) { -- case SCX_EXIT_UNREG: -- reason = "BPF scheduler unregistered"; -- break; -- case SCX_EXIT_SYSRQ: -- reason = "disabled by sysrq-S"; -- break; -- case SCX_EXIT_ERROR: -- reason = "runtime error"; -- break; -- case SCX_EXIT_ERROR_BPF: -- reason = "scx_bpf_error"; -- break; -- case SCX_EXIT_ERROR_STALL: -- reason = "runnable task stall"; -- break; -- default: -- reason = ""; -- } -- -- ei->type = type; -- strlcpy(ei->reason, reason, sizeof(ei->reason)); -- -- switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { -- case SCX_OPS_DISABLED: -- pr_warn("sched_ext: ops error detected without ops (%s)\n", -- scx_exit_info.msg); -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- return; -- case SCX_OPS_PREPPING: -- goto forward_progress_guaranteed; -- case SCX_OPS_DISABLING: -- /* shouldn't happen but handle it like ENABLING if it does */ -- WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); -- fallthrough; -- case SCX_OPS_ENABLING: -- case SCX_OPS_ENABLED: -- break; -- } -- -- /* -- * DISABLING is set and ops was either ENABLING or ENABLED indicating -- * that the ops and static branches are set. -- * -- * We must guarantee that all runnable tasks make forward progress -- * without trusting the BPF scheduler. We can't grab any mutexes or -- * rwsems as they might be held by tasks that the BPF scheduler is -- * forgetting to run, which unfortunately also excludes toggling the -- * static branches. -- * -- * Let's work around by overriding a couple ops and modifying behaviors -- * based on the DISABLING state and then cycling the tasks through -- * dequeue/enqueue to force global FIFO scheduling. -- * -- * a. ops.enqueue() and .dispatch() are overridden for simple global -- * FIFO scheduling. -- * -- * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value -- * can't be trusted. Whenever a tick triggers, the running task is -- * rotated to the tail of the queue with core_sched_at touched. -- * -- * c. pick_next_task() suppresses zero slice warning. -- * -- * d. scx_prio_less() reverts to the default core_sched_at order. -- */ -- scx_ops.enqueue = scx_ops_fallback_enqueue; -- scx_ops.dispatch = scx_ops_fallback_dispatch; -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- SCHED_CHANGE_BLOCK(task_rq(p), p, -- DEQUEUE_SAVE | DEQUEUE_MOVE) { -- /* cycling deq/enq is enough, see above */ -- } -- } -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* kick all CPUs to restore ticks */ -- for_each_possible_cpu(cpu) -- resched_cpu(cpu); -- --forward_progress_guaranteed: -- /* -- * Here, every runnable task is guaranteed to make forward progress and -- * we can safely use blocking synchronization constructs. Actually -- * disable ops. -- */ -- mutex_lock(&scx_ops_enable_mutex); -- -- static_branch_disable(&__scx_switched_all); -- WRITE_ONCE(scx_switching_all, false); -- -- /* avoid racing against fork and cgroup changes */ -- cpus_read_lock(); -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- bool alive = READ_ONCE(p->__state) != TASK_DEAD; -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL); -- -- __setscheduler_prio(p, p->prio); -- } -- -- if (alive) -- check_class_changed(task_rq(p), p, old_class, p->prio); -- -- scx_ops_disable_task(p); -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* no task is on scx, turn off all the switches and flush in-progress calls */ -- static_branch_disable_cpuslocked(&__scx_ops_enabled); -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- static_branch_disable_cpuslocked(&scx_has_op[i]); -- static_branch_disable_cpuslocked(&scx_ops_enq_last); -- static_branch_disable_cpuslocked(&scx_ops_enq_exiting); -- static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- synchronize_rcu(); -- -- scx_cgroup_exit(); -- -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- cpus_read_unlock(); -- -- if (ei->type >= SCX_EXIT_ERROR) { -- printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name); -- -- if (ei->msg[0] == '\0') -- printk(KERN_ERR "sched_ext: %s\n", ei->reason); -- else -- printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg); -- -- stack_trace_print(ei->bt, ei->bt_len, 2); -- } -- -- if (scx_ops.exit) -- SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); -- -- memset(&scx_ops, 0, sizeof(scx_ops)); -- -- free_percpu(scx_dsp_buf); -- scx_dsp_buf = NULL; -- scx_dsp_max_batch = 0; -- -- mutex_unlock(&scx_ops_enable_mutex); -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- -- scx_cgroup_config_knobs(); --} -- --static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); -- --static void schedule_scx_ops_disable_work(void) --{ -- struct kthread_worker *helper = READ_ONCE(scx_ops_helper); -- -- /* -- * We may be called spuriously before the first bpf_sched_ext_reg(). If -- * scx_ops_helper isn't set up yet, there's nothing to do. -- */ -- if (helper) -- kthread_queue_work(helper, &scx_ops_disable_work); --} -- --static void scx_ops_disable(enum scx_exit_type type) --{ -- int none = SCX_EXIT_NONE; -- -- if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)) -- type = SCX_EXIT_ERROR; -- -- atomic_try_cmpxchg(&scx_exit_type, &none, type); -- -- schedule_scx_ops_disable_work(); --} -- --static void scx_ops_error_irq_workfn(struct irq_work *irq_work) --{ -- schedule_scx_ops_disable_work(); --} -- --static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- int none = SCX_EXIT_NONE; -- va_list args; -- -- if (!atomic_try_cmpxchg(&scx_exit_type, &none, type)) -- return; -- -- ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1); -- -- va_start(args, fmt); -- vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args); -- va_end(args); -- -- irq_work_queue(&scx_ops_error_irq_work); --} -- --static struct kthread_worker *scx_create_rt_helper(const char *name) --{ -- struct kthread_worker *helper; -- -- helper = kthread_create_worker(0, name); -- if (helper) -- sched_set_fifo(helper->task); -- return helper; --} -- --static int scx_ops_enable(struct sched_ext_ops *ops) --{ -- struct scx_task_iter sti; -- struct task_struct *p; -- int i, ret; -- int tcnt = 0; -- unsigned long long start = 0; -- unsigned long long total_start = sched_clock(); -- -- mutex_lock(&scx_ops_enable_mutex); -- -- if (!scx_ops_helper) { -- WRITE_ONCE(scx_ops_helper, -- scx_create_rt_helper("sched_ext_ops_helper")); -- if (!scx_ops_helper) { -- ret = -ENOMEM; -- goto err_unlock; -- } -- } -- -- if (scx_ops_enable_state() != SCX_OPS_DISABLED) { -- ret = -EBUSY; -- goto err_unlock; -- } -- -- //slim_walt_enable(true); -- -- /* -- * Set scx_ops, transition to PREPPING and clear exit info to arm the -- * disable path. Failure triggers full disabling from here on. -- */ -- scx_ops = *ops; -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != -- SCX_OPS_DISABLED); -- -- memset(&scx_exit_info, 0, sizeof(scx_exit_info)); -- atomic_set(&scx_exit_type, SCX_EXIT_NONE); -- scx_warned_zero_slice = false; -- -- atomic64_set(&scx_nr_rejected, 0); -- -- /* -- * Keep CPUs stable during enable so that the BPF scheduler can track -- * online CPUs by watching ->on/offline_cpu() after ->init(). -- */ -- cpus_read_lock(); -- -- scx_switch_all_req = true; -- if (scx_ops.init) { -- ret = SCX_CALL_OP_RET(SCX_KF_INIT, init); -- if (ret) { -- ret = ops_sanitize_err("init", ret); -- goto err_disable; -- } -- -- /* -- * Exit early if ops.init() triggered scx_bpf_error(). Not -- * strictly necessary as we'll fail transitioning into ENABLING -- * later but that'd be after calling ops.prep_enable() on all -- * tasks and with -EBUSY which isn't very intuitive. Let's exit -- * early with success so that the condition is notified through -- * ops.exit() like other scx_bpf_error() invocations. -- */ -- if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE) -- goto err_disable; -- } -- -- WARN_ON_ONCE(scx_dsp_buf); -- scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; -- scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch, -- __alignof__(scx_dsp_buf[0])); -- if (!scx_dsp_buf) { -- ret = -ENOMEM; -- goto err_disable; -- } -- -- scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; -- if (ops->timeout_ms) -- scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); -- -- scx_watchdog_timestamp = jiffies; -- queue_delayed_work(system_unbound_wq, &scx_watchdog_work, -- scx_watchdog_timeout / 2); -- -- /* -- * Lock out forks, cgroup on/offlining and moves before opening the -- * floodgate so that they don't wander into the operations prematurely. -- */ -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- if (((void (**)(void))ops)[i]) -- static_branch_enable_cpuslocked(&scx_has_op[i]); -- -- if (ops->flags & SCX_OPS_ENQ_LAST) -- static_branch_enable_cpuslocked(&scx_ops_enq_last); -- -- if (ops->flags & SCX_OPS_ENQ_EXITING) -- static_branch_enable_cpuslocked(&scx_ops_enq_exiting); -- if (scx_ops.cpu_acquire || scx_ops.cpu_release) -- static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); -- -- if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { -- reset_idle_masks(); -- static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); -- } else { -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- } -- -- /* -- * All cgroups should be initialized before letting in tasks. cgroup -- * on/offlining and task migrations are already locked out. -- */ -- ret = scx_cgroup_init(); -- if (ret) -- goto err_disable_unlock; -- -- static_branch_enable_cpuslocked(&__scx_ops_enabled); -- -- /* -- * Enable ops for every task. Fork is excluded by scx_fork_rwsem -- * preventing new tasks from being added. No need to exclude tasks -- * leaving as sched_ext_free() can handle both prepped and enabled -- * tasks. Prep all tasks first and then enable them with preemption -- * disabled. -- */ -- spin_lock_irq(&scx_tasks_lock); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered(&sti))) { -- get_task_struct(p); -- spin_unlock_irq(&scx_tasks_lock); -- -- ret = scx_ops_prepare_task(p, task_group(p)); -- if (ret) { -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n", -- ret, p->comm, p->pid); -- goto err_disable_unlock; -- } -- -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- } -- scx_task_iter_exit(&sti); -- -- /* -- * All tasks are prepped but are still ops-disabled. Ensure that -- * %current can't be scheduled out and switch everyone. -- * preempt_disable() is necessary because we can't guarantee that -- * %current won't be starved if scheduled out while switching. -- */ -- preempt_disable(); -- -- /* -- * From here on, the disable path must assume that tasks have ops -- * enabled and need to be recovered. -- */ -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { -- preempt_enable(); -- spin_unlock_irq(&scx_tasks_lock); -- ret = -EBUSY; -- goto err_disable_unlock; -- } -- -- /* -- * We're fully committed and can't fail. The PREPPED -> ENABLED -- * transitions here are synchronized against sched_ext_free() through -- * scx_tasks_lock. -- */ -- WRITE_ONCE(scx_switching_all, scx_switch_all_req); -- start = sched_clock(); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- tcnt++; -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- scx_ops_enable_task(p); -- __setscheduler_prio(p, p->prio); -- } -- -- check_class_changed(task_rq(p), p, old_class, p->prio); -- } else { -- scx_ops_disable_task(p); -- } -- } -- printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id()); -- scx_task_iter_exit(&sti); -- -- spin_unlock_irq(&scx_tasks_lock); -- preempt_enable(); -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { -- ret = -EBUSY; -- goto err_disable; -- } -- -- if (scx_switch_all_req) -- static_branch_enable_cpuslocked(&__scx_switched_all); -- -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- -- scx_cgroup_config_knobs(); -- printk("\n total duration = %llu , current cpu = %d\n", sched_clock() - total_start, smp_processor_id()); -- -- return 0; -- --err_unlock: -- mutex_unlock(&scx_ops_enable_mutex); -- return ret; -- --err_disable_unlock: -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); --err_disable: -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- /* must be fully disabled before returning */ -- scx_ops_disable(SCX_EXIT_ERROR); -- kthread_flush_work(&scx_ops_disable_work); -- return ret; --} -- --#ifdef CONFIG_SCHED_DEBUG --static const char *scx_ops_enable_state_str[] = { -- [SCX_OPS_PREPPING] = "prepping", -- [SCX_OPS_ENABLING] = "enabling", -- [SCX_OPS_ENABLED] = "enabled", -- [SCX_OPS_DISABLING] = "disabling", -- [SCX_OPS_DISABLED] = "disabled", --}; -- --static int scx_debug_show(struct seq_file *m, void *v) --{ -- mutex_lock(&scx_ops_enable_mutex); -- seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name); -- seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled()); -- seq_printf(m, "%-30s: %d\n", "switching_all", -- READ_ONCE(scx_switching_all)); -- seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all()); -- seq_printf(m, "%-30s: %s\n", "enable_state", -- scx_ops_enable_state_str[scx_ops_enable_state()]); -- seq_printf(m, "%-30s: %llu\n", "nr_rejected", -- atomic64_read(&scx_nr_rejected)); -- mutex_unlock(&scx_ops_enable_mutex); -- return 0; --} -- --static int scx_debug_open(struct inode *inode, struct file *file) --{ -- return single_open(file, scx_debug_show, NULL); --} -- --const struct file_operations sched_ext_fops = { -- .open = scx_debug_open, -- .read = seq_read, -- .llseek = seq_lseek, -- .release = single_release, --}; --#endif -- --/******************************************************************************** -- * bpf_struct_ops plumbing. -- */ --#include --#include --#include -- --extern struct btf *btf_vmlinux; --static const struct btf_type *task_struct_type; -- --static bool bpf_scx_is_valid_access(int off, int size, -- enum bpf_access_type type, -- const struct bpf_prog *prog, -- struct bpf_insn_access_aux *info) --{ -- if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) -- return false; -- if (type != BPF_READ) -- return false; -- if (off % size != 0) -- return false; -- -- return btf_ctx_access(off, size, type, prog, info); --} -- --static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, -- const struct bpf_reg_state *reg, -- int off, int size) --{ -- return 0; --} -- --static const struct bpf_func_proto * --bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) --{ -- switch (func_id) { -- case BPF_FUNC_task_storage_get: -- return &bpf_task_storage_get_proto; -- case BPF_FUNC_task_storage_delete: -- return &bpf_task_storage_delete_proto; -- default: -- return bpf_base_func_proto(func_id); -- } --} -- --const struct bpf_verifier_ops bpf_scx_verifier_ops = { -- .get_func_proto = bpf_scx_get_func_proto, -- .is_valid_access = bpf_scx_is_valid_access, -- .btf_struct_access = bpf_scx_btf_struct_access, --}; -- --static int bpf_scx_init_member(const struct btf_type *t, -- const struct btf_member *member, -- void *kdata, const void *udata) --{ -- const struct sched_ext_ops *uops = udata; -- struct sched_ext_ops *ops = kdata; -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- int ret; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, dispatch_max_batch): -- if (*(u32 *)(udata + moff) > INT_MAX) -- return -E2BIG; -- ops->dispatch_max_batch = *(u32 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, flags): -- if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) -- return -EINVAL; -- ops->flags = *(u64 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, name): -- ret = bpf_obj_name_cpy(ops->name, uops->name, -- sizeof(ops->name)); -- if (ret < 0) -- return ret; -- if (ret == 0) -- return -EINVAL; -- return 1; -- case offsetof(struct sched_ext_ops, timeout_ms): -- if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT) -- return -E2BIG; -- ops->timeout_ms = *(u32 *)(udata + moff); -- return 1; -- } -- -- return 0; --} -- --static int bpf_scx_check_member(const struct btf_type *t, -- const struct btf_member *member, -- const struct bpf_prog *prog) --{ -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, prep_enable): -- case offsetof(struct sched_ext_ops, init): -- case offsetof(struct sched_ext_ops, exit): -- break; -- default: -- /* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */ -- /* if (prog->aux->sleepable) -- return -EINVAL; */ -- break; -- } -- -- return 0; --} -- --static int bpf_scx_reg(void *kdata) --{ -- return scx_ops_enable(kdata); --} -- --static void bpf_scx_unreg(void *kdata) --{ -- scx_ops_disable(SCX_EXIT_UNREG); -- kthread_flush_work(&scx_ops_disable_work); --} -- --static int bpf_scx_init(struct btf *btf) --{ -- u32 type_id; -- -- type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); -- if (type_id < 0) -- return -EINVAL; -- task_struct_type = btf_type_by_id(btf, type_id); -- -- return 0; --} -- --/* "extern" to avoid sparse warning, only used in this file */ --extern struct bpf_struct_ops bpf_sched_ext_ops; -- --struct bpf_struct_ops bpf_sched_ext_ops = { -- .verifier_ops = &bpf_scx_verifier_ops, -- .reg = bpf_scx_reg, -- .unreg = bpf_scx_unreg, -- .check_member = bpf_scx_check_member, -- .init_member = bpf_scx_init_member, -- .init = bpf_scx_init, -- .name = "sched_ext_ops", --}; -- --static void sysrq_handle_sched_ext_reset(u8 key) --{ -- if (scx_ops_helper) -- scx_ops_disable(SCX_EXIT_SYSRQ); -- else -- pr_info("sched_ext: BPF scheduler not yet used\n"); --} -- --static const struct sysrq_key_op sysrq_sched_ext_reset_op = { -- .handler = sysrq_handle_sched_ext_reset, -- .help_msg = "reset-sched-ext(S)", -- .action_msg = "Disable sched_ext and revert all tasks to CFS", -- .enable_mask = SYSRQ_ENABLE_RTNICE, --}; -- --static void kick_cpus_irq_workfn(struct irq_work *irq_work) --{ -- struct rq *this_rq = this_rq(); -- u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); -- int this_cpu = cpu_of(this_rq); -- int cpu; -- -- for_each_cpu(cpu, this_rq->scx->cpus_to_kick) { -- struct rq *rq = cpu_rq(cpu); -- unsigned long flags; -- -- raw_spin_rq_lock_irqsave(rq, flags); -- -- if (cpu_online(cpu) || cpu == this_cpu) { -- if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) && -- rq->curr->sched_class == &ext_sched_class) -- rq->curr->scx->slice = 0; -- pseqs[cpu] = rq->scx->pnt_seq; -- resched_curr(rq); -- } else { -- cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait); -- } -- -- raw_spin_rq_unlock_irqrestore(rq, flags); -- } -- -- for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait, -- cpumask_of(this_cpu)) { -- /* -- * Pairs with smp_store_release() issued by this CPU in -- * scx_notify_pick_next_task() on the resched path. -- * -- * We busy-wait here to guarantee that no other task can be -- * scheduled on our core before the target CPU has entered the -- * resched path. -- */ -- while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu]) -- cpu_relax(); -- } -- -- cpumask_clear(this_rq->scx->cpus_to_kick); -- cpumask_clear(this_rq->scx->cpus_to_preempt); -- cpumask_clear(this_rq->scx->cpus_to_wait); --} -- --void __init init_sched_ext_class(void) --{ -- int cpu; -- u32 v; -- -- /* -- * The following is to prevent the compiler from optimizing out the enum -- * definitions so that BPF scheduler implementations can use them -- * through the generated vmlinux.h. -- */ -- WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | -- SCX_TG_ONLINE | SCX_KICK_PREEMPT); -- -- init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); --#ifdef CONFIG_SMP -- BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -- BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); --#endif -- scx_kick_cpus_pnt_seqs = -- __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * -- num_possible_cpus(), -- __alignof__(scx_kick_cpus_pnt_seqs[0])); -- BUG_ON(!scx_kick_cpus_pnt_seqs); -- -- for_each_possible_cpu(cpu) { -- struct rq *rq = cpu_rq(cpu); -- -- rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL); -- if (rq->scx) -- rq->scx->rq = rq; -- else -- pr_err("fatal error : alloc rq->scx failed!!!\n"); -- -- init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL); -- INIT_LIST_HEAD(&rq->scx->watchdog_list); -- -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL)); -- init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn); -- } -- -- register_sysrq_key('S', &sysrq_sched_ext_reset_op); -- INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); -- scx_cgroup_config_knobs(); --} -- -- --/******************************************************************************** -- * Helpers that can be called from the BPF scheduler. -- */ --#include -- --/* Disables missing prototype warnings for kfuncs */ --__diag_push(); --__diag_ignore_all("-Wmissing-prototypes", -- "Global functions as their definitions will be in vmlinux BTF"); -- --/** -- * scx_bpf_switch_all - Switch all tasks into SCX -- * @into_scx: switch direction -- * -- * If @into_scx is %true, all existing and future non-dl/rt tasks are switched -- * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on -- * SCX. The actual switching is asynchronous. Can be called from ops.init(). -- */ --void scx_bpf_switch_all(void) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return; -- -- scx_switch_all_req = true; --} -- --BTF_SET8_START(scx_kfunc_ids_init) --BTF_ID_FLAGS(func, scx_bpf_switch_all) --BTF_SET8_END(scx_kfunc_ids_init) -- --static const struct btf_kfunc_id_set scx_kfunc_set_init = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_init, --}; -- --/** -- * scx_bpf_create_dsq - Create a custom DSQ -- * @dsq_id: DSQ to create -- * @node: NUMA node to allocate from -- * -- * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(), -- * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move(). -- */ --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return -EINVAL; -- -- if (unlikely(node >= (int)nr_node_ids || -- (node < 0 && node != NUMA_NO_NODE))) -- return -EINVAL; -- return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); --} -- --BTF_SET8_START(scx_kfunc_ids_sleepable) --BTF_ID_FLAGS(func, scx_bpf_create_dsq) --BTF_SET8_END(scx_kfunc_ids_sleepable) -- --static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_sleepable, --}; -- --static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) --{ -- if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) -- return false; -- -- lockdep_assert_irqs_disabled(); -- -- if (unlikely(!p)) { -- scx_ops_error("called with NULL task"); -- return false; -- } -- -- if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { -- scx_ops_error("invalid enq_flags 0x%llx", enq_flags); -- return false; -- } -- -- return true; --} -- --static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) --{ -- struct task_struct *ddsp_task; -- int idx; -- -- ddsp_task = __this_cpu_read(direct_dispatch_task); -- if (ddsp_task) { -- direct_dispatch(ddsp_task, p, dsq_id, enq_flags); -- return; -- } -- -- idx = __this_cpu_read(scx_dsp_ctx.buf_cursor); -- if (unlikely(idx >= scx_dsp_max_batch)) { -- scx_ops_error("dispatch buffer overflow"); -- return; -- } -- -- this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){ -- .task = p, -- .qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK, -- .dsq_id = dsq_id, -- .enq_flags = enq_flags, -- }; -- __this_cpu_inc(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe -- * to call this function spuriously. Can be called from ops.enqueue() and -- * ops.dispatch(). -- * -- * When called from ops.enqueue(), it's for direct dispatch and @p must match -- * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the -- * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be -- * on the target CPU in the first place. -- * -- * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id -- * and this function can be called upto ops.dispatch_max_batch times to dispatch -- * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the -- * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. -- * -- * This function doesn't have any locking restrictions and may be called under -- * BPF locks (in the future when BPF introduces more flexible locking). -- * -- * @p is allowed to run for @slice. The scheduling path is triggered on slice -- * exhaustion. If zero, the current residual slice is maintained. If -- * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with -- * scx_bpf_kick_cpu() to trigger scheduling. -- */ --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- scx_dispatch_commit(p, dsq_id, enq_flags); --} -- --/** -- * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. -- * Tasks queued into the priority queue are ordered by @vtime and always -- * consumed after the tasks in the FIFO queue. All other aspects are identical -- * to scx_bpf_dispatch(). -- * -- * @vtime ordering is according to time_before64() which considers wrapping. A -- * numerically larger vtime may indicate an earlier position in the ordering and -- * vice-versa. -- */ --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 vtime, u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- p->scx->dsq_vtime = vtime; -- -- scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); --} -- --BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime) --BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_enqueue_dispatch, --}; -- --/** -- * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots -- * -- * Can only be called from ops.dispatch(). -- */ --u32 scx_bpf_dispatch_nr_slots(void) --{ -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return 0; -- -- return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ -- * @dsq_id: DSQ to consume -- * -- * Consume a task from the non-local DSQ identified by @dsq_id and transfer it -- * to the current CPU's local DSQ for execution. Can only be called from -- * ops.dispatch(). -- * -- * This function flushes the in-flight dispatches from scx_bpf_dispatch() before -- * trying to consume the specified DSQ. It may also grab rq locks and thus can't -- * be called under any BPF locks. -- * -- * Returns %true if a task has been consumed, %false if there isn't any task to -- * consume. -- */ --bool scx_bpf_consume(u64 dsq_id) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- struct scx_dispatch_q *dsq; -- -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return false; -- -- flush_dispatch_buf(dspc->rq, dspc->rf); -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); -- return false; -- } -- -- if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) { -- /* -- * A successfully consumed task can be dequeued before it starts -- * running while the CPU is trying to migrate other dispatched -- * tasks. Bump nr_tasks to tell balance_scx() to retry on empty -- * local DSQ. -- */ -- dspc->nr_tasks++; -- return true; -- } else { -- return false; -- } --} -- --BTF_SET8_START(scx_kfunc_ids_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) --BTF_ID_FLAGS(func, scx_bpf_consume) --BTF_SET8_END(scx_kfunc_ids_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_dispatch, --}; -- --/** -- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ -- * -- * Iterate over all of the tasks currently enqueued on the local DSQ of the -- * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of -- * processed tasks. Can only be called from ops.cpu_release(). -- */ --u32 scx_bpf_reenqueue_local(void) --{ -- u32 nr_enqueued, i; -- struct rq *rq; -- struct scx_rq *scx_rq; -- -- if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) -- return 0; -- -- rq = cpu_rq(smp_processor_id()); -- lockdep_assert_rq_held(rq); -- scx_rq = rq->scx; -- -- /* -- * Get the number of tasks on the local DSQ before iterating over it to -- * pull off tasks. The enqueue callback below can signal that it wants -- * the task to stay on the local DSQ, and we want to prevent the BPF -- * scheduler from causing us to loop indefinitely. -- */ -- nr_enqueued = scx_rq->local_dsq.nr; -- for (i = 0; i < nr_enqueued; i++) { -- struct task_struct *p; -- -- p = first_local_task(rq); -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- WARN_ON_ONCE(p->scx->holding_cpu != -1); -- dispatch_dequeue(scx_rq, p); -- do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); -- } -- -- return nr_enqueued; --} -- --BTF_SET8_START(scx_kfunc_ids_cpu_release) --BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) --BTF_SET8_END(scx_kfunc_ids_cpu_release) -- --static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_cpu_release, --}; -- --/** -- * scx_bpf_kick_cpu - Trigger reschedule on a CPU -- * @cpu: cpu to kick -- * @flags: SCX_KICK_* flags -- * -- * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or -- * trigger rescheduling on a busy CPU. This can be called from any online -- * scx_ops operation and the actual kicking is performed asynchronously through -- * an irq work. -- */ --void scx_bpf_kick_cpu(s32 cpu, u64 flags) --{ -- struct rq *rq; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d", cpu); -- return; -- } -- -- preempt_disable(); -- rq = this_rq(); -- -- /* -- * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting -- * rq locks. We can probably be smarter and avoid bouncing if called -- * from ops which don't hold a rq lock. -- */ -- cpumask_set_cpu(cpu, rq->scx->cpus_to_kick); -- if (flags & SCX_KICK_PREEMPT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt); -- if (flags & SCX_KICK_WAIT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_wait); -- -- irq_work_queue(&rq->scx->kick_cpus_irq_work); -- preempt_enable(); --} -- --/** -- * scx_bpf_dsq_nr_queued - Return the number of queued tasks -- * @dsq_id: id of the DSQ -- * -- * Return the number of tasks in the DSQ matching @dsq_id. If not found, -- * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops -- * operations. -- */ --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) --{ -- struct scx_dispatch_q *dsq; -- -- lockdep_assert(rcu_read_lock_any_held()); -- -- if (dsq_id == SCX_DSQ_LOCAL) { -- return this_rq()->scx->local_dsq.nr; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (ops_cpu_valid(cpu)) -- return cpu_rq(cpu)->scx->local_dsq.nr; -- } else { -- dsq = find_non_local_dsq(dsq_id); -- if (dsq) -- return dsq->nr; -- } -- return -ENOENT; --} -- --/** -- * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state -- * @cpu: cpu to test and clear idle for -- * -- * Returns %true if @cpu was idle and its idle state was successfully cleared. -- * %false otherwise. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return false; -- } -- -- if (ops_cpu_valid(cpu)) -- return test_and_clear_cpu_idle(cpu); -- else -- return false; --} -- --/** -- * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu -- * @cpus_allowed: Allowed cpumask -- * -- * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked -- * idle cpu number on success. -%EBUSY if no matching cpu was found. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return -EBUSY; -- } -- -- return scx_pick_idle_cpu(cpus_allowed); --} -- --/** -- * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking -- * per-CPU cpumask. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_cpumask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.cpu; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, -- * per-physical-core cpumask. Can be used to determine if an entire physical -- * core is free. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_smtmask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.smt; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to -- * either the percpu, or SMT idle-tracking cpumask. -- */ --void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) --{ -- /* -- * Empty function body because we aren't actually acquiring or -- * releasing a reference to a global idle cpumask, which is read-only -- * in the caller and is never released. The acquire / release semantics -- * here are just used to make the cpumask is a trusted pointer in the -- * caller. -- */ --} -- --struct scx_bpf_error_bstr_bufs { -- u64 data[MAX_BPRINTF_VARARGS]; -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs); -- --/** -- * scx_bpf_error_bstr - Indicate fatal error -- * @fmt: error message format string -- * @data: format string parameters packaged using ___bpf_fill() macro -- * @data__sz: @data len, must end in '__sz' for the verifier -- * -- * Indicate that the BPF scheduler encountered a fatal error and initiate ops -- * disabling. -- */ --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) --{ -- struct scx_bpf_error_bstr_bufs *bufs; -- unsigned long flags; -- struct bpf_bprintf_data args = {}; -- int ret; -- -- local_irq_save(flags); -- bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs); -- -- if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || -- (data__sz && !data)) { -- scx_ops_error("invalid data=%p and data__sz=%u", -- (void *)data, data__sz); -- goto out_restore; -- } -- -- ret = copy_from_kernel_nofault(bufs->data, data, data__sz); -- if (ret) { -- scx_ops_error("failed to read data fields (%d)", ret); -- goto out_restore; -- } -- -- ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args); -- if (ret < 0) { -- scx_ops_error("failed to format prepration (%d)", ret); -- goto out_restore; -- } -- -- ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt, -- args.bin_args); -- bpf_bprintf_cleanup(&args); -- if (ret < 0) { -- scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format", -- fmt, data, data__sz); -- goto out_restore; -- } -- -- scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg); --out_restore: -- local_irq_restore(flags); --} -- --/** -- * scx_bpf_destroy_dsq - Destroy a custom DSQ -- * @dsq_id: DSQ to destroy -- * -- * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with -- * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is -- * empty and no further tasks are dispatched to it. Ignored if called on a DSQ -- * which doesn't exist. Can be called from any online scx_ops operations. -- */ --void scx_bpf_destroy_dsq(u64 dsq_id) --{ -- destroy_dsq(dsq_id); --} -- --/** -- * scx_bpf_task_running - Is task currently running? -- * @p: task of interest -- */ --bool scx_bpf_task_running(const struct task_struct *p) --{ -- return task_rq(p)->curr == p; --} -- --/** -- * scx_bpf_task_cpu - CPU a task is currently associated with -- * @p: task of interest -- */ --s32 scx_bpf_task_cpu(const struct task_struct *p) --{ -- return task_cpu(p); --} -- --/** -- * scx_bpf_task_cgroup - Return the sched cgroup of a task -- * @p: task of interest -- * -- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with -- * from the scheduler's POV. SCX operations should use this function to -- * determine @p's current cgroup as, unlike following @p->cgroups, -- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all -- * rq-locked operations. Can be called on the parameter tasks of rq-locked -- * operations. The restriction guarantees that @p's rq is locked by the caller. -- */ --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) --{ -- struct task_group *tg = p->sched_task_group; -- struct cgroup *cgrp = &cgrp_dfl_root.cgrp; -- -- if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) -- goto out; -- -- /* -- * A task_group may either be a cgroup or an autogroup. In the latter -- * case, @tg->css.cgroup is %NULL. A task_group can't become the other -- * kind once created. -- */ -- if (tg && tg->css.cgroup) -- cgrp = tg->css.cgroup; -- else -- cgrp = &cgrp_dfl_root.cgrp; --out: -- cgroup_get(cgrp); -- return cgrp; --} -- --BTF_SET8_START(scx_kfunc_ids_any) --BTF_ID_FLAGS(func, scx_bpf_kick_cpu) --BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) --BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) --BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu) --BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) --BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) --BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) --BTF_ID_FLAGS(func, scx_bpf_task_running) --BTF_ID_FLAGS(func, scx_bpf_task_cpu) --BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE) --BTF_SET8_END(scx_kfunc_ids_any) -- --static const struct btf_kfunc_id_set scx_kfunc_set_any = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_any, --}; -- --__diag_pop(); -- --/* -- * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set() -- * needs most of the system to be up. -- */ --static int __init register_ext_kfuncs(void) --{ -- int ret; -- -- /* -- * Some kfuncs are context-sensitive and can only be called from -- * specific SCX ops. They are grouped into BTF sets accordingly. -- * Unfortunately, BPF currently doesn't have a way of enforcing such -- * restrictions. Eventually, the verifier should be able to enforce -- * them. For now, register them the same and make each kfunc explicitly -- * check using scx_kf_allowed(). -- */ -- if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_init)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_sleepable)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_enqueue_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_cpu_release)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_any))) { -- pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret); -- return ret; -- } -- -- return 0; --} --__initcall(register_ext_kfuncs); -diff --git a/kernel/sched/ext.h b/kernel/sched/ext.h -deleted file mode 100755 -index fba8ed845f99..000000000000 ---- a/kernel/sched/ext.h -+++ /dev/null -@@ -1,257 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ -- -- --/* -- * Tag marking a kernel function as a kfunc. This is meant to minimize the -- * amount of copy-paste that kfunc authors have to include for correctness so -- * as to avoid issues such as the compiler inlining or eliding either a static -- * kfunc, or a global kfunc in an LTO build. -- */ --#define __bpf_kfunc __used noinline -- --enum scx_wake_flags { -- /* expose select WF_* flags as enums */ -- SCX_WAKE_EXEC = WF_EXEC, -- SCX_WAKE_FORK = WF_FORK, -- SCX_WAKE_TTWU = WF_TTWU, -- SCX_WAKE_SYNC = WF_SYNC, --}; -- --enum scx_enq_flags { -- /* expose select ENQUEUE_* flags as enums */ -- SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, -- SCX_ENQ_HEAD = ENQUEUE_HEAD, -- -- /* high 32bits are SCX specific */ -- -- /* -- * Set the following to trigger preemption when calling -- * scx_bpf_dispatch() with a local dsq as the target. The slice of the -- * current task is cleared to zero and the CPU is kicked into the -- * scheduling path. Implies %SCX_ENQ_HEAD. -- */ -- SCX_ENQ_PREEMPT = 1LLU << 32, -- -- /* -- * The task being enqueued was previously enqueued on the current CPU's -- * %SCX_DSQ_LOCAL, but was removed from it in a call to the -- * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was -- * invoked in a ->cpu_release() callback, and the task is again -- * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the -- * task will not be scheduled on the CPU until at least the next invocation -- * of the ->cpu_acquire() callback. -- */ -- SCX_ENQ_REENQ = 1LLU << 40, -- -- /* -- * The task being enqueued is the only task available for the cpu. By -- * default, ext core keeps executing such tasks but when -- * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with -- * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set. -- * -- * If the BPF scheduler wants to continue executing the task, -- * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately. -- * If the task gets queued on a different dsq or the BPF side, the BPF -- * scheduler is responsible for triggering a follow-up scheduling event. -- * Otherwise, Execution may stall. -- */ -- SCX_ENQ_LAST = 1LLU << 41, -- -- /* -- * A hint indicating that it's advisable to enqueue the task on the -- * local dsq of the currently selected CPU. Currently used by -- * select_cpu_dfl() and together with %SCX_ENQ_LAST. -- */ -- SCX_ENQ_LOCAL = 1LLU << 42, -- -- /* high 8 bits are internal */ -- __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, -- -- SCX_ENQ_CLEAR_OPSS = 1LLU << 56, -- SCX_ENQ_DSQ_PRIQ = 1LLU << 57, --}; -- --enum scx_deq_flags { -- /* expose select DEQUEUE_* flags as enums */ -- SCX_DEQ_SLEEP = DEQUEUE_SLEEP, -- -- /* high 32bits are SCX specific */ -- -- /* -- * The generic core-sched layer decided to execute the task even though -- * it hasn't been dispatched yet. Dequeue from the BPF side. -- */ -- SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, --}; -- --enum scx_tg_flags { -- SCX_TG_ONLINE = 1U << 0, -- SCX_TG_INITED = 1U << 1, --}; -- --enum scx_kick_flags { -- SCX_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -- SCX_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ --}; -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --extern const struct sched_class ext_sched_class; --extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops; --extern const struct file_operations sched_ext_fops; --extern unsigned long scx_watchdog_timeout; --extern unsigned long scx_watchdog_timestamp; -- --DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); --DECLARE_STATIC_KEY_FALSE(__scx_switched_all); --#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) --#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) -- --DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); -- --bool task_on_scx(struct task_struct *p); --void scx_pre_fork(struct task_struct *p); --int scx_fork(struct task_struct *p); --void scx_post_fork(struct task_struct *p); --void scx_cancel_fork(struct task_struct *p); --int scx_check_setscheduler(struct task_struct *p, int policy); --bool scx_can_stop_tick(struct rq *rq); --void init_sched_ext_class(void); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...); --#define scx_ops_error(fmt, args...) \ -- scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args) -- --void __scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active); -- --static inline void scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active) --{ -- if (!scx_enabled()) -- return; --#ifdef CONFIG_SMP -- /* -- * Pairs with the smp_load_acquire() issued by a CPU in -- * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -- * resched. -- */ -- smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1); --#endif -- if (!static_branch_unlikely(&scx_ops_cpu_preempt)) -- return; -- __scx_notify_pick_next_task(rq, p, active); --} -- --//extern void scx_scheduler_tick(void); --static inline void scx_notify_sched_tick(void) --{ -- unsigned long last_check; -- -- if (!scx_enabled()) -- return; -- //scx_scheduler_tick(); -- -- last_check = scx_watchdog_timestamp; -- if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "watchdog failed to check in for %u.%03us", -- dur_ms / 1000, dur_ms % 1000); -- } --} -- --static inline const struct sched_class *next_active_class(const struct sched_class *class) --{ -- class++; -- if (scx_switched_all() && class == &fair_sched_class) -- class++; -- if (!scx_enabled() && class == &ext_sched_class) -- class++; -- return class; --} -- --#define for_active_class_range(class, _from, _to) \ -- for (class = (_from); class != (_to); class = next_active_class(class)) -- --#define for_each_active_class(class) \ -- for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -- --/* -- * SCX requires a balance() call before every pick_next_task() call including -- * when waking up from idle. -- */ --#define for_balance_class_range(class, prev_class, end_class) \ -- for_active_class_range(class, (prev_class) > &ext_sched_class ? \ -- &ext_sched_class : (prev_class), (end_class)) -- --#ifdef CONFIG_SCHED_CORE --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi); --#endif -- --#else /* CONFIG_SCHED_CLASS_EXT */ -- --#define scx_enabled() false --#define scx_switched_all() false -- --static inline void scx_pre_fork(struct task_struct *p) {} --static inline int scx_fork(struct task_struct *p) { return 0; } --static inline void scx_post_fork(struct task_struct *p) {} --static inline void scx_cancel_fork(struct task_struct *p) {} --static inline int scx_check_setscheduler(struct task_struct *p, -- int policy) { return 0; } --static inline bool scx_can_stop_tick(struct rq *rq) { return true; } --static inline void init_sched_ext_class(void) {} --static inline void scx_notify_pick_next_task(struct rq *rq, -- const struct task_struct *p, -- const struct sched_class *active) {} --static inline void scx_notify_sched_tick(void) {} -- --#define for_each_active_class for_each_class --#define for_balance_class_range for_class_range -- --#endif /* CONFIG_SCHED_CLASS_EXT */ -- --#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) --void __scx_update_idle(struct rq *rq, bool idle); -- --static inline void scx_update_idle(struct rq *rq, bool idle) --{ -- if (scx_enabled()) -- __scx_update_idle(rq, idle); --} --#else --static inline void scx_update_idle(struct rq *rq, bool idle) {} --#endif -- --#ifdef CONFIG_CGROUP_SCHED --#ifdef CONFIG_EXT_GROUP_SCHED --int scx_tg_online(struct task_group *tg); --void scx_tg_offline(struct task_group *tg); --int scx_cgroup_can_attach(struct cgroup_taskset *tset); --void scx_move_task(struct task_struct *p); --void scx_cgroup_finish_attach(void); --void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); --void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); --#else /* CONFIG_EXT_GROUP_SCHED */ --static inline int scx_tg_online(struct task_group *tg) { return 0; } --static inline void scx_tg_offline(struct task_group *tg) {} --static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } --static inline void scx_move_task(struct task_struct *p) {} --static inline void scx_cgroup_finish_attach(void) {} --static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} --static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} --#endif /* CONFIG_EXT_GROUP_SCHED */ --#endif /* CONFIG_CGROUP_SCHED */ -diff --git a/kernel/sched/hmbird.h b/kernel/sched/hmbird.h -new file mode 100755 -index 000000000000..d0d84ddee13d ---- /dev/null -+++ b/kernel/sched/hmbird.h -@@ -0,0 +1,343 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#ifndef _HMBIRD_H_ -+#define _HMBIRD_H_ -+ -+/* -+ * Tag marking a kernel function as a kfunc. This is meant to minimize the -+ * amount of copy-paste that kfunc authors have to include for correctness so -+ * as to avoid issues such as the compiler inlining or eliding either a static -+ * kfunc, or a global kfunc in an LTO build. -+ */ -+ -+#define DEBUG_INTERNAL (1 << 0) -+#define DEBUG_INFO_TRACE (1 << 1) -+#define DEBUG_INFO_SYSTRACE (1 << 2) -+ -+#define NUMS_CGROUP_KINDS (512) -+#define SLIM_FOR_SGAME (1 << 0) -+#define SLIM_FOR_GENSHIN (1 << 1) -+ -+#ifdef MAX_TASK_NR -+#define MAX_KEY_THREAD_RECORD ((MAX_TASK_NR + 1) >> 1) -+#else -+#define MAX_KEY_THREAD_RECORD (1 << 3) -+#endif /* MAX_TASK_NR */ -+ -+#define TOP_TASK_SHIFT (8) -+#define TOP_TASK_MAX (1 << TOP_TASK_SHIFT) -+#ifndef TOP_TASK_BITS_MASK -+#define TOP_TASK_BITS_MASK (TOP_TASK_MAX - 1) -+#endif /* TOP_TASK_BITS_MASK */ -+ -+extern struct md_info_t *md_info; -+ -+#define debug_enabled() \ -+ (unlikely(hmbirdcore_debug)) -+ -+#define hmbird_debug(fmt, ...) \ -+ pr_info(":"fmt, ##__VA_ARGS__) -+ -+#define hmbird_err(id, fmt, ...) \ -+do { \ -+ pr_err(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_deferred_err(id, fmt, ...) \ -+do { \ -+ printk_deferred(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \ -+do { \ -+ if (unlikely(cond)) { \ -+ hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+ } \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_info_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_info_trace(fmt, ...) -+#endif -+ -+#define hmbird_info_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+ } \ -+} while (0) -+ -+#define hmbird_output_systrace(fmt, ...) \ -+do { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_internal_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_internal_trace(fmt, ...) -+#endif -+ -+#define hmbird_internal_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) { \ -+ hmbird_output_systrace(fmt, ##__VA_ARGS__); \ -+ } \ -+} while (0) -+ -+enum hmbird_wake_flags { -+ /* expose select WF_* flags as enums */ -+ HMBIRD_WAKE_EXEC = WF_EXEC, -+ HMBIRD_WAKE_FORK = WF_FORK, -+ HMBIRD_WAKE_TTWU = WF_TTWU, -+ HMBIRD_WAKE_SYNC = WF_SYNC, -+}; -+ -+enum hmbird_enq_flags { -+ /* expose select ENQUEUE_* flags as enums */ -+ HMBIRD_ENQ_WAKEUP = ENQUEUE_WAKEUP, -+ HMBIRD_ENQ_HEAD = ENQUEUE_HEAD, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * Set the following to trigger preemption when calling -+ * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the -+ * current task is cleared to zero and the CPU is kicked into the -+ * scheduling path. Implies %HMBIRD_ENQ_HEAD. -+ */ -+ HMBIRD_ENQ_PREEMPT = 1LLU << 32, -+ HMBIRD_ENQ_REENQ = 1LLU << 40, -+ HMBIRD_ENQ_LAST = 1LLU << 41, -+ -+ /* -+ * A hint indicating that it's advisable to enqueue the task on the -+ * local dsq of the currently selected CPU. Currently used by -+ * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST. -+ */ -+ HMBIRD_ENQ_LOCAL = 1LLU << 42, -+ -+ /* high 8 bits are internal */ -+ __HMBIRD_ENQ_INTERNAL_MASK = 0xffLLU << 56, -+ -+ HMBIRD_ENQ_CLEAR_OPSS = 1LLU << 56, -+ HMBIRD_ENQ_DSQ_PRIQ = 1LLU << 57, -+}; -+ -+enum hmbird_deq_flags { -+ /* expose select DEQUEUE_* flags as enums */ -+ HMBIRD_DEQ_SLEEP = DEQUEUE_SLEEP, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * The generic core-sched layer decided to execute the task even though -+ * it hasn't been dispatched yet. Dequeue from the BPF side. -+ */ -+ HMBIRD_DEQ_CORE_SCHED_EXEC = 1LLU << 32, -+}; -+ -+enum hmbird_kick_flags { -+ HMBIRD_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -+ HMBIRD_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ -+}; -+ -+enum hmbird_task_prop_type { -+ HMBIRD_TASK_PROP_COMMON, -+ HMBIRD_TASK_PROP_PIPELINE, -+ HMBIRD_TASK_PROP_DEBUG_OR_LOG, -+ HMBIRD_TASK_PROP_HIGH_LOAD, -+ HMBIRD_TASK_PROP_IO, -+ HMBIRD_TASK_PROP_NETWORK, -+ HMBIRD_TASK_PROP_PERIODIC, -+ HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL, -+ HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL, -+ HMBIRD_TASK_PROP_ISOLATE, -+ HMBIRD_TASK_PROP_MAX, -+}; -+ -+enum hmbird_preempt_policy_id { -+ HMBIRD_PREEMPT_POLICY_NONE, -+ HMBIRD_PREEMPT_POLICY_PRIO_BASED, -+}; -+ -+extern const struct sched_class hmbird_sched_class; -+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops; -+extern const struct file_operations sched_hmbird_fops; -+extern unsigned long hmbird_watchdog_timeout; -+extern unsigned long hmbird_watchdog_timestamp; -+ -+enum scx_rq_flags { -+ HMBIRD_RQ_CAN_STOP_TICK = 1 << 0, -+}; -+ -+struct hmbird_rq { -+ struct hmbird_dispatch_q local_dsq; -+ struct list_head watchdog_list; -+ u64 ops_qseq; -+ u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -+ u32 nr_running; -+ u32 flags; -+ bool cpu_released; -+ cpumask_var_t cpus_to_kick; -+ cpumask_var_t cpus_to_preempt; -+ cpumask_var_t cpus_to_wait; -+ u64 pnt_seq; -+ u64 *prev_runnable_sum_fixed; -+ u32 *prev_window_size; -+ struct irq_work kick_cpus_irq_work; -+ bool pipeline; -+ bool exclusive; -+ bool period_disallow; -+ bool nonperiod_disallow; -+ struct rq *rq; -+ struct hmbird_sched_rq_stats *srq; -+}; -+ -+struct hmbird_sched_change_guard { -+ struct task_struct *p; -+ struct rq *rq; -+ bool queued; -+ bool running; -+ bool done; -+}; -+ -+extern struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -+ -+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags); -+ -+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -+ for (struct hmbird_sched_change_guard __cg = \ -+ hmbird_sched_change_guard_init(__rq, __p, __flags); \ -+ !__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags)) -+ -+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+bool task_on_hmbird(struct task_struct *p); -+int hmbird_pre_fork(struct task_struct *p); -+int hmbird_fork(struct task_struct *p); -+void hmbird_post_fork(struct task_struct *p); -+void hmbird_cancel_fork(struct task_struct *p); -+int hmbird_check_setscheduler(struct task_struct *p, int policy); -+bool hmbird_can_stop_tick(struct rq *rq); -+void init_sched_hmbird_class(void); -+ -+void hmbird_ops_exit(void); -+#define hmbird_ops_error(fmt, ...) \ -+do { \ -+ hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__); \ -+ hmbird_ops_exit(); \ -+} while (0) -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active); -+ -+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight) -+{ -+ return clamp_t(unsigned long, -+ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -+ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); -+} -+ -+static inline void hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active) -+{ -+ if (!hmbird_enabled()) -+ return; -+#ifdef CONFIG_SMP -+ /* -+ * Pairs with the smp_load_acquire() issued by a CPU in -+ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -+ * resched. -+ */ -+ smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1); -+#endif -+ if (!static_branch_unlikely(&hmbird_ops_cpu_preempt)) -+ return; -+ __hmbird_notify_pick_next_task(rq, p, active); -+} -+ -+extern void hmbird_scheduler_tick(void); -+void scan_timeout(struct rq *rq); -+void hmbird_notify_sched_tick(void); -+ -+static inline const struct sched_class *next_active_class(const struct sched_class *class) -+{ -+ class++; -+ -+ if (hmbird_enabled() && class == &fair_sched_class) -+ class++; -+ if (!hmbird_enabled() && class == &hmbird_sched_class) -+ class++; -+ return class; -+} -+ -+#define for_active_class_range(class, _from, _to) \ -+ for (class = (_from); class != (_to); class = next_active_class(class)) -+ -+#define for_each_active_class(class) \ -+ for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -+ -+/* -+ * HMBIRD requires a balance() call before every pick_next_task() call including -+ * when waking up from idle. -+ */ -+#define for_balance_class_range(class, prev_class, end_class) \ -+ for_active_class_range(class, (prev_class) > &hmbird_sched_class ? \ -+ &hmbird_sched_class : (prev_class), (end_class)) -+ -+#define MIN_CGROUP_DL_IDX (5) /* 8ms */ -+#define DEFAULT_CGROUP_DL_IDX (8) /* 64ms */ -+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS]; -+ -+ -+void __hmbird_update_idle(struct rq *rq, bool idle); -+ -+static inline void hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ if (hmbird_enabled()) -+ __hmbird_update_idle(rq, idle); -+} -+ -+int hmbird_ctrl(bool enable); -+ -+int hmbird_tg_online(struct task_group *tg); -+ -+ -+static inline u16 hmbird_task_util(struct task_struct *p) -+{ -+ return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0; -+} -+u16 hmbird_cpu_util(int cpu); -+ -+bool get_hmbird_ops_enabled(void); -+bool get_non_hmbird_task(void); -+void set_cpu_cluster(u64 cpu_cluster); -+#endif /*_EXT_H_*/ -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -new file mode 100755 -index 000000000000..ce05928392bf ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird.c -@@ -0,0 +1,4313 @@ -+// SPDX-License-Identifier: GPL-2.0 -+ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#include -+#include -+ -+#include "slim.h" -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+#include -+ -+#define CLUSTER_SEPARATE -+ -+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0); -+atomic_t non_hmbird_task; -+atomic_t hmbird_module_loaded = ATOMIC_INIT(0); -+int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0}; -+ -+enum hmbird_internal_consts { -+ HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ, -+}; -+ -+enum hmbird_ops_enable_state { -+ HMBIRD_OPS_PREPPING, -+ HMBIRD_OPS_ENABLING, -+ HMBIRD_OPS_ENABLED, -+ HMBIRD_OPS_DISABLING, -+ HMBIRD_OPS_DISABLED, -+}; -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+/* -+ * hmbird_entity->ops_state -+ * -+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler. -+ * State transitions look as follows: -+ * -+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING -+ * ^ | | -+ * | v v -+ * \-------------------------------/ -+ * -+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -+ * sites for explanations on the conditions being waited upon and why they are -+ * safe. Transitions out of them into NONE or QUEUED must store_release and the -+ * waiters should load_acquire. -+ * -+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether -+ * any given task can be dispatched by the BPF scheduler at all times and thus -+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core -+ * can safely reject invalid dispatches. -+ */ -+enum hmbird_ops_state { -+ HMBIRD_OPSS_NONE, /* owned by the HMBIRD core */ -+ HMBIRD_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -+ HMBIRD_OPSS_QUEUED, /* owned by the BPF scheduler */ -+ HMBIRD_OPSS_DISPATCHING, /* in transit back to the HMBIRD core */ -+ -+ /* -+ * QSEQ brands each QUEUED instance so that, when dispatch races -+ * dequeue/requeue, the dispatcher can tell whether it still has a claim -+ * on the task being dispatched. -+ */ -+ HMBIRD_OPSS_QSEQ_SHIFT = 2, -+ HMBIRD_OPSS_STATE_MASK = (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1, -+ HMBIRD_OPSS_QSEQ_MASK = ~HMBIRD_OPSS_STATE_MASK, -+}; -+ -+enum switch_stat { -+ HMBIRD_DISABLED, -+ HMBIRD_SWITCH_PREP, -+ HMBIRD_RQ_SWITCH_BEGIN, -+ HMBIRD_RQ_SWITCH_DONE, -+ HMBIRD_ENABLED, -+}; -+enum switch_stat curr_ss; -+ -+/* -+ * During exit, a task may schedule after losing its PIDs. When disabling the -+ * BPF scheduler, we need to be able to iterate tasks in every state to -+ * guarantee system safety. Maintain a dedicated task list which contains every -+ * task between its fork and eventual free. -+ */ -+DEFINE_SPINLOCK(hmbird_tasks_lock); -+static LIST_HEAD(hmbird_tasks); -+ -+/* ops enable/disable */ -+static struct kthread_worker *hmbird_ops_helper; -+static DEFINE_MUTEX(hmbird_ops_enable_mutex); -+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem); -+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED); -+ -+static bool hmbird_warned_zero_slice; -+enum hmbird_switch_type sw_type; -+static int skip_num[MAX_GLOBAL_DSQS]; -+static int big_distribute_mask_prev; -+static int little_distribute_mask_prev; -+ -+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0); -+ -+/* -+ * The maximum amount of time in jiffies that a task may be runnable without -+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger -+ * hmbird_ops_error(). -+ */ -+unsigned long hmbird_watchdog_timeout; -+ -+/* -+ * The last time the delayed work was run. This delayed work relies on -+ * ksoftirqd being able to run to service timer interrupts, so it's possible -+ * that this work itself could get wedged. To account for this, we check that -+ * it's not stalled in the timer tick, and trigger an error if it is. -+ */ -+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES; -+ -+static struct delayed_work hmbird_watchdog_work; -+static struct work_struct hmbird_err_exit_work; -+ -+/* idle tracking */ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_CPUMASK_OFFSTACK -+#define CL_ALIGNED_IF_ONSTACK -+#else -+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp -+#endif -+ -+static struct { -+ cpumask_var_t cpu; -+ cpumask_var_t smt; -+} idle_masks CL_ALIGNED_IF_ONSTACK; -+ -+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus; -+#endif /* CONFIG_SMP */ -+ -+/* dispatch queues */ -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global; -+ -+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128}; -+u32 pcp_dsq_deadline = 20; -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS]; -+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq); -+ -+static u64 max_hmbird_dsq_internal_id; -+ -+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/ -+#define CLUSTER_SEPARATE_IDX (8) -+#define GDSQS_ID_BASE (3) -+#define UX_COMPATIBLE_IDX (4) -+#define NON_PERIOD_START (5) -+#define NON_PERIOD_END (MAX_GLOBAL_DSQS) -+#define CREATE_DSQ_LEVEL_WITHIN (1) -+ -+struct hmbird_sched_info { -+ spinlock_t lock; -+ int curr_idx[2]; -+ int rtime[MAX_GLOBAL_DSQS]; -+}; -+ -+struct pcp_sched_info { -+ s64 pcp_seq; -+ int rtime; -+ bool pcp_round; -+}; -+ -+/* -+ * pcp_info may rw by another cpu. -+ * protected by rq->lock. -+ */ -+atomic64_t pcp_dsq_round; -+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info); -+ -+struct md_info_t *md_info; -+ -+static int b_rescue_l, l_rescue_b; -+static struct hmbird_sched_info sinfo; -+ -+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC; -+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = { -+ 0, 0, 0, 0, 0, -+ 32 * NSEC_PER_MSEC, -+ 20 * NSEC_PER_MSEC, -+ 14 * NSEC_PER_MSEC, -+ 8 * NSEC_PER_MSEC, -+ 6 * NSEC_PER_MSEC -+}; -+ -+struct cluster_ctx { -+ /* cpu-dsq map must within [lower, upper) */ -+ int upper; -+ int lower; -+ int tidx; -+}; -+ -+enum stat_items { -+ GLOBAL_STAT, -+ CPU_ALLOW_FAIL, -+ RT_CNT, -+ KEY_TASK_CNT, -+ SWITCH_IDX, -+ TIMEOUT_CNT, -+ -+ TOTAL_DSP_CNT, -+ MOVE_RQ_CNT, -+ SELECT_CPU, -+ -+ DWORD_STAT_END = SELECT_CPU, -+ -+ GDSQ_CNT, -+ ERR_IDX, -+ PCP_TIMEOUT_CNT, -+ PCP_LDSQ_CNT, -+ PCP_ENQL_CNT, -+ -+ MAX_ITEMS, -+}; -+static DEFINE_SPINLOCK(stats_lock); -+static char *stats_str[MAX_ITEMS] = { -+ "global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt", -+ "switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt", -+ "select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt", -+ "pcp_ldsq_cnt", "pcp_enql_cnt" -+}; -+ -+ -+struct stats_struct { -+ u64 global_stat[2]; -+ u64 cpu_allow_fail[2]; -+ u64 rt_cnt[2]; -+ u64 key_task_cnt[2]; -+ u64 switch_idx[2]; -+ u64 timeout_cnt[2]; -+ -+ u64 total_dsp_cnt[2]; -+ u64 move_rq_cnt[2]; -+ u64 select_cpu[2]; -+ -+ u64 gdsq_count[MAX_GLOBAL_DSQS][2]; -+ u64 err_idx[5]; -+ u64 pcp_timeout_cnt[NR_CPUS]; -+ u64 pcp_ldsq_count[NR_CPUS][2]; -+ u64 pcp_enql_cnt[NR_CPUS]; -+} stats_data; -+ -+static struct { -+ cpumask_var_t ex_free; -+ cpumask_var_t exclusive; -+ cpumask_var_t partial; -+ cpumask_var_t big; -+ cpumask_var_t little; -+} iso_masks __read_mostly; -+ -+/* -+ * Need more synchronization for these two variables? -+ * I choose not to. -+ */ -+static int l_need_rescue, b_need_rescue; -+ -+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...) \ -+{ \ -+ char buf[MAX_FATAL_INFO]; \ -+ \ -+ scnprintf(buf, MAX_FATAL_INFO, fmt, ##args); \ -+ hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf); \ -+ trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable), \ -+ READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf); \ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); \ -+} \ -+ -+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2) -+{ -+ int c1, c2; -+ struct cpumask mask = {.bits = {0}}; -+ struct cpumask tmp = {.bits = {0}}; -+ -+ if (!slim_stats) -+ return false; -+ -+ if (!rq1 || !rq2) -+ return false; -+ -+ c1 = cpu_of(rq1); -+ c2 = cpu_of(rq2); -+ cpumask_set_cpu(c1, &mask); -+ cpumask_set_cpu(c2, &mask); -+ -+ if (cpumask_and(&tmp, iso_masks.little, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.big, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.partial, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ return false; -+} -+ -+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu) -+{ -+ unsigned long flags; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ if (!slim_stats) -+ return; -+ -+ switch (item) { -+ case GLOBAL_STAT: -+ fallthrough; -+ case CPU_ALLOW_FAIL: -+ fallthrough; -+ case RT_CNT: -+ fallthrough; -+ case KEY_TASK_CNT: -+ fallthrough; -+ case SWITCH_IDX: -+ fallthrough; -+ case TIMEOUT_CNT: -+ fallthrough; -+ case TOTAL_DSP_CNT: -+ fallthrough; -+ case MOVE_RQ_CNT: -+ fallthrough; -+ case SELECT_CPU: -+ pval = pbase + item * 2 + idx; -+ break; -+ case GDSQ_CNT: -+ pval = &stats_data.gdsq_count[dsq_id][idx]; -+ break; -+ case ERR_IDX: -+ pval = &stats_data.err_idx[idx]; -+ break; -+ case PCP_TIMEOUT_CNT: -+ pval = &stats_data.pcp_timeout_cnt[cpu]; -+ break; -+ case PCP_LDSQ_CNT: -+ pval = &stats_data.pcp_ldsq_count[cpu][idx]; -+ break; -+ case PCP_ENQL_CNT: -+ pval = &stats_data.pcp_enql_cnt[cpu]; -+ break; -+ default: -+ return; -+ } -+ -+ spin_lock_irqsave(&stats_lock, flags); -+ *pval += 1; -+ spin_unlock_irqrestore(&stats_lock, flags); -+} -+ -+static inline bool handle_ret(int ret, int *idx, int len) -+{ -+ if (ret < 0 || ret >= len - *idx) -+ return true; -+ *idx += ret; -+ return false; -+} -+ -+#define PRINT_INTV (5 * HZ) -+void stats_print(char *buf, int len) -+{ -+ int idx = 0, i, j, ret; -+ int item = 0; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ for (item = 0; item < MAX_ITEMS; item++) { -+ if (item <= DWORD_STAT_END) { -+ pval = pbase + item * 2; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n", -+ stats_str[item], pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == GDSQ_CNT) { -+ for (j = 0; j < MAX_GLOBAL_DSQS; j++) { -+ pval = (u64 *)&stats_data.gdsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == ERR_IDX) { -+ pval = (u64 *)&stats_data.err_idx; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n", -+ stats_str[item], pval[0], -+ pval[1], pval[2], pval[3], pval[4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == PCP_TIMEOUT_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_timeout_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_LDSQ_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_ldsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_ENQL_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_enql_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } -+ } -+ -+ if (!md_info) { -+ buf[idx] = '\0'; -+ return; -+ } -+ -+ ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_SWITCHS; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "sw_rec[%d] = %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.sw_rec[i].switch_at, -+ md_info->kern_dump.sw_rec[i].is_success, -+ md_info->kern_dump.sw_rec[i].end_state, -+ md_info->kern_dump.sw_rec[i].switch_reason); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_EXCEP_ID; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "excep[%d] = %llu %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.excep_rec[i][0], -+ md_info->kern_dump.excep_rec[i][1], -+ md_info->kern_dump.excep_rec[i][2], -+ md_info->kern_dump.excep_rec[i][3], -+ md_info->kern_dump.excep_rec[i][4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n", -+ i, md_info->kern_dump.excep_idx[i]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ -+ buf[idx] = '\0'; -+} -+ -+enum cpu_type { -+ LITTLE, -+ BIG, -+ PARTIAL, -+ EXCLUSIVE, -+ EX_FREE, -+ INVALID -+}; -+ -+enum dsq_type { -+ GLOBAL_DSQ, -+ PCP_DSQ, -+ OTHER, -+ MAX_DSQ_TYPE, -+}; -+ -+static enum cpu_type cpu_cluster(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (get_hmbird_rq(rq)->exclusive) { -+ return EXCLUSIVE; -+ } else { -+ if (cpumask_test_cpu(cpu, iso_masks.little)) -+ return LITTLE; -+ else if (cpumask_test_cpu(cpu, iso_masks.big)) -+ return BIG; -+ else if (cpumask_test_cpu(cpu, iso_masks.partial)) -+ return PARTIAL; -+ else if (cpumask_test_cpu(cpu, iso_masks.exclusive)) -+ return EXCLUSIVE; -+ else if (cpumask_test_cpu(cpu, iso_masks.ex_free)) -+ return EX_FREE; -+ } -+ return INVALID; -+} -+ -+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq) -+{ -+ if (!dsq) -+ return OTHER; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= GDSQS_ID_BASE) && -+ ((dsq->id & 0xff) < MAX_GLOBAL_DSQS)) -+ return GLOBAL_DSQ; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) && -+ ((dsq->id & 0xff) < max_hmbird_dsq_internal_id)) -+ return PCP_DSQ; -+ -+ return OTHER; -+} -+ -+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq) -+{ -+ enum dsq_type type; -+ -+ type = get_dsq_type(dsq); -+ switch (type) { -+ case GLOBAL_DSQ: -+ case PCP_DSQ: -+ return (dsq->id & 0xff) - GDSQS_ID_BASE; -+ default: -+ return -1; -+ } -+ return -1; -+} -+ -+static void update_cpus_idle(bool set, struct cpumask *mask) -+{ -+ int cpu; -+ struct rq *rq; -+ -+ if (set) { -+ for_each_cpu(cpu, mask) { -+ rq = cpu_rq(cpu); -+ if (is_idle_task(rq->curr)) -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ } -+ } else -+ cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask); -+} -+ -+static void set_partial_status(bool enable, bool little, bool big) -+{ -+ WRITE_ONCE(partial_enable, enable); -+ WRITE_ONCE(l_need_rescue, little); -+ WRITE_ONCE(b_need_rescue, big); -+} -+ -+static bool is_little_need_rescue(void) -+{ -+ return READ_ONCE(l_need_rescue); -+} -+ -+static bool is_big_need_rescue(void) -+{ -+ return READ_ONCE(b_need_rescue); -+} -+ -+static bool is_partial_enabled(void) -+{ -+ return READ_ONCE(partial_enable); -+} -+ -+static bool is_partial_cpu(int cpu) -+{ -+ return cpumask_test_cpu(cpu, iso_masks.partial); -+} -+ -+static void set_iso_par_free(bool enable) -+{ -+ WRITE_ONCE(isolate_ctrl, enable); -+} -+ -+static bool is_iso_par_free(void) -+{ -+ return READ_ONCE(isolate_ctrl); -+} -+ -+static bool skip_update_idle(void) -+{ -+ int cpu = smp_processor_id(); -+ enum cpu_type type = cpu_cluster(cpu); -+ -+ if ((type == EXCLUSIVE && !is_iso_par_free()) || -+ /* partial enable may changed during idle, it doesn't matter. */ -+ (!is_partial_enabled() && type == PARTIAL)) -+ return true; -+ -+ return false; -+} -+ -+static void init_isolate_cpus(void) -+{ -+ WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -+ cpumask_set_cpu(0, iso_masks.big); -+ cpumask_set_cpu(1, iso_masks.big); -+ cpumask_set_cpu(2, iso_masks.big); -+ cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(4, iso_masks.big); -+ cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(6, iso_masks.partial); -+ cpumask_set_cpu(7, iso_masks.ex_free); -+} -+ -+extern spinlock_t css_set_lock; -+ -+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp) -+{ -+ int i; -+ struct cgroup *anc; -+ -+ spin_lock_irq(&css_set_lock); -+ for (i = 0; i < cgrp->level; i++) { -+ anc = cgrp->ancestors[i]; -+ if (anc->level != CREATE_DSQ_LEVEL_WITHIN) -+ continue; -+ cgroup_get(anc); -+ spin_unlock_irq(&css_set_lock); -+ return anc; -+ } -+ spin_unlock_irq(&css_set_lock); -+ hmbird_err(NO_CGROUP_L1, ":error cgroup = %s\n", cgrp->kn->name); -+ return NULL; -+} -+ -+#define PCP_IDX_BIT (1 << 31) -+ -+static bool is_pcp_rt(struct task_struct *p) -+{ -+ return rt_prio(p->prio) && (p->nr_cpus_allowed == 1); -+} -+ -+static bool is_pcp_idx(int idx) -+{ -+ return idx >= MAX_GLOBAL_DSQS; -+} -+ -+static bool is_critical_system_task(struct task_struct *p) -+{ -+ int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK; -+ -+ return (p->prio < (MAX_RT_PRIO >> 1) && -+ (sp_dl < SCHED_PROP_DEADLINE_LEVEL3)); -+} -+ -+#define ISOLATE_TASK_TOP_BIT (1 << 17) -+#define PIPELINE_TASK_TOP_BIT (1 << 9) -+static bool is_pipeline_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT); -+} -+ -+static bool is_isolate_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT); -+} -+ -+static bool is_critical_app_task_without_isolate(struct task_struct *p) -+{ -+ return task_is_top_task(p) && !is_isolate_task(p); -+} -+ -+static int find_idx_from_task(struct task_struct *p) -+{ -+ int idx, cpu; -+ int sp_dl; -+ struct task_group *tg = p->sched_task_group; -+ -+ if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) { -+ cpu = cpumask_any(p->cpus_ptr); -+ idx = cpu + MAX_GLOBAL_DSQS; -+ return idx; -+ } -+ -+ if (is_critical_system_task(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL0; -+ goto done; -+ } -+ -+ if (is_critical_app_task_without_isolate(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL1; -+ goto done; -+ } -+ -+ sp_dl = hmbird_get_dsq_id(p); -+ if (sp_dl) { -+ idx = sp_dl; -+ goto done; -+ } -+ -+ if (rt_prio(p->prio)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL3; -+ goto done; -+ } -+ -+ if (tg && tg->css.cgroup && tg->css.cgroup->kn) { -+ if (likely(tg->css.cgroup->kn->id >= 0 && -+ tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS)) -+ idx = cgroup_ids_table[tg->css.cgroup->kn->id]; -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ -+done: -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) { -+ hmbird_err(DSQ_ID_ERR, " : idx error, idx = %d-----\n", idx); -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } -+ return idx; -+} -+ -+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p) -+{ -+ int idx; -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq; -+ -+ if (!p) -+ return NULL; -+ -+ idx = find_idx_from_task(p); -+ if (is_pcp_idx(idx)) { -+ idx -= MAX_GLOBAL_DSQS; -+ dsq = &per_cpu(pcp_ldsq, idx); -+ get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq); -+ slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx); -+ } else { -+ dsq = &gdsqs[idx]; -+ get_hmbird_ts(p)->gdsq_idx = idx; -+ slim_stats_record(GDSQ_CNT, 0, idx, 0); -+ } -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ dsq->last_consume_at = jiffies; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ return dsq; -+} -+ -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq); -+ -+static void set_partial_rescue(bool p_state, bool l_over, bool b_over) -+{ -+ set_partial_status(p_state, l_over, b_over); -+ update_cpus_idle(p_state, iso_masks.partial); -+ hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled()); -+ hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue()); -+ hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue()); -+} -+ -+static void free_isocpu(bool enable) -+{ -+ set_iso_par_free(enable); -+ update_cpus_idle(enable, iso_masks.ex_free); -+ hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free()); -+} -+ -+inline u64 get_hmbird_cpu_util(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed) -+ return 0; -+ u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed); -+ u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size); -+ -+ do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT); -+ -+ return prev_runnable_sum_fixed; -+} -+ -+static inline unsigned int get_scaling_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->max; -+} -+ -+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static u64 get_cpus_max_util(struct cpumask *mask) -+{ -+ int cpu; -+ u64 max = 0; -+ u64 util, ratio; -+ unsigned long effective_cap = 0; -+ for_each_cpu(cpu, mask) { -+ if (slim_walt_ctrl) -+ slim_get_cpu_util(cpu, &util); -+ else -+ util = get_hmbird_cpu_util(cpu); -+ -+ /* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/ -+ if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu))) -+ effective_cap = arch_scale_cpu_capacity(cpu); -+ else -+ effective_cap = arch_scale_cpu_capacity(cpu) * -+ get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu); -+ -+ ratio = util * 100 / effective_cap; -+ hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util); -+ hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n", -+ cpu, (u64)arch_scale_cpu_capacity(cpu)); -+ hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n", -+ cpu, (u64)effective_cap); -+ -+ if (ratio > 100) -+ ratio = 100; -+ -+ if (ratio > max) -+ max = ratio; -+ } -+ return max; -+} -+ -+static bool cluster_need_rescue(struct cpumask *mask, int hres) -+{ -+ return get_cpus_max_util(mask) > hres; -+} -+ -+void partial_dynamic_ctrl(void) -+{ -+ u64 lmax = 0, bmax = 0; -+ bool l_over, l_under, b_over, b_under; -+ static bool last_l_over, last_b_over; -+ static unsigned long last_check; -+ -+ /* Check partial every jiffies. need lock? */ -+ if (time_before_eq(jiffies, READ_ONCE(last_check))) -+ return; -+ -+ WRITE_ONCE(last_check, jiffies); -+ -+ lmax = get_cpus_max_util(iso_masks.little); -+ l_over = lmax > parctrl_high_ratio_l; -+ l_under = lmax < parctrl_low_ratio_l; -+ -+ bmax = get_cpus_max_util(iso_masks.big); -+ b_over = bmax > parctrl_high_ratio; -+ b_under = bmax < parctrl_low_ratio; -+ -+ if (is_partial_enabled() && (l_over || b_over)) { -+ if (last_l_over != l_over || last_b_over != b_over) -+ set_partial_rescue(true, l_over, b_over); -+ } else if (!is_partial_enabled() && (l_over || b_over)) { -+ set_partial_rescue(true, l_over, b_over); -+ } else if (is_partial_enabled() && l_under && b_under) { -+ set_partial_rescue(false, false, false); -+ } -+ last_l_over = l_over; -+ last_b_over = b_over; -+ -+ if (is_partial_enabled()) { -+ bmax = get_cpus_max_util(iso_masks.partial); -+ if (!is_iso_par_free() && bmax > isoctrl_high_ratio) { -+ hmbird_info_trace("partial max = %llu\n", bmax); -+ free_isocpu(true); -+ } else if (is_iso_par_free() && bmax < isoctrl_low_ratio) -+ free_isocpu(false); -+ } else if (is_iso_par_free()) { -+ free_isocpu(false); -+ } -+} -+ -+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx) -+{ -+ hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx); -+} -+ -+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx) -+{ -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[idx])) { -+ slim_stats_record(GDSQ_CNT, 1, idx, 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ int i; -+ -+ for (i = 0; i < UX_COMPATIBLE_IDX; i++) { -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ slim_stats_record(GDSQ_CNT, 1, i, 0); -+ return 1; -+ } -+ } -+ return 0; -+} -+ -+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) { -+ slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ goto clear_timeout; -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) -+ goto clear_timeout; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ hmbird_info_trace( -+ "dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n", -+ dsq_id_to_internal(dsq), entity->task->comm, -+ jiffies, msecs_to_jiffies(deadline), entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1); -+ return; -+ -+clear_timeout: -+ hmbird_info_trace("dsq[%d] clear timeout\n", -+ dsq_id_to_internal(dsq)); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0); -+ dsq->is_timeout = false; -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq)); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime) -+{ -+ hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime); -+} -+ -+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any) -+{ -+ bool is_timeout; -+ int cpu = cpu_of(rq); -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu); -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ is_timeout = dsq->is_timeout; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ /* -+ * dsq->is_timeout may change here, let it be. -+ * it won't cause serious problems. -+ * the same for consume_dispatch_q later. -+ */ -+ if (!is_timeout && !any) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, dsq)) { -+ if (is_timeout) { -+ hmbird_info_trace("dsq[%d] consume a pcp timeout task\n", -+ dsq_id_to_internal(dsq)); -+ update_timeout_stats(rq, dsq, pcp_dsq_deadline); -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu); -+ } -+ slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu); -+ return 1; -+ } -+ /* -+ * No pcp task, clear quota. -+ */ -+ if (any) { -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+ } -+ return 0; -+} -+ -+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf) -+{ -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ } -+ return 0; -+} -+ -+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf, -+ int tmp, int cidx, int tidx) -+{ -+ unsigned long flags; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) { -+ if (tmp != cidx) { -+ spin_lock(&sinfo.lock); -+ sinfo.curr_idx[tidx] = tmp; -+ spin_unlock(&sinfo.lock); -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]); -+ slim_stats_record(SWITCH_IDX, 0, 0, 0); -+ } -+ slim_stats_record(GDSQ_CNT, 1, tmp, 0); -+ -+ raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags); -+ gdsqs[tmp].last_consume_at = jiffies; -+ raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags); -+ return 1; -+ } -+ return 0; -+ -+} -+ -+static int get_cidx(struct cluster_ctx *ctx) -+{ -+ int cidx; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ if (cidx < ctx->lower || cidx >= ctx->upper) { -+ sinfo.curr_idx[ctx->tidx] = ctx->lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]); -+ slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ } -+ spin_unlock(&sinfo.lock); -+ -+ return cidx; -+} -+ -+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = CLUSTER_SEPARATE_IDX; -+ if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) { -+ pr_debug(" %s iso_masks.little first is %d\n", -+ __func__, cpumask_first(iso_masks.little)); -+ ctx->upper = NON_PERIOD_END; -+ } -+ ctx->tidx = 0; -+ break; -+ case LITTLE: -+ ctx->lower = CLUSTER_SEPARATE_IDX; -+ ctx->upper = NON_PERIOD_END; -+ if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) { -+ pr_debug(" %s iso_masks.big first is %d", -+ __func__, cpumask_first(iso_masks.big)); -+ ctx->lower = NON_PERIOD_START; -+ } -+ ctx->tidx = 1; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ case LITTLE: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = NON_PERIOD_END; -+ ctx->tidx = 0; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ if (cluster_separate) { -+ return gen_cluster_ctx_separate(ctx, type); -+ } else { -+ return gen_cluster_ctx_common(ctx, type); -+ } -+} -+ -+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ int i; -+ bool is_timeout; -+ unsigned long flags; -+ struct cluster_ctx ctx; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ /* Third param means only consume timeout pcp dsq. */ -+ if (consume_pcp_dsq(rq, rf, false)) -+ return 1; -+ -+ for (i = ctx.lower; i < ctx.upper; i++) { -+ raw_spin_lock_irqsave(&gdsqs[i].lock, flags); -+ is_timeout = gdsqs[i].is_timeout; -+ raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags); -+ /* gdsqs[i].is_timeout may change here, let it be... */ -+ if (is_timeout) { -+ /* -+ * consume_dispatch_q will acquire dsq-lock, -+ * So cannot keep lock here, annoy enough. -+ * may rewrite a consume_dispatch_q_locked, TODO. -+ */ -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ hmbird_info_trace("dsq[%d] consume a timeout task\n", i); -+ slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0); -+ update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]); -+ return 1; -+ } -+ } -+ } -+ return 0; -+} -+ -+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ struct cluster_ctx ctx; -+ int cidx; -+ int tmp; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ cidx = get_cidx(&ctx); -+ tmp = cidx; -+ do { -+ if (check_pcp_dsq_round(rq, rf)) -+ return 1; -+ if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx)) -+ return 1; -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[tmp] = 0; -+ systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]); -+ tmp++; -+ if (tmp >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ tmp = ctx.lower; -+ } -+ spin_unlock(&sinfo.lock); -+ } while (tmp != cidx); -+ -+ return consume_pcp_dsq(rq, rf, true); -+} -+ -+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ enum cpu_type type = cpu_cluster(cpu_of(rq)); -+ bool period_allowed = !get_hmbird_rq(rq)->period_disallow; -+ bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow; -+ -+ switch (type) { -+ case EX_FREE: -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ break; -+ case EXCLUSIVE: -+ if (!is_iso_par_free()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ } -+ /* Only non-period task can run on isolate cpus */ -+ if (!READ_ONCE(iso_free_rescue)) { -+ if (is_big_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ } else { -+ /* Free run, can run on any task. */ -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ case PARTIAL: -+ if (!is_partial_enabled()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ return 0; -+ } -+ if (is_big_need_rescue()) { -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ -+ case BIG: -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ if (is_iso_par_free() || (is_little_need_rescue() && -+ !cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) { -+ hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++); -+ return 1; -+ } -+ } -+ break; -+ -+ case LITTLE: -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ -+ if (is_iso_par_free() || (is_big_need_rescue() && -+ !cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) { -+ hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++); -+ return 1; -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return 0; -+} -+ -+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf) -+{ -+ return consume_hmbird_global_dsq(rq, rf); -+} -+ -+ -+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time) -+{ -+ int idx; -+ -+ /* which dsq belongs to while task enqueue, task will consume its running time. */ -+ idx = get_hmbird_ts(p)->gdsq_idx; -+ /* Only non-period dsq share running time between each other. */ -+ if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id) -+ return; -+ -+ if (idx >= MAX_GLOBAL_DSQS) { -+ per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } else { -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[idx] += exec_time; -+ spin_unlock(&sinfo.lock); -+ systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]); -+ } -+} -+ -+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type) -+{ -+ int cidx; -+ struct cluster_ctx ctx; -+ int cpu = cpu_of(rq); -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ if (cidx < ctx.lower || cidx >= ctx.upper) { -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ slim_stats_record(ERR_IDX, ctx.tidx, 0, 0); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ } -+ -+ while (1) { -+ if (per_cpu(pcp_info, cpu).pcp_round) { -+ if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) { -+ hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n", -+ cpu, per_cpu(pcp_info, cpu).rtime); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } -+ } -+ if (sinfo.rtime[cidx] < dsq_quota[cidx]) -+ break; -+ -+ /* clear current dsq rtime */ -+ sinfo.rtime[cidx] = 0; -+ systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]); -+ -+ sinfo.curr_idx[ctx.tidx]++; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", -+ ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ } -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ slim_stats_record(SWITCH_IDX, 1, 0, 0); -+ } -+ spin_unlock(&sinfo.lock); -+} -+ -+ -+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p) -+{ -+ enum cpu_type type; -+ -+ if (!rq || !p) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ switch (type) { -+ case PARTIAL: -+ return; -+ case EXCLUSIVE: -+ return; -+ case EX_FREE: -+ return; -+ default: -+ break; -+ } -+ update_dsq_idx(rq, p, type); -+} -+ -+ -+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ int dsq_id; -+ -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo) || dsq->is_timeout) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ hmbird_deferred_err(SCAN_ENTITY_NULL, -+ " : entity is NULL, dsq->id = %llu\n", dsq->id); -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ dsq->is_timeout = true; -+ dsq_id = dsq_id_to_internal(dsq); -+ hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n", -+ dsq_id, entity->task->comm, jiffies, entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1); -+ raw_spin_unlock(&dsq->lock); -+ -+ return true; -+} -+ -+void scan_timeout(struct rq *rq) -+{ -+ int i; -+ int cpu = cpu_of(rq); -+ struct hmbird_dispatch_q *dsq; -+ static u64 last_scan_at; -+ static DEFINE_PER_CPU(u64, pcp_last_scan_at); -+ -+ if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu))) -+ return; -+ per_cpu(pcp_last_scan_at, cpu) = jiffies; -+ -+ dsq = &per_cpu(pcp_ldsq, cpu); -+ scan_dsq_timeout(rq, dsq, pcp_dsq_deadline); -+ -+ if (time_before_eq(jiffies, (unsigned long)last_scan_at)) -+ return; -+ last_scan_at = jiffies; -+ -+ for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) { -+ dsq = &gdsqs[i]; -+ scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]); -+ } -+} -+ -+/*******************************Initialize***********************************/ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id); -+static void init_dsq_at_boot(void) -+{ -+ int i, cpu; -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i)); -+ } -+ for_each_possible_cpu(cpu) -+ init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i + cpu)); -+ -+ max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu; -+ spin_lock_init(&sinfo.lock); -+} -+ -+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgrp_name_to_idx(struct cgroup *cgrp) -+{ -+ int idx; -+ -+ if (!cgrp) -+ return -1; -+ -+ if (!strcmp(cgrp->kn->name, "display") -+ || !strcmp(cgrp->kn->name, "multimedia")) -+ idx = 5; /* 8ms */ -+ else if (!strcmp(cgrp->kn->name, "top-app") -+ || !strcmp(cgrp->kn->name, "ss-top")) -+ idx = 6; /* 16ms */ -+ else if (!strcmp(cgrp->kn->name, "ssfg") -+ || !strcmp(cgrp->kn->name, "foreground")) -+ idx = 7; /* 32ms */ -+ else if (!strcmp(cgrp->kn->name, "bg") -+ || !strcmp(cgrp->kn->name, "log") -+ || !strcmp(cgrp->kn->name, "dex2oat") -+ || !strcmp(cgrp->kn->name, "background")) -+ idx = 9; /* 128ms */ -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */ -+ -+ return idx; -+} -+ -+static void init_root_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX); -+} -+ -+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) { -+ pr_err("%s idx err!\n", __func__); -+ return; -+ } -+ -+ if (cgroup_ids_table[cgrp->kn->id] == -1) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp)); -+} -+ -+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ struct cgroup *l1cgrp; -+ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ l1cgrp = cgroup_ancestor_l1(cgrp); -+ if (l1cgrp) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp)); -+ cgroup_put(l1cgrp); -+} -+ -+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg) -+{ -+ switch (cgrp->level) { -+ case 0: -+ init_root_tg(cgrp, tg); -+ break; -+ case 1: -+ init_level1_tg(cgrp, tg); -+ break; -+ default: -+ init_child_tg(cgrp, tg); -+ break; -+ } -+} -+ -+/**************************************************************************/ -+ -+struct hmbird_task_iter { -+ struct hmbird_entity cursor; -+ struct task_struct *locked; -+ struct rq *rq; -+ struct rq_flags rf; -+}; -+ -+/** -+ * hmbird_task_iter_init - Initialize a task iterator -+ * @iter: iterator to init -+ * -+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized, -+ * @iter must eventually be exited with hmbird_task_iter_exit(). -+ * -+ * hmbird_tasks_lock may be released between this and the first next() call or -+ * between any two next() calls. If hmbird_tasks_lock is released between two -+ * next() calls, the caller is responsible for ensuring that the task being -+ * iterated remains accessible either through RCU read lock or obtaining a -+ * reference count. -+ * -+ * All tasks which existed when the iteration started are guaranteed to be -+ * visited as long as they still exist. -+ */ -+static void hmbird_task_iter_init(struct hmbird_task_iter *iter) -+{ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR }; -+ list_add(&iter->cursor.tasks_node, &hmbird_tasks); -+ iter->locked = NULL; -+} -+ -+/** -+ * hmbird_task_iter_exit - Exit a task iterator -+ * @iter: iterator to exit -+ * -+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held. -+ * If the iterator holds a task's rq lock, that rq lock is released. See -+ * hmbird_task_iter_init() for details. -+ */ -+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ if (list_empty(cursor)) -+ return; -+ -+ list_del_init(cursor); -+} -+ -+/** -+ * hmbird_task_iter_next - Next task -+ * @iter: iterator to walk -+ * -+ * Visit the next task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ struct hmbird_entity *pos; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ list_for_each_entry(pos, cursor, tasks_node) { -+ if (&pos->tasks_node == &hmbird_tasks) -+ return NULL; -+ if (!(pos->flags & HMBIRD_TASK_CURSOR)) { -+ list_move(cursor, &pos->tasks_node); -+ return pos->task; -+ } -+ } -+ -+ /* can't happen, should always terminate at hmbird_tasks above */ -+ hmbird_deferred_err(ITER_RET_NULL, " : unreachable path in scx_task_iter_next\n"); -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered - Next non-idle task -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ while ((p = hmbird_task_iter_next(iter))) { -+ if (!is_idle_task(p)) -+ return p; -+ } -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init() -+ * for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ p = hmbird_task_iter_next_filtered(iter); -+ if (!p) -+ return NULL; -+ -+ iter->rq = task_rq_lock(p, &iter->rf); -+ iter->locked = p; -+ return p; -+} -+ -+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void) -+{ -+ return atomic_read(&hmbird_ops_enable_state_var); -+} -+ -+static enum hmbird_ops_enable_state -+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to) -+{ -+ return atomic_xchg(&hmbird_ops_enable_state_var, to); -+} -+ -+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to, -+ enum hmbird_ops_enable_state from) -+{ -+ int from_v = from; -+ -+ return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to); -+} -+ -+static bool hmbird_ops_disabling(void) -+{ -+ return false; -+} -+ -+/** -+ * wait_ops_state - Busy-wait the specified ops state to end -+ * @p: target task -+ * @opss: state to wait the end of -+ * -+ * Busy-wait for @p to transition out of @opss. This can only be used when the -+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also -+ * has load_acquire semantics to ensure that the caller can see the updates made -+ * in the enqueueing and dispatching paths. -+ */ -+static void wait_ops_state(struct task_struct *p, u64 opss) -+{ -+ do { -+ cpu_relax(); -+ } while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss); -+} -+ -+ -+static void update_curr_hmbird(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ u64 now = rq_clock_task(rq); -+ u64 delta_exec; -+ -+ if (time_before_eq64(now, curr->se.exec_start)) -+ return; -+ -+ delta_exec = now - curr->se.exec_start; -+ curr->se.exec_start = now; -+ update_runningtime(rq, curr, delta_exec); -+ curr->se.sum_exec_runtime += delta_exec; -+ account_group_exec_runtime(curr, delta_exec); -+ cgroup_account_cputime(curr, delta_exec); -+ -+ if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF) -+ get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec); -+ -+ trace_sched_stat_runtime(curr, delta_exec, 0); -+} -+ -+static bool hmbird_dsq_priq_less(struct rb_node *node_a, -+ const struct rb_node *node_b) -+{ -+ const struct hmbird_entity *a = -+ container_of(node_a, struct hmbird_entity, dsq_node.priq); -+ const struct hmbird_entity *b = -+ container_of(node_b, struct hmbird_entity, dsq_node.priq); -+ -+ return time_before64(a->dsq_vtime, b->dsq_vtime); -+} -+ -+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ bool is_local = dsq->id == HMBIRD_DSQ_LOCAL; -+ unsigned long flags; -+ -+ hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq || -+ !list_empty(&get_hmbird_ts(p)->dsq_node.fifo), -+ "task = %s, dsq->id = %llu\n", p->comm, dsq->id); -+ hmbird_cond_deferred_err(ENQ_EXIST2, -+ (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq), -+ "task = %s\n", p->comm); -+ -+ if (!is_local) { -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ); -+ hmbird_ops_error(": %s\n", -+ "attempting to dispatch to a destroyed dsq"); -+ /* fall back to the global dsq */ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ dsq = &hmbird_dsq_global; -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ } -+ } -+ -+ if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) { -+ get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ; -+ rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq, -+ hmbird_dsq_priq_less); -+ } else { -+ if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT)) -+ list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ else -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ } -+ dsq->nr++; -+ get_hmbird_ts(p)->dsq = dsq; -+ -+ /* -+ * We're transitioning out of QUEUEING or DISPATCHING. store_release to -+ * match waiters' load_acquire. -+ */ -+ if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS) -+ atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE); -+ -+ if (is_local) { -+ struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq); -+ struct rq *rq = hmbird->rq; -+ bool preempt = false; -+ -+ if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr && -+ rq->curr->sched_class == &hmbird_sched_class) { -+ get_hmbird_ts(rq->curr)->slice = 0; -+ preempt = true; -+ } -+ -+ if (preempt || sched_class_above(&hmbird_sched_class, -+ rq->curr->sched_class)) -+ resched_curr(rq); -+ } else { -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ } -+} -+ -+static void task_unlink_from_dsq(struct task_struct *p, -+ struct hmbird_dispatch_q *dsq) -+{ -+ if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) { -+ rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ; -+ } else { -+ list_del_init(&get_hmbird_ts(p)->dsq_node.fifo); -+ } -+} -+ -+static bool task_linked_on_dsq(struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+} -+ -+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq; -+ bool is_local = dsq == &hmbird_rq->local_dsq; -+ -+ if (!dsq) { -+ hmbird_cond_deferred_err(TASK_LINKED1, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ /* -+ * When dispatching directly from the BPF scheduler to a local -+ * DSQ, the task isn't associated with any DSQ but -+ * @get_hmbird_ts(p)->holding_cpu may be set under the protection of -+ * %HMBIRD_OPSS_DISPATCHING. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu >= 0) -+ get_hmbird_ts(p)->holding_cpu = -1; -+ return; -+ } -+ -+ if (!is_local) -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ /* -+ * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node -+ * can't change underneath us. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu < 0) { -+ /* @p must still be on @dsq, dequeue */ -+ hmbird_cond_deferred_err(TASK_UNLINKED, -+ !task_linked_on_dsq(p), "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ } else { -+ /* -+ * We're racing against dispatch_to_local_dsq() which already -+ * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the -+ * holding_cpu which tells dispatch_to_local_dsq() that it lost -+ * the race. -+ */ -+ hmbird_cond_deferred_err(TASK_LINKED2, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ get_hmbird_ts(p)->holding_cpu = -1; -+ } -+ get_hmbird_ts(p)->dsq = NULL; -+ -+ if (!is_local) -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+ -+static bool test_rq_online(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ return rq->online; -+#else -+ return true; -+#endif -+} -+ -+static void refill_task_slice(struct task_struct *p) -+{ -+ if (is_isolate_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO; -+ else if (is_pipeline_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2; -+ else -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+} -+ -+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -+ int sticky_cpu) -+{ -+ struct hmbird_dispatch_q *d; -+ s32 cpu = -1; -+ -+ hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), -+ "task = %s\n", p->comm); -+ -+ if (is_pcp_rt(p)) { -+ /* Enqueue percpu rt task to local directly. */ -+ /* Or cause a bug when disable dispatch. */ -+ if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ } -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (cpu >= 0) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ -+ if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ /* rq migration */ -+ if (sticky_cpu == cpu_of(rq)) -+ goto local_norefill; -+ /* -+ * If !rq->online, we already told the BPF scheduler that the CPU is -+ * offline. We're just trying to on/offline the CPU. Don't bother the -+ * BPF scheduler. -+ */ -+ if (unlikely(!test_rq_online(rq))) -+ goto local; -+ -+ /* see %HMBIRD_OPS_ENQ_LAST */ -+ if (enq_flags & HMBIRD_ENQ_LAST) -+ goto local; -+ -+ if (enq_flags & HMBIRD_ENQ_LOCAL) -+ goto local; -+ else -+ goto global; -+local: -+ /* -+ * For task-ordering, slice refill must be treated as implying the end -+ * of the current slice. Otherwise, the longer @p stays on the CPU, the -+ * higher priority it becomes from hmbird_prio_less()'s POV. -+ */ -+ refill_task_slice(p); -+local_norefill: -+ dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags); -+ slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq)); -+ return; -+ -+global: -+ d = find_dsq_from_task(p); -+ if (d) { -+ refill_task_slice(p); -+ dispatch_enqueue(d, p, enq_flags); -+ return; -+ } -+ slim_stats_record(GLOBAL_STAT, 0, 0, 0); -+ refill_task_slice(p); -+ dispatch_enqueue(&hmbird_dsq_global, p, enq_flags); -+} -+ -+static bool watchdog_task_watched(const struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->watchdog_node); -+} -+ -+static void watchdog_watch_task(struct rq *rq, struct task_struct *p) -+{ -+ lockdep_assert_rq_held(rq); -+ if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ get_hmbird_ts(p)->runnable_at = jiffies; -+ clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list); -+} -+ -+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) -+{ -+ list_del_init(&get_hmbird_ts(p)->watchdog_node); -+ if (reset_timeout) -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags) -+{ -+ int sticky_cpu = get_hmbird_ts(p)->sticky_cpu; -+ -+ enq_flags |= get_hmbird_rq(rq)->extra_enq_flags; -+ -+ if (sticky_cpu >= 0) -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ -+ /* -+ * Restoring a running task will be immediately followed by -+ * set_next_task_hmbird() which expects the task to not be on the BPF -+ * scheduler as tasks can only start running through local DSQs. Force -+ * direct-dispatch into the local DSQ by setting the sticky_cpu. -+ */ -+ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -+ sticky_cpu = cpu_of(rq); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_UNWATCHED, -+ !watchdog_task_watched(p), "task = %s\n", p->comm); -+ return; -+ } -+ -+ watchdog_watch_task(rq, p); -+ set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ get_hmbird_rq(rq)->nr_running++; -+ add_nr_running(rq, 1); -+ -+ do_enqueue_task(rq, p, enq_flags, sticky_cpu); -+} -+ -+static void ops_dequeue(struct task_struct *p, u64 deq_flags) -+{ -+ u64 opss; -+ -+ watchdog_unwatch_task(p, false); -+ -+ /* acquire ensures that we see the preceding updates on QUEUED */ -+ opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state); -+ -+ switch (opss & HMBIRD_OPSS_STATE_MASK) { -+ case HMBIRD_OPSS_NONE: -+ break; -+ case HMBIRD_OPSS_QUEUEING: -+ /* -+ * QUEUEING is started and finished while holding @p's rq lock. -+ * As we're holding the rq lock now, we shouldn't see QUEUEING. -+ */ -+ hmbird_deferred_err(DEQ_DEQING, " : unreachable path in %s\n", __func__); -+ break; -+ case HMBIRD_OPSS_QUEUED: -+ if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss, -+ HMBIRD_OPSS_NONE)) -+ break; -+ fallthrough; -+ case HMBIRD_OPSS_DISPATCHING: -+ /* -+ * If @p is being dispatched from the BPF scheduler to a DSQ, -+ * wait for the transfer to complete so that @p doesn't get -+ * added to its DSQ after dequeueing is complete. -+ * -+ * As we're waiting on DISPATCHING with the rq locked, the -+ * dispatching side shouldn't try to lock the rq while -+ * DISPATCHING is set. See dispatch_to_local_dsq(). -+ * -+ * DISPATCHING shouldn't have qseq set and control can reach -+ * here with NONE @opss from the above QUEUED case block. -+ * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss. -+ */ -+ wait_ops_state(p, HMBIRD_OPSS_DISPATCHING); -+ hmbird_cond_deferred_err(HMBIRD_OPN, -+ atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE, -+ "task = %s\n", p->comm); -+ break; -+ } -+} -+ -+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ -+ if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p), -+ "task = %s\n", p->comm); -+ return; -+ } -+ -+ ops_dequeue(p, deq_flags); -+ -+ if (slim_walt_ctrl) { -+ if (task_current(rq, p)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ if (deq_flags & HMBIRD_DEQ_SLEEP) -+ set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else -+ clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), -+ (unsigned long *)&get_hmbird_ts(p)->flags); -+ -+ clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm); -+ hmbird_rq->nr_running--; -+ sub_nr_running(rq, 1); -+ dispatch_dequeue(hmbird_rq, p); -+} -+ -+static void yield_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ get_hmbird_ts(p)->slice = 0; -+} -+ -+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to) -+{ -+ return false; -+} -+ -+#ifdef CONFIG_SMP -+/** -+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -+ * @rq: rq to move the task into, currently locked -+ * @p: task to move -+ * @enq_flags: %HMBIRD_ENQ_* -+ * -+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller -+ * must: -+ * -+ * 1. Start with exclusive access to @p either through its DSQ lock or -+ * %HMBIRD_OPSS_DISPATCHING flag. -+ * -+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id(). -+ * -+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't -+ * deadlock with dequeue. -+ * -+ * 4. Lock @rq and the task_rq from #3. -+ * -+ * 5. Call this function. -+ * -+ * Returns %true if @p was successfully moved. %false after racing dequeue and -+ * losing. -+ */ -+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ struct rq *task_rq; -+ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * If dequeue got to @p while we were trying to lock both rq's, it'd -+ * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have -+ * updated it to different values afterwards, as this operation can't be -+ * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become -+ * raw_smp_processor_id() again before we're done. Thus, we can tell -+ * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is -+ * still raw_smp_processor_id(). -+ * -+ * See dispatch_dequeue() for the counterpart. -+ */ -+ if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id())) -+ return false; -+ -+ /* @p->rq couldn't have changed if we're still the holding cpu */ -+ task_rq = task_rq(p); -+ lockdep_assert_rq_held(task_rq); -+ deactivate_task(task_rq, p, 0); -+ set_task_cpu(p, cpu_of(rq)); -+ get_hmbird_ts(p)->sticky_cpu = cpu_of(rq); -+ -+ /* -+ * We want to pass hmbird-specific enq_flags but activate_task() will -+ * truncate the upper 32 bit. As we own @rq, we can pass them through -+ * @get_hmbird_rq(rq)->extra_enq_flags instead. -+ */ -+ hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags, -+ "task = %s\n", p->comm); -+ get_hmbird_rq(rq)->extra_enq_flags = enq_flags; -+ activate_task(rq, p, 0); -+ get_hmbird_rq(rq)->extra_enq_flags = 0; -+ -+ return true; -+} -+ -+#endif /* CONFIG_SMP */ -+ -+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu) -+{ -+ int fitable = 1; -+ -+ return fitable; -+} -+ -+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq, -+ struct hmbird_dispatch_q *dsq) -+{ -+ bool dsq_misfit; -+ int cpu = cpu_of(rq); -+ u64 task_util = 0; -+ struct cluster_ctx ctx; -+ int dsq_int = dsq_id_to_internal(dsq); -+ -+ if (!cpumask_test_cpu(cpu, iso_masks.little)) -+ return false; -+ -+ gen_cluster_ctx(&ctx, BIG); -+ dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 && -+ dsq_int <= SCHED_PROP_DEADLINE_LEVEL4); -+#ifdef CLUSTER_SEPARATE -+ /* In rescue mode, little will consume big cluster's dsq.*/ -+ dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper); -+#endif -+ if (!dsq_misfit) -+ return false; -+ -+ if (p) { -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &task_util); -+ else -+ task_util = get_hmbird_ts(p)->demand_scaled; -+ } -+ -+ if (task_util <= misfit_ds) -+ return false; -+ -+ hmbird_info_trace(":task %s can't run on cpu%d, util = %llu\n", -+ p->comm, cpu, task_util); -+ return true; -+} -+ -+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq) -+{ -+ if (!task_fits_cpu_hmbird(p, cpu_of(rq))) -+ return false; -+ -+ if (check_misfit_task_on_little(p, rq, dsq)) -+ return false; -+ -+ return likely(test_rq_online(rq)); -+} -+ -+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return; -+ -+ if (add) -+ skipn[idx]++; -+ else -+ skipn[idx] = 0; -+} -+ -+static bool skip_too_much(struct hmbird_dispatch_q *dsq) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return false; -+ -+ if (skip_num[idx] > 3) { -+ skip_num[idx] = 0; -+ return true; -+ } -+ -+ return false; -+} -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rb_node *rb_node; -+ struct rq *task_rq; -+ unsigned long flags; -+ bool moved = false; -+ struct task_struct *may_fit = NULL; -+ int skip = 0; -+ -+retry: -+ if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -+ return false; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) { -+ set_skip_num(dsq, skip_num, (bool)may_fit); -+ goto this_rq; -+ } -+ if (skip_too_much(dsq)) -+ goto remote_rq; -+ if (!may_fit) -+ may_fit = p; -+ if (++skip <= 3) -+ continue; -+ /* -+ * If the recent 3 tasks not fit, use the first one. -+ * and clear the skip, because the first one is consumed. -+ */ -+ set_skip_num(dsq, skip_num, false); -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ /* No more task, use the first may fit task.*/ -+ if (may_fit) { -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ -+ for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) -+ goto this_rq; -+ goto remote_rq; -+ } -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ return false; -+ -+this_rq: -+ /* @dsq is locked and @p is on this rq */ -+ hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo); -+ dsq->nr--; -+ hmbird_rq->local_dsq.nr++; -+ get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ -+remote_rq: -+#ifdef CONFIG_SMP -+ if (cpu_same_cluster_stat(p, rq, task_rq)) -+ slim_stats_record(MOVE_RQ_CNT, 0, 0, 0); -+ else -+ slim_stats_record(MOVE_RQ_CNT, 1, 0, 0); -+ /* -+ * @dsq is locked and @p is on a remote rq. @p is currently protected by -+ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -+ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -+ * rq lock or fail, do a little dancing from our side. See -+ * move_task_to_local_dsq(). -+ */ -+ hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id(); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ rq_unpin_lock(rq, rf); -+ double_lock_balance(rq, task_rq); -+ rq_repin_lock(rq, rf); -+ -+ moved = move_task_to_local_dsq(rq, p, 0); -+ -+ double_unlock_balance(rq, task_rq); -+#endif /* CONFIG_SMP */ -+ if (likely(moved)) { -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ } -+ may_fit = NULL; -+ goto retry; -+} -+ -+ -+static int balance_one(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf, bool local) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class; -+ -+ if (!hmbird_rq) -+ return 1; -+ -+ lockdep_assert_rq_held(rq); -+ -+ if (static_branch_unlikely(&hmbird_ops_cpu_preempt) && -+ unlikely(get_hmbird_rq(rq)->cpu_released)) { -+ /* -+ * If the previous sched_class for the current CPU was not HMBIRD, -+ * notify the BPF scheduler that it again has control of the -+ * core. This callback complements ->cpu_release(), which is -+ * emitted in hmbird_notify_pick_next_task(). -+ */ -+ get_hmbird_rq(rq)->cpu_released = false; -+ } -+ -+ if (prev_on_hmbird) -+ update_curr_hmbird(rq); -+ /* if there already are tasks to run, nothing to do */ -+ if (hmbird_rq->local_dsq.nr) -+ return 1; -+ -+ if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) { -+ slim_stats_record(GLOBAL_STAT, 1, 0, 0); -+ return 1; -+ } -+ -+ if (consume_dispatch_global(rq, rf)) -+ return 1; -+ -+ return 0; -+} -+ -+static int balance_hmbird(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf) -+{ -+ return balance_one(rq, prev, rf, true); -+} -+ -+/* -+ * output task util to systrace, only for debug mode. -+ * we can not output too many logs to systrace buffer even in debug mode -+ * only output debug-info while it exceed misfit_ds. -+ */ -+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p) -+{ -+ static DEFINE_PER_CPU(int, is_last_exceed); -+ int cpu = cpu_of(rq); -+ u64 util = 0; -+ -+ if (likely(!debug_enabled())) -+ return; -+ -+ if (!p) -+ return; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ util = uclamp_rq_util_with(rq, util, p); -+ -+ if (util >= misfit_ds) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util); -+ per_cpu(is_last_exceed, cpu) = true; -+ } else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0); -+ per_cpu(is_last_exceed, cpu) = false; -+ } else { -+ } -+} -+ -+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first) -+{ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ /* -+ * Core-sched might decide to execute @p before it is -+ * dispatched. Call ops_dequeue() to notify the BPF scheduler. -+ */ -+ ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC); -+ dispatch_dequeue(get_hmbird_rq(rq), p); -+ } -+ -+ p->se.exec_start = rq_clock_task(rq); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK); -+ } -+ -+ watchdog_unwatch_task(p, true); -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx); -+ systrace_output_cpu_ds(rq, p); -+ /* -+ * @p is getting newly scheduled or got kicked after someone updated its -+ * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird(). -+ */ -+ if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) != -+ (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) { -+ if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) -+ get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK; -+ else -+ get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK; -+ -+ sched_update_tick_dependency(rq); -+ } -+ -+ p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime; -+} -+ -+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p) -+{ -+ update_curr_hmbird(rq); -+ -+ update_dispatch_dsq_info(rq, p); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ watchdog_watch_task(rq, p); -+ -+ if (is_pipeline_task(p)) { -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1); -+ return; -+ } -+ -+ /* -+ * If we're in the pick_next_task path, balance_hmbird() should -+ * have already populated the local DSQ if there are any other -+ * available tasks. If empty, tell ops.enqueue() that @p is the -+ * only one available for this cpu. ops.enqueue() should put it -+ * on the local DSQ so that the subsequent pick_next_task_hmbird() -+ * can find the task unless it wants to trigger a separate -+ * follow-up scheduling event. -+ */ -+ if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1); -+ else -+ do_enqueue_task(rq, p, 0, -1); -+ } -+} -+ -+static struct task_struct *first_local_task(struct rq *rq) -+{ -+ struct rb_node *rb_node; -+ struct hmbird_entity *entity; -+ -+ if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) { -+ entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo, -+ struct hmbird_entity, dsq_node.fifo); -+ return entity->task; -+ } -+ -+ rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq); -+ if (rb_node) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ return entity->task; -+ } -+ return NULL; -+} -+ -+static struct task_struct *pick_next_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p; -+ -+ p = first_local_task(rq); -+ if (!p) -+ return NULL; -+ -+ if (unlikely(!get_hmbird_ts(p)->slice)) { -+ if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice) -+ hmbird_warned_zero_slice = true; -+ -+ refill_task_slice(p); -+ } -+ -+ set_next_task_hmbird(rq, p, true); -+ -+ return p; -+} -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task, -+ const struct sched_class *active) -+{ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * The callback is conceptually meant to convey that the CPU is no -+ * longer under the control of HMBIRD. Therefore, don't invoke the -+ * callback if the CPU is staying on HMBIRD, or going idle (in which -+ * case the HMBIRD scheduler has actively decided not to schedule any -+ * tasks on the CPU). -+ */ -+ if (likely(active >= &hmbird_sched_class)) -+ return; -+ -+ /* -+ * At this point we know that HMBIRD was preempted by a higher priority -+ * sched_class, so invoke the ->cpu_release() callback if we have not -+ * done so already. We only send the callback once between HMBIRD being -+ * preempted, and it regaining control of the CPU. -+ * -+ * ->cpu_release() complements ->cpu_acquire(), which is emitted the -+ * next time that balance_hmbird() is invoked. -+ */ -+ if (!get_hmbird_rq(rq)->cpu_released) -+ get_hmbird_rq(rq)->cpu_released = true; -+} -+ -+#ifdef CONFIG_SMP -+ -+static bool test_and_clear_cpu_idle(int cpu) -+{ -+ if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -+ if (cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ return true; -+ } else { -+ return false; -+ } -+} -+ -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) -+{ -+ int cpu; -+ -+ do { -+ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -+ if (cpu >= nr_cpu_ids) -+ return -EBUSY; -+ } while (!test_and_clear_cpu_idle(cpu)); -+ -+ return cpu; -+} -+ -+ -+static bool prev_cpu_misfit(int prev) -+{ -+ if (!is_partial_enabled() && is_partial_cpu(prev)) -+ return true; -+ -+ return false; -+} -+ -+static int heavy_rt_placement(struct task_struct *p, int prev) -+{ -+ struct cpumask tmp = {.bits = {0}}; -+ int cpu; -+ u64 util = 0; -+ -+ if (!rt_prio(p->prio)) -+ return -EFAULT; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ -+ if (util < misfit_ds) -+ return -EFAULT; -+ -+ if (is_partial_enabled()) -+ cpumask_or(&tmp, iso_masks.big, iso_masks.partial); -+ else -+ cpumask_copy(&tmp, iso_masks.big); -+ -+ cpu = hmbird_pick_idle_cpu(&tmp); -+ if (cpu >= 0) -+ return cpu; -+ -+ if (cpumask_test_cpu(prev, iso_masks.big) || -+ (is_partial_enabled() && is_partial_cpu(prev))) -+ return prev; -+ -+ return cpumask_first(&tmp); -+} -+ -+static int spec_task_before_pick_idle(struct task_struct *p, int prev) -+{ -+ int cpu; -+ -+ cpu = heavy_rt_placement(p, prev); -+ if (cpu >= 0) -+ return cpu; -+ return -EFAULT; -+} -+ -+static int cpumask_distribute_next(struct cpumask *mask, int *prev) -+{ -+ int p = *prev, n; -+ -+ n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1); -+ if (n < nr_cpu_ids) -+ WRITE_ONCE(*prev, n); -+ return n; -+} -+ -+static int repick_fallback_cpu(void) -+{ -+ /* -+ * partial cpu follow big cluster's Scheduling policy, -+ * simply return first bit cpu. -+ */ -+ return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev); -+} -+ -+/* -+ * Must return a valid cpu num, as this task's cpu. -+ */ -+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu, -+ struct cpumask *mask, int *prev_mask) -+{ -+ int cpu; -+ -+ cpu = hmbird_pick_idle_cpu(mask); -+ if (cpu >= 0) -+ return cpu; -+ return cpumask_distribute_next(mask, prev_mask); -+} -+ -+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type) -+{ -+ int idx; -+ struct cluster_ctx ctx; -+ -+ idx = find_idx_from_task(p); -+ if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS) -+ return false; -+ -+ gen_cluster_ctx(&ctx, type); -+ if (idx >= ctx.lower && idx < ctx.upper) -+ return true; -+ return false; -+} -+ -+static bool is_valid_cpu(int cpu) -+{ -+ return (cpu >= 0) && (cpu < nr_cpu_ids); -+} -+ -+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) -+{ -+ s32 cpu = -1; -+ struct cpumask mask = {.bits = {0}}; -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (is_valid_cpu(cpu)) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ partial_dynamic_ctrl(); -+ -+ if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ iso_masks.big, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ if (p->nr_cpus_allowed == 1) { -+ cpu = cpumask_any(p->cpus_ptr); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* For non-period global dsq, not contain pcp task. */ -+ if (task_only_blongs_to_cluster(p, LITTLE)) { -+ cpumask_copy(&mask, iso_masks.little); -+ if (unlikely(l_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &little_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ if (task_only_blongs_to_cluster(p, BIG)) { -+ cpumask_copy(&mask, iso_masks.big); -+ if (unlikely(b_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ cpu = spec_task_before_pick_idle(p, prev_cpu); -+ if (is_valid_cpu(cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* if the previous CPU is idle, dispatch directly to it */ -+ if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+ } -+ -+ cpu = hmbird_pick_idle_cpu(cpu_possible_mask); -+ if (is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ if (prev_cpu_misfit(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ cpu = repick_fallback_cpu(); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+} -+ -+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags) -+{ -+ return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags); -+} -+ -+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx) -+{ -+ set_cpus_allowed_common(p, ctx); -+} -+ -+static void reset_idle_masks(void) -+{ -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little); -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big); -+ if (is_partial_enabled()) -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial); -+ hmbird_has_idle_cpus = true; -+} -+ -+void __hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ int cpu = cpu_of(rq); -+ struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -+ -+ if (skip_update_idle()) -+ return; -+ -+ if (idle) { -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ if (!hmbird_has_idle_cpus) -+ hmbird_has_idle_cpus = true; -+ -+ /* -+ * idle_masks.smt handling is racy but that's fine as it's only -+ * for optimization and self-correcting. -+ */ -+ for_each_cpu(cpu, sib_mask) { -+ if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -+ return; -+ } -+ cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -+ } else { -+ cpumask_clear_cpu(cpu, idle_masks.cpu); -+ if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ -+ cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -+ } -+} -+ -+#else /* !CONFIG_SMP */ -+ -+static bool test_and_clear_cpu_idle(int cpu) { return false; } -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } -+static void reset_idle_masks(void) {} -+ -+#endif /* CONFIG_SMP */ -+ -+static bool check_rq_for_timeouts(struct rq *rq) -+{ -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rq_flags rf; -+ -+ rq_lock_irqsave(rq, &rf); -+ list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) { -+ unsigned long last_runnable; -+ -+ p = entity->task; -+ last_runnable = get_hmbird_ts(p)->runnable_at; -+ -+ if (unlikely(time_after(jiffies, -+ last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -+ -+ rq_unlock_irqrestore(rq, &rf); -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb", -+ p->comm, p->pid, -+ dur_ms / 1000, dur_ms % 1000, -+ get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0, -+ cpumask_pr_args(&p->cpus_mask)); -+ return 1; -+ } -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ return 0; -+} -+ -+static void hmbird_watchdog_workfn(struct work_struct *work) -+{ -+ int cpu; -+ bool timeout; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ -+ for_each_online_cpu(cpu) { -+ timeout = check_rq_for_timeouts(cpu_rq(cpu)); -+ if (unlikely(timeout)) -+ break; -+ -+ cond_resched(); -+ } -+ if (!timeout) -+ queue_delayed_work(system_unbound_wq, to_delayed_work(work), -+ hmbird_watchdog_timeout / 2); -+} -+ -+static void set_pcp_round(struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) { -+ per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round); -+ per_cpu(pcp_info, cpu).pcp_round = true; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+} -+ -+ -+/* -+ * Just for debug: output hmbird on/off state per 10s. -+ */ -+#define OUTPUT_INTVAL (msecs_to_jiffies(10 * 1000)) -+static void inform_hmbird_onoff_from_systrace(void) -+{ -+ static unsigned long __read_mostly next_print; -+ -+ if (time_before(jiffies, READ_ONCE(next_print))) -+ return; -+ -+ WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL); -+ hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio); -+} -+ -+void hmbird_notify_sched_tick(void) -+{ -+ unsigned long last_check; -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ hmbird_scheduler_tick(); -+ -+ if (!hmbird_enabled()) -+ return; -+ -+ last_check = hmbird_watchdog_timestamp; -+ if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -+ -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "watchdog failed to check in for %u.%03us", -+ dur_ms / 1000, dur_ms % 1000); -+ } -+ scan_timeout(rq); -+} -+ -+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued) -+{ -+ update_curr_hmbird(rq); -+ -+ set_pcp_round(rq); -+ -+ if (slim_walt_ctrl) -+ hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE); -+ /* -+ * While disabling, always resched and refresh core-sched timestamp as -+ * we can't trust the slice management or ops.core_sched_before(). -+ */ -+ if (hmbird_ops_disabling()) -+ get_hmbird_ts(curr)->slice = 0; -+ -+ if (!get_hmbird_ts(curr)->slice) -+ resched_curr(rq); -+ -+ inform_hmbird_onoff_from_systrace(); -+} -+ -+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg) -+{ -+ hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ get_hmbird_ts(p)->disallow = false; -+ -+ hmbird_sched_init_task(p); -+ -+ set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ return 0; -+} -+ -+static void hmbird_ops_enable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void hmbird_ops_disable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void set_task_hmbird_weight(struct task_struct *p) -+{ -+ u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -+ -+ get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight); -+} -+ -+/** -+ * refresh_hmbird_weight - Refresh a task's hmbird weight -+ * @p: task to refresh hmbird weight for -+ * -+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to -+ * enable easy access from the BPF scheduler. To keep it synchronized with the -+ * current task priority, this function should be called when a new task is -+ * created, priority is changed for a task on hmbird, and a task is switched -+ * to hmbird from other classes. -+ */ -+static void refresh_hmbird_weight(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ set_task_hmbird_weight(p); -+} -+ -+int hmbird_pre_fork(struct task_struct *p) -+{ -+ int ret = 0; -+ -+ p->android_oem_data1[HMBIRD_TS_IDX] = -+ (u64)(kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL)); -+ if (!get_hmbird_ts(p)) { -+ ret = -1; -+ goto lock; -+ } -+ -+ get_hmbird_ts(p)->dsq = NULL; -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node); -+ get_hmbird_ts(p)->flags = 0; -+ get_hmbird_ts(p)->weight = 0; -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ get_hmbird_ts(p)->holding_cpu = -1; -+ get_hmbird_ts(p)->kf_mask = 0; -+ atomic64_set(&get_hmbird_ts(p)->ops_state, 0); -+ get_hmbird_ts(p)->runnable_at = INITIAL_JIFFIES; -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+ get_hmbird_ts(p)->task = p; -+ hmbird_set_sched_prop(p, 0); -+ -+ get_hmbird_ts(p)->critical_affinity_cpu = -1; -+ get_hmbird_ts(p)->sched_class = &hmbird_sched_class; -+ -+ /* -+ * BPF scheduler enable/disable paths want to be able to iterate and -+ * update all tasks which can become complex when racing forks. As -+ * enable/disable are very cold paths, let's use a percpu_rwsem to -+ * exclude forks. -+ */ -+lock: -+ percpu_down_read(&hmbird_fork_rwsem); -+ -+ return ret; -+} -+ -+int hmbird_fork(struct task_struct *p) -+{ -+ percpu_rwsem_assert_held(&hmbird_fork_rwsem); -+ -+ if (hmbird_enabled()) -+ return hmbird_ops_prepare_task(p, task_group(p)); -+ else -+ return 0; -+} -+ -+void hmbird_post_fork(struct task_struct *p) -+{ -+ if (hmbird_enabled()) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ /* -+ * Set the weight manually before calling ops.enable() so that -+ * the scheduler doesn't see a stale value if they inspect the -+ * task struct. We'll invoke ops.set_weight() afterwards, as it -+ * would be odd to receive a callback on the task before we -+ * tell the scheduler that it's been fully enabled. -+ */ -+ set_task_hmbird_weight(p); -+ hmbird_ops_enable_task(p); -+ refresh_hmbird_weight(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ -+ spin_lock_irq(&hmbird_tasks_lock); -+ list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_cancel_fork(struct task_struct *p) -+{ -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ if (hmbird_enabled()) -+ hmbird_ops_disable_task(p); -+ -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_free(struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+ list_del_init(&get_hmbird_ts(p)->tasks_node); -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+ -+ /* -+ * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED -> -+ * ENABLED transitions can't race us. Disable ops for @p. -+ */ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) || -+ test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ hmbird_ops_disable_task(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+} -+ -+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio) -+{ -+} -+ -+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type) -+{ -+ return (prop >> TOP_TASK_SHIFT) & (1 << type); -+} -+ -+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p) -+{ -+ uint32_t prop = get_top_task_prop(p); -+ -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE)) -+ return HMBIRD_TASK_PROP_PIPELINE; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) || -+ !task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) { -+ return HMBIRD_TASK_PROP_COMMON; -+ } -+ return HMBIRD_TASK_PROP_DEBUG_OR_LOG; -+} -+ -+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b) -+{ -+ int type_a = hmbird_get_task_type(a); -+ int type_b = hmbird_get_task_type(b); -+ -+ return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b]; -+} -+ -+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ enum cpu_type type; -+ int sp_dl; -+ struct task_struct *curr = NULL; -+ -+ switch (hmbird_preempt_policy) { -+ case HMBIRD_PREEMPT_POLICY_PRIO_BASED: -+ curr = rq->curr; -+ if (curr && hmbird_prio_higher(p, curr)) -+ goto preempt; -+ break; -+ default: -+ break; -+ } -+ if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) || -+ (is_critical_system_task(p) && !is_critical_system_task(rq->curr))) -+ goto preempt; -+ -+ sp_dl = find_idx_from_task(p); -+ if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled())) -+ return; -+ -+ if (rq->curr->prio > p->prio) -+ goto preempt; -+ -+ return; -+ -+preempt: -+ resched_curr(rq); -+} -+ -+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {} -+ -+int hmbird_check_setscheduler(struct task_struct *p, int policy) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ /* if disallow, reject transitioning into HMBIRD */ -+ if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) && -+ p->policy != policy && policy == SCHED_HMBIRD) -+ return -EACCES; -+ -+ return 0; -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool hmbird_can_stop_tick(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ if (hmbird_ops_disabling()) -+ return false; -+ -+ if (p->sched_class != &hmbird_sched_class) -+ return true; -+ -+ return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK; -+} -+#endif -+ -+int hmbird_tg_online(struct task_group *tg) -+{ -+ struct cgroup *cgrp; -+ -+ if (!tg) -+ return 0; -+ cgrp = tg->css.cgroup; -+ if (!cgrp || !(cgrp->kn)) -+ return 0; -+ update_cgroup_ids_table(cgrp->kn->id, -1); -+ return 0; -+} -+ -+/* -+ * Omitted operations: -+ * -+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -+ * task isn't tied to the CPU at that point. Preemption is implemented by -+ * resetting the victim task's slice to 0 and triggering reschedule on the -+ * target CPU. -+ * -+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -+ * -+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -+ * their current sched_class. Call them directly from sched core instead. -+ * -+ * - task_woken, switched_from: Unnecessary. -+ */ -+DEFINE_SCHED_CLASS(hmbird) = { -+ .enqueue_task = enqueue_task_hmbird, -+ .dequeue_task = dequeue_task_hmbird, -+ .yield_task = yield_task_hmbird, -+ .yield_to_task = yield_to_task_hmbird, -+ -+ .check_preempt_curr = check_preempt_curr_hmbird, -+ -+ .pick_next_task = pick_next_task_hmbird, -+ -+ .put_prev_task = put_prev_task_hmbird, -+ .set_next_task = set_next_task_hmbird, -+ -+#ifdef CONFIG_SMP -+ .balance = balance_hmbird, -+ .select_task_rq = select_task_rq_hmbird, -+ .set_cpus_allowed = set_cpus_allowed_hmbird, -+#endif -+ .task_tick = task_tick_hmbird, -+ -+ .switched_to = switched_to_hmbird, -+ .prio_changed = prio_changed_hmbird, -+ -+ .update_curr = update_curr_hmbird, -+ -+#ifdef CONFIG_UCLAMP_TASK -+ .uclamp_enabled = 0, -+#endif -+}; -+ -+/* -+ * Must with rq lock held. -+ */ -+bool task_is_hmbird(struct task_struct *p) -+{ -+ return p->sched_class == &hmbird_sched_class; -+} -+ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id) -+{ -+ memset(dsq, 0, sizeof(*dsq)); -+ -+ raw_spin_lock_init(&dsq->lock); -+ INIT_LIST_HEAD(&dsq->fifo); -+ dsq->id = dsq_id; -+} -+ -+static int hmbird_cgroup_init(void) -+{ -+ struct cgroup_subsys_state *css; -+ -+ css_for_each_descendant_pre(css, &root_task_group.css) { -+ struct task_group *tg = css_tg(css); -+ -+ cgrp_dsq_idx_init(css->cgroup, tg); -+ } -+ return 0; -+} -+ -+/* -+ * Used by sched_fork() and __setscheduler_prio() to pick the matching -+ * sched_class. dl/rt are already handled. -+ */ -+bool task_on_hmbird(struct task_struct *p) -+{ -+ return hmbird_enabled(); -+} -+ -+static void __setscheduler_prio(struct task_struct *p, int prio) -+{ -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) { -+ p->prio = prio; -+ return; -+ } else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (on_hmbird && rt_prio(prio)) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ -+ p->prio = prio; -+} -+ -+/* -+ * Heartbeat, avoid humbird keep running while APP already exit. -+ * Check whether APP send alive-signal periodly. -+ */ -+#define HEARTBEAT_TIMEOUT (msecs_to_jiffies(2500)) -+#define HEARTBEAT_CHECK_INTERVAL (msecs_to_jiffies(1000)) -+static struct timer_list hb_timer; -+static unsigned long next_hb; -+ -+void hb_timer_handler(struct timer_list *timer) -+{ -+ if (!heartbeat_enable) -+ goto refill; -+ -+ pr_err(": enter timer.\n"); -+ if (heartbeat) { -+ heartbeat = 0; -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ } -+ -+ /* can't detect heartbeat, disable ext. */ -+ if (time_after(jiffies, READ_ONCE(next_hb))) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT, -+ "can't detect heartbeat, disable ext\n"); -+ } -+refill: -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_start(void) -+{ -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_init(void) -+{ -+ timer_setup(&hb_timer, hb_timer_handler, 0); -+} -+ -+static void hb_timer_exit(void) -+{ -+ del_timer(&hb_timer); -+} -+ -+static bool check_and_disable_cpuhp(void) -+{ -+ struct rq *rq; -+ struct rq_flags rf; -+ int cpu; -+ -+ cpu_hotplug_disable(); -+ cpus_read_lock(); -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ rq_lock_irqsave(rq, &rf); -+ if (!rq->online) { -+ rq_unlock_irqrestore(rq, &rf); -+ goto err_offline; -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ cpus_read_unlock(); -+ return true; -+ -+err_offline: -+ cpus_read_unlock(); -+ cpu_hotplug_enable(); -+ return false; -+} -+ -+static void reenable_cpuhp(void) -+{ -+ cpu_hotplug_enable(); -+} -+ -+static void scheduler_switch_done(bool final_state) -+{ -+ /* set scx_enable again, switch may fail. */ -+ scx_enable = final_state; -+ /* reset heartbeat*/ -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ -+ if (final_state) -+ hb_timer_start(); -+ else -+ hb_timer_exit(); -+} -+ -+/** -+ * ss : curr switch state -+ * finish : success or fail -+ * enable : curr operation is diabling or enabling? -+ * fail_reason : string output when failed, empty string when success. -+ */ -+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason) -+{ -+ char *s1 = finish ? "finished" : "failed"; -+ char *s2 = enable ? "enabled" : "disabled"; -+ -+ hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss); -+ if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) { -+ sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type)); -+ hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n", -+ s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason); -+ } -+ curr_ss = ss; -+} -+ -+bool get_hmbird_ops_enabled(void) -+{ -+ return atomic_read(&__hmbird_ops_enabled); -+} -+ -+bool get_non_hmbird_task(void) -+{ -+ return atomic_read(&non_hmbird_task); -+} -+ -+static void hmbird_ops_disable_workfn(struct kthread_work *work) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int cpu; -+ -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, ""); -+ cancel_delayed_work_sync(&hmbird_watchdog_work); -+ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) { -+ case HMBIRD_OPS_DISABLED: -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled"); -+ scheduler_switch_done(false); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return; -+ case HMBIRD_OPS_PREPPING: -+ fallthrough; -+ case HMBIRD_OPS_DISABLING: -+ /* shouldn't happen but handle it like ENABLING if it does */ -+ WARN_ONCE(true, "hmbird: duplicate disabling instance?"); -+ fallthrough; -+ case HMBIRD_OPS_ENABLING: -+ case HMBIRD_OPS_ENABLED: -+ break; -+ } -+ -+ /* kick all CPUs to restore ticks */ -+ for_each_possible_cpu(cpu) -+ resched_cpu(cpu); -+ -+ /* avoid racing against fork and cgroup changes */ -+ cpus_read_lock(); -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, ""); -+ spin_lock_irq(&hmbird_tasks_lock); -+ atomic_set(&__hmbird_ops_enabled, false); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ bool alive = READ_ONCE(p->__state) != TASK_DEAD; -+ -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ get_hmbird_ts(p)->slice = min_t(u64, -+ get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL); -+ -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ if (alive) -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ -+ hmbird_ops_disable_task(p); -+ } -+ hmbird_task_iter_exit(&sti); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, ""); -+ -+ atomic_set(&non_hmbird_task, true); -+ /* no task is on hmbird, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt); -+ synchronize_rcu(); -+ -+ percpu_up_write(&hmbird_fork_rwsem); -+ cpus_read_unlock(); -+ -+ if (slim_walt_ctrl) -+ slim_walt_enable(false); -+ -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 1, 0, ""); -+ scheduler_switch_done(false); -+ reenable_cpuhp(); -+} -+ -+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn); -+ -+static void schedule_hmbird_ops_disable_work(void) -+{ -+ struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper); -+ -+ /* -+ * We may be called spuriously before the first bpf_hmbird_reg(). If -+ * hmbird_ops_helper isn't set up yet, there's nothing to do. -+ */ -+ if (helper) -+ kthread_queue_work(helper, &hmbird_ops_disable_work); -+} -+ -+static void hmbird_ops_disable(void) -+{ -+ schedule_hmbird_ops_disable_work(); -+} -+ -+static void hmbird_err_exit_workfn(struct work_struct *work) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ hmbird_ctrl(false); -+ -+ for_each_present_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy) -+ continue; -+ if (cpu != policy->cpu) -+ goto put; -+ down_write(&policy->rwsem); -+ WARN_ON(store_scaling_governor(policy, -+ saved_gov[cpu], strlen(saved_gov[cpu])) <= 0); -+ up_write(&policy->rwsem); -+ hmbird_info_trace(":restore origin gov : %s\n", saved_gov[cpu]); -+put: -+ cpufreq_cpu_put(policy); -+ } -+ memset((char *)saved_gov, 0, sizeof(saved_gov)); -+} -+ -+void hmbird_ops_exit(void) -+{ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); -+} -+ -+static struct kthread_worker *hmbird_create_rt_helper(const char *name) -+{ -+ struct kthread_worker *helper; -+ -+ helper = kthread_create_worker(0, name); -+ if (helper) -+ sched_set_fifo(helper->task); -+ return helper; -+} -+ -+static inline void set_audio_thread_sched_prop(struct task_struct *p) -+{ -+ struct cgroup_subsys_state *css; -+ -+ if (likely(p->prio >= MAX_RT_PRIO)) -+ return; -+ -+ rcu_read_lock(); -+ css = task_css(p, cpuset_cgrp_id); -+ if (!css) { -+ rcu_read_unlock(); -+ return; -+ } -+ rcu_read_unlock(); -+ -+ if (!strcmp(css->cgroup->kn->name, "audio-app")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+} -+ -+static int hmbird_ops_enable(void *unused) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int ret; -+ int tcnt = 0; -+ unsigned long long start = 0; -+ -+ if (!check_and_disable_cpuhp()) { -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline"); -+ return -EBUSY; -+ } -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, ""); -+ mutex_lock(&hmbird_ops_enable_mutex); -+ -+ if (!hmbird_ops_helper) { -+ WRITE_ONCE(hmbird_ops_helper, -+ hmbird_create_rt_helper("hmbird_ops_helper")); -+ if (!hmbird_ops_helper) { -+ ret = -ENOMEM; -+ goto err_unlock; -+ } -+ } -+ -+ if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) { -+ ret = -EBUSY; -+ goto err_unlock; -+ } -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) != -+ HMBIRD_OPS_DISABLED); -+ -+ hmbird_warned_zero_slice = false; -+ -+ atomic64_set(&hmbird_nr_rejected, 0); -+ -+ /* -+ * Keep CPUs stable during enable so that the BPF scheduler can track -+ * online CPUs by watching ->on/offline_cpu() after ->init(). -+ */ -+ cpus_read_lock(); -+ -+ hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work, -+ hmbird_watchdog_timeout / 2); -+ -+ /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ */ -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ reset_idle_masks(); -+ -+ /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. -+ */ -+ ret = hmbird_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock; -+ -+ /* -+ * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem -+ * preventing new tasks from being added. No need to exclude tasks -+ * leaving as hmbird_free() can handle both prepped and enabled -+ * tasks. Prep all tasks first and then enable them with preemption -+ * disabled. -+ */ -+ spin_lock_irq(&hmbird_tasks_lock); -+ -+ atomic_set(&non_hmbird_task, false); -+ atomic_set(&__hmbird_ops_enabled, true); -+ -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered(&sti))) -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ hmbird_task_iter_exit(&sti); -+ -+ /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. -+ */ -+ preempt_disable(); -+ -+ /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ */ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) { -+ atomic_set(&non_hmbird_task, true); -+ atomic_set(&__hmbird_ops_enabled, false); -+ preempt_enable(); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ ret = -EBUSY; -+ goto err_disable_unlock; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED -+ * transitions here are synchronized against hmbird_free() through -+ * hmbird_tasks_lock. -+ */ -+ start = sched_clock(); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, ""); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ tcnt++; -+ if (READ_ONCE(p->__state) != TASK_DEAD) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ -+ set_audio_thread_sched_prop(p); -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ hmbird_ops_enable_task(p); -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ } else { -+ hmbird_ops_disable_task(p); -+ } -+ } -+ hmbird_task_iter_exit(&sti); -+ -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, ""); -+ preempt_enable(); -+ percpu_up_write(&hmbird_fork_rwsem); -+ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) { -+ ret = -EBUSY; -+ goto err_disable; -+ } -+ -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_ENABLED, 1, 1, ""); -+ scheduler_switch_done(true); -+ -+ return 0; -+ -+err_unlock: -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock"); -+ scheduler_switch_done(false); -+ return ret; -+ -+err_disable_unlock: -+ percpu_up_write(&hmbird_fork_rwsem); -+err_disable: -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ /* must be fully disabled before returning */ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable"); -+ scheduler_switch_done(false); -+ return ret; -+} -+ -+#ifdef CONFIG_SCHED_DEBUG -+static const char *hmbird_ops_enable_state_str[] = { -+ [HMBIRD_OPS_PREPPING] = "prepping", -+ [HMBIRD_OPS_ENABLING] = "enabling", -+ [HMBIRD_OPS_ENABLED] = "enabled", -+ [HMBIRD_OPS_DISABLING] = "disabling", -+ [HMBIRD_OPS_DISABLED] = "disabled", -+}; -+ -+static int hmbird_debug_show(struct seq_file *m, void *v) -+{ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled()); -+ seq_printf(m, "%-30s: %s\n", "enable_state", -+ hmbird_ops_enable_state_str[hmbird_ops_enable_state()]); -+ seq_printf(m, "%-30s: %llu\n", "nr_rejected", -+ atomic64_read(&hmbird_nr_rejected)); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return 0; -+} -+ -+static int hmbird_debug_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_debug_show, NULL); -+} -+ -+const struct file_operations sched_hmbird_fops = { -+ .open = hmbird_debug_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif -+ -+ -+static int bpf_hmbird_reg(void *kdata) -+{ -+ return hmbird_ops_enable(kdata); -+} -+ -+static int bpf_hmbird_unreg(void *kdata) -+{ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ return 0; -+} -+ -+void set_hmbird_module_loaded(int is_loaded) -+{ -+ atomic_set(&hmbird_module_loaded, is_loaded); -+} -+ -+/* -+ * MUST load hmbird module before enable hmbird scheduler -+ * load track & hmbird gover implemented in hmbird module -+ */ -+int hmbird_ctrl(bool enable) -+{ -+ if (!atomic_read(&hmbird_module_loaded)) { -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "ext module unloaded\n"); -+ return -EINVAL; -+ } -+ -+ if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED -+ || hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING -+ || hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already enabled(ing)\n"); -+ hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING || -+ hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already disabled(ing)\n"); -+ hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (enable) -+ return bpf_hmbird_reg(NULL); -+ else -+ return bpf_hmbird_unreg(NULL); -+} -+ -+void set_cpu_isomask(int cpu, cpumask_var_t *mask) -+{ -+ cpumask_clear_cpu(cpu, iso_masks.ex_free); -+ cpumask_clear_cpu(cpu, iso_masks.exclusive); -+ cpumask_clear_cpu(cpu, iso_masks.partial); -+ cpumask_clear_cpu(cpu, iso_masks.big); -+ cpumask_clear_cpu(cpu, iso_masks.little); -+ cpumask_set_cpu(cpu, *mask); -+} -+ -+void set_cpu_cluster(u64 cpu_cluster) -+{ -+ int cpu; -+ -+ for_each_present_cpu(cpu) { -+ u64 pos = 1 << cpu; -+ -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.ex_free)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.exclusive)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.partial)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.big)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) -+ set_cpu_isomask(cpu, &(iso_masks.little)); -+ } -+} -+ -+static void get_hmbird_snapshot(struct panic_snapshot_t *p) -+{ -+ struct hmbird_dispatch_q *dsq; -+ struct rq *rq; -+ int cpu, i; -+ -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ p->rq_nr[cpu] = rq->nr_running; -+ p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running; -+ } -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ struct hmbird_entity *entity; -+ -+ dsq = &gdsqs[i]; -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo)) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ p->runnable_at[i] = entity->runnable_at; -+ raw_spin_unlock(&dsq->lock); -+ -+ } -+ -+ p->snap_misc.hmbird_enabled = hmbird_enabled(); -+ p->snap_misc.curr_ss = curr_ss; -+ p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state(); -+ p->snap_misc.parctrl_high_ratio = parctrl_high_ratio; -+ p->snap_misc.parctrl_low_ratio = parctrl_low_ratio; -+ p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l; -+ p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l; -+ p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio; -+ p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio; -+ p->snap_misc.misfit_ds = misfit_ds; -+ p->snap_misc.partial_enable = partial_enable; -+ p->snap_misc.iso_free_rescue = iso_free_rescue; -+ p->snap_misc.isolate_ctrl = isolate_ctrl; -+ p->snap_misc.snap_jiffies = jiffies; -+ p->snap_misc.snap_time = local_clock(); -+} -+ -+// MTK minidump begin -+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3) -+{ -+ strscpy(m->desc_str, str, DESC_STR_LEN); -+ m->len = d1 * d2 * d3; -+ m->parse[0] = d1; -+ m->parse[1] = d2; -+ m->parse[2] = d3; -+} -+ -+static void init_desc_metas(struct md_info_t *m) -+{ -+ init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :", -+ 1, MAX_SWITCHS, SWITCH_ITEMS); -+ -+ init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1); -+ -+ init_desc_meta(&m->kern_dump.excep_rec_meta, -+ "excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS); -+ -+ init_desc_meta(&m->kern_dump.excep_idx_meta, -+ "excep idx :", 1, 1, MAX_EXCEP_ID); -+ -+ init_desc_meta(&m->kern_dump.snap.runnable_at_meta, -+ "each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS); -+ -+ init_desc_meta(&m->kern_dump.snap.rq_nr_meta, -+ "rq runnable task nr:", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta, -+ "scxrq runnable task nr :", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.snap_misc_meta, -+ "misc snap params :", 1, 1, SNAP_ITEMS); -+} -+ -+static void init_md_meta(struct md_info_t *m) -+{ -+ m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64); -+ m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64); -+ m->meta.unit_size = sizeof(u64); -+ m->meta.switches = MAX_SWITCHS; -+ m->meta.exceps = MAX_EXCEPS; -+ m->meta.global_dsqs = MAX_GLOBAL_DSQS; -+ m->meta.parse_dimens = PARSE_DIMENS; -+ m->meta.nr_cpus = num_possible_cpus(); -+ m->meta.real_cpus = nr_cpu_ids; -+ m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64); -+ m->meta.nr_meta_desc = 8; -+ m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64); -+ -+ init_desc_metas(m); -+} -+ -+#define MINIDUMP_DFL_SIZE (4 * 1024) -+ -+struct notifier_block hmbird_panic_blk; -+static int hmbird_panic_handler(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ if (!md_info) -+ return NOTIFY_DONE; -+ -+ get_hmbird_snapshot(&md_info->kern_dump.snap); -+ -+ return NOTIFY_DONE; -+} -+ -+static void panic_blk_init(void) -+{ -+ int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE); -+ -+ md_info = kzalloc(dump_size, GFP_KERNEL); -+ if (!md_info) -+ return; -+ init_md_meta(md_info); -+ -+ hmbird_panic_blk.notifier_call = hmbird_panic_handler; -+ /* make sure to execute before minidump. */ -+ hmbird_panic_blk.priority = INT_MAX; -+ atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk); -+ -+ hmbird_debug("register minidump.\n"); -+} -+ -+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size) -+{ -+ *vaddr = (unsigned long)md_info; -+ *size = sizeof(struct md_info_t); -+} -+// MTK minidump end -+ -+static inline void init_sched_prop_to_preempt_prio(void) -+{ -+ for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) { -+ switch (i) { -+ case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 5; -+ break; -+ -+ case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 4; -+ break; -+ -+ case HMBIRD_TASK_PROP_PIPELINE: -+ case HMBIRD_TASK_PROP_ISOLATE: -+ sched_prop_to_preempt_prio[i] = 3; -+ break; -+ -+ case HMBIRD_TASK_PROP_COMMON: -+ sched_prop_to_preempt_prio[i] = 1; -+ break; -+ -+ case HMBIRD_TASK_PROP_DEBUG_OR_LOG: -+ sched_prop_to_preempt_prio[i] = 0; -+ break; -+ -+ default: -+ sched_prop_to_preempt_prio[i] = 2; -+ break; -+ } -+ } -+} -+ -+void __init init_sched_hmbird_class(void) -+{ -+ int cpu; -+ u32 v; -+ struct hmbird_entity *init_hmbird; -+ -+ /* -+ * The following is to prevent the compiler from optimizing out the enum -+ * definitions so that BPF scheduler implementations can use them -+ * through the generated vmlinux.h. -+ */ -+ WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP | -+ HMBIRD_KICK_PREEMPT); -+ -+ init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL); -+ init_dsq_at_boot(); -+ init_isolate_cpus(); -+ hb_timer_init(); -+ init_sched_prop_to_preempt_prio(); -+#ifdef CONFIG_SMP -+ WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); -+#endif -+ -+ /* -+ * we can't static init init_task's hmbird struct, init here. -+ * init_task->hmbird would not use during boot. -+ */ -+ init_hmbird = kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL); -+ init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird; -+ if (init_hmbird) { -+ INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo); -+ INIT_LIST_HEAD(&init_hmbird->watchdog_node); -+ init_hmbird->sticky_cpu = -1; -+ init_hmbird->holding_cpu = -1; -+ atomic64_set(&init_hmbird->ops_state, 0); -+ init_hmbird->runnable_at = jiffies; -+ init_hmbird->slice = HMBIRD_SLICE_DFL; -+ init_hmbird->task = &init_task; -+ hmbird_set_sched_prop(&init_task, 0); -+ } else { -+ hmbird_err(INIT_TASK_FAIL, ":alloc init_task.scx failed!!!\n"); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ /* -+ * exec during boot phase, no need to care about alloc failed. -+ * lifecycle same to rq, no need to free. -+ */ -+ rq->android_oem_data1[HMBIRD_RQ_IDX] = -+ (u64)kmalloc(sizeof(struct hmbird_rq), GFP_KERNEL); -+ if (get_hmbird_rq(rq)) { -+ get_hmbird_rq(rq)->rq = rq; -+ get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window; -+ } else { -+ hmbird_err(ALLOC_RQSCX_FAIL, ":alloc rq->scx failed!!!\n"); -+ } -+ -+ rq->android_oem_data1[HMBIRD_OPS_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL); -+ if (!get_hmbird_ops(rq)) -+ pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n"); -+ init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL); -+ INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list); -+ -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL)); -+ -+ hmbird_ops_init(get_hmbird_ops(rq)); -+ } -+ -+ INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn); -+ INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn); -+ hmbird_misc_init(); -+ -+ panic_blk_init(); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_misc.c b/kernel/sched/hmbird/hmbird_misc.c -new file mode 100755 -index 000000000000..52582c22052c ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_misc.c -@@ -0,0 +1,124 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * @Description: -+ * @Version: -+ * @Author: wangrui8 -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include "hmbird_sched.h" -+#include -+#include -+#include "slim.h" -+ -+noinline int tracing_mark_write(const char *buf) -+{ -+ trace_printk(buf); -+ return 0; -+} -+ -+struct yield_opt_params yield_opt_params = { -+ .enable = 0, -+ .frame_per_sec = 120, -+ .frame_time_ns = NSEC_PER_SEC / 120, -+ .yield_headroom = 10, -+}; -+ -+DEFINE_PER_CPU(struct sched_yield_state, ystate); -+ -+static inline void hmbird_yield_state_update(struct sched_yield_state *ys) -+{ -+ if (!raw_spin_is_locked(&ys->lock)) -+ return; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ -+ if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1 -+ || ys->yield_cnt_after_sleep > yield_headroom) { -+ ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP); -+ } else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) { -+ ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP); -+ } -+ ys->yield_cnt = 0; -+ ys->sleep_times = 0; -+ ys->yield_cnt_after_sleep = 0; -+} -+ -+void hmbird_skip_yield(long *skip) -+{ -+ if (!get_hmbird_ops_enabled() || !yield_opt_params.enable) -+ return; -+ unsigned long flags, sleep_now = 0; -+ struct sched_yield_state *ys; -+ int cpu = raw_smp_processor_id(), cont_yield, new_frame; -+ int frame_time_ns = yield_opt_params.frame_time_ns; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ u64 wc; -+ -+ if (!(*skip)) { -+ wc = sched_clock(); -+ ys = &per_cpu(ystate, cpu); -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ -+ cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP; -+ new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1); -+ -+ if (!cont_yield && new_frame) { -+ hmbird_yield_state_update(ys); -+ ys->last_update_time = wc; -+ ys->sleep_end = ys->last_yield_time + frame_time_ns -+ - yield_headroom * YIELD_DURATION; -+ } -+ -+ if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) { -+ *skip = true; -+ -+ sleep_now = ys->sleep_times ? -+ max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep; -+ if (wc + sleep_now > ys->sleep_end) { -+ u64 delta = ys->sleep_end - wc; -+ -+ if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION) -+ sleep_now = delta; -+ else -+ sleep_now = 0; -+ } -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ if (sleep_now) { -+ sleep_now = div64_u64(sleep_now, 1000); -+ usleep_range_state(sleep_now, sleep_now, TASK_IDLE); -+ } -+ ys->sleep_times++; -+ ys->last_yield_time = sched_clock(); -+ return; -+ } -+ if (ys->sleep_times) -+ ys->yield_cnt_after_sleep++; -+ else -+ (ys->yield_cnt)++; -+ ys->last_yield_time = wc; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+} -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops) -+{ -+ hmbird_ops->scx_enable = get_hmbird_ops_enabled; -+ hmbird_ops->check_non_task = get_non_hmbird_task; -+ hmbird_ops->hmbird_get_md_info = hmbird_get_md_info; -+} -+ -+void hmbird_misc_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_init(&ys->lock); -+ } -+ -+ set_hmbird_module_loaded(1); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_sched.h b/kernel/sched/hmbird/hmbird_sched.h -new file mode 100755 -index 000000000000..6b5ef43cb827 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef __HMBIRD_SCHED__ -+#define __HMBIRD_SCHED__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include "../../time/tick-sched.h" -+#include "../../sched/sched.h" -+#include -+#include -+#include -+ -+#define REGISTER_TRACE_VH(vender_hook, handler) \ -+{ \ -+ ret = register_trace_##vender_hook(handler, NULL); \ -+ if (ret) { \ -+ pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \ -+ } \ -+} -+#define REGISTER_TRACE(vendor_hook, handler, data, err) \ -+do { \ -+ ret = register_trace_##vendor_hook(handler, data); \ -+ if (ret) { \ -+ pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret); \ -+ goto err; \ -+ } \ -+} while (0) -+ -+#define UNREGISTER_TRACE(vendor_hook, handler, data) \ -+ unregister_trace_##vendor_hook(handler, data) \ -+ -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+ -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int sched_ravg_window_frame_per_sec; -+extern int slim_gov_debug; -+extern int cpu7_tl; -+extern int scx_gov_ctrl; -+extern spinlock_t new_sched_ravg_window_lock; -+extern int cluster_separate; -+ -+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER BIT(31) -+#define MAX_YIELD_SLEEP (2000000ULL) -+#define MIN_YIELD_SLEEP (200000ULL) -+#define YIELD_DURATION (5000ULL) -+#define DEFAULT_YIELD_SLEEP_TH (10) -+ -+struct sched_yield_state { -+ raw_spinlock_t lock; -+ u64 last_yield_time; -+ u64 last_update_time; -+ u64 sleep_end; -+ unsigned long yield_cnt; -+ unsigned long yield_cnt_after_sleep; -+ unsigned long sleep; -+ int sleep_times; -+}; -+ -+DECLARE_PER_CPU(struct sched_yield_state, ystate); -+ -+void hmbird_window_rollover_run_once(struct rq *rq); -+void hmbird_yield_state_update_per_frame(void); -+void hmbird_misc_init(void); -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops); -+#endif /*__HMBIRD_SCHED__*/ -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.c b/kernel/sched/hmbird/hmbird_sched_proc.c -new file mode 100755 -index 000000000000..234768fc767a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.c -@@ -0,0 +1,527 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "hmbird_sched_proc.h" -+#include -+ -+#include "hmbird_util_track.h" -+#include "slim.h" -+ -+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched" -+#define SLIM_FREQ_GOV_DIR "slim_freq_gov" -+#define LOAD_TRACK_DIR "slim_walt" -+#define HMBIRD_PROC_PERMISSION 0666 -+ -+int scx_enable; -+int partial_enable; -+int cpuctrl_high_ratio = 55; -+int cpuctrl_low_ratio = 40; -+int slim_stats; -+int hmbirdcore_debug; -+int slim_for_app; -+int misfit_ds = 90; -+unsigned int highres_tick_ctrl; -+unsigned int highres_tick_ctrl_dbg; -+int cpu7_tl = 70; -+int slim_walt_ctrl; -+int slim_walt_dump; -+int slim_walt_policy; -+int slim_gov_debug; -+int scx_gov_ctrl = 1; -+int sched_ravg_window_frame_per_sec = 125; -+int parctrl_high_ratio = 55; -+int parctrl_low_ratio = 40; -+int parctrl_high_ratio_l = 65; -+int parctrl_low_ratio_l = 50; -+int isoctrl_high_ratio = 75; -+int isoctrl_low_ratio = 60; -+int isolate_ctrl; -+int iso_free_rescue; -+int heartbeat; -+int heartbeat_enable = 1; -+int watchdog_enable; -+int save_gov; -+u64 cpu_cluster_masks; -+int hmbird_preempt_policy; -+int cluster_separate; -+ -+char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= 32) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtoint(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoint\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+static int set_proc_buf_val_u64(struct file *file, const char __user *buf, -+ size_t count, u64 *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= sizeof(kbuf)) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtou64(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoul\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/* common ops begin */ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ return count; -+} -+ -+static int hmbird_common_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%d\n", *(int *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_show, pde_data(inode)); -+} -+ -+static int hmbird_common_ul_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%lu\n", *(unsigned long *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_ul_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_ul_show, pde_data(inode)); -+} -+ -+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write); -+/* common ops end */ -+ -+/* scx_enable ops begin */ -+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC); -+ if (hmbird_ctrl(*pval)) -+ return -EFAULT; -+ -+ return count; -+} -+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write); -+/* scx_enable ops end */ -+ -+/* hmbird_stats ops begin */ -+#define MAX_STATS_BUF (4096) -+static int hmbird_stats_proc_show(struct seq_file *m, void *v) -+{ -+ char *buf; -+ -+ buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ stats_print(buf, MAX_STATS_BUF); -+ -+ seq_printf(m, "%s\n", buf); -+ -+ kfree(buf); -+ return 0; -+} -+ -+static int hmbird_stats_proc_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_stats_proc_show, inode); -+} -+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL); -+/* hmbird_stats ops end */ -+ -+/* sched_ravg_window_frame_per_sec ops begin */ -+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file, -+ const char __user *buf, size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ sched_ravg_window_change(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open, -+ sched_ravg_window_frame_per_sec_proc_write); -+/* sched_ravg_window_frame_per_sec ops end */ -+ -+static ssize_t save_gov_str(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ for_each_possible_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy || (cpu != policy->cpu)) -+ continue; -+ WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0); -+ hmbird_info_systrace(":save origin gov : %s\n", saved_gov[cpu]); -+ } -+ return count; -+} -+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str); -+ -+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ u64 *pval = (u64 *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val_u64(file, buf, count, pval)) -+ return -EFAULT; -+ -+ if (scx_enable == 0) -+ set_cpu_cluster(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write); -+ -+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ int tmp_val; -+ -+ if (set_proc_buf_val(file, buf, count, &tmp_val)) -+ return -EFAULT; -+ -+ slim_walt_enable(tmp_val); -+ *pval = tmp_val; -+ -+ return count; -+} -+ -+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write); -+ -+/* yield_opt ops begin */ -+static int yield_opt_show(struct seq_file *m, void *v) -+{ -+ struct yield_opt_params *data = m->private; -+ -+ seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n", -+ data->enable, data->frame_per_sec, data->yield_headroom); -+ return 0; -+} -+ -+static int yield_opt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, yield_opt_show, pde_data(inode)); -+} -+ -+static ssize_t yield_opt_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu; -+ unsigned long flags; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if (copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp -+ != 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) || -+ (yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp; -+ yield_opt_params.frame_per_sec = frame_per_sec_tmp; -+ yield_opt_params.yield_headroom = yield_headroom_tmp; -+ yield_opt_params.enable = enable_tmp; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ ys->last_yield_time = 0; -+ ys->last_update_time = 0; -+ ys->sleep_end = 0; -+ ys->yield_cnt = 0; -+ ys->yield_cnt_after_sleep = 0; -+ ys->sleep = 0; -+ ys->sleep_times = 0; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write); -+ -+ -+static int __init hmbird_proc_init(void) -+{ -+ struct proc_dir_entry *hmbird_dir; -+ struct proc_dir_entry *load_track_dir; -+ struct proc_dir_entry *freq_gov_dir; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ -+ /* mkdir /proc/hmbird_sched */ -+ hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL); -+ if (!hmbird_dir) { -+ pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &scx_enable_proc_ops, -+ &scx_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &partial_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_stats); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbirdcore_debug); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_for_app); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &misfit_ds); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl_dbg); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpu7_tl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &cpu_cluster_masks_proc_ops, -+ &cpu_cluster_masks); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &save_gov_proc_ops, -+ &save_gov); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &watchdog_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isolate_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &iso_free_rescue); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_stats_proc_ops); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &yield_opt_proc_ops, -+ &yield_opt_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbird_preempt_policy); -+ /* /proc/hmbird_sched--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_walt */ -+ load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir); -+ if (!load_track_dir) { -+ pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_walt--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &slim_walt_ctrl_proc_ops, -+ &slim_walt_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_dump); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_policy); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &sched_ravg_window_frame_per_sec_proc_ops, -+ &sched_ravg_window_frame_per_sec); -+ /* /proc/hmbird_sched/slim_walt--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_freq_gov */ -+ freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir); -+ if (!freq_gov_dir) { -+ pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_freq_gov--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &slim_gov_debug); -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &scx_gov_ctrl); -+ /* /proc/hmbird_sched/slim_freq_gov--end */ -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cluster_separate); -+ -+ return 0; -+} -+ -+device_initcall(hmbird_proc_init); -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.h b/kernel/sched/hmbird/hmbird_sched_proc.h -new file mode 100755 -index 000000000000..e22bc75f1dd7 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SCHED_PROC_H__ -+#define __HMBIRD_SCHED_PROC_H__ -+ -+#include -+#include -+ -+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \ -+ do { \ -+ if (!proc_create(name, mode, parent, proc_ops)) { \ -+ pr_err("Error creating proc entry %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \ -+ do { \ -+ if (!proc_create_data(name, mode, parent, proc_ops, data)) { \ -+ pr_err("Error creating proc entry with data %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_PROC_OPS(name, open_func, write_func) \ -+ static const struct proc_ops name##_proc_ops = { \ -+ .proc_open = open_func, \ -+ .proc_write = write_func, \ -+ .proc_read = seq_read, \ -+ .proc_lseek = seq_lseek, \ -+ .proc_release = single_release, \ -+ } -+ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos); -+static int hmbird_common_show(struct seq_file *m, void *v); -+static int hmbird_common_open(struct inode *inode, struct file *file); -+ -+#endif -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.c b/kernel/sched/hmbird/hmbird_shadow_tick.c -new file mode 100755 -index 000000000000..21de551c5132 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.c -@@ -0,0 +1,159 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include -+#include "../../time/tick-sched.h" -+#include -+ -+#include "hmbird_shadow_tick.h" -+ -+#define HIGHRES_WATCH_CPU 0 -+ -+#include -+static bool shadow_tick_enable(void) {return highres_tick_ctrl; } -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; } -+#endif -+static bool shadow_tick_timer_init_flag; -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define shadow_tick_printk(fmt, args...) \ -+do { \ -+ int cpu = smp_processor_id(); \ -+ if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU) \ -+ trace_printk("hmbird shadow tick :"fmt, args); \ -+} while (0) -+#else -+#define shadow_tick_printk(fmt, args...) -+#endif -+ -+DEFINE_PER_CPU(struct hrtimer, stt); -+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu))) -+#define STOP_IDLE_TRIGGER (1) -+#define PERIODIC_TICK_TRIGGER (2) -+#define TICK_INTVAL (1000000ULL) -+/* -+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms, -+ * so we can't restart hrtimer in scheduler tick. -+ */ -+static DEFINE_PER_CPU(u8, trigger_event); -+ -+/* -+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick. -+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler. -+ */ -+static DEFINE_PER_CPU(u8, tick_phase); -+ -+static inline void highres_timer_ctrl(bool enable, int cpu) -+{ -+ if (enable && hmbird_enabled()) { -+ if (!hrtimer_active(shadow_tick_timer(cpu))) -+ hrtimer_start(shadow_tick_timer(cpu), -+ ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED); -+ } else { -+ if (!enable) -+ hrtimer_cancel(shadow_tick_timer(cpu)); -+ } -+} -+ -+static inline void high_res_clear_phase(int cpu) -+{ -+ per_cpu(tick_phase, cpu) = 0; -+} -+ -+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer) -+{ -+ per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3; -+ if (per_cpu(tick_phase, cpu)) { -+ hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) { -+ per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ struct rq_flags rf; -+ -+ rq_lock(rq, &rf); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ rq_unlock(rq, &rf); -+ -+ return highres_next_phase(cpu, timer); -+} -+ -+void shadow_tick_timer_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance; -+ } -+} -+ -+void start_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable()) { -+ if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER) -+ highres_timer_ctrl(false, cpu); -+ per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static void stop_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ per_cpu(trigger_event, cpu) = 0; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(false, cpu); -+} -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ stop_shadow_tick_timer(); -+} -+ -+void scheduler_tick_handler(void *unused, struct rq *rq) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ start_shadow_tick_timer(); -+} -+ -+static int __init hmbird_shadow_tick_init(void) -+{ -+ int ret = 0; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ shadow_tick_timer_init(); -+ shadow_tick_timer_init_flag = true; -+ return ret; -+} -+ -+device_initcall(hmbird_shadow_tick_init); -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.h b/kernel/sched/hmbird/hmbird_shadow_tick.h -new file mode 100755 -index 000000000000..0c26fd984f0a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.h -@@ -0,0 +1,11 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SHADOW_TICK_H__ -+#define __HMBIRD_SHADOW_TICK_H__ -+ -+#include -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+void scheduler_tick_handler(void *unused, struct rq *rq); -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state); -+#endif -diff --git a/kernel/sched/hmbird/hmbird_trace.h b/kernel/sched/hmbird/hmbird_trace.h -new file mode 100755 -index 000000000000..8468b406f347 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_trace.h -@@ -0,0 +1,92 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hmbird -+ -+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HMBIRD_H -+ -+#include -+#include -+#include -+ -+#define MAX_FATAL_INFO (64) -+ -+TRACE_EVENT(hmbird_fatal_info, -+ -+ TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info), -+ -+ TP_ARGS(type, pe, lnr, bnr, info), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, type) -+ __field(int, pe) -+ __field(int, lnr) -+ __field(int, bnr) -+ __array(char, info, MAX_FATAL_INFO)), -+ -+ TP_fast_assign( -+ __entry->type = type; -+ __entry->pe = pe; -+ __entry->lnr = lnr; -+ __entry->bnr = bnr; -+ memcpy(__entry->info, info, MAX_FATAL_INFO);), -+ -+ TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s", -+ __entry->type, __entry->pe, __entry->lnr, __entry->bnr, -+ __entry->info) -+); -+ -+TRACE_EVENT(hmbird_update_history, -+ -+ TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq, -+ struct task_struct *p, u32 runtime, int samples, int event), -+ -+ TP_ARGS(hmbird, rq, p, runtime, samples, event), -+ -+ TP_STRUCT__entry( -+ __array(char, comm, TASK_COMM_LEN) -+ __field(pid_t, pid) -+ __field(unsigned int, runtime) -+ __field(int, samples) -+ __field(int, event) -+ __field(unsigned int, demand) -+ __array(u32, hist, RAVG_HIST_SIZE) -+ __field(u16, task_util) -+ __field(int, cpu)), -+ -+ TP_fast_assign( -+ memcpy(__entry->comm, p->comm, TASK_COMM_LEN); -+ __entry->pid = p->pid; -+ __entry->runtime = runtime; -+ __entry->samples = samples; -+ __entry->event = event; -+ __entry->demand = hmbird->sts.demand; -+ memcpy(__entry->hist, hmbird->sts.sum_history, -+ RAVG_HIST_SIZE * sizeof(u32)); -+ __entry->task_util = hmbird->sts.demand_scaled; -+ __entry->cpu = rq->cpu;), -+ -+ TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d", -+ __entry->comm, __entry->pid, -+ __entry->runtime, __entry->samples, -+ __entry->event, -+ __entry->demand, -+ __entry->hist[0], __entry->hist[1], -+ __entry->hist[2], __entry->hist[3], -+ __entry->hist[4], -+ __entry->task_util, -+ __entry->cpu) -+); -+ -+#endif /*_TRACE_HMBIRD_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird -+ -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE hmbird_trace -+/* This part must be outside protection */ -+#include -diff --git a/kernel/sched/hmbird/hmbird_util_track.c b/kernel/sched/hmbird/hmbird_util_track.c -new file mode 100755 -index 000000000000..d520a8403401 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.c -@@ -0,0 +1,626 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+ -+#define CREATE_TRACE_POINTS -+#include "hmbird_trace.h" -+#undef CREATE_TRACE_POINTS -+ -+#define HMBIRD_DEBUG_PANIC (1 << 3) -+static bool init_irq_work_inited; -+ -+extern noinline int tracing_mark_write(const char *buf); -+ -+int hmbird_sched_ravg_window = 8000000; -+int new_hmbird_sched_ravg_window = 8000000; -+DEFINE_SPINLOCK(new_sched_ravg_window_lock); -+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+ -+static struct irq_work hmbird_slim_walt_irq_work; -+ -+/*Sysctl related interface*/ -+#define WINDOW_STATS_RECENT 0 -+#define WINDOW_STATS_MAX 1 -+#define WINDOW_STATS_MAX_RECENT_AVG 2 -+#define WINDOW_STATS_AVG 3 -+#define WINDOW_STATS_INVALID_POLICY 4 -+ -+ -+#define SCX_SCHED_CAPACITY_SHIFT 10 -+#define SCHED_ACCOUNT_WAIT_TIME 0 -+ -+atomic64_t hmbird_irq_work_lastq_ws; -+static u64 tick_sched_clock; -+ -+static inline u64 scale_exec_time(u64 delta, struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT; -+} -+ -+static inline u64 scale_time_to_util(u64 d) -+{ -+ do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT); -+ return d; -+} -+ -+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ delta = scale_exec_time(delta, rq); -+ sts->sum += delta; -+ if (unlikely(sts->sum > hmbird_sched_ravg_window)) -+ sts->sum = hmbird_sched_ravg_window; -+ -+ return delta; -+} -+ -+ -+static int -+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) -+{ -+ /* -+ * No need to bother updating task demand for the idle task. -+ */ -+ if (is_idle_task(p)) -+ return 0; -+ -+ /* -+ * When a task is waking up it is completing a segment of non-busy -+ * time. Likewise, if wait time is not treated as busy time, then -+ * when a task begins to run or is migrated, it is not running and -+ * is completing a segment of non-busy time. -+ */ -+ if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && -+ (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) -+ return 0; -+ -+ /* -+ * The idle exit time is not accounted for the first task _picked_ up to -+ * run on the idle CPU. -+ */ -+ if (event == PICK_NEXT_TASK && rq->curr == rq->idle) -+ return 0; -+ -+ /* -+ * TASK_UPDATE can be called on sleeping task, when its moved between -+ * related groups -+ */ -+ if (event == TASK_UPDATE) { -+ if (rq->curr == p) -+ return 1; -+ -+ return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; -+ } -+ -+ return 1; -+} -+ -+static void rollover_cpu_window(struct rq *rq, bool full_window) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 curr_sum = srq->curr_runnable_sum; -+ -+ if (unlikely(full_window)) -+ curr_sum = 0; -+ -+ srq->prev_runnable_sum = curr_sum; -+ srq->curr_runnable_sum = 0; -+} -+ -+static u64 -+update_window_start(struct rq *rq, u64 wallclock, int event) -+{ -+ s64 delta; -+ int nr_windows; -+ bool full_window; -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start = srq->window_start; -+ -+ if (wallclock < srq->latest_clock) -+ wallclock = srq->latest_clock; -+ delta = wallclock - srq->window_start; -+ if (delta < 0) { -+ delta = 0; -+ wallclock = srq->window_start; -+ } -+ srq->latest_clock = wallclock; -+ if (delta < hmbird_sched_ravg_window) -+ return old_window_start; -+ -+ nr_windows = div64_u64(delta, hmbird_sched_ravg_window); -+ srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window; -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ full_window = nr_windows > 1; -+ rollover_cpu_window(rq, full_window); -+ -+ return old_window_start; -+} -+ -+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -+ -+static inline unsigned int get_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static inline unsigned int cpu_cur_freq(int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cur; -+} -+ -+static void -+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, -+ u64 wallclock) -+{ -+ int cpu = cpu_of(rq); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * -+ arch_scale_cpu_capacity(cpu), get_max_freq(cpu)); -+} -+ -+/* -+ * Called when new window is starting for a task, to record cpu usage over -+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 -+ * when, say, a real-time task runs without preemption for several windows at a -+ * stretch. -+ */ -+static void update_history(struct rq *rq, struct task_struct *p, -+ u32 runtime, int samples, int event) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u32 *hist = &sts->sum_history[0]; -+ int i; -+ u32 max = 0, avg, demand; -+ u64 sum = 0; -+ u16 demand_scaled; -+ -+ /* Ignore windows where task had no activity */ -+ if (!runtime || is_idle_task(p) || !samples) -+ goto done; -+ -+ /* Push new 'runtime' value onto stack */ -+ for (; samples > 0; samples--) { -+ hist[sts->cidx] = runtime; -+ sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE; -+ } -+ -+ for (i = 0; i < RAVG_HIST_SIZE; i++) { -+ sum += hist[i]; -+ if (hist[i] > max) -+ max = hist[i]; -+ } -+ -+ sts->sum = 0; -+ avg = div64_u64(sum, RAVG_HIST_SIZE); -+ -+ switch (slim_walt_policy) { -+ case WINDOW_STATS_RECENT: -+ demand = runtime; -+ break; -+ case WINDOW_STATS_MAX: -+ demand = max; -+ break; -+ case WINDOW_STATS_AVG: -+ demand = avg; -+ break; -+ default: -+ demand = max(avg, runtime); -+ } -+ -+ demand_scaled = scale_time_to_util(demand); -+ -+ sts->demand = demand; -+ sts->demand_scaled = demand_scaled; -+ -+done: -+ return; -+} -+ -+ -+static u64 update_task_demand(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ -+ u64 delta, window_start = srq->window_start; -+ int new_window, nr_full_windows; -+ u32 window_size = hmbird_sched_ravg_window; -+ u64 runtime; -+ -+ new_window = mark_start < window_start; -+ if (!account_busy_for_task_demand(rq, p, event)) { -+ if (new_window) -+ /* -+ * If the time accounted isn't being accounted as -+ * busy time, and a new window started, only the -+ * previous window need be closed out with the -+ * pre-existing demand. Multiple windows may have -+ * elapsed, but since empty windows are dropped, -+ * it is not necessary to account those. -+ */ -+ update_history(rq, p, sts->sum, 1, event); -+ return 0; -+ } -+ -+ if (!new_window) { -+ /* -+ * The simple case - busy time contained within the existing -+ * window. -+ */ -+ return add_to_task_demand(rq, p, wallclock - mark_start); -+ } -+ -+ /* -+ * Busy time spans at least two windows. Temporarily rewind -+ * window_start to first window boundary after mark_start. -+ */ -+ delta = window_start - mark_start; -+ nr_full_windows = div64_u64(delta, window_size); -+ window_start -= (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (window_start - mark_start) first */ -+ runtime = add_to_task_demand(rq, p, window_start - mark_start); -+ -+ /* Push new sample(s) into task's demand history */ -+ update_history(rq, p, sts->sum, 1, event); -+ if (nr_full_windows) { -+ u64 scaled_window = scale_exec_time(window_size, rq); -+ -+ update_history(rq, p, scaled_window, nr_full_windows, event); -+ runtime += nr_full_windows * scaled_window; -+ } -+ -+ /* -+ * Roll window_start back to current to process any remainder -+ * in current window. -+ */ -+ window_start += (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (wallclock - window_start) next */ -+ mark_start = window_start; -+ runtime += add_to_task_demand(rq, p, wallclock - mark_start); -+ -+ return runtime; -+} -+ -+u16 slim_walt_cpu_util(int cpu) -+{ -+ u64 prev_runnable_sum; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ prev_runnable_sum = srq->prev_runnable_sum; -+ do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT); -+ -+ return (u16)prev_runnable_sum; -+} -+ -+static DEFINE_PER_CPU(u16, prev_cpu_util); -+static inline void cpu_util_update_systrace_c(int cpu) -+{ -+ char buf[256]; -+ u16 cpu_util = slim_walt_cpu_util(cpu); -+ -+ if (cpu_util != per_cpu(prev_cpu_util, cpu)) { -+ snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n", -+ cpu, cpu_util); -+ tracing_mark_write(buf); -+ per_cpu(prev_cpu_util, cpu) = cpu_util; -+ } -+} -+ -+ -+static inline int account_busy_for_cpu_time(struct rq *rq, -+ struct task_struct *p, -+ int event) -+{ -+ return !is_idle_task(p) && (event == PUT_PREV_TASK -+ || event == TASK_UPDATE); -+} -+ -+ -+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ int new_window, full_window = 0; -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 window_start = srq->window_start; -+ u32 window_size = srq->prev_window_size; -+ u64 delta; -+ u64 *curr_runnable_sum = &srq->curr_runnable_sum; -+ u64 *prev_runnable_sum = &srq->prev_runnable_sum; -+ -+ new_window = mark_start < window_start; -+ if (new_window) -+ full_window = (window_start - mark_start) >= window_size; -+ -+ -+ if (!account_busy_for_cpu_time(rq, p, event)) -+ goto done; -+ -+ -+ if (!new_window) { -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. No rollover -+ * since we didn't start a new window. An example of this is -+ * when a task starts execution and then sleeps within the -+ * same window. -+ */ -+ delta = wallclock - mark_start; -+ -+ delta = scale_exec_time(delta, rq); -+ *curr_runnable_sum += delta; -+ -+ goto done; -+ } -+ -+ /* -+ * situations below this need window rollover, -+ * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done -+ * in update_window_start() -+ * -+ * For task counters curr/prev_window[_cpu] are rolled over in the early part of -+ * this function. If full_window(s) have expired and time since last update needs -+ * to be accounted as busy time, set the prev to a complete window size time, else -+ * add the prev window portion. -+ * -+ * For task curr counters a new window has begun, always assign -+ */ -+ -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. A new window -+ * must have been started in udpate_window_start() -+ * If any of these three above conditions are true -+ * then this busy time can't be accounted as irqtime. -+ * -+ * Busy time for the idle task need not be accounted. -+ * -+ * An example of this would be a task that starts execution -+ * and then sleeps once a new window has begun. -+ */ -+ -+ /* -+ * A full window hasn't elapsed, account partial -+ * contribution to previous completed window. -+ */ -+ -+ -+ delta = full_window ? scale_exec_time(window_size, rq) : -+ scale_exec_time(window_start - mark_start, rq); -+ -+ *prev_runnable_sum += delta; -+ -+ /* Account piece of busy time in the current window. */ -+ delta = scale_exec_time(wallclock - window_start, rq); -+ *curr_runnable_sum += delta; -+ -+done: -+ if (slim_walt_dump && new_window) -+ cpu_util_update_systrace_c(rq->cpu); -+} -+ -+ -+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq) -+{ -+ u64 result; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 new_window_start = srq->window_start; -+ -+ if (old_window_start == new_window_start) -+ return; -+ -+ result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws, -+ old_window_start, new_window_start); -+ if (result != old_window_start) -+ return; -+ -+ if (likely(cpu_online(raw_smp_processor_id()))) -+ irq_work_queue(&hmbird_slim_walt_irq_work); -+ else -+ irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask)); -+} -+ -+/* -+ * In the core scheduler, most of the load update points update the rq_clock after -+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of -+ * obtaining the time, but to prevent the subsequent migration of the load update -+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock -+ * update here. -+ */ -+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p, -+ struct rq *rq, int event) -+{ -+ if (!(rq->clock_update_flags & RQCF_UPDATED)) -+ update_rq_clock(rq); -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock)); -+} -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start; -+ -+ if (!slim_walt_ctrl) -+ return; -+ -+ if (!srq->window_start || sts->mark_start == wallclock) -+ return; -+ -+ old_window_start = update_window_start(rq, wallclock, event); -+ -+ if (!sts->window_start) -+ sts->window_start = srq->window_start; -+ -+ if (!sts->mark_start) -+ goto done; -+ -+ update_task_rq_cpu_cycles(p, rq, event, wallclock); -+ update_task_demand(p, rq, event, wallclock); -+ update_cpu_busy_time(p, rq, event, wallclock); -+ -+ sts->window_start = srq->window_start; -+ -+done: -+ sts->mark_start = wallclock; -+ slim_walt_window_rollover_run_once(old_window_start, rq); -+} -+ -+static void slim_walt_irq_work(struct irq_work *irq_work) -+{ -+ cpumask_t lock_cpus; -+ struct hmbird_sched_rq_stats *srq; -+ struct rq *rq; -+ int cpu; -+ int level = 0; -+ u64 wc; -+ unsigned long flags; -+ -+ cpumask_copy(&lock_cpus, cpu_possible_mask); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ if (level == 0) -+ raw_spin_lock(&cpu_rq(cpu)->__lock); -+ else -+ raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); -+ level++; -+ } -+ -+ wc = sched_clock(); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ rq = cpu_rq(cpu); -+ hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc); -+ } -+ -+ cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER); -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) { -+ srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id()); -+ if (wc < srq->window_start + new_hmbird_sched_ravg_window) -+ hmbird_sched_ravg_window = new_hmbird_sched_ravg_window; -+ } -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ raw_spin_unlock(&cpu_rq(cpu)->__lock); -+ } -+} -+static void hmbird_sched_init_rq(struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ srq->task_exec_scale = 1024; -+ srq->window_start = 0; -+} -+ -+void hmbird_sched_init_task(struct task_struct *p) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ memset(sts, 0, sizeof(struct hmbird_sched_task_stats)); -+} -+ -+static void hmbird_sched_stats_init(void) -+{ -+ unsigned long flags; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ raw_spin_lock_irqsave(&rq->__lock, flags); -+ hmbird_sched_init_rq(rq); -+ raw_spin_unlock_irqrestore(&rq->__lock, flags); -+ } -+ slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG; -+ -+ if (false == init_irq_work_inited) { -+ init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work); -+ init_irq_work_inited = true; -+ } -+} -+ -+void hmbird_scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (unlikely(!tick_sched_clock)) { -+ /* -+ * Let the window begin 20us prior to the tick, -+ * that way we are guaranteed a rollover when the tick occurs. -+ * Use rq->clock directly instead of rq_clock() since -+ * we do not have the rq lock and -+ * rq->clock was updated in the tick callpath. -+ */ -+ if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000)) -+ return; -+ for_each_possible_cpu(cpu) { -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ srq->window_start = tick_sched_clock; -+ } -+ atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock); -+ } -+} -+ -+void slim_walt_enable(int enable) -+{ -+ if (1 == !!enable) { -+ hmbird_sched_stats_init(); -+ WRITE_ONCE(tick_sched_clock, 0); -+ } else -+ slim_walt_ctrl = 0; -+} -+ -+void slim_get_cpu_util(int cpu, u64 *util) -+{ -+ if (cpu < 0) -+ return; -+ -+ *util = slim_walt_cpu_util(cpu); -+} -+ -+void slim_get_task_util(struct task_struct *p, u64 *util) -+{ -+ if (p == NULL) -+ return; -+ -+ *util = get_hmbird_ts(p)->sts.demand_scaled; -+} -+ -+void sched_ravg_window_change(int frame_per_sec) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec; -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+} -diff --git a/kernel/sched/hmbird/hmbird_util_track.h b/kernel/sched/hmbird/hmbird_util_track.h -new file mode 100755 -index 000000000000..17b85df7f83b ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.h -@@ -0,0 +1,33 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef __HMBIRD_UTIL_TRACK_H__ -+#define __HMBIRD_UTIL_TRACK_H__ -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock); -+void hmbird_sched_init_task(struct task_struct *p); -+void slim_walt_enable(int enable); -+void slim_get_cpu_util(int cpu, u64 *util); -+void slim_get_task_util(struct task_struct *p, u64 *util); -+ -+extern atomic64_t hmbird_irq_work_lastq_ws; -+ -+enum task_event { -+ PUT_PREV_TASK = 0, -+ PICK_NEXT_TASK = 1, -+ TASK_WAKE = 2, -+ TASK_MIGRATE = 3, -+ TASK_UPDATE = 4, -+ IRQ_UPDATE = 5, -+}; -+ -+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+#endif /* __HMBIRD_UTIL_TRACK_H__ */ -diff --git a/kernel/sched/hmbird/slim.h b/kernel/sched/hmbird/slim.h -new file mode 100755 -index 000000000000..eb386260e950 ---- /dev/null -+++ b/kernel/sched/hmbird/slim.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+ -+#ifndef __SLIM_H -+#define __SLIM_H -+ -+extern atomic_t __hmbird_ops_enabled; -+extern atomic_t non_hmbird_task; -+extern int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+extern int heartbeat; -+extern int heartbeat_enable; -+extern int watchdog_enable; -+extern int isolate_ctrl; -+extern int parctrl_high_ratio; -+extern int parctrl_low_ratio; -+extern int isoctrl_high_ratio; -+extern int isoctrl_low_ratio; -+extern int iso_free_rescue; -+extern int yield_opt; -+ -+extern enum hmbird_switch_type sw_type; -+extern noinline int tracing_mark_write(const char *buf); -+int task_top_id(struct task_struct *p); -+void stats_print(char *buf, int len); -+void hmbird_skip_yield(long *skip); -+extern spinlock_t hmbird_tasks_lock; -+ -+struct yield_opt_params { -+ int enable; -+ int frame_per_sec; -+ u64 frame_time_ns; -+ int yield_headroom; -+}; -+ -+extern struct yield_opt_params yield_opt_params; -+ -+#define MAX_GOV_LEN (16) -+extern char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+#endif -diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c -index b33cefeb4188..487255ac6832 100755 ---- a/kernel/sched/idle.c -+++ b/kernel/sched/idle.c -@@ -408,13 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl - - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) - { -- scx_update_idle(rq, false); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, false); -+#endif - } - - static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) - { - update_idle_core(rq); -- scx_update_idle(rq, true); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, true); -+#endif - schedstat_inc(rq->sched_goidle); - } - -diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h -index 509e8dc616ab..ab546ffc244b 100755 ---- a/kernel/sched/sched.h -+++ b/kernel/sched/sched.h -@@ -188,18 +188,13 @@ static inline int idle_policy(int policy) - return policy == SCHED_IDLE; - } - --static inline int normal_policy(int policy) --{ --#ifdef CONFIG_SCHED_CLASS_EXT -- if (policy == SCHED_EXT) -- return true; --#endif -- return policy == SCHED_NORMAL; --} -- - static inline int fair_policy(int policy) - { -- return normal_policy(policy) || policy == SCHED_BATCH; -+#ifdef CONFIG_HMBIRD_SCHED -+ return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#else -+ return policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#endif - } - - static inline int rt_policy(int policy) -@@ -247,24 +242,6 @@ static inline void update_avg(u64 *avg, u64 sample) - #define shr_bound(val, shift) \ - (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) - --/* -- * cgroup weight knobs should use the common MIN, DFL and MAX values which are -- * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it -- * maps pretty well onto the shares value used by scheduler and the round-trip -- * conversions preserve the original value over the entire range. -- */ --static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) --{ -- return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); --} -- --static inline unsigned long sched_weight_to_cgroup(unsigned long weight) --{ -- return clamp_t(unsigned long, -- DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -- CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); --} -- - /* - * !! For sched_setattr_nocheck() (kernel) only !! - * -@@ -532,10 +509,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) { } - #endif /* CONFIG_SMP */ - #else /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_HMBIRD_SCHED - static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) - { - return 0; - } -+#endif - #endif /* CONFIG_FAIR_GROUP_SCHED */ - - #else /* CONFIG_CGROUP_SCHED */ -@@ -700,29 +679,6 @@ struct cfs_rq { - #endif /* CONFIG_FAIR_GROUP_SCHED */ - }; - --#ifdef CONFIG_SCHED_CLASS_EXT --/* scx_rq->flags, protected by the rq lock */ --enum scx_rq_flags { -- SCX_RQ_CAN_STOP_TICK = 1 << 0, --}; -- --struct scx_rq { -- struct scx_dispatch_q local_dsq; -- struct list_head watchdog_list; -- u64 ops_qseq; -- u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -- u32 nr_running; -- u32 flags; -- bool cpu_released; -- cpumask_var_t cpus_to_kick; -- cpumask_var_t cpus_to_preempt; -- cpumask_var_t cpus_to_wait; -- u64 pnt_seq; -- struct irq_work kick_cpus_irq_work; -- struct rq *rq; --}; --#endif /* CONFIG_SCHED_CLASS_EXT */ -- - static inline int rt_bandwidth_enabled(void) - { - return sysctl_sched_rt_runtime >= 0; -@@ -1258,11 +1214,7 @@ struct rq { - - ANDROID_OEM_DATA_ARRAY(1, 16); - --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, struct scx_rq *scx); --#else - ANDROID_KABI_RESERVE(1); --#endif - ANDROID_KABI_RESERVE(2); - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); -@@ -2340,11 +2292,6 @@ struct affinity_context { - unsigned int flags; - }; - --enum rq_onoff_reason { -- RQ_ONOFF_HOTPLUG, /* CPU is going on/offline */ -- RQ_ONOFF_TOPOLOGY, /* sched domain topology update */ --}; -- - struct sched_class { - - #ifdef CONFIG_UCLAMP_TASK -@@ -2551,7 +2498,6 @@ extern void init_sched_rt_class(void); - extern void init_sched_fair_class(void); - - extern void reweight_task(struct task_struct *p, int prio); --extern void __setscheduler_prio(struct task_struct *p, int prio); - - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); -@@ -2632,53 +2578,6 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) - extern void activate_task(struct rq *rq, struct task_struct *p, int flags); - extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); - --struct sched_change_guard { -- struct task_struct *p; -- struct rq *rq; -- bool queued; -- bool running; -- bool done; --}; -- --extern struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -- --extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags); -- --/** -- * SCHED_CHANGE_BLOCK - Nested block for task attribute updates -- * @__rq: Runqueue the target task belongs to -- * @__p: Target task -- * @__flags: DEQUEUE/ENQUEUE_* flags -- * -- * A task may need to be dequeued and put_prev_task'd for attribute updates and -- * set_next_task'd and re-enqueued afterwards. This helper defines a nested -- * block which automatically handles these preparation and cleanup operations. -- * -- * SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- * update_attribute(p); -- * ... -- * } -- * -- * If @__flags is a variable, the variable may be updated in the block body and -- * the updated value will be used when re-enqueueing @p. -- * -- * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling -- * update_rq_clock() beforehand. Otherwise, the rq clock is automatically -- * updated iff the task needs to be dequeued and re-enqueued. Only the former -- * case guarantees that the rq clock is up-to-date inside and after the block. -- */ --#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -- for (struct sched_change_guard __cg = \ -- sched_change_guard_init(__rq, __p, __flags); \ -- !__cg.done; sched_change_guard_fini(&__cg, __flags)) -- --extern void check_class_changing(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class); --extern void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio); -- - extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); - - #ifdef CONFIG_PREEMPT_RT -@@ -3710,6 +3609,8 @@ static inline bool cpu_busy_with_softirqs(int cpu) - } - #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ - --#include "ext.h" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird.h" -+#endif - - #endif /* _KERNEL_SCHED_SCHED_H */ -diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c -index 998c2eefe173..2a3f819b9e88 100755 ---- a/kernel/time/tick-sched.c -+++ b/kernel/time/tick-sched.c -@@ -34,6 +34,10 @@ - - #include - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "../sched/hmbird/hmbird_shadow_tick.h" -+#endif -+ - /* - * Per-CPU nohz control structure - */ -@@ -1112,6 +1116,9 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) - return true; - } - -+#ifdef CONFIG_HMBIRD_SCHED -+extern void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+#endif - /** - * tick_nohz_idle_stop_tick - stop the idle tick from the idle task - * -@@ -1124,6 +1131,9 @@ void tick_nohz_idle_stop_tick(void) - ktime_t expires; - - trace_android_vh_tick_nohz_idle_stop_tick(NULL); -+#ifdef CONFIG_HMBIRD_SCHED -+ android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL); -+#endif - /* - * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the - * tick timer expiration time is known already. -diff --git a/tools/sched_ext/.gitignore b/tools/sched_ext/.gitignore -deleted file mode 100644 -index a3240f9f7eba..000000000000 ---- a/tools/sched_ext/.gitignore -+++ /dev/null -@@ -1,9 +0,0 @@ --scx_example_simple --scx_example_qmap --scx_example_central --scx_example_pair --scx_example_flatcg --scx_example_userland --*.skel.h --*.subskel.h --/tools/ -diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile -deleted file mode 100644 -index 73c43782837d..000000000000 ---- a/tools/sched_ext/Makefile -+++ /dev/null -@@ -1,216 +0,0 @@ --# SPDX-License-Identifier: GPL-2.0 --# Copyright (c) 2022 Meta Platforms, Inc. and affiliates. --include ../build/Build.include --include ../scripts/Makefile.arch --include ../scripts/Makefile.include -- --ifneq ($(LLVM),) --ifneq ($(filter %/,$(LLVM)),) --LLVM_PREFIX := $(LLVM) --else ifneq ($(filter -%,$(LLVM)),) --LLVM_SUFFIX := $(LLVM) --endif -- --CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi --CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu --CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl --CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu --CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu --CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu --CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu --CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu --CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu --CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) -- --ifeq ($(CROSS_COMPILE),) --ifeq ($(CLANG_TARGET_FLAGS),) --$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk --else --CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) --endif # CLANG_TARGET_FLAGS --else --CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) --endif # CROSS_COMPILE -- --CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as --else --CC := $(CROSS_COMPILE)gcc --endif # LLVM -- --CURDIR := $(abspath .) --TOOLSDIR := $(abspath ..) --LIBDIR := $(TOOLSDIR)/lib --BPFDIR := $(LIBDIR)/bpf --TOOLSINCDIR := $(TOOLSDIR)/include --BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool --APIDIR := $(TOOLSINCDIR)/uapi --GENDIR := $(abspath ../../include/generated) --GENHDR := $(GENDIR)/autoconf.h -- --SCRATCH_DIR := $(CURDIR)/tools --BUILD_DIR := $(SCRATCH_DIR)/build --INCLUDE_DIR := $(SCRATCH_DIR)/include --BPFOBJ_DIR := $(BUILD_DIR)/libbpf --BPFOBJ := $(BPFOBJ_DIR)/libbpf.a --ifneq ($(CROSS_COMPILE),) --HOST_BUILD_DIR := $(BUILD_DIR)/host --HOST_SCRATCH_DIR := host-tools --HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include --else --HOST_BUILD_DIR := $(BUILD_DIR) --HOST_SCRATCH_DIR := $(SCRATCH_DIR) --HOST_INCLUDE_DIR := $(INCLUDE_DIR) --endif --HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a --RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids --DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -- --VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ -- $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ -- ../../vmlinux \ -- /sys/kernel/btf/vmlinux \ -- /boot/vmlinux-$(shell uname -r) --VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) --ifeq ($(VMLINUX_BTF),) --$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") --endif -- --BPFTOOL ?= $(DEFAULT_BPFTOOL) -- --ifneq ($(wildcard $(GENHDR)),) -- GENFLAGS := -DHAVE_GENHDR --endif -- --CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ -- -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -- -I$(TOOLSINCDIR) -I$(APIDIR) -- --CARGOFLAGS := --release -- --# Silence some warnings when compiled with clang --ifneq ($(LLVM),) --CFLAGS += -Wno-unused-command-line-argument --endif -- --LDFLAGS = -lelf -lz -lpthread -- --IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - &1 \ -- | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ --$(shell $(1) -dM -E - $@ --else -- $(call msg,CP,,$@) -- $(Q)cp "$(VMLINUX_H)" $@ --endif -- --%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \ -- | $(BPFOBJ) -- $(call msg,CLNG-BPF,,$@) -- $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ -- --%.skel.h: %.bpf.o $(BPFTOOL) -- $(call msg,GEN-SKEL,,$@) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) -- $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) -- $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@ -- $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h) -- --scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_userland: scx_example_userland.c scx_example_userland.skel.h \ -- scx_example_userland_common.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR) --atropos: export ATROPOS_CLANG = $(CLANG) --atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS) --atropos: $(INCLUDE_DIR)/vmlinux.h -- cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS) -- --clean: -- cargo clean --manifest-path=atropos/Cargo.toml -- rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) -- rm -f *.o *.bpf.o *.skel.h *.subskel.h -- rm -f scx_example_simple scx_example_qmap scx_example_central \ -- scx_example_pair scx_example_flatcg scx_example_userland -- --.PHONY: all atropos clean -- --# delete failed targets --.DELETE_ON_ERROR: -- --# keep intermediate (.skel.h, .bpf.o, etc) targets --.SECONDARY: -diff --git a/tools/sched_ext/atropos/.gitignore b/tools/sched_ext/atropos/.gitignore -deleted file mode 100644 -index 186dba259ec2..000000000000 ---- a/tools/sched_ext/atropos/.gitignore -+++ /dev/null -@@ -1,3 +0,0 @@ --src/bpf/.output --Cargo.lock --target -diff --git a/tools/sched_ext/atropos/Cargo.toml b/tools/sched_ext/atropos/Cargo.toml -deleted file mode 100644 -index 7462a836d53d..000000000000 ---- a/tools/sched_ext/atropos/Cargo.toml -+++ /dev/null -@@ -1,28 +0,0 @@ --[package] --name = "scx_atropos" --version = "0.5.0" --authors = ["Dan Schatzberg ", "Meta"] --edition = "2021" --description = "Userspace scheduling with BPF" --license = "GPL-2.0-only" -- --[dependencies] --anyhow = "1.0.65" --bitvec = { version = "1.0", features = ["serde"] } --clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } --ctrlc = { version = "3.1", features = ["termination"] } --fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"} --hex = "0.4.3" --libbpf-rs = "0.19.1" --libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] } --libc = "0.2.137" --log = "0.4.17" --ordered-float = "3.4.0" --simplelog = "0.12.0" -- --[build-dependencies] --bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false } --libbpf-cargo = "0.13.0" -- --[features] --enable_backtrace = [] -diff --git a/tools/sched_ext/atropos/build.rs b/tools/sched_ext/atropos/build.rs -deleted file mode 100644 -index 26e792c5e17e..000000000000 ---- a/tools/sched_ext/atropos/build.rs -+++ /dev/null -@@ -1,70 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --extern crate bindgen; -- --use std::env; --use std::fs::create_dir_all; --use std::path::Path; --use std::path::PathBuf; -- --use libbpf_cargo::SkeletonBuilder; -- --const HEADER_PATH: &str = "src/bpf/atropos.h"; -- --fn bindgen_atropos() { -- // Tell cargo to invalidate the built crate whenever the wrapper changes -- println!("cargo:rerun-if-changed={}", HEADER_PATH); -- -- // The bindgen::Builder is the main entry point -- // to bindgen, and lets you build up options for -- // the resulting bindings. -- let bindings = bindgen::Builder::default() -- // The input header we would like to generate -- // bindings for. -- .header(HEADER_PATH) -- // Tell cargo to invalidate the built crate whenever any of the -- // included header files changed. -- .parse_callbacks(Box::new(bindgen::CargoCallbacks)) -- // Finish the builder and generate the bindings. -- .generate() -- // Unwrap the Result and panic on failure. -- .expect("Unable to generate bindings"); -- -- // Write the bindings to the $OUT_DIR/bindings.rs file. -- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); -- bindings -- .write_to_file(out_path.join("atropos-sys.rs")) -- .expect("Couldn't write bindings!"); --} -- --fn gen_bpf_sched(name: &str) { -- let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap(); -- let clang = env::var("ATROPOS_CLANG").unwrap(); -- eprintln!("{}", clang); -- let outpath = format!("./src/bpf/.output/{}.skel.rs", name); -- let skel = Path::new(&outpath); -- let src = format!("./src/bpf/{}.bpf.c", name); -- SkeletonBuilder::new() -- .source(src.clone()) -- .clang(clang) -- .clang_args(bpf_cflags) -- .build_and_generate(&skel) -- .unwrap(); -- println!("cargo:rerun-if-changed={}", src); --} -- --fn main() { -- bindgen_atropos(); -- // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton. -- // Reasons are because the generated skeleton contains compiler attributes -- // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]` -- // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside -- // the path attribute either (see https://github.com/rust-lang/rust/pull/83366). -- // -- // However, there is hope! When the above feature stabilizes we can clean this -- // all up. -- create_dir_all("./src/bpf/.output").unwrap(); -- gen_bpf_sched("atropos"); --} -diff --git a/tools/sched_ext/atropos/rustfmt.toml b/tools/sched_ext/atropos/rustfmt.toml -deleted file mode 100644 -index b7258ed0a8d8..000000000000 ---- a/tools/sched_ext/atropos/rustfmt.toml -+++ /dev/null -@@ -1,8 +0,0 @@ --# Get help on options with `rustfmt --help=config` --# Please keep these in alphabetical order. --edition = "2021" --group_imports = "StdExternalCrate" --imports_granularity = "Item" --merge_derives = false --use_field_init_shorthand = true --version = "Two" -diff --git a/tools/sched_ext/atropos/src/atropos_sys.rs b/tools/sched_ext/atropos/src/atropos_sys.rs -deleted file mode 100644 -index bbeaf856d40e..000000000000 ---- a/tools/sched_ext/atropos/src/atropos_sys.rs -+++ /dev/null -@@ -1,10 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#![allow(non_upper_case_globals)] --#![allow(non_camel_case_types)] --#![allow(non_snake_case)] --#![allow(dead_code)] -- --include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs")); -diff --git a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -deleted file mode 100644 -index 3905a403e940..000000000000 ---- a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -+++ /dev/null -@@ -1,743 +0,0 @@ --/* Copyright (c) Meta Platforms, Inc. and affiliates. */ --/* -- * This software may be used and distributed according to the terms of the -- * GNU General Public License version 2. -- * -- * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF -- * part does simple round robin in each domain and the userspace part -- * calculates the load factor of each domain and tells the BPF part how to load -- * balance the domains. -- * -- * Every task has an entry in the task_data map which lists which domain the -- * task belongs to. When a task first enters the system (atropos_prep_enable), -- * they are round-robined to a domain. -- * -- * atropos_select_cpu is the primary scheduling logic, invoked when a task -- * becomes runnable. The lb_data map is populated by userspace to inform the BPF -- * scheduler that a task should be migrated to a new domain. Otherwise, the task -- * is scheduled in priority order as follows: -- * * The current core if the task was woken up synchronously and there are idle -- * cpus in the system -- * * The previous core, if idle -- * * The pinned-to core if the task is pinned to a specific core -- * * Any idle cpu in the domain -- * -- * If none of the above conditions are met, then the task is enqueued to a -- * dispatch queue corresponding to the domain (atropos_enqueue). -- * -- * atropos_dispatch will attempt to consume a task from its domain's -- * corresponding dispatch queue (this occurs after scheduling any tasks directly -- * assigned to it due to the logic in atropos_select_cpu). If no task is found, -- * then greedy load stealing will attempt to find a task on another dispatch -- * queue to run. -- * -- * Load balancing is almost entirely handled by userspace. BPF populates the -- * task weight, dom mask and current dom in the task_data map and executes the -- * load balance based on userspace populating the lb_data map. -- */ --#include "../../../scx_common.bpf.h" --#include "atropos.h" -- --#include --#include --#include --#include --#include --#include -- --char _license[] SEC("license") = "GPL"; -- --/* -- * const volatiles are set during initialization and treated as consts by the -- * jit compiler. -- */ -- --/* -- * Domains and cpus -- */ --const volatile __u32 nr_doms = 32; /* !0 for veristat, set during init */ --const volatile __u32 nr_cpus = 64; /* !0 for veristat, set during init */ --const volatile __u32 cpu_dom_id_map[MAX_CPUS]; --const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; -- --const volatile bool kthreads_local; --const volatile bool fifo_sched; --const volatile bool switch_partial; --const volatile __u32 greedy_threshold; -- --/* base slice duration */ --const volatile __u64 slice_us = 20000; -- --/* -- * Exit info -- */ --int exit_type = SCX_EXIT_NONE; --char exit_msg[SCX_EXIT_MSG_LEN]; -- --struct pcpu_ctx { -- __u32 dom_rr_cur; /* used when scanning other doms */ -- -- /* libbpf-rs does not respect the alignment, so pad out the struct explicitly */ -- __u8 _padding[CACHELINE_SIZE - sizeof(u64)]; --} __attribute__((aligned(CACHELINE_SIZE))); -- --struct pcpu_ctx pcpu_ctx[MAX_CPUS]; -- --/* -- * Domain context -- */ --struct dom_ctx { -- struct bpf_cpumask __kptr *cpumask; -- u64 vtime_now; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __type(key, u32); -- __type(value, struct dom_ctx); -- __uint(max_entries, MAX_DOMS); -- __uint(map_flags, 0); --} dom_ctx SEC(".maps"); -- --/* -- * Statistics -- */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, ATROPOS_NR_STATS); --} stats SEC(".maps"); -- --static inline void stat_add(enum stat_idx idx, u64 addend) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p) += addend; --} -- --/* Map pid -> task_ctx */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, struct task_ctx); -- __uint(max_entries, 1000000); -- __uint(map_flags, 0); --} task_data SEC(".maps"); -- --/* -- * This is populated from userspace to indicate which pids should be reassigned -- * to new doms. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, u32); -- __uint(max_entries, 1000); -- __uint(map_flags, 0); --} lb_data SEC(".maps"); -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p, -- u32 new_dom_id) --{ -- struct dom_ctx *old_domc, *new_domc; -- struct bpf_cpumask *d_cpumask, *t_cpumask; -- u32 old_dom_id = task_ctx->dom_id; -- s64 vtime_delta; -- -- old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id); -- if (!old_domc) { -- scx_bpf_error("No dom%u", old_dom_id); -- return false; -- } -- -- vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now; -- -- new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id); -- if (!new_domc) { -- scx_bpf_error("No dom%u", new_dom_id); -- return false; -- } -- -- d_cpumask = new_domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to get domain %u cpumask kptr", -- new_dom_id); -- return false; -- } -- -- t_cpumask = task_ctx->cpumask; -- if (!t_cpumask) { -- scx_bpf_error("Failed to look up task cpumask"); -- return false; -- } -- -- /* -- * set_cpumask might have happened between userspace requesting LB and -- * here and @p might not be able to run in @dom_id anymore. Verify. -- */ -- if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- p->cpus_ptr)) { -- p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta; -- task_ctx->dom_id = new_dom_id; -- bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask, -- p->cpus_ptr); -- } -- -- return task_ctx->dom_id == new_dom_id; --} -- --s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu, -- u32 wake_flags) --{ -- s32 cpu; -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- struct bpf_cpumask *p_cpumask; -- -- if (!task_ctx) -- return -ENOENT; -- -- if (kthreads_local && -- (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- cpu = prev_cpu; -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local dsq of the waker. -- */ -- if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) { -- struct task_struct *current = (void *)bpf_get_current_task(); -- -- if (!(BPF_CORE_READ(current, flags) & PF_EXITING) && -- task_ctx->dom_id < MAX_DOMS) { -- struct dom_ctx *domc; -- struct bpf_cpumask *d_cpumask; -- const struct cpumask *idle_cpumask; -- bool has_idle; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id); -- if (!domc) { -- scx_bpf_error("Failed to find dom%u", -- task_ctx->dom_id); -- return prev_cpu; -- } -- d_cpumask = domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to acquire domain %u cpumask kptr", -- task_ctx->dom_id); -- return prev_cpu; -- } -- -- idle_cpumask = scx_bpf_get_idle_cpumask(); -- -- has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- idle_cpumask); -- -- scx_bpf_put_idle_cpumask(idle_cpumask); -- -- if (has_idle) { -- cpu = bpf_get_smp_processor_id(); -- if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- stat_add(ATROPOS_STAT_WAKE_SYNC, 1); -- goto local; -- } -- } -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- stat_add(ATROPOS_STAT_PREV_IDLE, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- /* If only one core is allowed, dispatch */ -- if (p->nr_cpus_allowed == 1) { -- stat_add(ATROPOS_STAT_PINNED, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) -- return -ENOENT; -- -- /* If there is an eligible idle CPU, dispatch directly */ -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- if (cpu >= 0) { -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * @prev_cpu may be in a different domain. Returning an out-of-domain -- * CPU can lead to stalls as all in-domain CPUs may be idle by the time -- * @p gets enqueued. -- */ -- if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask)) -- cpu = prev_cpu; -- else -- cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask); -- -- return cpu; -- --local: -- task_ctx->dispatch_local = true; -- return cpu; --} -- --void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- u32 *new_dom; -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- new_dom = bpf_map_lookup_elem(&lb_data, &pid); -- if (new_dom && *new_dom != task_ctx->dom_id && -- task_set_dsq(task_ctx, p, *new_dom)) { -- struct bpf_cpumask *p_cpumask; -- s32 cpu; -- -- stat_add(ATROPOS_STAT_LOAD_BALANCE, 1); -- -- /* -- * If dispatch_local is set, We own @p's idle state but we are -- * not gonna put the task in the associated local dsq which can -- * cause the CPU to stall. Kick it. -- */ -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0); -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) { -- scx_bpf_error("Failed to get task_ctx->cpumask"); -- return; -- } -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- } -- -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags); -- return; -- } -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000, -- enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- u32 dom_id = task_ctx->dom_id; -- struct dom_ctx *domc; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, domc->vtime_now - slice_us * 1000)) -- vtime = domc->vtime_now - slice_us * 1000; -- -- scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --static u32 cpu_to_dom_id(s32 cpu) --{ -- const volatile u32 *dom_idp; -- -- if (nr_doms <= 1) -- return 0; -- -- dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]); -- if (!dom_idp) -- return MAX_DOMS; -- -- return *dom_idp; --} -- --static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) --{ -- s32 cpu; -- -- if (dom_id >= MAX_DOMS) -- return false; -- -- bpf_for(cpu, 0, nr_cpus) { -- if (bpf_cpumask_test_cpu(cpu, cpumask) && -- (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64)))) -- return true; -- } -- return false; --} -- --static u32 dom_rr_next(s32 cpu) --{ -- struct pcpu_ctx *pcpuc; -- u32 dom_id; -- -- pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]); -- if (!pcpuc) -- return 0; -- -- dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms; -- -- if (dom_id == cpu_to_dom_id(cpu)) -- dom_id = (dom_id + 1) % nr_doms; -- -- pcpuc->dom_rr_cur = dom_id; -- return dom_id; --} -- --void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 dom = cpu_to_dom_id(cpu); -- -- if (scx_bpf_consume(dom)) { -- stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1); -- return; -- } -- -- if (!greedy_threshold) -- return; -- -- bpf_repeat(nr_doms - 1) { -- u32 dom_id = dom_rr_next(cpu); -- -- if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold && -- scx_bpf_consume(dom_id)) { -- stat_add(ATROPOS_STAT_GREEDY, 1); -- break; -- } -- } --} -- --void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_at = bpf_ktime_get_ns(); --} -- --void BPF_STRUCT_OPS(atropos_running, struct task_struct *p) --{ -- struct task_ctx *taskc; -- struct dom_ctx *domc; -- pid_t pid = p->pid; -- u32 dom_id; -- -- if (fifo_sched) -- return; -- -- taskc = bpf_map_lookup_elem(&task_data, &pid); -- if (!taskc) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- dom_id = taskc->dom_id; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(domc->vtime_now, p->scx.dsq_vtime)) -- domc->vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at; -- task_ctx->runnable_at = 0; --} -- --void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->weight = weight; --} -- --struct pick_task_domain_loop_ctx { -- struct task_struct *p; -- const struct cpumask *cpumask; -- u64 dom_mask; -- u32 dom_rr_base; -- u32 dom_id; --}; -- --static int pick_task_domain_loopfn(u32 idx, void *data) --{ -- struct pick_task_domain_loop_ctx *lctx = data; -- u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms; -- -- if (dom_id >= MAX_DOMS) -- return 1; -- -- if (cpumask_intersects_domain(lctx->cpumask, dom_id)) { -- lctx->dom_mask |= 1LLU << dom_id; -- if (lctx->dom_id == MAX_DOMS) -- lctx->dom_id = dom_id; -- } -- return 0; --} -- --static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- struct pick_task_domain_loop_ctx lctx = { -- .p = p, -- .cpumask = cpumask, -- .dom_id = MAX_DOMS, -- }; -- s32 cpu = bpf_get_smp_processor_id(); -- -- if (cpu < 0 || cpu >= MAX_CPUS) -- return MAX_DOMS; -- -- lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur); -- -- bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0); -- task_ctx->dom_mask = lctx.dom_mask; -- -- return lctx.dom_id; --} -- --static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- u32 dom_id = 0; -- -- if (nr_doms > 1) -- dom_id = pick_task_domain(task_ctx, p, cpumask); -- -- if (!task_set_dsq(task_ctx, p, dom_id)) -- scx_bpf_error("Failed to set domain %d for %s[%d]", -- dom_id, p->comm, p->pid); --} -- --void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_set_domain(task_ctx, p, cpumask); --} -- --s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct bpf_cpumask *cpumask; -- struct task_ctx task_ctx, *map_value; -- long ret; -- pid_t pid; -- -- memset(&task_ctx, 0, sizeof(task_ctx)); -- -- pid = p->pid; -- ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return ret; -- } -- -- /* -- * Read the entry from the map immediately so we can add the cpumask -- * with bpf_kptr_xchg(). -- */ -- map_value = bpf_map_lookup_elem(&task_data, &pid); -- if (!map_value) -- /* Should never happen -- it was just inserted above. */ -- return -EINVAL; -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- bpf_map_delete_elem(&task_data, &pid); -- return -ENOMEM; -- } -- -- cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask); -- if (cpumask) { -- /* Should never happen as we just inserted it above. */ -- bpf_cpumask_release(cpumask); -- bpf_map_delete_elem(&task_data, &pid); -- return -EINVAL; -- } -- -- task_set_domain(map_value, p, p->cpus_ptr); -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p) --{ -- pid_t pid = p->pid; -- long ret = bpf_map_delete_elem(&task_data, &pid); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return; -- } --} -- --static int create_dom_dsq(u32 idx, void *data) --{ -- struct dom_ctx domc_init = {}, *domc; -- struct bpf_cpumask *cpumask; -- u32 cpu, dom_id = idx; -- s32 ret; -- -- ret = scx_bpf_create_dsq(dom_id, -1); -- if (ret < 0) { -- scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret); -- return 1; -- } -- -- ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0); -- if (ret) { -- scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret); -- return 1; -- } -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- /* Should never happen, we just inserted it above. */ -- scx_bpf_error("No dom%u", dom_id); -- return 1; -- } -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id); -- return 1; -- } -- -- for (cpu = 0; cpu < MAX_CPUS; cpu++) { -- const volatile __u64 *dmask; -- -- dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]); -- if (!dmask) { -- scx_bpf_error("array index error"); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- if (*dmask & (1LLU << (cpu % 64))) -- bpf_cpumask_set_cpu(cpu, cpumask); -- } -- -- cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask); -- if (cpumask) { -- scx_bpf_error("Domain %u was already present", dom_id); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(atropos_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- bpf_loop(nr_doms, create_dom_dsq, NULL, 0); -- -- for (u32 i = 0; i < nr_cpus; i++) -- pcpu_ctx[i].dom_rr_cur = i; -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg); -- exit_type = ei->type; --} -- --SEC(".struct_ops") --struct sched_ext_ops atropos = { -- .select_cpu = (void *)atropos_select_cpu, -- .enqueue = (void *)atropos_enqueue, -- .dispatch = (void *)atropos_dispatch, -- .runnable = (void *)atropos_runnable, -- .running = (void *)atropos_running, -- .stopping = (void *)atropos_stopping, -- .quiescent = (void *)atropos_quiescent, -- .set_weight = (void *)atropos_set_weight, -- .set_cpumask = (void *)atropos_set_cpumask, -- .prep_enable = (void *)atropos_prep_enable, -- .disable = (void *)atropos_disable, -- .init = (void *)atropos_init, -- .exit = (void *)atropos_exit, -- .flags = 0, -- .name = "atropos", --}; -diff --git a/tools/sched_ext/atropos/src/bpf/atropos.h b/tools/sched_ext/atropos/src/bpf/atropos.h -deleted file mode 100644 -index addf29ca104a..000000000000 ---- a/tools/sched_ext/atropos/src/bpf/atropos.h -+++ /dev/null -@@ -1,44 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#ifndef __ATROPOS_H --#define __ATROPOS_H -- --#include --#ifndef __kptr --#ifdef __KERNEL__ --#error "__kptr_ref not defined in the kernel" --#endif --#define __kptr --#endif -- --#define MAX_CPUS 512 --#define MAX_DOMS 64 /* limited to avoid complex bitmask ops */ --#define CACHELINE_SIZE 64 -- --/* Statistics */ --enum stat_idx { -- ATROPOS_STAT_TASK_GET_ERR, -- ATROPOS_STAT_WAKE_SYNC, -- ATROPOS_STAT_PREV_IDLE, -- ATROPOS_STAT_PINNED, -- ATROPOS_STAT_DIRECT_DISPATCH, -- ATROPOS_STAT_DSQ_DISPATCH, -- ATROPOS_STAT_GREEDY, -- ATROPOS_STAT_LOAD_BALANCE, -- ATROPOS_STAT_LAST_TASK, -- ATROPOS_NR_STATS, --}; -- --struct task_ctx { -- unsigned long long dom_mask; /* the domains this task can run on */ -- struct bpf_cpumask __kptr *cpumask; -- unsigned int dom_id; -- unsigned int weight; -- unsigned long long runnable_at; -- unsigned long long runnable_for; -- bool dispatch_local; --}; -- --#endif /* __ATROPOS_H */ -diff --git a/tools/sched_ext/atropos/src/main.rs b/tools/sched_ext/atropos/src/main.rs -deleted file mode 100644 -index 0d313662f713..000000000000 ---- a/tools/sched_ext/atropos/src/main.rs -+++ /dev/null -@@ -1,942 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#[path = "bpf/.output/atropos.skel.rs"] --mod atropos; --pub use atropos::*; --pub mod atropos_sys; -- --use std::cell::Cell; --use std::collections::{BTreeMap, BTreeSet}; --use std::ffi::CStr; --use std::ops::Bound::{Included, Unbounded}; --use std::sync::atomic::{AtomicBool, Ordering}; --use std::sync::Arc; --use std::time::{Duration, SystemTime}; -- --use ::fb_procfs as procfs; --use anyhow::{anyhow, bail, Context, Result}; --use bitvec::prelude::*; --use clap::Parser; --use log::{info, trace, warn}; --use ordered_float::OrderedFloat; -- --/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF --/// part does simple round robin in each domain and the userspace part --/// calculates the load factor of each domain and tells the BPF part how to load --/// balance the domains. -- --/// This scheduler demonstrates dividing scheduling logic between BPF and --/// userspace and using rust to build the userspace part. An earlier variant of --/// this scheduler was used to balance across six domains, each representing a --/// chiplet in a six-chiplet AMD processor, and could match the performance of --/// production setup using CFS. --#[derive(Debug, Parser)] --struct Opts { -- /// Scheduling slice duration in microseconds. -- #[clap(short, long, default_value = "20000")] -- slice_us: u64, -- -- /// Monitoring and load balance interval in seconds. -- #[clap(short, long, default_value = "2.0")] -- interval: f64, -- -- /// Build domains according to how CPUs are grouped at this cache level -- /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id. -- #[clap(short = 'c', long, default_value = "3")] -- cache_level: u32, -- -- /// Instead of using cache locality, set the cpumask for each domain -- /// manually, provide multiple --cpumasks, one for each domain. E.g. -- /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with -- /// the corresponding CPUs belonging to each domain. Each CPU must -- /// belong to precisely one domain. -- #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")] -- cpumasks: Vec, -- -- /// When non-zero, enable greedy task stealing. When a domain is idle, a -- /// cpu will attempt to steal tasks from a domain with at least -- /// greedy_threshold tasks enqueued. These tasks aren't permanently -- /// stolen from the domain. -- #[clap(short, long, default_value = "4")] -- greedy_threshold: u32, -- -- /// The load decay factor. Every interval, the existing load is decayed -- /// by this factor and new load is added. Must be in the range [0.0, -- /// 0.99]. The smaller the value, the more sensitive load calculation -- /// is to recent changes. When 0.0, history is ignored and the load -- /// value from the latest period is used directly. -- #[clap(short, long, default_value = "0.5")] -- load_decay_factor: f64, -- -- /// Disable load balancing. Unless disabled, periodically userspace will -- /// calculate the load factor of each domain and instruct BPF which -- /// processes to move. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- no_load_balance: bool, -- -- /// Put per-cpu kthreads directly into local dsq's. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- kthreads_local: bool, -- -- /// Use FIFO scheduling instead of weighted vtime scheduling. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- fifo_sched: bool, -- -- /// If specified, only tasks which have their scheduling policy set to -- /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all -- /// tasks are switched. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- partial: bool, -- -- /// Enable verbose output including libbpf details. Specify multiple -- /// times to increase verbosity. -- #[clap(short, long, action = clap::ArgAction::Count)] -- verbose: u8, --} -- --fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result { -- Ok(reader -- .read_stat() -- .context("Failed to read procfs")? -- .total_cpu -- .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?) --} -- --fn now_monotonic() -> u64 { -- let mut time = libc::timespec { -- tv_sec: 0, -- tv_nsec: 0, -- }; -- let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) }; -- assert!(ret == 0); -- time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 --} -- --fn clear_map(map: &mut libbpf_rs::Map) { -- // XXX: libbpf_rs has some design flaw that make it impossible to -- // delete while iterating despite it being safe so we alias it here -- let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) }; -- for key in map.keys() { -- let _ = deleter.delete(&key); -- } --} -- --#[derive(Debug)] --struct TaskLoad { -- runnable_for: u64, -- load: f64, --} -- --#[derive(Debug)] --struct TaskInfo { -- pid: i32, -- dom_mask: u64, -- migrated: Cell, --} -- --struct LoadBalancer<'a, 'b, 'c> { -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- -- tasks_by_load: Vec, TaskInfo>>, -- load_avg: f64, -- dom_loads: Vec, -- -- imbal: Vec, -- doms_to_push: BTreeMap, u32>, -- doms_to_pull: BTreeMap, u32>, -- -- nr_lb_data_errors: &'c mut u64, --} -- --impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> { -- const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10; -- const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1; -- const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50; -- -- fn new( -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- nr_lb_data_errors: &'c mut u64, -- ) -> Self { -- Self { -- maps, -- task_loads, -- nr_doms, -- load_decay_factor, -- -- tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(), -- load_avg: 0f64, -- dom_loads: vec![0.0; nr_doms], -- -- imbal: vec![0.0; nr_doms], -- doms_to_pull: BTreeMap::new(), -- doms_to_push: BTreeMap::new(), -- -- nr_lb_data_errors, -- } -- } -- -- fn read_task_loads(&mut self, period: Duration) -> Result<()> { -- let now_mono = now_monotonic(); -- let task_data = self.maps.task_data(); -- let mut this_task_loads = BTreeMap::::new(); -- let mut load_sum = 0.0f64; -- self.dom_loads = vec![0f64; self.nr_doms]; -- -- for key in task_data.keys() { -- if let Some(task_ctx_vec) = task_data -- .lookup(&key, libbpf_rs::MapFlags::ANY) -- .context("Failed to lookup task_data")? -- { -- let task_ctx = -- unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) }; -- let pid = i32::from_ne_bytes( -- key.as_slice() -- .try_into() -- .context("Invalid key length in task_data map")?, -- ); -- -- let (this_at, this_for, weight) = unsafe { -- ( -- std::ptr::read_volatile(&task_ctx.runnable_at as *const u64), -- std::ptr::read_volatile(&task_ctx.runnable_for as *const u64), -- std::ptr::read_volatile(&task_ctx.weight as *const u32), -- ) -- }; -- -- let (mut delta, prev_load) = match self.task_loads.get(&pid) { -- Some(prev) => (this_for - prev.runnable_for, Some(prev.load)), -- None => (this_for, None), -- }; -- -- // Non-zero this_at indicates that the task is currently -- // runnable. Note that we read runnable_at and runnable_for -- // without any synchronization and there is a small window -- // where we end up misaccounting. While this can cause -- // temporary error, it's unlikely to cause any noticeable -- // misbehavior especially given the load value clamping. -- if this_at > 0 && this_at < now_mono { -- delta += now_mono - this_at; -- } -- -- delta = delta.min(period.as_nanos() as u64); -- let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64) -- .clamp(0.0, weight as f64); -- -- let this_load = match prev_load { -- Some(prev_load) => { -- prev_load * self.load_decay_factor -- + this_load * (1.0 - self.load_decay_factor) -- } -- None => this_load, -- }; -- -- this_task_loads.insert( -- pid, -- TaskLoad { -- runnable_for: this_for, -- load: this_load, -- }, -- ); -- -- load_sum += this_load; -- self.dom_loads[task_ctx.dom_id as usize] += this_load; -- // Only record pids that are eligible for load balancing -- if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) { -- continue; -- } -- self.tasks_by_load[task_ctx.dom_id as usize].insert( -- OrderedFloat(this_load), -- TaskInfo { -- pid, -- dom_mask: task_ctx.dom_mask, -- migrated: Cell::new(false), -- }, -- ); -- } -- } -- -- self.load_avg = load_sum / self.nr_doms as f64; -- *self.task_loads = this_task_loads; -- Ok(()) -- } -- -- // To balance dom loads we identify doms with lower and higher load than average -- fn calculate_dom_load_balance(&mut self) -> Result<()> { -- for (dom, dom_load) in self.dom_loads.iter().enumerate() { -- let imbal = dom_load - self.load_avg; -- if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO { -- if imbal > 0f64 { -- self.doms_to_push.insert(OrderedFloat(imbal), dom as u32); -- } else { -- self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32); -- } -- self.imbal[dom] = imbal; -- } -- } -- Ok(()) -- } -- -- // Find the first candidate pid which hasn't already been migrated and -- // can run in @pull_dom. -- fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)> -- where -- I: IntoIterator, &'d TaskInfo)>, -- { -- match tasks_by_load -- .into_iter() -- .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0) -- .next() -- { -- Some((OrderedFloat(load), task)) => Some((*load, task)), -- None => None, -- } -- } -- -- fn pick_victim( -- &self, -- (push_dom, to_push): (u32, f64), -- (pull_dom, to_pull): (u32, f64), -- ) -> Option<(&TaskInfo, f64)> { -- let to_xfer = to_pull.min(to_push); -- -- trace!( -- "considering dom {}@{:.2} -> {}@{:.2}", -- push_dom, -- to_push, -- pull_dom, -- to_pull -- ); -- -- let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs(); -- -- trace!( -- "to_xfer={:.2} tasks_by_load={:?}", -- to_xfer, -- &self.tasks_by_load[push_dom as usize] -- ); -- -- // We want to pick a task to transfer from push_dom to pull_dom to -- // maximize the reduction of load imbalance between the two. IOW, -- // pick a task which has the closest load value to $to_xfer that can -- // be migrated. Find such task by locating the first migratable task -- // while scanning left from $to_xfer and the counterpart while -- // scanning right and picking the better of the two. -- let (load, task, new_imbal) = match ( -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Unbounded, Included(&OrderedFloat(to_xfer)))) -- .rev(), -- pull_dom, -- ), -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Included(&OrderedFloat(to_xfer)), Unbounded)), -- pull_dom, -- ), -- ) { -- (None, None) => return None, -- (Some((load, task)), None) | (None, Some((load, task))) => { -- (load, task, calc_new_imbal(load)) -- } -- (Some((load0, task0)), Some((load1, task1))) => { -- let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1)); -- if new_imbal0 <= new_imbal1 { -- (load0, task0, new_imbal0) -- } else { -- (load1, task1, new_imbal1) -- } -- } -- }; -- -- // If the best candidate can't reduce the imbalance, there's nothing -- // to do for this pair. -- let old_imbal = to_push + to_pull; -- if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal { -- trace!( -- "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal -- ); -- return None; -- } -- -- trace!( -- "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal, -- ); -- -- Some((task, load)) -- } -- -- // Actually execute the load balancing. Concretely this writes pid -> dom -- // entries into the lb_data map for bpf side to consume. -- fn load_balance(&mut self) -> Result<()> { -- clear_map(self.maps.lb_data()); -- -- trace!("imbal={:?}", &self.imbal); -- trace!("doms_to_push={:?}", &self.doms_to_push); -- trace!("doms_to_pull={:?}", &self.doms_to_pull); -- -- // Push from the most imbalanced to least. -- while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() { -- let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO; -- let mut pushed = 0f64; -- -- // Transfer tasks from push_dom to reduce imbalance. -- loop { -- let last_pushed = pushed; -- -- // Pull from the most imbalaned to least. -- let mut doms_to_pull = BTreeMap::<_, _>::new(); -- std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull); -- let mut pull_doms = doms_to_pull.into_iter().rev().collect::>(); -- -- for (to_pull, pull_dom) in pull_doms.iter_mut() { -- if let Some((task, load)) = -- self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull))) -- { -- // Execute migration. -- task.migrated.set(true); -- to_push -= load; -- *to_pull -= load; -- pushed += load; -- -- // Ask BPF code to execute the migration. -- let pid = task.pid; -- let cpid = (pid as libc::pid_t).to_ne_bytes(); -- if let Err(e) = self.maps.lb_data().update( -- &cpid, -- &pull_dom.to_ne_bytes(), -- libbpf_rs::MapFlags::NO_EXIST, -- ) { -- warn!( -- "Failed to update lb_data map for pid={} error={:?}", -- pid, &e -- ); -- *self.nr_lb_data_errors += 1; -- } -- -- // Always break after a successful migration so that -- // the pulling domains are always considered in the -- // descending imbalance order. -- break; -- } -- } -- -- pull_doms -- .into_iter() -- .map(|(k, v)| self.doms_to_pull.insert(k, v)) -- .count(); -- -- // Stop repeating if nothing got transferred or pushed enough. -- if pushed == last_pushed || pushed >= push_max { -- break; -- } -- } -- } -- Ok(()) -- } --} -- --struct Scheduler<'a> { -- skel: AtroposSkel<'a>, -- struct_ops: Option, -- -- nr_cpus: usize, -- nr_doms: usize, -- load_decay_factor: f64, -- balance_load: bool, -- -- proc_reader: procfs::ProcReader, -- -- prev_at: SystemTime, -- prev_total_cpu: procfs::CpuStat, -- task_loads: BTreeMap, -- -- nr_lb_data_errors: u64, --} -- --impl<'a> Scheduler<'a> { -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn parse_cpusets( -- cpumasks: &[String], -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- if cpumasks.len() > atropos_sys::MAX_DOMS as usize { -- bail!( -- "Number of requested DSQs ({}) is greater than MAX_DOMS ({})", -- cpumasks.len(), -- atropos_sys::MAX_DOMS -- ); -- } -- let mut cpus = vec![-1i32; nr_cpus]; -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()]; -- for (dq, cpumask) in cpumasks.iter().enumerate() { -- let hex_str = { -- let mut tmp_str = cpumask -- .strip_prefix("0x") -- .unwrap_or(cpumask) -- .replace('_', ""); -- if tmp_str.len() % 2 != 0 { -- tmp_str = "0".to_string() + &tmp_str; -- } -- tmp_str -- }; -- let byte_vec = hex::decode(&hex_str) -- .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?; -- -- for (index, &val) in byte_vec.iter().rev().enumerate() { -- let mut v = val; -- while v != 0 { -- let lsb = v.trailing_zeros() as usize; -- v &= !(1 << lsb); -- let cpu = index * 8 + lsb; -- if cpu > nr_cpus { -- bail!( -- concat!( -- "Found cpu ({}) in cpumask ({}) which is larger", -- " than the number of cpus on the machine ({})" -- ), -- cpu, -- cpumask, -- nr_cpus -- ); -- } -- if cpus[cpu] != -1 { -- bail!( -- "Found cpu ({}) with dq ({}) but also in cpumask ({})", -- cpu, -- cpus[cpu], -- cpumask -- ); -- } -- cpus[cpu] = dq as i32; -- cpusets[dq].set(cpu, true); -- } -- } -- cpusets[dq].set_uninitialized(false); -- } -- -- for (cpu, &dq) in cpus.iter().enumerate() { -- if dq < 0 { -- bail!( -- "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.", -- cpu -- ); -- } -- } -- -- Ok((cpusets, cpus)) -- } -- -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn cpusets_from_cache( -- level: u32, -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- let mut cpu_to_cache = vec![]; // (cpu_id, cache_id) -- let mut cache_ids = BTreeSet::::new(); -- let mut nr_not_found = 0; -- -- // Build cpu -> cache ID mapping. -- for cpu in 0..nr_cpus { -- let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level); -- let id = match std::fs::read_to_string(&path) { -- Ok(val) => val -- .trim() -- .parse::() -- .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?, -- Err(e) if e.kind() == std::io::ErrorKind::NotFound => { -- nr_not_found += 1; -- 0 -- } -- Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)), -- }; -- -- cpu_to_cache.push(id); -- cache_ids.insert(id); -- } -- -- if nr_not_found > 1 { -- warn!( -- "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0", -- level, nr_not_found, nr_cpus -- ); -- } -- -- // Cache IDs may have holes. Assign consecutive domain IDs to -- // existing cache IDs. -- let mut cache_to_dom = BTreeMap::::new(); -- let mut nr_doms = 0; -- for cache_id in cache_ids.iter() { -- cache_to_dom.insert(*cache_id, nr_doms); -- nr_doms += 1; -- } -- -- if nr_doms > atropos_sys::MAX_DOMS { -- bail!( -- "Total number of doms {} is greater than MAX_DOMS ({})", -- nr_doms, -- atropos_sys::MAX_DOMS -- ); -- } -- -- // Build and return dom -> cpumask and cpu -> dom mappings. -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize]; -- let mut cpu_to_dom = vec![]; -- -- for cpu in 0..nr_cpus { -- let dom_id = cache_to_dom[&cpu_to_cache[cpu]]; -- cpusets[dom_id as usize].set(cpu, true); -- cpu_to_dom.push(dom_id as i32); -- } -- -- Ok((cpusets, cpu_to_dom)) -- } -- -- fn init(opts: &Opts) -> Result { -- // Open the BPF prog first for verification. -- let mut skel_builder = AtroposSkelBuilder::default(); -- skel_builder.obj_builder.debug(opts.verbose > 0); -- let mut skel = skel_builder.open().context("Failed to open BPF program")?; -- -- let nr_cpus = libbpf_rs::num_possible_cpus().unwrap(); -- if nr_cpus > atropos_sys::MAX_CPUS as usize { -- bail!( -- "nr_cpus ({}) is greater than MAX_CPUS ({})", -- nr_cpus, -- atropos_sys::MAX_CPUS -- ); -- } -- -- // Initialize skel according to @opts. -- let (cpusets, cpus) = if opts.cpumasks.len() > 0 { -- Self::parse_cpusets(&opts.cpumasks, nr_cpus)? -- } else { -- Self::cpusets_from_cache(opts.cache_level, nr_cpus)? -- }; -- let nr_doms = cpusets.len(); -- skel.rodata().nr_doms = nr_doms as u32; -- skel.rodata().nr_cpus = nr_cpus as u32; -- -- for (cpu, dom) in cpus.iter().enumerate() { -- skel.rodata().cpu_dom_id_map[cpu] = *dom as u32; -- } -- -- for (dom, cpuset) in cpusets.iter().enumerate() { -- let raw_cpuset_slice = cpuset.as_raw_slice(); -- let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom]; -- let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len()); -- left.clone_from_slice(cpuset.as_raw_slice()); -- let cpumask_str = dom_cpumask_slice -- .iter() -- .take((nr_cpus + 63) / 64) -- .rev() -- .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x)); -- info!( -- "DOM[{:02}] cpumask{} ({} cpus)", -- dom, -- &cpumask_str, -- cpuset.count_ones() -- ); -- } -- -- skel.rodata().slice_us = opts.slice_us; -- skel.rodata().kthreads_local = opts.kthreads_local; -- skel.rodata().fifo_sched = opts.fifo_sched; -- skel.rodata().switch_partial = opts.partial; -- skel.rodata().greedy_threshold = opts.greedy_threshold; -- -- // Attach. -- let mut skel = skel.load().context("Failed to load BPF program")?; -- skel.attach().context("Failed to attach BPF program")?; -- let struct_ops = Some( -- skel.maps_mut() -- .atropos() -- .attach_struct_ops() -- .context("Failed to attach atropos struct ops")?, -- ); -- info!("Atropos Scheduler Attached"); -- -- // Other stuff. -- let mut proc_reader = procfs::ProcReader::new(); -- let prev_total_cpu = read_total_cpu(&mut proc_reader)?; -- -- Ok(Self { -- skel, -- struct_ops, // should be held to keep it attached -- -- nr_cpus, -- nr_doms, -- load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99), -- balance_load: !opts.no_load_balance, -- -- proc_reader, -- -- prev_at: SystemTime::now(), -- prev_total_cpu, -- task_loads: BTreeMap::new(), -- -- nr_lb_data_errors: 0, -- }) -- } -- -- fn get_cpu_busy(&mut self) -> Result { -- let total_cpu = read_total_cpu(&mut self.proc_reader)?; -- let busy = match (&self.prev_total_cpu, &total_cpu) { -- ( -- procfs::CpuStat { -- user_usec: Some(prev_user), -- nice_usec: Some(prev_nice), -- system_usec: Some(prev_system), -- idle_usec: Some(prev_idle), -- iowait_usec: Some(prev_iowait), -- irq_usec: Some(prev_irq), -- softirq_usec: Some(prev_softirq), -- stolen_usec: Some(prev_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- procfs::CpuStat { -- user_usec: Some(curr_user), -- nice_usec: Some(curr_nice), -- system_usec: Some(curr_system), -- idle_usec: Some(curr_idle), -- iowait_usec: Some(curr_iowait), -- irq_usec: Some(curr_irq), -- softirq_usec: Some(curr_softirq), -- stolen_usec: Some(curr_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- ) => { -- let idle_usec = curr_idle - prev_idle; -- let iowait_usec = curr_iowait - prev_iowait; -- let user_usec = curr_user - prev_user; -- let system_usec = curr_system - prev_system; -- let nice_usec = curr_nice - prev_nice; -- let irq_usec = curr_irq - prev_irq; -- let softirq_usec = curr_softirq - prev_softirq; -- let stolen_usec = curr_stolen - prev_stolen; -- -- let busy_usec = -- user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec; -- let total_usec = idle_usec + busy_usec + iowait_usec; -- busy_usec as f64 / total_usec as f64 -- } -- _ => { -- bail!("Some procfs stats are not populated!"); -- } -- }; -- -- self.prev_total_cpu = total_cpu; -- Ok(busy) -- } -- -- fn read_bpf_stats(&mut self) -> Result> { -- let mut maps = self.skel.maps_mut(); -- let stats_map = maps.stats(); -- let mut stats: Vec = Vec::new(); -- let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus]; -- -- for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS { -- let cpu_stat_vec = stats_map -- .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY) -- .with_context(|| format!("Failed to lookup stat {}", stat))? -- .expect("per-cpu stat should exist"); -- let sum = cpu_stat_vec -- .iter() -- .map(|val| { -- u64::from_ne_bytes( -- val.as_slice() -- .try_into() -- .expect("Invalid value length in stat map"), -- ) -- }) -- .sum(); -- stats_map -- .update_percpu( -- &(stat as u32).to_ne_bytes(), -- &zero_vec, -- libbpf_rs::MapFlags::ANY, -- ) -- .context("Failed to zero stat")?; -- stats.push(sum); -- } -- Ok(stats) -- } -- -- fn report( -- &self, -- stats: &Vec, -- cpu_busy: f64, -- processing_dur: Duration, -- load_avg: f64, -- dom_loads: &Vec, -- imbal: &Vec, -- ) { -- let stat = |idx| stats[idx as usize]; -- let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK); -- -- info!( -- "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms", -- cpu_busy * 100.0, -- load_avg, -- stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize], -- stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize], -- self.nr_lb_data_errors, -- processing_dur.as_millis(), -- ); -- -- let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0; -- -- info!( -- "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}", -- total, -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY), -- ); -- -- for i in 0..self.nr_doms { -- info!( -- "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}", -- i, -- dom_loads[i], -- if imbal[i] < 0.0 { -imbal[i] } else { 0.0 }, -- if imbal[i] > 0.0 { imbal[i] } else { 0.0 }, -- ); -- } -- } -- -- fn step(&mut self) -> Result<()> { -- let started_at = std::time::SystemTime::now(); -- let bpf_stats = self.read_bpf_stats()?; -- let cpu_busy = self.get_cpu_busy()?; -- -- let mut lb = LoadBalancer::new( -- self.skel.maps_mut(), -- &mut self.task_loads, -- self.nr_doms, -- self.load_decay_factor, -- &mut self.nr_lb_data_errors, -- ); -- -- lb.read_task_loads(started_at.duration_since(self.prev_at)?)?; -- lb.calculate_dom_load_balance()?; -- -- if self.balance_load { -- lb.load_balance()?; -- } -- -- // Extract fields needed for reporting and drop lb to release -- // mutable borrows. -- let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal); -- -- self.report( -- &bpf_stats, -- cpu_busy, -- std::time::SystemTime::now().duration_since(started_at)?, -- load_avg, -- &dom_loads, -- &imbal, -- ); -- -- self.prev_at = started_at; -- Ok(()) -- } -- -- fn read_bpf_exit_type(&mut self) -> i32 { -- unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) } -- } -- -- fn report_bpf_exit_type(&mut self) -> Result<()> { -- // Report msg if EXT_OPS_EXIT_ERROR. -- match self.read_bpf_exit_type() { -- 0 => Ok(()), -- etype if etype == 2 => { -- let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) }; -- let msg = cstr -- .to_str() -- .context("Failed to convert exit msg to string") -- .unwrap(); -- bail!("BPF exit_type={} msg={}", etype, msg); -- } -- etype => { -- info!("BPF exit_type={}", etype); -- Ok(()) -- } -- } -- } --} -- --impl<'a> Drop for Scheduler<'a> { -- fn drop(&mut self) { -- if let Some(struct_ops) = self.struct_ops.take() { -- drop(struct_ops); -- } -- } --} -- --fn main() -> Result<()> { -- let opts = Opts::parse(); -- -- let llv = match opts.verbose { -- 0 => simplelog::LevelFilter::Info, -- 1 => simplelog::LevelFilter::Debug, -- _ => simplelog::LevelFilter::Trace, -- }; -- let mut lcfg = simplelog::ConfigBuilder::new(); -- lcfg.set_time_level(simplelog::LevelFilter::Error) -- .set_location_level(simplelog::LevelFilter::Off) -- .set_target_level(simplelog::LevelFilter::Off) -- .set_thread_level(simplelog::LevelFilter::Off); -- simplelog::TermLogger::init( -- llv, -- lcfg.build(), -- simplelog::TerminalMode::Stderr, -- simplelog::ColorChoice::Auto, -- )?; -- -- let shutdown = Arc::new(AtomicBool::new(false)); -- let shutdown_clone = shutdown.clone(); -- ctrlc::set_handler(move || { -- shutdown_clone.store(true, Ordering::Relaxed); -- }) -- .context("Error setting Ctrl-C handler")?; -- -- let mut sched = Scheduler::init(&opts)?; -- -- while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 { -- std::thread::sleep(Duration::from_secs_f64(opts.interval)); -- sched.step()?; -- } -- -- sched.report_bpf_exit_type() --} -diff --git a/tools/sched_ext/gnu/stubs.h b/tools/sched_ext/gnu/stubs.h -deleted file mode 100644 -index 719225b16626..000000000000 ---- a/tools/sched_ext/gnu/stubs.h -+++ /dev/null -@@ -1 +0,0 @@ --/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ -diff --git a/tools/sched_ext/scx_common.bpf.h b/tools/sched_ext/scx_common.bpf.h -deleted file mode 100644 -index e56de9dc86f2..000000000000 ---- a/tools/sched_ext/scx_common.bpf.h -+++ /dev/null -@@ -1,288 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __SCHED_EXT_COMMON_BPF_H --#define __SCHED_EXT_COMMON_BPF_H -- --#include "vmlinux.h" --#include --#include --#include --#include "user_exit_info.h" -- --#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ --#define PF_EXITING 0x00000004 --#define CLOCK_MONOTONIC 1 -- --/* -- * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can -- * lead to really confusing misbehaviors. Let's trigger a build failure. -- */ --static inline void ___vmlinux_h_sanity_check___(void) --{ -- _Static_assert(SCX_DSQ_FLAG_BUILTIN, -- "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); --} -- --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; -- --static inline __attribute__((format(printf, 1, 2))) --void ___scx_bpf_error_format_checker(const char *fmt, ...) {} -- --/* -- * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments -- * instead of an array of u64. Note that __param[] must have at least one -- * element to keep the verifier happy. -- */ --#define scx_bpf_error(fmt, args...) \ --({ \ -- static char ___fmt[] = fmt; \ -- unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ -- \ -- _Pragma("GCC diagnostic push") \ -- _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ -- ___bpf_fill(___param, args); \ -- _Pragma("GCC diagnostic pop") \ -- \ -- scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ -- \ -- ___scx_bpf_error_format_checker(fmt, ##args); \ --}) -- --void scx_bpf_switch_all(void) __ksym; --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; --bool scx_bpf_consume(u64 dsq_id) __ksym; --u32 scx_bpf_dispatch_nr_slots(void) __ksym; --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym; --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym; --void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; --s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym; --const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; --const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; --void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; --void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; --bool scx_bpf_task_running(const struct task_struct *p) __ksym; --s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym; --u32 scx_bpf_reenqueue_local(void) __ksym; -- --#define BPF_STRUCT_OPS(name, args...) \ --SEC("struct_ops/"#name) \ --BPF_PROG(name, ##args) -- --#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ --SEC("struct_ops.s/"#name) \ --BPF_PROG(name, ##args) -- --/** -- * MEMBER_VPTR - Obtain the verified pointer to a struct or array member -- * @base: struct or array to index -- * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...) -- * -- * The verifier often gets confused by the instruction sequence the compiler -- * generates for indexing struct fields or arrays. This macro forces the -- * compiler to generate a code sequence which first calculates the byte offset, -- * checks it against the struct or array size and add that byte offset to -- * generate the pointer to the member to help the verifier. -- * -- * Ideally, we want to abort if the calculated offset is out-of-bounds. However, -- * BPF currently doesn't support abort, so evaluate to NULL instead. The caller -- * must check for NULL and take appropriate action to appease the verifier. To -- * avoid confusing the verifier, it's best to check for NULL and dereference -- * immediately. -- * -- * vptr = MEMBER_VPTR(my_array, [i][j]); -- * if (!vptr) -- * return error; -- * *vptr = new_value; -- */ --#define MEMBER_VPTR(base, member) (typeof(base member) *)({ \ -- u64 __base = (u64)base; \ -- u64 __addr = (u64)&(base member) - __base; \ -- asm volatile ( \ -- "if %0 <= %[max] goto +2\n" \ -- "%0 = 0\n" \ -- "goto +1\n" \ -- "%0 += %1\n" \ -- : "+r"(__addr) \ -- : "r"(__base), \ -- [max]"i"(sizeof(base) - sizeof(base member))); \ -- __addr; \ --}) -- --/* -- * BPF core and other generic helpers -- */ -- --/* list and rbtree */ --#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) --#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) -- --void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; --void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; -- --#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) --#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) -- --void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; --struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; --struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, -- struct bpf_rb_node *node) __ksym; --void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, -- bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym; --struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; -- --/* task */ --struct task_struct *bpf_task_from_pid(s32 pid) __ksym; --struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; --void bpf_task_release(struct task_struct *p) __ksym; -- --/* cgroup */ --struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; --void bpf_cgroup_release(struct cgroup *cgrp) __ksym; --struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; -- --/* cpumask */ --struct bpf_cpumask *bpf_cpumask_create(void) __ksym; --struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; --u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; --u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym; -- --/* rcu */ --void bpf_rcu_read_lock(void) __ksym; --void bpf_rcu_read_unlock(void) __ksym; -- --/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */ --struct bpf_iter_num; -- --extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym; --extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym; --extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym; -- --#ifndef bpf_for_each --/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for -- * using BPF open-coded iterators without having to write mundane explicit -- * low-level loop logic. Instead, it provides for()-like generic construct -- * that can be used pretty naturally. E.g., for some hypothetical cgroup -- * iterator, you'd write: -- * -- * struct cgroup *cg, *parent_cg = <...>; -- * -- * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { -- * bpf_printk("Child cgroup id = %d", cg->cgroup_id); -- * if (cg->cgroup_id == 123) -- * break; -- * } -- * -- * I.e., it looks almost like high-level for each loop in other languages, -- * supports continue/break, and is verifiable by BPF verifier. -- * -- * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) -- * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to -- * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int -- * *`, not just `int`. So for integers bpf_for() is more convenient. -- * -- * Note: this macro relies on C99 feature of allowing to declare variables -- * inside for() loop, bound to for() loop lifetime. It also utilizes GCC -- * extension: __attribute__((cleanup())), supported by both GCC and -- * Clang. -- */ --#define bpf_for_each(type, cur, args...) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ -- cleanup(bpf_iter_##type##_destroy))), \ -- /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_##type##_new(&___it, ##args), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_##type##_destroy, (void *)0); \ -- /* iteration and termination check */ \ -- (((cur) = bpf_iter_##type##_next(&___it))); \ --) --#endif /* bpf_for_each */ -- --#ifndef bpf_for --/* bpf_for(i, start, end) implements a for()-like looping construct that sets -- * provided integer variable *i* to values starting from *start* through, -- * but not including, *end*. It also proves to BPF verifier that *i* belongs -- * to range [start, end), so this can be used for accessing arrays without -- * extra checks. -- * -- * Note: *start* and *end* are assumed to be expressions with no side effects -- * and whose values do not change throughout bpf_for() loop execution. They do -- * not have to be statically known or constant, though. -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_for(i, start, end) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, (start), (end)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- ({ \ -- /* iteration step */ \ -- int *___t = bpf_iter_num_next(&___it); \ -- /* termination and bounds check */ \ -- (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ -- }); \ --) --#endif /* bpf_for */ -- --#ifndef bpf_repeat --/* bpf_repeat(N) performs N iterations without exposing iteration number -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_repeat(N) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, 0, (N)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- bpf_iter_num_next(&___it); \ -- /* nothing here */ \ --) --#endif /* bpf_repeat */ -- --#endif /* __SCHED_EXT_COMMON_BPF_H */ -diff --git a/tools/sched_ext/scx_example_central.bpf.c b/tools/sched_ext/scx_example_central.bpf.c -deleted file mode 100644 -index 4cec04b4c2ed..000000000000 ---- a/tools/sched_ext/scx_example_central.bpf.c -+++ /dev/null -@@ -1,334 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A central FIFO sched_ext scheduler which demonstrates the followings: -- * -- * a. Making all scheduling decisions from one CPU: -- * -- * The central CPU is the only one making scheduling decisions. All other -- * CPUs kick the central CPU when they run out of tasks to run. -- * -- * There is one global BPF queue and the central CPU schedules all CPUs by -- * dispatching from the global queue to each CPU's local dsq from dispatch(). -- * This isn't the most straightforward. e.g. It'd be easier to bounce -- * through per-CPU BPF queues. The current design is chosen to maximally -- * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching. -- * -- * b. Tickless operation -- * -- * All tasks are dispatched with the infinite slice which allows stopping the -- * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full -- * parameter. The tickless operation can be observed through -- * /proc/interrupts. -- * -- * Periodic switching is enforced by a periodic timer checking all CPUs and -- * preempting them as necessary. Unfortunately, BPF timer currently doesn't -- * have a way to pin to a specific CPU, so the periodic timer isn't pinned to -- * the central CPU. -- * -- * c. Preemption -- * -- * Kthreads are unconditionally queued to the head of a matching local dsq -- * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always -- * prioritized over user threads, which is required for ensuring forward -- * progress as e.g. the periodic timer may run on a ksoftirqd and if the -- * ksoftirqd gets starved by a user thread, there may not be anything else to -- * vacate that user thread. -- * -- * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the -- * next tasks. -- * -- * This scheduler is designed to maximize usage of various SCX mechanisms. A -- * more practical implementation would likely put the scheduling loop outside -- * the central CPU's dispatch() path and add some form of priority mechanism. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --enum { -- FALLBACK_DSQ_ID = 0, -- MAX_CPUS = 4096, -- MS_TO_NS = 1000LLU * 1000, -- TIMER_INTERVAL_NS = 1 * MS_TO_NS, --}; -- --const volatile bool switch_partial; --const volatile s32 central_cpu; --const volatile u32 nr_cpu_ids = 64; /* !0 for veristat, set during init */ -- --u64 nr_total, nr_locals, nr_queued, nr_lost_pids; --u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries; --u64 nr_overflows; -- --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, s32); --} central_q SEC(".maps"); -- --/* can't use percpu map due to bad lookups */ --static bool cpu_gimme_task[MAX_CPUS]; --static u64 cpu_started_at[MAX_CPUS]; -- --struct central_timer { -- struct bpf_timer timer; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, 1); -- __type(key, u32); -- __type(value, struct central_timer); --} central_timer SEC(".maps"); -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- /* -- * Steer wakeups to the central CPU as much as possible to avoid -- * disturbing other CPUs. It's safe to blindly return the central cpu as -- * select_cpu() is a hint and if @p can't be on it, the kernel will -- * automatically pick a fallback CPU. -- */ -- return central_cpu; --} -- --void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags) --{ -- s32 pid = p->pid; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- /* -- * Push per-cpu kthreads at the head of local dsq's and preempt the -- * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked -- * behind other threads which is necessary for forward progress -- * guarantee as we depend on the BPF timer which may run from ksoftirqd. -- */ -- if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- __sync_fetch_and_add(&nr_locals, 1); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF, -- enq_flags | SCX_ENQ_PREEMPT); -- return; -- } -- -- if (bpf_map_push_elem(¢ral_q, &pid, 0)) { -- __sync_fetch_and_add(&nr_overflows, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_queued, 1); -- -- if (!scx_bpf_task_running(p)) -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); --} -- --static bool dispatch_to_cpu(s32 cpu) --{ -- struct task_struct *p; -- s32 pid; -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (bpf_map_pop_elem(¢ral_q, &pid)) -- break; -- -- __sync_fetch_and_sub(&nr_queued, 1); -- -- p = bpf_task_from_pid(pid); -- if (!p) { -- __sync_fetch_and_add(&nr_lost_pids, 1); -- continue; -- } -- -- /* -- * If we can't run the task at the top, do the dumb thing and -- * bounce it to the fallback dsq. -- */ -- if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- __sync_fetch_and_add(&nr_mismatches, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0); -- bpf_task_release(p); -- continue; -- } -- -- /* dispatch to local and mark that @cpu doesn't need more */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0); -- -- if (cpu != central_cpu) -- scx_bpf_kick_cpu(cpu, 0); -- -- bpf_task_release(p); -- return true; -- } -- -- return false; --} -- --void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (cpu == central_cpu) { -- /* dispatch for all other CPUs first */ -- __sync_fetch_and_add(&nr_dispatches, 1); -- -- bpf_for(cpu, 0, nr_cpu_ids) { -- bool *gimme; -- -- if (!scx_bpf_dispatch_nr_slots()) -- break; -- -- /* central's gimme is never set */ -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme && !*gimme) -- continue; -- -- if (dispatch_to_cpu(cpu)) -- *gimme = false; -- } -- -- /* -- * Retry if we ran out of dispatch buffer slots as we might have -- * skipped some CPUs and also need to dispatch for self. The ext -- * core automatically retries if the local dsq is empty but we -- * can't rely on that as we're dispatching for other CPUs too. -- * Kick self explicitly to retry. -- */ -- if (!scx_bpf_dispatch_nr_slots()) { -- __sync_fetch_and_add(&nr_retries, 1); -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- return; -- } -- -- /* look for a task to run on the central CPU */ -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- dispatch_to_cpu(central_cpu); -- } else { -- bool *gimme; -- -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme) -- *gimme = true; -- -- /* -- * Force dispatch on the scheduling CPU so that it finds a task -- * to run for us. -- */ -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- } --} -- --void BPF_STRUCT_OPS(central_running, struct task_struct *p) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ --} -- --void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = 0; --} -- --static int central_timerfn(void *map, int *key, struct bpf_timer *timer) --{ -- u64 now = bpf_ktime_get_ns(); -- u64 nr_to_kick = nr_queued; -- s32 i; -- -- bpf_for(i, 0, nr_cpu_ids) { -- s32 cpu = (nr_timers + i) % nr_cpu_ids; -- u64 *started_at; -- -- if (cpu == central_cpu) -- continue; -- -- /* kick iff the current one exhausted its slice */ -- started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at && *started_at && -- vtime_before(now, *started_at + SCX_SLICE_DFL)) -- continue; -- -- /* and there's something pending */ -- if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) || -- scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu)) -- ; -- else if (nr_to_kick) -- nr_to_kick--; -- else -- continue; -- -- scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); -- } -- -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- -- bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- __sync_fetch_and_add(&nr_timers, 1); -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(central_init) --{ -- u32 key = 0; -- struct bpf_timer *timer; -- int ret; -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1); -- if (ret) -- return ret; -- -- timer = bpf_map_lookup_elem(¢ral_timer, &key); -- if (!timer) -- return -ESRCH; -- -- bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); -- bpf_timer_set_callback(timer, central_timerfn); -- ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- return ret; --} -- --void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops central_ops = { -- /* -- * We are offloading all scheduling decisions to the central CPU and -- * thus being the last task on a given CPU doesn't mean anything -- * special. Enqueue the last tasks like any other tasks. -- */ -- .flags = SCX_OPS_ENQ_LAST, -- -- .select_cpu = (void *)central_select_cpu, -- .enqueue = (void *)central_enqueue, -- .dispatch = (void *)central_dispatch, -- .running = (void *)central_running, -- .stopping = (void *)central_stopping, -- .init = (void *)central_init, -- .exit = (void *)central_exit, -- .name = "central", --}; -diff --git a/tools/sched_ext/scx_example_central.c b/tools/sched_ext/scx_example_central.c -deleted file mode 100644 -index 7ad591cbdc65..000000000000 ---- a/tools/sched_ext/scx_example_central.c -+++ /dev/null -@@ -1,94 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_central.skel.h" -- --const char help_fmt[] = --"A central FIFO sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-c CPU] [-p]\n" --"\n" --" -c CPU Override the central CPU (default: 0)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_central *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_central__open(); -- assert(skel); -- -- skel->rodata->central_cpu = 0; -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "c:ph")) != -1) { -- switch (opt) { -- case 'c': -- skel->rodata->central_cpu = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_central__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.central_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf("total :%10lu local:%10lu queued:%10lu lost:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_locals, -- skel->bss->nr_queued, -- skel->bss->nr_lost_pids); -- printf("timer :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n", -- skel->bss->nr_timers, -- skel->bss->nr_dispatches, -- skel->bss->nr_mismatches, -- skel->bss->nr_retries); -- printf("overflow:%10lu\n", -- skel->bss->nr_overflows); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_central__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_flatcg.bpf.c b/tools/sched_ext/scx_example_flatcg.bpf.c -deleted file mode 100644 -index f6078b9a681f..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.bpf.c -+++ /dev/null -@@ -1,872 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext flattened cgroup hierarchy scheduler. It implements -- * hierarchical weight-based cgroup CPU control by flattening the cgroup -- * hierarchy into a single layer by compounding the active weight share at each -- * level. Consider the following hierarchy with weights in parentheses: -- * -- * R + A (100) + B (100) -- * | \ C (100) -- * \ D (200) -- * -- * Ignoring the root and threaded cgroups, only B, C and D can contain tasks. -- * Let's say all three have runnable tasks. The total share that each of these -- * three cgroups is entitled to can be calculated by compounding its share at -- * each level. -- * -- * For example, B is competing against C and in that competition its share is -- * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's -- * share in that competition is 200/(200+100) == 1/3. B's eventual share in the -- * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's -- * eventual shaer is the same at 1/6. D is only competing at the top level and -- * its share is 200/(100+200) == 2/3. -- * -- * So, instead of hierarchically scheduling level-by-level, we can consider it -- * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3 -- * and keep updating the eventual shares as the cgroups' runnable states change. -- * -- * This flattening of hierarchy can bring a substantial performance gain when -- * the cgroup hierarchy is nested multiple levels. in a simple benchmark using -- * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it -- * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two -- * apache instances competing with 2:1 weight ratio nested four level deep. -- * -- * However, the gain comes at the cost of not being able to properly handle -- * thundering herd of cgroups. For example, if many cgroups which are nested -- * behind a low priority parent cgroup wake up around the same time, they may be -- * able to consume more CPU cycles than they are entitled to. In many use cases, -- * this isn't a real concern especially given the performance gain. Also, there -- * are ways to mitigate the problem further by e.g. introducing an extra -- * scheduling layer on cgroup delegation boundaries. -- * -- * The scheduler first picks the cgroup to run and then schedule the tasks -- * within by using nested weighted vtime scheduling by default. The -- * cgroup-internal scheduling can be switched to FIFO with the -f option. -- */ --#include "scx_common.bpf.h" --#include "user_exit_info.h" --#include "scx_example_flatcg.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */ --const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL; --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --u64 cvtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, u64); -- __uint(max_entries, FCG_NR_STATS); --} stats SEC(".maps"); -- --static void stat_inc(enum fcg_stat_idx idx) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p)++; --} -- --struct fcg_cpu_ctx { -- u64 cur_cgid; -- u64 cur_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, struct fcg_cpu_ctx); -- __uint(max_entries, 1); --} cpu_ctx SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_cgrp_ctx); --} cgrp_ctx SEC(".maps"); -- --struct cgv_node { -- struct bpf_rb_node rb_node; -- __u64 cvtime; -- __u64 cgid; --}; -- --private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock; --private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node); -- --struct cgv_node_stash { -- struct cgv_node __kptr *node; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, 16384); -- __type(key, __u64); -- __type(value, struct cgv_node_stash); --} cgv_node_stash SEC(".maps"); -- --struct fcg_task_ctx { -- u64 bypassed_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_task_ctx); --} task_ctx SEC(".maps"); -- --/* gets inc'd on weight tree changes to expire the cached hweights */ --unsigned long hweight_gen = 1; -- --static u64 div_round_up(u64 dividend, u64 divisor) --{ -- return (dividend + divisor - 1) / divisor; --} -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) --{ -- struct cgv_node *cgc_a, *cgc_b; -- -- cgc_a = container_of(a, struct cgv_node, rb_node); -- cgc_b = container_of(b, struct cgv_node, rb_node); -- -- return cgc_a->cvtime < cgc_b->cvtime; --} -- --static struct fcg_cpu_ctx *find_cpu_ctx(void) --{ -- struct fcg_cpu_ctx *cpuc; -- u32 idx = 0; -- -- cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx); -- if (!cpuc) { -- scx_bpf_error("cpu_ctx lookup failed"); -- return NULL; -- } -- return cpuc; --} -- --static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id); -- return NULL; -- } -- return cgc; --} -- --static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgrp = bpf_cgroup_ancestor(cgrp, level); -- if (!cgrp) { -- scx_bpf_error("ancestor cgroup lookup failed"); -- return NULL; -- } -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- scx_bpf_error("ancestor cgrp_ctx lookup failed"); -- bpf_cgroup_release(cgrp); -- return cgc; --} -- --static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- int level; -- -- if (!cgc->nr_active) { -- stat_inc(FCG_STAT_HWT_SKIP); -- return; -- } -- -- if (cgc->hweight_gen == hweight_gen) { -- stat_inc(FCG_STAT_HWT_CACHE); -- return; -- } -- -- stat_inc(FCG_STAT_HWT_UPDATES); -- bpf_for(level, 0, cgrp->level + 1) { -- struct fcg_cgrp_ctx *cgc; -- bool is_active; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- -- if (!level) { -- cgc->hweight = FCG_HWEIGHT_ONE; -- cgc->hweight_gen = hweight_gen; -- } else { -- struct fcg_cgrp_ctx *pcgc; -- -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- -- /* -- * We can be oppotunistic here and not grab the -- * cgv_tree_lock and deal with the occasional races. -- * However, hweight updates are already cached and -- * relatively low-frequency. Let's just do the -- * straightforward thing. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- is_active = cgc->nr_active; -- if (is_active) { -- cgc->hweight_gen = pcgc->hweight_gen; -- cgc->hweight = -- div_round_up(pcgc->hweight * cgc->weight, -- pcgc->child_weight_sum); -- } -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!is_active) { -- stat_inc(FCG_STAT_HWT_RACE); -- break; -- } -- } -- } --} -- --static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) --{ -- u64 delta, cvtime, max_budget; -- -- /* -- * A node which is on the rbtree can't be pointed to from elsewhere yet -- * and thus can't be updated and repositioned. Instead, we collect the -- * vtime deltas separately and apply it asynchronously here. -- */ -- delta = cgc->cvtime_delta; -- __sync_fetch_and_sub(&cgc->cvtime_delta, delta); -- cvtime = cgv_node->cvtime + delta; -- -- /* -- * Allow a cgroup to carry the maximum budget proportional to its -- * hweight such that a full-hweight cgroup can immediately take up half -- * of the CPUs at the most while staying at the front of the rbtree. -- */ -- max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / -- (2 * FCG_HWEIGHT_ONE); -- if (vtime_before(cvtime, cvtime_now - max_budget)) -- cvtime = cvtime_now - max_budget; -- -- cgv_node->cvtime = cvtime; --} -- --static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- u64 cgid = cgrp->kn->id; -- -- /* paired with cmpxchg in try_pick_next_cgroup() */ -- if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { -- stat_inc(FCG_STAT_ENQ_SKIP); -- return; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid); -- return; -- } -- -- /* NULL if the node is already on the rbtree */ -- cgv_node = bpf_kptr_xchg(&stash->node, NULL); -- if (!cgv_node) { -- stat_inc(FCG_STAT_ENQ_RACE); -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); --} -- --void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * If select_cpu_dfl() is recommending local enqueue, the target CPU is -- * idle. Follow it and charge the cgroup later in fcg_stopping() after -- * the fact. Use the same mechanism to deal with tasks with custom -- * affinities so that we don't have to worry about per-cgroup dq's -- * containing tasks that can't be executed from some CPUs. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) { -- /* -- * Tell fcg_stopping() that this bypassed the regular scheduling -- * path and should be force charged to the cgroup. 0 is used to -- * indicate that the task isn't bypassing, so if the current -- * runtime is 0, go back by one nanosecond. -- */ -- taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1; -- -- /* -- * The global dq is deprioritized as we don't want to let tasks -- * to boost themselves by constraining its cpumask. The -- * deprioritization is rather severe, so let's not apply that to -- * per-cpu kernel threads. This is ham-fisted. We probably wanna -- * implement per-cgroup fallback dq's instead so that we have -- * more control over when tasks with custom cpumask get issued. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || -- (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) { -- stat_inc(FCG_STAT_LOCAL); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- } else { -- stat_inc(FCG_STAT_GLOBAL); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } -- return; -- } -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- goto out_release; -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 tvtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) -- tvtime = cgc->tvtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, -- tvtime, enq_flags); -- } -- -- cgrp_enqueued(cgrp, cgc); --out_release: -- bpf_cgroup_release(cgrp); --} -- --/* -- * Walk the cgroup tree to update the active weight sums as tasks wake up and -- * sleep. The weight sums are used as the base when calculating the proportion a -- * given cgroup or task is entitled to at each level. -- */ --static void update_active_weight_sums(struct cgroup *cgrp, bool runnable) --{ -- struct fcg_cgrp_ctx *cgc; -- bool updated = false; -- int idx; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- /* -- * In most cases, a hot cgroup would have multiple threads going to -- * sleep and waking up while the whole cgroup stays active. In leaf -- * cgroups, ->nr_runnable which is updated with __sync operations gates -- * ->nr_active updates, so that we don't have to grab the cgv_tree_lock -- * repeatedly for a busy cgroup which is staying active. -- */ -- if (runnable) { -- if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_ACT); -- } else { -- if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_DEACT); -- } -- -- /* -- * If @cgrp is becoming runnable, its hweight should be refreshed after -- * it's added to the weight tree so that enqueue has the up-to-date -- * value. If @cgrp is becoming quiescent, the hweight should be -- * refreshed before it's removed from the weight tree so that the usage -- * charging which happens afterwards has access to the latest value. -- */ -- if (!runnable) -- cgrp_refresh_hweight(cgrp, cgc); -- -- /* propagate upwards */ -- bpf_for(idx, 0, cgrp->level) { -- int level = cgrp->level - idx; -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- bool propagate = false; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- if (level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- } -- -- /* -- * We need the propagation protected by a lock to synchronize -- * against weight changes. There's no reason to drop the lock at -- * each level but bpf_spin_lock() doesn't want any function -- * calls while locked. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- -- if (runnable) { -- if (!cgc->nr_active++) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum += cgc->weight; -- } -- } -- } else { -- if (!--cgc->nr_active) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum -= cgc->weight; -- } -- } -- } -- -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!propagate) -- break; -- } -- -- if (updated) -- __sync_fetch_and_add(&hweight_gen, 1); -- -- if (runnable) -- cgrp_refresh_hweight(cgrp, cgc); --} -- --void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, true); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) --{ -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- if (fifo_sched) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- /* -- * @cgc->tvtime_now always progresses forward as tasks start -- * executing. The test and update can be performed concurrently -- * from multiple CPUs and thus racy. Any error should be -- * contained and temporary. Let's just live with it. -- */ -- if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) -- cgc->tvtime_now = p->scx.dsq_vtime; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- /* scale the execution time by the inverse of the weight and charge */ -- if (!fifo_sched) -- p->scx.dsq_vtime += -- (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- if (!taskc->bypassed_at) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- __sync_fetch_and_add(&cgc->cvtime_delta, -- p->se.sum_exec_runtime - taskc->bypassed_at); -- taskc->bypassed_at = 0; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, false); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) --{ -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- if (cgrp->level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1); -- if (!pcgc) -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- if (pcgc && cgc->nr_active) -- pcgc->child_weight_sum += (s64)weight - cgc->weight; -- cgc->weight = weight; -- bpf_spin_unlock(&cgv_tree_lock); --} -- --static bool try_pick_next_cgroup(u64 *cgidp) --{ -- struct bpf_rb_node *rb_node; -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 cgid; -- -- /* pop the front cgroup and wind cvtime_now accordingly */ -- bpf_spin_lock(&cgv_tree_lock); -- -- rb_node = bpf_rbtree_first(&cgv_tree); -- if (!rb_node) { -- bpf_spin_unlock(&cgv_tree_lock); -- stat_inc(FCG_STAT_PNC_NO_CGRP); -- *cgidp = 0; -- return true; -- } -- -- rb_node = bpf_rbtree_remove(&cgv_tree, rb_node); -- bpf_spin_unlock(&cgv_tree_lock); -- -- cgv_node = container_of(rb_node, struct cgv_node, rb_node); -- cgid = cgv_node->cgid; -- -- if (vtime_before(cvtime_now, cgv_node->cvtime)) -- cvtime_now = cgv_node->cvtime; -- -- /* -- * If lookup fails, the cgroup's gone. Free and move on. See -- * fcg_cgroup_exit(). -- */ -- cgrp = bpf_cgroup_from_id(cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- if (!scx_bpf_consume(cgid)) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_EMPTY); -- goto out_stash; -- } -- -- /* -- * Successfully consumed from the cgroup. This will be our current -- * cgroup for the new slice. Refresh its hweight. -- */ -- cgrp_refresh_hweight(cgrp, cgc); -- -- bpf_cgroup_release(cgrp); -- -- /* -- * As the cgroup may have more tasks, add it back to the rbtree. Note -- * that here we charge the full slice upfront and then exact later -- * according to the actual consumption. This prevents lowpri thundering -- * herd from saturating the machine. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- -- *cgidp = cgid; -- stat_inc(FCG_STAT_PNC_NEXT); -- return true; -- --out_stash: -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- /* -- * Paired with cmpxchg in cgrp_enqueued(). If they see the following -- * transition, they'll enqueue the cgroup. If they are earlier, we'll -- * see their task in the dq below and requeue the cgroup. -- */ -- __sync_val_compare_and_swap(&cgc->queued, 1, 0); -- -- if (scx_bpf_dsq_nr_queued(cgid)) { -- bpf_spin_lock(&cgv_tree_lock); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- goto out_free; -- } -- } -- -- return false; -- --out_free: -- bpf_obj_drop(cgv_node); -- return false; --} -- --void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) --{ -- struct fcg_cpu_ctx *cpuc; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 now = bpf_ktime_get_ns(); -- -- cpuc = find_cpu_ctx(); -- if (!cpuc) -- return; -- -- if (!cpuc->cur_cgid) -- goto pick_next_cgroup; -- -- if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { -- if (scx_bpf_consume(cpuc->cur_cgid)) { -- stat_inc(FCG_STAT_CNS_KEEP); -- return; -- } -- stat_inc(FCG_STAT_CNS_EMPTY); -- } else { -- stat_inc(FCG_STAT_CNS_EXPIRE); -- } -- -- /* -- * The current cgroup is expiring. It was already charged a full slice. -- * Calculate the actual usage and accumulate the delta. -- */ -- cgrp = bpf_cgroup_from_id(cpuc->cur_cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_CNS_GONE); -- goto pick_next_cgroup; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (cgc) { -- /* -- * We want to update the vtime delta and then look for the next -- * cgroup to execute but the latter needs to be done in a loop -- * and we can't keep the lock held. Oh well... -- */ -- bpf_spin_lock(&cgv_tree_lock); -- __sync_fetch_and_add(&cgc->cvtime_delta, -- (cpuc->cur_at + cgrp_slice_ns - now) * -- FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- stat_inc(FCG_STAT_CNS_GONE); -- } -- -- bpf_cgroup_release(cgrp); -- --pick_next_cgroup: -- cpuc->cur_at = now; -- -- if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { -- cpuc->cur_cgid = 0; -- return; -- } -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_pick_next_cgroup(&cpuc->cur_cgid)) -- break; -- } --} -- --s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct fcg_task_ctx *taskc; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- taskc = bpf_task_storage_get(&task_ctx, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!taskc) -- return -ENOMEM; -- -- taskc->bypassed_at = 0; -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp, -- struct scx_cgroup_init_args *args) --{ -- struct fcg_cgrp_ctx *cgc; -- struct cgv_node *cgv_node; -- struct cgv_node_stash empty_stash = {}, *stash; -- u64 cgid = cgrp->kn->id; -- int ret; -- -- /* -- * Technically incorrect as cgroup ID is full 64bit while dq ID is -- * 63bit. Should not be a problem in practice and easy to spot in the -- * unlikely case that it breaks. -- */ -- ret = scx_bpf_create_dsq(cgid, -1); -- if (ret) -- return ret; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!cgc) { -- ret = -ENOMEM; -- goto err_destroy_dsq; -- } -- -- cgc->weight = args->weight; -- cgc->hweight = FCG_HWEIGHT_ONE; -- -- ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash, -- BPF_NOEXIST); -- if (ret) { -- if (ret != -ENOMEM) -- scx_bpf_error("unexpected stash creation error (%d)", -- ret); -- goto err_destroy_dsq; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("unexpected cgv_node stash lookup failure"); -- ret = -ENOENT; -- goto err_destroy_dsq; -- } -- -- cgv_node = bpf_obj_new(struct cgv_node); -- if (!cgv_node) { -- ret = -ENOMEM; -- goto err_del_cgv_node; -- } -- -- cgv_node->cgid = cgid; -- cgv_node->cvtime = cvtime_now; -- -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- ret = -EBUSY; -- goto err_drop; -- } -- -- return 0; -- --err_drop: -- bpf_obj_drop(cgv_node); --err_del_cgv_node: -- bpf_map_delete_elem(&cgv_node_stash, &cgid); --err_destroy_dsq: -- scx_bpf_destroy_dsq(cgid); -- return ret; --} -- --void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- -- /* -- * For now, there's no way find and remove the cgv_node if it's on the -- * cgv_tree. Let's drain them in the dispatch path as they get popped -- * off the front of the tree. -- */ -- bpf_map_delete_elem(&cgv_node_stash, &cgid); -- scx_bpf_destroy_dsq(cgid); --} -- --s32 BPF_STRUCT_OPS(fcg_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops flatcg_ops = { -- .enqueue = (void *)fcg_enqueue, -- .dispatch = (void *)fcg_dispatch, -- .runnable = (void *)fcg_runnable, -- .running = (void *)fcg_running, -- .stopping = (void *)fcg_stopping, -- .quiescent = (void *)fcg_quiescent, -- .prep_enable = (void *)fcg_prep_enable, -- .cgroup_set_weight = (void *)fcg_cgroup_set_weight, -- .cgroup_init = (void *)fcg_cgroup_init, -- .cgroup_exit = (void *)fcg_cgroup_exit, -- .init = (void *)fcg_init, -- .exit = (void *)fcg_exit, -- .flags = SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING, -- .name = "flatcg", --}; -diff --git a/tools/sched_ext/scx_example_flatcg.c b/tools/sched_ext/scx_example_flatcg.c -deleted file mode 100644 -index f9c8a5b84a70..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.c -+++ /dev/null -@@ -1,232 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2023 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2023 Tejun Heo -- * Copyright (c) 2023 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_flatcg.h" --#include "scx_example_flatcg.skel.h" -- --#ifndef FILEID_KERNFS --#define FILEID_KERNFS 0xfe --#endif -- --const char help_fmt[] = --"A flattened cgroup hierarchy sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -i INTERVAL Report interval\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --static float read_cpu_util(__u64 *last_sum, __u64 *last_idle) --{ -- FILE *fp; -- char buf[4096]; -- char *line, *cur = NULL, *tok; -- __u64 sum = 0, idle = 0; -- __u64 delta_sum, delta_idle; -- int idx; -- -- fp = fopen("/proc/stat", "r"); -- if (!fp) { -- perror("fopen(\"/proc/stat\")"); -- return 0.0; -- } -- -- if (!fgets(buf, sizeof(buf), fp)) { -- perror("fgets(\"/proc/stat\")"); -- fclose(fp); -- return 0.0; -- } -- fclose(fp); -- -- line = buf; -- for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) { -- char *endp = NULL; -- __u64 v; -- -- if (idx == 0) { -- line = NULL; -- continue; -- } -- v = strtoull(tok, &endp, 0); -- if (!endp || *endp != '\0') { -- fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n", -- idx, tok); -- continue; -- } -- sum += v; -- if (idx == 4) -- idle = v; -- } -- -- delta_sum = sum - *last_sum; -- delta_idle = idle - *last_idle; -- *last_sum = sum; -- *last_idle = idle; -- -- return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0; --} -- --static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats) --{ -- __u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus]; -- __u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); -- -- for (idx = 0; idx < FCG_NR_STATS; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_flatcg *skel; -- struct bpf_link *link; -- struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 }; -- bool dump_cgrps = false; -- __u64 last_cpu_sum = 0, last_cpu_idle = 0; -- __u64 last_stats[FCG_NR_STATS] = {}; -- unsigned long seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_flatcg__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open: %s\n", strerror(errno)); -- return 1; -- } -- -- skel->rodata->nr_cpus = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) { -- double v; -- -- switch (opt) { -- case 's': -- v = strtod(optarg, NULL); -- skel->rodata->cgrp_slice_ns = v * 1000; -- break; -- case 'i': -- v = strtod(optarg, NULL); -- intv_ts.tv_sec = v; -- intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000; -- break; -- case 'd': -- dump_cgrps = true; -- break; -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- case 'h': -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d", -- (double)skel->rodata->cgrp_slice_ns / 1000000.0, -- (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0, -- dump_cgrps); -- -- if (scx_example_flatcg__load(skel)) { -- fprintf(stderr, "Failed to load: %s\n", strerror(errno)); -- return 1; -- } -- -- link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops); -- if (!link) { -- fprintf(stderr, "Failed to attach_struct_ops: %s\n", -- strerror(errno)); -- return 1; -- } -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- __u64 acc_stats[FCG_NR_STATS]; -- __u64 stats[FCG_NR_STATS]; -- float cpu_util; -- int i; -- -- cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle); -- -- fcg_read_stats(skel, acc_stats); -- for (i = 0; i < FCG_NR_STATS; i++) -- stats[i] = acc_stats[i] - last_stats[i]; -- -- memcpy(last_stats, acc_stats, sizeof(acc_stats)); -- -- printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n", -- seq++, cpu_util * 100.0, skel->data->hweight_gen); -- printf(" act:%6llu deact:%6llu local:%6llu global:%6llu\n", -- stats[FCG_STAT_ACT], -- stats[FCG_STAT_DEACT], -- stats[FCG_STAT_LOCAL], -- stats[FCG_STAT_GLOBAL]); -- printf("HWT skip:%6llu race:%6llu cache:%6llu update:%6llu\n", -- stats[FCG_STAT_HWT_SKIP], -- stats[FCG_STAT_HWT_RACE], -- stats[FCG_STAT_HWT_CACHE], -- stats[FCG_STAT_HWT_UPDATES]); -- printf("ENQ skip:%6llu race:%6llu\n", -- stats[FCG_STAT_ENQ_SKIP], -- stats[FCG_STAT_ENQ_RACE]); -- printf("CNS keep:%6llu expire:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_CNS_KEEP], -- stats[FCG_STAT_CNS_EXPIRE], -- stats[FCG_STAT_CNS_EMPTY], -- stats[FCG_STAT_CNS_GONE]); -- printf("PNC nocgrp:%6llu next:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_PNC_NO_CGRP], -- stats[FCG_STAT_PNC_NEXT], -- stats[FCG_STAT_PNC_EMPTY], -- stats[FCG_STAT_PNC_GONE]); -- printf("BAD remove:%6llu\n", -- acc_stats[FCG_STAT_BAD_REMOVAL]); -- -- nanosleep(&intv_ts, NULL); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_flatcg__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_flatcg.h b/tools/sched_ext/scx_example_flatcg.h -deleted file mode 100644 -index 490758ed41f0..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.h -+++ /dev/null -@@ -1,49 +0,0 @@ --#ifndef __SCX_EXAMPLE_FLATCG_H --#define __SCX_EXAMPLE_FLATCG_H -- --enum { -- FCG_HWEIGHT_ONE = 1LLU << 16, --}; -- --enum fcg_stat_idx { -- FCG_STAT_ACT, -- FCG_STAT_DEACT, -- FCG_STAT_LOCAL, -- FCG_STAT_GLOBAL, -- -- FCG_STAT_HWT_UPDATES, -- FCG_STAT_HWT_CACHE, -- FCG_STAT_HWT_SKIP, -- FCG_STAT_HWT_RACE, -- -- FCG_STAT_ENQ_SKIP, -- FCG_STAT_ENQ_RACE, -- -- FCG_STAT_CNS_KEEP, -- FCG_STAT_CNS_EXPIRE, -- FCG_STAT_CNS_EMPTY, -- FCG_STAT_CNS_GONE, -- -- FCG_STAT_PNC_NO_CGRP, -- FCG_STAT_PNC_NEXT, -- FCG_STAT_PNC_EMPTY, -- FCG_STAT_PNC_GONE, -- -- FCG_STAT_BAD_REMOVAL, -- -- FCG_NR_STATS, --}; -- --struct fcg_cgrp_ctx { -- u32 nr_active; -- u32 nr_runnable; -- u32 queued; -- u32 weight; -- u32 hweight; -- u64 child_weight_sum; -- u64 hweight_gen; -- s64 cvtime_delta; -- u64 tvtime_now; --}; -- --#endif /* __SCX_EXAMPLE_FLATCG_H */ -diff --git a/tools/sched_ext/scx_example_pair.bpf.c b/tools/sched_ext/scx_example_pair.bpf.c -deleted file mode 100644 -index 279efe58b777..000000000000 ---- a/tools/sched_ext/scx_example_pair.bpf.c -+++ /dev/null -@@ -1,627 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext core-scheduler which always makes every sibling CPU pair -- * execute from the same CPU cgroup. -- * -- * This scheduler is a minimal implementation and would need some form of -- * priority handling both inside each cgroup and across the cgroups to be -- * practically useful. -- * -- * Each CPU in the system is paired with exactly one other CPU, according to a -- * "stride" value that can be specified when the BPF scheduler program is first -- * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee -- * that they will only ever schedule tasks that belong to the same CPU cgroup. -- * -- * Scheduler Initialization -- * ------------------------ -- * -- * The scheduler BPF program is first initialized from user space, before it is -- * enabled. During this initialization process, each CPU on the system is -- * assigned several values that are constant throughout its runtime: -- * -- * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling -- * decisions. Paired CPUs always schedule tasks from the same -- * CPU cgroup, and synchronize with each other to guarantee -- * that this constraint is not violated. -- * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access -- * a struct pair_ctx object that is shared between the pair. -- * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the -- * pair. Each struct pair_ctx has an active_mask field, -- * which is a bitmap used to indicate whether each core -- * in the pair currently has an actively running task. -- * This index specifies which entry in the bitmap corresponds -- * to each CPU in the pair. -- * -- * During this initialization, the CPUs are paired according to a "stride" that -- * may be specified when invoking the user space program that initializes and -- * loads the scheduler. By default, the stride is 1/2 the total number of CPUs. -- * -- * Tasks and cgroups -- * ----------------- -- * -- * Every cgroup in the system is registered with the scheduler using the -- * pair_cgroup_init() callback, and every task in the system is associated with -- * exactly one cgroup. At a high level, the idea with the pair scheduler is to -- * always schedule tasks from the same cgroup within a given CPU pair. When a -- * task is enqueued (i.e. passed to the pair_enqueue() callback function), its -- * cgroup ID is read from its task struct, and then a corresponding queue map -- * is used to FIFO-enqueue the task for that cgroup. -- * -- * If you look through the implementation of the scheduler, you'll notice that -- * there is quite a bit of complexity involved with looking up the per-cgroup -- * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash -- * BPF hash map that is used to map a cgroup ID to a globally unique ID that's -- * allocated in the BPF program. This is done because we use separate maps to -- * store the FIFO queue of tasks, and the length of that map, per cgroup. This -- * complexity is only present because of current deficiencies in BPF that will -- * soon be addressed. The main point to keep in mind is that newly enqueued -- * tasks are added to their cgroup's FIFO queue. -- * -- * Dispatching tasks -- * ----------------- -- * -- * This section will describe how enqueued tasks are dispatched and scheduled. -- * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is -- * as follows: -- * -- * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is -- * the structure that's used to synchronize amongst the two pair CPUs in their -- * scheduling decisions. After any of the following events have occurred: -- * -- * - The cgroup's slice run has expired, or -- * - The cgroup becomes empty, or -- * - Either CPU in the pair is preempted by a higher priority scheduling class -- * -- * The cgroup transitions to the draining state and stops executing new tasks -- * from the cgroup. -- * -- * 2. If the pair is still executing a task, mark the pair_ctx as draining, and -- * wait for the pair CPU to be preempted. -- * -- * 3. Otherwise, if the pair CPU is not running a task, we can move onto -- * scheduling new tasks. Pop the next cgroup id from the top_q queue. -- * -- * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it. -- * -- * Note again that this scheduling behavior is simple, but the implementation -- * is complex mostly because this it hits several BPF shortcomings and has to -- * work around in often awkward ways. Most of the shortcomings are expected to -- * be resolved in the near future which should allow greatly simplifying this -- * scheduler. -- * -- * Dealing with preemption -- * ----------------------- -- * -- * SCX is the lowest priority sched_class, and could be preempted by them at -- * any time. To address this, the scheduler implements pair_cpu_release() and -- * pair_cpu_acquire() callbacks which are invoked by the core scheduler when -- * the scheduler loses and gains control of the CPU respectively. -- * -- * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and -- * then invoke: -- * -- * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- * -- * This preempts the pair CPU, and waits until it has re-entered the scheduler -- * before returning. This is necessary to ensure that the higher priority -- * sched_class that preempted our scheduler does not schedule a task -- * concurrently with our pair CPU. -- * -- * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption -- * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable -- * pair scheduling. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include "scx_example_pair.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; -- --/* !0 for veristat, set during init */ --const volatile u32 nr_cpu_ids = 64; -- --/* a pair of CPUs stay on a cgroup for this duration */ --const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL; -- --/* cpu ID -> pair cpu ID */ --const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 }; -- --/* cpu ID -> pair_id */ --const volatile u32 pair_id[MAX_CPUS]; -- --/* CPU ID -> CPU # in the pair (0 or 1) */ --const volatile u32 in_pair_idx[MAX_CPUS]; -- --struct pair_ctx { -- struct bpf_spin_lock lock; -- -- /* the cgroup the pair is currently executing */ -- u64 cgid; -- -- /* the pair started executing the current cgroup at */ -- u64 started_at; -- -- /* whether the current cgroup is draining */ -- bool draining; -- -- /* the CPUs that are currently active on the cgroup */ -- u32 active_mask; -- -- /* -- * the CPUs that are currently preempted and running tasks in a -- * different scheduler. -- */ -- u32 preempted_mask; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, MAX_CPUS / 2); -- __type(key, u32); -- __type(value, struct pair_ctx); --} pair_ctx SEC(".maps"); -- --/* queue of cgrp_q's possibly with tasks on them */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- /* -- * Because it's difficult to build strong synchronization encompassing -- * multiple non-trivial operations in BPF, this queue is managed in an -- * opportunistic way so that we guarantee that a cgroup w/ active tasks -- * is always on it but possibly multiple times. Once we have more robust -- * synchronization constructs and e.g. linked list, we should be able to -- * do this in a prettier way but for now just size it big enough. -- */ -- __uint(max_entries, 4 * MAX_CGRPS); -- __type(value, u64); --} top_q SEC(".maps"); -- --/* per-cgroup q which FIFOs the tasks from the cgroup */ --struct cgrp_q { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, MAX_QUEUED); -- __type(value, u32); --}; -- --/* -- * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local -- * storage; however, a cgroup local storage can only be accessed from the BPF -- * progs attached to the cgroup. For now, work around by allocating array of -- * cgrp_q's and then allocating per-cgroup indices. -- * -- * Another caveat: It's difficult to populate a large array of maps statically -- * or from BPF. Initialize it from userland. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, MAX_CGRPS); -- __type(key, s32); -- __array(values, struct cgrp_q); --} cgrp_q_arr SEC(".maps"); -- --static u64 cgrp_q_len[MAX_CGRPS]; -- --/* -- * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be -- * useful to have as a map type. -- */ --static u32 cgrp_q_idx_cursor; --static u64 cgrp_q_idx_busy[MAX_CGRPS]; -- --/* -- * All added up, the following is what we do: -- * -- * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking -- * for a free ID. If not found, fail cgroup creation with -EBUSY. -- * -- * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following -- * cgrp_q_idx_hash. -- * -- * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from -- * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr. -- * -- * This is sadly complicated for something pretty simple. Hopefully, we should -- * be able to simplify in the future. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, MAX_CGRPS); -- __uint(key_size, sizeof(u64)); /* cgrp ID */ -- __uint(value_size, sizeof(s32)); /* cgrp_q idx */ --} cgrp_q_idx_hash SEC(".maps"); -- --/* statistics */ --u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions; --u64 nr_exps, nr_exp_waits, nr_exp_empty; --u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty; -- --struct user_exit_info uei; -- --static bool time_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- struct cgrp_q *cgq; -- s32 pid = p->pid; -- u64 cgid; -- u32 *q_idx; -- u64 *cgq_len; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- cgrp = scx_bpf_task_cgroup(p); -- cgid = cgrp->kn->id; -- bpf_cgroup_release(cgrp); -- -- /* find the cgroup's q and push @p into it */ -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!q_idx) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return; -- } -- -- cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx); -- if (!cgq) { -- scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]", -- cgid, *q_idx); -- return; -- } -- -- if (bpf_map_push_elem(cgq, &pid, 0)) { -- scx_bpf_error("cgroup[%llu] queue overflow", cgid); -- return; -- } -- -- /* bump q len, if going 0 -> 1, queue cgroup into the top_q */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len) { -- scx_bpf_error("MEMBER_VTPR malfunction"); -- return; -- } -- -- if (!__sync_fetch_and_add(cgq_len, 1) && -- bpf_map_push_elem(&top_q, &cgid, 0)) { -- scx_bpf_error("top_q overflow"); -- return; -- } --} -- --static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) --{ -- u32 *vptr; -- -- vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *pairc = bpf_map_lookup_elem(&pair_ctx, vptr); -- if (!(*pairc)) -- return -EINVAL; -- -- vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *mask = 1U << *vptr; -- -- return 0; --} -- --static int try_dispatch(s32 cpu) --{ -- struct pair_ctx *pairc; -- struct bpf_map *cgq_map; -- struct task_struct *p; -- u64 now = bpf_ktime_get_ns(); -- bool kick_pair = false; -- bool expired, pair_preempted; -- u32 *vptr, in_pair_mask; -- s32 pid, q_idx; -- u64 cgid; -- int ret; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) { -- scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]", -- cpu); -- return -ENOENT; -- } -- -- bpf_spin_lock(&pairc->lock); -- pairc->active_mask &= ~in_pair_mask; -- -- expired = time_before(pairc->started_at + pair_batch_dur_ns, now); -- if (expired || pairc->draining) { -- u64 new_cgid = 0; -- -- __sync_fetch_and_add(&nr_exps, 1); -- -- /* -- * We're done with the current cgid. An obvious optimization -- * would be not draining if the next cgroup is the current one. -- * For now, be dumb and always expire. -- */ -- pairc->draining = true; -- -- pair_preempted = pairc->preempted_mask; -- if (pairc->active_mask || pair_preempted) { -- /* -- * The other CPU is still active, or is no longer under -- * our control due to e.g. being preempted by a higher -- * priority sched_class. We want to wait until this -- * cgroup expires, or until control of our pair CPU has -- * been returned to us. -- * -- * If the pair controls its CPU, and the time already -- * expired, kick. When the other CPU arrives at -- * dispatch and clears its active mask, it'll push the -- * pair to the next cgroup and kick this CPU. -- */ -- __sync_fetch_and_add(&nr_exp_waits, 1); -- bpf_spin_unlock(&pairc->lock); -- if (expired && !pair_preempted) -- kick_pair = true; -- goto out_maybe_kick; -- } -- -- bpf_spin_unlock(&pairc->lock); -- -- /* -- * Pick the next cgroup. It'd be easier / cleaner to not drop -- * pairc->lock and use stronger synchronization here especially -- * given that we'll be switching cgroups significantly less -- * frequently than tasks. Unfortunately, bpf_spin_lock can't -- * really protect anything non-trivial. Let's do opportunistic -- * operations instead. -- */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u32 *q_idx; -- u64 *cgq_len; -- -- if (bpf_map_pop_elem(&top_q, &new_cgid)) { -- /* no active cgroup, go idle */ -- __sync_fetch_and_add(&nr_exp_empty, 1); -- return 0; -- } -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid); -- if (!q_idx) -- continue; -- -- /* -- * This is the only place where empty cgroups are taken -- * off the top_q. -- */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len || !*cgq_len) -- continue; -- -- /* -- * If it has any tasks, requeue as we may race and not -- * execute it. -- */ -- bpf_map_push_elem(&top_q, &new_cgid, 0); -- break; -- } -- -- bpf_spin_lock(&pairc->lock); -- -- /* -- * The other CPU may already have started on a new cgroup while -- * we dropped the lock. Make sure that we're still draining and -- * start on the new cgroup. -- */ -- if (pairc->draining && !pairc->active_mask) { -- __sync_fetch_and_add(&nr_cgrp_next, 1); -- pairc->cgid = new_cgid; -- pairc->started_at = now; -- pairc->draining = false; -- kick_pair = true; -- } else { -- __sync_fetch_and_add(&nr_cgrp_coll, 1); -- } -- } -- -- cgid = pairc->cgid; -- pairc->active_mask |= in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- -- /* again, it'd be better to do all these with the lock held, oh well */ -- vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!vptr) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return -ENOENT; -- } -- q_idx = *vptr; -- -- /* claim one task from cgrp_q w/ q_idx */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u64 *cgq_len, len; -- -- cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]); -- if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) { -- /* the cgroup must be empty, expire and repeat */ -- __sync_fetch_and_add(&nr_cgrp_empty, 1); -- bpf_spin_lock(&pairc->lock); -- pairc->draining = true; -- pairc->active_mask &= ~in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- return -EAGAIN; -- } -- -- if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len) -- continue; -- -- break; -- } -- -- cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx); -- if (!cgq_map) { -- scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- if (bpf_map_pop_elem(cgq_map, &pid)) { -- scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- p = bpf_task_from_pid(pid); -- if (p) { -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } else { -- /* we don't handle dequeues, retry on lost tasks */ -- __sync_fetch_and_add(&nr_missing, 1); -- return -EAGAIN; -- } -- --out_maybe_kick: -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } -- return 0; --} -- --void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) --{ -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_dispatch(cpu) != -EAGAIN) -- break; -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask &= ~in_pair_mask; -- /* Kick the pair CPU, unless it was also preempted. */ -- kick_pair = !pairc->preempted_mask; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask |= in_pair_mask; -- pairc->active_mask &= ~in_pair_mask; -- /* Kick the pair CPU if it's still running. */ -- kick_pair = pairc->active_mask; -- pairc->draining = true; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- } -- } -- __sync_fetch_and_add(&nr_preemptions, 1); --} -- --s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 i, q_idx; -- -- bpf_for(i, 0, MAX_CGRPS) { -- q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS; -- if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1)) -- break; -- } -- if (i == MAX_CGRPS) -- return -EBUSY; -- -- if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]); -- if (busy) -- *busy = 0; -- return -EBUSY; -- } -- -- return 0; --} -- --void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 *q_idx; -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (q_idx) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]); -- if (busy) -- *busy = 0; -- bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid); -- } --} -- --s32 BPF_STRUCT_OPS(pair_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops pair_ops = { -- .enqueue = (void *)pair_enqueue, -- .dispatch = (void *)pair_dispatch, -- .cpu_acquire = (void *)pair_cpu_acquire, -- .cpu_release = (void *)pair_cpu_release, -- .cgroup_init = (void *)pair_cgroup_init, -- .cgroup_exit = (void *)pair_cgroup_exit, -- .init = (void *)pair_init, -- .exit = (void *)pair_exit, -- .name = "pair", --}; -diff --git a/tools/sched_ext/scx_example_pair.c b/tools/sched_ext/scx_example_pair.c -deleted file mode 100644 -index 18e032bbc173..000000000000 ---- a/tools/sched_ext/scx_example_pair.c -+++ /dev/null -@@ -1,143 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_pair.h" --#include "scx_example_pair.skel.h" -- --const char help_fmt[] = --"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n" --"execute from the same CPU cgroup.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-S STRIDE] [-p]\n" --"\n" --" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_pair *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 stride, i, opt, outer_fd; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_pair__open(); -- assert(skel); -- -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- /* pair up the earlier half to the latter by default, override with -s */ -- stride = skel->rodata->nr_cpu_ids / 2; -- -- while ((opt = getopt(argc, argv, "S:ph")) != -1) { -- switch (opt) { -- case 'S': -- stride = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- for (i = 0; i < skel->rodata->nr_cpu_ids; i++) { -- if (skel->rodata->pair_cpu[i] < 0) { -- skel->rodata->pair_cpu[i] = i + stride; -- skel->rodata->pair_cpu[i + stride] = i; -- skel->rodata->pair_id[i] = i; -- skel->rodata->pair_id[i + stride] = i; -- skel->rodata->in_pair_idx[i] = 0; -- skel->rodata->in_pair_idx[i + stride] = 1; -- } -- } -- -- assert(!scx_example_pair__load(skel)); -- -- /* -- * Populate the cgrp_q_arr map which is an array containing per-cgroup -- * queues. It'd probably be better to do this from BPF but there are too -- * many to initialize statically and there's no way to dynamically -- * populate from BPF. -- */ -- outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr); -- assert(outer_fd >= 0); -- -- printf("Initializing"); -- for (i = 0; i < MAX_CGRPS; i++) { -- s32 inner_fd; -- -- if (exit_req) -- break; -- -- inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, -- sizeof(u32), MAX_QUEUED, NULL); -- assert(inner_fd >= 0); -- assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY)); -- close(inner_fd); -- -- if (!(i % 10)) -- printf("."); -- fflush(stdout); -- } -- printf("\n"); -- -- /* -- * Fully initialized, attach and run. -- */ -- link = bpf_map__attach_struct_ops(skel->maps.pair_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf(" total:%10lu dispatch:%10lu missing:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_dispatched, -- skel->bss->nr_missing); -- printf(" kicks:%10lu preemptions:%7lu\n", -- skel->bss->nr_kicks, -- skel->bss->nr_preemptions); -- printf(" exp:%10lu exp_wait:%10lu exp_empty:%10lu\n", -- skel->bss->nr_exps, -- skel->bss->nr_exp_waits, -- skel->bss->nr_exp_empty); -- printf("cgnext:%10lu cgcoll:%10lu cgempty:%10lu\n", -- skel->bss->nr_cgrp_next, -- skel->bss->nr_cgrp_coll, -- skel->bss->nr_cgrp_empty); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_pair__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_pair.h b/tools/sched_ext/scx_example_pair.h -deleted file mode 100644 -index f60b824272f7..000000000000 ---- a/tools/sched_ext/scx_example_pair.h -+++ /dev/null -@@ -1,10 +0,0 @@ --#ifndef __SCX_EXAMPLE_PAIR_H --#define __SCX_EXAMPLE_PAIR_H -- --enum { -- MAX_CPUS = 4096, -- MAX_QUEUED = 4096, -- MAX_CGRPS = 4096, --}; -- --#endif /* __SCX_EXAMPLE_PAIR_H */ -diff --git a/tools/sched_ext/scx_example_qmap.bpf.c b/tools/sched_ext/scx_example_qmap.bpf.c -deleted file mode 100644 -index 579ab21ae403..000000000000 ---- a/tools/sched_ext/scx_example_qmap.bpf.c -+++ /dev/null -@@ -1,401 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple five-level FIFO queue scheduler. -- * -- * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets -- * assigned to one depending on its compound weight. Each CPU round robins -- * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from -- * queue0, 2 from queue1, 4 from queue2 and so on. -- * -- * This scheduler demonstrates: -- * -- * - BPF-side queueing using PIDs. -- * - Sleepable per-task storage allocation using ops.prep_enable(). -- * - Using ops.cpu_release() to handle a higher priority scheduling class taking -- * the CPU away. -- * - Core-sched support. -- * -- * This scheduler is primarily for demonstration and testing of sched_ext -- * features and unlikely to be useful for actual workloads. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include -- --char _license[] SEC("license") = "GPL"; -- --const volatile u64 slice_ns = SCX_SLICE_DFL; --const volatile bool switch_partial; --const volatile u32 stall_user_nth; --const volatile u32 stall_kernel_nth; --const volatile u32 dsp_inf_loop_after; --const volatile s32 disallow_tgid; -- --u32 test_error_cnt; -- --struct user_exit_info uei; -- --struct qmap { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, u32); --} queue0 SEC(".maps"), -- queue1 SEC(".maps"), -- queue2 SEC(".maps"), -- queue3 SEC(".maps"), -- queue4 SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, 5); -- __type(key, int); -- __array(values, struct qmap); --} queue_arr SEC(".maps") = { -- .values = { -- [0] = &queue0, -- [1] = &queue1, -- [2] = &queue2, -- [3] = &queue3, -- [4] = &queue4, -- }, --}; -- --/* -- * Per-queue sequence numbers to implement core-sched ordering. -- * -- * Tail seq is assigned to each queued task and incremented. Head seq tracks the -- * sequence number of the latest dispatched task. The distance between the a -- * task's seq and the associated queue's head seq is called the queue distance -- * and used when comparing two tasks for ordering. See qmap_core_sched_before(). -- */ --static u64 core_sched_head_seqs[5]; --static u64 core_sched_tail_seqs[5]; -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local_dsq */ -- u64 core_sched_seq; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --/* Per-cpu dispatch index and remaining count */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(max_entries, 2); -- __type(key, u32); -- __type(value, u64); --} dispatch_idx_cnt SEC(".maps"); -- --/* Statistics */ --unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued; --unsigned long nr_core_sched_execed; -- --s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- struct task_ctx *tctx; -- s32 cpu; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- return cpu; -- -- return prev_cpu; --} -- --static int weight_to_idx(u32 weight) --{ -- /* Coarsely map the compound weight to a FIFO. */ -- if (weight <= 25) -- return 0; -- else if (weight <= 50) -- return 1; -- else if (weight < 200) -- return 2; -- else if (weight < 400) -- return 3; -- else -- return 4; --} -- --void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) --{ -- static u32 user_cnt, kernel_cnt; -- struct task_ctx *tctx; -- u32 pid = p->pid; -- int idx = weight_to_idx(p->scx.weight); -- void *ring; -- -- if (p->flags & PF_KTHREAD) { -- if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth)) -- return; -- } else { -- if (stall_user_nth && !(++user_cnt % stall_user_nth)) -- return; -- } -- -- if (test_error_cnt && !--test_error_cnt) -- scx_bpf_error("test triggering error"); -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * All enqueued tasks must have their core_sched_seq updated for correct -- * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in -- * qmap_ops.flags. -- */ -- tctx->core_sched_seq = core_sched_tail_seqs[idx]++; -- -- /* -- * If qmap_select_cpu() is telling us to or this is the last runnable -- * task on the CPU, enqueue locally. -- */ -- if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) { -- tctx->force_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); -- return; -- } -- -- /* -- * If the task was re-enqueued due to the CPU being preempted by a -- * higher priority scheduling class, just re-enqueue the task directly -- * on the global DSQ. As we want another CPU to pick it up, find and -- * kick an idle CPU. -- */ -- if (enq_flags & SCX_ENQ_REENQ) { -- s32 cpu; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags); -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- return; -- } -- -- ring = bpf_map_lookup_elem(&queue_arr, &idx); -- if (!ring) { -- scx_bpf_error("failed to find ring %d", idx); -- return; -- } -- -- /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */ -- if (bpf_map_push_elem(ring, &pid, 0)) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_enqueued, 1); --} -- --/* -- * The BPF queue map doesn't support removal and sched_ext can handle spurious -- * dispatches. qmap_dequeue() is only used to collect statistics. -- */ --void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags) --{ -- __sync_fetch_and_add(&nr_dequeued, 1); -- if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) -- __sync_fetch_and_add(&nr_core_sched_execed, 1); --} -- --static void update_core_sched_head_seq(struct task_struct *p) --{ -- struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- int idx = weight_to_idx(p->scx.weight); -- -- if (tctx) -- core_sched_head_seqs[idx] = tctx->core_sched_seq; -- else -- scx_bpf_error("task_ctx lookup failed"); --} -- --void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 zero = 0, one = 1; -- u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero); -- u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one); -- void *fifo; -- s32 pid; -- int i; -- -- if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) { -- struct task_struct *p; -- -- /* -- * PID 2 should be kthreadd which should mostly be idle and off -- * the scheduler. Let's keep dispatching it to force the kernel -- * to call this function over and over again. -- */ -- p = bpf_task_from_pid(2); -- if (p) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- if (!idx || !cnt) { -- scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt); -- return; -- } -- -- for (i = 0; i < 5; i++) { -- /* Advance the dispatch cursor and pick the fifo. */ -- if (!*cnt) { -- *idx = (*idx + 1) % 5; -- *cnt = 1 << *idx; -- } -- (*cnt)--; -- -- fifo = bpf_map_lookup_elem(&queue_arr, idx); -- if (!fifo) { -- scx_bpf_error("failed to find ring %llu", *idx); -- return; -- } -- -- /* Dispatch or advance. */ -- if (!bpf_map_pop_elem(fifo, &pid)) { -- struct task_struct *p; -- -- p = bpf_task_from_pid(pid); -- if (p) { -- update_core_sched_head_seq(p); -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- *cnt = 0; -- } --} -- --/* -- * The distance from the head of the queue scaled by the weight of the queue. -- * The lower the number, the older the task and the higher the priority. -- */ --static s64 task_qdist(struct task_struct *p) --{ -- int idx = weight_to_idx(p->scx.weight); -- struct task_ctx *tctx; -- s64 qdist; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return 0; -- } -- -- qdist = tctx->core_sched_seq - core_sched_head_seqs[idx]; -- -- /* -- * As queue index increments, the priority doubles. The queue w/ index 3 -- * is dispatched twice more frequently than 2. Reflect the difference by -- * scaling qdists accordingly. Note that the shift amount needs to be -- * flipped depending on the sign to avoid flipping priority direction. -- */ -- if (qdist >= 0) -- return qdist << (4 - idx); -- else -- return qdist << idx; --} -- --/* -- * This is called to determine the task ordering when core-sched is picking -- * tasks to execute on SMT siblings and should encode about the same ordering as -- * the regular scheduling path. Use the priority-scaled distances from the head -- * of the queues to compare the two tasks which should be consistent with the -- * dispatch path behavior. -- */ --bool BPF_STRUCT_OPS(qmap_core_sched_before, -- struct task_struct *a, struct task_struct *b) --{ -- return task_qdist(a) > task_qdist(b); --} -- --void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- u32 cnt; -- -- /* -- * Called when @cpu is taken by a higher priority scheduling class. This -- * makes @cpu no longer available for executing sched_ext tasks. As we -- * don't want the tasks in @cpu's local dsq to sit there until @cpu -- * becomes available again, re-enqueue them into the global dsq. See -- * %SCX_ENQ_REENQ handling in qmap_enqueue(). -- */ -- cnt = scx_bpf_reenqueue_local(); -- if (cnt) -- __sync_fetch_and_add(&nr_reenqueued, cnt); --} -- --s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (p->tgid == disallow_tgid) -- p->scx.disallow = true; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(qmap_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops qmap_ops = { -- .select_cpu = (void *)qmap_select_cpu, -- .enqueue = (void *)qmap_enqueue, -- .dequeue = (void *)qmap_dequeue, -- .dispatch = (void *)qmap_dispatch, -- .core_sched_before = (void *)qmap_core_sched_before, -- .cpu_release = (void *)qmap_cpu_release, -- .prep_enable = (void *)qmap_prep_enable, -- .init = (void *)qmap_init, -- .exit = (void *)qmap_exit, -- .flags = SCX_OPS_ENQ_LAST, -- .timeout_ms = 5000U, -- .name = "qmap", --}; -diff --git a/tools/sched_ext/scx_example_qmap.c b/tools/sched_ext/scx_example_qmap.c -deleted file mode 100644 -index ccb4814ee61b..000000000000 ---- a/tools/sched_ext/scx_example_qmap.c -+++ /dev/null -@@ -1,107 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_qmap.skel.h" -- --const char help_fmt[] = --"A simple five-level FIFO queue sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" --" -t COUNT Stall every COUNT'th user thread\n" --" -T COUNT Stall every COUNT'th kernel thread\n" --" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n" --" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_qmap *skel; -- struct bpf_link *link; -- int opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_qmap__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) { -- switch (opt) { -- case 's': -- skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; -- break; -- case 'e': -- skel->bss->test_error_cnt = strtoul(optarg, NULL, 0); -- break; -- case 't': -- skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0); -- break; -- case 'T': -- skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0); -- break; -- case 'l': -- skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0); -- break; -- case 'd': -- skel->rodata->disallow_tgid = strtol(optarg, NULL, 0); -- if (skel->rodata->disallow_tgid < 0) -- skel->rodata->disallow_tgid = getpid(); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_qmap__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.qmap_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- long nr_enqueued = skel->bss->nr_enqueued; -- long nr_dispatched = skel->bss->nr_dispatched; -- -- printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n", -- nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, -- skel->bss->nr_reenqueued, skel->bss->nr_dequeued, -- skel->bss->nr_core_sched_execed); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_qmap__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_simple.bpf.c b/tools/sched_ext/scx_example_simple.bpf.c -deleted file mode 100644 -index 4bccca3e2047..000000000000 ---- a/tools/sched_ext/scx_example_simple.bpf.c -+++ /dev/null -@@ -1,128 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple scheduler. -- * -- * By default, it operates as a simple global weighted vtime scheduler and can -- * be switched to FIFO scheduling. It also demonstrates the following niceties. -- * -- * - Statistics tracking how many tasks are queued to local and global dsq's. -- * - Termination notification for userspace. -- * -- * While very simple, this scheduler should work reasonably well on CPUs with a -- * uniform L3 cache topology. While preemption is not implemented, the fact that -- * the scheduling queue is shared across all CPUs means that whatever is at the -- * front of the queue is likely to be executed fairly quickly given enough -- * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads -- * but comes with the usual problems with FIFO scheduling where saturating -- * threads can easily drown out interactive ones. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --static u64 vtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, 2); /* [local, global] */ --} stats SEC(".maps"); -- --static void stat_inc(u32 idx) --{ -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); -- if (cnt_p) -- (*cnt_p)++; --} -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) --{ -- /* -- * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that -- * running @p on its CPU directly shouldn't affect fairness. Just queue -- * it on the local FIFO. -- */ -- if (enq_flags & SCX_ENQ_LOCAL) { -- stat_inc(0); /* count local queueing */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- return; -- } -- -- stat_inc(1); /* count global queueing */ -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) -- vtime = vtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --void BPF_STRUCT_OPS(simple_running, struct task_struct *p) --{ -- if (fifo_sched) -- return; -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(vtime_now, p->scx.dsq_vtime)) -- vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --s32 BPF_STRUCT_OPS(simple_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .running = (void *)simple_running, -- .stopping = (void *)simple_stopping, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", --}; -diff --git a/tools/sched_ext/scx_example_simple.c b/tools/sched_ext/scx_example_simple.c -deleted file mode 100644 -index 486b401f7c95..000000000000 ---- a/tools/sched_ext/scx_example_simple.c -+++ /dev/null -@@ -1,101 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_simple.skel.h" -- --const char help_fmt[] = --"A simple sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-f] [-p]\n" --"\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int simple) --{ -- exit_req = 1; --} -- --static void read_stats(struct scx_example_simple *skel, u64 *stats) --{ -- int nr_cpus = libbpf_num_possible_cpus(); -- u64 cnts[2][nr_cpus]; -- u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * 2); -- -- for (idx = 0; idx < 2; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_simple *skel; -- struct bpf_link *link; -- u32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_simple__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "fph")) != -1) { -- switch (opt) { -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_simple__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.simple_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- u64 stats[2]; -- -- read_stats(skel, stats); -- printf("local=%lu global=%lu\n", stats[0], stats[1]); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_simple__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_userland.bpf.c b/tools/sched_ext/scx_example_userland.bpf.c -deleted file mode 100644 -index a089bc6bbe86..000000000000 ---- a/tools/sched_ext/scx_example_userland.bpf.c -+++ /dev/null -@@ -1,269 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A minimal userland scheduler. -- * -- * In terms of scheduling, this provides two different types of behaviors: -- * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity. -- * All such tasks are direct-dispatched from the kernel, and are never -- * enqueued in user space. -- * 2. A primitive vruntime scheduler that is implemented in user space, for all -- * other tasks. -- * -- * Some parts of this example user space scheduler could be implemented more -- * efficiently using more complex and sophisticated data structures. For -- * example, rather than using BPF_MAP_TYPE_QUEUE's, -- * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between -- * user space and kernel space. Similarly, we use a simple vruntime-sorted list -- * in user space, but an rbtree could be used instead. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include --#include "scx_common.bpf.h" --#include "scx_example_userland_common.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; --const volatile s32 usersched_pid; -- --/* !0 for veristat, set during init */ --const volatile u32 num_possible_cpus = 64; -- --/* Stats that are printed by user space. */ --u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues; -- --struct user_exit_info uei; -- --/* -- * Whether the user space scheduler needs to be scheduled due to a task being -- * enqueued in user space. -- */ --static bool usersched_needed; -- --/* -- * The map containing tasks that are enqueued in user space from the kernel. -- * -- * This map is drained by the user space scheduler. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, struct scx_userland_enqueued_task); --} enqueued SEC(".maps"); -- --/* -- * The map containing tasks that are dispatched to the kernel from user space. -- * -- * Drained by the kernel in userland_dispatch(). -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, s32); --} dispatched SEC(".maps"); -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local DSQ */ --}; -- --/* Map that contains task-local storage. */ --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --static bool is_usersched_task(const struct task_struct *p) --{ -- return p->pid == usersched_pid; --} -- --static bool keep_in_kernel(const struct task_struct *p) --{ -- return p->nr_cpus_allowed < num_possible_cpus; --} -- --static struct task_struct *usersched_task(void) --{ -- struct task_struct *p; -- -- p = bpf_task_from_pid(usersched_pid); -- /* -- * Should never happen -- the usersched task should always be managed -- * by sched_ext. -- */ -- if (!p) { -- scx_bpf_error("Failed to find usersched task %d", usersched_pid); -- /* -- * We should never hit this path, and we error out of the -- * scheduler above just in case, so the scheduler will soon be -- * be evicted regardless. So as to simplify the logic in the -- * caller to not have to check for NULL, return an acquired -- * reference to the current task here rather than NULL. -- */ -- return bpf_task_acquire(bpf_get_current_task_btf()); -- } -- -- return p; --} -- --s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- if (keep_in_kernel(p)) { -- s32 cpu; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to look up task-local storage for %s", p->comm); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- tctx->force_local = true; -- return cpu; -- } -- } -- -- return prev_cpu; --} -- --static void dispatch_user_scheduler(void) --{ -- struct task_struct *p; -- -- usersched_needed = false; -- p = usersched_task(); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); --} -- --static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags) --{ -- struct scx_userland_enqueued_task task; -- -- memset(&task, 0, sizeof(task)); -- task.pid = p->pid; -- task.sum_exec_runtime = p->se.sum_exec_runtime; -- task.weight = p->scx.weight; -- -- if (bpf_map_push_elem(&enqueued, &task, 0)) { -- /* -- * If we fail to enqueue the task in user space, put it -- * directly on the global DSQ. -- */ -- __sync_fetch_and_add(&nr_failed_enqueues, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- __sync_fetch_and_add(&nr_user_enqueues, 1); -- usersched_needed = true; -- } --} -- --void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags) --{ -- if (keep_in_kernel(p)) { -- u64 dsq_id = SCX_DSQ_GLOBAL; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to lookup task ctx for %s", p->comm); -- return; -- } -- -- if (tctx->force_local) -- dsq_id = SCX_DSQ_LOCAL; -- tctx->force_local = false; -- scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags); -- __sync_fetch_and_add(&nr_kernel_enqueues, 1); -- return; -- } else if (!is_usersched_task(p)) { -- enqueue_task_in_user_space(p, enq_flags); -- } --} -- --void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (usersched_needed) -- dispatch_user_scheduler(); -- -- bpf_repeat(4096) { -- s32 pid; -- struct task_struct *p; -- -- if (bpf_map_pop_elem(&dispatched, &pid)) -- break; -- -- /* -- * The task could have exited by the time we get around to -- * dispatching it. Treat this as a normal occurrence, and simply -- * move onto the next iteration. -- */ -- p = bpf_task_from_pid(pid); -- if (!p) -- continue; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } --} -- --s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(userland_init) --{ -- if (num_possible_cpus == 0) { -- scx_bpf_error("User scheduler # CPUs uninitialized (%d)", -- num_possible_cpus); -- return -EINVAL; -- } -- -- if (usersched_pid <= 0) { -- scx_bpf_error("User scheduler pid uninitialized (%d)", -- usersched_pid); -- return -EINVAL; -- } -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops userland_ops = { -- .select_cpu = (void *)userland_select_cpu, -- .enqueue = (void *)userland_enqueue, -- .dispatch = (void *)userland_dispatch, -- .prep_enable = (void *)userland_prep_enable, -- .init = (void *)userland_init, -- .exit = (void *)userland_exit, -- .timeout_ms = 3000, -- .name = "userland", --}; -diff --git a/tools/sched_ext/scx_example_userland.c b/tools/sched_ext/scx_example_userland.c -deleted file mode 100644 -index 4152b1e65fe1..000000000000 ---- a/tools/sched_ext/scx_example_userland.c -+++ /dev/null -@@ -1,402 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext user space scheduler which provides vruntime semantics -- * using a simple ordered-list implementation. -- * -- * Each CPU in the system resides in a single, global domain. This precludes -- * the need to do any load balancing between domains. The scheduler could -- * easily be extended to support multiple domains, with load balancing -- * happening in user space. -- * -- * Any task which has any CPU affinity is scheduled entirely in BPF. This -- * program only schedules tasks which may run on any CPU. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include "user_exit_info.h" --#include "scx_example_userland_common.h" --#include "scx_example_userland.skel.h" -- --const char help_fmt[] = --"A minimal userland sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-b BATCH] [-p]\n" --"\n" --" -b BATCH The number of tasks to batch when dispatching (default: 8)\n" --" -p Don't switch all, switch only tasks on SCHED_EXT policy\n" --" -h Display this help and exit\n"; -- --/* Defined in UAPI */ --#define SCHED_EXT 7 -- --/* Number of tasks to batch when dispatching to user space. */ --static __u32 batch_size = 8; -- --static volatile int exit_req; --static int enqueued_fd, dispatched_fd; -- --static struct scx_example_userland *skel; --static struct bpf_link *ops_link; -- --/* Stats collected in user space. */ --static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches; -- --/* The data structure containing tasks that are enqueued in user space. */ --struct enqueued_task { -- LIST_ENTRY(enqueued_task) entries; -- __u64 sum_exec_runtime; -- double vruntime; --}; -- --/* -- * Use a vruntime-sorted list to store tasks. This could easily be extended to -- * a more optimal data structure, such as an rbtree as is done in CFS. We -- * currently elect to use a sorted list to simplify the example for -- * illustrative purposes. -- */ --LIST_HEAD(listhead, enqueued_task); -- --/* -- * A vruntime-sorted list of tasks. The head of the list contains the task with -- * the lowest vruntime. That is, the task that has the "highest" claim to be -- * scheduled. -- */ --static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head); -- --/* -- * The statically allocated array of tasks. We use a statically allocated list -- * here to avoid having to allocate on the enqueue path, which could cause a -- * deadlock. A more substantive user space scheduler could e.g. provide a hook -- * for newly enabled tasks that are passed to the scheduler from the -- * .prep_enable() callback to allows the scheduler to allocate on safe paths. -- */ --struct enqueued_task tasks[USERLAND_MAX_TASKS]; -- --static double min_vruntime; -- --static void sigint_handler(int userland) --{ -- exit_req = 1; --} -- --static __u32 task_pid(const struct enqueued_task *task) --{ -- return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task); --} -- --static int dispatch_task(s32 pid) --{ -- int err; -- -- err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d\n", pid); -- exit_req = 1; -- } else { -- nr_vruntime_dispatches++; -- } -- -- return err; --} -- --static struct enqueued_task *get_enqueued_task(__s32 pid) --{ -- if (pid >= USERLAND_MAX_TASKS) -- return NULL; -- -- return &tasks[pid]; --} -- --static double calc_vruntime_delta(__u64 weight, __u64 delta) --{ -- double weight_f = (double)weight / 100.0; -- double delta_f = (double)delta; -- -- return delta_f / weight_f; --} -- --static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task) --{ -- __u64 delta; -- -- delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime; -- -- enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta); -- if (min_vruntime > enqueued->vruntime) -- enqueued->vruntime = min_vruntime; -- enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime; --} -- --static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task) --{ -- struct enqueued_task *curr, *enqueued, *prev; -- -- curr = get_enqueued_task(bpf_task->pid); -- if (!curr) -- return ENOENT; -- -- update_enqueued(curr, bpf_task); -- nr_vruntime_enqueues++; -- -- /* -- * Enqueue the task in a vruntime-sorted list. A more optimal data -- * structure such as an rbtree could easily be used as well. We elect -- * to use a list here simply because it's less code, and thus the -- * example is less convoluted and better serves to illustrate what a -- * user space scheduler could look like. -- */ -- -- if (LIST_EMPTY(&vruntime_head)) { -- LIST_INSERT_HEAD(&vruntime_head, curr, entries); -- return 0; -- } -- -- LIST_FOREACH(enqueued, &vruntime_head, entries) { -- if (curr->vruntime <= enqueued->vruntime) { -- LIST_INSERT_BEFORE(enqueued, curr, entries); -- return 0; -- } -- prev = enqueued; -- } -- -- LIST_INSERT_AFTER(prev, curr, entries); -- -- return 0; --} -- --static void drain_enqueued_map(void) --{ -- while (1) { -- struct scx_userland_enqueued_task task; -- int err; -- -- if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) -- return; -- -- err = vruntime_enqueue(&task); -- if (err) { -- fprintf(stderr, "Failed to enqueue task %d: %s\n", -- task.pid, strerror(err)); -- exit_req = 1; -- return; -- } -- } --} -- --static void dispatch_batch(void) --{ -- __u32 i; -- -- for (i = 0; i < batch_size; i++) { -- struct enqueued_task *task; -- int err; -- __s32 pid; -- -- task = LIST_FIRST(&vruntime_head); -- if (!task) -- return; -- -- min_vruntime = task->vruntime; -- pid = task_pid(task); -- LIST_REMOVE(task, entries); -- err = dispatch_task(pid); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d in %u\n", -- pid, i); -- return; -- } -- } --} -- --static void *run_stats_printer(void *arg) --{ -- while (!exit_req) { -- __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; -- -- nr_failed_enqueues = skel->bss->nr_failed_enqueues; -- nr_kernel_enqueues = skel->bss->nr_kernel_enqueues; -- nr_user_enqueues = skel->bss->nr_user_enqueues; -- total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues; -- -- printf("o-----------------------o\n"); -- printf("| BPF ENQUEUES |\n"); -- printf("|-----------------------|\n"); -- printf("| kern: %10llu |\n", nr_kernel_enqueues); -- printf("| user: %10llu |\n", nr_user_enqueues); -- printf("| failed: %10llu |\n", nr_failed_enqueues); -- printf("| -------------------- |\n"); -- printf("| total: %10llu |\n", total); -- printf("| |\n"); -- printf("|-----------------------|\n"); -- printf("| VRUNTIME / USER |\n"); -- printf("|-----------------------|\n"); -- printf("| enq: %10llu |\n", nr_vruntime_enqueues); -- printf("| disp: %10llu |\n", nr_vruntime_dispatches); -- printf("o-----------------------o\n"); -- printf("\n\n"); -- sleep(1); -- } -- -- return NULL; --} -- --static int spawn_stats_thread(void) --{ -- pthread_t stats_printer; -- -- return pthread_create(&stats_printer, NULL, run_stats_printer, NULL); --} -- --static int bootstrap(int argc, char **argv) --{ -- int err; -- __u32 opt; -- struct sched_param sched_param = { -- .sched_priority = sched_get_priority_max(SCHED_EXT), -- }; -- bool switch_partial = false; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- /* -- * Enforce that the user scheduler task is managed by sched_ext. The -- * task eagerly drains the list of enqueued tasks in its main work -- * loop, and then yields the CPU. The BPF scheduler only schedules the -- * user space scheduler task when at least one other task in the system -- * needs to be scheduled. -- */ -- err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param); -- if (err) { -- fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err)); -- return err; -- } -- -- while ((opt = getopt(argc, argv, "b:ph")) != -1) { -- switch (opt) { -- case 'b': -- batch_size = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- exit(opt != 'h'); -- } -- } -- -- /* -- * It's not always safe to allocate in a user space scheduler, as an -- * enqueued task could hold a lock that we require in order to be able -- * to allocate. -- */ -- err = mlockall(MCL_CURRENT | MCL_FUTURE); -- if (err) { -- fprintf(stderr, "Failed to prefault and lock address space: %s\n", -- strerror(err)); -- return err; -- } -- -- skel = scx_example_userland__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno)); -- return errno; -- } -- skel->rodata->num_possible_cpus = libbpf_num_possible_cpus(); -- assert(skel->rodata->num_possible_cpus > 0); -- skel->rodata->usersched_pid = getpid(); -- assert(skel->rodata->usersched_pid > 0); -- skel->rodata->switch_partial = switch_partial; -- -- err = scx_example_userland__load(skel); -- if (err) { -- fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- enqueued_fd = bpf_map__fd(skel->maps.enqueued); -- dispatched_fd = bpf_map__fd(skel->maps.dispatched); -- assert(enqueued_fd > 0); -- assert(dispatched_fd > 0); -- -- err = spawn_stats_thread(); -- if (err) { -- fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops); -- if (!ops_link) { -- fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno)); -- err = errno; -- goto destroy_skel; -- } -- -- return 0; -- --destroy_skel: -- scx_example_userland__destroy(skel); -- exit_req = 1; -- return err; --} -- --static void sched_main_loop(void) --{ -- while (!exit_req) { -- /* -- * Perform the following work in the main user space scheduler -- * loop: -- * -- * 1. Drain all tasks from the enqueued map, and enqueue them -- * to the vruntime sorted list. -- * -- * 2. Dispatch a batch of tasks from the vruntime sorted list -- * down to the kernel. -- * -- * 3. Yield the CPU back to the system. The BPF scheduler will -- * reschedule the user space scheduler once another task has -- * been enqueued to user space. -- */ -- drain_enqueued_map(); -- dispatch_batch(); -- sched_yield(); -- } --} -- --int main(int argc, char **argv) --{ -- int err; -- -- err = bootstrap(argc, argv); -- if (err) { -- fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err)); -- return err; -- } -- -- sched_main_loop(); -- -- exit_req = 1; -- bpf_link__destroy(ops_link); -- uei_print(&skel->bss->uei); -- scx_example_userland__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_userland_common.h b/tools/sched_ext/scx_example_userland_common.h -deleted file mode 100644 -index 639c6809c5ff..000000000000 ---- a/tools/sched_ext/scx_example_userland_common.h -+++ /dev/null -@@ -1,19 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* Copyright (c) 2022 Meta, Inc */ -- --#ifndef __SCX_USERLAND_COMMON_H --#define __SCX_USERLAND_COMMON_H -- --#define USERLAND_MAX_TASKS 8192 -- --/* -- * An instance of a task that has been enqueued by the kernel for consumption -- * by a user space global scheduler thread. -- */ --struct scx_userland_enqueued_task { -- __s32 pid; -- u64 sum_exec_runtime; -- u64 weight; --}; -- --#endif // __SCX_USERLAND_COMMON_H -diff --git a/tools/sched_ext/user_exit_info.h b/tools/sched_ext/user_exit_info.h -deleted file mode 100644 -index e701ef0e0b86..000000000000 ---- a/tools/sched_ext/user_exit_info.h -+++ /dev/null -@@ -1,50 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Define struct user_exit_info which is shared between BPF and userspace parts -- * to communicate exit status and other information. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __USER_EXIT_INFO_H --#define __USER_EXIT_INFO_H -- --struct user_exit_info { -- int type; -- char reason[128]; -- char msg[1024]; --}; -- --#ifdef __bpf__ -- --#include "vmlinux.h" --#include -- --static inline void uei_record(struct user_exit_info *uei, -- const struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason); -- bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg); -- /* use __sync to force memory barrier */ -- __sync_val_compare_and_swap(&uei->type, uei->type, ei->type); --} -- --#else /* !__bpf__ */ -- --static inline bool uei_exited(struct user_exit_info *uei) --{ -- /* use __sync to force memory barrier */ -- return __sync_val_compare_and_swap(&uei->type, -1, -1); --} -- --static inline void uei_print(const struct user_exit_info *uei) --{ -- fprintf(stderr, "EXIT: %s", uei->reason); -- if (uei->msg[0] != '\0') -- fprintf(stderr, " (%s)", uei->msg); -- fputs("\n", stderr); --} -- --#endif /* __bpf__ */ --#endif /* __USER_EXIT_INFO_H */ -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -index ce05928392bf..f47f79079297 100755 ---- a/kernel/sched/hmbird/hmbird.c -+++ b/kernel/sched/hmbird/hmbird.c -@@ -633,14 +633,14 @@ static void init_isolate_cpus(void) - WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -- cpumask_set_cpu(0, iso_masks.big); -- cpumask_set_cpu(1, iso_masks.big); -- cpumask_set_cpu(2, iso_masks.big); -- cpumask_set_cpu(3, iso_masks.big); -- cpumask_set_cpu(4, iso_masks.big); -- cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(0, iso_masks.little); -+ cpumask_set_cpu(1, iso_masks.little); -+ cpumask_set_cpu(2, iso_masks.little); -+ cpumask_set_cpu(3, iso_masks.little); -+ cpumask_set_cpu(4, iso_masks.little); -+ cpumask_set_cpu(5, iso_masks.little); - cpumask_set_cpu(6, iso_masks.partial); -- cpumask_set_cpu(7, iso_masks.ex_free); -+ cpumask_set_cpu(7, iso_masks.big); - } - - extern spinlock_t css_set_lock; diff --git a/oneplus/hmbird/fengchi_OP13S_A16.patch b/oneplus/hmbird/fengchi_OP13S_A16.patch deleted file mode 100644 index e6008b42..00000000 --- a/oneplus/hmbird/fengchi_OP13S_A16.patch +++ /dev/null @@ -1,19909 +0,0 @@ -diff --git b/Documentation/scheduler/index.rst a/Documentation/scheduler/index.rst -index 0b650bb550e6..3170747226f6 100644 ---- b/Documentation/scheduler/index.rst -+++ a/Documentation/scheduler/index.rst -@@ -19,7 +19,6 @@ Scheduler - sched-nice-design - sched-rt-group - sched-stats -- sched-ext - sched-debug - - text_files -diff --git b/Documentation/scheduler/sched-ext.rst a/Documentation/scheduler/sched-ext.rst -deleted file mode 100644 -index 84c30b44f104..000000000000 ---- b/Documentation/scheduler/sched-ext.rst -+++ /dev/null -@@ -1,230 +0,0 @@ --========================== --Extensible Scheduler Class --========================== -- --sched_ext is a scheduler class whose behavior can be defined by a set of BPF --programs - the BPF scheduler. -- --* sched_ext exports a full scheduling interface so that any scheduling -- algorithm can be implemented on top. -- --* The BPF scheduler can group CPUs however it sees fit and schedule them -- together, as tasks aren't tied to specific CPUs at the time of wakeup. -- --* The BPF scheduler can be turned on and off dynamically anytime. -- --* The system integrity is maintained no matter what the BPF scheduler does. -- The default scheduling behavior is restored anytime an error is detected, -- a runnable task stalls, or on invoking the SysRq key sequence -- :kbd:`SysRq-S`. -- --Switching to and from sched_ext --=============================== -- --``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and --``tools/sched_ext`` contains the example schedulers. -- --sched_ext is used only when the BPF scheduler is loaded and running. -- --If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be --treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is --loaded. On load, such tasks will be switched to and scheduled by sched_ext. -- --The BPF scheduler can choose to schedule all normal and lower class tasks by --calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this --case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and --``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers, --this mode can be selected with the ``-a`` option. -- --Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or --detection of any internal error including stalled runnable tasks aborts the --BPF scheduler and reverts all tasks back to CFS. -- --.. code-block:: none -- -- # make -j16 -C tools/sched_ext -- # tools/sched_ext/scx_example_simple -- local=0 global=3 -- local=5 global=24 -- local=9 global=44 -- local=13 global=56 -- local=17 global=72 -- ^CEXIT: BPF scheduler unregistered -- --If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler --and whether a given task is on sched_ext can be determined as follows: -- --.. code-block:: none -- -- # cat /sys/kernel/debug/sched/ext -- ops : simple -- enabled : 1 -- switching_all : 1 -- switched_all : 1 -- enable_state : enabled -- -- # grep ext /proc/self/sched -- ext.enabled : 1 -- --The Basics --========== -- --Userspace can implement an arbitrary BPF scheduler by loading a set of BPF --programs that implement ``struct sched_ext_ops``. The only mandatory field --is ``ops.name`` which must be a valid BPF object name. All operations are --optional. The following modified excerpt is from --``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO --scheduler. -- --.. code-block:: c -- -- s32 BPF_STRUCT_OPS(simple_init) -- { -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; -- } -- -- void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) -- { -- if (enq_flags & SCX_ENQ_LOCAL) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags); -- } -- -- void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) -- { -- exit_type = ei->type; -- } -- -- SEC(".struct_ops") -- struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", -- }; -- --Dispatch Queues ----------------- -- --To match the impedance between the scheduler core and the BPF scheduler, --sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a --priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``), --and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage --an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and --``scx_bpf_destroy_dsq()``. -- --A CPU always executes a task from its local DSQ. A task is "dispatched" to a --DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's --local DSQ. -- --When a CPU is looking for the next task to run, if the local DSQ is not --empty, the first task is picked. Otherwise, the CPU tries to consume the --global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()`` --is invoked. -- --Scheduling Cycle ------------------ -- --The following briefly shows how a waking task is scheduled and executed. -- --1. When a task is waking up, ``ops.select_cpu()`` is the first operation -- invoked. This serves two purposes. First, CPU selection optimization -- hint. Second, waking up the selected CPU if idle. -- -- The CPU selected by ``ops.select_cpu()`` is an optimization hint and not -- binding. The actual decision is made at the last step of scheduling. -- However, there is a small performance gain if the CPU -- ``ops.select_cpu()`` returns matches the CPU the task eventually runs on. -- -- A side-effect of selecting a CPU is waking it up from idle. While a BPF -- scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper, -- using ``ops.select_cpu()`` judiciously can be simpler and more efficient. -- -- Note that the scheduler core will ignore an invalid CPU selection, for -- example, if it's outside the allowed cpumask of the task. -- --2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can -- make one of the following decisions: -- -- * Immediately dispatch the task to either the global or local DSQ by -- calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or -- ``SCX_DSQ_LOCAL``, respectively. -- -- * Immediately dispatch the task to a custom DSQ by calling -- ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63. -- -- * Queue the task on the BPF side. -- --3. When a CPU is ready to schedule, it first looks at its local DSQ. If -- empty, it then looks at the global DSQ. If there still isn't a task to -- run, ``ops.dispatch()`` is invoked which can use the following two -- functions to populate the local DSQ. -- -- * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can -- be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``, -- ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()`` -- currently can't be called with BPF locks held, this is being worked on -- and will be supported. ``scx_bpf_dispatch()`` schedules dispatching -- rather than performing them immediately. There can be up to -- ``ops.dispatch_max_batch`` pending tasks. -- -- * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ -- to the dispatching DSQ. This function cannot be called with any BPF -- locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks -- before trying to consume the specified DSQ. -- --4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ, -- the CPU runs the first one. If empty, the following steps are taken: -- -- * Try to consume the global DSQ. If successful, run the task. -- -- * If ``ops.dispatch()`` has dispatched any tasks, retry #3. -- -- * If the previous task is an SCX task and still runnable, keep executing -- it (see ``SCX_OPS_ENQ_LAST``). -- -- * Go idle. -- --Note that the BPF scheduler can always choose to dispatch tasks immediately --in ``ops.enqueue()`` as illustrated in the above simple example. If only the --built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as --a task is never queued on the BPF scheduler and both the local and global --DSQs are consumed automatically. -- --``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use --``scx_bpf_dispatch_vtime()`` for the priority queue. See the function --documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for --more information. -- --Where to Look --============= -- --* ``include/linux/sched/ext.h`` defines the core data structures, ops table -- and constants. -- --* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. -- The functions prefixed with ``scx_bpf_`` can be called from the BPF -- scheduler. -- --* ``tools/sched_ext/`` hosts example BPF scheduler implementations. -- -- * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example -- using a custom DSQ. -- -- * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting -- five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. -- --ABI Instability --=============== -- --The APIs provided by sched_ext to BPF schedulers programs have no stability --guarantees. This includes the ops table callbacks and constants defined in --``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in --``kernel/sched/ext.c``. -- --While we will attempt to provide a relatively stable API surface when --possible, they are subject to change without warning between kernel --versions. -diff --git b/MAINTAINERS a/MAINTAINERS -index 9fc6108503c3..2384b55682ee 100644 ---- b/MAINTAINERS -+++ a/MAINTAINERS -@@ -19132,8 +19132,6 @@ R: Ben Segall (CONFIG_CFS_BANDWIDTH) - R: Mel Gorman (CONFIG_NUMA_BALANCING) - R: Daniel Bristot de Oliveira (SCHED_DEADLINE) - R: Valentin Schneider (TOPOLOGY) --R: Tejun Heo (SCHED_EXT) --R: David Vernet (SCHED_EXT) - L: linux-kernel@vger.kernel.org - S: Maintained - T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core -@@ -19142,7 +19140,6 @@ F: include/linux/sched.h - F: include/linux/wait.h - F: include/uapi/linux/sched.h - F: kernel/sched/ --F: tools/sched_ext/ - - SCSI LIBSAS SUBSYSTEM - R: John Garry -diff --git b/arch/arm64/configs/defconfig a/arch/arm64/configs/defconfig -index f93980c09d7f..c2d530535980 100644 ---- b/arch/arm64/configs/defconfig -+++ a/arch/arm64/configs/defconfig -@@ -6,8 +6,6 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_BPF_SYSCALL=y - CONFIG_BPF_JIT=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_BSD_PROCESS_ACCT=y - CONFIG_BSD_PROCESS_ACCT_V3=y -@@ -1587,3 +1585,4 @@ CONFIG_CORESIGHT_STM=m - CONFIG_CORESIGHT_CPU_DEBUG=m - CONFIG_CORESIGHT_CTI=m - CONFIG_MEMTEST=y -+CONFIG_HMBIRD_SCHED=y -diff --git b/arch/arm64/configs/gki_defconfig a/arch/arm64/configs/gki_defconfig -index 4f756dca5e05..6c1bb94f875d 100644 ---- b/arch/arm64/configs/gki_defconfig -+++ a/arch/arm64/configs/gki_defconfig -@@ -10,8 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y - # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set - CONFIG_BPF_LSM=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_TASKSTATS=y - CONFIG_TASK_DELAY_ACCT=y -diff --git b/drivers/gpu/drm/msm/msm_drv.c a/drivers/gpu/drm/msm/msm_drv.c -index 4bd028fa7500..5825ce9d6d7b 100755 ---- b/drivers/gpu/drm/msm/msm_drv.c -+++ a/drivers/gpu/drm/msm/msm_drv.c -@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) - } - - sched_set_fifo(ev_thread->worker->task); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3); -+#endif - } - - ret = drm_vblank_init(ddev, priv->num_crtcs); -diff --git b/drivers/tty/sysrq.c a/drivers/tty/sysrq.c -index bd001445815e..dcf2e1ebf9c5 100644 ---- b/drivers/tty/sysrq.c -+++ a/drivers/tty/sysrq.c -@@ -524,7 +524,6 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { - NULL, /* P */ - NULL, /* Q */ - NULL, /* R */ -- /* S: May be registered by sched_ext for resetting */ - NULL, /* S */ - NULL, /* T */ - NULL, /* U */ -diff --git b/include/asm-generic/vmlinux.lds.h a/include/asm-generic/vmlinux.lds.h -index c45078a445c8..6c15659f874e 100755 ---- b/include/asm-generic/vmlinux.lds.h -+++ a/include/asm-generic/vmlinux.lds.h -@@ -131,7 +131,7 @@ - *(__dl_sched_class) \ - *(__rt_sched_class) \ - *(__fair_sched_class) \ -- *(__ext_sched_class) \ -+ *(__hmbird_sched_class) \ - *(__idle_sched_class) \ - __sched_class_lowest = .; - -diff --git b/include/linux/sched.h a/include/linux/sched.h -index ba49d7bd2b67..dd6b1d58af01 100755 ---- b/include/linux/sched.h -+++ a/include/linux/sched.h -@@ -73,7 +73,9 @@ struct task_delay_info; - struct task_group; - struct user_event_mm; - --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - - /* - * Task state bitmask. NOTE! These bits are also -@@ -298,11 +300,6 @@ enum { - - extern void scheduler_tick(void); - --#ifdef CONFIG_SLIM_SCHED --extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer); --extern void stop_shadow_tick_timer(void); --#endif -- - #define MAX_SCHEDULE_TIMEOUT LONG_MAX - - extern long schedule_timeout(long timeout); -@@ -1523,13 +1520,8 @@ struct task_struct { - */ - struct callback_head l1d_flush_kill; - #endif --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, unsigned long sched_prop); -- ANDROID_KABI_USE(2, struct sched_ext_entity *scx); --#else - ANDROID_KABI_RESERVE(1); - ANDROID_KABI_RESERVE(2); --#endif - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); - ANDROID_KABI_RESERVE(5); -@@ -1933,6 +1925,91 @@ static inline int task_nice(const struct task_struct *p) - return PRIO_TO_NICE((p)->static_prio); - } - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline bool task_is_top_task(struct task_struct *p) -+{ -+ return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ ->top_task_prop & TOP_TASK_BITS_MASK); -+} -+ -+static inline int get_top_task_prop(struct task_struct *p) -+{ -+ return get_hmbird_ts(p)->top_task_prop; -+} -+ -+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear) -+{ -+ if (set) -+ get_hmbird_ts(p)->top_task_prop |= set; -+ if (clear) -+ get_hmbird_ts(p)->top_task_prop &= ~clear; -+ return 0; -+} -+ -+static inline void reset_top_task_prop(struct task_struct *p) -+{ -+ get_hmbird_ts(p)->top_task_prop = 0; -+} -+ -+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ entity->sched_prop = sp; -+ } -+ return 0; -+} -+ -+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->sched_prop; -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq) -+{ -+ unsigned long new_dsq; -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq; -+ entity->sched_prop = new_dsq; -+ } -+} -+ -+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK); -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ entity->dsq_sync_ux = val; -+} -+ -+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->dsq_sync_ux; -+ else -+ return 0; -+} -+#endif - - extern int can_nice(const struct task_struct *p, const int nice); - extern int task_curr(const struct task_struct *p); -diff --git b/include/linux/sched/ext.h a/include/linux/sched/ext.h -deleted file mode 100755 -index b737a00c1373..000000000000 ---- b/include/linux/sched/ext.h -+++ /dev/null -@@ -1,647 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef _LINUX_SCHED_EXT_H --#define _LINUX_SCHED_EXT_H -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --#include -- --enum scx_consts { -- SCX_OPS_NAME_LEN = 128, -- SCX_EXIT_REASON_LEN = 128, -- SCX_EXIT_BT_LEN = 64, -- SCX_EXIT_MSG_LEN = 1024, -- -- SCX_SLICE_DFL = 20 * NSEC_PER_MSEC, -- SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */ --}; -- --/* -- * DSQ (dispatch queue) IDs are 64bit of the format: -- * -- * Bits: [63] [62 .. 0] -- * [ B] [ ID ] -- * -- * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -- * ID: 63 bit ID -- * -- * Built-in IDs: -- * -- * Bits: [63] [62] [61..32] [31 .. 0] -- * [ 1] [ L] [ R ] [ V ] -- * -- * 1: 1 for built-in DSQs. -- * L: 1 for LOCAL_ON DSQ IDs, 0 for others -- * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -- */ --enum scx_dsq_id_flags { -- SCX_DSQ_FLAG_BUILTIN = 1LLU << 63, -- SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -- -- SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0, -- SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, -- SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, -- SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, -- SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, --}; -- --enum scx_exit_type { -- SCX_EXIT_NONE, -- SCX_EXIT_DONE, -- -- SCX_EXIT_UNREG = 64, /* BPF unregistration */ -- SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -- SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ -- SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ --}; -- --/* -- * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is -- * being disabled. -- */ --struct scx_exit_info { -- /* %SCX_EXIT_* - broad category of the exit reason */ -- enum scx_exit_type type; -- /* textual representation of the above */ -- char reason[SCX_EXIT_REASON_LEN]; -- /* number of entries in the backtrace */ -- u32 bt_len; -- /* backtrace if exiting due to an error */ -- unsigned long bt[SCX_EXIT_BT_LEN]; -- /* extra message */ -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --/* sched_ext_ops.flags */ --enum scx_ops_flags { -- /* -- * Keep built-in idle tracking even if ops.update_idle() is implemented. -- */ -- SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, -- -- /* -- * By default, if there are no other task to run on the CPU, ext core -- * keeps running the current task even after its slice expires. If this -- * flag is specified, such tasks are passed to ops.enqueue() with -- * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. -- */ -- SCX_OPS_ENQ_LAST = 1LLU << 1, -- -- /* -- * An exiting task may schedule after PF_EXITING is set. In such cases, -- * bpf_task_from_pid() may not be able to find the task and if the BPF -- * scheduler depends on pid lookup for dispatching, the task will be -- * lost leading to various issues including RCU grace period stalls. -- * -- * To mask this problem, by default, unhashed tasks are automatically -- * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't -- * depend on pid lookups and wants to handle these tasks directly, the -- * following flag can be used. -- */ -- SCX_OPS_ENQ_EXITING = 1LLU << 2, -- -- /* -- * CPU cgroup knob enable flags -- */ -- SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16, /* cpu.weight */ -- -- SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | -- SCX_OPS_ENQ_LAST | -- SCX_OPS_ENQ_EXITING | -- SCX_OPS_CGROUP_KNOB_WEIGHT, --}; -- --/* argument container for ops.enable() and friends */ --struct scx_enable_args { --}; -- --/* argument container for ops->cgroup_init() */ --struct scx_cgroup_init_args { -- /* the weight of the cgroup [1..10000] */ -- u32 weight; --}; -- --enum scx_cpu_preempt_reason { -- /* next task is being scheduled by &sched_class_rt */ -- SCX_CPU_PREEMPT_RT, -- /* next task is being scheduled by &sched_class_dl */ -- SCX_CPU_PREEMPT_DL, -- /* next task is being scheduled by &sched_class_stop */ -- SCX_CPU_PREEMPT_STOP, -- /* unknown reason for SCX being preempted */ -- SCX_CPU_PREEMPT_UNKNOWN, --}; -- --/* -- * Argument container for ops->cpu_acquire(). Currently empty, but may be -- * expanded in the future. -- */ --struct scx_cpu_acquire_args {}; -- --/* argument container for ops->cpu_release() */ --struct scx_cpu_release_args { -- /* the reason the CPU was preempted */ -- enum scx_cpu_preempt_reason reason; -- -- /* the task that's going to be scheduled on the CPU */ -- struct task_struct *task; --}; -- --/** -- * struct sched_ext_ops - Operation table for BPF scheduler implementation -- * -- * Userland can implement an arbitrary scheduling policy by implementing and -- * loading operations in this table. -- */ --struct sched_ext_ops { -- /** -- * select_cpu - Pick the target CPU for a task which is being woken up -- * @p: task being woken up -- * @prev_cpu: the cpu @p was on before sleeping -- * @wake_flags: SCX_WAKE_* -- * -- * Decision made here isn't final. @p may be moved to any CPU while it -- * is getting dispatched for execution later. However, as @p is not on -- * the rq at this point, getting the eventual execution CPU right here -- * saves a small bit of overhead down the line. -- * -- * If an idle CPU is returned, the CPU is kicked and will try to -- * dispatch. While an explicit custom mechanism can be added, -- * select_cpu() serves as the default way to wake up idle CPUs. -- */ -- s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); -- -- /** -- * enqueue - Enqueue a task on the BPF scheduler -- * @p: task being enqueued -- * @enq_flags: %SCX_ENQ_* -- * -- * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() -- * or enqueue on the BPF scheduler. If not directly dispatched, the bpf -- * scheduler owns @p and if it fails to dispatch @p, the task will -- * stall. -- */ -- void (*enqueue)(struct task_struct *p, u64 enq_flags); -- -- /** -- * dequeue - Remove a task from the BPF scheduler -- * @p: task being dequeued -- * @deq_flags: %SCX_DEQ_* -- * -- * Remove @p from the BPF scheduler. This is usually called to isolate -- * the task while updating its scheduling properties (e.g. priority). -- * -- * The ext core keeps track of whether the BPF side owns a given task or -- * not and can gracefully ignore spurious dispatches from BPF side, -- * which makes it safe to not implement this method. However, depending -- * on the scheduling logic, this can lead to confusing behaviors - e.g. -- * scheduling position not being updated across a priority change. -- */ -- void (*dequeue)(struct task_struct *p, u64 deq_flags); -- -- /** -- * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs -- * @cpu: CPU to dispatch tasks for -- * @prev: previous task being switched out -- * -- * Called when a CPU's local dsq is empty. The operation should dispatch -- * one or more tasks from the BPF scheduler into the DSQs using -- * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using -- * scx_bpf_consume(). -- * -- * The maximum number of times scx_bpf_dispatch() can be called without -- * an intervening scx_bpf_consume() is specified by -- * ops.dispatch_max_batch. See the comments on top of the two functions -- * for more details. -- * -- * When not %NULL, @prev is an SCX task with its slice depleted. If -- * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in -- * @prev->scx.flags, it is not enqueued yet and will be enqueued after -- * ops.dispatch() returns. To keep executing @prev, return without -- * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. -- */ -- void (*dispatch)(s32 cpu, struct task_struct *prev); -- -- /** -- * runnable - A task is becoming runnable on its associated CPU -- * @p: task becoming runnable -- * @enq_flags: %SCX_ENQ_* -- * -- * This and the following three functions can be used to track a task's -- * execution state transitions. A task becomes ->runnable() on a CPU, -- * and then goes through one or more ->running() and ->stopping() pairs -- * as it runs on the CPU, and eventually becomes ->quiescent() when it's -- * done running on the CPU. -- * -- * @p is becoming runnable on the CPU because it's -- * -- * - waking up (%SCX_ENQ_WAKEUP) -- * - being moved from another CPU -- * - being restored after temporarily taken off the queue for an -- * attribute change. -- * -- * This and ->enqueue() are related but not coupled. This operation -- * notifies @p's state transition and may not be followed by ->enqueue() -- * e.g. when @p is being dispatched to a remote CPU. Likewise, a task -- * may be ->enqueue()'d without being preceded by this operation e.g. -- * after exhausting its slice. -- */ -- void (*runnable)(struct task_struct *p, u64 enq_flags); -- -- /** -- * running - A task is starting to run on its associated CPU -- * @p: task starting to run -- * -- * See ->runnable() for explanation on the task state notifiers. -- */ -- void (*running)(struct task_struct *p); -- -- /** -- * stopping - A task is stopping execution -- * @p: task stopping to run -- * @runnable: is task @p still runnable? -- * -- * See ->runnable() for explanation on the task state notifiers. If -- * !@runnable, ->quiescent() will be invoked after this operation -- * returns. -- */ -- void (*stopping)(struct task_struct *p, bool runnable); -- -- /** -- * quiescent - A task is becoming not runnable on its associated CPU -- * @p: task becoming not runnable -- * @deq_flags: %SCX_DEQ_* -- * -- * See ->runnable() for explanation on the task state notifiers. -- * -- * @p is becoming quiescent on the CPU because it's -- * -- * - sleeping (%SCX_DEQ_SLEEP) -- * - being moved to another CPU -- * - being temporarily taken off the queue for an attribute change -- * (%SCX_DEQ_SAVE) -- * -- * This and ->dequeue() are related but not coupled. This operation -- * notifies @p's state transition and may not be preceded by ->dequeue() -- * e.g. when @p is being dispatched to a remote CPU. -- */ -- void (*quiescent)(struct task_struct *p, u64 deq_flags); -- -- /** -- * yield - Yield CPU -- * @from: yielding task -- * @to: optional yield target task -- * -- * If @to is NULL, @from is yielding the CPU to other runnable tasks. -- * The BPF scheduler should ensure that other available tasks are -- * dispatched before the yielding task. Return value is ignored in this -- * case. -- * -- * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf -- * scheduler can implement the request, return %true; otherwise, %false. -- */ -- bool (*yield)(struct task_struct *from, struct task_struct *to); -- -- /** -- * core_sched_before - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Used by core-sched to determine the ordering between two tasks. See -- * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on -- * core-sched. -- * -- * Both @a and @b are runnable and may or may not currently be queued on -- * the BPF scheduler. Should return %true if @a should run before @b. -- * %false if there's no required ordering or @b should run before @a. -- * -- * If not specified, the default is ordering them according to when they -- * became runnable. -- */ -- bool (*core_sched_before)(struct task_struct *a,struct task_struct *b); -- -- /** -- * set_weight - Set task weight -- * @p: task to set weight for -- * @weight: new eight [1..10000] -- * -- * Update @p's weight to @weight. -- */ -- void (*set_weight)(struct task_struct *p, u32 weight); -- -- /** -- * set_cpumask - Set CPU affinity -- * @p: task to set CPU affinity for -- * @cpumask: cpumask of cpus that @p can run on -- * -- * Update @p's CPU affinity to @cpumask. -- */ -- void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask); -- -- /** -- * update_idle - Update the idle state of a CPU -- * @cpu: CPU to udpate the idle state for -- * @idle: whether entering or exiting the idle state -- * -- * This operation is called when @rq's CPU goes or leaves the idle -- * state. By default, implementing this operation disables the built-in -- * idle CPU tracking and the following helpers become unavailable: -- * -- * - scx_bpf_select_cpu_dfl() -- * - scx_bpf_test_and_clear_cpu_idle() -- * - scx_bpf_pick_idle_cpu() -- * - scx_bpf_any_idle_cpu() -- * -- * The user also must implement ops.select_cpu() as the default -- * implementation relies on scx_bpf_select_cpu_dfl(). -- * -- * If you keep the built-in idle tracking, specify the -- * %SCX_OPS_KEEP_BUILTIN_IDLE flag. -- */ -- void (*update_idle)(s32 cpu, bool idle); -- -- /** -- * cpu_acquire - A CPU is becoming available to the BPF scheduler -- * @cpu: The CPU being acquired by the BPF scheduler. -- * @args: Acquire arguments, see the struct definition. -- * -- * A CPU that was previously released from the BPF scheduler is now once -- * again under its control. -- */ -- void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); -- -- /** -- * cpu_release - A CPU is taken away from the BPF scheduler -- * @cpu: The CPU being released by the BPF scheduler. -- * @args: Release arguments, see the struct definition. -- * -- * The specified CPU is no longer under the control of the BPF -- * scheduler. This could be because it was preempted by a higher -- * priority sched_class, though there may be other reasons as well. The -- * caller should consult @args->reason to determine the cause. -- */ -- void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -- -- /** -- * cpu_online - A CPU became online -- * @cpu: CPU which just came up -- * -- * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs beforehand. -- */ -- void (*cpu_online)(s32 cpu); -- -- /** -- * cpu_offline - A CPU is going offline -- * @cpu: CPU which is going offline -- * -- * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs afterwards. -- */ -- void (*cpu_offline)(s32 cpu); -- -- /** -- * prep_enable - Prepare to enable BPF scheduling for a task -- * @p: task to prepare BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Either we're loading a BPF scheduler or a new task is being forked. -- * Prepare BPF scheduling for @p. This operation may block and can be -- * used for allocations. -- * -- * Return 0 for success, -errno for failure. An error return while -- * loading will abort loading of the BPF scheduler. During a fork, will -- * abort the specific fork. -- */ -- s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * enable - Enable BPF scheduling for a task -- * @p: task to enable BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Enable @p for BPF scheduling. @p is now in the cgroup specified for -- * the preceding prep_enable() and will start running soon. -- */ -- void (*enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * cancel_enable - Cancel prep_enable() -- * @p: task being canceled -- * @args: enable arguments, see the struct definition -- * -- * @p was prep_enable()'d but failed before reaching enable(). Undo the -- * preparation. -- */ -- void (*cancel_enable)(struct task_struct *p, -- struct scx_enable_args *args); -- -- /** -- * disable - Disable BPF scheduling for a task -- * @p: task to disable BPF scheduling for -- * -- * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. -- * Disable BPF scheduling for @p. -- */ -- void (*disable)(struct task_struct *p); -- -- /* -- * All online ops must come before ops.init(). -- */ -- -- /** -- * init - Initialize the BPF scheduler -- */ -- s32 (*init)(void); -- -- /** -- * exit - Clean up after the BPF scheduler -- * @info: Exit info -- */ -- void (*exit)(struct scx_exit_info *info); -- -- /** -- * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch -- */ -- u32 dispatch_max_batch; -- -- /** -- * flags - %SCX_OPS_* flags -- */ -- u64 flags; -- -- /** -- * timeout_ms - The maximum amount of time, in milliseconds, that a -- * runnable task should be able to wait before being scheduled. The -- * maximum timeout may not exceed the default timeout of 30 seconds. -- * -- * Defaults to the maximum allowed timeout value of 30 seconds. -- */ -- u32 timeout_ms; -- -- /** -- * name - BPF scheduler's name -- * -- * Must be a non-zero valid BPF object name including only isalnum(), -- * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the -- * BPF scheduler is enabled. -- */ -- char name[SCX_OPS_NAME_LEN]; --}; -- --/* -- * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -- * scheduler core and the BPF scheduler. See the documentation for more details. -- */ --struct scx_dispatch_q { -- raw_spinlock_t lock; -- struct list_head fifo; /* processed in dispatching order */ -- struct rb_root_cached priq; /* processed in p->scx.dsq_vtime order */ -- u32 nr; -- u64 id; -- struct llist_node free_node; -- struct rcu_head rcu; --}; -- --/* scx_entity.flags */ --enum scx_ent_flags { -- SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ -- SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -- SCX_TASK_ENQ_LOCAL = 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */ -- -- SCX_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -- SCX_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -- -- SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -- SCX_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -- -- SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ --}; -- --/* scx_entity.dsq_flags */ --enum scx_ent_dsq_flags { -- SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ --}; -- --/* -- * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from -- * everywhere and the following bits track which kfunc sets are currently -- * allowed for %current. This simple per-task tracking works because SCX ops -- * nest in a limited way. BPF will likely implement a way to allow and disallow -- * kfuncs depending on the calling context which will replace this manual -- * mechanism. See scx_kf_allow(). -- */ --enum scx_kf_mask { -- SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ -- /* all non-sleepables may be nested inside INIT and SLEEPABLE */ -- SCX_KF_INIT = 1 << 0, /* running ops.init() */ -- SCX_KF_SLEEPABLE = 1 << 1, /* other sleepable init operations */ -- /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ -- SCX_KF_CPU_RELEASE = 1 << 2, /* ops.cpu_release() */ -- /* ops.dequeue (in REST) may be nested inside DISPATCH */ -- SCX_KF_DISPATCH = 1 << 3, /* ops.dispatch() */ -- SCX_KF_ENQUEUE = 1 << 4, /* ops.enqueue() */ -- SCX_KF_REST = 1 << 5, /* other rq-locked operations */ -- -- __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | -- SCX_KF_ENQUEUE | SCX_KF_REST, -- __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_REST, --}; -- --#define RAVG_HIST_SIZE 5 --struct scx_sched_task_stats { -- u64 mark_start; -- u64 window_start; -- u32 sum; -- u32 sum_history[RAVG_HIST_SIZE]; -- int cidx; -- -- u32 demand; -- u16 demand_scaled; -- void *sdsq; --}; -- -- -- --/* -- * The following is embedded in task_struct and contains all fields necessary -- * for a task to be scheduled by SCX. -- */ --struct sched_ext_entity { -- struct scx_dispatch_q *dsq; -- struct { -- struct list_head fifo; /* dispatch order */ -- struct rb_node priq; /* p->scx.dsq_vtime order */ -- } dsq_node; -- struct list_head watchdog_node; -- u32 flags; /* protected by rq lock */ -- u32 dsq_flags; /* protected by dsq lock */ -- u32 weight; -- s32 sticky_cpu; -- s32 holding_cpu; -- u32 kf_mask; /* see scx_kf_mask above */ -- struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ -- atomic64_t ops_state; -- unsigned long runnable_at; --#ifdef CONFIG_SCHED_CORE -- u64 core_sched_at; /* see scx_prio_less() */ --#endif -- -- /* BPF scheduler modifiable fields */ -- -- /* -- * Runtime budget in nsecs. This is usually set through -- * scx_bpf_dispatch() but can also be modified directly by the BPF -- * scheduler. Automatically decreased by SCX as the task executes. On -- * depletion, a scheduling event is triggered. -- * -- * This value is cleared to zero if the task is preempted by -- * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the -- * task ran. Use p->se.sum_exec_runtime instead. -- */ -- u64 slice; -- -- /* -- * Used to order tasks when dispatching to the vtime-ordered priority -- * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime() -- * but can also be modified directly by the BPF scheduler. Modifying it -- * while a task is queued on a dsq may mangle the ordering and is not -- * recommended. -- */ -- u64 dsq_vtime; -- -- /* -- * If set, reject future sched_setscheduler(2) calls updating the policy -- * to %SCHED_EXT with -%EACCES. -- * -- * If set from ops.prep_enable() and the task's policy is already -- * %SCHED_EXT, which can happen while the BPF scheduler is being loaded -- * or by inhering the parent's policy during fork, the task's policy is -- * rejected and forcefully reverted to %SCHED_NORMAL. The number of such -- * events are reported through /sys/kernel/debug/sched_ext::nr_rejected. -- */ -- bool disallow; /* reject switching into SCX */ -- -- /* cold fields */ -- struct list_head tasks_node; -- struct task_struct *task; -- struct scx_sched_task_stats sts; --}; -- --void sched_ext_free(struct task_struct *p); -- --#else /* !CONFIG_SCHED_CLASS_EXT */ -- --static inline void sched_ext_free(struct task_struct *p) {} -- --#endif /* CONFIG_SCHED_CLASS_EXT */ --#endif /* _LINUX_SCHED_EXT_H */ -diff --git b/include/linux/sched/hmbird_proc_val.h a/include/linux/sched/hmbird_proc_val.h -new file mode 100755 -index 000000000000..e59708b16ea3 ---- /dev/null -+++ a/include/linux/sched/hmbird_proc_val.h -@@ -0,0 +1,22 @@ -+#ifndef __HMBORD_PROC_VAL_H__ -+#define __HMBORD_PROC_VAL_H__ -+ -+extern int scx_enable; -+extern int partial_enable; -+extern int cpuctrl_high_ratio; -+extern int cpuctrl_low_ratio; -+extern int slim_stats; -+extern int hmbirdcore_debug; -+extern int slim_for_app; -+extern int misfit_ds; -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+extern int cpu7_tl; -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int slim_gov_debug; -+extern int scx_gov_ctrl; -+extern int sched_ravg_window_frame_per_sec; -+ -+#endif -\ No newline at end of file -diff --git b/include/linux/sched/hmbird_version.h a/include/linux/sched/hmbird_version.h -new file mode 100755 -index 000000000000..b2843881c26f ---- /dev/null -+++ a/include/linux/sched/hmbird_version.h -@@ -0,0 +1,52 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#ifndef _OPLUS_HMBIRD_VERSION_H_ -+#define _OPLUS_HMBIRD_VERSION_H_ -+#include -+#include -+#include -+ -+enum hmbird_version { -+ HMBIRD_UNINIT, -+ HMBIRD_GKI_VERSION, -+ HMBIRD_OGKI_VERSION, -+ HMBIRD_UNKNOW_VERSION, -+}; -+ -+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT; -+ -+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type" -+ -+static inline enum hmbird_version get_hmbird_version_type(void) -+{ -+ struct device_node *np = NULL; -+ const char *hmbird_version_str = NULL; -+ if (HMBIRD_UNINIT != hmbird_version_type) -+ return hmbird_version_type; -+ np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH); -+ if (np) { -+ of_property_read_string(np, "type", &hmbird_version_str); -+ if (NULL != hmbird_version_str) { -+ if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) { -+ hmbird_version_type = HMBIRD_OGKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi"); -+ } else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) { -+ hmbird_version_type = HMBIRD_GKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi"); -+ } else { -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi"); -+ } -+ return hmbird_version_type; -+ } -+ } -+ -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION"); -+ return hmbird_version_type; -+} -+ -+#endif /*_OPLUS_HMBIRD_VERSION_H_ */ -diff --git b/include/linux/sched/task.h a/include/linux/sched/task.h -index 03d35e3eda3c..6a5f0c0d74bd 100644 ---- b/include/linux/sched/task.h -+++ a/include/linux/sched/task.h -@@ -61,8 +61,10 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); - extern void init_idle(struct task_struct *idle, int cpu); - - extern int sched_fork(unsigned long clone_flags, struct task_struct *p); --extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); --extern void sched_cancel_fork(struct task_struct *p); -+extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); -+#ifdef CONFIG_HMBIRD_SCHED -+extern void hmbird_cancel_fork(struct task_struct *p); -+#endif - extern void sched_post_fork(struct task_struct *p); - extern void sched_dead(struct task_struct *p); - -diff --git b/include/uapi/linux/sched.h a/include/uapi/linux/sched.h -index 359a14cc76a4..969716ab4320 100755 ---- b/include/uapi/linux/sched.h -+++ a/include/uapi/linux/sched.h -@@ -118,7 +118,7 @@ struct clone_args { - /* SCHED_ISO: reserved but not implemented yet */ - #define SCHED_IDLE 5 - #define SCHED_DEADLINE 6 --#define SCHED_EXT 7 -+#define SCHED_HMBIRD 7 - - /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ - #define SCHED_RESET_ON_FORK 0x40000000 -diff --git b/init/Kconfig a/init/Kconfig -index 337276cbc7f8..bf052ca532b4 100644 ---- b/init/Kconfig -+++ a/init/Kconfig -@@ -1030,10 +1030,6 @@ config RT_GROUP_SCHED - realtime bandwidth for them. - See Documentation/scheduler/sched-rt-group.rst for more information. - --config EXT_GROUP_SCHED -- bool -- depends on SCHED_CLASS_EXT && CGROUP_SCHED -- default y - endif #CGROUP_SCHED - - config SCHED_MM_CID -diff --git b/init/init_task.c a/init/init_task.c -index 69a046388995..31ceb0e469f7 100755 ---- b/init/init_task.c -+++ a/init/init_task.c -@@ -6,7 +6,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -102,9 +101,6 @@ struct task_struct init_task - #endif - #ifdef CONFIG_CGROUP_SCHED - .sched_task_group = &root_task_group, --#endif --#ifdef CONFIG_SLIM_SCHED -- .scx = NULL, - #endif - .ptraced = LIST_HEAD_INIT(init_task.ptraced), - .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), -diff --git b/kernel/Kconfig.preempt a/kernel/Kconfig.preempt -index cf22ef6107b8..ca8de6eb1e16 100755 ---- b/kernel/Kconfig.preempt -+++ a/kernel/Kconfig.preempt -@@ -133,12 +133,8 @@ config SCHED_CORE - which is the likely usage by Linux distributions, there should - be no measurable impact on performance. - --config SLIM_SCHED -- bool "slim sched" -- default n --config SCHED_CLASS_EXT -- bool "Extensible Scheduling Class" -- depends on BPF_SYSCALL && BPF_JIT -+config HMBIRD_SCHED -+ bool "hmbird Scheduling Class(base on ext scheduler)" - help - This option enables a new scheduler class sched_ext (SCX), which - allows scheduling policies to be implemented as BPF programs to -diff --git b/kernel/bpf/bpf_struct_ops_types.h a/kernel/bpf/bpf_struct_ops_types.h -index 3618769d853d..5678a9ddf817 100644 ---- b/kernel/bpf/bpf_struct_ops_types.h -+++ a/kernel/bpf/bpf_struct_ops_types.h -@@ -9,8 +9,4 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) - #include - BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) - #endif --#ifdef CONFIG_SCHED_CLASS_EXT --#include --BPF_STRUCT_OPS_TYPE(sched_ext_ops) --#endif - #endif -diff --git b/kernel/fork.c a/kernel/fork.c -index a43f97302332..7a91505b9378 100755 ---- b/kernel/fork.c -+++ a/kernel/fork.c -@@ -23,7 +23,9 @@ - #include - #include - #include --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - #include - #include - #include -@@ -999,8 +1001,9 @@ void __put_task_struct(struct task_struct *tsk) - WARN_ON(!tsk->exit_state); - WARN_ON(refcount_read(&tsk->usage)); - WARN_ON(tsk == current); -- -- sched_ext_free(tsk); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_free(tsk); -+#endif - io_uring_free(tsk); - cgroup_free(tsk); - task_numa_free(tsk, true); -@@ -2517,7 +2520,11 @@ __latent_entropy struct task_struct *copy_process( - - retval = perf_event_init_task(p, clone_flags); - if (retval) -- goto bad_fork_sched_cancel_fork; -+#ifdef CONFIG_HMBIRD_SCHED -+ goto bad_fork_hmbird_cancel_fork; -+#else -+ goto bad_fork_cleanup_policy; -+#endif - retval = audit_alloc(p); - if (retval) - goto bad_fork_cleanup_perf; -@@ -2649,9 +2656,7 @@ __latent_entropy struct task_struct *copy_process( - * cgroup specific, it unconditionally needs to place the task on a - * runqueue. - */ -- retval = sched_cgroup_fork(p, args); -- if (retval) -- goto bad_fork_cancel_cgroup; -+ sched_cgroup_fork(p, args); - - /* - * From this point on we must avoid any synchronous user-space -@@ -2697,13 +2702,13 @@ __latent_entropy struct task_struct *copy_process( - /* Don't start children in a dying pid namespace */ - if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { - retval = -ENOMEM; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* Let kill terminate clone/fork in the middle */ - if (fatal_signal_pending(current)) { - retval = -EINTR; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* No more failure paths after this point. */ -@@ -2780,11 +2785,10 @@ __latent_entropy struct task_struct *copy_process( - - return p; - --bad_fork_core_free: -+bad_fork_cancel_cgroup: - sched_core_free(p); - spin_unlock(¤t->sighand->siglock); - write_unlock_irq(&tasklist_lock); --bad_fork_cancel_cgroup: - cgroup_cancel_fork(p, args); - bad_fork_put_pidfd: - if (clone_flags & CLONE_PIDFD) { -@@ -2823,8 +2827,10 @@ __latent_entropy struct task_struct *copy_process( - audit_free(p); - bad_fork_cleanup_perf: - perf_event_free_task(p); --bad_fork_sched_cancel_fork: -- sched_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+bad_fork_hmbird_cancel_fork: -+ hmbird_cancel_fork(p); -+#endif - bad_fork_cleanup_policy: - lockdep_free_task(p); - #ifdef CONFIG_NUMA -diff --git b/kernel/sched/build_policy.c a/kernel/sched/build_policy.c -index 005025f55bea..c091539a0f60 100755 ---- b/kernel/sched/build_policy.c -+++ a/kernel/sched/build_policy.c -@@ -28,8 +28,10 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED - #include - #include -+#endif - - #include - -@@ -54,6 +56,10 @@ - #include "cputime.c" - #include "deadline.c" - --#ifdef CONFIG_SCHED_CLASS_EXT --# include "ext.c" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/hmbird_util_track.c" -+#include "hmbird/hmbird_sched_proc.c" -+#include "hmbird/hmbird_shadow_tick.c" -+#include "hmbird/hmbird.c" -+#include "hmbird/hmbird_misc.c" - #endif -diff --git b/kernel/sched/core.c a/kernel/sched/core.c -index 25d5703df992..b894417ce7df 100755 ---- b/kernel/sched/core.c -+++ a/kernel/sched/core.c -@@ -96,6 +96,12 @@ - #include "../../io_uring/io-wq.h" - #include "../smpboot.h" - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/slim.h" -+#include "hmbird/hmbird_shadow_tick.h" -+#include "hmbird.h" -+#endif -+ - #include - #include - #include -@@ -170,7 +176,6 @@ __read_mostly int scheduler_running; - - DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); - -- - /* kernel prio, less is more */ - static inline int __task_prio(const struct task_struct *p) - { -@@ -183,11 +188,10 @@ static inline int __task_prio(const struct task_struct *p) - if (p->sched_class == &idle_sched_class) - return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - --#ifdef CONFIG_SCHED_CLASS_EXT -- if (p->sched_class == &ext_sched_class) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (p->sched_class == &hmbird_sched_class) - return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ - #endif -- - return MAX_RT_PRIO + MAX_NICE; /* 119, squash fair */ - } - -@@ -217,9 +221,9 @@ static inline bool prio_less(const struct task_struct *a, - if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ - return cfs_prio_less(a, b, in_fi); - --#ifdef CONFIG_SCHED_CLASS_EXT -+#ifdef CONFIG_HMBIRD_SCHED - if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ -- return scx_prio_less(a, b, in_fi); -+ return hmbird_prio_less(a, b, in_fi); - #endif - - return false; -@@ -1279,17 +1283,26 @@ bool sched_can_stop_tick(struct rq *rq) - if (fifo_nr_running) - return true; - -+#ifdef CONFIG_HMBIRD_SCHED - /* -- * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks -+ * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks - * left. For CFS, if there's more than one we need the tick for -- * involuntary preemption. For SCX, ask. -+ * involuntary preemption. For HMBIRD, ask. - */ -- if (!scx_switched_all() && rq->nr_running > 1) -+ if (!hmbird_enabled() && rq->nr_running > 1) - return false; - -- if (scx_enabled() && !scx_can_stop_tick(rq)) -+ if (hmbird_enabled() && !hmbird_can_stop_tick(rq)) - return false; -- -+#else -+ /* -+ * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; -+ * if there's more than one we need the tick for involuntary -+ * preemption. -+ */ -+ if (rq->nr_running > 1) -+ return false; -+#endif - /* - * If there is one task and it has CFS runtime bandwidth constraints - * and it's on the cpu now we don't want to stop the tick. -@@ -2222,10 +2235,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) - } - EXPORT_SYMBOL_GPL(deactivate_task); - --struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) -+#ifdef CONFIG_HMBIRD_SCHED -+struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - { -- struct sched_change_guard cg = { -+ struct hmbird_sched_change_guard cg = { - .rq = rq, - .p = p, - .queued = task_on_rq_queued(p), -@@ -2247,7 +2261,7 @@ sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - return cg; - } - --void sched_change_guard_fini(struct sched_change_guard *cg, int flags) -+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags) - { - if (cg->queued) - enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); -@@ -2255,6 +2269,7 @@ void sched_change_guard_fini(struct sched_change_guard *cg, int flags) - set_next_task(cg->rq, cg->p); - cg->done = true; - } -+#endif - - static inline int __normal_prio(int policy, int rt_prio, int nice) - { -@@ -2320,9 +2335,9 @@ inline int task_curr(const struct task_struct *p) - * this means any call to check_class_changed() must be followed by a call to - * balance_callback(). - */ --void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio) -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) - { - if (prev_class != p->sched_class) { - if (prev_class->switched_from) -@@ -2885,6 +2900,7 @@ static void - __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - { - struct rq *rq = task_rq(p); -+ bool queued, running; - - /* - * This here violates the locking rules for affinity, since we're only -@@ -2903,9 +2919,26 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - else - lockdep_assert_held(&p->pi_lock); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->sched_class->set_cpus_allowed(p, ctx); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ -+ if (queued) { -+ /* -+ * Because __kthread_bind() calls this on blocked tasks without -+ * holding rq->lock. -+ */ -+ lockdep_assert_rq_held(rq); -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); - } -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->sched_class->set_cpus_allowed(p, ctx); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - } - - /* -@@ -3772,7 +3805,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class)) -+#else - if (unlikely(!is_cpu_allowed(p, cpu))) -+#endif - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -@@ -4891,8 +4928,9 @@ late_initcall(sched_core_sysctl_init); - */ - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { -- -+#ifdef CONFIG_HMBIRD_SCHED - int ret; -+#endif - - trace_android_rvh_sched_fork(p); - -@@ -4933,21 +4971,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_reset_on_fork = 0; - } - -- scx_pre_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ ret = hmbird_pre_fork(p); -+ if (ret) -+ goto out_cancel; - - if (dl_prio(p->prio)) { - ret = -EAGAIN; - goto out_cancel; --#ifdef CONFIG_SCHED_CLASS_EXT -- } else if (task_on_scx(p)) { -- p->sched_class = &ext_sched_class; --#endif -+ } else if (task_on_hmbird(p)) { -+ p->sched_class = &hmbird_sched_class; - } else if (rt_prio(p->prio)) { - p->sched_class = &rt_sched_class; - } else { - p->sched_class = &fair_sched_class; - } -- -+#else -+ if (dl_prio(p->prio)) -+ return -EAGAIN; -+ else if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - init_entity_runnable_average(&p->se); - trace_android_rvh_finish_prio_fork(p); - -@@ -4966,12 +5012,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - #endif - return 0; - -+#ifdef CONFIG_HMBIRD_SCHED - out_cancel: -- scx_cancel_fork(p); -+ hmbird_cancel_fork(p); - return ret; -+#endif - } - --int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) -+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - { - unsigned long flags; - -@@ -4998,20 +5046,14 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -- -- return scx_fork(p); --} -- --void sched_cancel_fork(struct task_struct *p) --{ -- scx_cancel_fork(p); - } - - void sched_post_fork(struct task_struct *p) - { - uclamp_post_fork(p); -- -- scx_post_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_post_fork(p); -+#endif - } - - unsigned long to_ratio(u64 period, u64 runtime) -@@ -5875,21 +5917,28 @@ void scheduler_tick(void) - - if (sched_feat(LATENCY_WARN) && resched_latency) - resched_latency_warn(cpu, resched_latency); -- -- scx_notify_sched_tick(); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_notify_sched_tick(); -+#endif - perf_event_task_tick(); - - if (curr->flags & PF_WQ_WORKER) - wq_worker_tick(curr); - - #ifdef CONFIG_SMP -- if (!scx_switched_all()) { -+#ifdef CONFIG_HMBIRD_SCHED -+ if (!hmbird_enabled()) { -+#endif - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); -+#ifdef CONFIG_HMBIRD_SCHED - } -+#endif - #endif - trace_android_vh_scheduler_tick(rq); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ scheduler_tick_handler(NULL, NULL); -+#endif - } - - #ifdef CONFIG_NO_HZ_FULL -@@ -6191,7 +6240,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ -+#ifdef CONFIG_HMBIRD_SCHED - for_balance_class_range(class, prev->sched_class, &idle_sched_class) { -+#else -+ for_class_range(class, prev->sched_class, &idle_sched_class) { -+#endif - if (class->balance(rq, prev, rf)) - break; - } -@@ -6209,9 +6262,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - const struct sched_class *class; - struct task_struct *p; - -- if (scx_enabled()) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (hmbird_enabled()) - goto restart; -- -+#endif - /* - * Optimization: we know that if all tasks are in the fair class we can - * call that function directly, but only if the @prev task wasn't of a -@@ -6237,13 +6291,21 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - restart: - put_prev_task_balance(rq, prev, rf); - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { - p = class->pick_next_task(rq); - if (p) { -- scx_notify_pick_next_task(rq, p, class); -+ hmbird_notify_pick_next_task(rq, p, class); - return p; - } - } -+#else -+ for_each_class(class) { -+ p = class->pick_next_task(rq); -+ if (p) -+ return p; -+ } -+#endif - - BUG(); /* The idle class should always have a runnable task. */ - } -@@ -6272,7 +6334,11 @@ static inline struct task_struct *pick_task(struct rq *rq) - const struct sched_class *class; - struct task_struct *p; - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { -+#else -+ for_each_class(class) { -+#endif - p = class->pick_task(rq); - if (p) - return p; -@@ -6916,7 +6982,9 @@ static void __sched notrace __schedule(unsigned int sched_mode) - psi_sched_switch(prev, next, !task_on_rq_queued(prev)); - - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#endif - /* Also unlocks the rq: */ - rq = context_switch(rq, prev, next, &rf); - } else { -@@ -7244,24 +7312,31 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag - } - EXPORT_SYMBOL(default_wake_function); - --void __setscheduler_prio(struct task_struct *p, int prio) -+static void __setscheduler_prio(struct task_struct *p, int prio) - { -- /* -- * After switching all rt and fair class to ext, -- * stop class is switched to ext class too. Just -- * skip it. */ -+#ifdef CONFIG_HMBIRD_SCHED -+ bool on_hmbird = task_on_hmbird(p); -+ - if (p->sched_class == &stop_sched_class) -- ;/* do nothing */ -+ ; - else if (dl_prio(prio)) - p->sched_class = &dl_sched_class; --#ifdef CONFIG_SCHED_CLASS_EXT -- else if (task_on_scx(p)) -- p->sched_class = &ext_sched_class; --#endif -+ else if (rt_prio(prio) && on_hmbird) -+ p->sched_class = &hmbird_sched_class; - else if (rt_prio(prio)) - p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; - else - p->sched_class = &fair_sched_class; -+#else -+ if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - - p->prio = prio; - trace_android_rvh_setscheduler_prio(p); -@@ -7297,7 +7372,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - */ - void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - { -- int prio, oldprio, queue_flag = -+ int prio, oldprio, queued, running, queue_flag = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *prev_class; - struct rq_flags rf; -@@ -7360,42 +7435,52 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - queue_flag &= ~DEQUEUE_MOVE; - - prev_class = p->sched_class; -- SCHED_CHANGE_BLOCK(rq, p, queue_flag) { -- /* -- * Boosting condition are: -- * 1. -rt task is running and holds mutex A -- * --> -dl task blocks on mutex A -- * -- * 2. -dl task is running and holds mutex A -- * --> -dl task blocks on mutex A and could preempt the -- * running task -- */ -- if (dl_prio(prio)) { -- if (!dl_prio(p->normal_prio) || -- (pi_task && dl_prio(pi_task->prio) && -- dl_entity_preempt(&pi_task->dl, &p->dl))) { -- p->dl.pi_se = pi_task->dl.pi_se; -- queue_flag |= ENQUEUE_REPLENISH; -- } else { -- p->dl.pi_se = &p->dl; -- } -- } else if (rt_prio(prio)) { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- if (oldprio < prio) -- queue_flag |= ENQUEUE_HEAD; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flag); -+ if (running) -+ put_prev_task(rq, p); -+ -+ /* -+ * Boosting condition are: -+ * 1. -rt task is running and holds mutex A -+ * --> -dl task blocks on mutex A -+ * -+ * 2. -dl task is running and holds mutex A -+ * --> -dl task blocks on mutex A and could preempt the -+ * running task -+ */ -+ if (dl_prio(prio)) { -+ if (!dl_prio(p->normal_prio) || -+ (pi_task && dl_prio(pi_task->prio) && -+ dl_entity_preempt(&pi_task->dl, &p->dl))) { -+ p->dl.pi_se = pi_task->dl.pi_se; -+ queue_flag |= ENQUEUE_REPLENISH; - } else { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- else if (rt_prio(oldprio)) -- p->rt.timeout = 0; -- else if (!task_has_idle_policy(p)) -- reweight_task(p, prio - MAX_RT_PRIO); -+ p->dl.pi_se = &p->dl; - } -- -- __setscheduler_prio(p, prio); -+ } else if (rt_prio(prio)) { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ if (oldprio < prio) -+ queue_flag |= ENQUEUE_HEAD; -+ } else { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ else if (rt_prio(oldprio)) -+ p->rt.timeout = 0; -+ else if (!task_has_idle_policy(p)) -+ reweight_task(p, prio - MAX_RT_PRIO); - } - -+ __setscheduler_prio(p, prio); -+ -+ if (queued) -+ enqueue_task(rq, p, queue_flag); -+ if (running) -+ set_next_task(rq, p); -+ - check_class_changed(rq, p, prev_class, oldprio); - out_unlock: - /* Avoid rq from going away on us: */ -@@ -7416,7 +7501,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - - void set_user_nice(struct task_struct *p, long nice) - { -- bool allowed = false; -+ bool queued, running, allowed = false; - int old_prio; - struct rq_flags rf; - struct rq *rq; -@@ -7445,13 +7530,22 @@ void set_user_nice(struct task_struct *p, long nice) - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); -+ if (running) -+ put_prev_task(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->static_prio = NICE_TO_PRIO(nice); -- set_load_weight(p, true); -- old_prio = p->prio; -- p->prio = effective_prio(p); -- } -+ p->static_prio = NICE_TO_PRIO(nice); -+ set_load_weight(p, true); -+ old_prio = p->prio; -+ p->prio = effective_prio(p); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - - /* - * If the task increased its priority or is running and -@@ -7868,7 +7962,7 @@ static int __sched_setscheduler(struct task_struct *p, - bool user, bool pi) - { - int oldpolicy = -1, policy = attr->sched_policy; -- int retval, oldprio, newprio; -+ int retval, oldprio, newprio, queued, running; - const struct sched_class *prev_class; - struct balance_callback *head; - struct rq_flags rf; -@@ -7876,7 +7970,9 @@ static int __sched_setscheduler(struct task_struct *p, - int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct rq *rq; - bool cpuset_locked = false; -- -+#ifdef CONFIG_HMBIRD_SCHED -+ unsigned long flags; -+#endif - - /* The pi code expects interrupts enabled */ - BUG_ON(pi && in_interrupt()); -@@ -7942,7 +8038,9 @@ static int __sched_setscheduler(struct task_struct *p, - * To be able to change p->policy safely, the appropriate - * runqueue lock must be held. - */ -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+#endif - rq = task_rq_lock(p, &rf); - update_rq_clock(rq); - -@@ -7954,10 +8052,11 @@ static int __sched_setscheduler(struct task_struct *p, - goto unlock; - } - -- retval = scx_check_setscheduler(p, policy); -+#ifdef CONFIG_HMBIRD_SCHED -+ retval = hmbird_check_setscheduler(p, policy); - if (retval) - goto unlock; -- -+#endif - /* - * If not changing anything there's no need to proceed further, - * but store a possible modification of reset_on_fork. -@@ -8014,7 +8113,9 @@ static int __sched_setscheduler(struct task_struct *p, - if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { - policy = oldpolicy = -1; - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - goto recheck; -@@ -8047,16 +8148,23 @@ static int __sched_setscheduler(struct task_struct *p, - queue_flags &= ~DEQUEUE_MOVE; - } - -- SCHED_CHANGE_BLOCK(rq, p, queue_flags) { -- prev_class = p->sched_class; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flags); -+ if (running) -+ put_prev_task(rq, p); -+ -+ prev_class = p->sched_class; - -- if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -- __setscheduler_params(p, attr); -- __setscheduler_prio(p, newprio); -- trace_android_rvh_setscheduler(p); -- } -- __setscheduler_uclamp(p, attr); -+ if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -+ __setscheduler_params(p, attr); -+ __setscheduler_prio(p, newprio); -+ trace_android_rvh_setscheduler(p); -+ } -+ __setscheduler_uclamp(p, attr); - -+ if (queued) { - /* - * We enqueue to tail when the priority of a task is - * increased (user space view). -@@ -8064,7 +8172,10 @@ static int __sched_setscheduler(struct task_struct *p, - if (oldprio < p->prio) - queue_flags |= ENQUEUE_HEAD; - -+ enqueue_task(rq, p, queue_flags); - } -+ if (running) -+ set_next_task(rq, p); - - check_class_changed(rq, p, prev_class, oldprio); - -@@ -8072,7 +8183,9 @@ static int __sched_setscheduler(struct task_struct *p, - preempt_disable(); - head = splice_balance_callbacks(rq); - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (pi) { - if (cpuset_locked) - cpuset_unlock(); -@@ -8087,7 +8200,9 @@ static int __sched_setscheduler(struct task_struct *p, - - unlock: - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - return retval; -@@ -8785,6 +8900,9 @@ static void do_sched_yield(void) - long skip = 0; - - trace_android_rvh_before_do_sched_yield(&skip); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_skip_yield(&skip); -+#endif - if (skip) - return; - -@@ -9309,7 +9427,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - break; - } -@@ -9337,7 +9457,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - } - return ret; -@@ -9638,15 +9760,25 @@ int migrate_task_to(struct task_struct *p, int target_cpu) - */ - void sched_setnuma(struct task_struct *p, int nid) - { -+ bool queued, running; - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) { -- p->numa_preferred_nid = nid; -- } -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE); -+ if (running) -+ put_prev_task(rq, p); - -+ p->numa_preferred_nid = nid; -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - task_rq_unlock(rq, p, &rf); - } - #endif /* CONFIG_NUMA_BALANCING */ -@@ -10212,17 +10344,23 @@ void __init sched_init(void) - int i; - - /* Make sure the linker didn't screw up */ -+#ifdef CONFIG_HMBIRD_SCHED - #ifdef CONFIG_SMP -- BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+#endif -+ WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class)); -+#else -+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 || -+ &fair_sched_class != &rt_sched_class + 1 || -+ &rt_sched_class != &dl_sched_class + 1); -+#ifdef CONFIG_SMP -+ BUG_ON(&dl_sched_class != &stop_sched_class + 1); - #endif -- BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); -- BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); -- BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); --#ifdef CONFIG_SCHED_CLASS_EXT -- BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); -- BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); - #endif -- - wait_bit_init(); - - #ifdef CONFIG_FAIR_GROUP_SCHED -@@ -10392,8 +10530,9 @@ void __init sched_init(void) - balance_push_set(smp_processor_id(), false); - #endif - init_sched_fair_class(); -- init_sched_ext_class(); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ init_sched_hmbird_class(); -+#endif - psi_init(); - - init_uclamp(); -@@ -10863,6 +11002,11 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) - struct task_group *tg = css_tg(css); - struct task_group *parent = css_tg(css->parent); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_tg_online(tg); -+#endif -+ -+ - if (parent) - sched_online_group(tg, parent); - -@@ -10912,13 +11056,77 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) - } - #endif - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 dl) -+{ -+ int i; -+ -+ for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) { -+ if (dl < HMBIRD_BPF_DSQS_DEADLINE[i]) -+ break; -+ } -+ -+ i = max_t(int, i-1, MIN_CGROUP_DL_IDX); -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return 0; -+ update_cgroup_ids_table(css->cgroup->kn->id, i); -+ -+ return 0; -+} -+ -+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ u8 i; -+ -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX]; -+ i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1); -+ if (i < 0) { -+ pr_err(" <%s> i is %d, less than 0, name is %s\n", -+ __func__, i, css->cgroup->kn->name); -+ i = DEFAULT_CGROUP_DL_IDX; -+ } -+ -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[i]; -+} -+ -+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css, -+ struct task_struct *p, int prio) -+{ -+ if (!css || !rt_prio(prio)) -+ return; -+ -+ if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) { -+ if (!strcmp(css->cgroup->kn->name, "display")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3); -+ else if (!strcmp(css->cgroup->kn->name, "touch")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2); -+ else if (!strcmp(css->cgroup->kn->name, "multimedia")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+ } -+} -+#endif -+ - static void cpu_cgroup_attach(struct cgroup_taskset *tset) - { - struct task_struct *task; - struct cgroup_subsys_state *css; - - cgroup_taskset_for_each(task, css, tset) { -- -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_change_rt_sched_prop(css, task, task->prio); -+#endif - sched_move_task(task); - } - -@@ -11600,7 +11808,13 @@ static struct cftype cpu_legacy_files[] = { - .write_u64 = cpu_uclamp_ls_write_u64, - }, - #endif -- -+#ifdef CONFIG_HMBIRD_SCHED -+ { -+ .name = "hmbird.deadline", -+ .read_u64 = cgroup_read_hmbird_deadline, -+ .write_u64 = cgroup_write_hmbird_deadline, -+ }, -+#endif - { } /* Terminate */ - }; - -@@ -11649,7 +11863,6 @@ static int cpu_local_stat_show(struct seq_file *sf, - } - - #ifdef CONFIG_FAIR_GROUP_SCHED -- - static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, - struct cftype *cft) - { -diff --git b/kernel/sched/debug.c a/kernel/sched/debug.c -index a6c99df9edf9..e09904f4d6e5 100755 ---- b/kernel/sched/debug.c -+++ a/kernel/sched/debug.c -@@ -375,9 +375,8 @@ static __init int sched_init_debug(void) - #endif - - debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); -- --#ifdef CONFIG_SCHED_CLASS_EXT -- debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops); -+#ifdef CONFIG_HMBIRD_SCHED -+ debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops); - #endif - return 0; - } -@@ -1006,9 +1005,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - SEQ_printf(m, - "---------------------------------------------------------" - "----------\n"); --#ifdef CONFIG_SLIM_SCHED -- SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop); --#endif - - #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) - #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) -@@ -1104,8 +1100,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - } else if (fair_policy(p->policy)) { - P(se.slice); - } --#ifdef CONFIG_SCHED_CLASS_EXT -- __PS("ext.enabled", p->sched_class == &ext_sched_class); -+#ifdef CONFIG_HMBIRD_SCHED -+ __PS("hmbird.enabled", p->sched_class == &hmbird_sched_class); -+ __PS("sched_prop", get_hmbird_ts(p)->sched_prop); -+ __PS("top_task_prop", get_hmbird_ts(p)->top_task_prop); - #endif - #undef PN_SCHEDSTAT - #undef P_SCHEDSTAT -diff --git b/kernel/sched/ext.c a/kernel/sched/ext.c -deleted file mode 100755 -index 4845d441df2b..000000000000 ---- b/kernel/sched/ext.c -+++ /dev/null -@@ -1,3978 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -- --enum scx_internal_consts { -- SCX_NR_ONLINE_OPS = SCX_OP_IDX(init), -- SCX_DSP_DFL_MAX_BATCH = 32, -- SCX_DSP_MAX_LOOPS = 32, -- SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, --}; -- --enum scx_ops_enable_state { -- SCX_OPS_PREPPING, -- SCX_OPS_ENABLING, -- SCX_OPS_ENABLED, -- SCX_OPS_DISABLING, -- SCX_OPS_DISABLED, --}; -- --static inline struct task_group *css_tg(struct cgroup_subsys_state *css) --{ -- return css ? container_of(css, struct task_group, css) : NULL; --} -- --/* -- * sched_ext_entity->ops_state -- * -- * Used to track the task ownership between the SCX core and the BPF scheduler. -- * State transitions look as follows: -- * -- * NONE -> QUEUEING -> QUEUED -> DISPATCHING -- * ^ | | -- * | v v -- * \-------------------------------/ -- * -- * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -- * sites for explanations on the conditions being waited upon and why they are -- * safe. Transitions out of them into NONE or QUEUED must store_release and the -- * waiters should load_acquire. -- * -- * Tracking scx_ops_state enables sched_ext core to reliably determine whether -- * any given task can be dispatched by the BPF scheduler at all times and thus -- * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -- * to try to dispatch any task anytime regardless of its state as the SCX core -- * can safely reject invalid dispatches. -- */ --enum scx_ops_state { -- SCX_OPSS_NONE, /* owned by the SCX core */ -- SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -- SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ -- SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ -- -- /* -- * QSEQ brands each QUEUED instance so that, when dispatch races -- * dequeue/requeue, the dispatcher can tell whether it still has a claim -- * on the task being dispatched. -- */ -- SCX_OPSS_QSEQ_SHIFT = 2, -- SCX_OPSS_STATE_MASK = (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1, -- SCX_OPSS_QSEQ_MASK = ~SCX_OPSS_STATE_MASK, --}; -- --/* -- * During exit, a task may schedule after losing its PIDs. When disabling the -- * BPF scheduler, we need to be able to iterate tasks in every state to -- * guarantee system safety. Maintain a dedicated task list which contains every -- * task between its fork and eventual free. -- */ --static DEFINE_SPINLOCK(scx_tasks_lock); --static LIST_HEAD(scx_tasks); -- --/* ops enable/disable */ --static struct kthread_worker *scx_ops_helper; --static DEFINE_MUTEX(scx_ops_enable_mutex); --DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); --DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); --static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); --static bool scx_switch_all_req; --static bool scx_switching_all; --DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -- --static struct sched_ext_ops scx_ops; --static bool scx_warned_zero_slice; -- --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); --DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); --static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); -- --struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] = -- { [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT }; -- --static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE); --static struct scx_exit_info scx_exit_info; -- --static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0); -- --/* -- * The maximum amount of time in jiffies that a task may be runnable without -- * being scheduled on a CPU. If this timeout is exceeded, it will trigger -- * scx_ops_error(). -- */ --unsigned long scx_watchdog_timeout; -- --/* -- * The last time the delayed work was run. This delayed work relies on -- * ksoftirqd being able to run to service timer interrupts, so it's possible -- * that this work itself could get wedged. To account for this, we check that -- * it's not stalled in the timer tick, and trigger an error if it is. -- */ --unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; -- --static struct delayed_work scx_watchdog_work; -- --/* idle tracking */ --#ifdef CONFIG_SMP --#ifdef CONFIG_CPUMASK_OFFSTACK --#define CL_ALIGNED_IF_ONSTACK --#else --#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp --#endif -- --static struct { -- cpumask_var_t cpu; -- cpumask_var_t smt; --} idle_masks CL_ALIGNED_IF_ONSTACK; -- --static bool __cacheline_aligned_in_smp scx_has_idle_cpus; --#endif /* CONFIG_SMP */ -- --/* for %SCX_KICK_WAIT */ --static u64 __percpu *scx_kick_cpus_pnt_seqs; -- --/* -- * Direct dispatch marker. -- * -- * Non-NULL values are used for direct dispatch from enqueue path. A valid -- * pointer points to the task currently being enqueued. An ERR_PTR value is used -- * to indicate that direct dispatch has already happened. -- */ --static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); -- --/* dispatch queues */ --static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; -- --static LLIST_HEAD(dsqs_to_free); -- --/* dispatch buf */ --struct scx_dsp_buf_ent { -- struct task_struct *task; -- u64 qseq; -- u64 dsq_id; -- u64 enq_flags; --}; -- --static u32 scx_dsp_max_batch; --static struct scx_dsp_buf_ent __percpu *scx_dsp_buf; -- --struct scx_dsp_ctx { -- struct rq *rq; -- struct rq_flags *rf; -- u32 buf_cursor; -- u32 nr_tasks; --}; -- --static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx); -- --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags); --void scx_bpf_kick_cpu(s32 cpu, u64 flags); -- --struct scx_task_iter { -- struct sched_ext_entity cursor; -- struct task_struct *locked; -- struct rq *rq; -- struct rq_flags rf; --}; -- --#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) -- --/* if the highest set bit is N, return a mask with bits [N+1, 31] set */ --static u32 higher_bits(u32 flags) --{ -- return ~((1 << fls(flags)) - 1); --} -- --/* return the mask with only the highest bit set */ --static u32 highest_bit(u32 flags) --{ -- int bit = fls(flags); -- return bit ? 1 << (bit - 1) : 0; --} -- --/* -- * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX -- * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate -- * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check -- * whether it's running from an allowed context. -- * -- * @mask is constant, always inline to cull the mask calculations. -- */ --static __always_inline void scx_kf_allow(u32 mask) --{ -- /* nesting is allowed only in increasing scx_kf_mask order */ -- WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask, -- "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n", -- current->scx->kf_mask, mask); -- current->scx->kf_mask |= mask; --} -- --static void scx_kf_disallow(u32 mask) --{ -- current->scx->kf_mask &= ~mask; --} -- --#define SCX_CALL_OP(mask, op, args...) \ --do { \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- scx_ops.op(args); \ -- } \ --} while (0) -- --#define SCX_CALL_OP_RET(mask, op, args...) \ --({ \ -- __typeof__(scx_ops.op(args)) __ret; \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- __ret = scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- __ret = scx_ops.op(args); \ -- } \ -- __ret; \ --}) -- --/* -- * Some kfuncs are allowed only on the tasks that are subjects of the -- * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such -- * restrictions, the following SCX_CALL_OP_*() variants should be used when -- * invoking scx_ops operations that take task arguments. These can only be used -- * for non-nesting operations due to the way the tasks are tracked. -- * -- * kfuncs which can only operate on such tasks can in turn use -- * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on -- * the specific task. -- */ --#define SCX_CALL_OP_TASK(mask, op, task, args...) \ --do { \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- SCX_CALL_OP(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ --} while (0) -- --#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ --({ \ -- __typeof__(scx_ops.op(task, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- __ret; \ --}) -- --#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ --({ \ -- __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task0; \ -- current->scx->kf_tasks[1] = task1; \ -- __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- current->scx->kf_tasks[1] = NULL; \ -- __ret; \ --}) -- --//#include "./slim_walt.c" -- --/* @mask is constant, always inline to cull unnecessary branches */ --static __always_inline bool scx_kf_allowed(u32 mask) --{ -- if (unlikely(!(current->scx->kf_mask & mask))) { -- scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", -- mask, current->scx->kf_mask); -- return false; -- } -- -- if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) && -- in_interrupt())) { -- scx_ops_error("sleepable kfunc called from non-sleepable context"); -- return false; -- } -- -- /* -- * Enforce nesting boundaries. e.g. A kfunc which can be called from -- * DISPATCH must not be called if we're running DEQUEUE which is nested -- * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE -- * boundary thanks to the above in_interrupt() check. -- */ -- if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && -- (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { -- scx_ops_error("cpu_release kfunc called from a nested operation"); -- return false; -- } -- -- if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && -- (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) { -- scx_ops_error("dispatch kfunc called from a nested operation"); -- return false; -- } -- -- return true; --} -- --/* see SCX_CALL_OP_TASK() */ --static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, -- struct task_struct *p) --{ -- if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED)) -- return false; -- -- if (unlikely((p != current->scx->kf_tasks[0] && -- p != current->scx->kf_tasks[1]))) { -- scx_ops_error("called on a task not being operated on"); -- return false; -- } -- -- return true; --} -- --/** -- * scx_task_iter_init - Initialize a task iterator -- * @iter: iterator to init -- * -- * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized, -- * @iter must eventually be exited with scx_task_iter_exit(). -- * -- * scx_tasks_lock may be released between this and the first next() call or -- * between any two next() calls. If scx_tasks_lock is released between two -- * next() calls, the caller is responsible for ensuring that the task being -- * iterated remains accessible either through RCU read lock or obtaining a -- * reference count. -- * -- * All tasks which existed when the iteration started are guaranteed to be -- * visited as long as they still exist. -- */ --static void scx_task_iter_init(struct scx_task_iter *iter) --{ -- lockdep_assert_held(&scx_tasks_lock); -- -- iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; -- list_add(&iter->cursor.tasks_node, &scx_tasks); -- iter->locked = NULL; --} -- --/** -- * scx_task_iter_exit - Exit a task iterator -- * @iter: iterator to exit -- * -- * Exit a previously initialized @iter. Must be called with scx_tasks_lock held. -- * If the iterator holds a task's rq lock, that rq lock is released. See -- * scx_task_iter_init() for details. -- */ --static void scx_task_iter_exit(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- if (list_empty(cursor)) -- return; -- -- list_del_init(cursor); --} -- --/** -- * scx_task_iter_next - Next task -- * @iter: iterator to walk -- * -- * Visit the next task. See scx_task_iter_init() for details. -- */ --static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- struct sched_ext_entity *pos; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- list_for_each_entry(pos, cursor, tasks_node) { -- if (&pos->tasks_node == &scx_tasks) -- return NULL; -- if (!(pos->flags & SCX_TASK_CURSOR)) { -- list_move(cursor, &pos->tasks_node); -- return pos->task; -- } -- } -- -- /* can't happen, should always terminate at scx_tasks above */ -- BUG(); --} -- --/** -- * scx_task_iter_next_filtered - Next non-idle task -- * @iter: iterator to walk -- * -- * Visit the next non-idle task. See scx_task_iter_init() for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- while ((p = scx_task_iter_next(iter))) { -- if (!is_idle_task(p)) -- return p; -- } -- return NULL; --} -- --/** -- * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked -- * @iter: iterator to walk -- * -- * Visit the next non-idle task with its rq lock held. See scx_task_iter_init() -- * for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered_locked(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- p = scx_task_iter_next_filtered(iter); -- if (!p) -- return NULL; -- -- iter->rq = task_rq_lock(p, &iter->rf); -- iter->locked = p; -- return p; --} -- --static enum scx_ops_enable_state scx_ops_enable_state(void) --{ -- return atomic_read(&scx_ops_enable_state_var); --} -- --static enum scx_ops_enable_state --scx_ops_set_enable_state(enum scx_ops_enable_state to) --{ -- return atomic_xchg(&scx_ops_enable_state_var, to); --} -- --static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, -- enum scx_ops_enable_state from) --{ -- int from_v = from; -- -- return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); --} -- --static bool scx_ops_disabling(void) --{ -- return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING); --} -- --/** -- * wait_ops_state - Busy-wait the specified ops state to end -- * @p: target task -- * @opss: state to wait the end of -- * -- * Busy-wait for @p to transition out of @opss. This can only be used when the -- * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also -- * has load_acquire semantics to ensure that the caller can see the updates made -- * in the enqueueing and dispatching paths. -- */ --static void wait_ops_state(struct task_struct *p, u64 opss) --{ -- do { -- cpu_relax(); -- } while (atomic64_read_acquire(&p->scx->ops_state) == opss); --} -- --/** -- * ops_cpu_valid - Verify a cpu number -- * @cpu: cpu number which came from a BPF ops -- * -- * @cpu is a cpu number which came from the BPF scheduler and can be any value. -- * Verify that it is in range and one of the possible cpus. -- */ --static bool ops_cpu_valid(s32 cpu) --{ -- return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); --} -- --/** -- * ops_sanitize_err - Sanitize a -errno value -- * @ops_name: operation to blame on failure -- * @err: -errno value to sanitize -- * -- * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return -- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can -- * cause misbehaviors. For an example, a large negative return from -- * ops.prep_enable() triggers an oops when passed up the call chain because the -- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is -- * handled as a pointer. -- */ --static int ops_sanitize_err(const char *ops_name, s32 err) --{ -- if (err < 0 && err >= -MAX_ERRNO) -- return err; -- -- scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); -- return -EPROTO; --} -- --/** -- * touch_core_sched - Update timestamp used for core-sched task ordering -- * @rq: rq to read clock from, must be locked -- * @p: task to update the timestamp for -- * -- * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to -- * implement global or local-DSQ FIFO ordering for core-sched. Should be called -- * when a task becomes runnable and its turn on the CPU ends (e.g. slice -- * exhaustion). -- */ --static void touch_core_sched(struct rq *rq, struct task_struct *p) --{ --#ifdef CONFIG_SCHED_CORE -- /* -- * It's okay to update the timestamp spuriously. Use -- * sched_core_disabled() which is cheaper than enabled(). -- */ -- if (!sched_core_disabled()) -- p->scx->core_sched_at = rq_clock_task(rq); --#endif --} -- --/** -- * touch_core_sched_dispatch - Update core-sched timestamp on dispatch -- * @rq: rq to read clock from, must be locked -- * @p: task being dispatched -- * -- * If the BPF scheduler implements custom core-sched ordering via -- * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO -- * ordering within each local DSQ. This function is called from dispatch paths -- * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect. -- */ --static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- assert_clock_updated(rq); -- --#ifdef CONFIG_SCHED_CORE -- if (SCX_HAS_OP(core_sched_before)) -- touch_core_sched(rq, p); --#endif --} -- --static void update_curr_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- u64 now = rq_clock_task(rq); -- u64 delta_exec; -- -- if (time_before_eq64(now, curr->se.exec_start)) -- return; -- -- delta_exec = now - curr->se.exec_start; -- curr->se.exec_start = now; -- curr->se.sum_exec_runtime += delta_exec; -- account_group_exec_runtime(curr, delta_exec); -- cgroup_account_cputime(curr, delta_exec); -- -- if (curr->scx->slice != SCX_SLICE_INF) { -- curr->scx->slice -= min(curr->scx->slice, delta_exec); -- if (!curr->scx->slice) -- touch_core_sched(rq, curr); -- } --} -- --static bool scx_dsq_priq_less(struct rb_node *node_a, -- const struct rb_node *node_b) --{ -- const struct sched_ext_entity *a = -- container_of(node_a, struct sched_ext_entity, dsq_node.priq); -- const struct sched_ext_entity *b = -- container_of(node_b, struct sched_ext_entity, dsq_node.priq); -- -- return time_before64(a->dsq_vtime, b->dsq_vtime); --} -- --static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, -- u64 enq_flags) --{ -- bool is_local = dsq->id == SCX_DSQ_LOCAL; -- -- WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo)); -- WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq)); -- -- if (!is_local) { -- raw_spin_lock(&dsq->lock); -- if (unlikely(dsq->id == SCX_DSQ_INVALID)) { -- scx_ops_error("attempting to dispatch to a destroyed dsq"); -- /* fall back to the global dsq */ -- raw_spin_unlock(&dsq->lock); -- dsq = &scx_dsq_global; -- raw_spin_lock(&dsq->lock); -- } -- } -- -- if (enq_flags & SCX_ENQ_DSQ_PRIQ) { -- p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; -- rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq, -- scx_dsq_priq_less); -- } else { -- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) -- list_add(&p->scx->dsq_node.fifo, &dsq->fifo); -- else -- list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo); -- } -- dsq->nr++; -- p->scx->dsq = dsq; -- -- /* -- * We're transitioning out of QUEUEING or DISPATCHING. store_release to -- * match waiters' load_acquire. -- */ -- if (enq_flags & SCX_ENQ_CLEAR_OPSS) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- if (is_local) { -- struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq); -- struct rq *rq = scx->rq; -- bool preempt = false; -- -- if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && -- rq->curr->sched_class == &ext_sched_class) { -- rq->curr->scx->slice = 0; -- preempt = true; -- } -- -- if (preempt || sched_class_above(&ext_sched_class, -- rq->curr->sched_class)) -- resched_curr(rq); -- } else { -- raw_spin_unlock(&dsq->lock); -- } --} -- --static void task_unlink_from_dsq(struct task_struct *p, -- struct scx_dispatch_q *dsq) --{ -- if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { -- rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; -- } else { -- list_del_init(&p->scx->dsq_node.fifo); -- } -- --} -- --static bool task_linked_on_dsq(struct task_struct *p) --{ -- return !list_empty(&p->scx->dsq_node.fifo) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq); --} -- --static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p) --{ -- struct scx_dispatch_q *dsq = p->scx->dsq; -- bool is_local = dsq == &scx_rq->local_dsq; -- -- if (!dsq) { -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- /* -- * When dispatching directly from the BPF scheduler to a local -- * DSQ, the task isn't associated with any DSQ but -- * @p->scx->holding_cpu may be set under the protection of -- * %SCX_OPSS_DISPATCHING. -- */ -- if (p->scx->holding_cpu >= 0) -- p->scx->holding_cpu = -1; -- return; -- } -- -- if (!is_local) -- raw_spin_lock(&dsq->lock); -- -- /* -- * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node -- * can't change underneath us. -- */ -- if (p->scx->holding_cpu < 0) { -- /* @p must still be on @dsq, dequeue */ -- WARN_ON_ONCE(!task_linked_on_dsq(p)); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- } else { -- /* -- * We're racing against dispatch_to_local_dsq() which already -- * removed @p from @dsq and set @p->scx->holding_cpu. Clear the -- * holding_cpu which tells dispatch_to_local_dsq() that it lost -- * the race. -- */ -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- p->scx->holding_cpu = -1; -- } -- p->scx->dsq = NULL; -- -- if (!is_local) -- raw_spin_unlock(&dsq->lock); --} -- --static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) --{ -- lockdep_assert(rcu_read_lock_any_held()); -- -- return &scx_dsq_global; --} -- --static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, -- struct task_struct *p) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id == SCX_DSQ_LOCAL) -- return &rq->scx->local_dsq; -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", -- dsq_id, p->comm, p->pid); -- return &scx_dsq_global; -- } -- -- return dsq; --} -- --static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- -- /* @p must match the task which is being enqueued */ -- if (unlikely(p != ddsp_task)) { -- if (IS_ERR(ddsp_task)) -- scx_ops_error("%s[%d] already direct-dispatched", -- p->comm, p->pid); -- else -- scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]", -- ddsp_task->comm, ddsp_task->pid, -- p->comm, p->pid); -- return; -- } -- -- /* -- * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because -- * dispatching to the local DSQ of a different CPU requires unlocking -- * the current rq which isn't allowed in the enqueue path. Use -- * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL. -- */ -- if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) { -- scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch"); -- return; -- } -- -- touch_core_sched_dispatch(task_rq(p), p); -- -- dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- -- /* -- * Mark that dispatch already happened by spoiling direct_dispatch_task -- * with a non-NULL value which can never match a valid task pointer. -- */ -- __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); --} -- --static bool test_rq_online(struct rq *rq) --{ --#ifdef CONFIG_SMP -- return rq->online; --#else -- return true; --#endif --} -- --static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- int sticky_cpu) --{ -- struct task_struct **ddsp_taskp; -- u64 qseq; -- -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- if (p->scx->flags & SCX_TASK_ENQ_LOCAL) { -- enq_flags |= SCX_ENQ_LOCAL; -- p->scx->flags &= ~SCX_TASK_ENQ_LOCAL; -- } -- -- /* rq migration */ -- if (sticky_cpu == cpu_of(rq)) -- goto local_norefill; -- -- /* -- * If !rq->online, we already told the BPF scheduler that the CPU is -- * offline. We're just trying to on/offline the CPU. Don't bother the -- * BPF scheduler. -- */ -- if (unlikely(!test_rq_online(rq))) -- goto local; -- -- /* see %SCX_OPS_ENQ_EXITING */ -- if (!static_branch_unlikely(&scx_ops_enq_exiting) && -- unlikely(p->flags & PF_EXITING)) -- goto local; -- -- /* see %SCX_OPS_ENQ_LAST */ -- if (!static_branch_unlikely(&scx_ops_enq_last) && -- (enq_flags & SCX_ENQ_LAST)) -- goto local; -- -- if (!SCX_HAS_OP(enqueue)) { -- if (enq_flags & SCX_ENQ_LOCAL) -- goto local; -- else -- goto global; -- } -- -- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ -- qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; -- -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq); -- -- ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); -- WARN_ON_ONCE(*ddsp_taskp); -- *ddsp_taskp = p; -- -- SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); -- -- /* -- * If not directly dispatched, QUEUEING isn't clear yet and dispatch or -- * dequeue may be waiting. The store_release matches their load_acquire. -- */ -- if (*ddsp_taskp == p) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq); -- *ddsp_taskp = NULL; -- return; -- --local: -- /* -- * For task-ordering, slice refill must be treated as implying the end -- * of the current slice. Otherwise, the longer @p stays on the CPU, the -- * higher priority it becomes from scx_prio_less()'s POV. -- */ -- touch_core_sched(rq, p); -- p->scx->slice = SCX_SLICE_DFL; --local_norefill: -- dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags); -- return; -- --global: -- touch_core_sched(rq, p); /* see the comment in local: */ -- p->scx->slice = SCX_SLICE_DFL; -- dispatch_enqueue(&scx_dsq_global, p, enq_flags); --} -- --static bool watchdog_task_watched(const struct task_struct *p) --{ -- return !list_empty(&p->scx->watchdog_node); --} -- --static void watchdog_watch_task(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- if (p->scx->flags & SCX_TASK_WATCHDOG_RESET) -- p->scx->runnable_at = jiffies; -- p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET; -- list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list); --} -- --static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) --{ -- list_del_init(&p->scx->watchdog_node); -- if (reset_timeout) -- p->scx->flags |= SCX_TASK_WATCHDOG_RESET; --} -- --static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) --{ -- int sticky_cpu = p->scx->sticky_cpu; -- -- enq_flags |= rq->scx->extra_enq_flags; -- -- if (sticky_cpu >= 0) -- p->scx->sticky_cpu = -1; -- -- /* -- * Restoring a running task will be immediately followed by -- * set_next_task_scx() which expects the task to not be on the BPF -- * scheduler as tasks can only start running through local DSQs. Force -- * direct-dispatch into the local DSQ by setting the sticky_cpu. -- */ -- if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -- sticky_cpu = cpu_of(rq); -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- WARN_ON_ONCE(!watchdog_task_watched(p)); -- return; -- } -- -- watchdog_watch_task(rq, p); -- p->scx->flags |= SCX_TASK_QUEUED; -- rq->scx->nr_running++; -- add_nr_running(rq, 1); -- -- if (SCX_HAS_OP(runnable)) -- SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); -- -- if (enq_flags & SCX_ENQ_WAKEUP) -- touch_core_sched(rq, p); -- -- do_enqueue_task(rq, p, enq_flags, sticky_cpu); --} -- --static void ops_dequeue(struct task_struct *p, u64 deq_flags) --{ -- u64 opss; -- -- watchdog_unwatch_task(p, false); -- -- /* acquire ensures that we see the preceding updates on QUEUED */ -- opss = atomic64_read_acquire(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_NONE: -- break; -- case SCX_OPSS_QUEUEING: -- /* -- * QUEUEING is started and finished while holding @p's rq lock. -- * As we're holding the rq lock now, we shouldn't see QUEUEING. -- */ -- BUG(); -- case SCX_OPSS_QUEUED: -- if (SCX_HAS_OP(dequeue)) -- SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); -- -- if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_NONE)) -- break; -- fallthrough; -- case SCX_OPSS_DISPATCHING: -- /* -- * If @p is being dispatched from the BPF scheduler to a DSQ, -- * wait for the transfer to complete so that @p doesn't get -- * added to its DSQ after dequeueing is complete. -- * -- * As we're waiting on DISPATCHING with the rq locked, the -- * dispatching side shouldn't try to lock the rq while -- * DISPATCHING is set. See dispatch_to_local_dsq(). -- * -- * DISPATCHING shouldn't have qseq set and control can reach -- * here with NONE @opss from the above QUEUED case block. -- * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. -- */ -- wait_ops_state(p, SCX_OPSS_DISPATCHING); -- BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- break; -- } --} -- --static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) --{ -- struct scx_rq *scx_rq = rq->scx; -- -- if (!(p->scx->flags & SCX_TASK_QUEUED)) { -- WARN_ON_ONCE(watchdog_task_watched(p)); -- return; -- } -- -- ops_dequeue(p, deq_flags); -- -- /* -- * A currently running task which is going off @rq first gets dequeued -- * and then stops running. As we want running <-> stopping transitions -- * to be contained within runnable <-> quiescent transitions, trigger -- * ->stopping() early here instead of in put_prev_task_scx(). -- * -- * @p may go through multiple stopping <-> running transitions between -- * here and put_prev_task_scx() if task attribute changes occur while -- * balance_scx() leaves @rq unlocked. However, they don't contain any -- * information meaningful to the BPF scheduler and can be suppressed by -- * skipping the callbacks if the task is !QUEUED. -- */ -- if (SCX_HAS_OP(stopping) && task_current(rq, p)) { -- update_curr_scx(rq); -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); -- } -- -- //if(task_current(rq, p)) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- if (SCX_HAS_OP(quiescent)) -- SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); -- -- if (deq_flags & SCX_DEQ_SLEEP) -- p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP; -- else -- p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP; -- -- p->scx->flags &= ~SCX_TASK_QUEUED; -- BUG_ON(!scx_rq->nr_running); -- scx_rq->nr_running--; -- sub_nr_running(rq, 1); -- -- dispatch_dequeue(scx_rq, p); --} -- --static void yield_task_scx(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); -- else -- p->scx->slice = 0; --} -- --static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) --{ -- struct task_struct *from = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); -- else -- return false; --} -- --#ifdef CONFIG_SMP --/** -- * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -- * @rq: rq to move the task into, currently locked -- * @p: task to move -- * @enq_flags: %SCX_ENQ_* -- * -- * Move @p which is currently on a different rq to @rq's local DSQ. The caller -- * must: -- * -- * 1. Start with exclusive access to @p either through its DSQ lock or -- * %SCX_OPSS_DISPATCHING flag. -- * -- * 2. Set @p->scx->holding_cpu to raw_smp_processor_id(). -- * -- * 3. Remember task_rq(@p). Release the exclusive access so that we don't -- * deadlock with dequeue. -- * -- * 4. Lock @rq and the task_rq from #3. -- * -- * 5. Call this function. -- * -- * Returns %true if @p was successfully moved. %false after racing dequeue and -- * losing. -- */ --static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -- u64 enq_flags) --{ -- struct rq *task_rq; -- -- lockdep_assert_rq_held(rq); -- -- /* -- * If dequeue got to @p while we were trying to lock both rq's, it'd -- * have cleared @p->scx->holding_cpu to -1. While other cpus may have -- * updated it to different values afterwards, as this operation can't be -- * preempted or recurse, @p->scx->holding_cpu can never become -- * raw_smp_processor_id() again before we're done. Thus, we can tell -- * whether we lost to dequeue by testing whether @p->scx->holding_cpu is -- * still raw_smp_processor_id(). -- * -- * See dispatch_dequeue() for the counterpart. -- */ -- if (unlikely(p->scx->holding_cpu != raw_smp_processor_id())) -- return false; -- -- /* @p->rq couldn't have changed if we're still the holding cpu */ -- task_rq = task_rq(p); -- lockdep_assert_rq_held(task_rq); -- -- WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)); -- deactivate_task(task_rq, p, 0); -- set_task_cpu(p, cpu_of(rq)); -- p->scx->sticky_cpu = cpu_of(rq); -- -- /* -- * We want to pass scx-specific enq_flags but activate_task() will -- * truncate the upper 32 bit. As we own @rq, we can pass them through -- * @rq->scx->extra_enq_flags instead. -- */ -- WARN_ON_ONCE(rq->scx->extra_enq_flags); -- rq->scx->extra_enq_flags = enq_flags; -- activate_task(rq, p, 0); -- rq->scx->extra_enq_flags = 0; -- -- return true; --} -- --/** -- * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * We're holding @rq lock and trying to dispatch a task from @src_rq to -- * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether -- * @rq stays locked isn't important as long as the state is restored after -- * dispatch_to_local_dsq_unlock(). -- */ --static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- rq_unpin_lock(rq, rf); -- -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(rq); -- raw_spin_rq_lock(dst_rq); -- } else if (rq == src_rq) { -- double_lock_balance(rq, dst_rq); -- rq_repin_lock(rq, rf); -- } else if (rq == dst_rq) { -- double_lock_balance(rq, src_rq); -- rq_repin_lock(rq, rf); -- } else { -- raw_spin_rq_unlock(rq); -- double_rq_lock(src_rq, dst_rq); -- } --} -- --/** -- * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock() -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return. -- */ --static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } else if (rq == src_rq) { -- double_unlock_balance(rq, dst_rq); -- } else if (rq == dst_rq) { -- double_unlock_balance(rq, src_rq); -- } else { -- double_rq_unlock(src_rq, dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } --} --#endif /* CONFIG_SMP */ -- -- --static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq) --{ -- return likely(test_rq_online(rq)) && !is_migration_disabled(p) && -- cpumask_test_cpu(cpu_of(rq), p->cpus_ptr); --} -- --static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -- struct scx_dispatch_q *dsq) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rb_node *rb_node; -- struct rq *task_rq; -- bool moved = false; --retry: -- if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -- return false; -- -- raw_spin_lock(&dsq->lock); -- -- list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- for (rb_node = rb_first_cached(&dsq->priq); rb_node; -- rb_node = rb_next(rb_node)) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- raw_spin_unlock(&dsq->lock); -- return false; -- --this_rq: -- /* @dsq is locked and @p is on this rq */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo); -- dsq->nr--; -- scx_rq->local_dsq.nr++; -- p->scx->dsq = &scx_rq->local_dsq; -- raw_spin_unlock(&dsq->lock); -- return true; -- --remote_rq: --#ifdef CONFIG_SMP -- /* -- * @dsq is locked and @p is on a remote rq. @p is currently protected by -- * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -- * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -- * rq lock or fail, do a little dancing from our side. See -- * move_task_to_local_dsq(). -- */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- p->scx->holding_cpu = raw_smp_processor_id(); -- raw_spin_unlock(&dsq->lock); -- -- rq_unpin_lock(rq, rf); -- double_lock_balance(rq, task_rq); -- rq_repin_lock(rq, rf); -- -- moved = move_task_to_local_dsq(rq, p, 0); -- -- double_unlock_balance(rq, task_rq); --#endif /* CONFIG_SMP */ -- if (likely(moved)) -- return true; -- goto retry; --} -- --enum dispatch_to_local_dsq_ret { -- DTL_DISPATCHED, /* successfully dispatched */ -- DTL_LOST, /* lost race to dequeue */ -- DTL_NOT_LOCAL, /* destination is not a local DSQ */ -- DTL_INVALID, /* invalid local dsq_id */ --}; -- --/** -- * dispatch_to_local_dsq - Dispatch a task to a local dsq -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @dsq_id: destination dsq ID -- * @p: task to dispatch -- * @enq_flags: %SCX_ENQ_* -- * -- * We're holding @rq lock and want to dispatch @p to the local DSQ identified by -- * @dsq_id. This function performs all the synchronization dancing needed -- * because local DSQs are protected with rq locks. -- * -- * The caller must have exclusive ownership of @p (e.g. through -- * %SCX_OPSS_DISPATCHING). -- */ --static enum dispatch_to_local_dsq_ret --dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id, -- struct task_struct *p, u64 enq_flags) --{ -- struct rq *src_rq = task_rq(p); -- struct rq *dst_rq; -- -- /* -- * We're synchronized against dequeue through DISPATCHING. As @p can't -- * be dequeued, its task_rq and cpus_allowed are stable too. -- */ -- if (dsq_id == SCX_DSQ_LOCAL) { -- dst_rq = rq; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]", -- cpu, p->comm, p->pid); -- return DTL_INVALID; -- } -- dst_rq = cpu_rq(cpu); -- } else { -- return DTL_NOT_LOCAL; -- } -- -- /* if dispatching to @rq that @p is already on, no lock dancing needed */ -- if (rq == src_rq && rq == dst_rq) { -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags | SCX_ENQ_CLEAR_OPSS); -- return DTL_DISPATCHED; -- } -- --#ifdef CONFIG_SMP -- if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { -- struct rq *locked_dst_rq = dst_rq; -- bool dsp; -- -- /* -- * @p is on a possibly remote @src_rq which we need to lock to -- * move the task. If dequeue is in progress, it'd be locking -- * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq -- * lock while holding DISPATCHING. -- * -- * As DISPATCHING guarantees that @p is wholly ours, we can -- * pretend that we're moving from a DSQ and use the same -- * mechanism - mark the task under transfer with holding_cpu, -- * release DISPATCHING and then follow the same protocol. -- */ -- p->scx->holding_cpu = raw_smp_processor_id(); -- -- /* store_release ensures that dequeue sees the above */ -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq); -- -- /* -- * We don't require the BPF scheduler to avoid dispatching to -- * offline CPUs mostly for convenience but also because CPUs can -- * go offline between scx_bpf_dispatch() calls and here. If @p -- * is destined to an offline CPU, queue it on its current CPU -- * instead, which should always be safe. As this is an allowed -- * behavior, don't trigger an ops error. -- */ -- if (unlikely(!test_rq_online(dst_rq))) -- dst_rq = src_rq; -- -- if (src_rq == dst_rq) { -- /* -- * As @p is staying on the same rq, there's no need to -- * go through the full deactivate/activate cycle. -- * Optimize by abbreviating the operations in -- * move_task_to_local_dsq(). -- */ -- dsp = p->scx->holding_cpu == raw_smp_processor_id(); -- if (likely(dsp)) { -- p->scx->holding_cpu = -1; -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags); -- } -- } else { -- dsp = move_task_to_local_dsq(dst_rq, p, enq_flags); -- } -- -- /* if the destination CPU is idle, wake it up */ -- if (dsp && p->sched_class > dst_rq->curr->sched_class) -- resched_curr(dst_rq); -- -- dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq); -- -- return dsp ? DTL_DISPATCHED : DTL_LOST; -- } --#endif /* CONFIG_SMP */ -- -- scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", -- cpu_of(dst_rq), p->comm, p->pid); -- return DTL_INVALID; --} -- --/** -- * finish_dispatch - Asynchronously finish dispatching a task -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @p: task to finish dispatching -- * @qseq_at_dispatch: qseq when @p started getting dispatched -- * @dsq_id: destination DSQ ID -- * @enq_flags: %SCX_ENQ_* -- * -- * Dispatching to local DSQs may need to wait for queueing to complete or -- * require rq lock dancing. As we don't wanna do either while inside -- * ops.dispatch() to avoid locking order inversion, we split dispatching into -- * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the -- * task and its qseq. Once ops.dispatch() returns, this function is called to -- * finish up. -- * -- * There is no guarantee that @p is still valid for dispatching or even that it -- * was valid in the first place. Make sure that the task is still owned by the -- * BPF scheduler and claim the ownership before dispatching. -- */ --static void finish_dispatch(struct rq *rq, struct rq_flags *rf, -- struct task_struct *p, u64 qseq_at_dispatch, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- u64 opss; -- -- touch_core_sched_dispatch(rq, p); --retry: -- /* -- * No need for _acquire here. @p is accessed only after a successful -- * try_cmpxchg to DISPATCHING. -- */ -- opss = atomic64_read(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_DISPATCHING: -- case SCX_OPSS_NONE: -- /* someone else already got to it */ -- return; -- case SCX_OPSS_QUEUED: -- /* -- * If qseq doesn't match, @p has gone through at least one -- * dispatch/dequeue and re-enqueue cycle between -- * scx_bpf_dispatch() and here and we have no claim on it. -- */ -- if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) -- return; -- -- /* -- * While we know @p is accessible, we don't yet have a claim on -- * it - the BPF scheduler is allowed to dispatch tasks -- * spuriously and there can be a racing dequeue attempt. Let's -- * claim @p by atomically transitioning it from QUEUED to -- * DISPATCHING. -- */ -- if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_DISPATCHING))) -- break; -- goto retry; -- case SCX_OPSS_QUEUEING: -- /* -- * do_enqueue_task() is in the process of transferring the task -- * to the BPF scheduler while holding @p's rq lock. As we aren't -- * holding any kernel or BPF resource that the enqueue path may -- * depend upon, it's safe to wait. -- */ -- wait_ops_state(p, opss); -- goto retry; -- } -- -- BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) { -- case DTL_DISPATCHED: -- break; -- case DTL_LOST: -- break; -- case DTL_INVALID: -- dsq_id = SCX_DSQ_GLOBAL; -- fallthrough; -- case DTL_NOT_LOCAL: -- dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()), -- dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- break; -- } --} -- --static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- u32 u; -- -- for (u = 0; u < dspc->buf_cursor; u++) { -- struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u]; -- -- finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id, -- ent->enq_flags); -- } -- -- dspc->nr_tasks += dspc->buf_cursor; -- dspc->buf_cursor = 0; --} -- --static int balance_one(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf, bool local) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- bool prev_on_scx = prev->sched_class == &ext_sched_class; -- int nr_loops = SCX_DSP_MAX_LOOPS; -- -- lockdep_assert_rq_held(rq); -- -- if (static_branch_unlikely(&scx_ops_cpu_preempt) && -- unlikely(rq->scx->cpu_released)) { -- /* -- * If the previous sched_class for the current CPU was not SCX, -- * notify the BPF scheduler that it again has control of the -- * core. This callback complements ->cpu_release(), which is -- * emitted in scx_notify_pick_next_task(). -- */ -- if (SCX_HAS_OP(cpu_acquire)) -- SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq), -- NULL); -- rq->scx->cpu_released = false; -- } -- -- if (prev_on_scx) { -- WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP)); -- update_curr_scx(rq); -- -- /* -- * If @prev is runnable & has slice left, it has priority and -- * fetching more just increases latency for the fetched tasks. -- * Tell put_prev_task_scx() to put @prev on local_dsq. If the -- * BPF scheduler wants to handle this explicitly, it should -- * implement ->cpu_released(). -- * -- * See scx_ops_disable_workfn() for the explanation on the -- * disabling() test. -- * -- * When balancing a remote CPU for core-sched, there won't be a -- * following put_prev_task_scx() call and we don't own -- * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the -- * same conditions later and pick @rq->curr accordingly. -- */ -- if ((prev->scx->flags & SCX_TASK_QUEUED) && -- prev->scx->slice && !scx_ops_disabling()) { -- if (local) -- prev->scx->flags |= SCX_TASK_BAL_KEEP; -- return 1; -- } -- } -- -- /* if there already are tasks to run, nothing to do */ -- if (scx_rq->local_dsq.nr) -- return 1; -- -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- if (!SCX_HAS_OP(dispatch)) -- return 0; -- -- dspc->rq = rq; -- dspc->rf = rf; -- -- /* -- * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, -- * the local DSQ might still end up empty after a successful -- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() -- * produced some tasks, retry. The BPF scheduler may depend on this -- * looping behavior to simplify its implementation. -- */ -- do { -- dspc->nr_tasks = 0; -- -- SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), -- prev_on_scx ? prev : NULL); -- -- flush_dispatch_buf(rq, rf); -- -- if (scx_rq->local_dsq.nr) -- return 1; -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- /* -- * ops.dispatch() can trap us in this loop by repeatedly -- * dispatching ineligible tasks. Break out once in a while to -- * allow the watchdog to run. As IRQ can't be enabled in -- * balance(), we want to complete this scheduling cycle and then -- * start a new one. IOW, we want to call resched_curr() on the -- * next, most likely idle, task, not the current one. Use -- * scx_bpf_kick_cpu() for deferred kicking. -- */ -- if (unlikely(!--nr_loops)) { -- scx_bpf_kick_cpu(cpu_of(rq), 0); -- break; -- } -- } while (dspc->nr_tasks); -- -- return 0; --} -- --static int balance_scx(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf) --{ -- int ret; -- -- ret = balance_one(rq, prev, rf, true); --#ifdef CONFIG_SCHED_SMT -- /* -- * When core-sched is enabled, this ops.balance() call will be followed -- * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx() -- * on the SMT siblings. Balance the siblings too. -- */ -- if (sched_core_enabled(rq)) { -- const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); -- int scpu; -- -- for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { -- struct rq *srq = cpu_rq(scpu); -- struct rq_flags srf; -- struct task_struct *sprev = srq->curr; -- -- /* -- * While core-scheduling, rq lock is shared among -- * siblings but the debug annotations and rq clock -- * aren't. Do pinning dance to transfer the ownership. -- */ -- WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); -- rq_unpin_lock(rq, rf); -- rq_pin_lock(srq, &srf); -- -- update_rq_clock(srq); -- balance_one(srq, sprev, &srf, false); -- -- rq_unpin_lock(srq, &srf); -- rq_repin_lock(rq, rf); -- } -- } --#endif -- return ret; --} -- --static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) --{ -- if (p->scx->flags & SCX_TASK_QUEUED) { -- /* -- * Core-sched might decide to execute @p before it is -- * dispatched. Call ops_dequeue() to notify the BPF scheduler. -- */ -- ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); -- dispatch_dequeue(rq->scx, p); -- } -- -- p->se.exec_start = rq_clock_task(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, running, p); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock); -- -- watchdog_unwatch_task(p, true); -- -- /* -- * @p is getting newly scheduled or got kicked after someone updated its -- * slice. Refresh whether tick can be stopped. See can_stop_tick_scx(). -- */ -- if ((p->scx->slice == SCX_SLICE_INF) != -- (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) { -- if (p->scx->slice == SCX_SLICE_INF) -- rq->scx->flags |= SCX_RQ_CAN_STOP_TICK; -- else -- rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK; -- -- sched_update_tick_dependency(rq); -- } --} -- --static void put_prev_task_scx(struct rq *rq, struct task_struct *p) --{ --#ifndef CONFIG_SMP -- /* -- * UP workaround. -- * -- * Because SCX may transfer tasks across CPUs during dispatch, dispatch -- * is performed from its balance operation which isn't called in UP. -- * Let's work around by calling it from the operations which come right -- * after. -- * -- * 1. If the prev task is on SCX, pick_next_task() calls -- * .put_prev_task() right after. As .put_prev_task() is also called -- * from other places, we need to distinguish the calls which can be -- * done by looking at the previous task's state - if still queued or -- * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task(). -- * This case is handled here. -- * -- * 2. If the prev task is not on SCX, the first following call into SCX -- * will be .pick_next_task(), which is covered by calling -- * balance_scx() from pick_next_task_scx(). -- * -- * Note that we can't merge the first case into the second as -- * balance_scx() must be called before the previous SCX task goes -- * through put_prev_task_scx(). -- * -- * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf. -- * Pass in %NULL. -- */ -- if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP)) -- balance_scx(rq, p, NULL); --#endif -- -- update_curr_scx(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- -- /* -- * If we're being called from put_prev_task_balance(), balance_scx() may -- * have decided that @p should keep running. -- */ -- if (p->scx->flags & SCX_TASK_BAL_KEEP) { -- p->scx->flags &= ~SCX_TASK_BAL_KEEP; -- watchdog_watch_task(rq, p); -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- watchdog_watch_task(rq, p); -- -- /* -- * If @p has slice left and balance_scx() didn't tag it for -- * keeping, @p is getting preempted by a higher priority -- * scheduler class or core-sched forcing a different task. Leave -- * it at the head of the local DSQ. -- */ -- if (p->scx->slice && !scx_ops_disabling()) { -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- /* -- * If we're in the pick_next_task path, balance_scx() should -- * have already populated the local DSQ if there are any other -- * available tasks. If empty, tell ops.enqueue() that @p is the -- * only one available for this cpu. ops.enqueue() should put it -- * on the local DSQ so that the subsequent pick_next_task_scx() -- * can find the task unless it wants to trigger a separate -- * follow-up scheduling event. -- */ -- if (list_empty(&rq->scx->local_dsq.fifo)) -- do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1); -- else -- do_enqueue_task(rq, p, 0, -1); -- } --} -- --static struct task_struct *first_local_task(struct rq *rq) --{ -- struct rb_node *rb_node; -- struct sched_ext_entity *entity; -- -- if (!list_empty(&rq->scx->local_dsq.fifo)) { -- entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo); -- return entity->task; -- } -- -- rb_node = rb_first_cached(&rq->scx->local_dsq.priq); -- if (rb_node) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- return entity->task; -- } -- -- return NULL; --} -- --static struct task_struct *pick_next_task_scx(struct rq *rq) --{ -- struct task_struct *p; -- --#ifndef CONFIG_SMP -- /* UP workaround - see the comment at the head of put_prev_task_scx() */ -- if (unlikely(rq->curr->sched_class != &ext_sched_class)) -- balance_scx(rq, rq->curr, NULL); --#endif -- -- p = first_local_task(rq); -- if (!p) -- return NULL; -- -- if (unlikely(!p->scx->slice)) { -- if (!scx_ops_disabling() && !scx_warned_zero_slice) { -- printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n", -- p->comm, p->pid); -- scx_warned_zero_slice = true; -- } -- p->scx->slice = SCX_SLICE_DFL; -- } -- -- set_next_task_scx(rq, p, true); -- -- return p; --} -- --#ifdef CONFIG_SCHED_CORE --/** -- * scx_prio_less - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Core-sched is implemented as an additional scheduling layer on top of the -- * usual sched_class'es and needs to find out the expected task ordering. For -- * SCX, core-sched calls this function to interrogate the task ordering. -- * -- * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used -- * to implement the default task ordering. The older the timestamp, the higher -- * prority the task - the global FIFO ordering matching the default scheduling -- * behavior. -- * -- * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to -- * implement FIFO ordering within each local DSQ. See pick_task_scx(). -- */ --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi) --{ -- /* -- * The const qualifiers are dropped from task_struct pointers when -- * calling ops.core_sched_before(). Accesses are controlled by the -- * verifier. -- */ -- if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling()) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, -- (struct task_struct *)a, -- (struct task_struct *)b); -- else -- return time_after64(a->scx->core_sched_at, b->scx->core_sched_at); --} -- --/** -- * pick_task_scx - Pick a candidate task for core-sched -- * @rq: rq to pick the candidate task from -- * -- * Core-sched calls this function on each SMT sibling to determine the next -- * tasks to run on the SMT siblings. balance_one() has been called on all -- * siblings and put_prev_task_scx() has been called only for the current CPU. -- * -- * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look -- * at the first task in the local dsq. @rq->curr has to be considered explicitly -- * to mimic %SCX_TASK_BAL_KEEP. -- */ --static struct task_struct *pick_task_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- struct task_struct *first = first_local_task(rq); -- -- if (curr->scx->flags & SCX_TASK_QUEUED) { -- /* is curr the only runnable task? */ -- if (!first) -- return curr; -- -- /* -- * Does curr trump first? We can always go by core_sched_at for -- * this comparison as it represents global FIFO ordering when -- * the default core-sched ordering is used and local-DSQ FIFO -- * ordering otherwise. -- * -- * We can have a task with an earlier timestamp on the DSQ. For -- * example, when a current task is preempted by a sibling -- * picking a different cookie, the task would be requeued at the -- * head of the local DSQ with an earlier timestamp than the -- * core-sched picked next task. Besides, the BPF scheduler may -- * dispatch any tasks to the local DSQ anytime. -- */ -- if (curr->scx->slice && time_before64(curr->scx->core_sched_at, -- first->scx->core_sched_at)) -- return curr; -- } -- -- return first; /* this may be %NULL */ --} --#endif /* CONFIG_SCHED_CORE */ -- --static enum scx_cpu_preempt_reason --preempt_reason_from_class(const struct sched_class *class) --{ --#ifdef CONFIG_SMP -- if (class == &stop_sched_class) -- return SCX_CPU_PREEMPT_STOP; --#endif -- if (class == &dl_sched_class) -- return SCX_CPU_PREEMPT_DL; -- if (class == &rt_sched_class) -- return SCX_CPU_PREEMPT_RT; -- return SCX_CPU_PREEMPT_UNKNOWN; --} -- --void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task, -- const struct sched_class *active) --{ -- lockdep_assert_rq_held(rq); -- -- /* -- * The callback is conceptually meant to convey that the CPU is no -- * longer under the control of SCX. Therefore, don't invoke the -- * callback if the CPU is is staying on SCX, or going idle (in which -- * case the SCX scheduler has actively decided not to schedule any -- * tasks on the CPU). -- */ -- if (likely(active >= &ext_sched_class)) -- return; -- -- /* -- * At this point we know that SCX was preempted by a higher priority -- * sched_class, so invoke the ->cpu_release() callback if we have not -- * done so already. We only send the callback once between SCX being -- * preempted, and it regaining control of the CPU. -- * -- * ->cpu_release() complements ->cpu_acquire(), which is emitted the -- * next time that balance_scx() is invoked. -- */ -- if (!rq->scx->cpu_released) { -- if (SCX_HAS_OP(cpu_release)) { -- struct scx_cpu_release_args args = { -- .reason = preempt_reason_from_class(active), -- .task = task, -- }; -- -- SCX_CALL_OP(SCX_KF_CPU_RELEASE, -- cpu_release, cpu_of(rq), &args); -- } -- rq->scx->cpu_released = true; -- } --} -- --#ifdef CONFIG_SMP -- --static bool test_and_clear_cpu_idle(int cpu) --{ -- if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -- if (cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- return true; -- } else { -- return false; -- } --} -- --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- int cpu; -- -- do { -- cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); -- if (cpu < nr_cpu_ids) { -- const struct cpumask *sbm = topology_sibling_cpumask(cpu); -- -- /* -- * If offline, @cpu is not its own sibling and we can -- * get caught in an infinite loop as @cpu is never -- * cleared from idle_masks.smt. Clear @cpu directly in -- * such cases. -- */ -- if (likely(cpumask_test_cpu(cpu, sbm))) -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm); -- else -- cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu)); -- } else { -- cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -- if (cpu >= nr_cpu_ids) -- return -EBUSY; -- } -- } while (!test_and_clear_cpu_idle(cpu)); -- -- return cpu; --} -- --static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) --{ -- s32 cpu; -- -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return prev_cpu; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local DSQ of the waker. -- */ -- if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 && -- scx_has_idle_cpus && !(current->flags & PF_EXITING)) { -- cpu = smp_processor_id(); -- if (cpumask_test_cpu(cpu, p->cpus_ptr)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (test_and_clear_cpu_idle(prev_cpu)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return prev_cpu; -- } -- -- if (p->nr_cpus_allowed == 1) -- return prev_cpu; -- -- cpu = scx_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- -- return prev_cpu; --} -- --static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) --{ -- if (SCX_HAS_OP(select_cpu)) { -- s32 cpu; -- -- cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu, -- wake_flags); -- if (ops_cpu_valid(cpu)) { -- return cpu; -- } else { -- scx_ops_error("select_cpu returned invalid cpu %d", cpu); -- return prev_cpu; -- } -- } else { -- return scx_select_cpu_dfl(p, prev_cpu, wake_flags); -- } --} -- --static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx) --{ -- set_cpus_allowed_common(p, ctx); -- -- /* -- * The effective cpumask is stored in @p->cpus_ptr which may temporarily -- * differ from the configured one in @p->cpus_mask. Always tell the bpf -- * scheduler the effective one. -- * -- * Fine-grained memory write control is enforced by BPF making the const -- * designation pointless. Cast it away when calling the operation. -- */ -- if (SCX_HAS_OP(set_cpumask)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, -- (struct cpumask *)p->cpus_ptr); --} -- --static void reset_idle_masks(void) --{ -- /* consider all cpus idle, should converge to the actual state quickly */ -- cpumask_setall(idle_masks.cpu); -- cpumask_setall(idle_masks.smt); -- scx_has_idle_cpus = true; --} -- --void __scx_update_idle(struct rq *rq, bool idle) --{ -- int cpu = cpu_of(rq); -- struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -- -- if (SCX_HAS_OP(update_idle)) { -- SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); -- if (!static_branch_unlikely(&scx_builtin_idle_enabled)) -- return; -- } -- -- if (idle) { -- cpumask_set_cpu(cpu, idle_masks.cpu); -- if (!scx_has_idle_cpus) -- scx_has_idle_cpus = true; -- -- /* -- * idle_masks.smt handling is racy but that's fine as it's only -- * for optimization and self-correcting. -- */ -- for_each_cpu(cpu, sib_mask) { -- if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -- return; -- } -- cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -- } else { -- cpumask_clear_cpu(cpu, idle_masks.cpu); -- if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -- } --} -- --#else /* !CONFIG_SMP */ -- --static bool test_and_clear_cpu_idle(int cpu) { return false; } --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } --static void reset_idle_masks(void) {} -- --#endif /* CONFIG_SMP */ -- --static bool check_rq_for_timeouts(struct rq *rq) --{ -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rq_flags rf; -- bool timed_out = false; -- -- rq_lock_irqsave(rq, &rf); -- list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) { -- unsigned long last_runnable; -- -- p = entity->task; -- last_runnable = p->scx->runnable_at; -- -- if (unlikely(time_after(jiffies, -- last_runnable + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "%s[%d] failed to run for %u.%03us", -- p->comm, p->pid, -- dur_ms / 1000, dur_ms % 1000); -- timed_out = true; -- break; -- } -- } -- rq_unlock_irqrestore(rq, &rf); -- -- return timed_out; --} -- --static void scx_watchdog_workfn(struct work_struct *work) --{ -- int cpu; -- -- scx_watchdog_timestamp = jiffies; -- -- for_each_online_cpu(cpu) { -- if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) -- break; -- -- cond_resched(); -- } -- queue_delayed_work(system_unbound_wq, to_delayed_work(work), -- scx_watchdog_timeout / 2); --} -- --static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) --{ -- update_curr_scx(rq); -- //scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock); -- /* -- * While disabling, always resched and refresh core-sched timestamp as -- * we can't trust the slice management or ops.core_sched_before(). -- */ -- if (scx_ops_disabling()) { -- curr->scx->slice = 0; -- touch_core_sched(rq, curr); -- } -- -- if (!curr->scx->slice) -- resched_curr(rq); --} -- --#define SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- --static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg) --{ -- int ret; -- -- WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED); -- -- p->scx->disallow = false; -- -- if (SCX_HAS_OP(prep_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- }; -- -- ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args); -- if (unlikely(ret)) { -- ret = ops_sanitize_err("prep_enable", ret); -- return ret; -- } -- } -- //scx_sched_init_task(p); -- -- if (p->scx->disallow) { -- struct rq *rq; -- struct rq_flags rf; -- -- rq = task_rq_lock(p, &rf); -- -- /* -- * We're either in fork or load path and @p->policy will be -- * applied right after. Reverting @p->policy here and rejecting -- * %SCHED_EXT transitions from scx_check_setscheduler() -- * guarantees that if ops.prep_enable() sets @p->disallow, @p -- * can never be in SCX. -- */ -- if (p->policy == SCHED_EXT) { -- p->policy = SCHED_NORMAL; -- atomic64_inc(&scx_nr_rejected); -- } -- -- task_rq_unlock(rq, p, &rf); -- } -- -- p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET); -- return 0; --} -- --static void scx_ops_enable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED)); -- -- if (SCX_HAS_OP(enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- p->scx->flags |= SCX_TASK_OPS_ENABLED; --} -- --static void scx_ops_disable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- if (p->scx->flags & SCX_TASK_OPS_PREPPED) { -- if (SCX_HAS_OP(cancel_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- } else if (p->scx->flags & SCX_TASK_OPS_ENABLED) { -- if (SCX_HAS_OP(disable)) -- SCX_CALL_OP(SCX_KF_REST, disable, p); -- p->scx->flags &= ~SCX_TASK_OPS_ENABLED; -- } --} -- --static void set_task_scx_weight(struct task_struct *p) --{ -- u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -- -- p->scx->weight = sched_weight_to_cgroup(weight); --} -- --/** -- * refresh_scx_weight - Refresh a task's ext weight -- * @p: task to refresh ext weight for -- * -- * @p->scx->weight carries the task's static priority in cgroup weight scale to -- * enable easy access from the BPF scheduler. To keep it synchronized with the -- * current task priority, this function should be called when a new task is -- * created, priority is changed for a task on sched_ext, and a task is switched -- * to sched_ext from other classes. -- */ --static void refresh_scx_weight(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- set_task_scx_weight(p); -- if (SCX_HAS_OP(set_weight)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight); --} -- --void scx_pre_fork(struct task_struct *p) --{ -- p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL); -- if (!p->scx) { -- goto lock; -- } -- -- p->scx->dsq = NULL; -- INIT_LIST_HEAD(&p->scx->dsq_node.fifo); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- INIT_LIST_HEAD(&p->scx->watchdog_node); -- p->scx->flags = 0; -- p->scx->weight = 0; -- p->scx->sticky_cpu = -1; -- p->scx->holding_cpu = -1; -- p->scx->kf_mask = 0; -- atomic64_set(&p->scx->ops_state, 0); -- p->scx->runnable_at = INITIAL_JIFFIES; -- p->scx->slice = SCX_SLICE_DFL; -- p->scx->task = p; -- p->sched_prop = 0; -- -- /* -- * BPF scheduler enable/disable paths want to be able to iterate and -- * update all tasks which can become complex when racing forks. As -- * enable/disable are very cold paths, let's use a percpu_rwsem to -- * exclude forks. -- */ --lock: -- percpu_down_read(&scx_fork_rwsem); --} -- --int scx_fork(struct task_struct *p) --{ -- percpu_rwsem_assert_held(&scx_fork_rwsem); -- -- if (scx_enabled()) -- return scx_ops_prepare_task(p, task_group(p)); -- else -- return 0; --} -- --void scx_post_fork(struct task_struct *p) --{ -- if (scx_enabled()) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- /* -- * Set the weight manually before calling ops.enable() so that -- * the scheduler doesn't see a stale value if they inspect the -- * task struct. We'll invoke ops.set_weight() afterwards, as it -- * would be odd to receive a callback on the task before we -- * tell the scheduler that it's been fully enabled. -- */ -- set_task_scx_weight(p); -- scx_ops_enable_task(p); -- refresh_scx_weight(p); -- task_rq_unlock(rq, p, &rf); -- } -- -- spin_lock_irq(&scx_tasks_lock); -- list_add_tail(&p->scx->tasks_node, &scx_tasks); -- spin_unlock_irq(&scx_tasks_lock); -- -- percpu_up_read(&scx_fork_rwsem); --} -- --void scx_cancel_fork(struct task_struct *p) --{ -- if (scx_enabled()) -- scx_ops_disable_task(p); -- percpu_up_read(&scx_fork_rwsem); --} -- --void sched_ext_free(struct task_struct *p) --{ -- unsigned long flags; -- -- spin_lock_irqsave(&scx_tasks_lock, flags); -- list_del_init(&p->scx->tasks_node); -- spin_unlock_irqrestore(&scx_tasks_lock, flags); -- -- /* -- * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED -> -- * ENABLED transitions can't race us. Disable ops for @p. -- */ -- if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- scx_ops_disable_task(p); -- task_rq_unlock(rq, p, &rf); -- } --} -- --static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) --{ --} -- --static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} --static void switched_to_scx(struct rq *rq, struct task_struct *p) {} -- --int scx_check_setscheduler(struct task_struct *p, int policy) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- /* if disallow, reject transitioning into SCX */ -- if (scx_enabled() && READ_ONCE(p->scx->disallow) && -- p->policy != policy && policy == SCHED_EXT) -- return -EACCES; -- -- return 0; --} -- --#ifdef CONFIG_NO_HZ_FULL --bool scx_can_stop_tick(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (scx_ops_disabling()) -- return false; -- -- if (p->sched_class != &ext_sched_class) -- return true; -- -- /* -- * @rq can dispatch from different DSQs, so we can't tell whether it -- * needs the tick or not by looking at nr_running. Allow stopping ticks -- * iff the BPF scheduler indicated so. See set_next_task_scx(). -- */ -- return rq->scx->flags & SCX_RQ_CAN_STOP_TICK; --} --#endif -- --static inline void scx_cgroup_lock(void) {} --static inline void scx_cgroup_unlock(void) {} -- --/* -- * Omitted operations: -- * -- * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -- * task isn't tied to the CPU at that point. Preemption is implemented by -- * resetting the victim task's slice to 0 and triggering reschedule on the -- * target CPU. -- * -- * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -- * -- * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -- * their current sched_class. Call them directly from sched core instead. -- * -- * - task_woken, switched_from: Unnecessary. -- */ --DEFINE_SCHED_CLASS(ext) = { -- .enqueue_task = enqueue_task_scx, -- .dequeue_task = dequeue_task_scx, -- .yield_task = yield_task_scx, -- .yield_to_task = yield_to_task_scx, -- -- .check_preempt_curr = check_preempt_curr_scx, -- -- .pick_next_task = pick_next_task_scx, -- -- .put_prev_task = put_prev_task_scx, -- .set_next_task = set_next_task_scx, -- --#ifdef CONFIG_SMP -- .balance = balance_scx, -- .select_task_rq = select_task_rq_scx, -- .set_cpus_allowed = set_cpus_allowed_scx, --#endif -- --#ifdef CONFIG_SCHED_CORE -- .pick_task = pick_task_scx, --#endif -- -- .task_tick = task_tick_scx, -- -- .switched_to = switched_to_scx, -- .prio_changed = prio_changed_scx, -- -- .update_curr = update_curr_scx, -- --#ifdef CONFIG_UCLAMP_TASK -- .uclamp_enabled = 0, --#endif --}; -- --static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) --{ -- memset(dsq, 0, sizeof(*dsq)); -- -- raw_spin_lock_init(&dsq->lock); -- INIT_LIST_HEAD(&dsq->fifo); -- dsq->id = dsq_id; --} -- --static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id & SCX_DSQ_FLAG_BUILTIN) -- return ERR_PTR(-EINVAL); -- -- dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node); -- if (!dsq) -- return ERR_PTR(-ENOMEM); -- -- init_dsq(dsq, dsq_id); -- -- return dsq; --} -- --static void free_dsq_irq_workfn(struct irq_work *irq_work) --{ -- struct llist_node *to_free = llist_del_all(&dsqs_to_free); -- struct scx_dispatch_q *dsq, *tmp_dsq; -- -- llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) -- kfree_rcu(dsq, rcu); --} -- --static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); -- --static void destroy_dsq(u64 dsq_id) --{ --} -- --static void scx_cgroup_exit(void) {} --static int scx_cgroup_init(void) { return 0; } --static void scx_cgroup_config_knobs(void) {} -- --/* -- * Used by sched_fork() and __setscheduler_prio() to pick the matching -- * sched_class. dl/rt are already handled. -- */ --bool task_on_scx(struct task_struct *p) --{ -- if (!scx_enabled() || scx_ops_disabling()) -- return false; -- if (READ_ONCE(scx_switching_all)) -- return true; -- return p->policy == SCHED_EXT; --} -- --static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags) --{ -- if (enq_flags & SCX_ENQ_LAST) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); --} -- --static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {} -- --static void scx_ops_disable_workfn(struct kthread_work *work) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- struct scx_task_iter sti; -- struct task_struct *p; -- const char *reason; -- int i, cpu, type; -- -- type = atomic_read(&scx_exit_type); -- while (true) { -- /* -- * NONE indicates that a new scx_ops has been registered since -- * disable was scheduled - don't kill the new ops. DONE -- * indicates that the ops has already been disabled. -- */ -- if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE) -- return; -- if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE)) -- break; -- } -- -- cancel_delayed_work_sync(&scx_watchdog_work); -- -- switch (type) { -- case SCX_EXIT_UNREG: -- reason = "BPF scheduler unregistered"; -- break; -- case SCX_EXIT_SYSRQ: -- reason = "disabled by sysrq-S"; -- break; -- case SCX_EXIT_ERROR: -- reason = "runtime error"; -- break; -- case SCX_EXIT_ERROR_BPF: -- reason = "scx_bpf_error"; -- break; -- case SCX_EXIT_ERROR_STALL: -- reason = "runnable task stall"; -- break; -- default: -- reason = ""; -- } -- -- ei->type = type; -- strlcpy(ei->reason, reason, sizeof(ei->reason)); -- -- switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { -- case SCX_OPS_DISABLED: -- pr_warn("sched_ext: ops error detected without ops (%s)\n", -- scx_exit_info.msg); -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- return; -- case SCX_OPS_PREPPING: -- goto forward_progress_guaranteed; -- case SCX_OPS_DISABLING: -- /* shouldn't happen but handle it like ENABLING if it does */ -- WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); -- fallthrough; -- case SCX_OPS_ENABLING: -- case SCX_OPS_ENABLED: -- break; -- } -- -- /* -- * DISABLING is set and ops was either ENABLING or ENABLED indicating -- * that the ops and static branches are set. -- * -- * We must guarantee that all runnable tasks make forward progress -- * without trusting the BPF scheduler. We can't grab any mutexes or -- * rwsems as they might be held by tasks that the BPF scheduler is -- * forgetting to run, which unfortunately also excludes toggling the -- * static branches. -- * -- * Let's work around by overriding a couple ops and modifying behaviors -- * based on the DISABLING state and then cycling the tasks through -- * dequeue/enqueue to force global FIFO scheduling. -- * -- * a. ops.enqueue() and .dispatch() are overridden for simple global -- * FIFO scheduling. -- * -- * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value -- * can't be trusted. Whenever a tick triggers, the running task is -- * rotated to the tail of the queue with core_sched_at touched. -- * -- * c. pick_next_task() suppresses zero slice warning. -- * -- * d. scx_prio_less() reverts to the default core_sched_at order. -- */ -- scx_ops.enqueue = scx_ops_fallback_enqueue; -- scx_ops.dispatch = scx_ops_fallback_dispatch; -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- SCHED_CHANGE_BLOCK(task_rq(p), p, -- DEQUEUE_SAVE | DEQUEUE_MOVE) { -- /* cycling deq/enq is enough, see above */ -- } -- } -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* kick all CPUs to restore ticks */ -- for_each_possible_cpu(cpu) -- resched_cpu(cpu); -- --forward_progress_guaranteed: -- /* -- * Here, every runnable task is guaranteed to make forward progress and -- * we can safely use blocking synchronization constructs. Actually -- * disable ops. -- */ -- mutex_lock(&scx_ops_enable_mutex); -- -- static_branch_disable(&__scx_switched_all); -- WRITE_ONCE(scx_switching_all, false); -- -- /* avoid racing against fork and cgroup changes */ -- cpus_read_lock(); -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- bool alive = READ_ONCE(p->__state) != TASK_DEAD; -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL); -- -- __setscheduler_prio(p, p->prio); -- } -- -- if (alive) -- check_class_changed(task_rq(p), p, old_class, p->prio); -- -- scx_ops_disable_task(p); -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* no task is on scx, turn off all the switches and flush in-progress calls */ -- static_branch_disable_cpuslocked(&__scx_ops_enabled); -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- static_branch_disable_cpuslocked(&scx_has_op[i]); -- static_branch_disable_cpuslocked(&scx_ops_enq_last); -- static_branch_disable_cpuslocked(&scx_ops_enq_exiting); -- static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- synchronize_rcu(); -- -- scx_cgroup_exit(); -- -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- cpus_read_unlock(); -- -- if (ei->type >= SCX_EXIT_ERROR) { -- printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name); -- -- if (ei->msg[0] == '\0') -- printk(KERN_ERR "sched_ext: %s\n", ei->reason); -- else -- printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg); -- -- stack_trace_print(ei->bt, ei->bt_len, 2); -- } -- -- if (scx_ops.exit) -- SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); -- -- memset(&scx_ops, 0, sizeof(scx_ops)); -- -- free_percpu(scx_dsp_buf); -- scx_dsp_buf = NULL; -- scx_dsp_max_batch = 0; -- -- mutex_unlock(&scx_ops_enable_mutex); -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- -- scx_cgroup_config_knobs(); --} -- --static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); -- --static void schedule_scx_ops_disable_work(void) --{ -- struct kthread_worker *helper = READ_ONCE(scx_ops_helper); -- -- /* -- * We may be called spuriously before the first bpf_sched_ext_reg(). If -- * scx_ops_helper isn't set up yet, there's nothing to do. -- */ -- if (helper) -- kthread_queue_work(helper, &scx_ops_disable_work); --} -- --static void scx_ops_disable(enum scx_exit_type type) --{ -- int none = SCX_EXIT_NONE; -- -- if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)) -- type = SCX_EXIT_ERROR; -- -- atomic_try_cmpxchg(&scx_exit_type, &none, type); -- -- schedule_scx_ops_disable_work(); --} -- --static void scx_ops_error_irq_workfn(struct irq_work *irq_work) --{ -- schedule_scx_ops_disable_work(); --} -- --static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- int none = SCX_EXIT_NONE; -- va_list args; -- -- if (!atomic_try_cmpxchg(&scx_exit_type, &none, type)) -- return; -- -- ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1); -- -- va_start(args, fmt); -- vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args); -- va_end(args); -- -- irq_work_queue(&scx_ops_error_irq_work); --} -- --static struct kthread_worker *scx_create_rt_helper(const char *name) --{ -- struct kthread_worker *helper; -- -- helper = kthread_create_worker(0, name); -- if (helper) -- sched_set_fifo(helper->task); -- return helper; --} -- --static int scx_ops_enable(struct sched_ext_ops *ops) --{ -- struct scx_task_iter sti; -- struct task_struct *p; -- int i, ret; -- int tcnt = 0; -- unsigned long long start = 0; -- unsigned long long total_start = sched_clock(); -- -- mutex_lock(&scx_ops_enable_mutex); -- -- if (!scx_ops_helper) { -- WRITE_ONCE(scx_ops_helper, -- scx_create_rt_helper("sched_ext_ops_helper")); -- if (!scx_ops_helper) { -- ret = -ENOMEM; -- goto err_unlock; -- } -- } -- -- if (scx_ops_enable_state() != SCX_OPS_DISABLED) { -- ret = -EBUSY; -- goto err_unlock; -- } -- -- //slim_walt_enable(true); -- -- /* -- * Set scx_ops, transition to PREPPING and clear exit info to arm the -- * disable path. Failure triggers full disabling from here on. -- */ -- scx_ops = *ops; -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != -- SCX_OPS_DISABLED); -- -- memset(&scx_exit_info, 0, sizeof(scx_exit_info)); -- atomic_set(&scx_exit_type, SCX_EXIT_NONE); -- scx_warned_zero_slice = false; -- -- atomic64_set(&scx_nr_rejected, 0); -- -- /* -- * Keep CPUs stable during enable so that the BPF scheduler can track -- * online CPUs by watching ->on/offline_cpu() after ->init(). -- */ -- cpus_read_lock(); -- -- scx_switch_all_req = true; -- if (scx_ops.init) { -- ret = SCX_CALL_OP_RET(SCX_KF_INIT, init); -- if (ret) { -- ret = ops_sanitize_err("init", ret); -- goto err_disable; -- } -- -- /* -- * Exit early if ops.init() triggered scx_bpf_error(). Not -- * strictly necessary as we'll fail transitioning into ENABLING -- * later but that'd be after calling ops.prep_enable() on all -- * tasks and with -EBUSY which isn't very intuitive. Let's exit -- * early with success so that the condition is notified through -- * ops.exit() like other scx_bpf_error() invocations. -- */ -- if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE) -- goto err_disable; -- } -- -- WARN_ON_ONCE(scx_dsp_buf); -- scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; -- scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch, -- __alignof__(scx_dsp_buf[0])); -- if (!scx_dsp_buf) { -- ret = -ENOMEM; -- goto err_disable; -- } -- -- scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; -- if (ops->timeout_ms) -- scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); -- -- scx_watchdog_timestamp = jiffies; -- queue_delayed_work(system_unbound_wq, &scx_watchdog_work, -- scx_watchdog_timeout / 2); -- -- /* -- * Lock out forks, cgroup on/offlining and moves before opening the -- * floodgate so that they don't wander into the operations prematurely. -- */ -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- if (((void (**)(void))ops)[i]) -- static_branch_enable_cpuslocked(&scx_has_op[i]); -- -- if (ops->flags & SCX_OPS_ENQ_LAST) -- static_branch_enable_cpuslocked(&scx_ops_enq_last); -- -- if (ops->flags & SCX_OPS_ENQ_EXITING) -- static_branch_enable_cpuslocked(&scx_ops_enq_exiting); -- if (scx_ops.cpu_acquire || scx_ops.cpu_release) -- static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); -- -- if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { -- reset_idle_masks(); -- static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); -- } else { -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- } -- -- /* -- * All cgroups should be initialized before letting in tasks. cgroup -- * on/offlining and task migrations are already locked out. -- */ -- ret = scx_cgroup_init(); -- if (ret) -- goto err_disable_unlock; -- -- static_branch_enable_cpuslocked(&__scx_ops_enabled); -- -- /* -- * Enable ops for every task. Fork is excluded by scx_fork_rwsem -- * preventing new tasks from being added. No need to exclude tasks -- * leaving as sched_ext_free() can handle both prepped and enabled -- * tasks. Prep all tasks first and then enable them with preemption -- * disabled. -- */ -- spin_lock_irq(&scx_tasks_lock); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered(&sti))) { -- get_task_struct(p); -- spin_unlock_irq(&scx_tasks_lock); -- -- ret = scx_ops_prepare_task(p, task_group(p)); -- if (ret) { -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n", -- ret, p->comm, p->pid); -- goto err_disable_unlock; -- } -- -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- } -- scx_task_iter_exit(&sti); -- -- /* -- * All tasks are prepped but are still ops-disabled. Ensure that -- * %current can't be scheduled out and switch everyone. -- * preempt_disable() is necessary because we can't guarantee that -- * %current won't be starved if scheduled out while switching. -- */ -- preempt_disable(); -- -- /* -- * From here on, the disable path must assume that tasks have ops -- * enabled and need to be recovered. -- */ -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { -- preempt_enable(); -- spin_unlock_irq(&scx_tasks_lock); -- ret = -EBUSY; -- goto err_disable_unlock; -- } -- -- /* -- * We're fully committed and can't fail. The PREPPED -> ENABLED -- * transitions here are synchronized against sched_ext_free() through -- * scx_tasks_lock. -- */ -- WRITE_ONCE(scx_switching_all, scx_switch_all_req); -- start = sched_clock(); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- tcnt++; -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- scx_ops_enable_task(p); -- __setscheduler_prio(p, p->prio); -- } -- -- check_class_changed(task_rq(p), p, old_class, p->prio); -- } else { -- scx_ops_disable_task(p); -- } -- } -- printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id()); -- scx_task_iter_exit(&sti); -- -- spin_unlock_irq(&scx_tasks_lock); -- preempt_enable(); -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { -- ret = -EBUSY; -- goto err_disable; -- } -- -- if (scx_switch_all_req) -- static_branch_enable_cpuslocked(&__scx_switched_all); -- -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- -- scx_cgroup_config_knobs(); -- printk("\n total duration = %llu , current cpu = %d\n", sched_clock() - total_start, smp_processor_id()); -- -- return 0; -- --err_unlock: -- mutex_unlock(&scx_ops_enable_mutex); -- return ret; -- --err_disable_unlock: -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); --err_disable: -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- /* must be fully disabled before returning */ -- scx_ops_disable(SCX_EXIT_ERROR); -- kthread_flush_work(&scx_ops_disable_work); -- return ret; --} -- --#ifdef CONFIG_SCHED_DEBUG --static const char *scx_ops_enable_state_str[] = { -- [SCX_OPS_PREPPING] = "prepping", -- [SCX_OPS_ENABLING] = "enabling", -- [SCX_OPS_ENABLED] = "enabled", -- [SCX_OPS_DISABLING] = "disabling", -- [SCX_OPS_DISABLED] = "disabled", --}; -- --static int scx_debug_show(struct seq_file *m, void *v) --{ -- mutex_lock(&scx_ops_enable_mutex); -- seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name); -- seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled()); -- seq_printf(m, "%-30s: %d\n", "switching_all", -- READ_ONCE(scx_switching_all)); -- seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all()); -- seq_printf(m, "%-30s: %s\n", "enable_state", -- scx_ops_enable_state_str[scx_ops_enable_state()]); -- seq_printf(m, "%-30s: %llu\n", "nr_rejected", -- atomic64_read(&scx_nr_rejected)); -- mutex_unlock(&scx_ops_enable_mutex); -- return 0; --} -- --static int scx_debug_open(struct inode *inode, struct file *file) --{ -- return single_open(file, scx_debug_show, NULL); --} -- --const struct file_operations sched_ext_fops = { -- .open = scx_debug_open, -- .read = seq_read, -- .llseek = seq_lseek, -- .release = single_release, --}; --#endif -- --/******************************************************************************** -- * bpf_struct_ops plumbing. -- */ --#include --#include --#include -- --extern struct btf *btf_vmlinux; --static const struct btf_type *task_struct_type; -- --static bool bpf_scx_is_valid_access(int off, int size, -- enum bpf_access_type type, -- const struct bpf_prog *prog, -- struct bpf_insn_access_aux *info) --{ -- if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) -- return false; -- if (type != BPF_READ) -- return false; -- if (off % size != 0) -- return false; -- -- return btf_ctx_access(off, size, type, prog, info); --} -- --static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, -- const struct bpf_reg_state *reg, -- int off, int size) --{ -- return 0; --} -- --static const struct bpf_func_proto * --bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) --{ -- switch (func_id) { -- case BPF_FUNC_task_storage_get: -- return &bpf_task_storage_get_proto; -- case BPF_FUNC_task_storage_delete: -- return &bpf_task_storage_delete_proto; -- default: -- return bpf_base_func_proto(func_id); -- } --} -- --const struct bpf_verifier_ops bpf_scx_verifier_ops = { -- .get_func_proto = bpf_scx_get_func_proto, -- .is_valid_access = bpf_scx_is_valid_access, -- .btf_struct_access = bpf_scx_btf_struct_access, --}; -- --static int bpf_scx_init_member(const struct btf_type *t, -- const struct btf_member *member, -- void *kdata, const void *udata) --{ -- const struct sched_ext_ops *uops = udata; -- struct sched_ext_ops *ops = kdata; -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- int ret; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, dispatch_max_batch): -- if (*(u32 *)(udata + moff) > INT_MAX) -- return -E2BIG; -- ops->dispatch_max_batch = *(u32 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, flags): -- if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) -- return -EINVAL; -- ops->flags = *(u64 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, name): -- ret = bpf_obj_name_cpy(ops->name, uops->name, -- sizeof(ops->name)); -- if (ret < 0) -- return ret; -- if (ret == 0) -- return -EINVAL; -- return 1; -- case offsetof(struct sched_ext_ops, timeout_ms): -- if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT) -- return -E2BIG; -- ops->timeout_ms = *(u32 *)(udata + moff); -- return 1; -- } -- -- return 0; --} -- --static int bpf_scx_check_member(const struct btf_type *t, -- const struct btf_member *member, -- const struct bpf_prog *prog) --{ -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, prep_enable): -- case offsetof(struct sched_ext_ops, init): -- case offsetof(struct sched_ext_ops, exit): -- break; -- default: -- /* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */ -- /* if (prog->aux->sleepable) -- return -EINVAL; */ -- break; -- } -- -- return 0; --} -- --static int bpf_scx_reg(void *kdata) --{ -- return scx_ops_enable(kdata); --} -- --static void bpf_scx_unreg(void *kdata) --{ -- scx_ops_disable(SCX_EXIT_UNREG); -- kthread_flush_work(&scx_ops_disable_work); --} -- --static int bpf_scx_init(struct btf *btf) --{ -- u32 type_id; -- -- type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); -- if (type_id < 0) -- return -EINVAL; -- task_struct_type = btf_type_by_id(btf, type_id); -- -- return 0; --} -- --/* "extern" to avoid sparse warning, only used in this file */ --extern struct bpf_struct_ops bpf_sched_ext_ops; -- --struct bpf_struct_ops bpf_sched_ext_ops = { -- .verifier_ops = &bpf_scx_verifier_ops, -- .reg = bpf_scx_reg, -- .unreg = bpf_scx_unreg, -- .check_member = bpf_scx_check_member, -- .init_member = bpf_scx_init_member, -- .init = bpf_scx_init, -- .name = "sched_ext_ops", --}; -- --static void sysrq_handle_sched_ext_reset(u8 key) --{ -- if (scx_ops_helper) -- scx_ops_disable(SCX_EXIT_SYSRQ); -- else -- pr_info("sched_ext: BPF scheduler not yet used\n"); --} -- --static const struct sysrq_key_op sysrq_sched_ext_reset_op = { -- .handler = sysrq_handle_sched_ext_reset, -- .help_msg = "reset-sched-ext(S)", -- .action_msg = "Disable sched_ext and revert all tasks to CFS", -- .enable_mask = SYSRQ_ENABLE_RTNICE, --}; -- --static void kick_cpus_irq_workfn(struct irq_work *irq_work) --{ -- struct rq *this_rq = this_rq(); -- u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); -- int this_cpu = cpu_of(this_rq); -- int cpu; -- -- for_each_cpu(cpu, this_rq->scx->cpus_to_kick) { -- struct rq *rq = cpu_rq(cpu); -- unsigned long flags; -- -- raw_spin_rq_lock_irqsave(rq, flags); -- -- if (cpu_online(cpu) || cpu == this_cpu) { -- if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) && -- rq->curr->sched_class == &ext_sched_class) -- rq->curr->scx->slice = 0; -- pseqs[cpu] = rq->scx->pnt_seq; -- resched_curr(rq); -- } else { -- cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait); -- } -- -- raw_spin_rq_unlock_irqrestore(rq, flags); -- } -- -- for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait, -- cpumask_of(this_cpu)) { -- /* -- * Pairs with smp_store_release() issued by this CPU in -- * scx_notify_pick_next_task() on the resched path. -- * -- * We busy-wait here to guarantee that no other task can be -- * scheduled on our core before the target CPU has entered the -- * resched path. -- */ -- while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu]) -- cpu_relax(); -- } -- -- cpumask_clear(this_rq->scx->cpus_to_kick); -- cpumask_clear(this_rq->scx->cpus_to_preempt); -- cpumask_clear(this_rq->scx->cpus_to_wait); --} -- --void __init init_sched_ext_class(void) --{ -- int cpu; -- u32 v; -- -- /* -- * The following is to prevent the compiler from optimizing out the enum -- * definitions so that BPF scheduler implementations can use them -- * through the generated vmlinux.h. -- */ -- WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | -- SCX_TG_ONLINE | SCX_KICK_PREEMPT); -- -- init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); --#ifdef CONFIG_SMP -- BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -- BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); --#endif -- scx_kick_cpus_pnt_seqs = -- __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * -- num_possible_cpus(), -- __alignof__(scx_kick_cpus_pnt_seqs[0])); -- BUG_ON(!scx_kick_cpus_pnt_seqs); -- -- for_each_possible_cpu(cpu) { -- struct rq *rq = cpu_rq(cpu); -- -- rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL); -- if (rq->scx) -- rq->scx->rq = rq; -- else -- pr_err("fatal error : alloc rq->scx failed!!!\n"); -- -- init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL); -- INIT_LIST_HEAD(&rq->scx->watchdog_list); -- -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL)); -- init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn); -- } -- -- register_sysrq_key('S', &sysrq_sched_ext_reset_op); -- INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); -- scx_cgroup_config_knobs(); --} -- -- --/******************************************************************************** -- * Helpers that can be called from the BPF scheduler. -- */ --#include -- --/* Disables missing prototype warnings for kfuncs */ --__diag_push(); --__diag_ignore_all("-Wmissing-prototypes", -- "Global functions as their definitions will be in vmlinux BTF"); -- --/** -- * scx_bpf_switch_all - Switch all tasks into SCX -- * @into_scx: switch direction -- * -- * If @into_scx is %true, all existing and future non-dl/rt tasks are switched -- * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on -- * SCX. The actual switching is asynchronous. Can be called from ops.init(). -- */ --void scx_bpf_switch_all(void) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return; -- -- scx_switch_all_req = true; --} -- --BTF_SET8_START(scx_kfunc_ids_init) --BTF_ID_FLAGS(func, scx_bpf_switch_all) --BTF_SET8_END(scx_kfunc_ids_init) -- --static const struct btf_kfunc_id_set scx_kfunc_set_init = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_init, --}; -- --/** -- * scx_bpf_create_dsq - Create a custom DSQ -- * @dsq_id: DSQ to create -- * @node: NUMA node to allocate from -- * -- * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(), -- * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move(). -- */ --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return -EINVAL; -- -- if (unlikely(node >= (int)nr_node_ids || -- (node < 0 && node != NUMA_NO_NODE))) -- return -EINVAL; -- return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); --} -- --BTF_SET8_START(scx_kfunc_ids_sleepable) --BTF_ID_FLAGS(func, scx_bpf_create_dsq) --BTF_SET8_END(scx_kfunc_ids_sleepable) -- --static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_sleepable, --}; -- --static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) --{ -- if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) -- return false; -- -- lockdep_assert_irqs_disabled(); -- -- if (unlikely(!p)) { -- scx_ops_error("called with NULL task"); -- return false; -- } -- -- if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { -- scx_ops_error("invalid enq_flags 0x%llx", enq_flags); -- return false; -- } -- -- return true; --} -- --static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) --{ -- struct task_struct *ddsp_task; -- int idx; -- -- ddsp_task = __this_cpu_read(direct_dispatch_task); -- if (ddsp_task) { -- direct_dispatch(ddsp_task, p, dsq_id, enq_flags); -- return; -- } -- -- idx = __this_cpu_read(scx_dsp_ctx.buf_cursor); -- if (unlikely(idx >= scx_dsp_max_batch)) { -- scx_ops_error("dispatch buffer overflow"); -- return; -- } -- -- this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){ -- .task = p, -- .qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK, -- .dsq_id = dsq_id, -- .enq_flags = enq_flags, -- }; -- __this_cpu_inc(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe -- * to call this function spuriously. Can be called from ops.enqueue() and -- * ops.dispatch(). -- * -- * When called from ops.enqueue(), it's for direct dispatch and @p must match -- * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the -- * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be -- * on the target CPU in the first place. -- * -- * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id -- * and this function can be called upto ops.dispatch_max_batch times to dispatch -- * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the -- * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. -- * -- * This function doesn't have any locking restrictions and may be called under -- * BPF locks (in the future when BPF introduces more flexible locking). -- * -- * @p is allowed to run for @slice. The scheduling path is triggered on slice -- * exhaustion. If zero, the current residual slice is maintained. If -- * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with -- * scx_bpf_kick_cpu() to trigger scheduling. -- */ --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- scx_dispatch_commit(p, dsq_id, enq_flags); --} -- --/** -- * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. -- * Tasks queued into the priority queue are ordered by @vtime and always -- * consumed after the tasks in the FIFO queue. All other aspects are identical -- * to scx_bpf_dispatch(). -- * -- * @vtime ordering is according to time_before64() which considers wrapping. A -- * numerically larger vtime may indicate an earlier position in the ordering and -- * vice-versa. -- */ --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 vtime, u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- p->scx->dsq_vtime = vtime; -- -- scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); --} -- --BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime) --BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_enqueue_dispatch, --}; -- --/** -- * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots -- * -- * Can only be called from ops.dispatch(). -- */ --u32 scx_bpf_dispatch_nr_slots(void) --{ -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return 0; -- -- return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ -- * @dsq_id: DSQ to consume -- * -- * Consume a task from the non-local DSQ identified by @dsq_id and transfer it -- * to the current CPU's local DSQ for execution. Can only be called from -- * ops.dispatch(). -- * -- * This function flushes the in-flight dispatches from scx_bpf_dispatch() before -- * trying to consume the specified DSQ. It may also grab rq locks and thus can't -- * be called under any BPF locks. -- * -- * Returns %true if a task has been consumed, %false if there isn't any task to -- * consume. -- */ --bool scx_bpf_consume(u64 dsq_id) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- struct scx_dispatch_q *dsq; -- -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return false; -- -- flush_dispatch_buf(dspc->rq, dspc->rf); -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); -- return false; -- } -- -- if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) { -- /* -- * A successfully consumed task can be dequeued before it starts -- * running while the CPU is trying to migrate other dispatched -- * tasks. Bump nr_tasks to tell balance_scx() to retry on empty -- * local DSQ. -- */ -- dspc->nr_tasks++; -- return true; -- } else { -- return false; -- } --} -- --BTF_SET8_START(scx_kfunc_ids_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) --BTF_ID_FLAGS(func, scx_bpf_consume) --BTF_SET8_END(scx_kfunc_ids_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_dispatch, --}; -- --/** -- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ -- * -- * Iterate over all of the tasks currently enqueued on the local DSQ of the -- * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of -- * processed tasks. Can only be called from ops.cpu_release(). -- */ --u32 scx_bpf_reenqueue_local(void) --{ -- u32 nr_enqueued, i; -- struct rq *rq; -- struct scx_rq *scx_rq; -- -- if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) -- return 0; -- -- rq = cpu_rq(smp_processor_id()); -- lockdep_assert_rq_held(rq); -- scx_rq = rq->scx; -- -- /* -- * Get the number of tasks on the local DSQ before iterating over it to -- * pull off tasks. The enqueue callback below can signal that it wants -- * the task to stay on the local DSQ, and we want to prevent the BPF -- * scheduler from causing us to loop indefinitely. -- */ -- nr_enqueued = scx_rq->local_dsq.nr; -- for (i = 0; i < nr_enqueued; i++) { -- struct task_struct *p; -- -- p = first_local_task(rq); -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- WARN_ON_ONCE(p->scx->holding_cpu != -1); -- dispatch_dequeue(scx_rq, p); -- do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); -- } -- -- return nr_enqueued; --} -- --BTF_SET8_START(scx_kfunc_ids_cpu_release) --BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) --BTF_SET8_END(scx_kfunc_ids_cpu_release) -- --static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_cpu_release, --}; -- --/** -- * scx_bpf_kick_cpu - Trigger reschedule on a CPU -- * @cpu: cpu to kick -- * @flags: SCX_KICK_* flags -- * -- * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or -- * trigger rescheduling on a busy CPU. This can be called from any online -- * scx_ops operation and the actual kicking is performed asynchronously through -- * an irq work. -- */ --void scx_bpf_kick_cpu(s32 cpu, u64 flags) --{ -- struct rq *rq; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d", cpu); -- return; -- } -- -- preempt_disable(); -- rq = this_rq(); -- -- /* -- * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting -- * rq locks. We can probably be smarter and avoid bouncing if called -- * from ops which don't hold a rq lock. -- */ -- cpumask_set_cpu(cpu, rq->scx->cpus_to_kick); -- if (flags & SCX_KICK_PREEMPT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt); -- if (flags & SCX_KICK_WAIT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_wait); -- -- irq_work_queue(&rq->scx->kick_cpus_irq_work); -- preempt_enable(); --} -- --/** -- * scx_bpf_dsq_nr_queued - Return the number of queued tasks -- * @dsq_id: id of the DSQ -- * -- * Return the number of tasks in the DSQ matching @dsq_id. If not found, -- * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops -- * operations. -- */ --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) --{ -- struct scx_dispatch_q *dsq; -- -- lockdep_assert(rcu_read_lock_any_held()); -- -- if (dsq_id == SCX_DSQ_LOCAL) { -- return this_rq()->scx->local_dsq.nr; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (ops_cpu_valid(cpu)) -- return cpu_rq(cpu)->scx->local_dsq.nr; -- } else { -- dsq = find_non_local_dsq(dsq_id); -- if (dsq) -- return dsq->nr; -- } -- return -ENOENT; --} -- --/** -- * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state -- * @cpu: cpu to test and clear idle for -- * -- * Returns %true if @cpu was idle and its idle state was successfully cleared. -- * %false otherwise. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return false; -- } -- -- if (ops_cpu_valid(cpu)) -- return test_and_clear_cpu_idle(cpu); -- else -- return false; --} -- --/** -- * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu -- * @cpus_allowed: Allowed cpumask -- * -- * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked -- * idle cpu number on success. -%EBUSY if no matching cpu was found. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return -EBUSY; -- } -- -- return scx_pick_idle_cpu(cpus_allowed); --} -- --/** -- * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking -- * per-CPU cpumask. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_cpumask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.cpu; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, -- * per-physical-core cpumask. Can be used to determine if an entire physical -- * core is free. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_smtmask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.smt; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to -- * either the percpu, or SMT idle-tracking cpumask. -- */ --void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) --{ -- /* -- * Empty function body because we aren't actually acquiring or -- * releasing a reference to a global idle cpumask, which is read-only -- * in the caller and is never released. The acquire / release semantics -- * here are just used to make the cpumask is a trusted pointer in the -- * caller. -- */ --} -- --struct scx_bpf_error_bstr_bufs { -- u64 data[MAX_BPRINTF_VARARGS]; -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs); -- --/** -- * scx_bpf_error_bstr - Indicate fatal error -- * @fmt: error message format string -- * @data: format string parameters packaged using ___bpf_fill() macro -- * @data__sz: @data len, must end in '__sz' for the verifier -- * -- * Indicate that the BPF scheduler encountered a fatal error and initiate ops -- * disabling. -- */ --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) --{ -- struct scx_bpf_error_bstr_bufs *bufs; -- unsigned long flags; -- struct bpf_bprintf_data args = {}; -- int ret; -- -- local_irq_save(flags); -- bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs); -- -- if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || -- (data__sz && !data)) { -- scx_ops_error("invalid data=%p and data__sz=%u", -- (void *)data, data__sz); -- goto out_restore; -- } -- -- ret = copy_from_kernel_nofault(bufs->data, data, data__sz); -- if (ret) { -- scx_ops_error("failed to read data fields (%d)", ret); -- goto out_restore; -- } -- -- ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args); -- if (ret < 0) { -- scx_ops_error("failed to format prepration (%d)", ret); -- goto out_restore; -- } -- -- ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt, -- args.bin_args); -- bpf_bprintf_cleanup(&args); -- if (ret < 0) { -- scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format", -- fmt, data, data__sz); -- goto out_restore; -- } -- -- scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg); --out_restore: -- local_irq_restore(flags); --} -- --/** -- * scx_bpf_destroy_dsq - Destroy a custom DSQ -- * @dsq_id: DSQ to destroy -- * -- * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with -- * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is -- * empty and no further tasks are dispatched to it. Ignored if called on a DSQ -- * which doesn't exist. Can be called from any online scx_ops operations. -- */ --void scx_bpf_destroy_dsq(u64 dsq_id) --{ -- destroy_dsq(dsq_id); --} -- --/** -- * scx_bpf_task_running - Is task currently running? -- * @p: task of interest -- */ --bool scx_bpf_task_running(const struct task_struct *p) --{ -- return task_rq(p)->curr == p; --} -- --/** -- * scx_bpf_task_cpu - CPU a task is currently associated with -- * @p: task of interest -- */ --s32 scx_bpf_task_cpu(const struct task_struct *p) --{ -- return task_cpu(p); --} -- --/** -- * scx_bpf_task_cgroup - Return the sched cgroup of a task -- * @p: task of interest -- * -- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with -- * from the scheduler's POV. SCX operations should use this function to -- * determine @p's current cgroup as, unlike following @p->cgroups, -- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all -- * rq-locked operations. Can be called on the parameter tasks of rq-locked -- * operations. The restriction guarantees that @p's rq is locked by the caller. -- */ --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) --{ -- struct task_group *tg = p->sched_task_group; -- struct cgroup *cgrp = &cgrp_dfl_root.cgrp; -- -- if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) -- goto out; -- -- /* -- * A task_group may either be a cgroup or an autogroup. In the latter -- * case, @tg->css.cgroup is %NULL. A task_group can't become the other -- * kind once created. -- */ -- if (tg && tg->css.cgroup) -- cgrp = tg->css.cgroup; -- else -- cgrp = &cgrp_dfl_root.cgrp; --out: -- cgroup_get(cgrp); -- return cgrp; --} -- --BTF_SET8_START(scx_kfunc_ids_any) --BTF_ID_FLAGS(func, scx_bpf_kick_cpu) --BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) --BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) --BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu) --BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) --BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) --BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) --BTF_ID_FLAGS(func, scx_bpf_task_running) --BTF_ID_FLAGS(func, scx_bpf_task_cpu) --BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE) --BTF_SET8_END(scx_kfunc_ids_any) -- --static const struct btf_kfunc_id_set scx_kfunc_set_any = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_any, --}; -- --__diag_pop(); -- --/* -- * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set() -- * needs most of the system to be up. -- */ --static int __init register_ext_kfuncs(void) --{ -- int ret; -- -- /* -- * Some kfuncs are context-sensitive and can only be called from -- * specific SCX ops. They are grouped into BTF sets accordingly. -- * Unfortunately, BPF currently doesn't have a way of enforcing such -- * restrictions. Eventually, the verifier should be able to enforce -- * them. For now, register them the same and make each kfunc explicitly -- * check using scx_kf_allowed(). -- */ -- if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_init)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_sleepable)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_enqueue_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_cpu_release)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_any))) { -- pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret); -- return ret; -- } -- -- return 0; --} --__initcall(register_ext_kfuncs); -diff --git b/kernel/sched/ext.h a/kernel/sched/ext.h -deleted file mode 100755 -index fba8ed845f99..000000000000 ---- b/kernel/sched/ext.h -+++ /dev/null -@@ -1,257 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ -- -- --/* -- * Tag marking a kernel function as a kfunc. This is meant to minimize the -- * amount of copy-paste that kfunc authors have to include for correctness so -- * as to avoid issues such as the compiler inlining or eliding either a static -- * kfunc, or a global kfunc in an LTO build. -- */ --#define __bpf_kfunc __used noinline -- --enum scx_wake_flags { -- /* expose select WF_* flags as enums */ -- SCX_WAKE_EXEC = WF_EXEC, -- SCX_WAKE_FORK = WF_FORK, -- SCX_WAKE_TTWU = WF_TTWU, -- SCX_WAKE_SYNC = WF_SYNC, --}; -- --enum scx_enq_flags { -- /* expose select ENQUEUE_* flags as enums */ -- SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, -- SCX_ENQ_HEAD = ENQUEUE_HEAD, -- -- /* high 32bits are SCX specific */ -- -- /* -- * Set the following to trigger preemption when calling -- * scx_bpf_dispatch() with a local dsq as the target. The slice of the -- * current task is cleared to zero and the CPU is kicked into the -- * scheduling path. Implies %SCX_ENQ_HEAD. -- */ -- SCX_ENQ_PREEMPT = 1LLU << 32, -- -- /* -- * The task being enqueued was previously enqueued on the current CPU's -- * %SCX_DSQ_LOCAL, but was removed from it in a call to the -- * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was -- * invoked in a ->cpu_release() callback, and the task is again -- * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the -- * task will not be scheduled on the CPU until at least the next invocation -- * of the ->cpu_acquire() callback. -- */ -- SCX_ENQ_REENQ = 1LLU << 40, -- -- /* -- * The task being enqueued is the only task available for the cpu. By -- * default, ext core keeps executing such tasks but when -- * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with -- * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set. -- * -- * If the BPF scheduler wants to continue executing the task, -- * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately. -- * If the task gets queued on a different dsq or the BPF side, the BPF -- * scheduler is responsible for triggering a follow-up scheduling event. -- * Otherwise, Execution may stall. -- */ -- SCX_ENQ_LAST = 1LLU << 41, -- -- /* -- * A hint indicating that it's advisable to enqueue the task on the -- * local dsq of the currently selected CPU. Currently used by -- * select_cpu_dfl() and together with %SCX_ENQ_LAST. -- */ -- SCX_ENQ_LOCAL = 1LLU << 42, -- -- /* high 8 bits are internal */ -- __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, -- -- SCX_ENQ_CLEAR_OPSS = 1LLU << 56, -- SCX_ENQ_DSQ_PRIQ = 1LLU << 57, --}; -- --enum scx_deq_flags { -- /* expose select DEQUEUE_* flags as enums */ -- SCX_DEQ_SLEEP = DEQUEUE_SLEEP, -- -- /* high 32bits are SCX specific */ -- -- /* -- * The generic core-sched layer decided to execute the task even though -- * it hasn't been dispatched yet. Dequeue from the BPF side. -- */ -- SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, --}; -- --enum scx_tg_flags { -- SCX_TG_ONLINE = 1U << 0, -- SCX_TG_INITED = 1U << 1, --}; -- --enum scx_kick_flags { -- SCX_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -- SCX_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ --}; -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --extern const struct sched_class ext_sched_class; --extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops; --extern const struct file_operations sched_ext_fops; --extern unsigned long scx_watchdog_timeout; --extern unsigned long scx_watchdog_timestamp; -- --DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); --DECLARE_STATIC_KEY_FALSE(__scx_switched_all); --#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) --#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) -- --DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); -- --bool task_on_scx(struct task_struct *p); --void scx_pre_fork(struct task_struct *p); --int scx_fork(struct task_struct *p); --void scx_post_fork(struct task_struct *p); --void scx_cancel_fork(struct task_struct *p); --int scx_check_setscheduler(struct task_struct *p, int policy); --bool scx_can_stop_tick(struct rq *rq); --void init_sched_ext_class(void); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...); --#define scx_ops_error(fmt, args...) \ -- scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args) -- --void __scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active); -- --static inline void scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active) --{ -- if (!scx_enabled()) -- return; --#ifdef CONFIG_SMP -- /* -- * Pairs with the smp_load_acquire() issued by a CPU in -- * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -- * resched. -- */ -- smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1); --#endif -- if (!static_branch_unlikely(&scx_ops_cpu_preempt)) -- return; -- __scx_notify_pick_next_task(rq, p, active); --} -- --//extern void scx_scheduler_tick(void); --static inline void scx_notify_sched_tick(void) --{ -- unsigned long last_check; -- -- if (!scx_enabled()) -- return; -- //scx_scheduler_tick(); -- -- last_check = scx_watchdog_timestamp; -- if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "watchdog failed to check in for %u.%03us", -- dur_ms / 1000, dur_ms % 1000); -- } --} -- --static inline const struct sched_class *next_active_class(const struct sched_class *class) --{ -- class++; -- if (scx_switched_all() && class == &fair_sched_class) -- class++; -- if (!scx_enabled() && class == &ext_sched_class) -- class++; -- return class; --} -- --#define for_active_class_range(class, _from, _to) \ -- for (class = (_from); class != (_to); class = next_active_class(class)) -- --#define for_each_active_class(class) \ -- for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -- --/* -- * SCX requires a balance() call before every pick_next_task() call including -- * when waking up from idle. -- */ --#define for_balance_class_range(class, prev_class, end_class) \ -- for_active_class_range(class, (prev_class) > &ext_sched_class ? \ -- &ext_sched_class : (prev_class), (end_class)) -- --#ifdef CONFIG_SCHED_CORE --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi); --#endif -- --#else /* CONFIG_SCHED_CLASS_EXT */ -- --#define scx_enabled() false --#define scx_switched_all() false -- --static inline void scx_pre_fork(struct task_struct *p) {} --static inline int scx_fork(struct task_struct *p) { return 0; } --static inline void scx_post_fork(struct task_struct *p) {} --static inline void scx_cancel_fork(struct task_struct *p) {} --static inline int scx_check_setscheduler(struct task_struct *p, -- int policy) { return 0; } --static inline bool scx_can_stop_tick(struct rq *rq) { return true; } --static inline void init_sched_ext_class(void) {} --static inline void scx_notify_pick_next_task(struct rq *rq, -- const struct task_struct *p, -- const struct sched_class *active) {} --static inline void scx_notify_sched_tick(void) {} -- --#define for_each_active_class for_each_class --#define for_balance_class_range for_class_range -- --#endif /* CONFIG_SCHED_CLASS_EXT */ -- --#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) --void __scx_update_idle(struct rq *rq, bool idle); -- --static inline void scx_update_idle(struct rq *rq, bool idle) --{ -- if (scx_enabled()) -- __scx_update_idle(rq, idle); --} --#else --static inline void scx_update_idle(struct rq *rq, bool idle) {} --#endif -- --#ifdef CONFIG_CGROUP_SCHED --#ifdef CONFIG_EXT_GROUP_SCHED --int scx_tg_online(struct task_group *tg); --void scx_tg_offline(struct task_group *tg); --int scx_cgroup_can_attach(struct cgroup_taskset *tset); --void scx_move_task(struct task_struct *p); --void scx_cgroup_finish_attach(void); --void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); --void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); --#else /* CONFIG_EXT_GROUP_SCHED */ --static inline int scx_tg_online(struct task_group *tg) { return 0; } --static inline void scx_tg_offline(struct task_group *tg) {} --static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } --static inline void scx_move_task(struct task_struct *p) {} --static inline void scx_cgroup_finish_attach(void) {} --static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} --static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} --#endif /* CONFIG_EXT_GROUP_SCHED */ --#endif /* CONFIG_CGROUP_SCHED */ -diff --git b/kernel/sched/hmbird.h a/kernel/sched/hmbird.h -new file mode 100755 -index 000000000000..fa76c7f09977 ---- /dev/null -+++ a/kernel/sched/hmbird.h -@@ -0,0 +1,342 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#ifndef _HMBIRD_H_ -+#define _HMBIRD_H_ -+ -+/* -+ * Tag marking a kernel function as a kfunc. This is meant to minimize the -+ * amount of copy-paste that kfunc authors have to include for correctness so -+ * as to avoid issues such as the compiler inlining or eliding either a static -+ * kfunc, or a global kfunc in an LTO build. -+ */ -+ -+#define DEBUG_INTERNAL (1 << 0) -+#define DEBUG_INFO_TRACE (1 << 1) -+#define DEBUG_INFO_SYSTRACE (1 << 2) -+ -+#define NUMS_CGROUP_KINDS (512) -+#define SLIM_FOR_SGAME (1 << 0) -+#define SLIM_FOR_GENSHIN (1 << 1) -+ -+#ifdef MAX_TASK_NR -+#define MAX_KEY_THREAD_RECORD ((MAX_TASK_NR + 1) >> 1) -+#else -+#define MAX_KEY_THREAD_RECORD (1 << 3) -+#endif /* MAX_TASK_NR */ -+ -+#define TOP_TASK_SHIFT (8) -+#define TOP_TASK_MAX (1 << TOP_TASK_SHIFT) -+#ifndef TOP_TASK_BITS_MASK -+#define TOP_TASK_BITS_MASK (TOP_TASK_MAX - 1) -+#endif /* TOP_TASK_BITS_MASK */ -+ -+extern struct md_info_t *md_info; -+ -+#define debug_enabled() \ -+ (unlikely(hmbirdcore_debug)) -+ -+#define hmbird_debug(fmt, ...) \ -+ pr_info(":"fmt, ##__VA_ARGS__) -+ -+#define hmbird_err(id, fmt, ...) \ -+do { \ -+ pr_err(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_deferred_err(id, fmt, ...) \ -+do { \ -+ printk_deferred(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \ -+do { \ -+ if (unlikely(cond)) { \ -+ hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+ } \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_info_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_info_trace(fmt, ...) -+#endif -+ -+#define hmbird_info_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+ } \ -+} while (0) -+ -+#define hmbird_output_systrace(fmt, ...) \ -+do { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_internal_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_internal_trace(fmt, ...) -+#endif -+ -+#define hmbird_internal_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) { \ -+ hmbird_output_systrace(fmt, ##__VA_ARGS__); \ -+ } \ -+} while (0) -+ -+enum hmbird_wake_flags { -+ /* expose select WF_* flags as enums */ -+ HMBIRD_WAKE_EXEC = WF_EXEC, -+ HMBIRD_WAKE_FORK = WF_FORK, -+ HMBIRD_WAKE_TTWU = WF_TTWU, -+ HMBIRD_WAKE_SYNC = WF_SYNC, -+}; -+ -+enum hmbird_enq_flags { -+ /* expose select ENQUEUE_* flags as enums */ -+ HMBIRD_ENQ_WAKEUP = ENQUEUE_WAKEUP, -+ HMBIRD_ENQ_HEAD = ENQUEUE_HEAD, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * Set the following to trigger preemption when calling -+ * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the -+ * current task is cleared to zero and the CPU is kicked into the -+ * scheduling path. Implies %HMBIRD_ENQ_HEAD. -+ */ -+ HMBIRD_ENQ_PREEMPT = 1LLU << 32, -+ HMBIRD_ENQ_REENQ = 1LLU << 40, -+ HMBIRD_ENQ_LAST = 1LLU << 41, -+ -+ /* -+ * A hint indicating that it's advisable to enqueue the task on the -+ * local dsq of the currently selected CPU. Currently used by -+ * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST. -+ */ -+ HMBIRD_ENQ_LOCAL = 1LLU << 42, -+ -+ /* high 8 bits are internal */ -+ __HMBIRD_ENQ_INTERNAL_MASK = 0xffLLU << 56, -+ -+ HMBIRD_ENQ_CLEAR_OPSS = 1LLU << 56, -+ HMBIRD_ENQ_DSQ_PRIQ = 1LLU << 57, -+}; -+ -+enum hmbird_deq_flags { -+ /* expose select DEQUEUE_* flags as enums */ -+ HMBIRD_DEQ_SLEEP = DEQUEUE_SLEEP, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * The generic core-sched layer decided to execute the task even though -+ * it hasn't been dispatched yet. Dequeue from the BPF side. -+ */ -+ HMBIRD_DEQ_CORE_SCHED_EXEC = 1LLU << 32, -+}; -+ -+enum hmbird_kick_flags { -+ HMBIRD_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -+ HMBIRD_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ -+}; -+ -+enum hmbird_task_prop_type { -+ HMBIRD_TASK_PROP_COMMON, -+ HMBIRD_TASK_PROP_PIPELINE, -+ HMBIRD_TASK_PROP_DEBUG_OR_LOG, -+ HMBIRD_TASK_PROP_HIGH_LOAD, -+ HMBIRD_TASK_PROP_IO, -+ HMBIRD_TASK_PROP_NETWORK, -+ HMBIRD_TASK_PROP_PERIODIC, -+ HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL, -+ HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL, -+ HMBIRD_TASK_PROP_ISOLATE, -+ HMBIRD_TASK_PROP_MAX, -+}; -+ -+enum hmbird_preempt_policy_id { -+ HMBIRD_PREEMPT_POLICY_NONE, -+ HMBIRD_PREEMPT_POLICY_PRIO_BASED, -+}; -+ -+extern const struct sched_class hmbird_sched_class; -+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops; -+extern const struct file_operations sched_hmbird_fops; -+extern unsigned long hmbird_watchdog_timeout; -+extern unsigned long hmbird_watchdog_timestamp; -+ -+enum scx_rq_flags { -+ HMBIRD_RQ_CAN_STOP_TICK = 1 << 0, -+}; -+ -+struct hmbird_rq { -+ struct hmbird_dispatch_q local_dsq; -+ struct list_head watchdog_list; -+ u64 ops_qseq; -+ u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -+ u32 nr_running; -+ u32 flags; -+ bool cpu_released; -+ cpumask_var_t cpus_to_kick; -+ cpumask_var_t cpus_to_preempt; -+ cpumask_var_t cpus_to_wait; -+ u64 pnt_seq; -+ u64 *prev_runnable_sum_fixed; -+ u32 *prev_window_size; -+ struct irq_work kick_cpus_irq_work; -+ bool pipeline; -+ bool exclusive; -+ bool period_disallow; -+ bool nonperiod_disallow; -+ struct rq *rq; -+ struct hmbird_sched_rq_stats *srq; -+}; -+ -+struct hmbird_sched_change_guard { -+ struct task_struct *p; -+ struct rq *rq; -+ bool queued; -+ bool running; -+ bool done; -+}; -+ -+extern struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -+ -+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags); -+ -+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -+ for (struct hmbird_sched_change_guard __cg = \ -+ hmbird_sched_change_guard_init(__rq, __p, __flags); \ -+ !__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags)) -+ -+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+bool task_on_hmbird(struct task_struct *p); -+int hmbird_pre_fork(struct task_struct *p); -+void hmbird_post_fork(struct task_struct *p); -+void hmbird_cancel_fork(struct task_struct *p); -+int hmbird_check_setscheduler(struct task_struct *p, int policy); -+bool hmbird_can_stop_tick(struct rq *rq); -+void init_sched_hmbird_class(void); -+ -+void hmbird_ops_exit(void); -+#define hmbird_ops_error(fmt, ...) \ -+do { \ -+ hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__); \ -+ hmbird_ops_exit(); \ -+} while (0) -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active); -+ -+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight) -+{ -+ return clamp_t(unsigned long, -+ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -+ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); -+} -+ -+static inline void hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active) -+{ -+ if (!hmbird_enabled()) -+ return; -+#ifdef CONFIG_SMP -+ /* -+ * Pairs with the smp_load_acquire() issued by a CPU in -+ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -+ * resched. -+ */ -+ smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1); -+#endif -+ if (!static_branch_unlikely(&hmbird_ops_cpu_preempt)) -+ return; -+ __hmbird_notify_pick_next_task(rq, p, active); -+} -+ -+extern void hmbird_scheduler_tick(void); -+void scan_timeout(struct rq *rq); -+void hmbird_notify_sched_tick(void); -+ -+static inline const struct sched_class *next_active_class(const struct sched_class *class) -+{ -+ class++; -+ -+ if (hmbird_enabled() && class == &fair_sched_class) -+ class++; -+ if (!hmbird_enabled() && class == &hmbird_sched_class) -+ class++; -+ return class; -+} -+ -+#define for_active_class_range(class, _from, _to) \ -+ for (class = (_from); class != (_to); class = next_active_class(class)) -+ -+#define for_each_active_class(class) \ -+ for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -+ -+/* -+ * HMBIRD requires a balance() call before every pick_next_task() call including -+ * when waking up from idle. -+ */ -+#define for_balance_class_range(class, prev_class, end_class) \ -+ for_active_class_range(class, (prev_class) > &hmbird_sched_class ? \ -+ &hmbird_sched_class : (prev_class), (end_class)) -+ -+#define MIN_CGROUP_DL_IDX (5) /* 8ms */ -+#define DEFAULT_CGROUP_DL_IDX (8) /* 64ms */ -+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS]; -+ -+ -+void __hmbird_update_idle(struct rq *rq, bool idle); -+ -+static inline void hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ if (hmbird_enabled()) -+ __hmbird_update_idle(rq, idle); -+} -+ -+int hmbird_ctrl(bool enable); -+ -+int hmbird_tg_online(struct task_group *tg); -+ -+ -+static inline u16 hmbird_task_util(struct task_struct *p) -+{ -+ return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0; -+} -+u16 hmbird_cpu_util(int cpu); -+ -+bool get_hmbird_ops_enabled(void); -+bool get_non_hmbird_task(void); -+void set_cpu_cluster(u64 cpu_cluster); -+#endif /*_EXT_H_*/ -diff --git b/kernel/sched/hmbird/hmbird.c a/kernel/sched/hmbird/hmbird.c -new file mode 100755 -index 000000000000..86f9fd092424 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird.c -@@ -0,0 +1,4354 @@ -+// SPDX-License-Identifier: GPL-2.0 -+ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#include -+#include -+ -+#include "slim.h" -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+#include -+ -+#define CLUSTER_SEPARATE -+ -+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0); -+atomic_t non_hmbird_task; -+atomic_t hmbird_module_loaded = ATOMIC_INIT(0); -+int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0}; -+ -+enum hmbird_internal_consts { -+ HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ, -+}; -+ -+enum hmbird_ops_enable_state { -+ HMBIRD_OPS_PREPPING, -+ HMBIRD_OPS_ENABLING, -+ HMBIRD_OPS_ENABLED, -+ HMBIRD_OPS_DISABLING, -+ HMBIRD_OPS_DISABLED, -+}; -+ -+static inline void put_hmbird_ts(struct task_struct *p) -+{ -+ kfree((void *)p->android_oem_data1[HMBIRD_TS_IDX]); -+ p->android_oem_data1[HMBIRD_TS_IDX] = 0; -+} -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+/* -+ * hmbird_entity->ops_state -+ * -+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler. -+ * State transitions look as follows: -+ * -+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING -+ * ^ | | -+ * | v v -+ * \-------------------------------/ -+ * -+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -+ * sites for explanations on the conditions being waited upon and why they are -+ * safe. Transitions out of them into NONE or QUEUED must store_release and the -+ * waiters should load_acquire. -+ * -+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether -+ * any given task can be dispatched by the BPF scheduler at all times and thus -+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core -+ * can safely reject invalid dispatches. -+ */ -+enum hmbird_ops_state { -+ HMBIRD_OPSS_NONE, /* owned by the HMBIRD core */ -+ HMBIRD_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -+ HMBIRD_OPSS_QUEUED, /* owned by the BPF scheduler */ -+ HMBIRD_OPSS_DISPATCHING, /* in transit back to the HMBIRD core */ -+ -+ /* -+ * QSEQ brands each QUEUED instance so that, when dispatch races -+ * dequeue/requeue, the dispatcher can tell whether it still has a claim -+ * on the task being dispatched. -+ */ -+ HMBIRD_OPSS_QSEQ_SHIFT = 2, -+ HMBIRD_OPSS_STATE_MASK = (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1, -+ HMBIRD_OPSS_QSEQ_MASK = ~HMBIRD_OPSS_STATE_MASK, -+}; -+ -+enum switch_stat { -+ HMBIRD_DISABLED, -+ HMBIRD_SWITCH_PREP, -+ HMBIRD_RQ_SWITCH_BEGIN, -+ HMBIRD_RQ_SWITCH_DONE, -+ HMBIRD_ENABLED, -+}; -+enum switch_stat curr_ss; -+ -+/* -+ * During exit, a task may schedule after losing its PIDs. When disabling the -+ * BPF scheduler, we need to be able to iterate tasks in every state to -+ * guarantee system safety. Maintain a dedicated task list which contains every -+ * task between its fork and eventual free. -+ */ -+DEFINE_SPINLOCK(hmbird_tasks_lock); -+static LIST_HEAD(hmbird_tasks); -+ -+/* ops enable/disable */ -+static struct kthread_worker *hmbird_ops_helper; -+static DEFINE_MUTEX(hmbird_ops_enable_mutex); -+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem); -+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED); -+ -+static bool hmbird_warned_zero_slice; -+enum hmbird_switch_type sw_type; -+static int skip_num[MAX_GLOBAL_DSQS]; -+static int big_distribute_mask_prev; -+static int little_distribute_mask_prev; -+ -+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0); -+ -+/* -+ * The maximum amount of time in jiffies that a task may be runnable without -+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger -+ * hmbird_ops_error(). -+ */ -+unsigned long hmbird_watchdog_timeout; -+ -+/* -+ * The last time the delayed work was run. This delayed work relies on -+ * ksoftirqd being able to run to service timer interrupts, so it's possible -+ * that this work itself could get wedged. To account for this, we check that -+ * it's not stalled in the timer tick, and trigger an error if it is. -+ */ -+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES; -+ -+static struct delayed_work hmbird_watchdog_work; -+static struct work_struct hmbird_err_exit_work; -+ -+/* idle tracking */ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_CPUMASK_OFFSTACK -+#define CL_ALIGNED_IF_ONSTACK -+#else -+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp -+#endif -+ -+static struct { -+ cpumask_var_t cpu; -+ cpumask_var_t smt; -+} idle_masks CL_ALIGNED_IF_ONSTACK; -+ -+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus; -+#endif /* CONFIG_SMP */ -+ -+/* dispatch queues */ -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global; -+ -+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128}; -+u32 pcp_dsq_deadline = 20; -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS]; -+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq); -+ -+static u64 max_hmbird_dsq_internal_id; -+ -+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/ -+#define CLUSTER_SEPARATE_IDX (8) -+#define GDSQS_ID_BASE (3) -+#define UX_COMPATIBLE_IDX (4) -+#define NON_PERIOD_START (5) -+#define NON_PERIOD_END (MAX_GLOBAL_DSQS) -+#define CREATE_DSQ_LEVEL_WITHIN (1) -+ -+struct hmbird_sched_info { -+ spinlock_t lock; -+ int curr_idx[2]; -+ int rtime[MAX_GLOBAL_DSQS]; -+}; -+ -+struct pcp_sched_info { -+ s64 pcp_seq; -+ int rtime; -+ bool pcp_round; -+}; -+ -+/* -+ * pcp_info may rw by another cpu. -+ * protected by rq->lock. -+ */ -+atomic64_t pcp_dsq_round; -+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info); -+ -+struct md_info_t *md_info; -+ -+static int b_rescue_l, l_rescue_b; -+static struct hmbird_sched_info sinfo; -+ -+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC; -+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = { -+ 0, 0, 0, 0, 0, -+ 32 * NSEC_PER_MSEC, -+ 20 * NSEC_PER_MSEC, -+ 14 * NSEC_PER_MSEC, -+ 8 * NSEC_PER_MSEC, -+ 6 * NSEC_PER_MSEC -+}; -+ -+struct cluster_ctx { -+ /* cpu-dsq map must within [lower, upper) */ -+ int upper; -+ int lower; -+ int tidx; -+}; -+ -+enum stat_items { -+ GLOBAL_STAT, -+ CPU_ALLOW_FAIL, -+ RT_CNT, -+ KEY_TASK_CNT, -+ SWITCH_IDX, -+ TIMEOUT_CNT, -+ -+ TOTAL_DSP_CNT, -+ MOVE_RQ_CNT, -+ SELECT_CPU, -+ -+ DWORD_STAT_END = SELECT_CPU, -+ -+ GDSQ_CNT, -+ ERR_IDX, -+ PCP_TIMEOUT_CNT, -+ PCP_LDSQ_CNT, -+ PCP_ENQL_CNT, -+ -+ MAX_ITEMS, -+}; -+static DEFINE_SPINLOCK(stats_lock); -+static char *stats_str[MAX_ITEMS] = { -+ "global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt", -+ "switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt", -+ "select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt", -+ "pcp_ldsq_cnt", "pcp_enql_cnt" -+}; -+ -+ -+struct stats_struct { -+ u64 global_stat[2]; -+ u64 cpu_allow_fail[2]; -+ u64 rt_cnt[2]; -+ u64 key_task_cnt[2]; -+ u64 switch_idx[2]; -+ u64 timeout_cnt[2]; -+ -+ u64 total_dsp_cnt[2]; -+ u64 move_rq_cnt[2]; -+ u64 select_cpu[2]; -+ -+ u64 gdsq_count[MAX_GLOBAL_DSQS][2]; -+ u64 err_idx[5]; -+ u64 pcp_timeout_cnt[NR_CPUS]; -+ u64 pcp_ldsq_count[NR_CPUS][2]; -+ u64 pcp_enql_cnt[NR_CPUS]; -+} stats_data; -+ -+static struct { -+ cpumask_var_t ex_free; -+ cpumask_var_t exclusive; -+ cpumask_var_t partial; -+ cpumask_var_t big; -+ cpumask_var_t little; -+} iso_masks __read_mostly; -+ -+/* -+ * Need more synchronization for these two variables? -+ * I choose not to. -+ */ -+static int l_need_rescue, b_need_rescue; -+ -+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...) \ -+{ \ -+ char buf[MAX_FATAL_INFO]; \ -+ \ -+ scnprintf(buf, MAX_FATAL_INFO, fmt, ##args); \ -+ hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf); \ -+ trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable), \ -+ READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf); \ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); \ -+} \ -+ -+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2) -+{ -+ int c1, c2; -+ struct cpumask mask = {.bits = {0}}; -+ struct cpumask tmp = {.bits = {0}}; -+ -+ if (!slim_stats) -+ return false; -+ -+ if (!rq1 || !rq2) -+ return false; -+ -+ c1 = cpu_of(rq1); -+ c2 = cpu_of(rq2); -+ cpumask_set_cpu(c1, &mask); -+ cpumask_set_cpu(c2, &mask); -+ -+ if (cpumask_and(&tmp, iso_masks.little, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.big, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.partial, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ return false; -+} -+ -+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu) -+{ -+ unsigned long flags; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ if (!slim_stats) -+ return; -+ -+ switch (item) { -+ case GLOBAL_STAT: -+ fallthrough; -+ case CPU_ALLOW_FAIL: -+ fallthrough; -+ case RT_CNT: -+ fallthrough; -+ case KEY_TASK_CNT: -+ fallthrough; -+ case SWITCH_IDX: -+ fallthrough; -+ case TIMEOUT_CNT: -+ fallthrough; -+ case TOTAL_DSP_CNT: -+ fallthrough; -+ case MOVE_RQ_CNT: -+ fallthrough; -+ case SELECT_CPU: -+ pval = pbase + item * 2 + idx; -+ break; -+ case GDSQ_CNT: -+ pval = &stats_data.gdsq_count[dsq_id][idx]; -+ break; -+ case ERR_IDX: -+ pval = &stats_data.err_idx[idx]; -+ break; -+ case PCP_TIMEOUT_CNT: -+ pval = &stats_data.pcp_timeout_cnt[cpu]; -+ break; -+ case PCP_LDSQ_CNT: -+ pval = &stats_data.pcp_ldsq_count[cpu][idx]; -+ break; -+ case PCP_ENQL_CNT: -+ pval = &stats_data.pcp_enql_cnt[cpu]; -+ break; -+ default: -+ return; -+ } -+ -+ spin_lock_irqsave(&stats_lock, flags); -+ *pval += 1; -+ spin_unlock_irqrestore(&stats_lock, flags); -+} -+ -+static inline bool handle_ret(int ret, int *idx, int len) -+{ -+ if (ret < 0 || ret >= len - *idx) -+ return true; -+ *idx += ret; -+ return false; -+} -+ -+#define PRINT_INTV (5 * HZ) -+void stats_print(char *buf, int len) -+{ -+ int idx = 0, i, j, ret; -+ int item = 0; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ for (item = 0; item < MAX_ITEMS; item++) { -+ if (item <= DWORD_STAT_END) { -+ pval = pbase + item * 2; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n", -+ stats_str[item], pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == GDSQ_CNT) { -+ for (j = 0; j < MAX_GLOBAL_DSQS; j++) { -+ pval = (u64 *)&stats_data.gdsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == ERR_IDX) { -+ pval = (u64 *)&stats_data.err_idx; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n", -+ stats_str[item], pval[0], -+ pval[1], pval[2], pval[3], pval[4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == PCP_TIMEOUT_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_timeout_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_LDSQ_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_ldsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_ENQL_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_enql_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } -+ } -+ -+ if (!md_info) { -+ buf[idx] = '\0'; -+ return; -+ } -+ -+ ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_SWITCHS; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "sw_rec[%d] = %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.sw_rec[i].switch_at, -+ md_info->kern_dump.sw_rec[i].is_success, -+ md_info->kern_dump.sw_rec[i].end_state, -+ md_info->kern_dump.sw_rec[i].switch_reason); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_EXCEP_ID; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "excep[%d] = %llu %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.excep_rec[i][0], -+ md_info->kern_dump.excep_rec[i][1], -+ md_info->kern_dump.excep_rec[i][2], -+ md_info->kern_dump.excep_rec[i][3], -+ md_info->kern_dump.excep_rec[i][4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n", -+ i, md_info->kern_dump.excep_idx[i]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ -+ buf[idx] = '\0'; -+} -+ -+enum cpu_type { -+ LITTLE, -+ BIG, -+ PARTIAL, -+ EXCLUSIVE, -+ EX_FREE, -+ INVALID -+}; -+ -+enum dsq_type { -+ GLOBAL_DSQ, -+ PCP_DSQ, -+ OTHER, -+ MAX_DSQ_TYPE, -+}; -+ -+static enum cpu_type cpu_cluster(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (get_hmbird_rq(rq)->exclusive) { -+ return EXCLUSIVE; -+ } else { -+ if (cpumask_test_cpu(cpu, iso_masks.little)) -+ return LITTLE; -+ else if (cpumask_test_cpu(cpu, iso_masks.big)) -+ return BIG; -+ else if (cpumask_test_cpu(cpu, iso_masks.partial)) -+ return PARTIAL; -+ else if (cpumask_test_cpu(cpu, iso_masks.exclusive)) -+ return EXCLUSIVE; -+ else if (cpumask_test_cpu(cpu, iso_masks.ex_free)) -+ return EX_FREE; -+ } -+ return INVALID; -+} -+ -+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq) -+{ -+ if (!dsq) -+ return OTHER; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= GDSQS_ID_BASE) && -+ ((dsq->id & 0xff) < MAX_GLOBAL_DSQS)) -+ return GLOBAL_DSQ; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) && -+ ((dsq->id & 0xff) < max_hmbird_dsq_internal_id)) -+ return PCP_DSQ; -+ -+ return OTHER; -+} -+ -+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq) -+{ -+ enum dsq_type type; -+ -+ type = get_dsq_type(dsq); -+ switch (type) { -+ case GLOBAL_DSQ: -+ case PCP_DSQ: -+ return (dsq->id & 0xff) - GDSQS_ID_BASE; -+ default: -+ return -1; -+ } -+ return -1; -+} -+ -+static void update_cpus_idle(bool set, struct cpumask *mask) -+{ -+ int cpu; -+ struct rq *rq; -+ -+ if (set) { -+ for_each_cpu(cpu, mask) { -+ rq = cpu_rq(cpu); -+ if (is_idle_task(rq->curr)) -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ } -+ } else -+ cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask); -+} -+ -+static void set_partial_status(bool enable, bool little, bool big) -+{ -+ WRITE_ONCE(partial_enable, enable); -+ WRITE_ONCE(l_need_rescue, little); -+ WRITE_ONCE(b_need_rescue, big); -+} -+ -+static bool is_little_need_rescue(void) -+{ -+ return READ_ONCE(l_need_rescue); -+} -+ -+static bool is_big_need_rescue(void) -+{ -+ return READ_ONCE(b_need_rescue); -+} -+ -+static bool is_partial_enabled(void) -+{ -+ return READ_ONCE(partial_enable); -+} -+ -+static bool is_partial_cpu(int cpu) -+{ -+ return cpumask_test_cpu(cpu, iso_masks.partial); -+} -+ -+static void set_iso_par_free(bool enable) -+{ -+ WRITE_ONCE(isolate_ctrl, enable); -+} -+ -+static bool is_iso_par_free(void) -+{ -+ return READ_ONCE(isolate_ctrl); -+} -+ -+static bool skip_update_idle(void) -+{ -+ int cpu = smp_processor_id(); -+ enum cpu_type type = cpu_cluster(cpu); -+ -+ if ((type == EXCLUSIVE && !is_iso_par_free()) || -+ /* partial enable may changed during idle, it doesn't matter. */ -+ (!is_partial_enabled() && type == PARTIAL)) -+ return true; -+ -+ return false; -+} -+ -+static void init_isolate_cpus(void) -+{ -+ WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -+ cpumask_set_cpu(0, iso_masks.big); -+ cpumask_set_cpu(1, iso_masks.big); -+ cpumask_set_cpu(2, iso_masks.big); -+ cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(4, iso_masks.big); -+ cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(6, iso_masks.partial); -+ cpumask_set_cpu(7, iso_masks.ex_free); -+} -+ -+extern spinlock_t css_set_lock; -+ -+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp) -+{ -+ int i; -+ struct cgroup *anc; -+ -+ spin_lock_irq(&css_set_lock); -+ for (i = 0; i < cgrp->level; i++) { -+ anc = cgrp->ancestors[i]; -+ if (anc->level != CREATE_DSQ_LEVEL_WITHIN) -+ continue; -+ cgroup_get(anc); -+ spin_unlock_irq(&css_set_lock); -+ return anc; -+ } -+ spin_unlock_irq(&css_set_lock); -+ hmbird_err(NO_CGROUP_L1, ":error cgroup = %s\n", cgrp->kn->name); -+ return NULL; -+} -+ -+#define PCP_IDX_BIT (1 << 31) -+ -+static bool is_pcp_rt(struct task_struct *p) -+{ -+ return rt_prio(p->prio) && (p->nr_cpus_allowed == 1); -+} -+ -+static bool is_pcp_idx(int idx) -+{ -+ return idx >= MAX_GLOBAL_DSQS; -+} -+ -+static bool is_critical_system_task(struct task_struct *p) -+{ -+ int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK; -+ -+ return (p->prio < (MAX_RT_PRIO >> 1) && -+ (sp_dl < SCHED_PROP_DEADLINE_LEVEL3)); -+} -+ -+#define ISOLATE_TASK_TOP_BIT (1 << 17) -+#define PIPELINE_TASK_TOP_BIT (1 << 9) -+static bool is_pipeline_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT); -+} -+ -+static bool is_isolate_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT); -+} -+ -+static bool is_critical_app_task_without_isolate(struct task_struct *p) -+{ -+ return task_is_top_task(p) && !is_isolate_task(p); -+} -+ -+static int find_idx_from_task(struct task_struct *p) -+{ -+ int idx, cpu; -+ int sp_dl; -+ struct task_group *tg = p->sched_task_group; -+ -+ if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) { -+ cpu = cpumask_any(p->cpus_ptr); -+ idx = cpu + MAX_GLOBAL_DSQS; -+ return idx; -+ } -+ -+ if (is_critical_system_task(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL0; -+ goto done; -+ } -+ -+ if (is_critical_app_task_without_isolate(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL1; -+ goto done; -+ } -+ -+ if (p->pid == scx_systemui_pid) { -+ idx = SCHED_PROP_DEADLINE_LEVEL4; -+ goto done; -+ } -+ -+ sp_dl = hmbird_get_dsq_id(p); -+ if (sp_dl) { -+ idx = sp_dl; -+ goto done; -+ } -+ -+ if (rt_prio(p->prio)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL3; -+ goto done; -+ } -+ -+ if (tg && tg->css.cgroup && tg->css.cgroup->kn) { -+ if (likely(tg->css.cgroup->kn->id >= 0 && -+ tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS)) -+ idx = cgroup_ids_table[tg->css.cgroup->kn->id]; -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ -+done: -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) { -+ hmbird_err(DSQ_ID_ERR, " : idx error, idx = %d-----\n", idx); -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } -+ return idx; -+} -+ -+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p) -+{ -+ int idx; -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq; -+ -+ if (!p) -+ return NULL; -+ -+ idx = find_idx_from_task(p); -+ if (is_pcp_idx(idx)) { -+ idx -= MAX_GLOBAL_DSQS; -+ dsq = &per_cpu(pcp_ldsq, idx); -+ get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq); -+ slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx); -+ } else { -+ dsq = &gdsqs[idx]; -+ get_hmbird_ts(p)->gdsq_idx = idx; -+ slim_stats_record(GDSQ_CNT, 0, idx, 0); -+ } -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ dsq->last_consume_at = jiffies; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ return dsq; -+} -+ -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq); -+ -+static void set_partial_rescue(bool p_state, bool l_over, bool b_over) -+{ -+ set_partial_status(p_state, l_over, b_over); -+ update_cpus_idle(p_state, iso_masks.partial); -+ hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled()); -+ hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue()); -+ hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue()); -+} -+ -+static void free_isocpu(bool enable) -+{ -+ set_iso_par_free(enable); -+ update_cpus_idle(enable, iso_masks.ex_free); -+ hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free()); -+} -+ -+inline u64 get_hmbird_cpu_util(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed) -+ return 0; -+ u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed); -+ u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size); -+ -+ do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT); -+ -+ return prev_runnable_sum_fixed; -+} -+ -+static inline unsigned int get_scaling_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->max; -+} -+ -+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static u64 get_cpus_max_util(struct cpumask *mask) -+{ -+ int cpu; -+ u64 max = 0; -+ u64 util, ratio; -+ unsigned long effective_cap = 0; -+ -+ for_each_cpu(cpu, mask) { -+ if (slim_walt_ctrl) -+ slim_get_cpu_util(cpu, &util); -+ else -+ util = get_hmbird_cpu_util(cpu); -+ -+ /* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/ -+ if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu))) -+ effective_cap = arch_scale_cpu_capacity(cpu); -+ else -+ effective_cap = arch_scale_cpu_capacity(cpu) * -+ get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu); -+ -+ ratio = util * 100 / effective_cap; -+ hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util); -+ hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n", -+ cpu, (u64)arch_scale_cpu_capacity(cpu)); -+ hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n", -+ cpu, (u64)effective_cap); -+ -+ if (ratio > 100) -+ ratio = 100; -+ -+ if (ratio > max) -+ max = ratio; -+ } -+ return max; -+} -+ -+static bool cluster_need_rescue(struct cpumask *mask, int hres) -+{ -+ return get_cpus_max_util(mask) > hres; -+} -+ -+void partial_dynamic_ctrl(void) -+{ -+ u64 lmax = 0, bmax = 0; -+ bool l_over, l_under, b_over, b_under; -+ static bool last_l_over, last_b_over; -+ static unsigned long last_check; -+ -+ /* Check partial every jiffies. need lock? */ -+ if (time_before_eq(jiffies, READ_ONCE(last_check))) -+ return; -+ -+ WRITE_ONCE(last_check, jiffies); -+ -+ lmax = get_cpus_max_util(iso_masks.little); -+ l_over = lmax > parctrl_high_ratio_l; -+ l_under = lmax < parctrl_low_ratio_l; -+ -+ bmax = get_cpus_max_util(iso_masks.big); -+ b_over = bmax > parctrl_high_ratio; -+ b_under = bmax < parctrl_low_ratio; -+ -+ if (is_partial_enabled() && (l_over || b_over)) { -+ if (last_l_over != l_over || last_b_over != b_over) -+ set_partial_rescue(true, l_over, b_over); -+ } else if (!is_partial_enabled() && (l_over || b_over)) { -+ set_partial_rescue(true, l_over, b_over); -+ } else if (is_partial_enabled() && l_under && b_under) { -+ set_partial_rescue(false, false, false); -+ } -+ last_l_over = l_over; -+ last_b_over = b_over; -+ -+ if (is_partial_enabled()) { -+ if (cpumask_empty(iso_masks.partial)) { -+ bmax = bmax > lmax ? bmax : lmax; -+ } else { -+ bmax = get_cpus_max_util(iso_masks.partial); -+ } -+ if (!is_iso_par_free() && bmax > isoctrl_high_ratio) { -+ hmbird_info_trace("partial max = %llu\n", bmax); -+ free_isocpu(true); -+ } else if (is_iso_par_free() && bmax < isoctrl_low_ratio) -+ free_isocpu(false); -+ } else if (is_iso_par_free()) { -+ free_isocpu(false); -+ } -+} -+ -+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx) -+{ -+ hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx); -+} -+ -+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx) -+{ -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[idx])) { -+ slim_stats_record(GDSQ_CNT, 1, idx, 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ int i; -+ -+ for (i = 0; i < UX_COMPATIBLE_IDX; i++) { -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ slim_stats_record(GDSQ_CNT, 1, i, 0); -+ return 1; -+ } -+ } -+ return 0; -+} -+ -+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) { -+ slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ goto clear_timeout; -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) -+ goto clear_timeout; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ hmbird_info_trace( -+ "dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n", -+ dsq_id_to_internal(dsq), entity->task->comm, -+ jiffies, msecs_to_jiffies(deadline), entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1); -+ return; -+ -+clear_timeout: -+ hmbird_info_trace("dsq[%d] clear timeout\n", -+ dsq_id_to_internal(dsq)); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0); -+ dsq->is_timeout = false; -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq)); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime) -+{ -+ hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime); -+} -+ -+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any) -+{ -+ bool is_timeout; -+ int cpu = cpu_of(rq); -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu); -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ is_timeout = dsq->is_timeout; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ /* -+ * dsq->is_timeout may change here, let it be. -+ * it won't cause serious problems. -+ * the same for consume_dispatch_q later. -+ */ -+ if (!is_timeout && !any) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, dsq)) { -+ if (is_timeout) { -+ hmbird_info_trace("dsq[%d] consume a pcp timeout task\n", -+ dsq_id_to_internal(dsq)); -+ update_timeout_stats(rq, dsq, pcp_dsq_deadline); -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu); -+ } -+ slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu); -+ return 1; -+ } -+ /* -+ * No pcp task, clear quota. -+ */ -+ if (any) { -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+ } -+ return 0; -+} -+ -+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf) -+{ -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ } -+ return 0; -+} -+ -+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf, -+ int tmp, int cidx, int tidx) -+{ -+ unsigned long flags; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) { -+ if (tmp != cidx) { -+ spin_lock(&sinfo.lock); -+ sinfo.curr_idx[tidx] = tmp; -+ spin_unlock(&sinfo.lock); -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]); -+ slim_stats_record(SWITCH_IDX, 0, 0, 0); -+ } -+ slim_stats_record(GDSQ_CNT, 1, tmp, 0); -+ -+ raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags); -+ gdsqs[tmp].last_consume_at = jiffies; -+ raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags); -+ return 1; -+ } -+ return 0; -+ -+} -+ -+static int get_cidx(struct cluster_ctx *ctx) -+{ -+ int cidx; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ if (cidx < ctx->lower || cidx >= ctx->upper) { -+ sinfo.curr_idx[ctx->tidx] = ctx->lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]); -+ slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ } -+ spin_unlock(&sinfo.lock); -+ -+ return cidx; -+} -+ -+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = CLUSTER_SEPARATE_IDX; -+ if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) { -+ pr_debug(" %s iso_masks.little first is %d\n", -+ __func__, cpumask_first(iso_masks.little)); -+ ctx->upper = NON_PERIOD_END; -+ } -+ ctx->tidx = 0; -+ break; -+ case LITTLE: -+ ctx->lower = CLUSTER_SEPARATE_IDX; -+ ctx->upper = NON_PERIOD_END; -+ if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) { -+ pr_debug(" %s iso_masks.big first is %d", -+ __func__, cpumask_first(iso_masks.big)); -+ ctx->lower = NON_PERIOD_START; -+ } -+ ctx->tidx = 1; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ case LITTLE: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = NON_PERIOD_END; -+ ctx->tidx = 0; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ if (cluster_separate) { -+ return gen_cluster_ctx_separate(ctx, type); -+ } else { -+ return gen_cluster_ctx_common(ctx, type); -+ } -+} -+ -+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ int i; -+ bool is_timeout; -+ unsigned long flags; -+ struct cluster_ctx ctx; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ /* Third param means only consume timeout pcp dsq. */ -+ if (consume_pcp_dsq(rq, rf, false)) -+ return 1; -+ -+ for (i = ctx.lower; i < ctx.upper; i++) { -+ raw_spin_lock_irqsave(&gdsqs[i].lock, flags); -+ is_timeout = gdsqs[i].is_timeout; -+ raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags); -+ /* gdsqs[i].is_timeout may change here, let it be... */ -+ if (is_timeout) { -+ /* -+ * consume_dispatch_q will acquire dsq-lock, -+ * So cannot keep lock here, annoy enough. -+ * may rewrite a consume_dispatch_q_locked, TODO. -+ */ -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ hmbird_info_trace("dsq[%d] consume a timeout task\n", i); -+ slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0); -+ update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]); -+ return 1; -+ } -+ } -+ } -+ return 0; -+} -+ -+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ struct cluster_ctx ctx; -+ int cidx; -+ int tmp; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ cidx = get_cidx(&ctx); -+ tmp = cidx; -+ do { -+ if (check_pcp_dsq_round(rq, rf)) -+ return 1; -+ if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx)) -+ return 1; -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[tmp] = 0; -+ systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]); -+ tmp++; -+ if (tmp >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ tmp = ctx.lower; -+ } -+ spin_unlock(&sinfo.lock); -+ } while (tmp != cidx); -+ -+ return consume_pcp_dsq(rq, rf, true); -+} -+ -+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ enum cpu_type type = cpu_cluster(cpu_of(rq)); -+ bool period_allowed = !get_hmbird_rq(rq)->period_disallow; -+ bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow; -+ -+ switch (type) { -+ case EX_FREE: -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ break; -+ case EXCLUSIVE: -+ if (!is_iso_par_free()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ } -+ /* Only non-period task can run on isolate cpus */ -+ if (!READ_ONCE(iso_free_rescue)) { -+ if (is_big_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ } else { -+ /* Free run, can run on any task. */ -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ case PARTIAL: -+ if (!is_partial_enabled()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ return 0; -+ } -+ if (is_big_need_rescue()) { -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ -+ case BIG: -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ if (is_iso_par_free() || (is_little_need_rescue() && -+ !cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) { -+ hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++); -+ return 1; -+ } -+ } -+ break; -+ -+ case LITTLE: -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ -+ if (is_iso_par_free() || (is_big_need_rescue() && -+ !cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) { -+ hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++); -+ return 1; -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return 0; -+} -+ -+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf) -+{ -+ return consume_hmbird_global_dsq(rq, rf); -+} -+ -+ -+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time) -+{ -+ int idx; -+ -+ /* which dsq belongs to while task enqueue, task will consume its running time. */ -+ idx = get_hmbird_ts(p)->gdsq_idx; -+ /* Only non-period dsq share running time between each other. */ -+ if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id) -+ return; -+ -+ if (idx >= MAX_GLOBAL_DSQS) { -+ per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } else { -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[idx] += exec_time; -+ spin_unlock(&sinfo.lock); -+ systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]); -+ } -+} -+ -+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type) -+{ -+ int cidx; -+ struct cluster_ctx ctx; -+ int cpu = cpu_of(rq); -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ if (cidx < ctx.lower || cidx >= ctx.upper) { -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ slim_stats_record(ERR_IDX, ctx.tidx, 0, 0); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ } -+ -+ while (1) { -+ if (per_cpu(pcp_info, cpu).pcp_round) { -+ if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) { -+ hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n", -+ cpu, per_cpu(pcp_info, cpu).rtime); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } -+ } -+ if (sinfo.rtime[cidx] < dsq_quota[cidx]) -+ break; -+ -+ /* clear current dsq rtime */ -+ sinfo.rtime[cidx] = 0; -+ systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]); -+ -+ sinfo.curr_idx[ctx.tidx]++; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", -+ ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ } -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ slim_stats_record(SWITCH_IDX, 1, 0, 0); -+ } -+ spin_unlock(&sinfo.lock); -+} -+ -+ -+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p) -+{ -+ enum cpu_type type; -+ -+ if (!rq || !p) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ switch (type) { -+ case PARTIAL: -+ return; -+ case EXCLUSIVE: -+ return; -+ case EX_FREE: -+ return; -+ default: -+ break; -+ } -+ update_dsq_idx(rq, p, type); -+} -+ -+ -+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ int dsq_id; -+ -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo) || dsq->is_timeout) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ hmbird_deferred_err(SCAN_ENTITY_NULL, -+ " : entity is NULL, dsq->id = %llu\n", dsq->id); -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ dsq->is_timeout = true; -+ dsq_id = dsq_id_to_internal(dsq); -+ hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n", -+ dsq_id, entity->task->comm, jiffies, entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1); -+ raw_spin_unlock(&dsq->lock); -+ -+ return true; -+} -+ -+void scan_timeout(struct rq *rq) -+{ -+ int i; -+ int cpu = cpu_of(rq); -+ struct hmbird_dispatch_q *dsq; -+ static u64 last_scan_at; -+ static DEFINE_PER_CPU(u64, pcp_last_scan_at); -+ -+ if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu))) -+ return; -+ per_cpu(pcp_last_scan_at, cpu) = jiffies; -+ -+ dsq = &per_cpu(pcp_ldsq, cpu); -+ scan_dsq_timeout(rq, dsq, pcp_dsq_deadline); -+ -+ if (time_before_eq(jiffies, (unsigned long)last_scan_at)) -+ return; -+ last_scan_at = jiffies; -+ -+ for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) { -+ dsq = &gdsqs[i]; -+ scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]); -+ } -+} -+ -+/*******************************Initialize***********************************/ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id); -+static void init_dsq_at_boot(void) -+{ -+ int i, cpu; -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i)); -+ } -+ for_each_possible_cpu(cpu) -+ init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i + cpu)); -+ -+ max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu; -+ spin_lock_init(&sinfo.lock); -+} -+ -+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgrp_name_to_idx(struct cgroup *cgrp) -+{ -+ int idx; -+ -+ if (!cgrp) -+ return -1; -+ -+ if (!strcmp(cgrp->kn->name, "display") -+ || !strcmp(cgrp->kn->name, "multimedia")) -+ idx = 5; /* 8ms */ -+ else if (!strcmp(cgrp->kn->name, "top-app") -+ || !strcmp(cgrp->kn->name, "ss-top")) -+ idx = 6; /* 16ms */ -+ else if (!strcmp(cgrp->kn->name, "ssfg") -+ || !strcmp(cgrp->kn->name, "foreground")) -+ idx = 7; /* 32ms */ -+ else if (!strcmp(cgrp->kn->name, "bg") -+ || !strcmp(cgrp->kn->name, "log") -+ || !strcmp(cgrp->kn->name, "dex2oat") -+ || !strcmp(cgrp->kn->name, "background")) -+ idx = 9; /* 128ms */ -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */ -+ -+ return idx; -+} -+ -+static void init_root_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX); -+} -+ -+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) { -+ pr_err("%s idx err!\n", __func__); -+ return; -+ } -+ -+ if (cgroup_ids_table[cgrp->kn->id] == -1) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp)); -+} -+ -+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ struct cgroup *l1cgrp; -+ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ l1cgrp = cgroup_ancestor_l1(cgrp); -+ if (l1cgrp) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp)); -+ cgroup_put(l1cgrp); -+} -+ -+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg) -+{ -+ switch (cgrp->level) { -+ case 0: -+ init_root_tg(cgrp, tg); -+ break; -+ case 1: -+ init_level1_tg(cgrp, tg); -+ break; -+ default: -+ init_child_tg(cgrp, tg); -+ break; -+ } -+} -+ -+/**************************************************************************/ -+ -+struct hmbird_task_iter { -+ struct hmbird_entity cursor; -+ struct task_struct *locked; -+ struct rq *rq; -+ struct rq_flags rf; -+}; -+ -+/** -+ * hmbird_task_iter_init - Initialize a task iterator -+ * @iter: iterator to init -+ * -+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized, -+ * @iter must eventually be exited with hmbird_task_iter_exit(). -+ * -+ * hmbird_tasks_lock may be released between this and the first next() call or -+ * between any two next() calls. If hmbird_tasks_lock is released between two -+ * next() calls, the caller is responsible for ensuring that the task being -+ * iterated remains accessible either through RCU read lock or obtaining a -+ * reference count. -+ * -+ * All tasks which existed when the iteration started are guaranteed to be -+ * visited as long as they still exist. -+ */ -+static void hmbird_task_iter_init(struct hmbird_task_iter *iter) -+{ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR }; -+ list_add(&iter->cursor.tasks_node, &hmbird_tasks); -+ iter->locked = NULL; -+} -+ -+/** -+ * hmbird_task_iter_exit - Exit a task iterator -+ * @iter: iterator to exit -+ * -+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held. -+ * If the iterator holds a task's rq lock, that rq lock is released. See -+ * hmbird_task_iter_init() for details. -+ */ -+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ if (list_empty(cursor)) -+ return; -+ -+ list_del_init(cursor); -+} -+ -+/** -+ * hmbird_task_iter_next - Next task -+ * @iter: iterator to walk -+ * -+ * Visit the next task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ struct hmbird_entity *pos; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ list_for_each_entry(pos, cursor, tasks_node) { -+ if (&pos->tasks_node == &hmbird_tasks) -+ return NULL; -+ if (!(pos->flags & HMBIRD_TASK_CURSOR)) { -+ list_move(cursor, &pos->tasks_node); -+ return pos->task; -+ } -+ } -+ -+ /* can't happen, should always terminate at hmbird_tasks above */ -+ hmbird_deferred_err(ITER_RET_NULL, " : unreachable path in scx_task_iter_next\n"); -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered - Next non-idle task -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ while ((p = hmbird_task_iter_next(iter))) { -+ if (!is_idle_task(p)) -+ return p; -+ } -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init() -+ * for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ p = hmbird_task_iter_next_filtered(iter); -+ if (!p) -+ return NULL; -+ -+ iter->rq = task_rq_lock(p, &iter->rf); -+ iter->locked = p; -+ return p; -+} -+ -+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void) -+{ -+ return atomic_read(&hmbird_ops_enable_state_var); -+} -+ -+static enum hmbird_ops_enable_state -+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to) -+{ -+ return atomic_xchg(&hmbird_ops_enable_state_var, to); -+} -+ -+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to, -+ enum hmbird_ops_enable_state from) -+{ -+ int from_v = from; -+ -+ return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to); -+} -+ -+static bool hmbird_ops_disabling(void) -+{ -+ return false; -+} -+ -+/** -+ * wait_ops_state - Busy-wait the specified ops state to end -+ * @p: target task -+ * @opss: state to wait the end of -+ * -+ * Busy-wait for @p to transition out of @opss. This can only be used when the -+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also -+ * has load_acquire semantics to ensure that the caller can see the updates made -+ * in the enqueueing and dispatching paths. -+ */ -+static void wait_ops_state(struct task_struct *p, u64 opss) -+{ -+ do { -+ cpu_relax(); -+ } while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss); -+} -+ -+ -+static void update_curr_hmbird(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ u64 now = rq_clock_task(rq); -+ u64 delta_exec; -+ -+ if (time_before_eq64(now, curr->se.exec_start)) -+ return; -+ -+ delta_exec = now - curr->se.exec_start; -+ curr->se.exec_start = now; -+ update_runningtime(rq, curr, delta_exec); -+ curr->se.sum_exec_runtime += delta_exec; -+ account_group_exec_runtime(curr, delta_exec); -+ cgroup_account_cputime(curr, delta_exec); -+ -+ if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF) -+ get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec); -+ -+ trace_sched_stat_runtime(curr, delta_exec, 0); -+} -+ -+static bool hmbird_dsq_priq_less(struct rb_node *node_a, -+ const struct rb_node *node_b) -+{ -+ const struct hmbird_entity *a = -+ container_of(node_a, struct hmbird_entity, dsq_node.priq); -+ const struct hmbird_entity *b = -+ container_of(node_b, struct hmbird_entity, dsq_node.priq); -+ -+ return time_before64(a->dsq_vtime, b->dsq_vtime); -+} -+ -+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ bool is_local = dsq->id == HMBIRD_DSQ_LOCAL; -+ unsigned long flags; -+ -+ hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq || -+ !list_empty(&get_hmbird_ts(p)->dsq_node.fifo), -+ "task = %s, dsq->id = %llu\n", p->comm, dsq->id); -+ hmbird_cond_deferred_err(ENQ_EXIST2, -+ (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq), -+ "task = %s\n", p->comm); -+ -+ if (!is_local) { -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ); -+ hmbird_ops_error(": %s\n", -+ "attempting to dispatch to a destroyed dsq"); -+ /* fall back to the global dsq */ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ dsq = &hmbird_dsq_global; -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ } -+ } -+ -+ if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) { -+ get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ; -+ rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq, -+ hmbird_dsq_priq_less); -+ } else { -+ if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT)) -+ list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ else -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ } -+ dsq->nr++; -+ get_hmbird_ts(p)->dsq = dsq; -+ -+ /* -+ * We're transitioning out of QUEUEING or DISPATCHING. store_release to -+ * match waiters' load_acquire. -+ */ -+ if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS) -+ atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE); -+ -+ if (is_local) { -+ struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq); -+ struct rq *rq = hmbird->rq; -+ bool preempt = false; -+ -+ if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr && -+ rq->curr->sched_class == &hmbird_sched_class) { -+ get_hmbird_ts(rq->curr)->slice = 0; -+ preempt = true; -+ } -+ -+ if (preempt || sched_class_above(&hmbird_sched_class, -+ rq->curr->sched_class)) -+ resched_curr(rq); -+ } else { -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ } -+} -+ -+static void task_unlink_from_dsq(struct task_struct *p, -+ struct hmbird_dispatch_q *dsq) -+{ -+ if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) { -+ rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ; -+ } else { -+ list_del_init(&get_hmbird_ts(p)->dsq_node.fifo); -+ } -+} -+ -+static bool task_linked_on_dsq(struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+} -+ -+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq; -+ bool is_local = dsq == &hmbird_rq->local_dsq; -+ -+ if (!dsq) { -+ hmbird_cond_deferred_err(TASK_LINKED1, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ /* -+ * When dispatching directly from the BPF scheduler to a local -+ * DSQ, the task isn't associated with any DSQ but -+ * @get_hmbird_ts(p)->holding_cpu may be set under the protection of -+ * %HMBIRD_OPSS_DISPATCHING. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu >= 0) -+ get_hmbird_ts(p)->holding_cpu = -1; -+ return; -+ } -+ -+ if (!is_local) -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ /* -+ * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node -+ * can't change underneath us. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu < 0) { -+ /* @p must still be on @dsq, dequeue */ -+ hmbird_cond_deferred_err(TASK_UNLINKED, -+ !task_linked_on_dsq(p), "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ } else { -+ /* -+ * We're racing against dispatch_to_local_dsq() which already -+ * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the -+ * holding_cpu which tells dispatch_to_local_dsq() that it lost -+ * the race. -+ */ -+ hmbird_cond_deferred_err(TASK_LINKED2, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ get_hmbird_ts(p)->holding_cpu = -1; -+ } -+ get_hmbird_ts(p)->dsq = NULL; -+ -+ if (!is_local) -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+ -+static bool test_rq_online(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ return rq->online; -+#else -+ return true; -+#endif -+} -+ -+static void refill_task_slice(struct task_struct *p) -+{ -+ if (is_isolate_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO; -+ else if (is_pipeline_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2; -+ else -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+} -+ -+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -+ int sticky_cpu) -+{ -+ struct hmbird_dispatch_q *d; -+ s32 cpu = -1; -+ -+ hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), -+ "task = %s\n", p->comm); -+ -+ if (is_pcp_rt(p)) { -+ /* Enqueue percpu rt task to local directly. */ -+ /* Or cause a bug when disable dispatch. */ -+ if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ } -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (cpu >= 0) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ -+ if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ /* rq migration */ -+ if (sticky_cpu == cpu_of(rq)) -+ goto local_norefill; -+ /* -+ * If !rq->online, we already told the BPF scheduler that the CPU is -+ * offline. We're just trying to on/offline the CPU. Don't bother the -+ * BPF scheduler. -+ */ -+ if (unlikely(!test_rq_online(rq))) -+ goto local; -+ -+ /* see %HMBIRD_OPS_ENQ_LAST */ -+ if (enq_flags & HMBIRD_ENQ_LAST) -+ goto local; -+ -+ if (enq_flags & HMBIRD_ENQ_LOCAL) -+ goto local; -+ else -+ goto global; -+local: -+ /* -+ * For task-ordering, slice refill must be treated as implying the end -+ * of the current slice. Otherwise, the longer @p stays on the CPU, the -+ * higher priority it becomes from hmbird_prio_less()'s POV. -+ */ -+ refill_task_slice(p); -+local_norefill: -+ dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags); -+ slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq)); -+ return; -+ -+global: -+ d = find_dsq_from_task(p); -+ if (d) { -+ refill_task_slice(p); -+ dispatch_enqueue(d, p, enq_flags); -+ return; -+ } -+ slim_stats_record(GLOBAL_STAT, 0, 0, 0); -+ refill_task_slice(p); -+ dispatch_enqueue(&hmbird_dsq_global, p, enq_flags); -+} -+ -+static bool watchdog_task_watched(const struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->watchdog_node); -+} -+ -+static void watchdog_watch_task(struct rq *rq, struct task_struct *p) -+{ -+ lockdep_assert_rq_held(rq); -+ if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ get_hmbird_ts(p)->runnable_at = jiffies; -+ clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list); -+} -+ -+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) -+{ -+ list_del_init(&get_hmbird_ts(p)->watchdog_node); -+ if (reset_timeout) -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags) -+{ -+ int sticky_cpu = get_hmbird_ts(p)->sticky_cpu; -+ -+ enq_flags |= get_hmbird_rq(rq)->extra_enq_flags; -+ -+ if (sticky_cpu >= 0) -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ -+ /* -+ * Restoring a running task will be immediately followed by -+ * set_next_task_hmbird() which expects the task to not be on the BPF -+ * scheduler as tasks can only start running through local DSQs. Force -+ * direct-dispatch into the local DSQ by setting the sticky_cpu. -+ */ -+ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -+ sticky_cpu = cpu_of(rq); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_UNWATCHED, -+ !watchdog_task_watched(p), "task = %s\n", p->comm); -+ return; -+ } -+ -+ watchdog_watch_task(rq, p); -+ set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ get_hmbird_rq(rq)->nr_running++; -+ add_nr_running(rq, 1); -+ -+ do_enqueue_task(rq, p, enq_flags, sticky_cpu); -+} -+ -+static void ops_dequeue(struct task_struct *p, u64 deq_flags) -+{ -+ u64 opss; -+ -+ watchdog_unwatch_task(p, false); -+ -+ /* acquire ensures that we see the preceding updates on QUEUED */ -+ opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state); -+ -+ switch (opss & HMBIRD_OPSS_STATE_MASK) { -+ case HMBIRD_OPSS_NONE: -+ break; -+ case HMBIRD_OPSS_QUEUEING: -+ /* -+ * QUEUEING is started and finished while holding @p's rq lock. -+ * As we're holding the rq lock now, we shouldn't see QUEUEING. -+ */ -+ hmbird_deferred_err(DEQ_DEQING, " : unreachable path in %s\n", __func__); -+ break; -+ case HMBIRD_OPSS_QUEUED: -+ if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss, -+ HMBIRD_OPSS_NONE)) -+ break; -+ fallthrough; -+ case HMBIRD_OPSS_DISPATCHING: -+ /* -+ * If @p is being dispatched from the BPF scheduler to a DSQ, -+ * wait for the transfer to complete so that @p doesn't get -+ * added to its DSQ after dequeueing is complete. -+ * -+ * As we're waiting on DISPATCHING with the rq locked, the -+ * dispatching side shouldn't try to lock the rq while -+ * DISPATCHING is set. See dispatch_to_local_dsq(). -+ * -+ * DISPATCHING shouldn't have qseq set and control can reach -+ * here with NONE @opss from the above QUEUED case block. -+ * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss. -+ */ -+ wait_ops_state(p, HMBIRD_OPSS_DISPATCHING); -+ hmbird_cond_deferred_err(HMBIRD_OPN, -+ atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE, -+ "task = %s\n", p->comm); -+ break; -+ } -+} -+ -+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ -+ if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p), -+ "task = %s\n", p->comm); -+ return; -+ } -+ -+ ops_dequeue(p, deq_flags); -+ -+ if (slim_walt_ctrl) { -+ if (task_current(rq, p)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ if (deq_flags & HMBIRD_DEQ_SLEEP) -+ set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else -+ clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), -+ (unsigned long *)&get_hmbird_ts(p)->flags); -+ -+ clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm); -+ hmbird_rq->nr_running--; -+ sub_nr_running(rq, 1); -+ dispatch_dequeue(hmbird_rq, p); -+} -+ -+static void yield_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ get_hmbird_ts(p)->slice = 0; -+} -+ -+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to) -+{ -+ return false; -+} -+ -+#ifdef CONFIG_SMP -+/** -+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -+ * @rq: rq to move the task into, currently locked -+ * @p: task to move -+ * @enq_flags: %HMBIRD_ENQ_* -+ * -+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller -+ * must: -+ * -+ * 1. Start with exclusive access to @p either through its DSQ lock or -+ * %HMBIRD_OPSS_DISPATCHING flag. -+ * -+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id(). -+ * -+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't -+ * deadlock with dequeue. -+ * -+ * 4. Lock @rq and the task_rq from #3. -+ * -+ * 5. Call this function. -+ * -+ * Returns %true if @p was successfully moved. %false after racing dequeue and -+ * losing. -+ */ -+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ struct rq *task_rq; -+ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * If dequeue got to @p while we were trying to lock both rq's, it'd -+ * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have -+ * updated it to different values afterwards, as this operation can't be -+ * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become -+ * raw_smp_processor_id() again before we're done. Thus, we can tell -+ * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is -+ * still raw_smp_processor_id(). -+ * -+ * See dispatch_dequeue() for the counterpart. -+ */ -+ if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id())) -+ return false; -+ -+ /* @p->rq couldn't have changed if we're still the holding cpu */ -+ task_rq = task_rq(p); -+ lockdep_assert_rq_held(task_rq); -+ deactivate_task(task_rq, p, 0); -+ set_task_cpu(p, cpu_of(rq)); -+ get_hmbird_ts(p)->sticky_cpu = cpu_of(rq); -+ -+ /* -+ * We want to pass hmbird-specific enq_flags but activate_task() will -+ * truncate the upper 32 bit. As we own @rq, we can pass them through -+ * @get_hmbird_rq(rq)->extra_enq_flags instead. -+ */ -+ hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags, -+ "task = %s\n", p->comm); -+ get_hmbird_rq(rq)->extra_enq_flags = enq_flags; -+ activate_task(rq, p, 0); -+ get_hmbird_rq(rq)->extra_enq_flags = 0; -+ -+ return true; -+} -+ -+#endif /* CONFIG_SMP */ -+ -+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu) -+{ -+ int fitable = 1; -+ -+ return fitable; -+} -+ -+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq, -+ struct hmbird_dispatch_q *dsq) -+{ -+ bool dsq_misfit; -+ int cpu = cpu_of(rq); -+ u64 task_util = 0; -+ struct cluster_ctx ctx; -+ int dsq_int = dsq_id_to_internal(dsq); -+ -+ if (!cpumask_test_cpu(cpu, iso_masks.little)) -+ return false; -+ if (p->pid == scx_systemui_pid) -+ return true; -+ -+ gen_cluster_ctx(&ctx, BIG); -+ dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 && -+ dsq_int <= SCHED_PROP_DEADLINE_LEVEL4); -+#ifdef CLUSTER_SEPARATE -+ /* In rescue mode, little will consume big cluster's dsq.*/ -+ dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper); -+#endif -+ if (!dsq_misfit) -+ return false; -+ -+ if (p) { -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &task_util); -+ else -+ task_util = get_hmbird_ts(p)->demand_scaled; -+ } -+ -+ if (task_util <= misfit_ds) -+ return false; -+ -+ hmbird_info_trace(":task %s can't run on cpu%d, util = %llu\n", -+ p->comm, cpu, task_util); -+ return true; -+} -+ -+static int check_misfit_task_on_fake_big(struct task_struct *p, struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (likely(p->pid != scx_systemui_pid)) -+ return false; -+ -+ if (topology_cluster_id(num_possible_cpus() - 1) > 1 && -+ arch_scale_cpu_capacity(cpu) == arch_scale_cpu_capacity(0) && -+ (parctrl_high_ratio <= 0 || parctrl_high_ratio_l <= 0)) -+ return true; -+ -+ return false; -+} -+ -+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq) -+{ -+ if (!cpumask_test_cpu(cpu_of(rq), task_cpu_possible_mask(p))) -+ return false; -+ -+ if (!task_fits_cpu_hmbird(p, cpu_of(rq))) -+ return false; -+ -+ if (check_misfit_task_on_little(p, rq, dsq)) -+ return false; -+ -+ if (check_misfit_task_on_fake_big(p, rq)) -+ return false; -+ -+ return likely(test_rq_online(rq)); -+} -+ -+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return; -+ -+ if (add) -+ skipn[idx]++; -+ else -+ skipn[idx] = 0; -+} -+ -+static bool skip_too_much(struct hmbird_dispatch_q *dsq) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return false; -+ -+ if (skip_num[idx] > 3) { -+ skip_num[idx] = 0; -+ return true; -+ } -+ -+ return false; -+} -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rb_node *rb_node; -+ struct rq *task_rq; -+ unsigned long flags; -+ bool moved = false; -+ struct task_struct *may_fit = NULL; -+ int skip = 0; -+ -+retry: -+ if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -+ return false; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) { -+ set_skip_num(dsq, skip_num, (bool)may_fit); -+ goto this_rq; -+ } -+ if (skip_too_much(dsq)) -+ goto remote_rq; -+ if (!may_fit) -+ may_fit = p; -+ if (++skip <= 3) -+ continue; -+ /* -+ * If the recent 3 tasks not fit, use the first one. -+ * and clear the skip, because the first one is consumed. -+ */ -+ set_skip_num(dsq, skip_num, false); -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ /* No more task, use the first may fit task.*/ -+ if (may_fit) { -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ -+ for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) -+ goto this_rq; -+ goto remote_rq; -+ } -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ return false; -+ -+this_rq: -+ /* @dsq is locked and @p is on this rq */ -+ hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo); -+ dsq->nr--; -+ hmbird_rq->local_dsq.nr++; -+ get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ -+remote_rq: -+#ifdef CONFIG_SMP -+ if (cpu_same_cluster_stat(p, rq, task_rq)) -+ slim_stats_record(MOVE_RQ_CNT, 0, 0, 0); -+ else -+ slim_stats_record(MOVE_RQ_CNT, 1, 0, 0); -+ /* -+ * @dsq is locked and @p is on a remote rq. @p is currently protected by -+ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -+ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -+ * rq lock or fail, do a little dancing from our side. See -+ * move_task_to_local_dsq(). -+ */ -+ hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id(); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ rq_unpin_lock(rq, rf); -+ double_lock_balance(rq, task_rq); -+ rq_repin_lock(rq, rf); -+ -+ moved = move_task_to_local_dsq(rq, p, 0); -+ -+ double_unlock_balance(rq, task_rq); -+#endif /* CONFIG_SMP */ -+ if (likely(moved)) { -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ } -+ may_fit = NULL; -+ goto retry; -+} -+ -+ -+static int balance_one(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf, bool local) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class; -+ -+ if (!hmbird_rq) -+ return 1; -+ -+ lockdep_assert_rq_held(rq); -+ -+ if (static_branch_unlikely(&hmbird_ops_cpu_preempt) && -+ unlikely(get_hmbird_rq(rq)->cpu_released)) { -+ /* -+ * If the previous sched_class for the current CPU was not HMBIRD, -+ * notify the BPF scheduler that it again has control of the -+ * core. This callback complements ->cpu_release(), which is -+ * emitted in hmbird_notify_pick_next_task(). -+ */ -+ get_hmbird_rq(rq)->cpu_released = false; -+ } -+ -+ if (prev_on_hmbird) -+ update_curr_hmbird(rq); -+ /* if there already are tasks to run, nothing to do */ -+ if (hmbird_rq->local_dsq.nr) -+ return 1; -+ -+ if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) { -+ slim_stats_record(GLOBAL_STAT, 1, 0, 0); -+ return 1; -+ } -+ -+ if (consume_dispatch_global(rq, rf)) -+ return 1; -+ -+ return 0; -+} -+ -+static int balance_hmbird(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf) -+{ -+ return balance_one(rq, prev, rf, true); -+} -+ -+/* -+ * output task util to systrace, only for debug mode. -+ * we can not output too many logs to systrace buffer even in debug mode -+ * only output debug-info while it exceed misfit_ds. -+ */ -+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p) -+{ -+ static DEFINE_PER_CPU(int, is_last_exceed); -+ int cpu = cpu_of(rq); -+ u64 util = 0; -+ -+ if (likely(!debug_enabled())) -+ return; -+ -+ if (!p) -+ return; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ util = uclamp_rq_util_with(rq, util, p); -+ -+ if (util >= misfit_ds) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util); -+ per_cpu(is_last_exceed, cpu) = true; -+ } else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0); -+ per_cpu(is_last_exceed, cpu) = false; -+ } else { -+ } -+} -+ -+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first) -+{ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ /* -+ * Core-sched might decide to execute @p before it is -+ * dispatched. Call ops_dequeue() to notify the BPF scheduler. -+ */ -+ ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC); -+ dispatch_dequeue(get_hmbird_rq(rq), p); -+ } -+ -+ p->se.exec_start = rq_clock_task(rq); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK); -+ } -+ -+ watchdog_unwatch_task(p, true); -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx); -+ systrace_output_cpu_ds(rq, p); -+ /* -+ * @p is getting newly scheduled or got kicked after someone updated its -+ * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird(). -+ */ -+ if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) != -+ (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) { -+ if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) -+ get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK; -+ else -+ get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK; -+ -+ sched_update_tick_dependency(rq); -+ } -+ -+ p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime; -+} -+ -+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p) -+{ -+ update_curr_hmbird(rq); -+ -+ update_dispatch_dsq_info(rq, p); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ watchdog_watch_task(rq, p); -+ -+ if (is_pipeline_task(p)) { -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1); -+ return; -+ } -+ -+ /* -+ * If we're in the pick_next_task path, balance_hmbird() should -+ * have already populated the local DSQ if there are any other -+ * available tasks. If empty, tell ops.enqueue() that @p is the -+ * only one available for this cpu. ops.enqueue() should put it -+ * on the local DSQ so that the subsequent pick_next_task_hmbird() -+ * can find the task unless it wants to trigger a separate -+ * follow-up scheduling event. -+ */ -+ if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1); -+ else -+ do_enqueue_task(rq, p, 0, -1); -+ } -+} -+ -+static struct task_struct *first_local_task(struct rq *rq) -+{ -+ struct rb_node *rb_node; -+ struct hmbird_entity *entity; -+ -+ if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) { -+ entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo, -+ struct hmbird_entity, dsq_node.fifo); -+ return entity->task; -+ } -+ -+ rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq); -+ if (rb_node) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ return entity->task; -+ } -+ return NULL; -+} -+ -+static struct task_struct *pick_next_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p; -+ -+ p = first_local_task(rq); -+ if (!p) -+ return NULL; -+ -+ if (unlikely(!get_hmbird_ts(p)->slice)) { -+ if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice) -+ hmbird_warned_zero_slice = true; -+ -+ refill_task_slice(p); -+ } -+ -+ set_next_task_hmbird(rq, p, true); -+ -+ return p; -+} -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task, -+ const struct sched_class *active) -+{ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * The callback is conceptually meant to convey that the CPU is no -+ * longer under the control of HMBIRD. Therefore, don't invoke the -+ * callback if the CPU is staying on HMBIRD, or going idle (in which -+ * case the HMBIRD scheduler has actively decided not to schedule any -+ * tasks on the CPU). -+ */ -+ if (likely(active >= &hmbird_sched_class)) -+ return; -+ -+ /* -+ * At this point we know that HMBIRD was preempted by a higher priority -+ * sched_class, so invoke the ->cpu_release() callback if we have not -+ * done so already. We only send the callback once between HMBIRD being -+ * preempted, and it regaining control of the CPU. -+ * -+ * ->cpu_release() complements ->cpu_acquire(), which is emitted the -+ * next time that balance_hmbird() is invoked. -+ */ -+ if (!get_hmbird_rq(rq)->cpu_released) -+ get_hmbird_rq(rq)->cpu_released = true; -+} -+ -+#ifdef CONFIG_SMP -+ -+static bool test_and_clear_cpu_idle(int cpu) -+{ -+ if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -+ if (cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ return true; -+ } else { -+ return false; -+ } -+} -+ -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) -+{ -+ int cpu; -+ -+ do { -+ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -+ if (cpu >= nr_cpu_ids) -+ return -EBUSY; -+ } while (!test_and_clear_cpu_idle(cpu)); -+ -+ return cpu; -+} -+ -+ -+static bool prev_cpu_misfit(int prev) -+{ -+ if (!is_partial_enabled() && is_partial_cpu(prev)) -+ return true; -+ -+ return false; -+} -+ -+static int heavy_rt_placement(struct task_struct *p, int prev) -+{ -+ struct cpumask tmp = {.bits = {0}}; -+ int cpu; -+ u64 util = 0; -+ -+ if (!rt_prio(p->prio)) -+ return -EFAULT; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ -+ if (util < misfit_ds) -+ return -EFAULT; -+ -+ if (is_partial_enabled()) -+ cpumask_or(&tmp, iso_masks.big, iso_masks.partial); -+ else -+ cpumask_copy(&tmp, iso_masks.big); -+ -+ cpu = hmbird_pick_idle_cpu(&tmp); -+ if (cpu >= 0) -+ return cpu; -+ -+ if (cpumask_test_cpu(prev, iso_masks.big) || -+ (is_partial_enabled() && is_partial_cpu(prev))) -+ return prev; -+ -+ return cpumask_first(&tmp); -+} -+ -+static int spec_task_before_pick_idle(struct task_struct *p, int prev) -+{ -+ int cpu; -+ -+ cpu = heavy_rt_placement(p, prev); -+ if (cpu >= 0) -+ return cpu; -+ return -EFAULT; -+} -+ -+static int cpumask_distribute_next(struct cpumask *mask, int *prev) -+{ -+ int p = *prev, n; -+ -+ n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1); -+ if (n < nr_cpu_ids) -+ WRITE_ONCE(*prev, n); -+ return n; -+} -+ -+static int repick_fallback_cpu(void) -+{ -+ /* -+ * partial cpu follow big cluster's Scheduling policy, -+ * simply return first bit cpu. -+ */ -+ return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev); -+} -+ -+/* -+ * Must return a valid cpu num, as this task's cpu. -+ */ -+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu, -+ struct cpumask *mask, int *prev_mask) -+{ -+ int cpu; -+ -+ cpu = hmbird_pick_idle_cpu(mask); -+ if (cpu >= 0) -+ return cpu; -+ return cpumask_distribute_next(mask, prev_mask); -+} -+ -+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type) -+{ -+ int idx; -+ struct cluster_ctx ctx; -+ -+ idx = find_idx_from_task(p); -+ if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS) -+ return false; -+ -+ gen_cluster_ctx(&ctx, type); -+ if (idx >= ctx.lower && idx < ctx.upper) -+ return true; -+ return false; -+} -+ -+static bool is_valid_cpu(int cpu) -+{ -+ return (cpu >= 0) && (cpu < nr_cpu_ids); -+} -+ -+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) -+{ -+ s32 cpu = -1; -+ struct cpumask mask = {.bits = {0}}; -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (is_valid_cpu(cpu)) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ partial_dynamic_ctrl(); -+ -+ if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ iso_masks.big, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ if (p->nr_cpus_allowed == 1) { -+ cpu = cpumask_any(p->cpus_ptr); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* For non-period global dsq, not contain pcp task. */ -+ if (task_only_blongs_to_cluster(p, LITTLE)) { -+ cpumask_copy(&mask, iso_masks.little); -+ if (unlikely(l_need_rescue)) { -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ cpumask_or(&mask, iso_masks.ex_free, &mask); -+ } -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &little_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ if (task_only_blongs_to_cluster(p, BIG)) { -+ cpumask_copy(&mask, iso_masks.big); -+ if (unlikely(b_need_rescue)) { -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ cpumask_or(&mask, iso_masks.ex_free, &mask); -+ } -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ cpu = spec_task_before_pick_idle(p, prev_cpu); -+ if (is_valid_cpu(cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* if the previous CPU is idle, dispatch directly to it */ -+ if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+ } -+ -+ cpu = hmbird_pick_idle_cpu(cpu_possible_mask); -+ if (is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ if (prev_cpu_misfit(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ cpu = repick_fallback_cpu(); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+} -+ -+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags) -+{ -+ return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags); -+} -+ -+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx) -+{ -+ set_cpus_allowed_common(p, ctx); -+} -+ -+static void reset_idle_masks(void) -+{ -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little); -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big); -+ if (is_partial_enabled()) -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial); -+ hmbird_has_idle_cpus = true; -+} -+ -+void __hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ int cpu = cpu_of(rq); -+ struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -+ -+ if (skip_update_idle()) -+ return; -+ -+ if (idle) { -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ if (!hmbird_has_idle_cpus) -+ hmbird_has_idle_cpus = true; -+ -+ /* -+ * idle_masks.smt handling is racy but that's fine as it's only -+ * for optimization and self-correcting. -+ */ -+ for_each_cpu(cpu, sib_mask) { -+ if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -+ return; -+ } -+ cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -+ } else { -+ cpumask_clear_cpu(cpu, idle_masks.cpu); -+ if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ -+ cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -+ } -+} -+ -+#else /* !CONFIG_SMP */ -+ -+static bool test_and_clear_cpu_idle(int cpu) { return false; } -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } -+static void reset_idle_masks(void) {} -+ -+#endif /* CONFIG_SMP */ -+ -+static bool check_rq_for_timeouts(struct rq *rq) -+{ -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rq_flags rf; -+ -+ rq_lock_irqsave(rq, &rf); -+ list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) { -+ unsigned long last_runnable; -+ -+ p = entity->task; -+ last_runnable = get_hmbird_ts(p)->runnable_at; -+ -+ if (unlikely(time_after(jiffies, -+ last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -+ -+ rq_unlock_irqrestore(rq, &rf); -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb", -+ p->comm, p->pid, -+ dur_ms / 1000, dur_ms % 1000, -+ get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0, -+ cpumask_pr_args(&p->cpus_mask)); -+ return 1; -+ } -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ return 0; -+} -+ -+static void hmbird_watchdog_workfn(struct work_struct *work) -+{ -+ int cpu; -+ bool timeout; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ -+ for_each_online_cpu(cpu) { -+ timeout = check_rq_for_timeouts(cpu_rq(cpu)); -+ if (unlikely(timeout)) -+ break; -+ -+ cond_resched(); -+ } -+ if (!timeout) -+ queue_delayed_work(system_unbound_wq, to_delayed_work(work), -+ hmbird_watchdog_timeout / 2); -+} -+ -+static void set_pcp_round(struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) { -+ per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round); -+ per_cpu(pcp_info, cpu).pcp_round = true; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+} -+ -+ -+/* -+ * Just for debug: output hmbird on/off state per 10s. -+ */ -+#define OUTPUT_INTVAL (msecs_to_jiffies(10 * 1000)) -+static void inform_hmbird_onoff_from_systrace(void) -+{ -+ static unsigned long __read_mostly next_print; -+ -+ if (time_before(jiffies, READ_ONCE(next_print))) -+ return; -+ -+ WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL); -+ hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio); -+} -+ -+void hmbird_notify_sched_tick(void) -+{ -+ unsigned long last_check; -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ hmbird_scheduler_tick(); -+ -+ if (!hmbird_enabled()) -+ return; -+ -+ last_check = hmbird_watchdog_timestamp; -+ if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -+ -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "watchdog failed to check in for %u.%03us", -+ dur_ms / 1000, dur_ms % 1000); -+ } -+ scan_timeout(rq); -+} -+ -+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued) -+{ -+ update_curr_hmbird(rq); -+ -+ set_pcp_round(rq); -+ -+ if (slim_walt_ctrl) -+ hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE); -+ /* -+ * While disabling, always resched and refresh core-sched timestamp as -+ * we can't trust the slice management or ops.core_sched_before(). -+ */ -+ if (hmbird_ops_disabling()) -+ get_hmbird_ts(curr)->slice = 0; -+ -+ if (!get_hmbird_ts(curr)->slice) -+ resched_curr(rq); -+ -+ inform_hmbird_onoff_from_systrace(); -+} -+ -+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg) -+{ -+ hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ get_hmbird_ts(p)->disallow = false; -+ -+ hmbird_sched_init_task(p); -+ -+ set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ return 0; -+} -+ -+static void hmbird_ops_enable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void hmbird_ops_disable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void set_task_hmbird_weight(struct task_struct *p) -+{ -+ u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -+ -+ get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight); -+} -+ -+/** -+ * refresh_hmbird_weight - Refresh a task's hmbird weight -+ * @p: task to refresh hmbird weight for -+ * -+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to -+ * enable easy access from the BPF scheduler. To keep it synchronized with the -+ * current task priority, this function should be called when a new task is -+ * created, priority is changed for a task on hmbird, and a task is switched -+ * to hmbird from other classes. -+ */ -+static void refresh_hmbird_weight(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ set_task_hmbird_weight(p); -+} -+ -+int hmbird_pre_fork(struct task_struct *p) -+{ -+ int ret = 0; -+ -+ p->android_oem_data1[HMBIRD_TS_IDX] = -+ (u64)(kzalloc(sizeof(struct hmbird_entity), GFP_KERNEL)); -+ if (!get_hmbird_ts(p)) -+ return -1; -+ -+ get_hmbird_ts(p)->dsq = NULL; -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node); -+ get_hmbird_ts(p)->flags = 0; -+ get_hmbird_ts(p)->weight = 0; -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ get_hmbird_ts(p)->holding_cpu = -1; -+ get_hmbird_ts(p)->kf_mask = 0; -+ atomic64_set(&get_hmbird_ts(p)->ops_state, 0); -+ get_hmbird_ts(p)->runnable_at = INITIAL_JIFFIES; -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+ get_hmbird_ts(p)->task = p; -+ hmbird_set_sched_prop(p, 0); -+ -+ get_hmbird_ts(p)->critical_affinity_cpu = -1; -+ get_hmbird_ts(p)->sched_class = &hmbird_sched_class; -+ get_hmbird_ts(p)->tick_hit_count = 0; -+ get_hmbird_ts(p)->start_jiffies = 0; -+ -+ /* -+ * BPF scheduler enable/disable paths want to be able to iterate and -+ * update all tasks which can become complex when racing forks. As -+ * enable/disable are very cold paths, let's use a percpu_rwsem to -+ * exclude forks. -+ */ -+ -+ return ret; -+} -+ -+void hmbird_post_fork(struct task_struct *p) -+{ -+ percpu_down_read(&hmbird_fork_rwsem); -+ -+ if (hmbird_enabled()) { -+ struct rq_flags rf; -+ struct rq *rq; -+ p->sched_class = &hmbird_sched_class; -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ rq = task_rq_lock(p, &rf); -+ /* -+ * Set the weight manually before calling ops.enable() so that -+ * the scheduler doesn't see a stale value if they inspect the -+ * task struct. We'll invoke ops.set_weight() afterwards, as it -+ * would be odd to receive a callback on the task before we -+ * tell the scheduler that it's been fully enabled. -+ */ -+ set_task_hmbird_weight(p); -+ hmbird_ops_enable_task(p); -+ refresh_hmbird_weight(p); -+ task_rq_unlock(rq, p, &rf); -+ } else { -+ if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ } -+ -+ spin_lock_irq(&hmbird_tasks_lock); -+ list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_cancel_fork(struct task_struct *p) -+{ -+ if (hmbird_enabled()) -+ hmbird_ops_disable_task(p); -+ put_hmbird_ts(p); -+} -+ -+void hmbird_free(struct task_struct *p) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+ list_del_init(&get_hmbird_ts(p)->tasks_node); -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+ -+ /* -+ * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED -> -+ * ENABLED transitions can't race us. Disable ops for @p. -+ */ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) || -+ test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ hmbird_ops_disable_task(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ put_hmbird_ts(p); -+} -+ -+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio) -+{ -+} -+ -+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type) -+{ -+ return (prop >> TOP_TASK_SHIFT) & (1 << type); -+} -+ -+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p) -+{ -+ uint32_t prop = get_top_task_prop(p); -+ -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE)) -+ return HMBIRD_TASK_PROP_PIPELINE; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) || -+ !task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) { -+ return HMBIRD_TASK_PROP_COMMON; -+ } -+ return HMBIRD_TASK_PROP_DEBUG_OR_LOG; -+} -+ -+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b) -+{ -+ int type_a = hmbird_get_task_type(a); -+ int type_b = hmbird_get_task_type(b); -+ -+ return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b]; -+} -+ -+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ enum cpu_type type; -+ int sp_dl; -+ struct task_struct *curr = NULL; -+ -+ switch (hmbird_preempt_policy) { -+ case HMBIRD_PREEMPT_POLICY_PRIO_BASED: -+ curr = rq->curr; -+ if (curr && hmbird_prio_higher(p, curr)) -+ goto preempt; -+ break; -+ default: -+ break; -+ } -+ if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) || -+ (is_critical_system_task(p) && !is_critical_system_task(rq->curr))) -+ goto preempt; -+ -+ sp_dl = find_idx_from_task(p); -+ if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled())) -+ return; -+ -+ if (rq->curr->prio > p->prio) -+ goto preempt; -+ -+ return; -+ -+preempt: -+ resched_curr(rq); -+} -+ -+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {} -+ -+int hmbird_check_setscheduler(struct task_struct *p, int policy) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ /* if disallow, reject transitioning into HMBIRD */ -+ if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) && -+ p->policy != policy && policy == SCHED_HMBIRD) -+ return -EACCES; -+ -+ return 0; -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool hmbird_can_stop_tick(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ if (hmbird_ops_disabling()) -+ return false; -+ -+ if (p->sched_class != &hmbird_sched_class) -+ return true; -+ -+ return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK; -+} -+#endif -+ -+int hmbird_tg_online(struct task_group *tg) -+{ -+ struct cgroup *cgrp; -+ -+ if (!tg) -+ return 0; -+ cgrp = tg->css.cgroup; -+ if (!cgrp || !(cgrp->kn)) -+ return 0; -+ update_cgroup_ids_table(cgrp->kn->id, -1); -+ return 0; -+} -+ -+/* -+ * Omitted operations: -+ * -+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -+ * task isn't tied to the CPU at that point. Preemption is implemented by -+ * resetting the victim task's slice to 0 and triggering reschedule on the -+ * target CPU. -+ * -+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -+ * -+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -+ * their current sched_class. Call them directly from sched core instead. -+ * -+ * - task_woken, switched_from: Unnecessary. -+ */ -+DEFINE_SCHED_CLASS(hmbird) = { -+ .enqueue_task = enqueue_task_hmbird, -+ .dequeue_task = dequeue_task_hmbird, -+ .yield_task = yield_task_hmbird, -+ .yield_to_task = yield_to_task_hmbird, -+ -+ .check_preempt_curr = check_preempt_curr_hmbird, -+ -+ .pick_next_task = pick_next_task_hmbird, -+ -+ .put_prev_task = put_prev_task_hmbird, -+ .set_next_task = set_next_task_hmbird, -+ -+#ifdef CONFIG_SMP -+ .balance = balance_hmbird, -+ .select_task_rq = select_task_rq_hmbird, -+ .set_cpus_allowed = set_cpus_allowed_hmbird, -+#endif -+ .task_tick = task_tick_hmbird, -+ -+ .switched_to = switched_to_hmbird, -+ .prio_changed = prio_changed_hmbird, -+ -+ .update_curr = update_curr_hmbird, -+ -+#ifdef CONFIG_UCLAMP_TASK -+ .uclamp_enabled = 0, -+#endif -+}; -+ -+/* -+ * Must with rq lock held. -+ */ -+bool task_is_hmbird(struct task_struct *p) -+{ -+ return p->sched_class == &hmbird_sched_class; -+} -+ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id) -+{ -+ memset(dsq, 0, sizeof(*dsq)); -+ -+ raw_spin_lock_init(&dsq->lock); -+ INIT_LIST_HEAD(&dsq->fifo); -+ dsq->id = dsq_id; -+} -+ -+static int hmbird_cgroup_init(void) -+{ -+ struct cgroup_subsys_state *css; -+ -+ css_for_each_descendant_pre(css, &root_task_group.css) { -+ struct task_group *tg = css_tg(css); -+ -+ cgrp_dsq_idx_init(css->cgroup, tg); -+ } -+ return 0; -+} -+ -+/* -+ * Used by sched_fork() and __setscheduler_prio() to pick the matching -+ * sched_class. dl/rt are already handled. -+ */ -+bool task_on_hmbird(struct task_struct *p) -+{ -+ return hmbird_enabled(); -+} -+ -+static void __setscheduler_prio(struct task_struct *p, int prio) -+{ -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) { -+ p->prio = prio; -+ return; -+ } else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (on_hmbird && rt_prio(prio)) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ -+ p->prio = prio; -+} -+ -+/* -+ * Heartbeat, avoid humbird keep running while APP already exit. -+ * Check whether APP send alive-signal periodly. -+ */ -+#define HEARTBEAT_TIMEOUT (msecs_to_jiffies(2500)) -+#define HEARTBEAT_CHECK_INTERVAL (msecs_to_jiffies(1000)) -+static struct timer_list hb_timer; -+static unsigned long next_hb; -+ -+void hb_timer_handler(struct timer_list *timer) -+{ -+ if (!heartbeat_enable) -+ goto refill; -+ -+ pr_info(": enter timer.\n"); -+ if (heartbeat) { -+ heartbeat = 0; -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ } -+ -+ /* can't detect heartbeat, disable ext. */ -+ if (time_after(jiffies, READ_ONCE(next_hb))) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT, -+ "can't detect heartbeat, disable ext\n"); -+ } -+refill: -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_start(void) -+{ -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_init(void) -+{ -+ timer_setup(&hb_timer, hb_timer_handler, 0); -+} -+ -+static void hb_timer_exit(void) -+{ -+ del_timer(&hb_timer); -+} -+ -+static bool check_and_disable_cpuhp(void) -+{ -+ struct rq *rq; -+ struct rq_flags rf; -+ int cpu; -+ -+ cpu_hotplug_disable(); -+ cpus_read_lock(); -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ rq_lock_irqsave(rq, &rf); -+ if (!rq->online) { -+ rq_unlock_irqrestore(rq, &rf); -+ goto err_offline; -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ cpus_read_unlock(); -+ return true; -+ -+err_offline: -+ cpus_read_unlock(); -+ cpu_hotplug_enable(); -+ return false; -+} -+ -+static void reenable_cpuhp(void) -+{ -+ cpu_hotplug_enable(); -+} -+ -+static void scheduler_switch_done(bool final_state) -+{ -+ /* set scx_enable again, switch may fail. */ -+ scx_enable = final_state; -+ /* reset heartbeat*/ -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ -+ if (final_state) -+ hb_timer_start(); -+ else -+ hb_timer_exit(); -+} -+ -+/** -+ * ss : curr switch state -+ * finish : success or fail -+ * enable : curr operation is diabling or enabling? -+ * fail_reason : string output when failed, empty string when success. -+ */ -+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason) -+{ -+ char *s1 = finish ? "finished" : "failed"; -+ char *s2 = enable ? "enabled" : "disabled"; -+ -+ hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss); -+ if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) { -+ sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type)); -+ hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n", -+ s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason); -+ } -+ curr_ss = ss; -+} -+ -+bool get_hmbird_ops_enabled(void) -+{ -+ return atomic_read(&__hmbird_ops_enabled); -+} -+ -+bool get_non_hmbird_task(void) -+{ -+ return atomic_read(&non_hmbird_task); -+} -+ -+static void hmbird_ops_disable_workfn(struct kthread_work *work) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int cpu; -+ -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, ""); -+ cancel_delayed_work_sync(&hmbird_watchdog_work); -+ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) { -+ case HMBIRD_OPS_DISABLED: -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled"); -+ scheduler_switch_done(false); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return; -+ case HMBIRD_OPS_PREPPING: -+ fallthrough; -+ case HMBIRD_OPS_DISABLING: -+ /* shouldn't happen but handle it like ENABLING if it does */ -+ WARN_ONCE(true, "hmbird: duplicate disabling instance?"); -+ fallthrough; -+ case HMBIRD_OPS_ENABLING: -+ case HMBIRD_OPS_ENABLED: -+ break; -+ } -+ -+ /* kick all CPUs to restore ticks */ -+ for_each_possible_cpu(cpu) -+ resched_cpu(cpu); -+ -+ /* avoid racing against fork and cgroup changes */ -+ cpus_read_lock(); -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, ""); -+ spin_lock_irq(&hmbird_tasks_lock); -+ atomic_set(&__hmbird_ops_enabled, false); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ bool alive = READ_ONCE(p->__state) != TASK_DEAD; -+ -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ get_hmbird_ts(p)->slice = min_t(u64, -+ get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL); -+ -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ if (alive) -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ -+ hmbird_ops_disable_task(p); -+ } -+ hmbird_task_iter_exit(&sti); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, ""); -+ -+ atomic_set(&non_hmbird_task, true); -+ /* no task is on hmbird, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt); -+ synchronize_rcu(); -+ -+ percpu_up_write(&hmbird_fork_rwsem); -+ cpus_read_unlock(); -+ -+ if (slim_walt_ctrl) -+ slim_walt_enable(false); -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ -+ hmbird_switch_log(HMBIRD_DISABLED, 1, 0, ""); -+ scheduler_switch_done(false); -+ reenable_cpuhp(); -+} -+ -+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn); -+ -+static void schedule_hmbird_ops_disable_work(void) -+{ -+ struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper); -+ -+ /* -+ * We may be called spuriously before the first bpf_hmbird_reg(). If -+ * hmbird_ops_helper isn't set up yet, there's nothing to do. -+ */ -+ if (helper) -+ kthread_queue_work(helper, &hmbird_ops_disable_work); -+} -+ -+static void hmbird_ops_disable(void) -+{ -+ schedule_hmbird_ops_disable_work(); -+} -+ -+static void hmbird_err_exit_workfn(struct work_struct *work) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ hmbird_ctrl(false); -+ -+ for_each_present_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy) -+ continue; -+ if (cpu != policy->cpu) -+ goto put; -+ down_write(&policy->rwsem); -+ WARN_ON(store_scaling_governor(policy, -+ saved_gov[cpu], strlen(saved_gov[cpu])) <= 0); -+ up_write(&policy->rwsem); -+ hmbird_info_trace(":restore origin gov : %s\n", saved_gov[cpu]); -+put: -+ cpufreq_cpu_put(policy); -+ } -+ memset((char *)saved_gov, 0, sizeof(saved_gov)); -+} -+ -+void hmbird_ops_exit(void) -+{ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); -+} -+ -+static struct kthread_worker *hmbird_create_rt_helper(const char *name) -+{ -+ struct kthread_worker *helper; -+ -+ helper = kthread_create_worker(KTW_FREEZABLE, name); -+ if (helper) -+ sched_set_fifo(helper->task); -+ return helper; -+} -+ -+static inline void set_audio_thread_sched_prop(struct task_struct *p) -+{ -+ struct cgroup_subsys_state *css; -+ -+ if (likely(p->prio >= MAX_RT_PRIO)) -+ return; -+ -+ rcu_read_lock(); -+ css = task_css(p, cpuset_cgrp_id); -+ if (!css) { -+ rcu_read_unlock(); -+ return; -+ } -+ rcu_read_unlock(); -+ -+ if (!strcmp(css->cgroup->kn->name, "audio-app")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+} -+ -+int scx_systemui_pid = -1; -+void set_systemui_thread_pid(struct task_struct *p) -+{ -+ if ((strcmp(p->comm, "ndroid.systemui") == 0) && (p->pid == p->tgid)) -+ scx_systemui_pid = p->pid; -+} -+ -+static int hmbird_ops_enable(void *unused) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int ret; -+ int tcnt = 0; -+ unsigned long long start = 0; -+ -+ if (!check_and_disable_cpuhp()) { -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline"); -+ return -EBUSY; -+ } -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, ""); -+ mutex_lock(&hmbird_ops_enable_mutex); -+ -+ if (!hmbird_ops_helper) { -+ WRITE_ONCE(hmbird_ops_helper, -+ hmbird_create_rt_helper("hmbird_ops_helper")); -+ if (!hmbird_ops_helper) { -+ ret = -ENOMEM; -+ goto err_unlock; -+ } -+ } -+ -+ if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) { -+ ret = -EBUSY; -+ goto err_unlock; -+ } -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) != -+ HMBIRD_OPS_DISABLED); -+ -+ hmbird_warned_zero_slice = false; -+ -+ atomic64_set(&hmbird_nr_rejected, 0); -+ -+ /* -+ * Keep CPUs stable during enable so that the BPF scheduler can track -+ * online CPUs by watching ->on/offline_cpu() after ->init(). -+ */ -+ cpus_read_lock(); -+ -+ hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work, -+ hmbird_watchdog_timeout / 2); -+ -+ /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ */ -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ reset_idle_masks(); -+ -+ /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. -+ */ -+ ret = hmbird_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock; -+ -+ /* -+ * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem -+ * preventing new tasks from being added. No need to exclude tasks -+ * leaving as hmbird_free() can handle both prepped and enabled -+ * tasks. Prep all tasks first and then enable them with preemption -+ * disabled. -+ */ -+ spin_lock_irq(&hmbird_tasks_lock); -+ -+ atomic_set(&non_hmbird_task, false); -+ atomic_set(&__hmbird_ops_enabled, true); -+ -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered(&sti))) -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ hmbird_task_iter_exit(&sti); -+ -+ /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. -+ */ -+ preempt_disable(); -+ -+ /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ */ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) { -+ atomic_set(&non_hmbird_task, true); -+ atomic_set(&__hmbird_ops_enabled, false); -+ preempt_enable(); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ ret = -EBUSY; -+ goto err_disable_unlock; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED -+ * transitions here are synchronized against hmbird_free() through -+ * hmbird_tasks_lock. -+ */ -+ start = sched_clock(); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, ""); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ tcnt++; -+ if (READ_ONCE(p->__state) != TASK_DEAD) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ -+ set_audio_thread_sched_prop(p); -+ set_systemui_thread_pid(p); -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ hmbird_ops_enable_task(p); -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ } else { -+ hmbird_ops_disable_task(p); -+ } -+ } -+ hmbird_task_iter_exit(&sti); -+ -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, ""); -+ preempt_enable(); -+ percpu_up_write(&hmbird_fork_rwsem); -+ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) { -+ ret = -EBUSY; -+ goto err_disable; -+ } -+ -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_ENABLED, 1, 1, ""); -+ scheduler_switch_done(true); -+ -+ return 0; -+ -+err_unlock: -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock"); -+ scheduler_switch_done(false); -+ return ret; -+ -+err_disable_unlock: -+ percpu_up_write(&hmbird_fork_rwsem); -+err_disable: -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ /* must be fully disabled before returning */ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable"); -+ scheduler_switch_done(false); -+ return ret; -+} -+ -+#ifdef CONFIG_SCHED_DEBUG -+static const char *hmbird_ops_enable_state_str[] = { -+ [HMBIRD_OPS_PREPPING] = "prepping", -+ [HMBIRD_OPS_ENABLING] = "enabling", -+ [HMBIRD_OPS_ENABLED] = "enabled", -+ [HMBIRD_OPS_DISABLING] = "disabling", -+ [HMBIRD_OPS_DISABLED] = "disabled", -+}; -+ -+static int hmbird_debug_show(struct seq_file *m, void *v) -+{ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled()); -+ seq_printf(m, "%-30s: %s\n", "enable_state", -+ hmbird_ops_enable_state_str[hmbird_ops_enable_state()]); -+ seq_printf(m, "%-30s: %llu\n", "nr_rejected", -+ atomic64_read(&hmbird_nr_rejected)); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return 0; -+} -+ -+static int hmbird_debug_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_debug_show, NULL); -+} -+ -+const struct file_operations sched_hmbird_fops = { -+ .open = hmbird_debug_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif -+ -+ -+static int bpf_hmbird_reg(void *kdata) -+{ -+ return hmbird_ops_enable(kdata); -+} -+ -+static int bpf_hmbird_unreg(void *kdata) -+{ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ return 0; -+} -+ -+void set_hmbird_module_loaded(int is_loaded) -+{ -+ atomic_set(&hmbird_module_loaded, is_loaded); -+} -+ -+/* -+ * MUST load hmbird module before enable hmbird scheduler -+ * load track & hmbird gover implemented in hmbird module -+ */ -+int hmbird_ctrl(bool enable) -+{ -+ if (!atomic_read(&hmbird_module_loaded)) { -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "ext module unloaded\n"); -+ return -EINVAL; -+ } -+ -+ if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED -+ || hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING -+ || hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already enabled(ing)\n"); -+ hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING || -+ hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already disabled(ing)\n"); -+ hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (enable) -+ return bpf_hmbird_reg(NULL); -+ else -+ return bpf_hmbird_unreg(NULL); -+} -+ -+void set_cpu_isomask(int cpu, cpumask_var_t *mask) -+{ -+ cpumask_clear_cpu(cpu, iso_masks.ex_free); -+ cpumask_clear_cpu(cpu, iso_masks.exclusive); -+ cpumask_clear_cpu(cpu, iso_masks.partial); -+ cpumask_clear_cpu(cpu, iso_masks.big); -+ cpumask_clear_cpu(cpu, iso_masks.little); -+ cpumask_set_cpu(cpu, *mask); -+} -+ -+void set_cpu_cluster(u64 cpu_cluster) -+{ -+ int cpu; -+ -+ for_each_present_cpu(cpu) { -+ u64 pos = 1 << cpu; -+ -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.ex_free)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.exclusive)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.partial)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.big)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) -+ set_cpu_isomask(cpu, &(iso_masks.little)); -+ } -+} -+ -+static void get_hmbird_snapshot(struct panic_snapshot_t *p) -+{ -+ struct hmbird_dispatch_q *dsq; -+ struct rq *rq; -+ int cpu, i; -+ -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ p->rq_nr[cpu] = rq->nr_running; -+ p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running; -+ } -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ struct hmbird_entity *entity; -+ -+ dsq = &gdsqs[i]; -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo)) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ p->runnable_at[i] = entity->runnable_at; -+ raw_spin_unlock(&dsq->lock); -+ -+ } -+ -+ p->snap_misc.hmbird_enabled = hmbird_enabled(); -+ p->snap_misc.curr_ss = curr_ss; -+ p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state(); -+ p->snap_misc.parctrl_high_ratio = parctrl_high_ratio; -+ p->snap_misc.parctrl_low_ratio = parctrl_low_ratio; -+ p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l; -+ p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l; -+ p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio; -+ p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio; -+ p->snap_misc.misfit_ds = misfit_ds; -+ p->snap_misc.partial_enable = partial_enable; -+ p->snap_misc.iso_free_rescue = iso_free_rescue; -+ p->snap_misc.isolate_ctrl = isolate_ctrl; -+ p->snap_misc.snap_jiffies = jiffies; -+ p->snap_misc.snap_time = local_clock(); -+} -+ -+// MTK minidump begin -+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3) -+{ -+ strscpy(m->desc_str, str, DESC_STR_LEN); -+ m->len = d1 * d2 * d3; -+ m->parse[0] = d1; -+ m->parse[1] = d2; -+ m->parse[2] = d3; -+} -+ -+static void init_desc_metas(struct md_info_t *m) -+{ -+ init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :", -+ 1, MAX_SWITCHS, SWITCH_ITEMS); -+ -+ init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1); -+ -+ init_desc_meta(&m->kern_dump.excep_rec_meta, -+ "excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS); -+ -+ init_desc_meta(&m->kern_dump.excep_idx_meta, -+ "excep idx :", 1, 1, MAX_EXCEP_ID); -+ -+ init_desc_meta(&m->kern_dump.snap.runnable_at_meta, -+ "each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS); -+ -+ init_desc_meta(&m->kern_dump.snap.rq_nr_meta, -+ "rq runnable task nr:", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta, -+ "scxrq runnable task nr :", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.snap_misc_meta, -+ "misc snap params :", 1, 1, SNAP_ITEMS); -+} -+ -+static void init_md_meta(struct md_info_t *m) -+{ -+ m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64); -+ m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64); -+ m->meta.unit_size = sizeof(u64); -+ m->meta.switches = MAX_SWITCHS; -+ m->meta.exceps = MAX_EXCEPS; -+ m->meta.global_dsqs = MAX_GLOBAL_DSQS; -+ m->meta.parse_dimens = PARSE_DIMENS; -+ m->meta.nr_cpus = num_possible_cpus(); -+ m->meta.real_cpus = nr_cpu_ids; -+ m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64); -+ m->meta.nr_meta_desc = 8; -+ m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64); -+ -+ init_desc_metas(m); -+} -+ -+#define MINIDUMP_DFL_SIZE (4 * 1024) -+ -+struct notifier_block hmbird_panic_blk; -+static int hmbird_panic_handler(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ if (!md_info) -+ return NOTIFY_DONE; -+ -+ get_hmbird_snapshot(&md_info->kern_dump.snap); -+ -+ return NOTIFY_DONE; -+} -+ -+static void panic_blk_init(void) -+{ -+ int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE); -+ -+ md_info = kzalloc(dump_size, GFP_KERNEL); -+ if (!md_info) -+ return; -+ init_md_meta(md_info); -+ -+ hmbird_panic_blk.notifier_call = hmbird_panic_handler; -+ /* make sure to execute before minidump. */ -+ hmbird_panic_blk.priority = INT_MAX; -+ atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk); -+ -+ hmbird_debug("register minidump.\n"); -+} -+ -+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size) -+{ -+ *vaddr = (unsigned long)md_info; -+ *size = sizeof(struct md_info_t); -+} -+// MTK minidump end -+ -+static inline void init_sched_prop_to_preempt_prio(void) -+{ -+ for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) { -+ switch (i) { -+ case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 5; -+ break; -+ -+ case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 4; -+ break; -+ -+ case HMBIRD_TASK_PROP_PIPELINE: -+ case HMBIRD_TASK_PROP_ISOLATE: -+ sched_prop_to_preempt_prio[i] = 3; -+ break; -+ -+ case HMBIRD_TASK_PROP_COMMON: -+ sched_prop_to_preempt_prio[i] = 1; -+ break; -+ -+ case HMBIRD_TASK_PROP_DEBUG_OR_LOG: -+ sched_prop_to_preempt_prio[i] = 0; -+ break; -+ -+ default: -+ sched_prop_to_preempt_prio[i] = 2; -+ break; -+ } -+ } -+} -+ -+void __init init_sched_hmbird_class(void) -+{ -+ int cpu; -+ u32 v; -+ struct hmbird_entity *init_hmbird; -+ -+ /* -+ * The following is to prevent the compiler from optimizing out the enum -+ * definitions so that BPF scheduler implementations can use them -+ * through the generated vmlinux.h. -+ */ -+ WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP | -+ HMBIRD_KICK_PREEMPT); -+ -+ init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL); -+ init_dsq_at_boot(); -+ init_isolate_cpus(); -+ hb_timer_init(); -+ init_sched_prop_to_preempt_prio(); -+#ifdef CONFIG_SMP -+ WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); -+#endif -+ -+ /* -+ * we can't static init init_task's hmbird struct, init here. -+ * init_task->hmbird would not use during boot. -+ */ -+ init_hmbird = kzalloc(sizeof(struct hmbird_entity), GFP_KERNEL); -+ init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird; -+ if (init_hmbird) { -+ INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo); -+ INIT_LIST_HEAD(&init_hmbird->watchdog_node); -+ init_hmbird->sticky_cpu = -1; -+ init_hmbird->holding_cpu = -1; -+ atomic64_set(&init_hmbird->ops_state, 0); -+ init_hmbird->runnable_at = jiffies; -+ init_hmbird->slice = HMBIRD_SLICE_DFL; -+ init_hmbird->task = &init_task; -+ hmbird_set_sched_prop(&init_task, 0); -+ } else { -+ hmbird_err(INIT_TASK_FAIL, ":alloc init_task.scx failed!!!\n"); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ /* -+ * exec during boot phase, no need to care about alloc failed. -+ * lifecycle same to rq, no need to free. -+ */ -+ rq->android_oem_data1[HMBIRD_RQ_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_rq), GFP_KERNEL); -+ if (get_hmbird_rq(rq)) { -+ get_hmbird_rq(rq)->rq = rq; -+ get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window; -+ } else { -+ hmbird_err(ALLOC_RQSCX_FAIL, ":alloc rq->scx failed!!!\n"); -+ } -+ -+ rq->android_oem_data1[HMBIRD_OPS_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL); -+ if (!get_hmbird_ops(rq)) -+ pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n"); -+ init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL); -+ INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list); -+ -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL)); -+ -+ hmbird_ops_init(get_hmbird_ops(rq)); -+ } -+ -+ INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn); -+ INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn); -+ hmbird_misc_init(); -+ -+ panic_blk_init(); -+} -+ -diff --git b/kernel/sched/hmbird/hmbird_misc.c a/kernel/sched/hmbird/hmbird_misc.c -new file mode 100755 -index 000000000000..895e8a29df33 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_misc.c -@@ -0,0 +1,148 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * @Description: -+ * @Version: -+ * @Author: wangrui8 -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include "hmbird_sched.h" -+#include -+#include -+#include "slim.h" -+ -+noinline int tracing_mark_write(const char *buf) -+{ -+ trace_printk(buf); -+ return 0; -+} -+ -+struct yield_opt_params yield_opt_params = { -+ .enable = 0, -+ .frame_per_sec = 120, -+ .frame_time_ns = NSEC_PER_SEC / 120, -+ .yield_headroom = 10, -+}; -+ -+DEFINE_PER_CPU(struct sched_yield_state, ystate); -+ -+static inline void hmbird_yield_state_update(struct sched_yield_state *ys) -+{ -+ if (!raw_spin_is_locked(&ys->lock)) -+ return; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ -+ if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1 -+ || ys->yield_cnt_after_sleep > yield_headroom) { -+ ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP); -+ } else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) { -+ ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP); -+ } -+ ys->yield_cnt = 0; -+ ys->sleep_times = 0; -+ ys->yield_cnt_after_sleep = 0; -+} -+ -+void hmbird_skip_yield(long *skip) -+{ -+ if (!get_hmbird_ops_enabled() || !yield_opt_params.enable) -+ return; -+ unsigned long flags, sleep_now = 0; -+ struct sched_yield_state *ys; -+ int cpu = raw_smp_processor_id(), cont_yield, new_frame; -+ int frame_time_ns = yield_opt_params.frame_time_ns; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ u64 wc; -+ -+ if (!(*skip)) { -+ wc = sched_clock(); -+ ys = &per_cpu(ystate, cpu); -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ -+ cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP; -+ new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1); -+ -+ if (!cont_yield && new_frame) { -+ hmbird_yield_state_update(ys); -+ ys->last_update_time = wc; -+ ys->sleep_end = ys->last_yield_time + frame_time_ns -+ - yield_headroom * YIELD_DURATION; -+ } -+ -+ if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) { -+ *skip = true; -+ -+ sleep_now = ys->sleep_times ? -+ max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep; -+ if (wc + sleep_now > ys->sleep_end) { -+ u64 delta = ys->sleep_end - wc; -+ -+ if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION) -+ sleep_now = delta; -+ else -+ sleep_now = 0; -+ } -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ if (sleep_now) { -+ sleep_now = div64_u64(sleep_now, 1000); -+ usleep_range_state(sleep_now, sleep_now, TASK_IDLE); -+ } -+ ys->sleep_times++; -+ ys->last_yield_time = sched_clock(); -+ return; -+ } -+ if (ys->sleep_times) -+ ys->yield_cnt_after_sleep++; -+ else -+ (ys->yield_cnt)++; -+ ys->last_yield_time = wc; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+} -+ -+struct boost_policy_params boost_policy_params = { -+ .enable = 0, -+ .bottom_freq = 1200000, -+ .boost_weight = 120, -+}; -+ -+int hmbird_get_boost_enable(void) -+{ -+ return boost_policy_params.enable; -+} -+ -+unsigned int hmbird_get_boost_bottom_freq(void) -+{ -+ return boost_policy_params.bottom_freq; -+} -+ -+int hmbird_get_boost_weight(void) -+{ -+ return boost_policy_params.boost_weight; -+} -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops) -+{ -+ hmbird_ops->scx_enable = get_hmbird_ops_enabled; -+ hmbird_ops->check_non_task = get_non_hmbird_task; -+ hmbird_ops->hmbird_get_md_info = hmbird_get_md_info; -+ hmbird_ops->hmbird_get_boost_enable = hmbird_get_boost_enable; -+ hmbird_ops->hmbird_get_boost_bottom_freq = hmbird_get_boost_bottom_freq; -+ hmbird_ops->hmbird_get_boost_weight = hmbird_get_boost_weight; -+} -+ -+void hmbird_misc_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_init(&ys->lock); -+ } -+ -+ set_hmbird_module_loaded(1); -+} -+ -diff --git b/kernel/sched/hmbird/hmbird_sched.h a/kernel/sched/hmbird/hmbird_sched.h -new file mode 100755 -index 000000000000..76acbe9685e4 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef __HMBIRD_SCHED__ -+#define __HMBIRD_SCHED__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include <../../kernel/time/tick-sched.h> -+#include <../../kernel/sched/sched.h> -+#include -+#include -+#include -+ -+#define REGISTER_TRACE_VH(vender_hook, handler) \ -+{ \ -+ ret = register_trace_##vender_hook(handler, NULL); \ -+ if (ret) { \ -+ pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \ -+ } \ -+} -+#define REGISTER_TRACE(vendor_hook, handler, data, err) \ -+do { \ -+ ret = register_trace_##vendor_hook(handler, data); \ -+ if (ret) { \ -+ pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret); \ -+ goto err; \ -+ } \ -+} while (0) -+ -+#define UNREGISTER_TRACE(vendor_hook, handler, data) \ -+ unregister_trace_##vendor_hook(handler, data) \ -+ -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+ -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int sched_ravg_window_frame_per_sec; -+extern int slim_gov_debug; -+extern int cpu7_tl; -+extern int scx_gov_ctrl; -+extern spinlock_t new_sched_ravg_window_lock; -+extern int cluster_separate; -+ -+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER BIT(31) -+#define MAX_YIELD_SLEEP (2000000ULL) -+#define MIN_YIELD_SLEEP (200000ULL) -+#define YIELD_DURATION (5000ULL) -+#define DEFAULT_YIELD_SLEEP_TH (10) -+ -+struct sched_yield_state { -+ raw_spinlock_t lock; -+ u64 last_yield_time; -+ u64 last_update_time; -+ u64 sleep_end; -+ unsigned long yield_cnt; -+ unsigned long yield_cnt_after_sleep; -+ unsigned long sleep; -+ int sleep_times; -+}; -+ -+DECLARE_PER_CPU(struct sched_yield_state, ystate); -+ -+void hmbird_window_rollover_run_once(struct rq *rq); -+void hmbird_yield_state_update_per_frame(void); -+void hmbird_misc_init(void); -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops); -+#endif /*__HMBIRD_SCHED__*/ -diff --git b/kernel/sched/hmbird/hmbird_sched_proc.c a/kernel/sched/hmbird/hmbird_sched_proc.c -new file mode 100755 -index 000000000000..295d45404608 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched_proc.c -@@ -0,0 +1,640 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "hmbird_sched_proc.h" -+#include -+ -+#include "hmbird_util_track.h" -+#include "slim.h" -+ -+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched" -+#define SLIM_FREQ_GOV_DIR "slim_freq_gov" -+#define LOAD_TRACK_DIR "slim_walt" -+#define HMBIRD_PROC_PERMISSION 0666 -+ -+int scx_enable; -+int partial_enable; -+int cpuctrl_high_ratio = 55; -+int cpuctrl_low_ratio = 40; -+int slim_stats; -+int hmbirdcore_debug; -+int slim_for_app; -+int misfit_ds = 90; -+unsigned int highres_tick_ctrl; -+unsigned int highres_tick_ctrl_dbg; -+int cpu7_tl = 70; -+int slim_walt_ctrl; -+int slim_walt_dump; -+int slim_walt_policy; -+int slim_gov_debug; -+int scx_gov_ctrl = 1; -+int sched_ravg_window_frame_per_sec = 125; -+int parctrl_high_ratio = 55; -+int parctrl_low_ratio = 40; -+int parctrl_high_ratio_l = 65; -+int parctrl_low_ratio_l = 50; -+int isoctrl_high_ratio = 75; -+int isoctrl_low_ratio = 60; -+int isolate_ctrl; -+int iso_free_rescue; -+int heartbeat; -+int heartbeat_enable = 1; -+int watchdog_enable; -+int save_gov; -+u64 cpu_cluster_masks; -+int hmbird_preempt_policy; -+int cluster_separate; -+ -+char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= 32) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtoint(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoint\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+static int set_proc_buf_val_u64(struct file *file, const char __user *buf, -+ size_t count, u64 *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= sizeof(kbuf)) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtou64(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoul\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/* common ops begin */ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ return count; -+} -+ -+static int hmbird_common_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%d\n", *(int *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_show, pde_data(inode)); -+} -+ -+static int hmbird_common_ul_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%lu\n", *(unsigned long *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_ul_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_ul_show, pde_data(inode)); -+} -+ -+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write); -+/* common ops end */ -+ -+/* scx_enable ops begin */ -+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC); -+ if (hmbird_ctrl(*pval)) -+ return -EFAULT; -+ -+ return count; -+} -+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write); -+/* scx_enable ops end */ -+ -+/* hmbird_stats ops begin */ -+#define MAX_STATS_BUF (4096) -+static int hmbird_stats_proc_show(struct seq_file *m, void *v) -+{ -+ char *buf; -+ -+ buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ stats_print(buf, MAX_STATS_BUF); -+ -+ seq_printf(m, "%s\n", buf); -+ -+ kfree(buf); -+ return 0; -+} -+ -+static int hmbird_stats_proc_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_stats_proc_show, inode); -+} -+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL); -+/* hmbird_stats ops end */ -+ -+/* sched_ravg_window_frame_per_sec ops begin */ -+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file, -+ const char __user *buf, size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ sched_ravg_window_change(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open, -+ sched_ravg_window_frame_per_sec_proc_write); -+/* sched_ravg_window_frame_per_sec ops end */ -+ -+static ssize_t save_gov_str(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ for_each_possible_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy || (cpu != policy->cpu)) -+ continue; -+ WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0); -+ hmbird_info_systrace(":save origin gov : %s\n", saved_gov[cpu]); -+ } -+ return count; -+} -+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str); -+ -+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ u64 *pval = (u64 *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val_u64(file, buf, count, pval)) -+ return -EFAULT; -+ -+ if (scx_enable == 0) -+ set_cpu_cluster(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write); -+ -+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ int tmp_val; -+ -+ if (set_proc_buf_val(file, buf, count, &tmp_val)) -+ return -EFAULT; -+ -+ slim_walt_enable(tmp_val); -+ *pval = tmp_val; -+ -+ return count; -+} -+ -+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write); -+ -+/* yield_opt ops begin */ -+static int yield_opt_show(struct seq_file *m, void *v) -+{ -+ struct yield_opt_params *data = m->private; -+ -+ seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n", -+ data->enable, data->frame_per_sec, data->yield_headroom); -+ return 0; -+} -+ -+static int yield_opt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, yield_opt_show, pde_data(inode)); -+} -+ -+static ssize_t yield_opt_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu; -+ unsigned long flags; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if (copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp -+ != 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) || -+ (yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp; -+ yield_opt_params.frame_per_sec = frame_per_sec_tmp; -+ yield_opt_params.yield_headroom = yield_headroom_tmp; -+ yield_opt_params.enable = enable_tmp; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ ys->last_yield_time = 0; -+ ys->last_update_time = 0; -+ ys->sleep_end = 0; -+ ys->yield_cnt = 0; -+ ys->yield_cnt_after_sleep = 0; -+ ys->sleep = 0; -+ ys->sleep_times = 0; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write); -+ -+/* boost_policy ops begin */ -+static int boost_policy_show(struct seq_file *m, void *v) -+{ -+ struct boost_policy_params *data = m->private; -+ -+ seq_printf(m, "boost_policy:{\"enable\":%d; \"bottom_freq\":%u; \"boost_weight\":%d}\n", -+ data->enable, data->bottom_freq, data->boost_weight); -+ return 0; -+} -+ -+static int boost_policy_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, boost_policy_show, pde_data(inode)); -+} -+ -+static ssize_t boost_policy_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, bottom_freq_tmp, boost_weight_tmp; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if(copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &bottom_freq_tmp, &boost_weight_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || -+ (boost_weight_tmp < 50 || boost_weight_tmp > 300) || -+ (bottom_freq_tmp < 400000 || bottom_freq_tmp > 2200000)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ boost_policy_params.bottom_freq = bottom_freq_tmp; -+ boost_policy_params.boost_weight = boost_weight_tmp; -+ boost_policy_params.enable = enable_tmp; -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(boost_policy, boost_policy_open, boost_policy_write); -+ -+/* tick_hit ops begin */ -+static int tick_hit_show(struct seq_file *m, void *v) -+{ -+ struct tick_hit_params *data = m->private; -+ -+ seq_printf(m, "tick_hit:{\"enable\":%d; \"hit_count_thres\":%d; \"jiffies_num\":%lu}\n", -+ data->enable, data->hit_count_thres, data->jiffies_num); -+ return 0; -+} -+ -+static int tick_hit_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, tick_hit_show, pde_data(inode)); -+} -+ -+static ssize_t tick_hit_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, hit_count_thres_tmp, jiffies_num_tmp; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if(copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &hit_count_thres_tmp, &jiffies_num_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ tick_hit_params.hit_count_thres = hit_count_thres_tmp; -+ tick_hit_params.jiffies_num = jiffies_num_tmp; -+ tick_hit_params.enable = enable_tmp; -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(tick_hit, tick_hit_open, tick_hit_write); -+ -+static int __init hmbird_proc_init(void) -+{ -+ struct proc_dir_entry *hmbird_dir; -+ struct proc_dir_entry *load_track_dir; -+ struct proc_dir_entry *freq_gov_dir; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ -+ /* mkdir /proc/hmbird_sched */ -+ hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL); -+ if (!hmbird_dir) { -+ pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &scx_enable_proc_ops, -+ &scx_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &partial_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_stats); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbirdcore_debug); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_for_app); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &misfit_ds); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl_dbg); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpu7_tl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &cpu_cluster_masks_proc_ops, -+ &cpu_cluster_masks); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &save_gov_proc_ops, -+ &save_gov); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &watchdog_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isolate_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &iso_free_rescue); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_stats_proc_ops); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &yield_opt_proc_ops, -+ &yield_opt_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("boost_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &boost_policy_proc_ops, -+ &boost_policy_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("tick_hit", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &tick_hit_proc_ops, -+ &tick_hit_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbird_preempt_policy); -+ /* /proc/hmbird_sched--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_walt */ -+ load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir); -+ if (!load_track_dir) { -+ pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_walt--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &slim_walt_ctrl_proc_ops, -+ &slim_walt_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_dump); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_policy); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &sched_ravg_window_frame_per_sec_proc_ops, -+ &sched_ravg_window_frame_per_sec); -+ /* /proc/hmbird_sched/slim_walt--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_freq_gov */ -+ freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir); -+ if (!freq_gov_dir) { -+ pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_freq_gov--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &slim_gov_debug); -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &scx_gov_ctrl); -+ /* /proc/hmbird_sched/slim_freq_gov--end */ -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cluster_separate); -+ -+ return 0; -+} -+ -+device_initcall(hmbird_proc_init); -diff --git b/kernel/sched/hmbird/hmbird_sched_proc.h a/kernel/sched/hmbird/hmbird_sched_proc.h -new file mode 100755 -index 000000000000..e22bc75f1dd7 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched_proc.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SCHED_PROC_H__ -+#define __HMBIRD_SCHED_PROC_H__ -+ -+#include -+#include -+ -+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \ -+ do { \ -+ if (!proc_create(name, mode, parent, proc_ops)) { \ -+ pr_err("Error creating proc entry %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \ -+ do { \ -+ if (!proc_create_data(name, mode, parent, proc_ops, data)) { \ -+ pr_err("Error creating proc entry with data %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_PROC_OPS(name, open_func, write_func) \ -+ static const struct proc_ops name##_proc_ops = { \ -+ .proc_open = open_func, \ -+ .proc_write = write_func, \ -+ .proc_read = seq_read, \ -+ .proc_lseek = seq_lseek, \ -+ .proc_release = single_release, \ -+ } -+ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos); -+static int hmbird_common_show(struct seq_file *m, void *v); -+static int hmbird_common_open(struct inode *inode, struct file *file); -+ -+#endif -diff --git b/kernel/sched/hmbird/hmbird_shadow_tick.c a/kernel/sched/hmbird/hmbird_shadow_tick.c -new file mode 100755 -index 000000000000..2744cd42bbfd ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_shadow_tick.c -@@ -0,0 +1,198 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include -+#include <../../kernel/time/tick-sched.h> -+#include -+ -+#include "hmbird_shadow_tick.h" -+#include "slim.h" -+ -+#define HIGHRES_WATCH_CPU 0 -+ -+#include -+static bool shadow_tick_enable(void) {return highres_tick_ctrl; } -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; } -+#endif -+static bool shadow_tick_timer_init_flag; -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define shadow_tick_printk(fmt, args...) \ -+do { \ -+ int cpu = smp_processor_id(); \ -+ if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU) \ -+ trace_printk("hmbird shadow tick :"fmt, args); \ -+} while (0) -+#else -+#define shadow_tick_printk(fmt, args...) -+#endif -+ -+DEFINE_PER_CPU(struct hrtimer, stt); -+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu))) -+#define STOP_IDLE_TRIGGER (1) -+#define PERIODIC_TICK_TRIGGER (2) -+#define TICK_INTVAL (1000000ULL) -+#define HMBIRD_TICK_HIT_BOOST BIT(29) -+/* -+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms, -+ * so we can't restart hrtimer in scheduler tick. -+ */ -+static DEFINE_PER_CPU(u8, trigger_event); -+ -+/* -+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick. -+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler. -+ */ -+static DEFINE_PER_CPU(u8, tick_phase); -+ -+static inline void highres_timer_ctrl(bool enable, int cpu) -+{ -+ if (enable && hmbird_enabled()) { -+ if (!hrtimer_active(shadow_tick_timer(cpu))) -+ hrtimer_start(shadow_tick_timer(cpu), -+ ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED); -+ } else { -+ if (!enable) -+ hrtimer_cancel(shadow_tick_timer(cpu)); -+ } -+} -+ -+static inline void high_res_clear_phase(int cpu) -+{ -+ per_cpu(tick_phase, cpu) = 0; -+} -+ -+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer) -+{ -+ per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3; -+ if (per_cpu(tick_phase, cpu)) { -+ hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) { -+ per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+struct tick_hit_params tick_hit_params = { -+ .enable = 0, -+ .jiffies_num = 2, -+ .hit_count_thres = 6, -+}; -+ -+static void tick_hit_critical_task(struct task_struct *curr, struct rq *rq) -+{ -+ struct hmbird_entity *see = get_hmbird_ts(curr); -+ if (!tick_hit_params.enable || !see) -+ return; -+ -+ if ((!task_is_top_task(curr) || curr->pid != curr->tgid) && (curr->pid != scx_systemui_pid)) -+ return; -+ -+ if (see->tick_hit_count == 0) { -+ if (see->start_jiffies == 0) { -+ see->start_jiffies = jiffies; -+ see->tick_hit_count = 1; -+ } else { -+ see->start_jiffies = 0; /* status reset */ -+ see->tick_hit_count = 0; -+ } -+ } else { -+ if (time_before_eq(jiffies, see->start_jiffies + tick_hit_params.jiffies_num)) { -+ see->tick_hit_count++; -+ if (see->tick_hit_count >= tick_hit_params.hit_count_thres) { -+ cpufreq_update_util(rq, HMBIRD_TICK_HIT_BOOST); -+ } -+ } else { -+ see->tick_hit_count = 1; -+ see->start_jiffies = jiffies; -+ } -+ } -+} -+ -+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ struct rq_flags rf; -+ -+ rq_lock(rq, &rf); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ tick_hit_critical_task(curr, rq); -+ rq_unlock(rq, &rf); -+ -+ return highres_next_phase(cpu, timer); -+} -+ -+void shadow_tick_timer_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance; -+ } -+} -+ -+void start_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable()) { -+ if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER) -+ highres_timer_ctrl(false, cpu); -+ per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static void stop_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ per_cpu(trigger_event, cpu) = 0; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(false, cpu); -+} -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ stop_shadow_tick_timer(); -+} -+ -+void scheduler_tick_handler(void *unused, struct rq *rq) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ start_shadow_tick_timer(); -+} -+ -+static int __init hmbird_shadow_tick_init(void) -+{ -+ int ret = 0; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ shadow_tick_timer_init(); -+ shadow_tick_timer_init_flag = true; -+ return ret; -+} -+ -+device_initcall(hmbird_shadow_tick_init); -diff --git b/kernel/sched/hmbird/hmbird_shadow_tick.h a/kernel/sched/hmbird/hmbird_shadow_tick.h -new file mode 100755 -index 000000000000..0c26fd984f0a ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_shadow_tick.h -@@ -0,0 +1,11 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SHADOW_TICK_H__ -+#define __HMBIRD_SHADOW_TICK_H__ -+ -+#include -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+void scheduler_tick_handler(void *unused, struct rq *rq); -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state); -+#endif -diff --git b/kernel/sched/hmbird/hmbird_trace.h a/kernel/sched/hmbird/hmbird_trace.h -new file mode 100755 -index 000000000000..8468b406f347 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_trace.h -@@ -0,0 +1,92 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hmbird -+ -+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HMBIRD_H -+ -+#include -+#include -+#include -+ -+#define MAX_FATAL_INFO (64) -+ -+TRACE_EVENT(hmbird_fatal_info, -+ -+ TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info), -+ -+ TP_ARGS(type, pe, lnr, bnr, info), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, type) -+ __field(int, pe) -+ __field(int, lnr) -+ __field(int, bnr) -+ __array(char, info, MAX_FATAL_INFO)), -+ -+ TP_fast_assign( -+ __entry->type = type; -+ __entry->pe = pe; -+ __entry->lnr = lnr; -+ __entry->bnr = bnr; -+ memcpy(__entry->info, info, MAX_FATAL_INFO);), -+ -+ TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s", -+ __entry->type, __entry->pe, __entry->lnr, __entry->bnr, -+ __entry->info) -+); -+ -+TRACE_EVENT(hmbird_update_history, -+ -+ TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq, -+ struct task_struct *p, u32 runtime, int samples, int event), -+ -+ TP_ARGS(hmbird, rq, p, runtime, samples, event), -+ -+ TP_STRUCT__entry( -+ __array(char, comm, TASK_COMM_LEN) -+ __field(pid_t, pid) -+ __field(unsigned int, runtime) -+ __field(int, samples) -+ __field(int, event) -+ __field(unsigned int, demand) -+ __array(u32, hist, RAVG_HIST_SIZE) -+ __field(u16, task_util) -+ __field(int, cpu)), -+ -+ TP_fast_assign( -+ memcpy(__entry->comm, p->comm, TASK_COMM_LEN); -+ __entry->pid = p->pid; -+ __entry->runtime = runtime; -+ __entry->samples = samples; -+ __entry->event = event; -+ __entry->demand = hmbird->sts.demand; -+ memcpy(__entry->hist, hmbird->sts.sum_history, -+ RAVG_HIST_SIZE * sizeof(u32)); -+ __entry->task_util = hmbird->sts.demand_scaled; -+ __entry->cpu = rq->cpu;), -+ -+ TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d", -+ __entry->comm, __entry->pid, -+ __entry->runtime, __entry->samples, -+ __entry->event, -+ __entry->demand, -+ __entry->hist[0], __entry->hist[1], -+ __entry->hist[2], __entry->hist[3], -+ __entry->hist[4], -+ __entry->task_util, -+ __entry->cpu) -+); -+ -+#endif /*_TRACE_HMBIRD_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird -+ -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE hmbird_trace -+/* This part must be outside protection */ -+#include -diff --git b/kernel/sched/hmbird/hmbird_util_track.c a/kernel/sched/hmbird/hmbird_util_track.c -new file mode 100755 -index 000000000000..d520a8403401 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_util_track.c -@@ -0,0 +1,626 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+ -+#define CREATE_TRACE_POINTS -+#include "hmbird_trace.h" -+#undef CREATE_TRACE_POINTS -+ -+#define HMBIRD_DEBUG_PANIC (1 << 3) -+static bool init_irq_work_inited; -+ -+extern noinline int tracing_mark_write(const char *buf); -+ -+int hmbird_sched_ravg_window = 8000000; -+int new_hmbird_sched_ravg_window = 8000000; -+DEFINE_SPINLOCK(new_sched_ravg_window_lock); -+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+ -+static struct irq_work hmbird_slim_walt_irq_work; -+ -+/*Sysctl related interface*/ -+#define WINDOW_STATS_RECENT 0 -+#define WINDOW_STATS_MAX 1 -+#define WINDOW_STATS_MAX_RECENT_AVG 2 -+#define WINDOW_STATS_AVG 3 -+#define WINDOW_STATS_INVALID_POLICY 4 -+ -+ -+#define SCX_SCHED_CAPACITY_SHIFT 10 -+#define SCHED_ACCOUNT_WAIT_TIME 0 -+ -+atomic64_t hmbird_irq_work_lastq_ws; -+static u64 tick_sched_clock; -+ -+static inline u64 scale_exec_time(u64 delta, struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT; -+} -+ -+static inline u64 scale_time_to_util(u64 d) -+{ -+ do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT); -+ return d; -+} -+ -+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ delta = scale_exec_time(delta, rq); -+ sts->sum += delta; -+ if (unlikely(sts->sum > hmbird_sched_ravg_window)) -+ sts->sum = hmbird_sched_ravg_window; -+ -+ return delta; -+} -+ -+ -+static int -+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) -+{ -+ /* -+ * No need to bother updating task demand for the idle task. -+ */ -+ if (is_idle_task(p)) -+ return 0; -+ -+ /* -+ * When a task is waking up it is completing a segment of non-busy -+ * time. Likewise, if wait time is not treated as busy time, then -+ * when a task begins to run or is migrated, it is not running and -+ * is completing a segment of non-busy time. -+ */ -+ if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && -+ (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) -+ return 0; -+ -+ /* -+ * The idle exit time is not accounted for the first task _picked_ up to -+ * run on the idle CPU. -+ */ -+ if (event == PICK_NEXT_TASK && rq->curr == rq->idle) -+ return 0; -+ -+ /* -+ * TASK_UPDATE can be called on sleeping task, when its moved between -+ * related groups -+ */ -+ if (event == TASK_UPDATE) { -+ if (rq->curr == p) -+ return 1; -+ -+ return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; -+ } -+ -+ return 1; -+} -+ -+static void rollover_cpu_window(struct rq *rq, bool full_window) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 curr_sum = srq->curr_runnable_sum; -+ -+ if (unlikely(full_window)) -+ curr_sum = 0; -+ -+ srq->prev_runnable_sum = curr_sum; -+ srq->curr_runnable_sum = 0; -+} -+ -+static u64 -+update_window_start(struct rq *rq, u64 wallclock, int event) -+{ -+ s64 delta; -+ int nr_windows; -+ bool full_window; -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start = srq->window_start; -+ -+ if (wallclock < srq->latest_clock) -+ wallclock = srq->latest_clock; -+ delta = wallclock - srq->window_start; -+ if (delta < 0) { -+ delta = 0; -+ wallclock = srq->window_start; -+ } -+ srq->latest_clock = wallclock; -+ if (delta < hmbird_sched_ravg_window) -+ return old_window_start; -+ -+ nr_windows = div64_u64(delta, hmbird_sched_ravg_window); -+ srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window; -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ full_window = nr_windows > 1; -+ rollover_cpu_window(rq, full_window); -+ -+ return old_window_start; -+} -+ -+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -+ -+static inline unsigned int get_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static inline unsigned int cpu_cur_freq(int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cur; -+} -+ -+static void -+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, -+ u64 wallclock) -+{ -+ int cpu = cpu_of(rq); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * -+ arch_scale_cpu_capacity(cpu), get_max_freq(cpu)); -+} -+ -+/* -+ * Called when new window is starting for a task, to record cpu usage over -+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 -+ * when, say, a real-time task runs without preemption for several windows at a -+ * stretch. -+ */ -+static void update_history(struct rq *rq, struct task_struct *p, -+ u32 runtime, int samples, int event) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u32 *hist = &sts->sum_history[0]; -+ int i; -+ u32 max = 0, avg, demand; -+ u64 sum = 0; -+ u16 demand_scaled; -+ -+ /* Ignore windows where task had no activity */ -+ if (!runtime || is_idle_task(p) || !samples) -+ goto done; -+ -+ /* Push new 'runtime' value onto stack */ -+ for (; samples > 0; samples--) { -+ hist[sts->cidx] = runtime; -+ sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE; -+ } -+ -+ for (i = 0; i < RAVG_HIST_SIZE; i++) { -+ sum += hist[i]; -+ if (hist[i] > max) -+ max = hist[i]; -+ } -+ -+ sts->sum = 0; -+ avg = div64_u64(sum, RAVG_HIST_SIZE); -+ -+ switch (slim_walt_policy) { -+ case WINDOW_STATS_RECENT: -+ demand = runtime; -+ break; -+ case WINDOW_STATS_MAX: -+ demand = max; -+ break; -+ case WINDOW_STATS_AVG: -+ demand = avg; -+ break; -+ default: -+ demand = max(avg, runtime); -+ } -+ -+ demand_scaled = scale_time_to_util(demand); -+ -+ sts->demand = demand; -+ sts->demand_scaled = demand_scaled; -+ -+done: -+ return; -+} -+ -+ -+static u64 update_task_demand(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ -+ u64 delta, window_start = srq->window_start; -+ int new_window, nr_full_windows; -+ u32 window_size = hmbird_sched_ravg_window; -+ u64 runtime; -+ -+ new_window = mark_start < window_start; -+ if (!account_busy_for_task_demand(rq, p, event)) { -+ if (new_window) -+ /* -+ * If the time accounted isn't being accounted as -+ * busy time, and a new window started, only the -+ * previous window need be closed out with the -+ * pre-existing demand. Multiple windows may have -+ * elapsed, but since empty windows are dropped, -+ * it is not necessary to account those. -+ */ -+ update_history(rq, p, sts->sum, 1, event); -+ return 0; -+ } -+ -+ if (!new_window) { -+ /* -+ * The simple case - busy time contained within the existing -+ * window. -+ */ -+ return add_to_task_demand(rq, p, wallclock - mark_start); -+ } -+ -+ /* -+ * Busy time spans at least two windows. Temporarily rewind -+ * window_start to first window boundary after mark_start. -+ */ -+ delta = window_start - mark_start; -+ nr_full_windows = div64_u64(delta, window_size); -+ window_start -= (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (window_start - mark_start) first */ -+ runtime = add_to_task_demand(rq, p, window_start - mark_start); -+ -+ /* Push new sample(s) into task's demand history */ -+ update_history(rq, p, sts->sum, 1, event); -+ if (nr_full_windows) { -+ u64 scaled_window = scale_exec_time(window_size, rq); -+ -+ update_history(rq, p, scaled_window, nr_full_windows, event); -+ runtime += nr_full_windows * scaled_window; -+ } -+ -+ /* -+ * Roll window_start back to current to process any remainder -+ * in current window. -+ */ -+ window_start += (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (wallclock - window_start) next */ -+ mark_start = window_start; -+ runtime += add_to_task_demand(rq, p, wallclock - mark_start); -+ -+ return runtime; -+} -+ -+u16 slim_walt_cpu_util(int cpu) -+{ -+ u64 prev_runnable_sum; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ prev_runnable_sum = srq->prev_runnable_sum; -+ do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT); -+ -+ return (u16)prev_runnable_sum; -+} -+ -+static DEFINE_PER_CPU(u16, prev_cpu_util); -+static inline void cpu_util_update_systrace_c(int cpu) -+{ -+ char buf[256]; -+ u16 cpu_util = slim_walt_cpu_util(cpu); -+ -+ if (cpu_util != per_cpu(prev_cpu_util, cpu)) { -+ snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n", -+ cpu, cpu_util); -+ tracing_mark_write(buf); -+ per_cpu(prev_cpu_util, cpu) = cpu_util; -+ } -+} -+ -+ -+static inline int account_busy_for_cpu_time(struct rq *rq, -+ struct task_struct *p, -+ int event) -+{ -+ return !is_idle_task(p) && (event == PUT_PREV_TASK -+ || event == TASK_UPDATE); -+} -+ -+ -+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ int new_window, full_window = 0; -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 window_start = srq->window_start; -+ u32 window_size = srq->prev_window_size; -+ u64 delta; -+ u64 *curr_runnable_sum = &srq->curr_runnable_sum; -+ u64 *prev_runnable_sum = &srq->prev_runnable_sum; -+ -+ new_window = mark_start < window_start; -+ if (new_window) -+ full_window = (window_start - mark_start) >= window_size; -+ -+ -+ if (!account_busy_for_cpu_time(rq, p, event)) -+ goto done; -+ -+ -+ if (!new_window) { -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. No rollover -+ * since we didn't start a new window. An example of this is -+ * when a task starts execution and then sleeps within the -+ * same window. -+ */ -+ delta = wallclock - mark_start; -+ -+ delta = scale_exec_time(delta, rq); -+ *curr_runnable_sum += delta; -+ -+ goto done; -+ } -+ -+ /* -+ * situations below this need window rollover, -+ * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done -+ * in update_window_start() -+ * -+ * For task counters curr/prev_window[_cpu] are rolled over in the early part of -+ * this function. If full_window(s) have expired and time since last update needs -+ * to be accounted as busy time, set the prev to a complete window size time, else -+ * add the prev window portion. -+ * -+ * For task curr counters a new window has begun, always assign -+ */ -+ -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. A new window -+ * must have been started in udpate_window_start() -+ * If any of these three above conditions are true -+ * then this busy time can't be accounted as irqtime. -+ * -+ * Busy time for the idle task need not be accounted. -+ * -+ * An example of this would be a task that starts execution -+ * and then sleeps once a new window has begun. -+ */ -+ -+ /* -+ * A full window hasn't elapsed, account partial -+ * contribution to previous completed window. -+ */ -+ -+ -+ delta = full_window ? scale_exec_time(window_size, rq) : -+ scale_exec_time(window_start - mark_start, rq); -+ -+ *prev_runnable_sum += delta; -+ -+ /* Account piece of busy time in the current window. */ -+ delta = scale_exec_time(wallclock - window_start, rq); -+ *curr_runnable_sum += delta; -+ -+done: -+ if (slim_walt_dump && new_window) -+ cpu_util_update_systrace_c(rq->cpu); -+} -+ -+ -+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq) -+{ -+ u64 result; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 new_window_start = srq->window_start; -+ -+ if (old_window_start == new_window_start) -+ return; -+ -+ result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws, -+ old_window_start, new_window_start); -+ if (result != old_window_start) -+ return; -+ -+ if (likely(cpu_online(raw_smp_processor_id()))) -+ irq_work_queue(&hmbird_slim_walt_irq_work); -+ else -+ irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask)); -+} -+ -+/* -+ * In the core scheduler, most of the load update points update the rq_clock after -+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of -+ * obtaining the time, but to prevent the subsequent migration of the load update -+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock -+ * update here. -+ */ -+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p, -+ struct rq *rq, int event) -+{ -+ if (!(rq->clock_update_flags & RQCF_UPDATED)) -+ update_rq_clock(rq); -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock)); -+} -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start; -+ -+ if (!slim_walt_ctrl) -+ return; -+ -+ if (!srq->window_start || sts->mark_start == wallclock) -+ return; -+ -+ old_window_start = update_window_start(rq, wallclock, event); -+ -+ if (!sts->window_start) -+ sts->window_start = srq->window_start; -+ -+ if (!sts->mark_start) -+ goto done; -+ -+ update_task_rq_cpu_cycles(p, rq, event, wallclock); -+ update_task_demand(p, rq, event, wallclock); -+ update_cpu_busy_time(p, rq, event, wallclock); -+ -+ sts->window_start = srq->window_start; -+ -+done: -+ sts->mark_start = wallclock; -+ slim_walt_window_rollover_run_once(old_window_start, rq); -+} -+ -+static void slim_walt_irq_work(struct irq_work *irq_work) -+{ -+ cpumask_t lock_cpus; -+ struct hmbird_sched_rq_stats *srq; -+ struct rq *rq; -+ int cpu; -+ int level = 0; -+ u64 wc; -+ unsigned long flags; -+ -+ cpumask_copy(&lock_cpus, cpu_possible_mask); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ if (level == 0) -+ raw_spin_lock(&cpu_rq(cpu)->__lock); -+ else -+ raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); -+ level++; -+ } -+ -+ wc = sched_clock(); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ rq = cpu_rq(cpu); -+ hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc); -+ } -+ -+ cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER); -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) { -+ srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id()); -+ if (wc < srq->window_start + new_hmbird_sched_ravg_window) -+ hmbird_sched_ravg_window = new_hmbird_sched_ravg_window; -+ } -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ raw_spin_unlock(&cpu_rq(cpu)->__lock); -+ } -+} -+static void hmbird_sched_init_rq(struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ srq->task_exec_scale = 1024; -+ srq->window_start = 0; -+} -+ -+void hmbird_sched_init_task(struct task_struct *p) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ memset(sts, 0, sizeof(struct hmbird_sched_task_stats)); -+} -+ -+static void hmbird_sched_stats_init(void) -+{ -+ unsigned long flags; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ raw_spin_lock_irqsave(&rq->__lock, flags); -+ hmbird_sched_init_rq(rq); -+ raw_spin_unlock_irqrestore(&rq->__lock, flags); -+ } -+ slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG; -+ -+ if (false == init_irq_work_inited) { -+ init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work); -+ init_irq_work_inited = true; -+ } -+} -+ -+void hmbird_scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (unlikely(!tick_sched_clock)) { -+ /* -+ * Let the window begin 20us prior to the tick, -+ * that way we are guaranteed a rollover when the tick occurs. -+ * Use rq->clock directly instead of rq_clock() since -+ * we do not have the rq lock and -+ * rq->clock was updated in the tick callpath. -+ */ -+ if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000)) -+ return; -+ for_each_possible_cpu(cpu) { -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ srq->window_start = tick_sched_clock; -+ } -+ atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock); -+ } -+} -+ -+void slim_walt_enable(int enable) -+{ -+ if (1 == !!enable) { -+ hmbird_sched_stats_init(); -+ WRITE_ONCE(tick_sched_clock, 0); -+ } else -+ slim_walt_ctrl = 0; -+} -+ -+void slim_get_cpu_util(int cpu, u64 *util) -+{ -+ if (cpu < 0) -+ return; -+ -+ *util = slim_walt_cpu_util(cpu); -+} -+ -+void slim_get_task_util(struct task_struct *p, u64 *util) -+{ -+ if (p == NULL) -+ return; -+ -+ *util = get_hmbird_ts(p)->sts.demand_scaled; -+} -+ -+void sched_ravg_window_change(int frame_per_sec) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec; -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+} -diff --git b/kernel/sched/hmbird/hmbird_util_track.h a/kernel/sched/hmbird/hmbird_util_track.h -new file mode 100644 -index 000000000000..17b85df7f83b ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_util_track.h -@@ -0,0 +1,33 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef __HMBIRD_UTIL_TRACK_H__ -+#define __HMBIRD_UTIL_TRACK_H__ -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock); -+void hmbird_sched_init_task(struct task_struct *p); -+void slim_walt_enable(int enable); -+void slim_get_cpu_util(int cpu, u64 *util); -+void slim_get_task_util(struct task_struct *p, u64 *util); -+ -+extern atomic64_t hmbird_irq_work_lastq_ws; -+ -+enum task_event { -+ PUT_PREV_TASK = 0, -+ PICK_NEXT_TASK = 1, -+ TASK_WAKE = 2, -+ TASK_MIGRATE = 3, -+ TASK_UPDATE = 4, -+ IRQ_UPDATE = 5, -+}; -+ -+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+#endif /* __HMBIRD_UTIL_TRACK_H__ */ -diff --git b/kernel/sched/hmbird/slim.h a/kernel/sched/hmbird/slim.h -new file mode 100755 -index 000000000000..dffe6506658e ---- /dev/null -+++ a/kernel/sched/hmbird/slim.h -@@ -0,0 +1,56 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+ -+#ifndef __SLIM_H -+#define __SLIM_H -+ -+extern atomic_t __hmbird_ops_enabled; -+extern atomic_t non_hmbird_task; -+extern int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+extern int heartbeat; -+extern int heartbeat_enable; -+extern int watchdog_enable; -+extern int isolate_ctrl; -+extern int parctrl_high_ratio; -+extern int parctrl_low_ratio; -+extern int isoctrl_high_ratio; -+extern int isoctrl_low_ratio; -+extern int iso_free_rescue; -+extern int yield_opt; -+ -+extern enum hmbird_switch_type sw_type; -+extern noinline int tracing_mark_write(const char *buf); -+int task_top_id(struct task_struct *p); -+void stats_print(char *buf, int len); -+void hmbird_skip_yield(long *skip); -+extern spinlock_t hmbird_tasks_lock; -+extern int scx_systemui_pid; -+ -+struct yield_opt_params { -+ int enable; -+ int frame_per_sec; -+ u64 frame_time_ns; -+ int yield_headroom; -+}; -+ -+extern struct yield_opt_params yield_opt_params; -+ -+struct tick_hit_params { -+ int enable; -+ unsigned long jiffies_num; -+ int hit_count_thres; -+}; -+ -+extern struct tick_hit_params tick_hit_params; -+ -+struct boost_policy_params { -+ int enable; -+ unsigned int bottom_freq; -+ int boost_weight; -+}; -+ -+extern struct boost_policy_params boost_policy_params; -+ -+#define MAX_GOV_LEN (16) -+extern char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+#endif -diff --git b/kernel/sched/idle.c a/kernel/sched/idle.c -index b33cefeb4188..487255ac6832 100755 ---- b/kernel/sched/idle.c -+++ a/kernel/sched/idle.c -@@ -408,13 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl - - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) - { -- scx_update_idle(rq, false); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, false); -+#endif - } - - static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) - { - update_idle_core(rq); -- scx_update_idle(rq, true); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, true); -+#endif - schedstat_inc(rq->sched_goidle); - } - -diff --git b/kernel/sched/sched.h a/kernel/sched/sched.h -index 509e8dc616ab..a7e9caea1efe 100755 ---- b/kernel/sched/sched.h -+++ a/kernel/sched/sched.h -@@ -188,18 +188,13 @@ static inline int idle_policy(int policy) - return policy == SCHED_IDLE; - } - --static inline int normal_policy(int policy) --{ --#ifdef CONFIG_SCHED_CLASS_EXT -- if (policy == SCHED_EXT) -- return true; --#endif -- return policy == SCHED_NORMAL; --} -- - static inline int fair_policy(int policy) - { -- return normal_policy(policy) || policy == SCHED_BATCH; -+#ifdef CONFIG_HMBIRD_SCHED -+ return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#else -+ return policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#endif - } - - static inline int rt_policy(int policy) -@@ -247,25 +242,7 @@ static inline void update_avg(u64 *avg, u64 sample) - #define shr_bound(val, shift) \ - (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) - --/* -- * cgroup weight knobs should use the common MIN, DFL and MAX values which are -- * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it -- * maps pretty well onto the shares value used by scheduler and the round-trip -- * conversions preserve the original value over the entire range. -- */ --static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) --{ -- return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); --} -- --static inline unsigned long sched_weight_to_cgroup(unsigned long weight) --{ -- return clamp_t(unsigned long, -- DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -- CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); --} -- --/* -+/* - * !! For sched_setattr_nocheck() (kernel) only !! - * - * This is actually gross. :( -@@ -532,10 +509,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) { } - #endif /* CONFIG_SMP */ - #else /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_HMBIRD_SCHED - static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) - { - return 0; - } -+#endif - #endif /* CONFIG_FAIR_GROUP_SCHED */ - - #else /* CONFIG_CGROUP_SCHED */ -@@ -700,29 +679,6 @@ struct cfs_rq { - #endif /* CONFIG_FAIR_GROUP_SCHED */ - }; - --#ifdef CONFIG_SCHED_CLASS_EXT --/* scx_rq->flags, protected by the rq lock */ --enum scx_rq_flags { -- SCX_RQ_CAN_STOP_TICK = 1 << 0, --}; -- --struct scx_rq { -- struct scx_dispatch_q local_dsq; -- struct list_head watchdog_list; -- u64 ops_qseq; -- u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -- u32 nr_running; -- u32 flags; -- bool cpu_released; -- cpumask_var_t cpus_to_kick; -- cpumask_var_t cpus_to_preempt; -- cpumask_var_t cpus_to_wait; -- u64 pnt_seq; -- struct irq_work kick_cpus_irq_work; -- struct rq *rq; --}; --#endif /* CONFIG_SCHED_CLASS_EXT */ -- - static inline int rt_bandwidth_enabled(void) - { - return sysctl_sched_rt_runtime >= 0; -@@ -1258,11 +1214,7 @@ struct rq { - - ANDROID_OEM_DATA_ARRAY(1, 16); - --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, struct scx_rq *scx); --#else - ANDROID_KABI_RESERVE(1); --#endif - ANDROID_KABI_RESERVE(2); - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); -@@ -2340,11 +2292,6 @@ struct affinity_context { - unsigned int flags; - }; - --enum rq_onoff_reason { -- RQ_ONOFF_HOTPLUG, /* CPU is going on/offline */ -- RQ_ONOFF_TOPOLOGY, /* sched domain topology update */ --}; -- - struct sched_class { - - #ifdef CONFIG_UCLAMP_TASK -@@ -2551,7 +2498,6 @@ extern void init_sched_rt_class(void); - extern void init_sched_fair_class(void); - - extern void reweight_task(struct task_struct *p, int prio); --extern void __setscheduler_prio(struct task_struct *p, int prio); - - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); -@@ -2632,53 +2578,6 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) - extern void activate_task(struct rq *rq, struct task_struct *p, int flags); - extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); - --struct sched_change_guard { -- struct task_struct *p; -- struct rq *rq; -- bool queued; -- bool running; -- bool done; --}; -- --extern struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -- --extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags); -- --/** -- * SCHED_CHANGE_BLOCK - Nested block for task attribute updates -- * @__rq: Runqueue the target task belongs to -- * @__p: Target task -- * @__flags: DEQUEUE/ENQUEUE_* flags -- * -- * A task may need to be dequeued and put_prev_task'd for attribute updates and -- * set_next_task'd and re-enqueued afterwards. This helper defines a nested -- * block which automatically handles these preparation and cleanup operations. -- * -- * SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- * update_attribute(p); -- * ... -- * } -- * -- * If @__flags is a variable, the variable may be updated in the block body and -- * the updated value will be used when re-enqueueing @p. -- * -- * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling -- * update_rq_clock() beforehand. Otherwise, the rq clock is automatically -- * updated iff the task needs to be dequeued and re-enqueued. Only the former -- * case guarantees that the rq clock is up-to-date inside and after the block. -- */ --#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -- for (struct sched_change_guard __cg = \ -- sched_change_guard_init(__rq, __p, __flags); \ -- !__cg.done; sched_change_guard_fini(&__cg, __flags)) -- --extern void check_class_changing(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class); --extern void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio); -- - extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); - - #ifdef CONFIG_PREEMPT_RT -@@ -3710,6 +3609,8 @@ static inline bool cpu_busy_with_softirqs(int cpu) - } - #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ - --#include "ext.h" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird.h" -+#endif - - #endif /* _KERNEL_SCHED_SCHED_H */ -diff --git b/kernel/time/tick-sched.c a/kernel/time/tick-sched.c -index 998c2eefe173..c13772ee823c 100755 ---- b/kernel/time/tick-sched.c -+++ a/kernel/time/tick-sched.c -@@ -34,6 +34,10 @@ - - #include - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "../sched/hmbird/hmbird_shadow_tick.h" -+#endif -+ - /* - * Per-CPU nohz control structure - */ -@@ -1124,6 +1128,9 @@ void tick_nohz_idle_stop_tick(void) - ktime_t expires; - - trace_android_vh_tick_nohz_idle_stop_tick(NULL); -+#ifdef CONFIG_HMBIRD_SCHED -+ android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL); -+#endif - /* - * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the - * tick timer expiration time is known already. -diff --git b/tools/sched_ext/.gitignore a/tools/sched_ext/.gitignore -deleted file mode 100644 -index a3240f9f7eba..000000000000 ---- b/tools/sched_ext/.gitignore -+++ /dev/null -@@ -1,9 +0,0 @@ --scx_example_simple --scx_example_qmap --scx_example_central --scx_example_pair --scx_example_flatcg --scx_example_userland --*.skel.h --*.subskel.h --/tools/ -diff --git b/tools/sched_ext/Makefile a/tools/sched_ext/Makefile -deleted file mode 100644 -index 73c43782837d..000000000000 ---- b/tools/sched_ext/Makefile -+++ /dev/null -@@ -1,216 +0,0 @@ --# SPDX-License-Identifier: GPL-2.0 --# Copyright (c) 2022 Meta Platforms, Inc. and affiliates. --include ../build/Build.include --include ../scripts/Makefile.arch --include ../scripts/Makefile.include -- --ifneq ($(LLVM),) --ifneq ($(filter %/,$(LLVM)),) --LLVM_PREFIX := $(LLVM) --else ifneq ($(filter -%,$(LLVM)),) --LLVM_SUFFIX := $(LLVM) --endif -- --CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi --CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu --CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl --CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu --CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu --CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu --CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu --CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu --CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu --CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) -- --ifeq ($(CROSS_COMPILE),) --ifeq ($(CLANG_TARGET_FLAGS),) --$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk --else --CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) --endif # CLANG_TARGET_FLAGS --else --CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) --endif # CROSS_COMPILE -- --CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as --else --CC := $(CROSS_COMPILE)gcc --endif # LLVM -- --CURDIR := $(abspath .) --TOOLSDIR := $(abspath ..) --LIBDIR := $(TOOLSDIR)/lib --BPFDIR := $(LIBDIR)/bpf --TOOLSINCDIR := $(TOOLSDIR)/include --BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool --APIDIR := $(TOOLSINCDIR)/uapi --GENDIR := $(abspath ../../include/generated) --GENHDR := $(GENDIR)/autoconf.h -- --SCRATCH_DIR := $(CURDIR)/tools --BUILD_DIR := $(SCRATCH_DIR)/build --INCLUDE_DIR := $(SCRATCH_DIR)/include --BPFOBJ_DIR := $(BUILD_DIR)/libbpf --BPFOBJ := $(BPFOBJ_DIR)/libbpf.a --ifneq ($(CROSS_COMPILE),) --HOST_BUILD_DIR := $(BUILD_DIR)/host --HOST_SCRATCH_DIR := host-tools --HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include --else --HOST_BUILD_DIR := $(BUILD_DIR) --HOST_SCRATCH_DIR := $(SCRATCH_DIR) --HOST_INCLUDE_DIR := $(INCLUDE_DIR) --endif --HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a --RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids --DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -- --VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ -- $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ -- ../../vmlinux \ -- /sys/kernel/btf/vmlinux \ -- /boot/vmlinux-$(shell uname -r) --VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) --ifeq ($(VMLINUX_BTF),) --$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") --endif -- --BPFTOOL ?= $(DEFAULT_BPFTOOL) -- --ifneq ($(wildcard $(GENHDR)),) -- GENFLAGS := -DHAVE_GENHDR --endif -- --CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ -- -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -- -I$(TOOLSINCDIR) -I$(APIDIR) -- --CARGOFLAGS := --release -- --# Silence some warnings when compiled with clang --ifneq ($(LLVM),) --CFLAGS += -Wno-unused-command-line-argument --endif -- --LDFLAGS = -lelf -lz -lpthread -- --IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - &1 \ -- | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ --$(shell $(1) -dM -E - $@ --else -- $(call msg,CP,,$@) -- $(Q)cp "$(VMLINUX_H)" $@ --endif -- --%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \ -- | $(BPFOBJ) -- $(call msg,CLNG-BPF,,$@) -- $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ -- --%.skel.h: %.bpf.o $(BPFTOOL) -- $(call msg,GEN-SKEL,,$@) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) -- $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) -- $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@ -- $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h) -- --scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_userland: scx_example_userland.c scx_example_userland.skel.h \ -- scx_example_userland_common.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR) --atropos: export ATROPOS_CLANG = $(CLANG) --atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS) --atropos: $(INCLUDE_DIR)/vmlinux.h -- cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS) -- --clean: -- cargo clean --manifest-path=atropos/Cargo.toml -- rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) -- rm -f *.o *.bpf.o *.skel.h *.subskel.h -- rm -f scx_example_simple scx_example_qmap scx_example_central \ -- scx_example_pair scx_example_flatcg scx_example_userland -- --.PHONY: all atropos clean -- --# delete failed targets --.DELETE_ON_ERROR: -- --# keep intermediate (.skel.h, .bpf.o, etc) targets --.SECONDARY: -diff --git b/tools/sched_ext/atropos/.gitignore a/tools/sched_ext/atropos/.gitignore -deleted file mode 100644 -index 186dba259ec2..000000000000 ---- b/tools/sched_ext/atropos/.gitignore -+++ /dev/null -@@ -1,3 +0,0 @@ --src/bpf/.output --Cargo.lock --target -diff --git b/tools/sched_ext/atropos/Cargo.toml a/tools/sched_ext/atropos/Cargo.toml -deleted file mode 100644 -index 7462a836d53d..000000000000 ---- b/tools/sched_ext/atropos/Cargo.toml -+++ /dev/null -@@ -1,28 +0,0 @@ --[package] --name = "scx_atropos" --version = "0.5.0" --authors = ["Dan Schatzberg ", "Meta"] --edition = "2021" --description = "Userspace scheduling with BPF" --license = "GPL-2.0-only" -- --[dependencies] --anyhow = "1.0.65" --bitvec = { version = "1.0", features = ["serde"] } --clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } --ctrlc = { version = "3.1", features = ["termination"] } --fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"} --hex = "0.4.3" --libbpf-rs = "0.19.1" --libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] } --libc = "0.2.137" --log = "0.4.17" --ordered-float = "3.4.0" --simplelog = "0.12.0" -- --[build-dependencies] --bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false } --libbpf-cargo = "0.13.0" -- --[features] --enable_backtrace = [] -diff --git b/tools/sched_ext/atropos/build.rs a/tools/sched_ext/atropos/build.rs -deleted file mode 100644 -index 26e792c5e17e..000000000000 ---- b/tools/sched_ext/atropos/build.rs -+++ /dev/null -@@ -1,70 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --extern crate bindgen; -- --use std::env; --use std::fs::create_dir_all; --use std::path::Path; --use std::path::PathBuf; -- --use libbpf_cargo::SkeletonBuilder; -- --const HEADER_PATH: &str = "src/bpf/atropos.h"; -- --fn bindgen_atropos() { -- // Tell cargo to invalidate the built crate whenever the wrapper changes -- println!("cargo:rerun-if-changed={}", HEADER_PATH); -- -- // The bindgen::Builder is the main entry point -- // to bindgen, and lets you build up options for -- // the resulting bindings. -- let bindings = bindgen::Builder::default() -- // The input header we would like to generate -- // bindings for. -- .header(HEADER_PATH) -- // Tell cargo to invalidate the built crate whenever any of the -- // included header files changed. -- .parse_callbacks(Box::new(bindgen::CargoCallbacks)) -- // Finish the builder and generate the bindings. -- .generate() -- // Unwrap the Result and panic on failure. -- .expect("Unable to generate bindings"); -- -- // Write the bindings to the $OUT_DIR/bindings.rs file. -- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); -- bindings -- .write_to_file(out_path.join("atropos-sys.rs")) -- .expect("Couldn't write bindings!"); --} -- --fn gen_bpf_sched(name: &str) { -- let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap(); -- let clang = env::var("ATROPOS_CLANG").unwrap(); -- eprintln!("{}", clang); -- let outpath = format!("./src/bpf/.output/{}.skel.rs", name); -- let skel = Path::new(&outpath); -- let src = format!("./src/bpf/{}.bpf.c", name); -- SkeletonBuilder::new() -- .source(src.clone()) -- .clang(clang) -- .clang_args(bpf_cflags) -- .build_and_generate(&skel) -- .unwrap(); -- println!("cargo:rerun-if-changed={}", src); --} -- --fn main() { -- bindgen_atropos(); -- // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton. -- // Reasons are because the generated skeleton contains compiler attributes -- // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]` -- // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside -- // the path attribute either (see https://github.com/rust-lang/rust/pull/83366). -- // -- // However, there is hope! When the above feature stabilizes we can clean this -- // all up. -- create_dir_all("./src/bpf/.output").unwrap(); -- gen_bpf_sched("atropos"); --} -diff --git b/tools/sched_ext/atropos/rustfmt.toml a/tools/sched_ext/atropos/rustfmt.toml -deleted file mode 100644 -index b7258ed0a8d8..000000000000 ---- b/tools/sched_ext/atropos/rustfmt.toml -+++ /dev/null -@@ -1,8 +0,0 @@ --# Get help on options with `rustfmt --help=config` --# Please keep these in alphabetical order. --edition = "2021" --group_imports = "StdExternalCrate" --imports_granularity = "Item" --merge_derives = false --use_field_init_shorthand = true --version = "Two" -diff --git b/tools/sched_ext/atropos/src/atropos_sys.rs a/tools/sched_ext/atropos/src/atropos_sys.rs -deleted file mode 100644 -index bbeaf856d40e..000000000000 ---- b/tools/sched_ext/atropos/src/atropos_sys.rs -+++ /dev/null -@@ -1,10 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#![allow(non_upper_case_globals)] --#![allow(non_camel_case_types)] --#![allow(non_snake_case)] --#![allow(dead_code)] -- --include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs")); -diff --git b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -deleted file mode 100644 -index 3905a403e940..000000000000 ---- b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -+++ /dev/null -@@ -1,743 +0,0 @@ --/* Copyright (c) Meta Platforms, Inc. and affiliates. */ --/* -- * This software may be used and distributed according to the terms of the -- * GNU General Public License version 2. -- * -- * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF -- * part does simple round robin in each domain and the userspace part -- * calculates the load factor of each domain and tells the BPF part how to load -- * balance the domains. -- * -- * Every task has an entry in the task_data map which lists which domain the -- * task belongs to. When a task first enters the system (atropos_prep_enable), -- * they are round-robined to a domain. -- * -- * atropos_select_cpu is the primary scheduling logic, invoked when a task -- * becomes runnable. The lb_data map is populated by userspace to inform the BPF -- * scheduler that a task should be migrated to a new domain. Otherwise, the task -- * is scheduled in priority order as follows: -- * * The current core if the task was woken up synchronously and there are idle -- * cpus in the system -- * * The previous core, if idle -- * * The pinned-to core if the task is pinned to a specific core -- * * Any idle cpu in the domain -- * -- * If none of the above conditions are met, then the task is enqueued to a -- * dispatch queue corresponding to the domain (atropos_enqueue). -- * -- * atropos_dispatch will attempt to consume a task from its domain's -- * corresponding dispatch queue (this occurs after scheduling any tasks directly -- * assigned to it due to the logic in atropos_select_cpu). If no task is found, -- * then greedy load stealing will attempt to find a task on another dispatch -- * queue to run. -- * -- * Load balancing is almost entirely handled by userspace. BPF populates the -- * task weight, dom mask and current dom in the task_data map and executes the -- * load balance based on userspace populating the lb_data map. -- */ --#include "../../../scx_common.bpf.h" --#include "atropos.h" -- --#include --#include --#include --#include --#include --#include -- --char _license[] SEC("license") = "GPL"; -- --/* -- * const volatiles are set during initialization and treated as consts by the -- * jit compiler. -- */ -- --/* -- * Domains and cpus -- */ --const volatile __u32 nr_doms = 32; /* !0 for veristat, set during init */ --const volatile __u32 nr_cpus = 64; /* !0 for veristat, set during init */ --const volatile __u32 cpu_dom_id_map[MAX_CPUS]; --const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; -- --const volatile bool kthreads_local; --const volatile bool fifo_sched; --const volatile bool switch_partial; --const volatile __u32 greedy_threshold; -- --/* base slice duration */ --const volatile __u64 slice_us = 20000; -- --/* -- * Exit info -- */ --int exit_type = SCX_EXIT_NONE; --char exit_msg[SCX_EXIT_MSG_LEN]; -- --struct pcpu_ctx { -- __u32 dom_rr_cur; /* used when scanning other doms */ -- -- /* libbpf-rs does not respect the alignment, so pad out the struct explicitly */ -- __u8 _padding[CACHELINE_SIZE - sizeof(u64)]; --} __attribute__((aligned(CACHELINE_SIZE))); -- --struct pcpu_ctx pcpu_ctx[MAX_CPUS]; -- --/* -- * Domain context -- */ --struct dom_ctx { -- struct bpf_cpumask __kptr *cpumask; -- u64 vtime_now; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __type(key, u32); -- __type(value, struct dom_ctx); -- __uint(max_entries, MAX_DOMS); -- __uint(map_flags, 0); --} dom_ctx SEC(".maps"); -- --/* -- * Statistics -- */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, ATROPOS_NR_STATS); --} stats SEC(".maps"); -- --static inline void stat_add(enum stat_idx idx, u64 addend) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p) += addend; --} -- --/* Map pid -> task_ctx */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, struct task_ctx); -- __uint(max_entries, 1000000); -- __uint(map_flags, 0); --} task_data SEC(".maps"); -- --/* -- * This is populated from userspace to indicate which pids should be reassigned -- * to new doms. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, u32); -- __uint(max_entries, 1000); -- __uint(map_flags, 0); --} lb_data SEC(".maps"); -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p, -- u32 new_dom_id) --{ -- struct dom_ctx *old_domc, *new_domc; -- struct bpf_cpumask *d_cpumask, *t_cpumask; -- u32 old_dom_id = task_ctx->dom_id; -- s64 vtime_delta; -- -- old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id); -- if (!old_domc) { -- scx_bpf_error("No dom%u", old_dom_id); -- return false; -- } -- -- vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now; -- -- new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id); -- if (!new_domc) { -- scx_bpf_error("No dom%u", new_dom_id); -- return false; -- } -- -- d_cpumask = new_domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to get domain %u cpumask kptr", -- new_dom_id); -- return false; -- } -- -- t_cpumask = task_ctx->cpumask; -- if (!t_cpumask) { -- scx_bpf_error("Failed to look up task cpumask"); -- return false; -- } -- -- /* -- * set_cpumask might have happened between userspace requesting LB and -- * here and @p might not be able to run in @dom_id anymore. Verify. -- */ -- if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- p->cpus_ptr)) { -- p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta; -- task_ctx->dom_id = new_dom_id; -- bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask, -- p->cpus_ptr); -- } -- -- return task_ctx->dom_id == new_dom_id; --} -- --s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu, -- u32 wake_flags) --{ -- s32 cpu; -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- struct bpf_cpumask *p_cpumask; -- -- if (!task_ctx) -- return -ENOENT; -- -- if (kthreads_local && -- (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- cpu = prev_cpu; -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local dsq of the waker. -- */ -- if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) { -- struct task_struct *current = (void *)bpf_get_current_task(); -- -- if (!(BPF_CORE_READ(current, flags) & PF_EXITING) && -- task_ctx->dom_id < MAX_DOMS) { -- struct dom_ctx *domc; -- struct bpf_cpumask *d_cpumask; -- const struct cpumask *idle_cpumask; -- bool has_idle; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id); -- if (!domc) { -- scx_bpf_error("Failed to find dom%u", -- task_ctx->dom_id); -- return prev_cpu; -- } -- d_cpumask = domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to acquire domain %u cpumask kptr", -- task_ctx->dom_id); -- return prev_cpu; -- } -- -- idle_cpumask = scx_bpf_get_idle_cpumask(); -- -- has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- idle_cpumask); -- -- scx_bpf_put_idle_cpumask(idle_cpumask); -- -- if (has_idle) { -- cpu = bpf_get_smp_processor_id(); -- if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- stat_add(ATROPOS_STAT_WAKE_SYNC, 1); -- goto local; -- } -- } -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- stat_add(ATROPOS_STAT_PREV_IDLE, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- /* If only one core is allowed, dispatch */ -- if (p->nr_cpus_allowed == 1) { -- stat_add(ATROPOS_STAT_PINNED, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) -- return -ENOENT; -- -- /* If there is an eligible idle CPU, dispatch directly */ -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- if (cpu >= 0) { -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * @prev_cpu may be in a different domain. Returning an out-of-domain -- * CPU can lead to stalls as all in-domain CPUs may be idle by the time -- * @p gets enqueued. -- */ -- if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask)) -- cpu = prev_cpu; -- else -- cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask); -- -- return cpu; -- --local: -- task_ctx->dispatch_local = true; -- return cpu; --} -- --void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- u32 *new_dom; -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- new_dom = bpf_map_lookup_elem(&lb_data, &pid); -- if (new_dom && *new_dom != task_ctx->dom_id && -- task_set_dsq(task_ctx, p, *new_dom)) { -- struct bpf_cpumask *p_cpumask; -- s32 cpu; -- -- stat_add(ATROPOS_STAT_LOAD_BALANCE, 1); -- -- /* -- * If dispatch_local is set, We own @p's idle state but we are -- * not gonna put the task in the associated local dsq which can -- * cause the CPU to stall. Kick it. -- */ -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0); -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) { -- scx_bpf_error("Failed to get task_ctx->cpumask"); -- return; -- } -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- } -- -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags); -- return; -- } -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000, -- enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- u32 dom_id = task_ctx->dom_id; -- struct dom_ctx *domc; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, domc->vtime_now - slice_us * 1000)) -- vtime = domc->vtime_now - slice_us * 1000; -- -- scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --static u32 cpu_to_dom_id(s32 cpu) --{ -- const volatile u32 *dom_idp; -- -- if (nr_doms <= 1) -- return 0; -- -- dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]); -- if (!dom_idp) -- return MAX_DOMS; -- -- return *dom_idp; --} -- --static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) --{ -- s32 cpu; -- -- if (dom_id >= MAX_DOMS) -- return false; -- -- bpf_for(cpu, 0, nr_cpus) { -- if (bpf_cpumask_test_cpu(cpu, cpumask) && -- (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64)))) -- return true; -- } -- return false; --} -- --static u32 dom_rr_next(s32 cpu) --{ -- struct pcpu_ctx *pcpuc; -- u32 dom_id; -- -- pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]); -- if (!pcpuc) -- return 0; -- -- dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms; -- -- if (dom_id == cpu_to_dom_id(cpu)) -- dom_id = (dom_id + 1) % nr_doms; -- -- pcpuc->dom_rr_cur = dom_id; -- return dom_id; --} -- --void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 dom = cpu_to_dom_id(cpu); -- -- if (scx_bpf_consume(dom)) { -- stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1); -- return; -- } -- -- if (!greedy_threshold) -- return; -- -- bpf_repeat(nr_doms - 1) { -- u32 dom_id = dom_rr_next(cpu); -- -- if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold && -- scx_bpf_consume(dom_id)) { -- stat_add(ATROPOS_STAT_GREEDY, 1); -- break; -- } -- } --} -- --void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_at = bpf_ktime_get_ns(); --} -- --void BPF_STRUCT_OPS(atropos_running, struct task_struct *p) --{ -- struct task_ctx *taskc; -- struct dom_ctx *domc; -- pid_t pid = p->pid; -- u32 dom_id; -- -- if (fifo_sched) -- return; -- -- taskc = bpf_map_lookup_elem(&task_data, &pid); -- if (!taskc) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- dom_id = taskc->dom_id; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(domc->vtime_now, p->scx.dsq_vtime)) -- domc->vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at; -- task_ctx->runnable_at = 0; --} -- --void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->weight = weight; --} -- --struct pick_task_domain_loop_ctx { -- struct task_struct *p; -- const struct cpumask *cpumask; -- u64 dom_mask; -- u32 dom_rr_base; -- u32 dom_id; --}; -- --static int pick_task_domain_loopfn(u32 idx, void *data) --{ -- struct pick_task_domain_loop_ctx *lctx = data; -- u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms; -- -- if (dom_id >= MAX_DOMS) -- return 1; -- -- if (cpumask_intersects_domain(lctx->cpumask, dom_id)) { -- lctx->dom_mask |= 1LLU << dom_id; -- if (lctx->dom_id == MAX_DOMS) -- lctx->dom_id = dom_id; -- } -- return 0; --} -- --static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- struct pick_task_domain_loop_ctx lctx = { -- .p = p, -- .cpumask = cpumask, -- .dom_id = MAX_DOMS, -- }; -- s32 cpu = bpf_get_smp_processor_id(); -- -- if (cpu < 0 || cpu >= MAX_CPUS) -- return MAX_DOMS; -- -- lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur); -- -- bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0); -- task_ctx->dom_mask = lctx.dom_mask; -- -- return lctx.dom_id; --} -- --static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- u32 dom_id = 0; -- -- if (nr_doms > 1) -- dom_id = pick_task_domain(task_ctx, p, cpumask); -- -- if (!task_set_dsq(task_ctx, p, dom_id)) -- scx_bpf_error("Failed to set domain %d for %s[%d]", -- dom_id, p->comm, p->pid); --} -- --void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_set_domain(task_ctx, p, cpumask); --} -- --s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct bpf_cpumask *cpumask; -- struct task_ctx task_ctx, *map_value; -- long ret; -- pid_t pid; -- -- memset(&task_ctx, 0, sizeof(task_ctx)); -- -- pid = p->pid; -- ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return ret; -- } -- -- /* -- * Read the entry from the map immediately so we can add the cpumask -- * with bpf_kptr_xchg(). -- */ -- map_value = bpf_map_lookup_elem(&task_data, &pid); -- if (!map_value) -- /* Should never happen -- it was just inserted above. */ -- return -EINVAL; -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- bpf_map_delete_elem(&task_data, &pid); -- return -ENOMEM; -- } -- -- cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask); -- if (cpumask) { -- /* Should never happen as we just inserted it above. */ -- bpf_cpumask_release(cpumask); -- bpf_map_delete_elem(&task_data, &pid); -- return -EINVAL; -- } -- -- task_set_domain(map_value, p, p->cpus_ptr); -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p) --{ -- pid_t pid = p->pid; -- long ret = bpf_map_delete_elem(&task_data, &pid); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return; -- } --} -- --static int create_dom_dsq(u32 idx, void *data) --{ -- struct dom_ctx domc_init = {}, *domc; -- struct bpf_cpumask *cpumask; -- u32 cpu, dom_id = idx; -- s32 ret; -- -- ret = scx_bpf_create_dsq(dom_id, -1); -- if (ret < 0) { -- scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret); -- return 1; -- } -- -- ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0); -- if (ret) { -- scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret); -- return 1; -- } -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- /* Should never happen, we just inserted it above. */ -- scx_bpf_error("No dom%u", dom_id); -- return 1; -- } -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id); -- return 1; -- } -- -- for (cpu = 0; cpu < MAX_CPUS; cpu++) { -- const volatile __u64 *dmask; -- -- dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]); -- if (!dmask) { -- scx_bpf_error("array index error"); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- if (*dmask & (1LLU << (cpu % 64))) -- bpf_cpumask_set_cpu(cpu, cpumask); -- } -- -- cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask); -- if (cpumask) { -- scx_bpf_error("Domain %u was already present", dom_id); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(atropos_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- bpf_loop(nr_doms, create_dom_dsq, NULL, 0); -- -- for (u32 i = 0; i < nr_cpus; i++) -- pcpu_ctx[i].dom_rr_cur = i; -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg); -- exit_type = ei->type; --} -- --SEC(".struct_ops") --struct sched_ext_ops atropos = { -- .select_cpu = (void *)atropos_select_cpu, -- .enqueue = (void *)atropos_enqueue, -- .dispatch = (void *)atropos_dispatch, -- .runnable = (void *)atropos_runnable, -- .running = (void *)atropos_running, -- .stopping = (void *)atropos_stopping, -- .quiescent = (void *)atropos_quiescent, -- .set_weight = (void *)atropos_set_weight, -- .set_cpumask = (void *)atropos_set_cpumask, -- .prep_enable = (void *)atropos_prep_enable, -- .disable = (void *)atropos_disable, -- .init = (void *)atropos_init, -- .exit = (void *)atropos_exit, -- .flags = 0, -- .name = "atropos", --}; -diff --git b/tools/sched_ext/atropos/src/bpf/atropos.h a/tools/sched_ext/atropos/src/bpf/atropos.h -deleted file mode 100644 -index addf29ca104a..000000000000 ---- b/tools/sched_ext/atropos/src/bpf/atropos.h -+++ /dev/null -@@ -1,44 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#ifndef __ATROPOS_H --#define __ATROPOS_H -- --#include --#ifndef __kptr --#ifdef __KERNEL__ --#error "__kptr_ref not defined in the kernel" --#endif --#define __kptr --#endif -- --#define MAX_CPUS 512 --#define MAX_DOMS 64 /* limited to avoid complex bitmask ops */ --#define CACHELINE_SIZE 64 -- --/* Statistics */ --enum stat_idx { -- ATROPOS_STAT_TASK_GET_ERR, -- ATROPOS_STAT_WAKE_SYNC, -- ATROPOS_STAT_PREV_IDLE, -- ATROPOS_STAT_PINNED, -- ATROPOS_STAT_DIRECT_DISPATCH, -- ATROPOS_STAT_DSQ_DISPATCH, -- ATROPOS_STAT_GREEDY, -- ATROPOS_STAT_LOAD_BALANCE, -- ATROPOS_STAT_LAST_TASK, -- ATROPOS_NR_STATS, --}; -- --struct task_ctx { -- unsigned long long dom_mask; /* the domains this task can run on */ -- struct bpf_cpumask __kptr *cpumask; -- unsigned int dom_id; -- unsigned int weight; -- unsigned long long runnable_at; -- unsigned long long runnable_for; -- bool dispatch_local; --}; -- --#endif /* __ATROPOS_H */ -diff --git b/tools/sched_ext/atropos/src/main.rs a/tools/sched_ext/atropos/src/main.rs -deleted file mode 100644 -index 0d313662f713..000000000000 ---- b/tools/sched_ext/atropos/src/main.rs -+++ /dev/null -@@ -1,942 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#[path = "bpf/.output/atropos.skel.rs"] --mod atropos; --pub use atropos::*; --pub mod atropos_sys; -- --use std::cell::Cell; --use std::collections::{BTreeMap, BTreeSet}; --use std::ffi::CStr; --use std::ops::Bound::{Included, Unbounded}; --use std::sync::atomic::{AtomicBool, Ordering}; --use std::sync::Arc; --use std::time::{Duration, SystemTime}; -- --use ::fb_procfs as procfs; --use anyhow::{anyhow, bail, Context, Result}; --use bitvec::prelude::*; --use clap::Parser; --use log::{info, trace, warn}; --use ordered_float::OrderedFloat; -- --/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF --/// part does simple round robin in each domain and the userspace part --/// calculates the load factor of each domain and tells the BPF part how to load --/// balance the domains. -- --/// This scheduler demonstrates dividing scheduling logic between BPF and --/// userspace and using rust to build the userspace part. An earlier variant of --/// this scheduler was used to balance across six domains, each representing a --/// chiplet in a six-chiplet AMD processor, and could match the performance of --/// production setup using CFS. --#[derive(Debug, Parser)] --struct Opts { -- /// Scheduling slice duration in microseconds. -- #[clap(short, long, default_value = "20000")] -- slice_us: u64, -- -- /// Monitoring and load balance interval in seconds. -- #[clap(short, long, default_value = "2.0")] -- interval: f64, -- -- /// Build domains according to how CPUs are grouped at this cache level -- /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id. -- #[clap(short = 'c', long, default_value = "3")] -- cache_level: u32, -- -- /// Instead of using cache locality, set the cpumask for each domain -- /// manually, provide multiple --cpumasks, one for each domain. E.g. -- /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with -- /// the corresponding CPUs belonging to each domain. Each CPU must -- /// belong to precisely one domain. -- #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")] -- cpumasks: Vec, -- -- /// When non-zero, enable greedy task stealing. When a domain is idle, a -- /// cpu will attempt to steal tasks from a domain with at least -- /// greedy_threshold tasks enqueued. These tasks aren't permanently -- /// stolen from the domain. -- #[clap(short, long, default_value = "4")] -- greedy_threshold: u32, -- -- /// The load decay factor. Every interval, the existing load is decayed -- /// by this factor and new load is added. Must be in the range [0.0, -- /// 0.99]. The smaller the value, the more sensitive load calculation -- /// is to recent changes. When 0.0, history is ignored and the load -- /// value from the latest period is used directly. -- #[clap(short, long, default_value = "0.5")] -- load_decay_factor: f64, -- -- /// Disable load balancing. Unless disabled, periodically userspace will -- /// calculate the load factor of each domain and instruct BPF which -- /// processes to move. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- no_load_balance: bool, -- -- /// Put per-cpu kthreads directly into local dsq's. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- kthreads_local: bool, -- -- /// Use FIFO scheduling instead of weighted vtime scheduling. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- fifo_sched: bool, -- -- /// If specified, only tasks which have their scheduling policy set to -- /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all -- /// tasks are switched. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- partial: bool, -- -- /// Enable verbose output including libbpf details. Specify multiple -- /// times to increase verbosity. -- #[clap(short, long, action = clap::ArgAction::Count)] -- verbose: u8, --} -- --fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result { -- Ok(reader -- .read_stat() -- .context("Failed to read procfs")? -- .total_cpu -- .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?) --} -- --fn now_monotonic() -> u64 { -- let mut time = libc::timespec { -- tv_sec: 0, -- tv_nsec: 0, -- }; -- let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) }; -- assert!(ret == 0); -- time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 --} -- --fn clear_map(map: &mut libbpf_rs::Map) { -- // XXX: libbpf_rs has some design flaw that make it impossible to -- // delete while iterating despite it being safe so we alias it here -- let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) }; -- for key in map.keys() { -- let _ = deleter.delete(&key); -- } --} -- --#[derive(Debug)] --struct TaskLoad { -- runnable_for: u64, -- load: f64, --} -- --#[derive(Debug)] --struct TaskInfo { -- pid: i32, -- dom_mask: u64, -- migrated: Cell, --} -- --struct LoadBalancer<'a, 'b, 'c> { -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- -- tasks_by_load: Vec, TaskInfo>>, -- load_avg: f64, -- dom_loads: Vec, -- -- imbal: Vec, -- doms_to_push: BTreeMap, u32>, -- doms_to_pull: BTreeMap, u32>, -- -- nr_lb_data_errors: &'c mut u64, --} -- --impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> { -- const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10; -- const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1; -- const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50; -- -- fn new( -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- nr_lb_data_errors: &'c mut u64, -- ) -> Self { -- Self { -- maps, -- task_loads, -- nr_doms, -- load_decay_factor, -- -- tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(), -- load_avg: 0f64, -- dom_loads: vec![0.0; nr_doms], -- -- imbal: vec![0.0; nr_doms], -- doms_to_pull: BTreeMap::new(), -- doms_to_push: BTreeMap::new(), -- -- nr_lb_data_errors, -- } -- } -- -- fn read_task_loads(&mut self, period: Duration) -> Result<()> { -- let now_mono = now_monotonic(); -- let task_data = self.maps.task_data(); -- let mut this_task_loads = BTreeMap::::new(); -- let mut load_sum = 0.0f64; -- self.dom_loads = vec![0f64; self.nr_doms]; -- -- for key in task_data.keys() { -- if let Some(task_ctx_vec) = task_data -- .lookup(&key, libbpf_rs::MapFlags::ANY) -- .context("Failed to lookup task_data")? -- { -- let task_ctx = -- unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) }; -- let pid = i32::from_ne_bytes( -- key.as_slice() -- .try_into() -- .context("Invalid key length in task_data map")?, -- ); -- -- let (this_at, this_for, weight) = unsafe { -- ( -- std::ptr::read_volatile(&task_ctx.runnable_at as *const u64), -- std::ptr::read_volatile(&task_ctx.runnable_for as *const u64), -- std::ptr::read_volatile(&task_ctx.weight as *const u32), -- ) -- }; -- -- let (mut delta, prev_load) = match self.task_loads.get(&pid) { -- Some(prev) => (this_for - prev.runnable_for, Some(prev.load)), -- None => (this_for, None), -- }; -- -- // Non-zero this_at indicates that the task is currently -- // runnable. Note that we read runnable_at and runnable_for -- // without any synchronization and there is a small window -- // where we end up misaccounting. While this can cause -- // temporary error, it's unlikely to cause any noticeable -- // misbehavior especially given the load value clamping. -- if this_at > 0 && this_at < now_mono { -- delta += now_mono - this_at; -- } -- -- delta = delta.min(period.as_nanos() as u64); -- let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64) -- .clamp(0.0, weight as f64); -- -- let this_load = match prev_load { -- Some(prev_load) => { -- prev_load * self.load_decay_factor -- + this_load * (1.0 - self.load_decay_factor) -- } -- None => this_load, -- }; -- -- this_task_loads.insert( -- pid, -- TaskLoad { -- runnable_for: this_for, -- load: this_load, -- }, -- ); -- -- load_sum += this_load; -- self.dom_loads[task_ctx.dom_id as usize] += this_load; -- // Only record pids that are eligible for load balancing -- if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) { -- continue; -- } -- self.tasks_by_load[task_ctx.dom_id as usize].insert( -- OrderedFloat(this_load), -- TaskInfo { -- pid, -- dom_mask: task_ctx.dom_mask, -- migrated: Cell::new(false), -- }, -- ); -- } -- } -- -- self.load_avg = load_sum / self.nr_doms as f64; -- *self.task_loads = this_task_loads; -- Ok(()) -- } -- -- // To balance dom loads we identify doms with lower and higher load than average -- fn calculate_dom_load_balance(&mut self) -> Result<()> { -- for (dom, dom_load) in self.dom_loads.iter().enumerate() { -- let imbal = dom_load - self.load_avg; -- if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO { -- if imbal > 0f64 { -- self.doms_to_push.insert(OrderedFloat(imbal), dom as u32); -- } else { -- self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32); -- } -- self.imbal[dom] = imbal; -- } -- } -- Ok(()) -- } -- -- // Find the first candidate pid which hasn't already been migrated and -- // can run in @pull_dom. -- fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)> -- where -- I: IntoIterator, &'d TaskInfo)>, -- { -- match tasks_by_load -- .into_iter() -- .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0) -- .next() -- { -- Some((OrderedFloat(load), task)) => Some((*load, task)), -- None => None, -- } -- } -- -- fn pick_victim( -- &self, -- (push_dom, to_push): (u32, f64), -- (pull_dom, to_pull): (u32, f64), -- ) -> Option<(&TaskInfo, f64)> { -- let to_xfer = to_pull.min(to_push); -- -- trace!( -- "considering dom {}@{:.2} -> {}@{:.2}", -- push_dom, -- to_push, -- pull_dom, -- to_pull -- ); -- -- let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs(); -- -- trace!( -- "to_xfer={:.2} tasks_by_load={:?}", -- to_xfer, -- &self.tasks_by_load[push_dom as usize] -- ); -- -- // We want to pick a task to transfer from push_dom to pull_dom to -- // maximize the reduction of load imbalance between the two. IOW, -- // pick a task which has the closest load value to $to_xfer that can -- // be migrated. Find such task by locating the first migratable task -- // while scanning left from $to_xfer and the counterpart while -- // scanning right and picking the better of the two. -- let (load, task, new_imbal) = match ( -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Unbounded, Included(&OrderedFloat(to_xfer)))) -- .rev(), -- pull_dom, -- ), -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Included(&OrderedFloat(to_xfer)), Unbounded)), -- pull_dom, -- ), -- ) { -- (None, None) => return None, -- (Some((load, task)), None) | (None, Some((load, task))) => { -- (load, task, calc_new_imbal(load)) -- } -- (Some((load0, task0)), Some((load1, task1))) => { -- let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1)); -- if new_imbal0 <= new_imbal1 { -- (load0, task0, new_imbal0) -- } else { -- (load1, task1, new_imbal1) -- } -- } -- }; -- -- // If the best candidate can't reduce the imbalance, there's nothing -- // to do for this pair. -- let old_imbal = to_push + to_pull; -- if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal { -- trace!( -- "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal -- ); -- return None; -- } -- -- trace!( -- "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal, -- ); -- -- Some((task, load)) -- } -- -- // Actually execute the load balancing. Concretely this writes pid -> dom -- // entries into the lb_data map for bpf side to consume. -- fn load_balance(&mut self) -> Result<()> { -- clear_map(self.maps.lb_data()); -- -- trace!("imbal={:?}", &self.imbal); -- trace!("doms_to_push={:?}", &self.doms_to_push); -- trace!("doms_to_pull={:?}", &self.doms_to_pull); -- -- // Push from the most imbalanced to least. -- while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() { -- let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO; -- let mut pushed = 0f64; -- -- // Transfer tasks from push_dom to reduce imbalance. -- loop { -- let last_pushed = pushed; -- -- // Pull from the most imbalaned to least. -- let mut doms_to_pull = BTreeMap::<_, _>::new(); -- std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull); -- let mut pull_doms = doms_to_pull.into_iter().rev().collect::>(); -- -- for (to_pull, pull_dom) in pull_doms.iter_mut() { -- if let Some((task, load)) = -- self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull))) -- { -- // Execute migration. -- task.migrated.set(true); -- to_push -= load; -- *to_pull -= load; -- pushed += load; -- -- // Ask BPF code to execute the migration. -- let pid = task.pid; -- let cpid = (pid as libc::pid_t).to_ne_bytes(); -- if let Err(e) = self.maps.lb_data().update( -- &cpid, -- &pull_dom.to_ne_bytes(), -- libbpf_rs::MapFlags::NO_EXIST, -- ) { -- warn!( -- "Failed to update lb_data map for pid={} error={:?}", -- pid, &e -- ); -- *self.nr_lb_data_errors += 1; -- } -- -- // Always break after a successful migration so that -- // the pulling domains are always considered in the -- // descending imbalance order. -- break; -- } -- } -- -- pull_doms -- .into_iter() -- .map(|(k, v)| self.doms_to_pull.insert(k, v)) -- .count(); -- -- // Stop repeating if nothing got transferred or pushed enough. -- if pushed == last_pushed || pushed >= push_max { -- break; -- } -- } -- } -- Ok(()) -- } --} -- --struct Scheduler<'a> { -- skel: AtroposSkel<'a>, -- struct_ops: Option, -- -- nr_cpus: usize, -- nr_doms: usize, -- load_decay_factor: f64, -- balance_load: bool, -- -- proc_reader: procfs::ProcReader, -- -- prev_at: SystemTime, -- prev_total_cpu: procfs::CpuStat, -- task_loads: BTreeMap, -- -- nr_lb_data_errors: u64, --} -- --impl<'a> Scheduler<'a> { -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn parse_cpusets( -- cpumasks: &[String], -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- if cpumasks.len() > atropos_sys::MAX_DOMS as usize { -- bail!( -- "Number of requested DSQs ({}) is greater than MAX_DOMS ({})", -- cpumasks.len(), -- atropos_sys::MAX_DOMS -- ); -- } -- let mut cpus = vec![-1i32; nr_cpus]; -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()]; -- for (dq, cpumask) in cpumasks.iter().enumerate() { -- let hex_str = { -- let mut tmp_str = cpumask -- .strip_prefix("0x") -- .unwrap_or(cpumask) -- .replace('_', ""); -- if tmp_str.len() % 2 != 0 { -- tmp_str = "0".to_string() + &tmp_str; -- } -- tmp_str -- }; -- let byte_vec = hex::decode(&hex_str) -- .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?; -- -- for (index, &val) in byte_vec.iter().rev().enumerate() { -- let mut v = val; -- while v != 0 { -- let lsb = v.trailing_zeros() as usize; -- v &= !(1 << lsb); -- let cpu = index * 8 + lsb; -- if cpu > nr_cpus { -- bail!( -- concat!( -- "Found cpu ({}) in cpumask ({}) which is larger", -- " than the number of cpus on the machine ({})" -- ), -- cpu, -- cpumask, -- nr_cpus -- ); -- } -- if cpus[cpu] != -1 { -- bail!( -- "Found cpu ({}) with dq ({}) but also in cpumask ({})", -- cpu, -- cpus[cpu], -- cpumask -- ); -- } -- cpus[cpu] = dq as i32; -- cpusets[dq].set(cpu, true); -- } -- } -- cpusets[dq].set_uninitialized(false); -- } -- -- for (cpu, &dq) in cpus.iter().enumerate() { -- if dq < 0 { -- bail!( -- "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.", -- cpu -- ); -- } -- } -- -- Ok((cpusets, cpus)) -- } -- -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn cpusets_from_cache( -- level: u32, -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- let mut cpu_to_cache = vec![]; // (cpu_id, cache_id) -- let mut cache_ids = BTreeSet::::new(); -- let mut nr_not_found = 0; -- -- // Build cpu -> cache ID mapping. -- for cpu in 0..nr_cpus { -- let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level); -- let id = match std::fs::read_to_string(&path) { -- Ok(val) => val -- .trim() -- .parse::() -- .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?, -- Err(e) if e.kind() == std::io::ErrorKind::NotFound => { -- nr_not_found += 1; -- 0 -- } -- Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)), -- }; -- -- cpu_to_cache.push(id); -- cache_ids.insert(id); -- } -- -- if nr_not_found > 1 { -- warn!( -- "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0", -- level, nr_not_found, nr_cpus -- ); -- } -- -- // Cache IDs may have holes. Assign consecutive domain IDs to -- // existing cache IDs. -- let mut cache_to_dom = BTreeMap::::new(); -- let mut nr_doms = 0; -- for cache_id in cache_ids.iter() { -- cache_to_dom.insert(*cache_id, nr_doms); -- nr_doms += 1; -- } -- -- if nr_doms > atropos_sys::MAX_DOMS { -- bail!( -- "Total number of doms {} is greater than MAX_DOMS ({})", -- nr_doms, -- atropos_sys::MAX_DOMS -- ); -- } -- -- // Build and return dom -> cpumask and cpu -> dom mappings. -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize]; -- let mut cpu_to_dom = vec![]; -- -- for cpu in 0..nr_cpus { -- let dom_id = cache_to_dom[&cpu_to_cache[cpu]]; -- cpusets[dom_id as usize].set(cpu, true); -- cpu_to_dom.push(dom_id as i32); -- } -- -- Ok((cpusets, cpu_to_dom)) -- } -- -- fn init(opts: &Opts) -> Result { -- // Open the BPF prog first for verification. -- let mut skel_builder = AtroposSkelBuilder::default(); -- skel_builder.obj_builder.debug(opts.verbose > 0); -- let mut skel = skel_builder.open().context("Failed to open BPF program")?; -- -- let nr_cpus = libbpf_rs::num_possible_cpus().unwrap(); -- if nr_cpus > atropos_sys::MAX_CPUS as usize { -- bail!( -- "nr_cpus ({}) is greater than MAX_CPUS ({})", -- nr_cpus, -- atropos_sys::MAX_CPUS -- ); -- } -- -- // Initialize skel according to @opts. -- let (cpusets, cpus) = if opts.cpumasks.len() > 0 { -- Self::parse_cpusets(&opts.cpumasks, nr_cpus)? -- } else { -- Self::cpusets_from_cache(opts.cache_level, nr_cpus)? -- }; -- let nr_doms = cpusets.len(); -- skel.rodata().nr_doms = nr_doms as u32; -- skel.rodata().nr_cpus = nr_cpus as u32; -- -- for (cpu, dom) in cpus.iter().enumerate() { -- skel.rodata().cpu_dom_id_map[cpu] = *dom as u32; -- } -- -- for (dom, cpuset) in cpusets.iter().enumerate() { -- let raw_cpuset_slice = cpuset.as_raw_slice(); -- let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom]; -- let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len()); -- left.clone_from_slice(cpuset.as_raw_slice()); -- let cpumask_str = dom_cpumask_slice -- .iter() -- .take((nr_cpus + 63) / 64) -- .rev() -- .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x)); -- info!( -- "DOM[{:02}] cpumask{} ({} cpus)", -- dom, -- &cpumask_str, -- cpuset.count_ones() -- ); -- } -- -- skel.rodata().slice_us = opts.slice_us; -- skel.rodata().kthreads_local = opts.kthreads_local; -- skel.rodata().fifo_sched = opts.fifo_sched; -- skel.rodata().switch_partial = opts.partial; -- skel.rodata().greedy_threshold = opts.greedy_threshold; -- -- // Attach. -- let mut skel = skel.load().context("Failed to load BPF program")?; -- skel.attach().context("Failed to attach BPF program")?; -- let struct_ops = Some( -- skel.maps_mut() -- .atropos() -- .attach_struct_ops() -- .context("Failed to attach atropos struct ops")?, -- ); -- info!("Atropos Scheduler Attached"); -- -- // Other stuff. -- let mut proc_reader = procfs::ProcReader::new(); -- let prev_total_cpu = read_total_cpu(&mut proc_reader)?; -- -- Ok(Self { -- skel, -- struct_ops, // should be held to keep it attached -- -- nr_cpus, -- nr_doms, -- load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99), -- balance_load: !opts.no_load_balance, -- -- proc_reader, -- -- prev_at: SystemTime::now(), -- prev_total_cpu, -- task_loads: BTreeMap::new(), -- -- nr_lb_data_errors: 0, -- }) -- } -- -- fn get_cpu_busy(&mut self) -> Result { -- let total_cpu = read_total_cpu(&mut self.proc_reader)?; -- let busy = match (&self.prev_total_cpu, &total_cpu) { -- ( -- procfs::CpuStat { -- user_usec: Some(prev_user), -- nice_usec: Some(prev_nice), -- system_usec: Some(prev_system), -- idle_usec: Some(prev_idle), -- iowait_usec: Some(prev_iowait), -- irq_usec: Some(prev_irq), -- softirq_usec: Some(prev_softirq), -- stolen_usec: Some(prev_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- procfs::CpuStat { -- user_usec: Some(curr_user), -- nice_usec: Some(curr_nice), -- system_usec: Some(curr_system), -- idle_usec: Some(curr_idle), -- iowait_usec: Some(curr_iowait), -- irq_usec: Some(curr_irq), -- softirq_usec: Some(curr_softirq), -- stolen_usec: Some(curr_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- ) => { -- let idle_usec = curr_idle - prev_idle; -- let iowait_usec = curr_iowait - prev_iowait; -- let user_usec = curr_user - prev_user; -- let system_usec = curr_system - prev_system; -- let nice_usec = curr_nice - prev_nice; -- let irq_usec = curr_irq - prev_irq; -- let softirq_usec = curr_softirq - prev_softirq; -- let stolen_usec = curr_stolen - prev_stolen; -- -- let busy_usec = -- user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec; -- let total_usec = idle_usec + busy_usec + iowait_usec; -- busy_usec as f64 / total_usec as f64 -- } -- _ => { -- bail!("Some procfs stats are not populated!"); -- } -- }; -- -- self.prev_total_cpu = total_cpu; -- Ok(busy) -- } -- -- fn read_bpf_stats(&mut self) -> Result> { -- let mut maps = self.skel.maps_mut(); -- let stats_map = maps.stats(); -- let mut stats: Vec = Vec::new(); -- let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus]; -- -- for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS { -- let cpu_stat_vec = stats_map -- .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY) -- .with_context(|| format!("Failed to lookup stat {}", stat))? -- .expect("per-cpu stat should exist"); -- let sum = cpu_stat_vec -- .iter() -- .map(|val| { -- u64::from_ne_bytes( -- val.as_slice() -- .try_into() -- .expect("Invalid value length in stat map"), -- ) -- }) -- .sum(); -- stats_map -- .update_percpu( -- &(stat as u32).to_ne_bytes(), -- &zero_vec, -- libbpf_rs::MapFlags::ANY, -- ) -- .context("Failed to zero stat")?; -- stats.push(sum); -- } -- Ok(stats) -- } -- -- fn report( -- &self, -- stats: &Vec, -- cpu_busy: f64, -- processing_dur: Duration, -- load_avg: f64, -- dom_loads: &Vec, -- imbal: &Vec, -- ) { -- let stat = |idx| stats[idx as usize]; -- let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK); -- -- info!( -- "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms", -- cpu_busy * 100.0, -- load_avg, -- stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize], -- stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize], -- self.nr_lb_data_errors, -- processing_dur.as_millis(), -- ); -- -- let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0; -- -- info!( -- "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}", -- total, -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY), -- ); -- -- for i in 0..self.nr_doms { -- info!( -- "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}", -- i, -- dom_loads[i], -- if imbal[i] < 0.0 { -imbal[i] } else { 0.0 }, -- if imbal[i] > 0.0 { imbal[i] } else { 0.0 }, -- ); -- } -- } -- -- fn step(&mut self) -> Result<()> { -- let started_at = std::time::SystemTime::now(); -- let bpf_stats = self.read_bpf_stats()?; -- let cpu_busy = self.get_cpu_busy()?; -- -- let mut lb = LoadBalancer::new( -- self.skel.maps_mut(), -- &mut self.task_loads, -- self.nr_doms, -- self.load_decay_factor, -- &mut self.nr_lb_data_errors, -- ); -- -- lb.read_task_loads(started_at.duration_since(self.prev_at)?)?; -- lb.calculate_dom_load_balance()?; -- -- if self.balance_load { -- lb.load_balance()?; -- } -- -- // Extract fields needed for reporting and drop lb to release -- // mutable borrows. -- let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal); -- -- self.report( -- &bpf_stats, -- cpu_busy, -- std::time::SystemTime::now().duration_since(started_at)?, -- load_avg, -- &dom_loads, -- &imbal, -- ); -- -- self.prev_at = started_at; -- Ok(()) -- } -- -- fn read_bpf_exit_type(&mut self) -> i32 { -- unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) } -- } -- -- fn report_bpf_exit_type(&mut self) -> Result<()> { -- // Report msg if EXT_OPS_EXIT_ERROR. -- match self.read_bpf_exit_type() { -- 0 => Ok(()), -- etype if etype == 2 => { -- let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) }; -- let msg = cstr -- .to_str() -- .context("Failed to convert exit msg to string") -- .unwrap(); -- bail!("BPF exit_type={} msg={}", etype, msg); -- } -- etype => { -- info!("BPF exit_type={}", etype); -- Ok(()) -- } -- } -- } --} -- --impl<'a> Drop for Scheduler<'a> { -- fn drop(&mut self) { -- if let Some(struct_ops) = self.struct_ops.take() { -- drop(struct_ops); -- } -- } --} -- --fn main() -> Result<()> { -- let opts = Opts::parse(); -- -- let llv = match opts.verbose { -- 0 => simplelog::LevelFilter::Info, -- 1 => simplelog::LevelFilter::Debug, -- _ => simplelog::LevelFilter::Trace, -- }; -- let mut lcfg = simplelog::ConfigBuilder::new(); -- lcfg.set_time_level(simplelog::LevelFilter::Error) -- .set_location_level(simplelog::LevelFilter::Off) -- .set_target_level(simplelog::LevelFilter::Off) -- .set_thread_level(simplelog::LevelFilter::Off); -- simplelog::TermLogger::init( -- llv, -- lcfg.build(), -- simplelog::TerminalMode::Stderr, -- simplelog::ColorChoice::Auto, -- )?; -- -- let shutdown = Arc::new(AtomicBool::new(false)); -- let shutdown_clone = shutdown.clone(); -- ctrlc::set_handler(move || { -- shutdown_clone.store(true, Ordering::Relaxed); -- }) -- .context("Error setting Ctrl-C handler")?; -- -- let mut sched = Scheduler::init(&opts)?; -- -- while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 { -- std::thread::sleep(Duration::from_secs_f64(opts.interval)); -- sched.step()?; -- } -- -- sched.report_bpf_exit_type() --} -diff --git b/tools/sched_ext/gnu/stubs.h a/tools/sched_ext/gnu/stubs.h -deleted file mode 100644 -index 719225b16626..000000000000 ---- b/tools/sched_ext/gnu/stubs.h -+++ /dev/null -@@ -1 +0,0 @@ --/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ -diff --git b/tools/sched_ext/scx_common.bpf.h a/tools/sched_ext/scx_common.bpf.h -deleted file mode 100644 -index e56de9dc86f2..000000000000 ---- b/tools/sched_ext/scx_common.bpf.h -+++ /dev/null -@@ -1,288 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __SCHED_EXT_COMMON_BPF_H --#define __SCHED_EXT_COMMON_BPF_H -- --#include "vmlinux.h" --#include --#include --#include --#include "user_exit_info.h" -- --#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ --#define PF_EXITING 0x00000004 --#define CLOCK_MONOTONIC 1 -- --/* -- * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can -- * lead to really confusing misbehaviors. Let's trigger a build failure. -- */ --static inline void ___vmlinux_h_sanity_check___(void) --{ -- _Static_assert(SCX_DSQ_FLAG_BUILTIN, -- "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); --} -- --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; -- --static inline __attribute__((format(printf, 1, 2))) --void ___scx_bpf_error_format_checker(const char *fmt, ...) {} -- --/* -- * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments -- * instead of an array of u64. Note that __param[] must have at least one -- * element to keep the verifier happy. -- */ --#define scx_bpf_error(fmt, args...) \ --({ \ -- static char ___fmt[] = fmt; \ -- unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ -- \ -- _Pragma("GCC diagnostic push") \ -- _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ -- ___bpf_fill(___param, args); \ -- _Pragma("GCC diagnostic pop") \ -- \ -- scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ -- \ -- ___scx_bpf_error_format_checker(fmt, ##args); \ --}) -- --void scx_bpf_switch_all(void) __ksym; --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; --bool scx_bpf_consume(u64 dsq_id) __ksym; --u32 scx_bpf_dispatch_nr_slots(void) __ksym; --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym; --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym; --void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; --s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym; --const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; --const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; --void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; --void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; --bool scx_bpf_task_running(const struct task_struct *p) __ksym; --s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym; --u32 scx_bpf_reenqueue_local(void) __ksym; -- --#define BPF_STRUCT_OPS(name, args...) \ --SEC("struct_ops/"#name) \ --BPF_PROG(name, ##args) -- --#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ --SEC("struct_ops.s/"#name) \ --BPF_PROG(name, ##args) -- --/** -- * MEMBER_VPTR - Obtain the verified pointer to a struct or array member -- * @base: struct or array to index -- * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...) -- * -- * The verifier often gets confused by the instruction sequence the compiler -- * generates for indexing struct fields or arrays. This macro forces the -- * compiler to generate a code sequence which first calculates the byte offset, -- * checks it against the struct or array size and add that byte offset to -- * generate the pointer to the member to help the verifier. -- * -- * Ideally, we want to abort if the calculated offset is out-of-bounds. However, -- * BPF currently doesn't support abort, so evaluate to NULL instead. The caller -- * must check for NULL and take appropriate action to appease the verifier. To -- * avoid confusing the verifier, it's best to check for NULL and dereference -- * immediately. -- * -- * vptr = MEMBER_VPTR(my_array, [i][j]); -- * if (!vptr) -- * return error; -- * *vptr = new_value; -- */ --#define MEMBER_VPTR(base, member) (typeof(base member) *)({ \ -- u64 __base = (u64)base; \ -- u64 __addr = (u64)&(base member) - __base; \ -- asm volatile ( \ -- "if %0 <= %[max] goto +2\n" \ -- "%0 = 0\n" \ -- "goto +1\n" \ -- "%0 += %1\n" \ -- : "+r"(__addr) \ -- : "r"(__base), \ -- [max]"i"(sizeof(base) - sizeof(base member))); \ -- __addr; \ --}) -- --/* -- * BPF core and other generic helpers -- */ -- --/* list and rbtree */ --#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) --#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) -- --void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; --void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; -- --#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) --#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) -- --void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; --struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; --struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, -- struct bpf_rb_node *node) __ksym; --void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, -- bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym; --struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; -- --/* task */ --struct task_struct *bpf_task_from_pid(s32 pid) __ksym; --struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; --void bpf_task_release(struct task_struct *p) __ksym; -- --/* cgroup */ --struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; --void bpf_cgroup_release(struct cgroup *cgrp) __ksym; --struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; -- --/* cpumask */ --struct bpf_cpumask *bpf_cpumask_create(void) __ksym; --struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; --u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; --u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym; -- --/* rcu */ --void bpf_rcu_read_lock(void) __ksym; --void bpf_rcu_read_unlock(void) __ksym; -- --/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */ --struct bpf_iter_num; -- --extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym; --extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym; --extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym; -- --#ifndef bpf_for_each --/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for -- * using BPF open-coded iterators without having to write mundane explicit -- * low-level loop logic. Instead, it provides for()-like generic construct -- * that can be used pretty naturally. E.g., for some hypothetical cgroup -- * iterator, you'd write: -- * -- * struct cgroup *cg, *parent_cg = <...>; -- * -- * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { -- * bpf_printk("Child cgroup id = %d", cg->cgroup_id); -- * if (cg->cgroup_id == 123) -- * break; -- * } -- * -- * I.e., it looks almost like high-level for each loop in other languages, -- * supports continue/break, and is verifiable by BPF verifier. -- * -- * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) -- * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to -- * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int -- * *`, not just `int`. So for integers bpf_for() is more convenient. -- * -- * Note: this macro relies on C99 feature of allowing to declare variables -- * inside for() loop, bound to for() loop lifetime. It also utilizes GCC -- * extension: __attribute__((cleanup())), supported by both GCC and -- * Clang. -- */ --#define bpf_for_each(type, cur, args...) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ -- cleanup(bpf_iter_##type##_destroy))), \ -- /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_##type##_new(&___it, ##args), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_##type##_destroy, (void *)0); \ -- /* iteration and termination check */ \ -- (((cur) = bpf_iter_##type##_next(&___it))); \ --) --#endif /* bpf_for_each */ -- --#ifndef bpf_for --/* bpf_for(i, start, end) implements a for()-like looping construct that sets -- * provided integer variable *i* to values starting from *start* through, -- * but not including, *end*. It also proves to BPF verifier that *i* belongs -- * to range [start, end), so this can be used for accessing arrays without -- * extra checks. -- * -- * Note: *start* and *end* are assumed to be expressions with no side effects -- * and whose values do not change throughout bpf_for() loop execution. They do -- * not have to be statically known or constant, though. -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_for(i, start, end) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, (start), (end)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- ({ \ -- /* iteration step */ \ -- int *___t = bpf_iter_num_next(&___it); \ -- /* termination and bounds check */ \ -- (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ -- }); \ --) --#endif /* bpf_for */ -- --#ifndef bpf_repeat --/* bpf_repeat(N) performs N iterations without exposing iteration number -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_repeat(N) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, 0, (N)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- bpf_iter_num_next(&___it); \ -- /* nothing here */ \ --) --#endif /* bpf_repeat */ -- --#endif /* __SCHED_EXT_COMMON_BPF_H */ -diff --git b/tools/sched_ext/scx_example_central.bpf.c a/tools/sched_ext/scx_example_central.bpf.c -deleted file mode 100644 -index 4cec04b4c2ed..000000000000 ---- b/tools/sched_ext/scx_example_central.bpf.c -+++ /dev/null -@@ -1,334 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A central FIFO sched_ext scheduler which demonstrates the followings: -- * -- * a. Making all scheduling decisions from one CPU: -- * -- * The central CPU is the only one making scheduling decisions. All other -- * CPUs kick the central CPU when they run out of tasks to run. -- * -- * There is one global BPF queue and the central CPU schedules all CPUs by -- * dispatching from the global queue to each CPU's local dsq from dispatch(). -- * This isn't the most straightforward. e.g. It'd be easier to bounce -- * through per-CPU BPF queues. The current design is chosen to maximally -- * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching. -- * -- * b. Tickless operation -- * -- * All tasks are dispatched with the infinite slice which allows stopping the -- * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full -- * parameter. The tickless operation can be observed through -- * /proc/interrupts. -- * -- * Periodic switching is enforced by a periodic timer checking all CPUs and -- * preempting them as necessary. Unfortunately, BPF timer currently doesn't -- * have a way to pin to a specific CPU, so the periodic timer isn't pinned to -- * the central CPU. -- * -- * c. Preemption -- * -- * Kthreads are unconditionally queued to the head of a matching local dsq -- * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always -- * prioritized over user threads, which is required for ensuring forward -- * progress as e.g. the periodic timer may run on a ksoftirqd and if the -- * ksoftirqd gets starved by a user thread, there may not be anything else to -- * vacate that user thread. -- * -- * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the -- * next tasks. -- * -- * This scheduler is designed to maximize usage of various SCX mechanisms. A -- * more practical implementation would likely put the scheduling loop outside -- * the central CPU's dispatch() path and add some form of priority mechanism. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --enum { -- FALLBACK_DSQ_ID = 0, -- MAX_CPUS = 4096, -- MS_TO_NS = 1000LLU * 1000, -- TIMER_INTERVAL_NS = 1 * MS_TO_NS, --}; -- --const volatile bool switch_partial; --const volatile s32 central_cpu; --const volatile u32 nr_cpu_ids = 64; /* !0 for veristat, set during init */ -- --u64 nr_total, nr_locals, nr_queued, nr_lost_pids; --u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries; --u64 nr_overflows; -- --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, s32); --} central_q SEC(".maps"); -- --/* can't use percpu map due to bad lookups */ --static bool cpu_gimme_task[MAX_CPUS]; --static u64 cpu_started_at[MAX_CPUS]; -- --struct central_timer { -- struct bpf_timer timer; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, 1); -- __type(key, u32); -- __type(value, struct central_timer); --} central_timer SEC(".maps"); -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- /* -- * Steer wakeups to the central CPU as much as possible to avoid -- * disturbing other CPUs. It's safe to blindly return the central cpu as -- * select_cpu() is a hint and if @p can't be on it, the kernel will -- * automatically pick a fallback CPU. -- */ -- return central_cpu; --} -- --void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags) --{ -- s32 pid = p->pid; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- /* -- * Push per-cpu kthreads at the head of local dsq's and preempt the -- * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked -- * behind other threads which is necessary for forward progress -- * guarantee as we depend on the BPF timer which may run from ksoftirqd. -- */ -- if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- __sync_fetch_and_add(&nr_locals, 1); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF, -- enq_flags | SCX_ENQ_PREEMPT); -- return; -- } -- -- if (bpf_map_push_elem(¢ral_q, &pid, 0)) { -- __sync_fetch_and_add(&nr_overflows, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_queued, 1); -- -- if (!scx_bpf_task_running(p)) -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); --} -- --static bool dispatch_to_cpu(s32 cpu) --{ -- struct task_struct *p; -- s32 pid; -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (bpf_map_pop_elem(¢ral_q, &pid)) -- break; -- -- __sync_fetch_and_sub(&nr_queued, 1); -- -- p = bpf_task_from_pid(pid); -- if (!p) { -- __sync_fetch_and_add(&nr_lost_pids, 1); -- continue; -- } -- -- /* -- * If we can't run the task at the top, do the dumb thing and -- * bounce it to the fallback dsq. -- */ -- if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- __sync_fetch_and_add(&nr_mismatches, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0); -- bpf_task_release(p); -- continue; -- } -- -- /* dispatch to local and mark that @cpu doesn't need more */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0); -- -- if (cpu != central_cpu) -- scx_bpf_kick_cpu(cpu, 0); -- -- bpf_task_release(p); -- return true; -- } -- -- return false; --} -- --void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (cpu == central_cpu) { -- /* dispatch for all other CPUs first */ -- __sync_fetch_and_add(&nr_dispatches, 1); -- -- bpf_for(cpu, 0, nr_cpu_ids) { -- bool *gimme; -- -- if (!scx_bpf_dispatch_nr_slots()) -- break; -- -- /* central's gimme is never set */ -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme && !*gimme) -- continue; -- -- if (dispatch_to_cpu(cpu)) -- *gimme = false; -- } -- -- /* -- * Retry if we ran out of dispatch buffer slots as we might have -- * skipped some CPUs and also need to dispatch for self. The ext -- * core automatically retries if the local dsq is empty but we -- * can't rely on that as we're dispatching for other CPUs too. -- * Kick self explicitly to retry. -- */ -- if (!scx_bpf_dispatch_nr_slots()) { -- __sync_fetch_and_add(&nr_retries, 1); -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- return; -- } -- -- /* look for a task to run on the central CPU */ -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- dispatch_to_cpu(central_cpu); -- } else { -- bool *gimme; -- -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme) -- *gimme = true; -- -- /* -- * Force dispatch on the scheduling CPU so that it finds a task -- * to run for us. -- */ -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- } --} -- --void BPF_STRUCT_OPS(central_running, struct task_struct *p) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ --} -- --void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = 0; --} -- --static int central_timerfn(void *map, int *key, struct bpf_timer *timer) --{ -- u64 now = bpf_ktime_get_ns(); -- u64 nr_to_kick = nr_queued; -- s32 i; -- -- bpf_for(i, 0, nr_cpu_ids) { -- s32 cpu = (nr_timers + i) % nr_cpu_ids; -- u64 *started_at; -- -- if (cpu == central_cpu) -- continue; -- -- /* kick iff the current one exhausted its slice */ -- started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at && *started_at && -- vtime_before(now, *started_at + SCX_SLICE_DFL)) -- continue; -- -- /* and there's something pending */ -- if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) || -- scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu)) -- ; -- else if (nr_to_kick) -- nr_to_kick--; -- else -- continue; -- -- scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); -- } -- -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- -- bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- __sync_fetch_and_add(&nr_timers, 1); -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(central_init) --{ -- u32 key = 0; -- struct bpf_timer *timer; -- int ret; -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1); -- if (ret) -- return ret; -- -- timer = bpf_map_lookup_elem(¢ral_timer, &key); -- if (!timer) -- return -ESRCH; -- -- bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); -- bpf_timer_set_callback(timer, central_timerfn); -- ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- return ret; --} -- --void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops central_ops = { -- /* -- * We are offloading all scheduling decisions to the central CPU and -- * thus being the last task on a given CPU doesn't mean anything -- * special. Enqueue the last tasks like any other tasks. -- */ -- .flags = SCX_OPS_ENQ_LAST, -- -- .select_cpu = (void *)central_select_cpu, -- .enqueue = (void *)central_enqueue, -- .dispatch = (void *)central_dispatch, -- .running = (void *)central_running, -- .stopping = (void *)central_stopping, -- .init = (void *)central_init, -- .exit = (void *)central_exit, -- .name = "central", --}; -diff --git b/tools/sched_ext/scx_example_central.c a/tools/sched_ext/scx_example_central.c -deleted file mode 100644 -index 7ad591cbdc65..000000000000 ---- b/tools/sched_ext/scx_example_central.c -+++ /dev/null -@@ -1,94 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_central.skel.h" -- --const char help_fmt[] = --"A central FIFO sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-c CPU] [-p]\n" --"\n" --" -c CPU Override the central CPU (default: 0)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_central *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_central__open(); -- assert(skel); -- -- skel->rodata->central_cpu = 0; -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "c:ph")) != -1) { -- switch (opt) { -- case 'c': -- skel->rodata->central_cpu = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_central__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.central_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf("total :%10lu local:%10lu queued:%10lu lost:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_locals, -- skel->bss->nr_queued, -- skel->bss->nr_lost_pids); -- printf("timer :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n", -- skel->bss->nr_timers, -- skel->bss->nr_dispatches, -- skel->bss->nr_mismatches, -- skel->bss->nr_retries); -- printf("overflow:%10lu\n", -- skel->bss->nr_overflows); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_central__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_flatcg.bpf.c a/tools/sched_ext/scx_example_flatcg.bpf.c -deleted file mode 100644 -index f6078b9a681f..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.bpf.c -+++ /dev/null -@@ -1,872 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext flattened cgroup hierarchy scheduler. It implements -- * hierarchical weight-based cgroup CPU control by flattening the cgroup -- * hierarchy into a single layer by compounding the active weight share at each -- * level. Consider the following hierarchy with weights in parentheses: -- * -- * R + A (100) + B (100) -- * | \ C (100) -- * \ D (200) -- * -- * Ignoring the root and threaded cgroups, only B, C and D can contain tasks. -- * Let's say all three have runnable tasks. The total share that each of these -- * three cgroups is entitled to can be calculated by compounding its share at -- * each level. -- * -- * For example, B is competing against C and in that competition its share is -- * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's -- * share in that competition is 200/(200+100) == 1/3. B's eventual share in the -- * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's -- * eventual shaer is the same at 1/6. D is only competing at the top level and -- * its share is 200/(100+200) == 2/3. -- * -- * So, instead of hierarchically scheduling level-by-level, we can consider it -- * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3 -- * and keep updating the eventual shares as the cgroups' runnable states change. -- * -- * This flattening of hierarchy can bring a substantial performance gain when -- * the cgroup hierarchy is nested multiple levels. in a simple benchmark using -- * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it -- * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two -- * apache instances competing with 2:1 weight ratio nested four level deep. -- * -- * However, the gain comes at the cost of not being able to properly handle -- * thundering herd of cgroups. For example, if many cgroups which are nested -- * behind a low priority parent cgroup wake up around the same time, they may be -- * able to consume more CPU cycles than they are entitled to. In many use cases, -- * this isn't a real concern especially given the performance gain. Also, there -- * are ways to mitigate the problem further by e.g. introducing an extra -- * scheduling layer on cgroup delegation boundaries. -- * -- * The scheduler first picks the cgroup to run and then schedule the tasks -- * within by using nested weighted vtime scheduling by default. The -- * cgroup-internal scheduling can be switched to FIFO with the -f option. -- */ --#include "scx_common.bpf.h" --#include "user_exit_info.h" --#include "scx_example_flatcg.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */ --const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL; --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --u64 cvtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, u64); -- __uint(max_entries, FCG_NR_STATS); --} stats SEC(".maps"); -- --static void stat_inc(enum fcg_stat_idx idx) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p)++; --} -- --struct fcg_cpu_ctx { -- u64 cur_cgid; -- u64 cur_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, struct fcg_cpu_ctx); -- __uint(max_entries, 1); --} cpu_ctx SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_cgrp_ctx); --} cgrp_ctx SEC(".maps"); -- --struct cgv_node { -- struct bpf_rb_node rb_node; -- __u64 cvtime; -- __u64 cgid; --}; -- --private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock; --private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node); -- --struct cgv_node_stash { -- struct cgv_node __kptr *node; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, 16384); -- __type(key, __u64); -- __type(value, struct cgv_node_stash); --} cgv_node_stash SEC(".maps"); -- --struct fcg_task_ctx { -- u64 bypassed_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_task_ctx); --} task_ctx SEC(".maps"); -- --/* gets inc'd on weight tree changes to expire the cached hweights */ --unsigned long hweight_gen = 1; -- --static u64 div_round_up(u64 dividend, u64 divisor) --{ -- return (dividend + divisor - 1) / divisor; --} -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) --{ -- struct cgv_node *cgc_a, *cgc_b; -- -- cgc_a = container_of(a, struct cgv_node, rb_node); -- cgc_b = container_of(b, struct cgv_node, rb_node); -- -- return cgc_a->cvtime < cgc_b->cvtime; --} -- --static struct fcg_cpu_ctx *find_cpu_ctx(void) --{ -- struct fcg_cpu_ctx *cpuc; -- u32 idx = 0; -- -- cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx); -- if (!cpuc) { -- scx_bpf_error("cpu_ctx lookup failed"); -- return NULL; -- } -- return cpuc; --} -- --static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id); -- return NULL; -- } -- return cgc; --} -- --static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgrp = bpf_cgroup_ancestor(cgrp, level); -- if (!cgrp) { -- scx_bpf_error("ancestor cgroup lookup failed"); -- return NULL; -- } -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- scx_bpf_error("ancestor cgrp_ctx lookup failed"); -- bpf_cgroup_release(cgrp); -- return cgc; --} -- --static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- int level; -- -- if (!cgc->nr_active) { -- stat_inc(FCG_STAT_HWT_SKIP); -- return; -- } -- -- if (cgc->hweight_gen == hweight_gen) { -- stat_inc(FCG_STAT_HWT_CACHE); -- return; -- } -- -- stat_inc(FCG_STAT_HWT_UPDATES); -- bpf_for(level, 0, cgrp->level + 1) { -- struct fcg_cgrp_ctx *cgc; -- bool is_active; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- -- if (!level) { -- cgc->hweight = FCG_HWEIGHT_ONE; -- cgc->hweight_gen = hweight_gen; -- } else { -- struct fcg_cgrp_ctx *pcgc; -- -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- -- /* -- * We can be oppotunistic here and not grab the -- * cgv_tree_lock and deal with the occasional races. -- * However, hweight updates are already cached and -- * relatively low-frequency. Let's just do the -- * straightforward thing. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- is_active = cgc->nr_active; -- if (is_active) { -- cgc->hweight_gen = pcgc->hweight_gen; -- cgc->hweight = -- div_round_up(pcgc->hweight * cgc->weight, -- pcgc->child_weight_sum); -- } -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!is_active) { -- stat_inc(FCG_STAT_HWT_RACE); -- break; -- } -- } -- } --} -- --static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) --{ -- u64 delta, cvtime, max_budget; -- -- /* -- * A node which is on the rbtree can't be pointed to from elsewhere yet -- * and thus can't be updated and repositioned. Instead, we collect the -- * vtime deltas separately and apply it asynchronously here. -- */ -- delta = cgc->cvtime_delta; -- __sync_fetch_and_sub(&cgc->cvtime_delta, delta); -- cvtime = cgv_node->cvtime + delta; -- -- /* -- * Allow a cgroup to carry the maximum budget proportional to its -- * hweight such that a full-hweight cgroup can immediately take up half -- * of the CPUs at the most while staying at the front of the rbtree. -- */ -- max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / -- (2 * FCG_HWEIGHT_ONE); -- if (vtime_before(cvtime, cvtime_now - max_budget)) -- cvtime = cvtime_now - max_budget; -- -- cgv_node->cvtime = cvtime; --} -- --static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- u64 cgid = cgrp->kn->id; -- -- /* paired with cmpxchg in try_pick_next_cgroup() */ -- if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { -- stat_inc(FCG_STAT_ENQ_SKIP); -- return; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid); -- return; -- } -- -- /* NULL if the node is already on the rbtree */ -- cgv_node = bpf_kptr_xchg(&stash->node, NULL); -- if (!cgv_node) { -- stat_inc(FCG_STAT_ENQ_RACE); -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); --} -- --void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * If select_cpu_dfl() is recommending local enqueue, the target CPU is -- * idle. Follow it and charge the cgroup later in fcg_stopping() after -- * the fact. Use the same mechanism to deal with tasks with custom -- * affinities so that we don't have to worry about per-cgroup dq's -- * containing tasks that can't be executed from some CPUs. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) { -- /* -- * Tell fcg_stopping() that this bypassed the regular scheduling -- * path and should be force charged to the cgroup. 0 is used to -- * indicate that the task isn't bypassing, so if the current -- * runtime is 0, go back by one nanosecond. -- */ -- taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1; -- -- /* -- * The global dq is deprioritized as we don't want to let tasks -- * to boost themselves by constraining its cpumask. The -- * deprioritization is rather severe, so let's not apply that to -- * per-cpu kernel threads. This is ham-fisted. We probably wanna -- * implement per-cgroup fallback dq's instead so that we have -- * more control over when tasks with custom cpumask get issued. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || -- (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) { -- stat_inc(FCG_STAT_LOCAL); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- } else { -- stat_inc(FCG_STAT_GLOBAL); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } -- return; -- } -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- goto out_release; -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 tvtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) -- tvtime = cgc->tvtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, -- tvtime, enq_flags); -- } -- -- cgrp_enqueued(cgrp, cgc); --out_release: -- bpf_cgroup_release(cgrp); --} -- --/* -- * Walk the cgroup tree to update the active weight sums as tasks wake up and -- * sleep. The weight sums are used as the base when calculating the proportion a -- * given cgroup or task is entitled to at each level. -- */ --static void update_active_weight_sums(struct cgroup *cgrp, bool runnable) --{ -- struct fcg_cgrp_ctx *cgc; -- bool updated = false; -- int idx; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- /* -- * In most cases, a hot cgroup would have multiple threads going to -- * sleep and waking up while the whole cgroup stays active. In leaf -- * cgroups, ->nr_runnable which is updated with __sync operations gates -- * ->nr_active updates, so that we don't have to grab the cgv_tree_lock -- * repeatedly for a busy cgroup which is staying active. -- */ -- if (runnable) { -- if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_ACT); -- } else { -- if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_DEACT); -- } -- -- /* -- * If @cgrp is becoming runnable, its hweight should be refreshed after -- * it's added to the weight tree so that enqueue has the up-to-date -- * value. If @cgrp is becoming quiescent, the hweight should be -- * refreshed before it's removed from the weight tree so that the usage -- * charging which happens afterwards has access to the latest value. -- */ -- if (!runnable) -- cgrp_refresh_hweight(cgrp, cgc); -- -- /* propagate upwards */ -- bpf_for(idx, 0, cgrp->level) { -- int level = cgrp->level - idx; -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- bool propagate = false; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- if (level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- } -- -- /* -- * We need the propagation protected by a lock to synchronize -- * against weight changes. There's no reason to drop the lock at -- * each level but bpf_spin_lock() doesn't want any function -- * calls while locked. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- -- if (runnable) { -- if (!cgc->nr_active++) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum += cgc->weight; -- } -- } -- } else { -- if (!--cgc->nr_active) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum -= cgc->weight; -- } -- } -- } -- -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!propagate) -- break; -- } -- -- if (updated) -- __sync_fetch_and_add(&hweight_gen, 1); -- -- if (runnable) -- cgrp_refresh_hweight(cgrp, cgc); --} -- --void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, true); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) --{ -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- if (fifo_sched) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- /* -- * @cgc->tvtime_now always progresses forward as tasks start -- * executing. The test and update can be performed concurrently -- * from multiple CPUs and thus racy. Any error should be -- * contained and temporary. Let's just live with it. -- */ -- if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) -- cgc->tvtime_now = p->scx.dsq_vtime; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- /* scale the execution time by the inverse of the weight and charge */ -- if (!fifo_sched) -- p->scx.dsq_vtime += -- (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- if (!taskc->bypassed_at) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- __sync_fetch_and_add(&cgc->cvtime_delta, -- p->se.sum_exec_runtime - taskc->bypassed_at); -- taskc->bypassed_at = 0; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, false); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) --{ -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- if (cgrp->level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1); -- if (!pcgc) -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- if (pcgc && cgc->nr_active) -- pcgc->child_weight_sum += (s64)weight - cgc->weight; -- cgc->weight = weight; -- bpf_spin_unlock(&cgv_tree_lock); --} -- --static bool try_pick_next_cgroup(u64 *cgidp) --{ -- struct bpf_rb_node *rb_node; -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 cgid; -- -- /* pop the front cgroup and wind cvtime_now accordingly */ -- bpf_spin_lock(&cgv_tree_lock); -- -- rb_node = bpf_rbtree_first(&cgv_tree); -- if (!rb_node) { -- bpf_spin_unlock(&cgv_tree_lock); -- stat_inc(FCG_STAT_PNC_NO_CGRP); -- *cgidp = 0; -- return true; -- } -- -- rb_node = bpf_rbtree_remove(&cgv_tree, rb_node); -- bpf_spin_unlock(&cgv_tree_lock); -- -- cgv_node = container_of(rb_node, struct cgv_node, rb_node); -- cgid = cgv_node->cgid; -- -- if (vtime_before(cvtime_now, cgv_node->cvtime)) -- cvtime_now = cgv_node->cvtime; -- -- /* -- * If lookup fails, the cgroup's gone. Free and move on. See -- * fcg_cgroup_exit(). -- */ -- cgrp = bpf_cgroup_from_id(cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- if (!scx_bpf_consume(cgid)) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_EMPTY); -- goto out_stash; -- } -- -- /* -- * Successfully consumed from the cgroup. This will be our current -- * cgroup for the new slice. Refresh its hweight. -- */ -- cgrp_refresh_hweight(cgrp, cgc); -- -- bpf_cgroup_release(cgrp); -- -- /* -- * As the cgroup may have more tasks, add it back to the rbtree. Note -- * that here we charge the full slice upfront and then exact later -- * according to the actual consumption. This prevents lowpri thundering -- * herd from saturating the machine. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- -- *cgidp = cgid; -- stat_inc(FCG_STAT_PNC_NEXT); -- return true; -- --out_stash: -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- /* -- * Paired with cmpxchg in cgrp_enqueued(). If they see the following -- * transition, they'll enqueue the cgroup. If they are earlier, we'll -- * see their task in the dq below and requeue the cgroup. -- */ -- __sync_val_compare_and_swap(&cgc->queued, 1, 0); -- -- if (scx_bpf_dsq_nr_queued(cgid)) { -- bpf_spin_lock(&cgv_tree_lock); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- goto out_free; -- } -- } -- -- return false; -- --out_free: -- bpf_obj_drop(cgv_node); -- return false; --} -- --void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) --{ -- struct fcg_cpu_ctx *cpuc; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 now = bpf_ktime_get_ns(); -- -- cpuc = find_cpu_ctx(); -- if (!cpuc) -- return; -- -- if (!cpuc->cur_cgid) -- goto pick_next_cgroup; -- -- if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { -- if (scx_bpf_consume(cpuc->cur_cgid)) { -- stat_inc(FCG_STAT_CNS_KEEP); -- return; -- } -- stat_inc(FCG_STAT_CNS_EMPTY); -- } else { -- stat_inc(FCG_STAT_CNS_EXPIRE); -- } -- -- /* -- * The current cgroup is expiring. It was already charged a full slice. -- * Calculate the actual usage and accumulate the delta. -- */ -- cgrp = bpf_cgroup_from_id(cpuc->cur_cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_CNS_GONE); -- goto pick_next_cgroup; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (cgc) { -- /* -- * We want to update the vtime delta and then look for the next -- * cgroup to execute but the latter needs to be done in a loop -- * and we can't keep the lock held. Oh well... -- */ -- bpf_spin_lock(&cgv_tree_lock); -- __sync_fetch_and_add(&cgc->cvtime_delta, -- (cpuc->cur_at + cgrp_slice_ns - now) * -- FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- stat_inc(FCG_STAT_CNS_GONE); -- } -- -- bpf_cgroup_release(cgrp); -- --pick_next_cgroup: -- cpuc->cur_at = now; -- -- if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { -- cpuc->cur_cgid = 0; -- return; -- } -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_pick_next_cgroup(&cpuc->cur_cgid)) -- break; -- } --} -- --s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct fcg_task_ctx *taskc; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- taskc = bpf_task_storage_get(&task_ctx, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!taskc) -- return -ENOMEM; -- -- taskc->bypassed_at = 0; -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp, -- struct scx_cgroup_init_args *args) --{ -- struct fcg_cgrp_ctx *cgc; -- struct cgv_node *cgv_node; -- struct cgv_node_stash empty_stash = {}, *stash; -- u64 cgid = cgrp->kn->id; -- int ret; -- -- /* -- * Technically incorrect as cgroup ID is full 64bit while dq ID is -- * 63bit. Should not be a problem in practice and easy to spot in the -- * unlikely case that it breaks. -- */ -- ret = scx_bpf_create_dsq(cgid, -1); -- if (ret) -- return ret; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!cgc) { -- ret = -ENOMEM; -- goto err_destroy_dsq; -- } -- -- cgc->weight = args->weight; -- cgc->hweight = FCG_HWEIGHT_ONE; -- -- ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash, -- BPF_NOEXIST); -- if (ret) { -- if (ret != -ENOMEM) -- scx_bpf_error("unexpected stash creation error (%d)", -- ret); -- goto err_destroy_dsq; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("unexpected cgv_node stash lookup failure"); -- ret = -ENOENT; -- goto err_destroy_dsq; -- } -- -- cgv_node = bpf_obj_new(struct cgv_node); -- if (!cgv_node) { -- ret = -ENOMEM; -- goto err_del_cgv_node; -- } -- -- cgv_node->cgid = cgid; -- cgv_node->cvtime = cvtime_now; -- -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- ret = -EBUSY; -- goto err_drop; -- } -- -- return 0; -- --err_drop: -- bpf_obj_drop(cgv_node); --err_del_cgv_node: -- bpf_map_delete_elem(&cgv_node_stash, &cgid); --err_destroy_dsq: -- scx_bpf_destroy_dsq(cgid); -- return ret; --} -- --void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- -- /* -- * For now, there's no way find and remove the cgv_node if it's on the -- * cgv_tree. Let's drain them in the dispatch path as they get popped -- * off the front of the tree. -- */ -- bpf_map_delete_elem(&cgv_node_stash, &cgid); -- scx_bpf_destroy_dsq(cgid); --} -- --s32 BPF_STRUCT_OPS(fcg_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops flatcg_ops = { -- .enqueue = (void *)fcg_enqueue, -- .dispatch = (void *)fcg_dispatch, -- .runnable = (void *)fcg_runnable, -- .running = (void *)fcg_running, -- .stopping = (void *)fcg_stopping, -- .quiescent = (void *)fcg_quiescent, -- .prep_enable = (void *)fcg_prep_enable, -- .cgroup_set_weight = (void *)fcg_cgroup_set_weight, -- .cgroup_init = (void *)fcg_cgroup_init, -- .cgroup_exit = (void *)fcg_cgroup_exit, -- .init = (void *)fcg_init, -- .exit = (void *)fcg_exit, -- .flags = SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING, -- .name = "flatcg", --}; -diff --git b/tools/sched_ext/scx_example_flatcg.c a/tools/sched_ext/scx_example_flatcg.c -deleted file mode 100644 -index f9c8a5b84a70..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.c -+++ /dev/null -@@ -1,232 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2023 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2023 Tejun Heo -- * Copyright (c) 2023 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_flatcg.h" --#include "scx_example_flatcg.skel.h" -- --#ifndef FILEID_KERNFS --#define FILEID_KERNFS 0xfe --#endif -- --const char help_fmt[] = --"A flattened cgroup hierarchy sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -i INTERVAL Report interval\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --static float read_cpu_util(__u64 *last_sum, __u64 *last_idle) --{ -- FILE *fp; -- char buf[4096]; -- char *line, *cur = NULL, *tok; -- __u64 sum = 0, idle = 0; -- __u64 delta_sum, delta_idle; -- int idx; -- -- fp = fopen("/proc/stat", "r"); -- if (!fp) { -- perror("fopen(\"/proc/stat\")"); -- return 0.0; -- } -- -- if (!fgets(buf, sizeof(buf), fp)) { -- perror("fgets(\"/proc/stat\")"); -- fclose(fp); -- return 0.0; -- } -- fclose(fp); -- -- line = buf; -- for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) { -- char *endp = NULL; -- __u64 v; -- -- if (idx == 0) { -- line = NULL; -- continue; -- } -- v = strtoull(tok, &endp, 0); -- if (!endp || *endp != '\0') { -- fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n", -- idx, tok); -- continue; -- } -- sum += v; -- if (idx == 4) -- idle = v; -- } -- -- delta_sum = sum - *last_sum; -- delta_idle = idle - *last_idle; -- *last_sum = sum; -- *last_idle = idle; -- -- return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0; --} -- --static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats) --{ -- __u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus]; -- __u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); -- -- for (idx = 0; idx < FCG_NR_STATS; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_flatcg *skel; -- struct bpf_link *link; -- struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 }; -- bool dump_cgrps = false; -- __u64 last_cpu_sum = 0, last_cpu_idle = 0; -- __u64 last_stats[FCG_NR_STATS] = {}; -- unsigned long seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_flatcg__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open: %s\n", strerror(errno)); -- return 1; -- } -- -- skel->rodata->nr_cpus = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) { -- double v; -- -- switch (opt) { -- case 's': -- v = strtod(optarg, NULL); -- skel->rodata->cgrp_slice_ns = v * 1000; -- break; -- case 'i': -- v = strtod(optarg, NULL); -- intv_ts.tv_sec = v; -- intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000; -- break; -- case 'd': -- dump_cgrps = true; -- break; -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- case 'h': -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d", -- (double)skel->rodata->cgrp_slice_ns / 1000000.0, -- (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0, -- dump_cgrps); -- -- if (scx_example_flatcg__load(skel)) { -- fprintf(stderr, "Failed to load: %s\n", strerror(errno)); -- return 1; -- } -- -- link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops); -- if (!link) { -- fprintf(stderr, "Failed to attach_struct_ops: %s\n", -- strerror(errno)); -- return 1; -- } -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- __u64 acc_stats[FCG_NR_STATS]; -- __u64 stats[FCG_NR_STATS]; -- float cpu_util; -- int i; -- -- cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle); -- -- fcg_read_stats(skel, acc_stats); -- for (i = 0; i < FCG_NR_STATS; i++) -- stats[i] = acc_stats[i] - last_stats[i]; -- -- memcpy(last_stats, acc_stats, sizeof(acc_stats)); -- -- printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n", -- seq++, cpu_util * 100.0, skel->data->hweight_gen); -- printf(" act:%6llu deact:%6llu local:%6llu global:%6llu\n", -- stats[FCG_STAT_ACT], -- stats[FCG_STAT_DEACT], -- stats[FCG_STAT_LOCAL], -- stats[FCG_STAT_GLOBAL]); -- printf("HWT skip:%6llu race:%6llu cache:%6llu update:%6llu\n", -- stats[FCG_STAT_HWT_SKIP], -- stats[FCG_STAT_HWT_RACE], -- stats[FCG_STAT_HWT_CACHE], -- stats[FCG_STAT_HWT_UPDATES]); -- printf("ENQ skip:%6llu race:%6llu\n", -- stats[FCG_STAT_ENQ_SKIP], -- stats[FCG_STAT_ENQ_RACE]); -- printf("CNS keep:%6llu expire:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_CNS_KEEP], -- stats[FCG_STAT_CNS_EXPIRE], -- stats[FCG_STAT_CNS_EMPTY], -- stats[FCG_STAT_CNS_GONE]); -- printf("PNC nocgrp:%6llu next:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_PNC_NO_CGRP], -- stats[FCG_STAT_PNC_NEXT], -- stats[FCG_STAT_PNC_EMPTY], -- stats[FCG_STAT_PNC_GONE]); -- printf("BAD remove:%6llu\n", -- acc_stats[FCG_STAT_BAD_REMOVAL]); -- -- nanosleep(&intv_ts, NULL); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_flatcg__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_flatcg.h a/tools/sched_ext/scx_example_flatcg.h -deleted file mode 100644 -index 490758ed41f0..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.h -+++ /dev/null -@@ -1,49 +0,0 @@ --#ifndef __SCX_EXAMPLE_FLATCG_H --#define __SCX_EXAMPLE_FLATCG_H -- --enum { -- FCG_HWEIGHT_ONE = 1LLU << 16, --}; -- --enum fcg_stat_idx { -- FCG_STAT_ACT, -- FCG_STAT_DEACT, -- FCG_STAT_LOCAL, -- FCG_STAT_GLOBAL, -- -- FCG_STAT_HWT_UPDATES, -- FCG_STAT_HWT_CACHE, -- FCG_STAT_HWT_SKIP, -- FCG_STAT_HWT_RACE, -- -- FCG_STAT_ENQ_SKIP, -- FCG_STAT_ENQ_RACE, -- -- FCG_STAT_CNS_KEEP, -- FCG_STAT_CNS_EXPIRE, -- FCG_STAT_CNS_EMPTY, -- FCG_STAT_CNS_GONE, -- -- FCG_STAT_PNC_NO_CGRP, -- FCG_STAT_PNC_NEXT, -- FCG_STAT_PNC_EMPTY, -- FCG_STAT_PNC_GONE, -- -- FCG_STAT_BAD_REMOVAL, -- -- FCG_NR_STATS, --}; -- --struct fcg_cgrp_ctx { -- u32 nr_active; -- u32 nr_runnable; -- u32 queued; -- u32 weight; -- u32 hweight; -- u64 child_weight_sum; -- u64 hweight_gen; -- s64 cvtime_delta; -- u64 tvtime_now; --}; -- --#endif /* __SCX_EXAMPLE_FLATCG_H */ -diff --git b/tools/sched_ext/scx_example_pair.bpf.c a/tools/sched_ext/scx_example_pair.bpf.c -deleted file mode 100644 -index 279efe58b777..000000000000 ---- b/tools/sched_ext/scx_example_pair.bpf.c -+++ /dev/null -@@ -1,627 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext core-scheduler which always makes every sibling CPU pair -- * execute from the same CPU cgroup. -- * -- * This scheduler is a minimal implementation and would need some form of -- * priority handling both inside each cgroup and across the cgroups to be -- * practically useful. -- * -- * Each CPU in the system is paired with exactly one other CPU, according to a -- * "stride" value that can be specified when the BPF scheduler program is first -- * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee -- * that they will only ever schedule tasks that belong to the same CPU cgroup. -- * -- * Scheduler Initialization -- * ------------------------ -- * -- * The scheduler BPF program is first initialized from user space, before it is -- * enabled. During this initialization process, each CPU on the system is -- * assigned several values that are constant throughout its runtime: -- * -- * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling -- * decisions. Paired CPUs always schedule tasks from the same -- * CPU cgroup, and synchronize with each other to guarantee -- * that this constraint is not violated. -- * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access -- * a struct pair_ctx object that is shared between the pair. -- * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the -- * pair. Each struct pair_ctx has an active_mask field, -- * which is a bitmap used to indicate whether each core -- * in the pair currently has an actively running task. -- * This index specifies which entry in the bitmap corresponds -- * to each CPU in the pair. -- * -- * During this initialization, the CPUs are paired according to a "stride" that -- * may be specified when invoking the user space program that initializes and -- * loads the scheduler. By default, the stride is 1/2 the total number of CPUs. -- * -- * Tasks and cgroups -- * ----------------- -- * -- * Every cgroup in the system is registered with the scheduler using the -- * pair_cgroup_init() callback, and every task in the system is associated with -- * exactly one cgroup. At a high level, the idea with the pair scheduler is to -- * always schedule tasks from the same cgroup within a given CPU pair. When a -- * task is enqueued (i.e. passed to the pair_enqueue() callback function), its -- * cgroup ID is read from its task struct, and then a corresponding queue map -- * is used to FIFO-enqueue the task for that cgroup. -- * -- * If you look through the implementation of the scheduler, you'll notice that -- * there is quite a bit of complexity involved with looking up the per-cgroup -- * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash -- * BPF hash map that is used to map a cgroup ID to a globally unique ID that's -- * allocated in the BPF program. This is done because we use separate maps to -- * store the FIFO queue of tasks, and the length of that map, per cgroup. This -- * complexity is only present because of current deficiencies in BPF that will -- * soon be addressed. The main point to keep in mind is that newly enqueued -- * tasks are added to their cgroup's FIFO queue. -- * -- * Dispatching tasks -- * ----------------- -- * -- * This section will describe how enqueued tasks are dispatched and scheduled. -- * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is -- * as follows: -- * -- * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is -- * the structure that's used to synchronize amongst the two pair CPUs in their -- * scheduling decisions. After any of the following events have occurred: -- * -- * - The cgroup's slice run has expired, or -- * - The cgroup becomes empty, or -- * - Either CPU in the pair is preempted by a higher priority scheduling class -- * -- * The cgroup transitions to the draining state and stops executing new tasks -- * from the cgroup. -- * -- * 2. If the pair is still executing a task, mark the pair_ctx as draining, and -- * wait for the pair CPU to be preempted. -- * -- * 3. Otherwise, if the pair CPU is not running a task, we can move onto -- * scheduling new tasks. Pop the next cgroup id from the top_q queue. -- * -- * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it. -- * -- * Note again that this scheduling behavior is simple, but the implementation -- * is complex mostly because this it hits several BPF shortcomings and has to -- * work around in often awkward ways. Most of the shortcomings are expected to -- * be resolved in the near future which should allow greatly simplifying this -- * scheduler. -- * -- * Dealing with preemption -- * ----------------------- -- * -- * SCX is the lowest priority sched_class, and could be preempted by them at -- * any time. To address this, the scheduler implements pair_cpu_release() and -- * pair_cpu_acquire() callbacks which are invoked by the core scheduler when -- * the scheduler loses and gains control of the CPU respectively. -- * -- * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and -- * then invoke: -- * -- * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- * -- * This preempts the pair CPU, and waits until it has re-entered the scheduler -- * before returning. This is necessary to ensure that the higher priority -- * sched_class that preempted our scheduler does not schedule a task -- * concurrently with our pair CPU. -- * -- * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption -- * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable -- * pair scheduling. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include "scx_example_pair.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; -- --/* !0 for veristat, set during init */ --const volatile u32 nr_cpu_ids = 64; -- --/* a pair of CPUs stay on a cgroup for this duration */ --const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL; -- --/* cpu ID -> pair cpu ID */ --const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 }; -- --/* cpu ID -> pair_id */ --const volatile u32 pair_id[MAX_CPUS]; -- --/* CPU ID -> CPU # in the pair (0 or 1) */ --const volatile u32 in_pair_idx[MAX_CPUS]; -- --struct pair_ctx { -- struct bpf_spin_lock lock; -- -- /* the cgroup the pair is currently executing */ -- u64 cgid; -- -- /* the pair started executing the current cgroup at */ -- u64 started_at; -- -- /* whether the current cgroup is draining */ -- bool draining; -- -- /* the CPUs that are currently active on the cgroup */ -- u32 active_mask; -- -- /* -- * the CPUs that are currently preempted and running tasks in a -- * different scheduler. -- */ -- u32 preempted_mask; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, MAX_CPUS / 2); -- __type(key, u32); -- __type(value, struct pair_ctx); --} pair_ctx SEC(".maps"); -- --/* queue of cgrp_q's possibly with tasks on them */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- /* -- * Because it's difficult to build strong synchronization encompassing -- * multiple non-trivial operations in BPF, this queue is managed in an -- * opportunistic way so that we guarantee that a cgroup w/ active tasks -- * is always on it but possibly multiple times. Once we have more robust -- * synchronization constructs and e.g. linked list, we should be able to -- * do this in a prettier way but for now just size it big enough. -- */ -- __uint(max_entries, 4 * MAX_CGRPS); -- __type(value, u64); --} top_q SEC(".maps"); -- --/* per-cgroup q which FIFOs the tasks from the cgroup */ --struct cgrp_q { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, MAX_QUEUED); -- __type(value, u32); --}; -- --/* -- * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local -- * storage; however, a cgroup local storage can only be accessed from the BPF -- * progs attached to the cgroup. For now, work around by allocating array of -- * cgrp_q's and then allocating per-cgroup indices. -- * -- * Another caveat: It's difficult to populate a large array of maps statically -- * or from BPF. Initialize it from userland. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, MAX_CGRPS); -- __type(key, s32); -- __array(values, struct cgrp_q); --} cgrp_q_arr SEC(".maps"); -- --static u64 cgrp_q_len[MAX_CGRPS]; -- --/* -- * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be -- * useful to have as a map type. -- */ --static u32 cgrp_q_idx_cursor; --static u64 cgrp_q_idx_busy[MAX_CGRPS]; -- --/* -- * All added up, the following is what we do: -- * -- * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking -- * for a free ID. If not found, fail cgroup creation with -EBUSY. -- * -- * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following -- * cgrp_q_idx_hash. -- * -- * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from -- * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr. -- * -- * This is sadly complicated for something pretty simple. Hopefully, we should -- * be able to simplify in the future. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, MAX_CGRPS); -- __uint(key_size, sizeof(u64)); /* cgrp ID */ -- __uint(value_size, sizeof(s32)); /* cgrp_q idx */ --} cgrp_q_idx_hash SEC(".maps"); -- --/* statistics */ --u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions; --u64 nr_exps, nr_exp_waits, nr_exp_empty; --u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty; -- --struct user_exit_info uei; -- --static bool time_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- struct cgrp_q *cgq; -- s32 pid = p->pid; -- u64 cgid; -- u32 *q_idx; -- u64 *cgq_len; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- cgrp = scx_bpf_task_cgroup(p); -- cgid = cgrp->kn->id; -- bpf_cgroup_release(cgrp); -- -- /* find the cgroup's q and push @p into it */ -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!q_idx) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return; -- } -- -- cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx); -- if (!cgq) { -- scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]", -- cgid, *q_idx); -- return; -- } -- -- if (bpf_map_push_elem(cgq, &pid, 0)) { -- scx_bpf_error("cgroup[%llu] queue overflow", cgid); -- return; -- } -- -- /* bump q len, if going 0 -> 1, queue cgroup into the top_q */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len) { -- scx_bpf_error("MEMBER_VTPR malfunction"); -- return; -- } -- -- if (!__sync_fetch_and_add(cgq_len, 1) && -- bpf_map_push_elem(&top_q, &cgid, 0)) { -- scx_bpf_error("top_q overflow"); -- return; -- } --} -- --static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) --{ -- u32 *vptr; -- -- vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *pairc = bpf_map_lookup_elem(&pair_ctx, vptr); -- if (!(*pairc)) -- return -EINVAL; -- -- vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *mask = 1U << *vptr; -- -- return 0; --} -- --static int try_dispatch(s32 cpu) --{ -- struct pair_ctx *pairc; -- struct bpf_map *cgq_map; -- struct task_struct *p; -- u64 now = bpf_ktime_get_ns(); -- bool kick_pair = false; -- bool expired, pair_preempted; -- u32 *vptr, in_pair_mask; -- s32 pid, q_idx; -- u64 cgid; -- int ret; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) { -- scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]", -- cpu); -- return -ENOENT; -- } -- -- bpf_spin_lock(&pairc->lock); -- pairc->active_mask &= ~in_pair_mask; -- -- expired = time_before(pairc->started_at + pair_batch_dur_ns, now); -- if (expired || pairc->draining) { -- u64 new_cgid = 0; -- -- __sync_fetch_and_add(&nr_exps, 1); -- -- /* -- * We're done with the current cgid. An obvious optimization -- * would be not draining if the next cgroup is the current one. -- * For now, be dumb and always expire. -- */ -- pairc->draining = true; -- -- pair_preempted = pairc->preempted_mask; -- if (pairc->active_mask || pair_preempted) { -- /* -- * The other CPU is still active, or is no longer under -- * our control due to e.g. being preempted by a higher -- * priority sched_class. We want to wait until this -- * cgroup expires, or until control of our pair CPU has -- * been returned to us. -- * -- * If the pair controls its CPU, and the time already -- * expired, kick. When the other CPU arrives at -- * dispatch and clears its active mask, it'll push the -- * pair to the next cgroup and kick this CPU. -- */ -- __sync_fetch_and_add(&nr_exp_waits, 1); -- bpf_spin_unlock(&pairc->lock); -- if (expired && !pair_preempted) -- kick_pair = true; -- goto out_maybe_kick; -- } -- -- bpf_spin_unlock(&pairc->lock); -- -- /* -- * Pick the next cgroup. It'd be easier / cleaner to not drop -- * pairc->lock and use stronger synchronization here especially -- * given that we'll be switching cgroups significantly less -- * frequently than tasks. Unfortunately, bpf_spin_lock can't -- * really protect anything non-trivial. Let's do opportunistic -- * operations instead. -- */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u32 *q_idx; -- u64 *cgq_len; -- -- if (bpf_map_pop_elem(&top_q, &new_cgid)) { -- /* no active cgroup, go idle */ -- __sync_fetch_and_add(&nr_exp_empty, 1); -- return 0; -- } -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid); -- if (!q_idx) -- continue; -- -- /* -- * This is the only place where empty cgroups are taken -- * off the top_q. -- */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len || !*cgq_len) -- continue; -- -- /* -- * If it has any tasks, requeue as we may race and not -- * execute it. -- */ -- bpf_map_push_elem(&top_q, &new_cgid, 0); -- break; -- } -- -- bpf_spin_lock(&pairc->lock); -- -- /* -- * The other CPU may already have started on a new cgroup while -- * we dropped the lock. Make sure that we're still draining and -- * start on the new cgroup. -- */ -- if (pairc->draining && !pairc->active_mask) { -- __sync_fetch_and_add(&nr_cgrp_next, 1); -- pairc->cgid = new_cgid; -- pairc->started_at = now; -- pairc->draining = false; -- kick_pair = true; -- } else { -- __sync_fetch_and_add(&nr_cgrp_coll, 1); -- } -- } -- -- cgid = pairc->cgid; -- pairc->active_mask |= in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- -- /* again, it'd be better to do all these with the lock held, oh well */ -- vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!vptr) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return -ENOENT; -- } -- q_idx = *vptr; -- -- /* claim one task from cgrp_q w/ q_idx */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u64 *cgq_len, len; -- -- cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]); -- if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) { -- /* the cgroup must be empty, expire and repeat */ -- __sync_fetch_and_add(&nr_cgrp_empty, 1); -- bpf_spin_lock(&pairc->lock); -- pairc->draining = true; -- pairc->active_mask &= ~in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- return -EAGAIN; -- } -- -- if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len) -- continue; -- -- break; -- } -- -- cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx); -- if (!cgq_map) { -- scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- if (bpf_map_pop_elem(cgq_map, &pid)) { -- scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- p = bpf_task_from_pid(pid); -- if (p) { -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } else { -- /* we don't handle dequeues, retry on lost tasks */ -- __sync_fetch_and_add(&nr_missing, 1); -- return -EAGAIN; -- } -- --out_maybe_kick: -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } -- return 0; --} -- --void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) --{ -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_dispatch(cpu) != -EAGAIN) -- break; -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask &= ~in_pair_mask; -- /* Kick the pair CPU, unless it was also preempted. */ -- kick_pair = !pairc->preempted_mask; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask |= in_pair_mask; -- pairc->active_mask &= ~in_pair_mask; -- /* Kick the pair CPU if it's still running. */ -- kick_pair = pairc->active_mask; -- pairc->draining = true; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- } -- } -- __sync_fetch_and_add(&nr_preemptions, 1); --} -- --s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 i, q_idx; -- -- bpf_for(i, 0, MAX_CGRPS) { -- q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS; -- if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1)) -- break; -- } -- if (i == MAX_CGRPS) -- return -EBUSY; -- -- if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]); -- if (busy) -- *busy = 0; -- return -EBUSY; -- } -- -- return 0; --} -- --void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 *q_idx; -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (q_idx) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]); -- if (busy) -- *busy = 0; -- bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid); -- } --} -- --s32 BPF_STRUCT_OPS(pair_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops pair_ops = { -- .enqueue = (void *)pair_enqueue, -- .dispatch = (void *)pair_dispatch, -- .cpu_acquire = (void *)pair_cpu_acquire, -- .cpu_release = (void *)pair_cpu_release, -- .cgroup_init = (void *)pair_cgroup_init, -- .cgroup_exit = (void *)pair_cgroup_exit, -- .init = (void *)pair_init, -- .exit = (void *)pair_exit, -- .name = "pair", --}; -diff --git b/tools/sched_ext/scx_example_pair.c a/tools/sched_ext/scx_example_pair.c -deleted file mode 100644 -index 18e032bbc173..000000000000 ---- b/tools/sched_ext/scx_example_pair.c -+++ /dev/null -@@ -1,143 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_pair.h" --#include "scx_example_pair.skel.h" -- --const char help_fmt[] = --"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n" --"execute from the same CPU cgroup.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-S STRIDE] [-p]\n" --"\n" --" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_pair *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 stride, i, opt, outer_fd; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_pair__open(); -- assert(skel); -- -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- /* pair up the earlier half to the latter by default, override with -s */ -- stride = skel->rodata->nr_cpu_ids / 2; -- -- while ((opt = getopt(argc, argv, "S:ph")) != -1) { -- switch (opt) { -- case 'S': -- stride = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- for (i = 0; i < skel->rodata->nr_cpu_ids; i++) { -- if (skel->rodata->pair_cpu[i] < 0) { -- skel->rodata->pair_cpu[i] = i + stride; -- skel->rodata->pair_cpu[i + stride] = i; -- skel->rodata->pair_id[i] = i; -- skel->rodata->pair_id[i + stride] = i; -- skel->rodata->in_pair_idx[i] = 0; -- skel->rodata->in_pair_idx[i + stride] = 1; -- } -- } -- -- assert(!scx_example_pair__load(skel)); -- -- /* -- * Populate the cgrp_q_arr map which is an array containing per-cgroup -- * queues. It'd probably be better to do this from BPF but there are too -- * many to initialize statically and there's no way to dynamically -- * populate from BPF. -- */ -- outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr); -- assert(outer_fd >= 0); -- -- printf("Initializing"); -- for (i = 0; i < MAX_CGRPS; i++) { -- s32 inner_fd; -- -- if (exit_req) -- break; -- -- inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, -- sizeof(u32), MAX_QUEUED, NULL); -- assert(inner_fd >= 0); -- assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY)); -- close(inner_fd); -- -- if (!(i % 10)) -- printf("."); -- fflush(stdout); -- } -- printf("\n"); -- -- /* -- * Fully initialized, attach and run. -- */ -- link = bpf_map__attach_struct_ops(skel->maps.pair_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf(" total:%10lu dispatch:%10lu missing:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_dispatched, -- skel->bss->nr_missing); -- printf(" kicks:%10lu preemptions:%7lu\n", -- skel->bss->nr_kicks, -- skel->bss->nr_preemptions); -- printf(" exp:%10lu exp_wait:%10lu exp_empty:%10lu\n", -- skel->bss->nr_exps, -- skel->bss->nr_exp_waits, -- skel->bss->nr_exp_empty); -- printf("cgnext:%10lu cgcoll:%10lu cgempty:%10lu\n", -- skel->bss->nr_cgrp_next, -- skel->bss->nr_cgrp_coll, -- skel->bss->nr_cgrp_empty); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_pair__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_pair.h a/tools/sched_ext/scx_example_pair.h -deleted file mode 100644 -index f60b824272f7..000000000000 ---- b/tools/sched_ext/scx_example_pair.h -+++ /dev/null -@@ -1,10 +0,0 @@ --#ifndef __SCX_EXAMPLE_PAIR_H --#define __SCX_EXAMPLE_PAIR_H -- --enum { -- MAX_CPUS = 4096, -- MAX_QUEUED = 4096, -- MAX_CGRPS = 4096, --}; -- --#endif /* __SCX_EXAMPLE_PAIR_H */ -diff --git b/tools/sched_ext/scx_example_qmap.bpf.c a/tools/sched_ext/scx_example_qmap.bpf.c -deleted file mode 100644 -index 579ab21ae403..000000000000 ---- b/tools/sched_ext/scx_example_qmap.bpf.c -+++ /dev/null -@@ -1,401 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple five-level FIFO queue scheduler. -- * -- * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets -- * assigned to one depending on its compound weight. Each CPU round robins -- * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from -- * queue0, 2 from queue1, 4 from queue2 and so on. -- * -- * This scheduler demonstrates: -- * -- * - BPF-side queueing using PIDs. -- * - Sleepable per-task storage allocation using ops.prep_enable(). -- * - Using ops.cpu_release() to handle a higher priority scheduling class taking -- * the CPU away. -- * - Core-sched support. -- * -- * This scheduler is primarily for demonstration and testing of sched_ext -- * features and unlikely to be useful for actual workloads. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include -- --char _license[] SEC("license") = "GPL"; -- --const volatile u64 slice_ns = SCX_SLICE_DFL; --const volatile bool switch_partial; --const volatile u32 stall_user_nth; --const volatile u32 stall_kernel_nth; --const volatile u32 dsp_inf_loop_after; --const volatile s32 disallow_tgid; -- --u32 test_error_cnt; -- --struct user_exit_info uei; -- --struct qmap { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, u32); --} queue0 SEC(".maps"), -- queue1 SEC(".maps"), -- queue2 SEC(".maps"), -- queue3 SEC(".maps"), -- queue4 SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, 5); -- __type(key, int); -- __array(values, struct qmap); --} queue_arr SEC(".maps") = { -- .values = { -- [0] = &queue0, -- [1] = &queue1, -- [2] = &queue2, -- [3] = &queue3, -- [4] = &queue4, -- }, --}; -- --/* -- * Per-queue sequence numbers to implement core-sched ordering. -- * -- * Tail seq is assigned to each queued task and incremented. Head seq tracks the -- * sequence number of the latest dispatched task. The distance between the a -- * task's seq and the associated queue's head seq is called the queue distance -- * and used when comparing two tasks for ordering. See qmap_core_sched_before(). -- */ --static u64 core_sched_head_seqs[5]; --static u64 core_sched_tail_seqs[5]; -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local_dsq */ -- u64 core_sched_seq; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --/* Per-cpu dispatch index and remaining count */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(max_entries, 2); -- __type(key, u32); -- __type(value, u64); --} dispatch_idx_cnt SEC(".maps"); -- --/* Statistics */ --unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued; --unsigned long nr_core_sched_execed; -- --s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- struct task_ctx *tctx; -- s32 cpu; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- return cpu; -- -- return prev_cpu; --} -- --static int weight_to_idx(u32 weight) --{ -- /* Coarsely map the compound weight to a FIFO. */ -- if (weight <= 25) -- return 0; -- else if (weight <= 50) -- return 1; -- else if (weight < 200) -- return 2; -- else if (weight < 400) -- return 3; -- else -- return 4; --} -- --void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) --{ -- static u32 user_cnt, kernel_cnt; -- struct task_ctx *tctx; -- u32 pid = p->pid; -- int idx = weight_to_idx(p->scx.weight); -- void *ring; -- -- if (p->flags & PF_KTHREAD) { -- if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth)) -- return; -- } else { -- if (stall_user_nth && !(++user_cnt % stall_user_nth)) -- return; -- } -- -- if (test_error_cnt && !--test_error_cnt) -- scx_bpf_error("test triggering error"); -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * All enqueued tasks must have their core_sched_seq updated for correct -- * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in -- * qmap_ops.flags. -- */ -- tctx->core_sched_seq = core_sched_tail_seqs[idx]++; -- -- /* -- * If qmap_select_cpu() is telling us to or this is the last runnable -- * task on the CPU, enqueue locally. -- */ -- if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) { -- tctx->force_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); -- return; -- } -- -- /* -- * If the task was re-enqueued due to the CPU being preempted by a -- * higher priority scheduling class, just re-enqueue the task directly -- * on the global DSQ. As we want another CPU to pick it up, find and -- * kick an idle CPU. -- */ -- if (enq_flags & SCX_ENQ_REENQ) { -- s32 cpu; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags); -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- return; -- } -- -- ring = bpf_map_lookup_elem(&queue_arr, &idx); -- if (!ring) { -- scx_bpf_error("failed to find ring %d", idx); -- return; -- } -- -- /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */ -- if (bpf_map_push_elem(ring, &pid, 0)) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_enqueued, 1); --} -- --/* -- * The BPF queue map doesn't support removal and sched_ext can handle spurious -- * dispatches. qmap_dequeue() is only used to collect statistics. -- */ --void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags) --{ -- __sync_fetch_and_add(&nr_dequeued, 1); -- if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) -- __sync_fetch_and_add(&nr_core_sched_execed, 1); --} -- --static void update_core_sched_head_seq(struct task_struct *p) --{ -- struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- int idx = weight_to_idx(p->scx.weight); -- -- if (tctx) -- core_sched_head_seqs[idx] = tctx->core_sched_seq; -- else -- scx_bpf_error("task_ctx lookup failed"); --} -- --void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 zero = 0, one = 1; -- u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero); -- u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one); -- void *fifo; -- s32 pid; -- int i; -- -- if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) { -- struct task_struct *p; -- -- /* -- * PID 2 should be kthreadd which should mostly be idle and off -- * the scheduler. Let's keep dispatching it to force the kernel -- * to call this function over and over again. -- */ -- p = bpf_task_from_pid(2); -- if (p) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- if (!idx || !cnt) { -- scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt); -- return; -- } -- -- for (i = 0; i < 5; i++) { -- /* Advance the dispatch cursor and pick the fifo. */ -- if (!*cnt) { -- *idx = (*idx + 1) % 5; -- *cnt = 1 << *idx; -- } -- (*cnt)--; -- -- fifo = bpf_map_lookup_elem(&queue_arr, idx); -- if (!fifo) { -- scx_bpf_error("failed to find ring %llu", *idx); -- return; -- } -- -- /* Dispatch or advance. */ -- if (!bpf_map_pop_elem(fifo, &pid)) { -- struct task_struct *p; -- -- p = bpf_task_from_pid(pid); -- if (p) { -- update_core_sched_head_seq(p); -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- *cnt = 0; -- } --} -- --/* -- * The distance from the head of the queue scaled by the weight of the queue. -- * The lower the number, the older the task and the higher the priority. -- */ --static s64 task_qdist(struct task_struct *p) --{ -- int idx = weight_to_idx(p->scx.weight); -- struct task_ctx *tctx; -- s64 qdist; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return 0; -- } -- -- qdist = tctx->core_sched_seq - core_sched_head_seqs[idx]; -- -- /* -- * As queue index increments, the priority doubles. The queue w/ index 3 -- * is dispatched twice more frequently than 2. Reflect the difference by -- * scaling qdists accordingly. Note that the shift amount needs to be -- * flipped depending on the sign to avoid flipping priority direction. -- */ -- if (qdist >= 0) -- return qdist << (4 - idx); -- else -- return qdist << idx; --} -- --/* -- * This is called to determine the task ordering when core-sched is picking -- * tasks to execute on SMT siblings and should encode about the same ordering as -- * the regular scheduling path. Use the priority-scaled distances from the head -- * of the queues to compare the two tasks which should be consistent with the -- * dispatch path behavior. -- */ --bool BPF_STRUCT_OPS(qmap_core_sched_before, -- struct task_struct *a, struct task_struct *b) --{ -- return task_qdist(a) > task_qdist(b); --} -- --void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- u32 cnt; -- -- /* -- * Called when @cpu is taken by a higher priority scheduling class. This -- * makes @cpu no longer available for executing sched_ext tasks. As we -- * don't want the tasks in @cpu's local dsq to sit there until @cpu -- * becomes available again, re-enqueue them into the global dsq. See -- * %SCX_ENQ_REENQ handling in qmap_enqueue(). -- */ -- cnt = scx_bpf_reenqueue_local(); -- if (cnt) -- __sync_fetch_and_add(&nr_reenqueued, cnt); --} -- --s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (p->tgid == disallow_tgid) -- p->scx.disallow = true; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(qmap_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops qmap_ops = { -- .select_cpu = (void *)qmap_select_cpu, -- .enqueue = (void *)qmap_enqueue, -- .dequeue = (void *)qmap_dequeue, -- .dispatch = (void *)qmap_dispatch, -- .core_sched_before = (void *)qmap_core_sched_before, -- .cpu_release = (void *)qmap_cpu_release, -- .prep_enable = (void *)qmap_prep_enable, -- .init = (void *)qmap_init, -- .exit = (void *)qmap_exit, -- .flags = SCX_OPS_ENQ_LAST, -- .timeout_ms = 5000U, -- .name = "qmap", --}; -diff --git b/tools/sched_ext/scx_example_qmap.c a/tools/sched_ext/scx_example_qmap.c -deleted file mode 100644 -index ccb4814ee61b..000000000000 ---- b/tools/sched_ext/scx_example_qmap.c -+++ /dev/null -@@ -1,107 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_qmap.skel.h" -- --const char help_fmt[] = --"A simple five-level FIFO queue sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" --" -t COUNT Stall every COUNT'th user thread\n" --" -T COUNT Stall every COUNT'th kernel thread\n" --" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n" --" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_qmap *skel; -- struct bpf_link *link; -- int opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_qmap__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) { -- switch (opt) { -- case 's': -- skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; -- break; -- case 'e': -- skel->bss->test_error_cnt = strtoul(optarg, NULL, 0); -- break; -- case 't': -- skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0); -- break; -- case 'T': -- skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0); -- break; -- case 'l': -- skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0); -- break; -- case 'd': -- skel->rodata->disallow_tgid = strtol(optarg, NULL, 0); -- if (skel->rodata->disallow_tgid < 0) -- skel->rodata->disallow_tgid = getpid(); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_qmap__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.qmap_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- long nr_enqueued = skel->bss->nr_enqueued; -- long nr_dispatched = skel->bss->nr_dispatched; -- -- printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n", -- nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, -- skel->bss->nr_reenqueued, skel->bss->nr_dequeued, -- skel->bss->nr_core_sched_execed); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_qmap__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_simple.bpf.c a/tools/sched_ext/scx_example_simple.bpf.c -deleted file mode 100644 -index 4bccca3e2047..000000000000 ---- b/tools/sched_ext/scx_example_simple.bpf.c -+++ /dev/null -@@ -1,128 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple scheduler. -- * -- * By default, it operates as a simple global weighted vtime scheduler and can -- * be switched to FIFO scheduling. It also demonstrates the following niceties. -- * -- * - Statistics tracking how many tasks are queued to local and global dsq's. -- * - Termination notification for userspace. -- * -- * While very simple, this scheduler should work reasonably well on CPUs with a -- * uniform L3 cache topology. While preemption is not implemented, the fact that -- * the scheduling queue is shared across all CPUs means that whatever is at the -- * front of the queue is likely to be executed fairly quickly given enough -- * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads -- * but comes with the usual problems with FIFO scheduling where saturating -- * threads can easily drown out interactive ones. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --static u64 vtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, 2); /* [local, global] */ --} stats SEC(".maps"); -- --static void stat_inc(u32 idx) --{ -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); -- if (cnt_p) -- (*cnt_p)++; --} -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) --{ -- /* -- * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that -- * running @p on its CPU directly shouldn't affect fairness. Just queue -- * it on the local FIFO. -- */ -- if (enq_flags & SCX_ENQ_LOCAL) { -- stat_inc(0); /* count local queueing */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- return; -- } -- -- stat_inc(1); /* count global queueing */ -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) -- vtime = vtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --void BPF_STRUCT_OPS(simple_running, struct task_struct *p) --{ -- if (fifo_sched) -- return; -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(vtime_now, p->scx.dsq_vtime)) -- vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --s32 BPF_STRUCT_OPS(simple_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .running = (void *)simple_running, -- .stopping = (void *)simple_stopping, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", --}; -diff --git b/tools/sched_ext/scx_example_simple.c a/tools/sched_ext/scx_example_simple.c -deleted file mode 100644 -index 486b401f7c95..000000000000 ---- b/tools/sched_ext/scx_example_simple.c -+++ /dev/null -@@ -1,101 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_simple.skel.h" -- --const char help_fmt[] = --"A simple sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-f] [-p]\n" --"\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int simple) --{ -- exit_req = 1; --} -- --static void read_stats(struct scx_example_simple *skel, u64 *stats) --{ -- int nr_cpus = libbpf_num_possible_cpus(); -- u64 cnts[2][nr_cpus]; -- u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * 2); -- -- for (idx = 0; idx < 2; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_simple *skel; -- struct bpf_link *link; -- u32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_simple__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "fph")) != -1) { -- switch (opt) { -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_simple__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.simple_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- u64 stats[2]; -- -- read_stats(skel, stats); -- printf("local=%lu global=%lu\n", stats[0], stats[1]); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_simple__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_userland.bpf.c a/tools/sched_ext/scx_example_userland.bpf.c -deleted file mode 100644 -index a089bc6bbe86..000000000000 ---- b/tools/sched_ext/scx_example_userland.bpf.c -+++ /dev/null -@@ -1,269 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A minimal userland scheduler. -- * -- * In terms of scheduling, this provides two different types of behaviors: -- * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity. -- * All such tasks are direct-dispatched from the kernel, and are never -- * enqueued in user space. -- * 2. A primitive vruntime scheduler that is implemented in user space, for all -- * other tasks. -- * -- * Some parts of this example user space scheduler could be implemented more -- * efficiently using more complex and sophisticated data structures. For -- * example, rather than using BPF_MAP_TYPE_QUEUE's, -- * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between -- * user space and kernel space. Similarly, we use a simple vruntime-sorted list -- * in user space, but an rbtree could be used instead. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include --#include "scx_common.bpf.h" --#include "scx_example_userland_common.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; --const volatile s32 usersched_pid; -- --/* !0 for veristat, set during init */ --const volatile u32 num_possible_cpus = 64; -- --/* Stats that are printed by user space. */ --u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues; -- --struct user_exit_info uei; -- --/* -- * Whether the user space scheduler needs to be scheduled due to a task being -- * enqueued in user space. -- */ --static bool usersched_needed; -- --/* -- * The map containing tasks that are enqueued in user space from the kernel. -- * -- * This map is drained by the user space scheduler. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, struct scx_userland_enqueued_task); --} enqueued SEC(".maps"); -- --/* -- * The map containing tasks that are dispatched to the kernel from user space. -- * -- * Drained by the kernel in userland_dispatch(). -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, s32); --} dispatched SEC(".maps"); -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local DSQ */ --}; -- --/* Map that contains task-local storage. */ --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --static bool is_usersched_task(const struct task_struct *p) --{ -- return p->pid == usersched_pid; --} -- --static bool keep_in_kernel(const struct task_struct *p) --{ -- return p->nr_cpus_allowed < num_possible_cpus; --} -- --static struct task_struct *usersched_task(void) --{ -- struct task_struct *p; -- -- p = bpf_task_from_pid(usersched_pid); -- /* -- * Should never happen -- the usersched task should always be managed -- * by sched_ext. -- */ -- if (!p) { -- scx_bpf_error("Failed to find usersched task %d", usersched_pid); -- /* -- * We should never hit this path, and we error out of the -- * scheduler above just in case, so the scheduler will soon be -- * be evicted regardless. So as to simplify the logic in the -- * caller to not have to check for NULL, return an acquired -- * reference to the current task here rather than NULL. -- */ -- return bpf_task_acquire(bpf_get_current_task_btf()); -- } -- -- return p; --} -- --s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- if (keep_in_kernel(p)) { -- s32 cpu; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to look up task-local storage for %s", p->comm); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- tctx->force_local = true; -- return cpu; -- } -- } -- -- return prev_cpu; --} -- --static void dispatch_user_scheduler(void) --{ -- struct task_struct *p; -- -- usersched_needed = false; -- p = usersched_task(); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); --} -- --static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags) --{ -- struct scx_userland_enqueued_task task; -- -- memset(&task, 0, sizeof(task)); -- task.pid = p->pid; -- task.sum_exec_runtime = p->se.sum_exec_runtime; -- task.weight = p->scx.weight; -- -- if (bpf_map_push_elem(&enqueued, &task, 0)) { -- /* -- * If we fail to enqueue the task in user space, put it -- * directly on the global DSQ. -- */ -- __sync_fetch_and_add(&nr_failed_enqueues, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- __sync_fetch_and_add(&nr_user_enqueues, 1); -- usersched_needed = true; -- } --} -- --void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags) --{ -- if (keep_in_kernel(p)) { -- u64 dsq_id = SCX_DSQ_GLOBAL; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to lookup task ctx for %s", p->comm); -- return; -- } -- -- if (tctx->force_local) -- dsq_id = SCX_DSQ_LOCAL; -- tctx->force_local = false; -- scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags); -- __sync_fetch_and_add(&nr_kernel_enqueues, 1); -- return; -- } else if (!is_usersched_task(p)) { -- enqueue_task_in_user_space(p, enq_flags); -- } --} -- --void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (usersched_needed) -- dispatch_user_scheduler(); -- -- bpf_repeat(4096) { -- s32 pid; -- struct task_struct *p; -- -- if (bpf_map_pop_elem(&dispatched, &pid)) -- break; -- -- /* -- * The task could have exited by the time we get around to -- * dispatching it. Treat this as a normal occurrence, and simply -- * move onto the next iteration. -- */ -- p = bpf_task_from_pid(pid); -- if (!p) -- continue; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } --} -- --s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(userland_init) --{ -- if (num_possible_cpus == 0) { -- scx_bpf_error("User scheduler # CPUs uninitialized (%d)", -- num_possible_cpus); -- return -EINVAL; -- } -- -- if (usersched_pid <= 0) { -- scx_bpf_error("User scheduler pid uninitialized (%d)", -- usersched_pid); -- return -EINVAL; -- } -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops userland_ops = { -- .select_cpu = (void *)userland_select_cpu, -- .enqueue = (void *)userland_enqueue, -- .dispatch = (void *)userland_dispatch, -- .prep_enable = (void *)userland_prep_enable, -- .init = (void *)userland_init, -- .exit = (void *)userland_exit, -- .timeout_ms = 3000, -- .name = "userland", --}; -diff --git b/tools/sched_ext/scx_example_userland.c a/tools/sched_ext/scx_example_userland.c -deleted file mode 100644 -index 4152b1e65fe1..000000000000 ---- b/tools/sched_ext/scx_example_userland.c -+++ /dev/null -@@ -1,402 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext user space scheduler which provides vruntime semantics -- * using a simple ordered-list implementation. -- * -- * Each CPU in the system resides in a single, global domain. This precludes -- * the need to do any load balancing between domains. The scheduler could -- * easily be extended to support multiple domains, with load balancing -- * happening in user space. -- * -- * Any task which has any CPU affinity is scheduled entirely in BPF. This -- * program only schedules tasks which may run on any CPU. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include "user_exit_info.h" --#include "scx_example_userland_common.h" --#include "scx_example_userland.skel.h" -- --const char help_fmt[] = --"A minimal userland sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-b BATCH] [-p]\n" --"\n" --" -b BATCH The number of tasks to batch when dispatching (default: 8)\n" --" -p Don't switch all, switch only tasks on SCHED_EXT policy\n" --" -h Display this help and exit\n"; -- --/* Defined in UAPI */ --#define SCHED_EXT 7 -- --/* Number of tasks to batch when dispatching to user space. */ --static __u32 batch_size = 8; -- --static volatile int exit_req; --static int enqueued_fd, dispatched_fd; -- --static struct scx_example_userland *skel; --static struct bpf_link *ops_link; -- --/* Stats collected in user space. */ --static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches; -- --/* The data structure containing tasks that are enqueued in user space. */ --struct enqueued_task { -- LIST_ENTRY(enqueued_task) entries; -- __u64 sum_exec_runtime; -- double vruntime; --}; -- --/* -- * Use a vruntime-sorted list to store tasks. This could easily be extended to -- * a more optimal data structure, such as an rbtree as is done in CFS. We -- * currently elect to use a sorted list to simplify the example for -- * illustrative purposes. -- */ --LIST_HEAD(listhead, enqueued_task); -- --/* -- * A vruntime-sorted list of tasks. The head of the list contains the task with -- * the lowest vruntime. That is, the task that has the "highest" claim to be -- * scheduled. -- */ --static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head); -- --/* -- * The statically allocated array of tasks. We use a statically allocated list -- * here to avoid having to allocate on the enqueue path, which could cause a -- * deadlock. A more substantive user space scheduler could e.g. provide a hook -- * for newly enabled tasks that are passed to the scheduler from the -- * .prep_enable() callback to allows the scheduler to allocate on safe paths. -- */ --struct enqueued_task tasks[USERLAND_MAX_TASKS]; -- --static double min_vruntime; -- --static void sigint_handler(int userland) --{ -- exit_req = 1; --} -- --static __u32 task_pid(const struct enqueued_task *task) --{ -- return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task); --} -- --static int dispatch_task(s32 pid) --{ -- int err; -- -- err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d\n", pid); -- exit_req = 1; -- } else { -- nr_vruntime_dispatches++; -- } -- -- return err; --} -- --static struct enqueued_task *get_enqueued_task(__s32 pid) --{ -- if (pid >= USERLAND_MAX_TASKS) -- return NULL; -- -- return &tasks[pid]; --} -- --static double calc_vruntime_delta(__u64 weight, __u64 delta) --{ -- double weight_f = (double)weight / 100.0; -- double delta_f = (double)delta; -- -- return delta_f / weight_f; --} -- --static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task) --{ -- __u64 delta; -- -- delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime; -- -- enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta); -- if (min_vruntime > enqueued->vruntime) -- enqueued->vruntime = min_vruntime; -- enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime; --} -- --static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task) --{ -- struct enqueued_task *curr, *enqueued, *prev; -- -- curr = get_enqueued_task(bpf_task->pid); -- if (!curr) -- return ENOENT; -- -- update_enqueued(curr, bpf_task); -- nr_vruntime_enqueues++; -- -- /* -- * Enqueue the task in a vruntime-sorted list. A more optimal data -- * structure such as an rbtree could easily be used as well. We elect -- * to use a list here simply because it's less code, and thus the -- * example is less convoluted and better serves to illustrate what a -- * user space scheduler could look like. -- */ -- -- if (LIST_EMPTY(&vruntime_head)) { -- LIST_INSERT_HEAD(&vruntime_head, curr, entries); -- return 0; -- } -- -- LIST_FOREACH(enqueued, &vruntime_head, entries) { -- if (curr->vruntime <= enqueued->vruntime) { -- LIST_INSERT_BEFORE(enqueued, curr, entries); -- return 0; -- } -- prev = enqueued; -- } -- -- LIST_INSERT_AFTER(prev, curr, entries); -- -- return 0; --} -- --static void drain_enqueued_map(void) --{ -- while (1) { -- struct scx_userland_enqueued_task task; -- int err; -- -- if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) -- return; -- -- err = vruntime_enqueue(&task); -- if (err) { -- fprintf(stderr, "Failed to enqueue task %d: %s\n", -- task.pid, strerror(err)); -- exit_req = 1; -- return; -- } -- } --} -- --static void dispatch_batch(void) --{ -- __u32 i; -- -- for (i = 0; i < batch_size; i++) { -- struct enqueued_task *task; -- int err; -- __s32 pid; -- -- task = LIST_FIRST(&vruntime_head); -- if (!task) -- return; -- -- min_vruntime = task->vruntime; -- pid = task_pid(task); -- LIST_REMOVE(task, entries); -- err = dispatch_task(pid); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d in %u\n", -- pid, i); -- return; -- } -- } --} -- --static void *run_stats_printer(void *arg) --{ -- while (!exit_req) { -- __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; -- -- nr_failed_enqueues = skel->bss->nr_failed_enqueues; -- nr_kernel_enqueues = skel->bss->nr_kernel_enqueues; -- nr_user_enqueues = skel->bss->nr_user_enqueues; -- total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues; -- -- printf("o-----------------------o\n"); -- printf("| BPF ENQUEUES |\n"); -- printf("|-----------------------|\n"); -- printf("| kern: %10llu |\n", nr_kernel_enqueues); -- printf("| user: %10llu |\n", nr_user_enqueues); -- printf("| failed: %10llu |\n", nr_failed_enqueues); -- printf("| -------------------- |\n"); -- printf("| total: %10llu |\n", total); -- printf("| |\n"); -- printf("|-----------------------|\n"); -- printf("| VRUNTIME / USER |\n"); -- printf("|-----------------------|\n"); -- printf("| enq: %10llu |\n", nr_vruntime_enqueues); -- printf("| disp: %10llu |\n", nr_vruntime_dispatches); -- printf("o-----------------------o\n"); -- printf("\n\n"); -- sleep(1); -- } -- -- return NULL; --} -- --static int spawn_stats_thread(void) --{ -- pthread_t stats_printer; -- -- return pthread_create(&stats_printer, NULL, run_stats_printer, NULL); --} -- --static int bootstrap(int argc, char **argv) --{ -- int err; -- __u32 opt; -- struct sched_param sched_param = { -- .sched_priority = sched_get_priority_max(SCHED_EXT), -- }; -- bool switch_partial = false; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- /* -- * Enforce that the user scheduler task is managed by sched_ext. The -- * task eagerly drains the list of enqueued tasks in its main work -- * loop, and then yields the CPU. The BPF scheduler only schedules the -- * user space scheduler task when at least one other task in the system -- * needs to be scheduled. -- */ -- err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param); -- if (err) { -- fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err)); -- return err; -- } -- -- while ((opt = getopt(argc, argv, "b:ph")) != -1) { -- switch (opt) { -- case 'b': -- batch_size = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- exit(opt != 'h'); -- } -- } -- -- /* -- * It's not always safe to allocate in a user space scheduler, as an -- * enqueued task could hold a lock that we require in order to be able -- * to allocate. -- */ -- err = mlockall(MCL_CURRENT | MCL_FUTURE); -- if (err) { -- fprintf(stderr, "Failed to prefault and lock address space: %s\n", -- strerror(err)); -- return err; -- } -- -- skel = scx_example_userland__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno)); -- return errno; -- } -- skel->rodata->num_possible_cpus = libbpf_num_possible_cpus(); -- assert(skel->rodata->num_possible_cpus > 0); -- skel->rodata->usersched_pid = getpid(); -- assert(skel->rodata->usersched_pid > 0); -- skel->rodata->switch_partial = switch_partial; -- -- err = scx_example_userland__load(skel); -- if (err) { -- fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- enqueued_fd = bpf_map__fd(skel->maps.enqueued); -- dispatched_fd = bpf_map__fd(skel->maps.dispatched); -- assert(enqueued_fd > 0); -- assert(dispatched_fd > 0); -- -- err = spawn_stats_thread(); -- if (err) { -- fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops); -- if (!ops_link) { -- fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno)); -- err = errno; -- goto destroy_skel; -- } -- -- return 0; -- --destroy_skel: -- scx_example_userland__destroy(skel); -- exit_req = 1; -- return err; --} -- --static void sched_main_loop(void) --{ -- while (!exit_req) { -- /* -- * Perform the following work in the main user space scheduler -- * loop: -- * -- * 1. Drain all tasks from the enqueued map, and enqueue them -- * to the vruntime sorted list. -- * -- * 2. Dispatch a batch of tasks from the vruntime sorted list -- * down to the kernel. -- * -- * 3. Yield the CPU back to the system. The BPF scheduler will -- * reschedule the user space scheduler once another task has -- * been enqueued to user space. -- */ -- drain_enqueued_map(); -- dispatch_batch(); -- sched_yield(); -- } --} -- --int main(int argc, char **argv) --{ -- int err; -- -- err = bootstrap(argc, argv); -- if (err) { -- fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err)); -- return err; -- } -- -- sched_main_loop(); -- -- exit_req = 1; -- bpf_link__destroy(ops_link); -- uei_print(&skel->bss->uei); -- scx_example_userland__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_userland_common.h a/tools/sched_ext/scx_example_userland_common.h -deleted file mode 100644 -index 639c6809c5ff..000000000000 ---- b/tools/sched_ext/scx_example_userland_common.h -+++ /dev/null -@@ -1,19 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* Copyright (c) 2022 Meta, Inc */ -- --#ifndef __SCX_USERLAND_COMMON_H --#define __SCX_USERLAND_COMMON_H -- --#define USERLAND_MAX_TASKS 8192 -- --/* -- * An instance of a task that has been enqueued by the kernel for consumption -- * by a user space global scheduler thread. -- */ --struct scx_userland_enqueued_task { -- __s32 pid; -- u64 sum_exec_runtime; -- u64 weight; --}; -- --#endif // __SCX_USERLAND_COMMON_H -diff --git b/tools/sched_ext/user_exit_info.h a/tools/sched_ext/user_exit_info.h -deleted file mode 100644 -index e701ef0e0b86..000000000000 ---- b/tools/sched_ext/user_exit_info.h -+++ /dev/null -@@ -1,50 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Define struct user_exit_info which is shared between BPF and userspace parts -- * to communicate exit status and other information. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __USER_EXIT_INFO_H --#define __USER_EXIT_INFO_H -- --struct user_exit_info { -- int type; -- char reason[128]; -- char msg[1024]; --}; -- --#ifdef __bpf__ -- --#include "vmlinux.h" --#include -- --static inline void uei_record(struct user_exit_info *uei, -- const struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason); -- bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg); -- /* use __sync to force memory barrier */ -- __sync_val_compare_and_swap(&uei->type, uei->type, ei->type); --} -- --#else /* !__bpf__ */ -- --static inline bool uei_exited(struct user_exit_info *uei) --{ -- /* use __sync to force memory barrier */ -- return __sync_val_compare_and_swap(&uei->type, -1, -1); --} -- --static inline void uei_print(const struct user_exit_info *uei) --{ -- fprintf(stderr, "EXIT: %s", uei->reason); -- if (uei->msg[0] != '\0') -- fprintf(stderr, " (%s)", uei->msg); -- fputs("\n", stderr); --} -- --#endif /* __bpf__ */ --#endif /* __USER_EXIT_INFO_H */ diff --git a/oneplus/hmbird/fengchi_OP13T_A15.patch b/oneplus/hmbird/fengchi_OP13T_A15.patch deleted file mode 100644 index e5d21faa..00000000 --- a/oneplus/hmbird/fengchi_OP13T_A15.patch +++ /dev/null @@ -1,21125 +0,0 @@ -diff --git a/Documentation/scheduler/index.rst b/Documentation/scheduler/index.rst -index 0b650bb550e6..3170747226f6 100644 ---- a/Documentation/scheduler/index.rst -+++ b/Documentation/scheduler/index.rst -@@ -19,7 +19,6 @@ Scheduler - sched-nice-design - sched-rt-group - sched-stats -- sched-ext - sched-debug - - text_files -diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst -deleted file mode 100644 -index 84c30b44f104..000000000000 ---- a/Documentation/scheduler/sched-ext.rst -+++ /dev/null -@@ -1,230 +0,0 @@ --========================== --Extensible Scheduler Class --========================== -- --sched_ext is a scheduler class whose behavior can be defined by a set of BPF --programs - the BPF scheduler. -- --* sched_ext exports a full scheduling interface so that any scheduling -- algorithm can be implemented on top. -- --* The BPF scheduler can group CPUs however it sees fit and schedule them -- together, as tasks aren't tied to specific CPUs at the time of wakeup. -- --* The BPF scheduler can be turned on and off dynamically anytime. -- --* The system integrity is maintained no matter what the BPF scheduler does. -- The default scheduling behavior is restored anytime an error is detected, -- a runnable task stalls, or on invoking the SysRq key sequence -- :kbd:`SysRq-S`. -- --Switching to and from sched_ext --=============================== -- --``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and --``tools/sched_ext`` contains the example schedulers. -- --sched_ext is used only when the BPF scheduler is loaded and running. -- --If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be --treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is --loaded. On load, such tasks will be switched to and scheduled by sched_ext. -- --The BPF scheduler can choose to schedule all normal and lower class tasks by --calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this --case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and --``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers, --this mode can be selected with the ``-a`` option. -- --Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or --detection of any internal error including stalled runnable tasks aborts the --BPF scheduler and reverts all tasks back to CFS. -- --.. code-block:: none -- -- # make -j16 -C tools/sched_ext -- # tools/sched_ext/scx_example_simple -- local=0 global=3 -- local=5 global=24 -- local=9 global=44 -- local=13 global=56 -- local=17 global=72 -- ^CEXIT: BPF scheduler unregistered -- --If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler --and whether a given task is on sched_ext can be determined as follows: -- --.. code-block:: none -- -- # cat /sys/kernel/debug/sched/ext -- ops : simple -- enabled : 1 -- switching_all : 1 -- switched_all : 1 -- enable_state : enabled -- -- # grep ext /proc/self/sched -- ext.enabled : 1 -- --The Basics --========== -- --Userspace can implement an arbitrary BPF scheduler by loading a set of BPF --programs that implement ``struct sched_ext_ops``. The only mandatory field --is ``ops.name`` which must be a valid BPF object name. All operations are --optional. The following modified excerpt is from --``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO --scheduler. -- --.. code-block:: c -- -- s32 BPF_STRUCT_OPS(simple_init) -- { -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; -- } -- -- void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) -- { -- if (enq_flags & SCX_ENQ_LOCAL) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags); -- } -- -- void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) -- { -- exit_type = ei->type; -- } -- -- SEC(".struct_ops") -- struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", -- }; -- --Dispatch Queues ----------------- -- --To match the impedance between the scheduler core and the BPF scheduler, --sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a --priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``), --and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage --an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and --``scx_bpf_destroy_dsq()``. -- --A CPU always executes a task from its local DSQ. A task is "dispatched" to a --DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's --local DSQ. -- --When a CPU is looking for the next task to run, if the local DSQ is not --empty, the first task is picked. Otherwise, the CPU tries to consume the --global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()`` --is invoked. -- --Scheduling Cycle ------------------ -- --The following briefly shows how a waking task is scheduled and executed. -- --1. When a task is waking up, ``ops.select_cpu()`` is the first operation -- invoked. This serves two purposes. First, CPU selection optimization -- hint. Second, waking up the selected CPU if idle. -- -- The CPU selected by ``ops.select_cpu()`` is an optimization hint and not -- binding. The actual decision is made at the last step of scheduling. -- However, there is a small performance gain if the CPU -- ``ops.select_cpu()`` returns matches the CPU the task eventually runs on. -- -- A side-effect of selecting a CPU is waking it up from idle. While a BPF -- scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper, -- using ``ops.select_cpu()`` judiciously can be simpler and more efficient. -- -- Note that the scheduler core will ignore an invalid CPU selection, for -- example, if it's outside the allowed cpumask of the task. -- --2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can -- make one of the following decisions: -- -- * Immediately dispatch the task to either the global or local DSQ by -- calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or -- ``SCX_DSQ_LOCAL``, respectively. -- -- * Immediately dispatch the task to a custom DSQ by calling -- ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63. -- -- * Queue the task on the BPF side. -- --3. When a CPU is ready to schedule, it first looks at its local DSQ. If -- empty, it then looks at the global DSQ. If there still isn't a task to -- run, ``ops.dispatch()`` is invoked which can use the following two -- functions to populate the local DSQ. -- -- * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can -- be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``, -- ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()`` -- currently can't be called with BPF locks held, this is being worked on -- and will be supported. ``scx_bpf_dispatch()`` schedules dispatching -- rather than performing them immediately. There can be up to -- ``ops.dispatch_max_batch`` pending tasks. -- -- * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ -- to the dispatching DSQ. This function cannot be called with any BPF -- locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks -- before trying to consume the specified DSQ. -- --4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ, -- the CPU runs the first one. If empty, the following steps are taken: -- -- * Try to consume the global DSQ. If successful, run the task. -- -- * If ``ops.dispatch()`` has dispatched any tasks, retry #3. -- -- * If the previous task is an SCX task and still runnable, keep executing -- it (see ``SCX_OPS_ENQ_LAST``). -- -- * Go idle. -- --Note that the BPF scheduler can always choose to dispatch tasks immediately --in ``ops.enqueue()`` as illustrated in the above simple example. If only the --built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as --a task is never queued on the BPF scheduler and both the local and global --DSQs are consumed automatically. -- --``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use --``scx_bpf_dispatch_vtime()`` for the priority queue. See the function --documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for --more information. -- --Where to Look --============= -- --* ``include/linux/sched/ext.h`` defines the core data structures, ops table -- and constants. -- --* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. -- The functions prefixed with ``scx_bpf_`` can be called from the BPF -- scheduler. -- --* ``tools/sched_ext/`` hosts example BPF scheduler implementations. -- -- * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example -- using a custom DSQ. -- -- * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting -- five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. -- --ABI Instability --=============== -- --The APIs provided by sched_ext to BPF schedulers programs have no stability --guarantees. This includes the ops table callbacks and constants defined in --``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in --``kernel/sched/ext.c``. -- --While we will attempt to provide a relatively stable API surface when --possible, they are subject to change without warning between kernel --versions. -diff --git a/MAINTAINERS b/MAINTAINERS -index ff6b9ea9b788..46ca82b0b506 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -19125,8 +19125,6 @@ R: Ben Segall (CONFIG_CFS_BANDWIDTH) - R: Mel Gorman (CONFIG_NUMA_BALANCING) - R: Daniel Bristot de Oliveira (SCHED_DEADLINE) - R: Valentin Schneider (TOPOLOGY) --R: Tejun Heo (SCHED_EXT) --R: David Vernet (SCHED_EXT) - L: linux-kernel@vger.kernel.org - S: Maintained - T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core -@@ -19135,7 +19133,6 @@ F: include/linux/sched.h - F: include/linux/wait.h - F: include/uapi/linux/sched.h - F: kernel/sched/ --F: tools/sched_ext/ - - SCSI LIBSAS SUBSYSTEM - R: John Garry -diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig -index f93980c09d7f..5ba590235a8c 100644 ---- a/arch/arm64/configs/defconfig -+++ b/arch/arm64/configs/defconfig -@@ -6,8 +6,7 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_BPF_SYSCALL=y - CONFIG_BPF_JIT=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_BSD_PROCESS_ACCT=y - CONFIG_BSD_PROCESS_ACCT_V3=y -diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig -index 42c199371c79..ef087e70fba4 100644 ---- a/arch/arm64/configs/gki_defconfig -+++ b/arch/arm64/configs/gki_defconfig -@@ -10,8 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y - # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set - CONFIG_BPF_LSM=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_TASKSTATS=y - CONFIG_TASK_DELAY_ACCT=y -diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c -index 4bd028fa7500..5825ce9d6d7b 100755 ---- a/drivers/gpu/drm/msm/msm_drv.c -+++ b/drivers/gpu/drm/msm/msm_drv.c -@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) - } - - sched_set_fifo(ev_thread->worker->task); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3); -+#endif - } - - ret = drm_vblank_init(ddev, priv->num_crtcs); -diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c -index bd001445815e..dcf2e1ebf9c5 100644 ---- a/drivers/tty/sysrq.c -+++ b/drivers/tty/sysrq.c -@@ -524,7 +524,6 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { - NULL, /* P */ - NULL, /* Q */ - NULL, /* R */ -- /* S: May be registered by sched_ext for resetting */ - NULL, /* S */ - NULL, /* T */ - NULL, /* U */ -diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h -index 3b9dde0c15c0..c0fee44f5417 100755 ---- a/include/asm-generic/vmlinux.lds.h -+++ b/include/asm-generic/vmlinux.lds.h -@@ -131,7 +131,7 @@ - *(__dl_sched_class) \ - *(__rt_sched_class) \ - *(__fair_sched_class) \ -- *(__ext_sched_class) \ -+ *(__hmbird_sched_class) \ - *(__idle_sched_class) \ - __sched_class_lowest = .; - -diff --git a/include/linux/sched.h b/include/linux/sched.h -index 6991b190b8dc..39c797c5cc75 100755 ---- a/include/linux/sched.h -+++ b/include/linux/sched.h -@@ -73,7 +73,9 @@ struct task_delay_info; - struct task_group; - struct user_event_mm; - --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - - /* - * Task state bitmask. NOTE! These bits are also -@@ -298,11 +300,6 @@ enum { - - extern void scheduler_tick(void); - --#ifdef CONFIG_SLIM_SCHED --extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer); --extern void stop_shadow_tick_timer(void); --#endif -- - #define MAX_SCHEDULE_TIMEOUT LONG_MAX - - extern long schedule_timeout(long timeout); -@@ -1523,13 +1520,8 @@ struct task_struct { - */ - struct callback_head l1d_flush_kill; - #endif --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, unsigned long sched_prop); -- ANDROID_KABI_USE(2, struct sched_ext_entity *scx); --#else - ANDROID_KABI_RESERVE(1); - ANDROID_KABI_RESERVE(2); --#endif - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); - ANDROID_KABI_RESERVE(5); -@@ -1932,6 +1924,91 @@ static inline int task_nice(const struct task_struct *p) - return PRIO_TO_NICE((p)->static_prio); - } - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline bool task_is_top_task(struct task_struct *p) -+{ -+ return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ ->top_task_prop & TOP_TASK_BITS_MASK); -+} -+ -+static inline int get_top_task_prop(struct task_struct *p) -+{ -+ return get_hmbird_ts(p)->top_task_prop; -+} -+ -+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear) -+{ -+ if (set) -+ get_hmbird_ts(p)->top_task_prop |= set; -+ if (clear) -+ get_hmbird_ts(p)->top_task_prop &= ~clear; -+ return 0; -+} -+ -+static inline void reset_top_task_prop(struct task_struct *p) -+{ -+ get_hmbird_ts(p)->top_task_prop = 0; -+} -+ -+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ entity->sched_prop = sp; -+ } -+ return 0; -+} -+ -+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->sched_prop; -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq) -+{ -+ unsigned long new_dsq; -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq; -+ entity->sched_prop = new_dsq; -+ } -+} -+ -+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK); -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ entity->dsq_sync_ux = val; -+} -+ -+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->dsq_sync_ux; -+ else -+ return 0; -+} -+#endif - - extern int can_nice(const struct task_struct *p, const int nice); - extern int task_curr(const struct task_struct *p); -diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h -deleted file mode 100755 -index b737a00c1373..000000000000 ---- a/include/linux/sched/ext.h -+++ /dev/null -@@ -1,647 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef _LINUX_SCHED_EXT_H --#define _LINUX_SCHED_EXT_H -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --#include -- --enum scx_consts { -- SCX_OPS_NAME_LEN = 128, -- SCX_EXIT_REASON_LEN = 128, -- SCX_EXIT_BT_LEN = 64, -- SCX_EXIT_MSG_LEN = 1024, -- -- SCX_SLICE_DFL = 20 * NSEC_PER_MSEC, -- SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */ --}; -- --/* -- * DSQ (dispatch queue) IDs are 64bit of the format: -- * -- * Bits: [63] [62 .. 0] -- * [ B] [ ID ] -- * -- * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -- * ID: 63 bit ID -- * -- * Built-in IDs: -- * -- * Bits: [63] [62] [61..32] [31 .. 0] -- * [ 1] [ L] [ R ] [ V ] -- * -- * 1: 1 for built-in DSQs. -- * L: 1 for LOCAL_ON DSQ IDs, 0 for others -- * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -- */ --enum scx_dsq_id_flags { -- SCX_DSQ_FLAG_BUILTIN = 1LLU << 63, -- SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -- -- SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0, -- SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, -- SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, -- SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, -- SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, --}; -- --enum scx_exit_type { -- SCX_EXIT_NONE, -- SCX_EXIT_DONE, -- -- SCX_EXIT_UNREG = 64, /* BPF unregistration */ -- SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -- SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ -- SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ --}; -- --/* -- * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is -- * being disabled. -- */ --struct scx_exit_info { -- /* %SCX_EXIT_* - broad category of the exit reason */ -- enum scx_exit_type type; -- /* textual representation of the above */ -- char reason[SCX_EXIT_REASON_LEN]; -- /* number of entries in the backtrace */ -- u32 bt_len; -- /* backtrace if exiting due to an error */ -- unsigned long bt[SCX_EXIT_BT_LEN]; -- /* extra message */ -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --/* sched_ext_ops.flags */ --enum scx_ops_flags { -- /* -- * Keep built-in idle tracking even if ops.update_idle() is implemented. -- */ -- SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, -- -- /* -- * By default, if there are no other task to run on the CPU, ext core -- * keeps running the current task even after its slice expires. If this -- * flag is specified, such tasks are passed to ops.enqueue() with -- * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. -- */ -- SCX_OPS_ENQ_LAST = 1LLU << 1, -- -- /* -- * An exiting task may schedule after PF_EXITING is set. In such cases, -- * bpf_task_from_pid() may not be able to find the task and if the BPF -- * scheduler depends on pid lookup for dispatching, the task will be -- * lost leading to various issues including RCU grace period stalls. -- * -- * To mask this problem, by default, unhashed tasks are automatically -- * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't -- * depend on pid lookups and wants to handle these tasks directly, the -- * following flag can be used. -- */ -- SCX_OPS_ENQ_EXITING = 1LLU << 2, -- -- /* -- * CPU cgroup knob enable flags -- */ -- SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16, /* cpu.weight */ -- -- SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | -- SCX_OPS_ENQ_LAST | -- SCX_OPS_ENQ_EXITING | -- SCX_OPS_CGROUP_KNOB_WEIGHT, --}; -- --/* argument container for ops.enable() and friends */ --struct scx_enable_args { --}; -- --/* argument container for ops->cgroup_init() */ --struct scx_cgroup_init_args { -- /* the weight of the cgroup [1..10000] */ -- u32 weight; --}; -- --enum scx_cpu_preempt_reason { -- /* next task is being scheduled by &sched_class_rt */ -- SCX_CPU_PREEMPT_RT, -- /* next task is being scheduled by &sched_class_dl */ -- SCX_CPU_PREEMPT_DL, -- /* next task is being scheduled by &sched_class_stop */ -- SCX_CPU_PREEMPT_STOP, -- /* unknown reason for SCX being preempted */ -- SCX_CPU_PREEMPT_UNKNOWN, --}; -- --/* -- * Argument container for ops->cpu_acquire(). Currently empty, but may be -- * expanded in the future. -- */ --struct scx_cpu_acquire_args {}; -- --/* argument container for ops->cpu_release() */ --struct scx_cpu_release_args { -- /* the reason the CPU was preempted */ -- enum scx_cpu_preempt_reason reason; -- -- /* the task that's going to be scheduled on the CPU */ -- struct task_struct *task; --}; -- --/** -- * struct sched_ext_ops - Operation table for BPF scheduler implementation -- * -- * Userland can implement an arbitrary scheduling policy by implementing and -- * loading operations in this table. -- */ --struct sched_ext_ops { -- /** -- * select_cpu - Pick the target CPU for a task which is being woken up -- * @p: task being woken up -- * @prev_cpu: the cpu @p was on before sleeping -- * @wake_flags: SCX_WAKE_* -- * -- * Decision made here isn't final. @p may be moved to any CPU while it -- * is getting dispatched for execution later. However, as @p is not on -- * the rq at this point, getting the eventual execution CPU right here -- * saves a small bit of overhead down the line. -- * -- * If an idle CPU is returned, the CPU is kicked and will try to -- * dispatch. While an explicit custom mechanism can be added, -- * select_cpu() serves as the default way to wake up idle CPUs. -- */ -- s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); -- -- /** -- * enqueue - Enqueue a task on the BPF scheduler -- * @p: task being enqueued -- * @enq_flags: %SCX_ENQ_* -- * -- * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() -- * or enqueue on the BPF scheduler. If not directly dispatched, the bpf -- * scheduler owns @p and if it fails to dispatch @p, the task will -- * stall. -- */ -- void (*enqueue)(struct task_struct *p, u64 enq_flags); -- -- /** -- * dequeue - Remove a task from the BPF scheduler -- * @p: task being dequeued -- * @deq_flags: %SCX_DEQ_* -- * -- * Remove @p from the BPF scheduler. This is usually called to isolate -- * the task while updating its scheduling properties (e.g. priority). -- * -- * The ext core keeps track of whether the BPF side owns a given task or -- * not and can gracefully ignore spurious dispatches from BPF side, -- * which makes it safe to not implement this method. However, depending -- * on the scheduling logic, this can lead to confusing behaviors - e.g. -- * scheduling position not being updated across a priority change. -- */ -- void (*dequeue)(struct task_struct *p, u64 deq_flags); -- -- /** -- * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs -- * @cpu: CPU to dispatch tasks for -- * @prev: previous task being switched out -- * -- * Called when a CPU's local dsq is empty. The operation should dispatch -- * one or more tasks from the BPF scheduler into the DSQs using -- * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using -- * scx_bpf_consume(). -- * -- * The maximum number of times scx_bpf_dispatch() can be called without -- * an intervening scx_bpf_consume() is specified by -- * ops.dispatch_max_batch. See the comments on top of the two functions -- * for more details. -- * -- * When not %NULL, @prev is an SCX task with its slice depleted. If -- * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in -- * @prev->scx.flags, it is not enqueued yet and will be enqueued after -- * ops.dispatch() returns. To keep executing @prev, return without -- * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. -- */ -- void (*dispatch)(s32 cpu, struct task_struct *prev); -- -- /** -- * runnable - A task is becoming runnable on its associated CPU -- * @p: task becoming runnable -- * @enq_flags: %SCX_ENQ_* -- * -- * This and the following three functions can be used to track a task's -- * execution state transitions. A task becomes ->runnable() on a CPU, -- * and then goes through one or more ->running() and ->stopping() pairs -- * as it runs on the CPU, and eventually becomes ->quiescent() when it's -- * done running on the CPU. -- * -- * @p is becoming runnable on the CPU because it's -- * -- * - waking up (%SCX_ENQ_WAKEUP) -- * - being moved from another CPU -- * - being restored after temporarily taken off the queue for an -- * attribute change. -- * -- * This and ->enqueue() are related but not coupled. This operation -- * notifies @p's state transition and may not be followed by ->enqueue() -- * e.g. when @p is being dispatched to a remote CPU. Likewise, a task -- * may be ->enqueue()'d without being preceded by this operation e.g. -- * after exhausting its slice. -- */ -- void (*runnable)(struct task_struct *p, u64 enq_flags); -- -- /** -- * running - A task is starting to run on its associated CPU -- * @p: task starting to run -- * -- * See ->runnable() for explanation on the task state notifiers. -- */ -- void (*running)(struct task_struct *p); -- -- /** -- * stopping - A task is stopping execution -- * @p: task stopping to run -- * @runnable: is task @p still runnable? -- * -- * See ->runnable() for explanation on the task state notifiers. If -- * !@runnable, ->quiescent() will be invoked after this operation -- * returns. -- */ -- void (*stopping)(struct task_struct *p, bool runnable); -- -- /** -- * quiescent - A task is becoming not runnable on its associated CPU -- * @p: task becoming not runnable -- * @deq_flags: %SCX_DEQ_* -- * -- * See ->runnable() for explanation on the task state notifiers. -- * -- * @p is becoming quiescent on the CPU because it's -- * -- * - sleeping (%SCX_DEQ_SLEEP) -- * - being moved to another CPU -- * - being temporarily taken off the queue for an attribute change -- * (%SCX_DEQ_SAVE) -- * -- * This and ->dequeue() are related but not coupled. This operation -- * notifies @p's state transition and may not be preceded by ->dequeue() -- * e.g. when @p is being dispatched to a remote CPU. -- */ -- void (*quiescent)(struct task_struct *p, u64 deq_flags); -- -- /** -- * yield - Yield CPU -- * @from: yielding task -- * @to: optional yield target task -- * -- * If @to is NULL, @from is yielding the CPU to other runnable tasks. -- * The BPF scheduler should ensure that other available tasks are -- * dispatched before the yielding task. Return value is ignored in this -- * case. -- * -- * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf -- * scheduler can implement the request, return %true; otherwise, %false. -- */ -- bool (*yield)(struct task_struct *from, struct task_struct *to); -- -- /** -- * core_sched_before - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Used by core-sched to determine the ordering between two tasks. See -- * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on -- * core-sched. -- * -- * Both @a and @b are runnable and may or may not currently be queued on -- * the BPF scheduler. Should return %true if @a should run before @b. -- * %false if there's no required ordering or @b should run before @a. -- * -- * If not specified, the default is ordering them according to when they -- * became runnable. -- */ -- bool (*core_sched_before)(struct task_struct *a,struct task_struct *b); -- -- /** -- * set_weight - Set task weight -- * @p: task to set weight for -- * @weight: new eight [1..10000] -- * -- * Update @p's weight to @weight. -- */ -- void (*set_weight)(struct task_struct *p, u32 weight); -- -- /** -- * set_cpumask - Set CPU affinity -- * @p: task to set CPU affinity for -- * @cpumask: cpumask of cpus that @p can run on -- * -- * Update @p's CPU affinity to @cpumask. -- */ -- void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask); -- -- /** -- * update_idle - Update the idle state of a CPU -- * @cpu: CPU to udpate the idle state for -- * @idle: whether entering or exiting the idle state -- * -- * This operation is called when @rq's CPU goes or leaves the idle -- * state. By default, implementing this operation disables the built-in -- * idle CPU tracking and the following helpers become unavailable: -- * -- * - scx_bpf_select_cpu_dfl() -- * - scx_bpf_test_and_clear_cpu_idle() -- * - scx_bpf_pick_idle_cpu() -- * - scx_bpf_any_idle_cpu() -- * -- * The user also must implement ops.select_cpu() as the default -- * implementation relies on scx_bpf_select_cpu_dfl(). -- * -- * If you keep the built-in idle tracking, specify the -- * %SCX_OPS_KEEP_BUILTIN_IDLE flag. -- */ -- void (*update_idle)(s32 cpu, bool idle); -- -- /** -- * cpu_acquire - A CPU is becoming available to the BPF scheduler -- * @cpu: The CPU being acquired by the BPF scheduler. -- * @args: Acquire arguments, see the struct definition. -- * -- * A CPU that was previously released from the BPF scheduler is now once -- * again under its control. -- */ -- void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); -- -- /** -- * cpu_release - A CPU is taken away from the BPF scheduler -- * @cpu: The CPU being released by the BPF scheduler. -- * @args: Release arguments, see the struct definition. -- * -- * The specified CPU is no longer under the control of the BPF -- * scheduler. This could be because it was preempted by a higher -- * priority sched_class, though there may be other reasons as well. The -- * caller should consult @args->reason to determine the cause. -- */ -- void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -- -- /** -- * cpu_online - A CPU became online -- * @cpu: CPU which just came up -- * -- * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs beforehand. -- */ -- void (*cpu_online)(s32 cpu); -- -- /** -- * cpu_offline - A CPU is going offline -- * @cpu: CPU which is going offline -- * -- * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs afterwards. -- */ -- void (*cpu_offline)(s32 cpu); -- -- /** -- * prep_enable - Prepare to enable BPF scheduling for a task -- * @p: task to prepare BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Either we're loading a BPF scheduler or a new task is being forked. -- * Prepare BPF scheduling for @p. This operation may block and can be -- * used for allocations. -- * -- * Return 0 for success, -errno for failure. An error return while -- * loading will abort loading of the BPF scheduler. During a fork, will -- * abort the specific fork. -- */ -- s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * enable - Enable BPF scheduling for a task -- * @p: task to enable BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Enable @p for BPF scheduling. @p is now in the cgroup specified for -- * the preceding prep_enable() and will start running soon. -- */ -- void (*enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * cancel_enable - Cancel prep_enable() -- * @p: task being canceled -- * @args: enable arguments, see the struct definition -- * -- * @p was prep_enable()'d but failed before reaching enable(). Undo the -- * preparation. -- */ -- void (*cancel_enable)(struct task_struct *p, -- struct scx_enable_args *args); -- -- /** -- * disable - Disable BPF scheduling for a task -- * @p: task to disable BPF scheduling for -- * -- * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. -- * Disable BPF scheduling for @p. -- */ -- void (*disable)(struct task_struct *p); -- -- /* -- * All online ops must come before ops.init(). -- */ -- -- /** -- * init - Initialize the BPF scheduler -- */ -- s32 (*init)(void); -- -- /** -- * exit - Clean up after the BPF scheduler -- * @info: Exit info -- */ -- void (*exit)(struct scx_exit_info *info); -- -- /** -- * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch -- */ -- u32 dispatch_max_batch; -- -- /** -- * flags - %SCX_OPS_* flags -- */ -- u64 flags; -- -- /** -- * timeout_ms - The maximum amount of time, in milliseconds, that a -- * runnable task should be able to wait before being scheduled. The -- * maximum timeout may not exceed the default timeout of 30 seconds. -- * -- * Defaults to the maximum allowed timeout value of 30 seconds. -- */ -- u32 timeout_ms; -- -- /** -- * name - BPF scheduler's name -- * -- * Must be a non-zero valid BPF object name including only isalnum(), -- * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the -- * BPF scheduler is enabled. -- */ -- char name[SCX_OPS_NAME_LEN]; --}; -- --/* -- * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -- * scheduler core and the BPF scheduler. See the documentation for more details. -- */ --struct scx_dispatch_q { -- raw_spinlock_t lock; -- struct list_head fifo; /* processed in dispatching order */ -- struct rb_root_cached priq; /* processed in p->scx.dsq_vtime order */ -- u32 nr; -- u64 id; -- struct llist_node free_node; -- struct rcu_head rcu; --}; -- --/* scx_entity.flags */ --enum scx_ent_flags { -- SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ -- SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -- SCX_TASK_ENQ_LOCAL = 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */ -- -- SCX_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -- SCX_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -- -- SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -- SCX_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -- -- SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ --}; -- --/* scx_entity.dsq_flags */ --enum scx_ent_dsq_flags { -- SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ --}; -- --/* -- * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from -- * everywhere and the following bits track which kfunc sets are currently -- * allowed for %current. This simple per-task tracking works because SCX ops -- * nest in a limited way. BPF will likely implement a way to allow and disallow -- * kfuncs depending on the calling context which will replace this manual -- * mechanism. See scx_kf_allow(). -- */ --enum scx_kf_mask { -- SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ -- /* all non-sleepables may be nested inside INIT and SLEEPABLE */ -- SCX_KF_INIT = 1 << 0, /* running ops.init() */ -- SCX_KF_SLEEPABLE = 1 << 1, /* other sleepable init operations */ -- /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ -- SCX_KF_CPU_RELEASE = 1 << 2, /* ops.cpu_release() */ -- /* ops.dequeue (in REST) may be nested inside DISPATCH */ -- SCX_KF_DISPATCH = 1 << 3, /* ops.dispatch() */ -- SCX_KF_ENQUEUE = 1 << 4, /* ops.enqueue() */ -- SCX_KF_REST = 1 << 5, /* other rq-locked operations */ -- -- __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | -- SCX_KF_ENQUEUE | SCX_KF_REST, -- __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_REST, --}; -- --#define RAVG_HIST_SIZE 5 --struct scx_sched_task_stats { -- u64 mark_start; -- u64 window_start; -- u32 sum; -- u32 sum_history[RAVG_HIST_SIZE]; -- int cidx; -- -- u32 demand; -- u16 demand_scaled; -- void *sdsq; --}; -- -- -- --/* -- * The following is embedded in task_struct and contains all fields necessary -- * for a task to be scheduled by SCX. -- */ --struct sched_ext_entity { -- struct scx_dispatch_q *dsq; -- struct { -- struct list_head fifo; /* dispatch order */ -- struct rb_node priq; /* p->scx.dsq_vtime order */ -- } dsq_node; -- struct list_head watchdog_node; -- u32 flags; /* protected by rq lock */ -- u32 dsq_flags; /* protected by dsq lock */ -- u32 weight; -- s32 sticky_cpu; -- s32 holding_cpu; -- u32 kf_mask; /* see scx_kf_mask above */ -- struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ -- atomic64_t ops_state; -- unsigned long runnable_at; --#ifdef CONFIG_SCHED_CORE -- u64 core_sched_at; /* see scx_prio_less() */ --#endif -- -- /* BPF scheduler modifiable fields */ -- -- /* -- * Runtime budget in nsecs. This is usually set through -- * scx_bpf_dispatch() but can also be modified directly by the BPF -- * scheduler. Automatically decreased by SCX as the task executes. On -- * depletion, a scheduling event is triggered. -- * -- * This value is cleared to zero if the task is preempted by -- * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the -- * task ran. Use p->se.sum_exec_runtime instead. -- */ -- u64 slice; -- -- /* -- * Used to order tasks when dispatching to the vtime-ordered priority -- * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime() -- * but can also be modified directly by the BPF scheduler. Modifying it -- * while a task is queued on a dsq may mangle the ordering and is not -- * recommended. -- */ -- u64 dsq_vtime; -- -- /* -- * If set, reject future sched_setscheduler(2) calls updating the policy -- * to %SCHED_EXT with -%EACCES. -- * -- * If set from ops.prep_enable() and the task's policy is already -- * %SCHED_EXT, which can happen while the BPF scheduler is being loaded -- * or by inhering the parent's policy during fork, the task's policy is -- * rejected and forcefully reverted to %SCHED_NORMAL. The number of such -- * events are reported through /sys/kernel/debug/sched_ext::nr_rejected. -- */ -- bool disallow; /* reject switching into SCX */ -- -- /* cold fields */ -- struct list_head tasks_node; -- struct task_struct *task; -- struct scx_sched_task_stats sts; --}; -- --void sched_ext_free(struct task_struct *p); -- --#else /* !CONFIG_SCHED_CLASS_EXT */ -- --static inline void sched_ext_free(struct task_struct *p) {} -- --#endif /* CONFIG_SCHED_CLASS_EXT */ --#endif /* _LINUX_SCHED_EXT_H */ -diff --git a/include/linux/sched/hmbird.h b/include/linux/sched/hmbird.h -index 780a545667e9..3c7861e37ade 100755 ---- a/include/linux/sched/hmbird.h -+++ b/include/linux/sched/hmbird.h -@@ -54,6 +54,7 @@ - extern atomic_t non_hmbird_task; - extern atomic_t __hmbird_ops_enabled; - #define hmbird_enabled() atomic_read(&__hmbird_ops_enabled) -+#define MAX_GLOBAL_DSQS (10) - - enum hmbird_consts { - HMBIRD_SLICE_DFL = 1 * NSEC_PER_MSEC, -@@ -90,15 +91,13 @@ enum hmbird_dsq_id_flags { - HMBIRD_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, - }; - --enum hmbird_exit_type { -- HMBIRD_EXIT_NONE, -- HMBIRD_EXIT_DONE, -- -- HMBIRD_EXIT_UNREG = 64, /* BPF unregistration */ -- HMBIRD_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- HMBIRD_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -+enum hmbird_switch_type { -+ HMBIRD_SWITCH_PROC, -+ HMBIRD_SWITCH_ERR_WDT, -+ HMBIRD_SWITCH_ERR_HB, -+ HMBIRD_SWITCH_ERR_DSQ, - HMBIRD_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ -+ HMBIRD_EXIT_ERROR_HEARTBEAT, /* heart beat has stopped */ - }; - - /* -@@ -199,11 +198,176 @@ struct hmbird_entity { - int dsq_sync_ux; - }; - -+ -+/* -+ * All variables use 64bits width, -+ * Avoid parsing problems caused by automatic alignment(with padding) of structures. -+ */ -+ -+/* NOTING : Must align to 64bits. */ -+#define DESC_STR_LEN (32) -+/* Supporti up to three-dimensional arrays. */ -+#define PARSE_DIMENS (3) -+struct meta_desc_t { -+ char desc_str[DESC_STR_LEN]; -+ u64 len; -+ u64 parse[PARSE_DIMENS]; -+}; -+ -+#define MAX_SWITCHS (5) -+struct hmbird_switch_t { -+ u64 switch_at; -+ u64 is_success; -+ u64 end_state; -+ u64 switch_reason; -+}; -+#define SWITCH_ITEMS (sizeof(struct hmbird_switch_t) / sizeof(u64)) -+ -+#define MAX_EXCEPS (5) -+enum excep_id { -+ NO_CGROUP_L1, -+ MODULE_UNLOAD, -+ ALREADY_ENABLED, -+ ALREADY_DISABLED, -+ INIT_TASK_FAIL, -+ ALLOC_RQSCX_FAIL, -+ DSQ_ID_ERR, -+ CPU_NO_MASK, -+ SCAN_ENTITY_NULL, -+ ITER_RET_NULL, -+ DEQ_DEQING, -+ ENQ_EXIST1, -+ ENQ_EXIST2, -+ TASK_LINKED1, -+ TASK_UNLINKED, -+ TASK_LINKED2, -+ TASK_UNQUED, -+ TASK_UNWATCHED, -+ HMBIRD_OPN, -+ TASK_WATCHED, -+ RQ_NO_RUNNING, -+ EXTRA_FLAGS, -+ HOLDING_CPU1, -+ HOLDING_CPU2, -+ TASK_OPS_PREPPED, -+ TASK_OPS_UNPREPPED, -+ HMBIRD_OPS_ERR, -+ MAX_EXCEP_ID, -+}; -+ -+struct snap_misc_t { -+ u64 hmbird_enabled; -+ u64 curr_ss; -+ u64 hmbird_ops_enable_state_var; -+ u64 non_ext_task; -+ u64 parctrl_high_ratio; -+ u64 parctrl_low_ratio; -+ u64 parctrl_high_ratio_l; -+ u64 parctrl_low_ratio_l; -+ u64 isoctrl_high_ratio; -+ u64 isoctrl_low_ratio; -+ u64 misfit_ds; -+ u64 partial_enable; -+ u64 iso_free_rescue; -+ u64 isolate_ctrl; -+ u64 snap_jiffies; -+ u64 snap_time; -+}; -+#define SNAP_ITEMS (sizeof(struct snap_misc_t) / sizeof(u64)) -+ -+struct panic_snapshot_t { -+ /* what time dose the first task of dsq turn in runnable, check starvation */ -+ struct meta_desc_t runnable_at_meta; -+ u64 runnable_at[MAX_GLOBAL_DSQS]; -+ -+ struct meta_desc_t rq_nr_meta; -+ u64 rq_nr[NR_CPUS]; -+ -+ struct meta_desc_t scxrq_nr_meta; -+ u64 scxrq_nr[NR_CPUS]; -+ -+ struct meta_desc_t snap_misc_meta; -+ struct snap_misc_t snap_misc; -+}; -+ -+/* -+ * Do not record info in hot paths unless absolutely necessary, -+ * The impact on performance should be minimized. -+ */ -+struct kernel_info_t { -+ struct meta_desc_t sw_rec_meta; -+ struct hmbird_switch_t sw_rec[MAX_SWITCHS]; -+ -+ struct meta_desc_t sw_idx_meta; -+ u64 sw_idx; -+ -+ struct meta_desc_t excep_rec_meta; -+ u64 excep_rec[MAX_EXCEP_ID][MAX_EXCEPS]; -+ -+ struct meta_desc_t excep_idx_meta; -+ u64 excep_idx[MAX_EXCEP_ID]; -+ -+ /* snapshot while panic. */ -+ struct panic_snapshot_t snap; -+}; -+ -+struct ko_info_t {}; -+ -+struct md_meta_t { -+ u64 self_len; -+ u64 unit_size; -+ u64 desc_meta_len; -+ u64 desc_str_len; -+ u64 switches; -+ u64 exceps; -+ u64 global_dsqs; -+ u64 parse_dimens; -+ u64 nr_cpus; -+ u64 real_cpus; -+ u64 nr_meta_desc; -+ u64 dump_real_size; -+}; -+ -+struct md_info_t { -+ struct md_meta_t meta; -+ struct kernel_info_t kern_dump; -+ struct ko_info_t ko_dump; -+}; -+ -+static inline void exceps_update(struct md_info_t *rec, int id, unsigned long jiffies) -+{ -+ u64 *idx; -+ -+ if (!rec) -+ return; -+ -+ idx = &rec->kern_dump.excep_idx[id]; -+ rec->kern_dump.excep_rec[id][*idx] = jiffies; -+ *idx = ++(*idx) % MAX_EXCEPS; -+} -+ -+static inline void sw_update(struct md_info_t *rec, u64 switch_at, -+ u64 is_success, u64 end_state, u64 switch_reason) -+{ -+ u64 *idx; -+ -+ if (!rec) -+ return; -+ -+ idx = &rec->kern_dump.sw_idx; -+ rec->kern_dump.sw_rec[*idx].switch_at = switch_at; -+ rec->kern_dump.sw_rec[*idx].is_success = is_success; -+ rec->kern_dump.sw_rec[*idx].end_state = end_state; -+ rec->kern_dump.sw_rec[*idx].switch_reason = switch_reason; -+ *idx = ++(*idx) % MAX_SWITCHS; -+} -+ - struct hmbird_ops { - bool (*scx_enable)(void); - bool (*check_non_task)(void); - void (*do_sched_yield_before)(long *skip); - void (*window_rollover_run_once)(struct rq *rq); -+ void (*hmbird_get_md_info)(unsigned long *vaddr, unsigned long *size); - }; - - void hmbird_free(struct task_struct *p); -diff --git a/include/linux/sched/hmbird_proc_val.h b/include/linux/sched/hmbird_proc_val.h -new file mode 100755 -index 000000000000..e59708b16ea3 ---- /dev/null -+++ b/include/linux/sched/hmbird_proc_val.h -@@ -0,0 +1,22 @@ -+#ifndef __HMBORD_PROC_VAL_H__ -+#define __HMBORD_PROC_VAL_H__ -+ -+extern int scx_enable; -+extern int partial_enable; -+extern int cpuctrl_high_ratio; -+extern int cpuctrl_low_ratio; -+extern int slim_stats; -+extern int hmbirdcore_debug; -+extern int slim_for_app; -+extern int misfit_ds; -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+extern int cpu7_tl; -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int slim_gov_debug; -+extern int scx_gov_ctrl; -+extern int sched_ravg_window_frame_per_sec; -+ -+#endif -\ No newline at end of file -diff --git a/include/linux/sched/hmbird_version.h b/include/linux/sched/hmbird_version.h -new file mode 100755 -index 000000000000..b2843881c26f ---- /dev/null -+++ b/include/linux/sched/hmbird_version.h -@@ -0,0 +1,52 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#ifndef _OPLUS_HMBIRD_VERSION_H_ -+#define _OPLUS_HMBIRD_VERSION_H_ -+#include -+#include -+#include -+ -+enum hmbird_version { -+ HMBIRD_UNINIT, -+ HMBIRD_GKI_VERSION, -+ HMBIRD_OGKI_VERSION, -+ HMBIRD_UNKNOW_VERSION, -+}; -+ -+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT; -+ -+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type" -+ -+static inline enum hmbird_version get_hmbird_version_type(void) -+{ -+ struct device_node *np = NULL; -+ const char *hmbird_version_str = NULL; -+ if (HMBIRD_UNINIT != hmbird_version_type) -+ return hmbird_version_type; -+ np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH); -+ if (np) { -+ of_property_read_string(np, "type", &hmbird_version_str); -+ if (NULL != hmbird_version_str) { -+ if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) { -+ hmbird_version_type = HMBIRD_OGKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi"); -+ } else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) { -+ hmbird_version_type = HMBIRD_GKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi"); -+ } else { -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi"); -+ } -+ return hmbird_version_type; -+ } -+ } -+ -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION"); -+ return hmbird_version_type; -+} -+ -+#endif /*_OPLUS_HMBIRD_VERSION_H_ */ -diff --git a/include/linux/sched/sa_common.h b/include/linux/sched/sa_common.h -new file mode 100755 -index 000000000000..6f76d7cfd7bf ---- /dev/null -+++ b/include/linux/sched/sa_common.h -@@ -0,0 +1,797 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_COMMON_H_ -+#define _OPLUS_SA_COMMON_H_ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+#include -+#endif -+#include -+#include -+#include -+#include -+#include -+ -+#include "sa_oemdata.h" -+#include "sa_common_struct.h" -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) -+#define OPLUS_UX_EEVDF_COMPATIBLE 1 -+#endif -+ -+#define SA_DEBUG_ON 0 -+ -+#if (SA_DEBUG_ON >= 1) -+#define DEBUG_BUG_ON(x) BUG_ON(x) -+#define DEBUG_WARN_ON(x) WARN_ON(x) -+#else -+#define DEBUG_BUG_ON(x) -+#define DEBUG_WARN_ON(x) -+#endif -+ -+#define ux_err(fmt, ...) \ -+ pr_err("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_warn(fmt, ...) \ -+ pr_warn("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+#define ux_debug(fmt, ...) \ -+ pr_info("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__) -+ -+#define UX_MSG_LEN 64 -+#define UX_DEPTH_MAX 5 -+ -+/* define for debug */ -+#define DEBUG_SYSTRACE (1 << 0) -+#define DEBUG_FTRACE (1 << 1) -+#define DEBUG_KMSG (1 << 2) /* used for frameboost */ -+#define DEBUG_PIPELINE (1 << 3) -+#define DEBUG_DYNAMIC_HZ (1 << 4) -+#define DEBUG_DYNAMIC_PREEMPT (1 << 5) -+#define DEBUG_AMU_INSTRUCTION (1 << 6) -+#define DEBUG_VERBOSE (1 << 10) /* used for frameboost */ -+#define DEBUG_SET_DSQ_ID (1 << 11) /* used for hmbird */ -+ -+/* define for sched assist feature */ -+#define FEATURE_COMMON (1 << 0) -+#define FEATURE_SPREAD (1 << 1) -+ -+#define UX_EXEC_SLICE (4000000U) -+ -+/* define for sched assist thread type, keep same as the define in java file */ -+#define SA_OPT_CLEAR (0) -+#define SA_TYPE_LIGHT (1 << 0) -+#define SA_TYPE_HEAVY (1 << 1) -+#define SA_TYPE_ANIMATOR (1 << 2) -+/* SA_TYPE_LISTPICK for camera */ -+#define SA_TYPE_LISTPICK (1 << 3) -+#define SA_TYPE_MQ_VIP (1 << 4) -+#define SA_OPT_SET (1 << 7) -+#define SA_OPT_RESET (1 << 8) -+#define SA_OPT_SET_PRIORITY (1 << 9) -+ -+/* The following ux value only used in kernel */ -+#define SA_TYPE_SWIFT (1 << 14) -+/* clear ux type when dequeue */ -+#define SA_TYPE_ONCE (1 << 15) -+#define SA_TYPE_INHERIT (1 << 16) -+/* SA_TYPE_COMBINED isn't marked in ux_state, it only exists in return value of function */ -+#define SA_TYPE_COMBINED (1 << 17) -+#define SA_TYPE_URGENT_MASK (SA_TYPE_LIGHT|SA_TYPE_ANIMATOR|SA_TYPE_SWIFT) -+#define SCHED_ASSIST_UX_MASK (SA_TYPE_LIGHT|SA_TYPE_HEAVY|SA_TYPE_ANIMATOR|SA_TYPE_LISTPICK|SA_TYPE_SWIFT) -+ -+/* load balance operation is performed on the following ux types */ -+#define SCHED_ASSIST_LB_UX (SA_TYPE_SWIFT | SA_TYPE_ANIMATOR | SA_TYPE_LIGHT | SA_TYPE_HEAVY) -+#define POSSIBLE_UX_MASK (SA_TYPE_SWIFT | \ -+ SA_TYPE_LIGHT | \ -+ SA_TYPE_HEAVY | \ -+ SA_TYPE_ANIMATOR | \ -+ SA_TYPE_LISTPICK | \ -+ SA_TYPE_ONCE | \ -+ SA_TYPE_INHERIT) -+ -+#define SCHED_ASSIST_UX_PRIORITY_MASK (0xFF000000) -+#define SCHED_ASSIST_UX_PRIORITY_SHIFT 24 -+ -+#define UX_PRIORITY_TOP_APP 0x0A000000 -+#define UX_PRIORITY_AUDIO 0x0A000000 -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+#define UX_PRIORITY_PIPELINE_UI 0x06000000 -+#define UX_PRIORITY_PIPELINE 0x05000000 -+#endif -+ -+/* define for sched assist scene type, keep same as the define in java file */ -+#define SA_SCENE_OPT_CLEAR (0) -+#define SA_LAUNCH (1 << 0) -+#define SA_SLIDE (1 << 1) -+#define SA_CAMERA (1 << 2) -+#define SA_ANIM_START (1 << 3) /* we care about both launcher and top app */ -+#define SA_ANIM (1 << 4) /* we only care about launcher */ -+#define SA_INPUT (1 << 5) -+#define SA_LAUNCHER_SI (1 << 6) -+#define SA_SCENE_OPT_SET (1 << 7) -+ -+#define ROOT_UID 0 -+#define SYSTEM_UID 1000 -+#define FIRST_APPLICATION_UID 10000 -+#define LAST_APPLICATION_UID 19999 -+#define PER_USER_RANGE 100000 -+/* define for clear IM_FLAG -+if im_flag is 70, it should clear im_flag_audio (70 - 64 = 6) -+eg: gerrit patchset "30438485" -+ */ -+#define IM_FLAG_CLEAR 64 -+ -+extern pid_t save_audio_tgid; -+extern pid_t save_top_app_tgid; -+extern unsigned int top_app_type; -+extern int global_lowend_plat_opt; -+ -+#ifdef CONFIG_OPLUS_SCHED_HALT_MASK_PRT -+/* This must be the same as the definition of pause_type in walt_halt.c */ -+enum oplus_pause_type { -+ OPLUS_HALT, -+ OPLUS_PARTIAL_HALT, -+ -+ OPLUS_MAX_PAUSE_TYPE -+}; -+extern cpumask_t cur_cpus_halt_mask; -+extern cpumask_t cur_cpus_phalt_mask; -+DECLARE_PER_CPU(int[OPLUS_MAX_PAUSE_TYPE], oplus_cur_pause_client); -+extern void sa_corectl_systrace_c(void); -+#endif /* CONFIG_OPLUS_SCHED_HALT_MASK_PRT */ -+ -+/* define for boost threshold unit */ -+#define BOOST_THRESHOLD_UNIT (51) -+ -+ -+enum UX_STATE_TYPE { -+ UX_STATE_INVALID = 0, -+ UX_STATE_NONE, -+ UX_STATE_STATIC, -+ UX_STATE_INHERIT, -+ UX_STATE_COMBINED, -+ MAX_UX_STATE_TYPE, -+}; -+ -+enum INHERIT_UX_TYPE { -+ INHERIT_UX_BINDER = 0, -+ INHERIT_UX_RWSEM, -+ INHERIT_UX_MUTEX, -+ INHERIT_UX_FUTEX, -+ INHERIT_UX_PIFUTEX, -+ INHERIT_UX_MAX, -+}; -+ -+/* -+ * WANNING: -+ * new flag should be add before MAX_IM_FLAG_TYPE, never change -+ * the value of those existed flag type. -+ */ -+enum IM_FLAG_TYPE { -+ IM_FLAG_NONE = 0, -+ IM_FLAG_SURFACEFLINGER, -+ IM_FLAG_HWC, /* Discarded */ -+ IM_FLAG_RENDERENGINE, -+ IM_FLAG_WEBVIEW, -+ IM_FLAG_CAMERA_HAL, -+ IM_FLAG_AUDIO, -+ IM_FLAG_HWBINDER, -+ IM_FLAG_LAUNCHER, -+ IM_FLAG_LAUNCHER_NON_UX_RENDER, -+ IM_FLAG_SS_LOCK_OWNER, -+ IM_FLAG_FORBID_SET_CPU_AFFINITY = 11, /* forbid setting cpu affinity from app */ -+ IM_FLAG_SYSTEMSERVER_PID, -+ IM_FLAG_MIDASD, -+ IM_FLAG_AUDIO_CAMERA_HAL, /* audio mode disable camera hal ux */ -+ IM_FLAG_AFFINITY_THREAD, -+ IM_FLAG_TPD_SET_CPU_AFFINITY = 16, -+ IM_FLAG_COMPRESS_THREAD = 17, /* compress thread skips locking protect */ -+ IM_FLAG_RENDER_THREAD = 18, -+ MAX_IM_FLAG_TYPE, -+}; -+ -+#define MAX_IM_FLAG_PRIO MAX_IM_FLAG_TYPE -+ -+enum ots_state { -+ OTS_STATE_SET_AFFINITY, -+ OTS_STATE_DDL_ACTIVE, -+ OTS_STATE_DDL_ACTIVE_PREEMPTED, -+ OTS_STATE_MAX, -+}; -+ -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+DECLARE_PER_CPU(u64, retired_instrs); -+DECLARE_PER_CPU(u64, nvcsw); -+DECLARE_PER_CPU(u64, nivcsw); -+#endif -+ -+struct ux_sched_cluster { -+ struct cpumask cpus; -+ unsigned long capacity; -+}; -+ -+#define OPLUS_NR_CPUS (8) -+#define OPLUS_MAX_CLS (5) -+struct ux_sched_cputopo { -+ int cls_nr; -+ struct ux_sched_cluster sched_cls[OPLUS_NR_CPUS]; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ cpumask_t oplus_cpu_array[2*OPLUS_MAX_CLS][OPLUS_MAX_CLS]; -+#endif -+}; -+ -+struct oplus_rq { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_root_cached ux_list; -+ /* a tree to track minimum exec vruntime */ -+ struct rb_root_cached exec_timeline; -+ /* malloc this spinlock_t instead of built-in to shrank size of oplus_rq */ -+ spinlock_t *ux_list_lock; -+ int nr_running; -+ u64 min_vruntime; -+ u64 load_weight; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) -+ struct rb_root_cached ddl_root; -+ spinlock_t *ddl_lock; -+#endif -+ -+#ifdef CONFIG_LOCKING_PROTECT -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ struct list_head locking_thread_list; -+ spinlock_t *locking_list_lock; -+ int rq_locking_task; -+#else -+ struct sched_entity *last_entity; -+#endif -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ struct oplus_lb lb; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+ struct hrtimer *resched_timer; -+ int cpu; -+#endif -+}; -+ -+extern int global_debug_enabled; -+extern int global_sched_assist_enabled; -+extern int global_sched_assist_scene; -+extern int global_silver_perf_core; -+extern int global_sched_group_enabled; -+ -+struct rq; -+ -+#ifdef CONFIG_LOCKING_PROTECT -+struct sched_assist_locking_ops { -+ void (*check_preempt_tick)(struct task_struct *p, -+ unsigned long *ideal_runtime, bool *skip_preempt, -+ unsigned long delta_exec, struct cfs_rq *cfs_rq, -+ struct sched_entity *curr, unsigned int granularity); -+ void (*check_preempt_wakeup)(struct rq *rq, struct task_struct *p, bool *preempt, bool *nopreempt); -+ void (*state_systrace_c)(unsigned int cpu, struct task_struct *p); -+ void (*locking_tick_hit)(struct task_struct *prev, struct task_struct *next); -+#ifndef CONFIG_LOCKING_LAST_ENTITY -+ void (*replace_next_task_fair)(struct rq *rq, -+ struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*enqueue_entity)(struct rq *rq, struct task_struct *p); -+ void (*dequeue_entity)(struct rq *rq, struct task_struct *p); -+#else -+ void (*set_last_entity)(struct task_struct *prev, struct task_struct *next, struct rq *rq); -+ void (*pick_last_entity)(struct rq *rq, struct task_struct **p, struct sched_entity **se, bool *repick, bool simple); -+ void (*clear_last_entity)(struct rq *rq, struct task_struct *p); -+#endif -+ void (*opt_ss_lock_contention)(struct task_struct *p, int old_im, int new_im); -+}; -+ -+extern struct sched_assist_locking_ops *locking_ops; -+ -+ -+#define LOCKING_CALL_OP(op, args...) \ -+do { \ -+ if (locking_ops && locking_ops->op) { \ -+ locking_ops->op(args); \ -+ } \ -+} while (0) -+ -+#define LOCKING_CALL_OP_RET(op, args...) \ -+({ \ -+ __typeof__(locking_ops->op(args)) __ret = 0; \ -+ if (locking_ops && locking_ops->op) \ -+ __ret = locking_ops->op(args); \ -+ __ret; \ -+}) -+ -+void register_sched_assist_locking_ops(struct sched_assist_locking_ops *ops); -+#endif -+ -+/* attention: before insert .ko, task's list->prev/next is init with 0 */ -+static inline bool oplus_list_empty(struct list_head *list) -+{ -+ return list_empty(list) || (list->prev == 0 && list->next == 0); -+} -+ -+static inline bool oplus_rbtree_empty(struct rb_root_cached *list) -+{ -+ return (rb_first_cached(list) == NULL); -+} -+ -+/** -+ * Check if there are ux tasks waiting to run on the specified cpu -+ */ -+static inline bool orq_has_ux_tasks(struct oplus_rq *orq) -+{ -+ return !oplus_rbtree_empty(&orq->ux_list); -+} -+ -+/* attention: before insert .ko, task's node->__rb_parent_color is init with 0 */ -+static inline bool oplus_rbnode_empty(struct rb_node *node) -+{ -+ return RB_EMPTY_NODE(node) || (node->__rb_parent_color == 0); -+} -+ -+static inline struct oplus_task_struct *ux_list_first_entry(struct rb_root_cached *list) -+{ -+ struct rb_node *leftmost = rb_first_cached(list); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, ux_entry); -+} -+ -+static inline struct oplus_task_struct *exec_timeline_first_entry(struct rb_root_cached *tree) -+{ -+ struct rb_node *leftmost = rb_first_cached(tree); -+ if (leftmost == NULL) { -+ return NULL; -+ } -+ return rb_entry(leftmost, struct oplus_task_struct, exec_time_node); -+} -+ -+typedef bool (*migrate_task_callback_t)(struct task_struct *tsk, int src_cpu, int dst_cpu); -+extern migrate_task_callback_t fbg_migrate_task_callback; -+typedef void (*android_rvh_schedule_handler_t)(struct task_struct *prev, -+ struct task_struct *next, struct rq *rq); -+extern android_rvh_schedule_handler_t fbg_android_rvh_schedule_callback; -+ -+extern struct kmem_cache *oplus_task_struct_cachep; -+ -+#define ots_to_ts(ots) (ots->task) -+#define OTS_IDX 0 -+#define ORQ_IDX 0 -+ -+static inline bool test_task_is_fair(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid CFS priority is MAX_RT_PRIO..MAX_PRIO-1 */ -+ if ((task->prio >= MAX_RT_PRIO) && (task->prio <= MAX_PRIO-1)) -+ return true; -+ return false; -+} -+ -+static inline bool test_task_is_rt(struct task_struct *task) -+{ -+ DEBUG_BUG_ON(!task); -+ -+ /* valid RT priority is 0..MAX_RT_PRIO-1 */ -+ if (rt_prio(task->prio)) -+ return true; -+ -+ return false; -+} -+ -+static inline struct oplus_task_struct *get_oplus_task_struct(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = NULL; -+ -+ /* not Skip idle thread */ -+ if (!t) -+ return NULL; -+ -+ ots = (struct oplus_task_struct *) READ_ONCE(t->android_oem_data1[OTS_IDX]); -+ if (IS_ERR_OR_NULL(ots)) -+ return NULL; -+ -+ return ots; -+} -+ -+static inline unsigned long oplus_get_im_flag(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return IM_FLAG_NONE; -+ -+ return ots->im_flag; -+} -+ -+static inline bool is_optimized_audio_thread(struct task_struct *t) -+{ -+ unsigned long im_flag; -+ -+ im_flag = oplus_get_im_flag(t); -+ if (test_bit(IM_FLAG_AUDIO, &im_flag)) -+ return true; -+ -+ return false; -+} -+ -+static inline void oplus_set_im_flag(struct task_struct *t, int im_flag) -+{ -+ struct oplus_task_struct *ots = NULL; -+ ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ set_bit(im_flag, &ots->im_flag); -+} -+ -+static inline int get_ots_ux_state(struct oplus_task_struct *ots) -+{ -+ int ux_state; -+ int sub_ux_state; -+ int ux_prio; -+ int ret; -+ -+ ux_prio = ots->ux_state & SCHED_ASSIST_UX_PRIORITY_MASK; -+ ux_state = ots->ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ sub_ux_state = ots->sub_ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT); -+ -+ if (sub_ux_state) { -+ ret = ux_state | (sub_ux_state & ~SA_TYPE_INHERIT) | SA_TYPE_COMBINED; -+ } else { -+ ret = ux_state; -+ } -+ -+ if (ret & SCHED_ASSIST_UX_MASK) { -+ return ux_prio | ret; -+ } -+ -+ return 0; -+} -+ -+bool is_multiple_ux(struct oplus_task_struct *ots); -+ -+static inline int oplus_get_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return get_ots_ux_state(ots); -+} -+ -+static inline int oplus_get_static_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return 0; -+ } -+ return ots->ux_state; -+} -+ -+static inline int oplus_get_sub_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->sub_ux_state; -+} -+ -+static inline int oplus_get_inherited_ux_state(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ if (ots->ux_state & SA_TYPE_INHERIT) { -+ return ots->ux_state; -+ } -+ return ots->sub_ux_state; -+} -+ -+void oplus_set_ux_state_lock(struct task_struct *t, int ux_state, int inherit_type, bool need_lock_rq); -+ -+static inline s64 oplus_get_inherit_ux(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return atomic64_read(&ots->inherit_ux); -+} -+ -+static inline void oplus_set_inherit_ux(struct task_struct *t, s64 inherit_ux) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ atomic64_set(&ots->inherit_ux, inherit_ux); -+} -+ -+static inline int oplus_get_ux_depth(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->ux_depth; -+} -+ -+static inline void oplus_set_ux_depth(struct task_struct *t, int ux_depth) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->ux_depth = ux_depth; -+} -+ -+static inline u64 oplus_get_enqueue_time(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return 0; -+ -+ return ots->enqueue_time; -+} -+ -+static inline void oplus_set_enqueue_time(struct task_struct *t, u64 enqueue_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->enqueue_time = enqueue_time; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* -+ * Record the number of task context switches -+ * and scheduling delays when enqueueing a task. -+ */ -+ ots->snap_pcount = t->sched_info.pcount; -+ ots->snap_run_delay = t->sched_info.run_delay; -+#endif -+} -+ -+static inline void oplus_set_inherit_ux_start(struct task_struct *t, u64 start_time) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ ots->inherit_ux_start = t->se.sum_exec_runtime; -+} -+ -+static inline void init_task_ux_info(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return; -+ -+ RB_CLEAR_NODE(&ots->ux_entry); -+ RB_CLEAR_NODE(&ots->exec_time_node); -+ ots->ux_state = 0; -+ ots->sub_ux_state = 0; -+ atomic64_set(&ots->inherit_ux, 0); -+ ots->ux_depth = 0; -+ ots->enqueue_time = 0; -+ ots->inherit_ux_start = 0; -+ ots->ux_priority = -1; -+ ots->ux_nice = -1; -+ ots->vruntime = 0; -+ ots->preset_vruntime = 0; -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG) -+ ots->abnormal_flag = 0; -+#endif -+#ifdef CONFIG_OPLUS_FEATURE_SCHED_SPREAD -+ ots->lb_state = 0; -+ ots->ld_flag = 0; -+#endif -+ ots->exec_calc_runtime = 0; -+ ots->is_update_runtime = 0; -+ ots->target_process = -1; -+ ots->wake_tid = 0; -+ ots->running_start_time = 0; -+ ots->update_running_start_time = false; -+ ots->last_wake_ts = 0; -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ memset(&ots->lkinfo, 0, sizeof(struct locking_info)); -+ INIT_LIST_HEAD(&ots->lkinfo.node); -+/*#endif*/ -+ ots->block_start_time = 0; -+#ifdef CONFIG_LOCKING_PROTECT -+ INIT_LIST_HEAD(&ots->locking_entry); -+ ots->locking_start_time = 0; -+ ots->locking_depth = 0; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+ /* for loadbalance */ -+ ots->snap_pcount = 0; -+ ots->snap_run_delay = 0; -+ plist_node_init(&ots->rtb, MAX_IM_FLAG_PRIO); -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE) -+ atomic_set(&ots->pipeline_cpu, -1); -+#endif -+ -+#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO) -+ ots->amu_cycle = 0; -+ ots->amu_instruct = 0; -+#endif -+}; -+ -+static inline bool test_ux_type(struct task_struct *task, int ux_type) -+{ -+ return oplus_get_ux_state(task) & ux_type; -+} -+ -+static inline bool is_heavy_ux_task(struct task_struct *t) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(t); -+ -+ if (IS_ERR_OR_NULL(ots)) -+ return false; -+ -+ return get_ots_ux_state(ots) & SA_TYPE_HEAVY; -+} -+ -+static inline bool sched_assist_scene(unsigned int scene) -+{ -+ if (unlikely(!global_sched_assist_enabled)) -+ return false; -+ -+ return global_sched_assist_scene & scene; -+} -+ -+static inline unsigned long oplus_task_util(struct task_struct *p) -+{ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+ struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; -+ -+ return wts->demand_scaled; -+#else -+ return READ_ONCE(p->se.avg.util_avg); -+#endif -+} -+ -+#if IS_ENABLED(CONFIG_SCHED_WALT) -+static inline u32 task_wts_sum(struct task_struct *tsk) -+{ -+ struct walt_task_struct *wts = (struct walt_task_struct *) tsk->android_vendor_data1; -+ -+ return wts->sum; -+} -+#endif -+ -+bool is_min_cluster(int cpu); -+bool is_max_cluster(int cpu); -+bool is_mid_cluster(int cpu); -+bool im_mali(const char *comm); -+bool is_top(struct task_struct *p); -+bool task_is_runnable(struct task_struct *task); -+struct oplus_rq *get_oplus_rq(struct rq *rq); -+ -+typedef unsigned long (*kallsyms_lookup_name_t)(const char *name); -+extern kallsyms_lookup_name_t _kallsyms_lookup_name; -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE) -+int ux_mask_to_prio(int ux_mask); -+int ux_prio_to_mask(int prio); -+#endif -+ -+noinline int tracing_mark_write(const char *buf); -+void hwbinder_systrace_c(unsigned int cpu, int flag); -+void sched_assist_init_oplus_rq(void); -+void queue_ux_thread(struct rq *rq, struct task_struct *p, int enqueue); -+ -+void inherit_ux_inc(struct task_struct *task, int type); -+void inherit_ux_sub(struct task_struct *task, int type, int value); -+void set_inherit_ux(struct task_struct *task, int type, int depth, int inherit_val); -+void reset_inherit_ux(struct task_struct *inherit_task, struct task_struct *ux_task, int reset_type); -+void unset_inherit_ux(struct task_struct *task, int type); -+void unset_inherit_ux_value(struct task_struct *task, int type, int value); -+void inc_inherit_ux_refs(struct task_struct *task, int type); -+void clear_all_inherit_type(struct task_struct *p); -+int get_max_inherit_gran(struct task_struct *p); -+ -+bool is_heavy_load_top_task(struct task_struct *p); -+bool test_task_is_fair(struct task_struct *task); -+bool test_task_is_rt(struct task_struct *task); -+ -+bool test_task_ux(struct task_struct *task); -+bool test_task_ux_depth(int ux_depth); -+bool test_inherit_ux(struct task_struct *task, int type); -+bool test_set_inherit_ux(struct task_struct *task); -+int get_ux_state_type(struct task_struct *task); -+void sched_assist_target_comm(struct task_struct *task, const char *comm); -+unsigned int ux_task_exec_limit(struct task_struct *p); -+ -+void update_ux_sched_cputopo(void); -+bool is_task_util_over(struct task_struct *tsk, int threshold); -+bool oplus_task_misfit(struct task_struct *tsk, int cpu); -+ssize_t oplus_show_cpus(const struct cpumask *mask, char *buf); -+void adjust_rt_lowest_mask(struct task_struct *p, struct cpumask *local_cpu_mask, int ret, bool force_adjust); -+bool sa_skip_rt_sync(struct rq *rq, struct task_struct *p, bool *sync); -+void ux_state_systrace_c(unsigned int cpu, struct task_struct *p); -+bool sa_rt_skip_ux_cpu(int cpu); -+int is_vip_mvp(struct task_struct *p); -+ -+/* s64 account_ux_runtime(struct rq *rq, struct task_struct *curr); */ -+void opt_ss_lock_contention(struct task_struct *p, unsigned long old_im, int new_im); -+ -+/* register vender hook in kernel/sched/topology.c */ -+void android_vh_build_sched_domains_handler(void *unused, bool has_asym); -+ -+/* register vender hook in kernel/sched/rt.c */ -+void android_rvh_select_task_rq_rt_handler(void *unused, struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags, int *new_cpu); -+void android_rvh_find_lowest_rq_handler(void *unused, struct task_struct *p, struct cpumask *local_cpu_mask, int ret, int *best_cpu); -+ -+/* register vender hook in kernel/sched/core.c */ -+void android_rvh_sched_fork_handler(void *unused, struct task_struct *p); -+void android_rvh_after_enqueue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_dequeue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags); -+void android_rvh_schedule_handler(void *unused, struct task_struct *prev, struct task_struct *next, struct rq *rq); -+void android_vh_scheduler_tick_handler(void *unused, struct rq *rq); -+ -+void android_vh_account_process_tick_gran_handler(void *unused, struct task_struct *p, struct rq *rq, int user_tick, int *ticks); -+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN -+void sa_sched_switch_handler(void *unused, bool preempt, struct task_struct *prev, struct task_struct *next, unsigned int prev_state); -+#endif -+ -+void set_im_flag_with_bit(int im_flag, struct task_struct *task); -+ -+/* register vendor hook in kernel/cgroup/cgroup-v1.c */ -+void android_vh_cgroup_set_task_handler(void *unused, int ret, struct task_struct *task); -+/* register vendor hook in kernel/signal.c */ -+void android_vh_exit_signal_handler(void *unused, struct task_struct *p); -+void android_rvh_set_cpus_allowed_by_task_handler(void *unused, const struct cpumask *cpu_valid_mask, -+ const struct cpumask *new_mask, struct task_struct *task, unsigned int *dest_cpu); -+void android_rvh_setscheduler_handler(void *unused, struct task_struct *p); -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_BAN_APP_SET_AFFINITY) -+void android_vh_sched_setaffinity_early_handler(void *unused, struct task_struct *task, const struct cpumask *new_mask, bool *skip); -+#endif -+ -+#ifdef CONFIG_OPLUS_SCHED_GROUP_OPT -+void android_vh_reweight_entity_handler(void *unused, struct sched_entity *se); -+#endif -+ -+#ifdef CONFIG_BLOCKIO_UX_OPT -+int sa_blockio_init(void); -+void sa_blockio_exit(void); -+#endif -+extern struct notifier_block process_exit_notifier_block; -+#endif /* _OPLUS_SA_COMMON_H_ */ -diff --git a/include/linux/sched/sa_common_struct.h b/include/linux/sched/sa_common_struct.h -new file mode 100755 -index 000000000000..876feff48b36 ---- /dev/null -+++ b/include/linux/sched/sa_common_struct.h -@@ -0,0 +1,277 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2020-2022 Oplus. All rights reserved. -+ */ -+ -+/* -+this file is splited from the sa_common.h to adapt the OKI, -+IS_ENABLED is not allowed in here, because the macro will not work in OKI. -+*/ -+#ifndef _OPLUS_SA_COMMON_STRUCT_H_ -+#define _OPLUS_SA_COMMON_STRUCT_H_ -+ -+#define MAX_CLUSTER (4) -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+/* hot-thread */ -+struct task_record { -+#define RECOED_WINSIZE (1 << 8) -+#define RECOED_WINIDX_MASK (RECOED_WINSIZE - 1) -+ u8 winidx; -+ u8 count; -+}; -+ -+#define MAX_TASK_COMM_LEN 256 -+struct uid_struct { -+ uid_t uid; -+ u64 uid_total_cycle; -+ u64 uid_total_inst; -+ spinlock_t lock; -+ char leader_comm[TASK_COMM_LEN]; -+ char cmdline[MAX_TASK_COMM_LEN]; -+}; -+ -+struct amu_uid_entry { -+ uid_t uid; -+ struct uid_struct *uid_struct; -+ struct hlist_node node; -+}; -+ -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+struct locking_info { -+ u64 waittime_stamp; -+ u64 holdtime_stamp; -+ /* Used in torture acquire latency statistic.*/ -+ u64 acquire_stamp; -+ /* -+ * mutex or rwsem optimistic spin start time. Because a task -+ * can't spin both on mutex and rwsem at one time, use one common -+ * threshold time is OK. -+ */ -+ u64 opt_spin_start_time; -+ struct task_struct *holder; -+ u32 waittype; -+ bool ux_contrib; -+ /* -+ * Whether task is ux when it's going to be added to mutex or -+ * rwsem waiter list. It helps us check whether there is ux -+ * task on mutex or rwsem waiter list. Also, a task can't be -+ * added to both mutex and rwsem at one time, so use one common -+ * field is OK. -+ */ -+ bool is_block_ux; -+ u32 kill_flag; -+ /* for cfs enqueue smoothly.*/ -+ struct list_head node; -+ struct task_struct *owner; -+ struct list_head lock_head; -+ u64 clear_seq; -+ atomic_t lock_depth; -+}; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+#define RAVG_HIST_SIZE 5 -+#define SCX_SLICE_DFL (1 * NSEC_PER_MSEC) -+#define SCX_SLICE_INF U64_MAX -+#define DEFAULT_CGROUP_DL_IDX (8) -+#define EXT_FLAG_RT_CHANGED (1 << 0) -+#define EXT_FLAG_CFS_CHANGED (1 << 1) -+struct scx_task_stats { -+ u64 mark_start; -+ u64 window_start; -+ u32 sum; -+ u32 sum_history[RAVG_HIST_SIZE]; -+ int cidx; -+ u32 demand; -+ u16 demand_scaled; -+ void *sdsq; -+}; -+/* -+ * The following is embedded in task_struct and contains all fields necessary -+ * for a task to be scheduled by SCX. -+ */ -+struct scx_entity { -+ struct scx_dispatch_q *dsq; -+ struct { -+ struct list_head fifo; /* dispatch order */ -+ struct rb_node priq; /* p->scx.dsq_vtime order */ -+ } dsq_node; -+ u32 flags; /* protected by rq lock */ -+ u32 dsq_flags; /* protected by dsq lock */ -+ s32 sticky_cpu; -+ unsigned long runnable_at; -+ u64 slice; -+ u64 dsq_vtime; -+ int gdsq_idx; -+ int ext_flags; -+ int prio_backup; -+ unsigned long sched_prop; -+ struct scx_task_stats sts; -+}; -+/*#endif*/ -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ -+#define MAX_CPU_FREQ_STATE 32 -+#define MAX_CPU_CNT 8 -+ -+#define STATE_IN_POOL 0 -+#define STATE_ACTIVE 1 -+ -+struct powermodel_freq_task_state { -+ u8 powermodel_enqueued:1; -+ u8 powermodel_index:7; -+}; -+ -+struct powermodel_cpu_task_state { -+ struct oplus_task_struct *ots; -+ struct powermodel_freq_task_state powermodel_freq_task_states[MAX_CPU_FREQ_STATE]; -+ u64 powermodel_last_seq; -+ u64 last_seq; -+ struct list_head node; -+ int state; -+ int pool_id; -+}; -+ -+#endif -+ -+ -+/* Please add your own members of task_struct here :) */ -+struct oplus_task_struct { -+ /* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */ -+ struct rb_node ux_entry; -+ struct rb_node exec_time_node; -+ struct task_struct *task; -+ atomic64_t inherit_ux; -+ u64 enqueue_time; -+ u64 inherit_ux_start; -+ /* u64 sum_exec_baseline; */ -+ u64 total_exec; -+ u64 vruntime; -+ u64 preset_vruntime; -+ /* contains ux state -+ 1. if static and inherited ux both exist, static ux stores in ux_state, inherited ux in sub_ux_state. -+ 2. if only static ux exists, static ux stores in ux_state. -+ 2. if only inherited ux exists, inherited ux stores in ux_state */ -+ int ux_state; -+ int sub_ux_state; -+ u8 ux_depth; -+ s8 ux_priority; -+ s8 ux_nice; -+ pid_t affinity_pid; -+ pid_t affinity_tgid; -+ unsigned long state; -+ unsigned long im_flag; -+ atomic_t is_vip_mvp; -+ -+/* #if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) */ -+ u64 ddl; -+ u64 ddl_active_ts; -+ u64 runnable_ts; -+ struct rb_node ddl_node; -+/* #endif */ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG)*/ -+ int abnormal_flag; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_SCHED_SPREAD */ -+ int lb_state; -+ int ld_flag:1; -+ /* CONFIG_OPLUS_FEATURE_TASK_LOAD */ -+ int is_update_runtime:1; -+ int target_process; -+ u64 wake_tid; -+ u64 running_start_time; -+ bool update_running_start_time; -+ u64 exec_calc_runtime; -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct task_record record[MAX_CLUSTER]; /* 2*u64 */ -+ u64 block_start_time; -+/*#endif*/ -+ /* CONFIG_OPLUS_FEATURE_FRAME_BOOST */ -+ struct list_head fbg_list; -+ raw_spinlock_t fbg_list_entry_lock; -+ bool fbg_running; /* task belongs to a group, and in running */ -+ u16 fbg_state; -+ s8 preferred_cluster_id; -+ s8 fbg_depth; -+ u64 last_wake_ts; -+ int fbg_cur_group; -+/*#ifdef CONFIG_LOCKING_PROTECT*/ -+ unsigned long locking_start_time; -+ unsigned long last_jiffies; -+ struct list_head locking_entry; -+ int locking_depth; -+ int lk_tick_hit; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/ -+ struct locking_info lkinfo; -+/*#endif*/ -+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/ -+ struct scx_entity scx; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_FDLEAK_CHECK)*/ -+ u8 fdleak_flag; -+/*#endif*/ -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+ /* for loadbalance */ -+ struct plist_node rtb; /* rt boost task */ -+ -+ /* -+ * The following variables are used to calculate the time -+ * a task spends in the running/runnable state. -+ */ -+ u64 snap_run_delay; -+ unsigned long snap_pcount; -+/*#endif*/ -+#if IS_ENABLED(CONFIG_OPLUS_SCHED_TUNE) -+ int stune_idx; -+#endif -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE)*/ -+ atomic_t pipeline_cpu; -+/*#endif*/ -+ -+ /* for oplus secure guard */ -+ int sg_flag; -+ int sg_scno; -+ uid_t sg_uid; -+ uid_t sg_euid; -+ gid_t sg_gid; -+ gid_t sg_egid; -+/*#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/ -+ struct uid_struct *uid_struct; -+ u64 amu_instruct; -+ u64 amu_cycle; -+/*#endif*/ -+ /* for binder ux */ -+ int binder_async_ux_enable; -+ bool binder_async_ux_sts; -+ int binder_thread_mode; -+ struct binder_node *binder_thread_node; -+ -+/* for powermodel */ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL) -+ struct powermodel_cpu_task_state *powermodel_cpu_task_states[MAX_CPU_CNT]; -+ u64 exec_runtime; -+#endif -+ -+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_CFBT) -+ int cfbt_cur_group; -+ bool cfbt_running; -+#endif /* CONFIG_OPLUS_FEATURE_SCHED_CFBT */ -+} ____cacheline_aligned; -+ -+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/ -+#define INVALID_PID (-1) -+struct oplus_lb { -+ /* used for active_balance to record the running task. */ -+ pid_t pid; -+}; -+/*#endif*/ -+ -+#endif /* _OPLUS_SA_COMMON_STRUCT_H_ */ -+ -diff --git a/include/linux/sched/sa_oemdata.h b/include/linux/sched/sa_oemdata.h -new file mode 100755 -index 000000000000..ebbbc754e391 ---- /dev/null -+++ b/include/linux/sched/sa_oemdata.h -@@ -0,0 +1,13 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2022 Oplus. All rights reserved. -+ */ -+ -+ -+#ifndef _OPLUS_SA_OEMDATA_H_ -+#define _OPLUS_SA_OEMDATA_H_ -+ -+int sa_oemdata_init(void); -+void sa_oemdata_deinit(void); -+ -+#endif /* _OPLUS_SA_OEMDATA_H_ */ -diff --git a/include/linux/sched/sched_ext.h b/include/linux/sched/sched_ext.h -new file mode 100755 -index 000000000000..9f1afdb8a04b ---- /dev/null -+++ b/include/linux/sched/sched_ext.h -@@ -0,0 +1,57 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef _OPLUS_SCHED_EXT_H -+#define _OPLUS_SCHED_EXT_H -+#include "sa_common.h" -+ -+#define SCHED_PROP_TOP_THREAD_SHIFT (8) -+#define SCHED_PROP_TOP_THREAD_MASK (0xf << SCHED_PROP_TOP_THREAD_SHIFT) -+#define SCHED_PROP_DEADLINE_MASK (0xFF) /* deadline for ext sched class */ -+#define SCHED_PROP_DEADLINE_LEVEL1 (1) /* 1ms for user-aware audio tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL2 (2) /* 2ms for user-aware touch tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL3 (3) /* 4ms for user aware dispaly tasks */ -+#define SCHED_PROP_DEADLINE_LEVEL4 (4) /* 6ms */ -+#define SCHED_PROP_DEADLINE_LEVEL5 (5) /* 8ms */ -+#define SCHED_PROP_DEADLINE_LEVEL6 (6) /* 16ms */ -+#define SCHED_PROP_DEADLINE_LEVEL7 (7) /* 32ms */ -+#define SCHED_PROP_DEADLINE_LEVEL8 (8) /* 64ms */ -+#define SCHED_PROP_DEADLINE_LEVEL9 (9) /* 128ms */ -+ -+static inline int sched_prop_get_top_thread_id(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ return -EPERM; -+ } -+ -+ return ((ots->scx.sched_prop & SCHED_PROP_TOP_THREAD_MASK) >> SCHED_PROP_TOP_THREAD_SHIFT); -+} -+ -+static inline int sched_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_set_sched_prop failed! fn=%s\n", __func__); -+ return -EPERM; -+ } -+ -+ ots->scx.sched_prop = sp; -+ return 0; -+} -+ -+static inline unsigned long sched_get_sched_prop(struct task_struct *p) -+{ -+ struct oplus_task_struct *ots = get_oplus_task_struct(p); -+ -+ if (!ots) { -+ pr_err("scx_sched_ext: sched_get_sched_prop failed! fn=%s\n", __func__); -+ return (unsigned long)-1; -+ } -+ return ots->scx.sched_prop; -+} -+ -+#endif /*_OPLUS_SCHED_EXT_H */ -diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h -index 03d35e3eda3c..6a5f0c0d74bd 100644 ---- a/include/linux/sched/task.h -+++ b/include/linux/sched/task.h -@@ -61,8 +61,10 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); - extern void init_idle(struct task_struct *idle, int cpu); - - extern int sched_fork(unsigned long clone_flags, struct task_struct *p); --extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); --extern void sched_cancel_fork(struct task_struct *p); -+extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); -+#ifdef CONFIG_HMBIRD_SCHED -+extern void hmbird_cancel_fork(struct task_struct *p); -+#endif - extern void sched_post_fork(struct task_struct *p); - extern void sched_dead(struct task_struct *p); - -diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h -index 359a14cc76a4..969716ab4320 100755 ---- a/include/uapi/linux/sched.h -+++ b/include/uapi/linux/sched.h -@@ -118,7 +118,7 @@ struct clone_args { - /* SCHED_ISO: reserved but not implemented yet */ - #define SCHED_IDLE 5 - #define SCHED_DEADLINE 6 --#define SCHED_EXT 7 -+#define SCHED_HMBIRD 7 - - /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ - #define SCHED_RESET_ON_FORK 0x40000000 -diff --git a/init/Kconfig b/init/Kconfig -index 48c86ee02f26..f0e97bc54978 100644 ---- a/init/Kconfig -+++ b/init/Kconfig -@@ -1021,10 +1021,6 @@ config RT_GROUP_SCHED - realtime bandwidth for them. - See Documentation/scheduler/sched-rt-group.rst for more information. - --config EXT_GROUP_SCHED -- bool -- depends on SCHED_CLASS_EXT && CGROUP_SCHED -- default y - endif #CGROUP_SCHED - - config SCHED_MM_CID -diff --git a/init/init_task.c b/init/init_task.c -index 69a046388995..31ceb0e469f7 100755 ---- a/init/init_task.c -+++ b/init/init_task.c -@@ -6,7 +6,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -102,9 +101,6 @@ struct task_struct init_task - #endif - #ifdef CONFIG_CGROUP_SCHED - .sched_task_group = &root_task_group, --#endif --#ifdef CONFIG_SLIM_SCHED -- .scx = NULL, - #endif - .ptraced = LIST_HEAD_INIT(init_task.ptraced), - .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), -diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt -index cf22ef6107b8..b131d5662b48 100755 ---- a/kernel/Kconfig.preempt -+++ b/kernel/Kconfig.preempt -@@ -133,12 +133,8 @@ config SCHED_CORE - which is the likely usage by Linux distributions, there should - be no measurable impact on performance. - --config SLIM_SCHED -- bool "slim sched" -- default n --config SCHED_CLASS_EXT -- bool "Extensible Scheduling Class" -- depends on BPF_SYSCALL && BPF_JIT -+config HMBIRD_SCHED -+ bool "hmbird Scheduling Class(base on ext scheduler)" - help - This option enables a new scheduler class sched_ext (SCX), which - allows scheduling policies to be implemented as BPF programs to -@@ -159,3 +155,10 @@ config SCHED_CLASS_EXT - similar to struct sched_class. - - See Documentation/scheduler/sched-ext.rst for more details. -+ -+config DYNAMIC_WALT_SUPPORT -+ bool "Dynamic WALT Support for Extensible Scheduling Class" -+ depends on HMBIRD_SCHED -+ default n -+ help -+ Enable dynamic WALT support for Extensible Scheduling Class. -diff --git a/kernel/bpf/bpf_struct_ops_types.h b/kernel/bpf/bpf_struct_ops_types.h -index 3618769d853d..5678a9ddf817 100644 ---- a/kernel/bpf/bpf_struct_ops_types.h -+++ b/kernel/bpf/bpf_struct_ops_types.h -@@ -9,8 +9,4 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) - #include - BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) - #endif --#ifdef CONFIG_SCHED_CLASS_EXT --#include --BPF_STRUCT_OPS_TYPE(sched_ext_ops) --#endif - #endif -diff --git a/kernel/fork.c b/kernel/fork.c -index f202fd6d20a6..6a393a0d638e 100755 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -23,7 +23,9 @@ - #include - #include - #include --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - #include - #include - #include -@@ -997,7 +999,9 @@ void __put_task_struct(struct task_struct *tsk) - WARN_ON(refcount_read(&tsk->usage)); - WARN_ON(tsk == current); - -- sched_ext_free(tsk); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_free(tsk); -+#endif - io_uring_free(tsk); - cgroup_free(tsk); - task_numa_free(tsk, true); -@@ -2508,7 +2512,11 @@ __latent_entropy struct task_struct *copy_process( - - retval = perf_event_init_task(p, clone_flags); - if (retval) -- goto bad_fork_sched_cancel_fork; -+#ifdef CONFIG_HMBIRD_SCHED -+ goto bad_fork_hmbird_cancel_fork; -+#else -+ goto bad_fork_cleanup_policy; -+#endif - retval = audit_alloc(p); - if (retval) - goto bad_fork_cleanup_perf; -@@ -2640,9 +2648,7 @@ __latent_entropy struct task_struct *copy_process( - * cgroup specific, it unconditionally needs to place the task on a - * runqueue. - */ -- retval = sched_cgroup_fork(p, args); -- if (retval) -- goto bad_fork_cancel_cgroup; -+ sched_cgroup_fork(p, args); - - /* - * From this point on we must avoid any synchronous user-space -@@ -2688,13 +2694,13 @@ __latent_entropy struct task_struct *copy_process( - /* Don't start children in a dying pid namespace */ - if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { - retval = -ENOMEM; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* Let kill terminate clone/fork in the middle */ - if (fatal_signal_pending(current)) { - retval = -EINTR; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* No more failure paths after this point. */ -@@ -2771,11 +2777,10 @@ __latent_entropy struct task_struct *copy_process( - - return p; - --bad_fork_core_free: -+bad_fork_cancel_cgroup: - sched_core_free(p); - spin_unlock(¤t->sighand->siglock); - write_unlock_irq(&tasklist_lock); --bad_fork_cancel_cgroup: - cgroup_cancel_fork(p, args); - bad_fork_put_pidfd: - if (clone_flags & CLONE_PIDFD) { -@@ -2814,8 +2819,10 @@ __latent_entropy struct task_struct *copy_process( - audit_free(p); - bad_fork_cleanup_perf: - perf_event_free_task(p); --bad_fork_sched_cancel_fork: -- sched_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+bad_fork_hmbird_cancel_fork: -+ hmbird_cancel_fork(p); -+#endif - bad_fork_cleanup_policy: - lockdep_free_task(p); - #ifdef CONFIG_NUMA -diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c -index 005025f55bea..c091539a0f60 100755 ---- a/kernel/sched/build_policy.c -+++ b/kernel/sched/build_policy.c -@@ -28,8 +28,10 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED - #include - #include -+#endif - - #include - -@@ -54,6 +56,10 @@ - #include "cputime.c" - #include "deadline.c" - --#ifdef CONFIG_SCHED_CLASS_EXT --# include "ext.c" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/hmbird_util_track.c" -+#include "hmbird/hmbird_sched_proc.c" -+#include "hmbird/hmbird_shadow_tick.c" -+#include "hmbird/hmbird.c" -+#include "hmbird/hmbird_misc.c" - #endif -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index 85ca05ced366..a258adcea40c 100755 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -96,6 +96,12 @@ - #include "../../io_uring/io-wq.h" - #include "../smpboot.h" - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/slim.h" -+#include "hmbird/hmbird_shadow_tick.h" -+#include "hmbird.h" -+#endif -+ - #include - #include - #include -@@ -170,7 +176,6 @@ __read_mostly int scheduler_running; - - DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); - -- - /* kernel prio, less is more */ - static inline int __task_prio(const struct task_struct *p) - { -@@ -183,8 +188,8 @@ static inline int __task_prio(const struct task_struct *p) - if (p->sched_class == &idle_sched_class) - return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - --#ifdef CONFIG_SCHED_CLASS_EXT -- if (p->sched_class == &ext_sched_class) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (p->sched_class == &hmbird_sched_class) - return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ - #endif - -@@ -217,9 +222,9 @@ static inline bool prio_less(const struct task_struct *a, - if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ - return cfs_prio_less(a, b, in_fi); - --#ifdef CONFIG_SCHED_CLASS_EXT -+#ifdef CONFIG_HMBIRD_SCHED - if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ -- return scx_prio_less(a, b, in_fi); -+ return hmbird_prio_less(a, b, in_fi); - #endif - - return false; -@@ -1276,17 +1281,26 @@ bool sched_can_stop_tick(struct rq *rq) - if (fifo_nr_running) - return true; - -+#ifdef CONFIG_HMBIRD_SCHED - /* -- * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks -+ * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks - * left. For CFS, if there's more than one we need the tick for -- * involuntary preemption. For SCX, ask. -+ * involuntary preemption. For HMBIRD, ask. - */ -- if (!scx_switched_all() && rq->nr_running > 1) -+ if (!hmbird_enabled() && rq->nr_running > 1) - return false; - -- if (scx_enabled() && !scx_can_stop_tick(rq)) -+ if (hmbird_enabled() && !hmbird_can_stop_tick(rq)) - return false; -- -+#else -+ /* -+ * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; -+ * if there's more than one we need the tick for involuntary -+ * preemption. -+ */ -+ if (rq->nr_running > 1) -+ return false; -+#endif - /* - * If there is one task and it has CFS runtime bandwidth constraints - * and it's on the cpu now we don't want to stop the tick. -@@ -2211,10 +2225,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) - } - EXPORT_SYMBOL_GPL(deactivate_task); - --struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) -+#ifdef CONFIG_HMBIRD_SCHED -+struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - { -- struct sched_change_guard cg = { -+ struct hmbird_sched_change_guard cg = { - .rq = rq, - .p = p, - .queued = task_on_rq_queued(p), -@@ -2236,7 +2251,7 @@ sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - return cg; - } - --void sched_change_guard_fini(struct sched_change_guard *cg, int flags) -+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags) - { - if (cg->queued) - enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); -@@ -2244,6 +2259,7 @@ void sched_change_guard_fini(struct sched_change_guard *cg, int flags) - set_next_task(cg->rq, cg->p); - cg->done = true; - } -+#endif - - static inline int __normal_prio(int policy, int rt_prio, int nice) - { -@@ -2309,9 +2325,9 @@ inline int task_curr(const struct task_struct *p) - * this means any call to check_class_changed() must be followed by a call to - * balance_callback(). - */ --void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio) -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) - { - if (prev_class != p->sched_class) { - if (prev_class->switched_from) -@@ -2874,6 +2890,7 @@ static void - __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - { - struct rq *rq = task_rq(p); -+ bool queued, running; - - /* - * This here violates the locking rules for affinity, since we're only -@@ -2892,9 +2909,26 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - else - lockdep_assert_held(&p->pi_lock); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->sched_class->set_cpus_allowed(p, ctx); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ -+ if (queued) { -+ /* -+ * Because __kthread_bind() calls this on blocked tasks without -+ * holding rq->lock. -+ */ -+ lockdep_assert_rq_held(rq); -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); - } -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->sched_class->set_cpus_allowed(p, ctx); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - } - - /* -@@ -3761,7 +3795,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class)) -+#else - if (unlikely(!is_cpu_allowed(p, cpu))) -+#endif - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -@@ -4879,8 +4917,9 @@ late_initcall(sched_core_sysctl_init); - */ - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { -- -+#ifdef CONFIG_HMBIRD_SCHED - int ret; -+#endif - - trace_android_rvh_sched_fork(p); - -@@ -4921,21 +4960,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_reset_on_fork = 0; - } - -- scx_pre_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ ret = hmbird_pre_fork(p); -+ if (ret) -+ goto out_cancel; - - if (dl_prio(p->prio)) { - ret = -EAGAIN; - goto out_cancel; --#ifdef CONFIG_SCHED_CLASS_EXT -- } else if (task_on_scx(p)) { -- p->sched_class = &ext_sched_class; --#endif -+ } else if (task_on_hmbird(p)) { -+ p->sched_class = &hmbird_sched_class; - } else if (rt_prio(p->prio)) { - p->sched_class = &rt_sched_class; - } else { - p->sched_class = &fair_sched_class; - } -- -+#else -+ if (dl_prio(p->prio)) -+ return -EAGAIN; -+ else if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - init_entity_runnable_average(&p->se); - trace_android_rvh_finish_prio_fork(p); - -@@ -4954,12 +5001,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - #endif - return 0; - -+#ifdef CONFIG_HMBIRD_SCHED - out_cancel: -- scx_cancel_fork(p); -+ hmbird_cancel_fork(p); - return ret; -+#endif - } - --int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) -+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - { - unsigned long flags; - -@@ -4986,20 +5035,17 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -- -- return scx_fork(p); --} -- --void sched_cancel_fork(struct task_struct *p) --{ -- scx_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_fork(p); -+#endif - } - - void sched_post_fork(struct task_struct *p) - { - uclamp_post_fork(p); -- -- scx_post_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_post_fork(p); -+#endif - } - - unsigned long to_ratio(u64 period, u64 runtime) -@@ -5864,20 +5910,28 @@ void scheduler_tick(void) - if (sched_feat(LATENCY_WARN) && resched_latency) - resched_latency_warn(cpu, resched_latency); - -- scx_notify_sched_tick(); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_notify_sched_tick(); -+#endif - perf_event_task_tick(); - - if (curr->flags & PF_WQ_WORKER) - wq_worker_tick(curr); - - #ifdef CONFIG_SMP -- if (!scx_switched_all()) { -+#ifdef CONFIG_HMBIRD_SCHED -+ if (!hmbird_enabled()) { -+#endif - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); -+#ifdef CONFIG_HMBIRD_SCHED - } -+#endif - #endif - trace_android_vh_scheduler_tick(rq); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ scheduler_tick_handler(NULL, NULL); -+#endif - } - - #ifdef CONFIG_NO_HZ_FULL -@@ -6179,7 +6233,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ -+#ifdef CONFIG_HMBIRD_SCHED - for_balance_class_range(class, prev->sched_class, &idle_sched_class) { -+#else -+ for_class_range(class, prev->sched_class, &idle_sched_class) { -+#endif - if (class->balance(rq, prev, rf)) - break; - } -@@ -6197,9 +6255,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - const struct sched_class *class; - struct task_struct *p; - -- if (scx_enabled()) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (hmbird_enabled()) - goto restart; -- -+#endif - /* - * Optimization: we know that if all tasks are in the fair class we can - * call that function directly, but only if the @prev task wasn't of a -@@ -6225,13 +6284,21 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - restart: - put_prev_task_balance(rq, prev, rf); - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { - p = class->pick_next_task(rq); - if (p) { -- scx_notify_pick_next_task(rq, p, class); -+ hmbird_notify_pick_next_task(rq, p, class); - return p; - } - } -+#else -+ for_each_class(class) { -+ p = class->pick_next_task(rq); -+ if (p) -+ return p; -+ } -+#endif - - BUG(); /* The idle class should always have a runnable task. */ - } -@@ -6260,7 +6327,11 @@ static inline struct task_struct *pick_task(struct rq *rq) - const struct sched_class *class; - struct task_struct *p; - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { -+#else -+ for_each_class(class) { -+#endif - p = class->pick_task(rq); - if (p) - return p; -@@ -6904,6 +6975,9 @@ static void __sched notrace __schedule(unsigned int sched_mode) - psi_sched_switch(prev, next, !task_on_rq_queued(prev)); - - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#ifdef CONFIG_HMBIRD_SCHED -+ sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#endif - - /* Also unlocks the rq: */ - rq = context_switch(rq, prev, next, &rf); -@@ -7232,24 +7306,31 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag - } - EXPORT_SYMBOL(default_wake_function); - --void __setscheduler_prio(struct task_struct *p, int prio) -+static void __setscheduler_prio(struct task_struct *p, int prio) - { -- /* -- * After switching all rt and fair class to ext, -- * stop class is switched to ext class too. Just -- * skip it. */ -+#ifdef CONFIG_HMBIRD_SCHED -+ bool on_hmbird = task_on_hmbird(p); -+ - if (p->sched_class == &stop_sched_class) -- ;/* do nothing */ -+ ; - else if (dl_prio(prio)) - p->sched_class = &dl_sched_class; --#ifdef CONFIG_SCHED_CLASS_EXT -- else if (task_on_scx(p)) -- p->sched_class = &ext_sched_class; --#endif -+ else if (rt_prio(prio) && on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#else -+ if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; - else if (rt_prio(prio)) - p->sched_class = &rt_sched_class; - else - p->sched_class = &fair_sched_class; -+#endif - - p->prio = prio; - } -@@ -7284,7 +7365,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - */ - void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - { -- int prio, oldprio, queue_flag = -+ int prio, oldprio, queued, running, queue_flag = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *prev_class; - struct rq_flags rf; -@@ -7347,42 +7428,52 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - queue_flag &= ~DEQUEUE_MOVE; - - prev_class = p->sched_class; -- SCHED_CHANGE_BLOCK(rq, p, queue_flag) { -- /* -- * Boosting condition are: -- * 1. -rt task is running and holds mutex A -- * --> -dl task blocks on mutex A -- * -- * 2. -dl task is running and holds mutex A -- * --> -dl task blocks on mutex A and could preempt the -- * running task -- */ -- if (dl_prio(prio)) { -- if (!dl_prio(p->normal_prio) || -- (pi_task && dl_prio(pi_task->prio) && -- dl_entity_preempt(&pi_task->dl, &p->dl))) { -- p->dl.pi_se = pi_task->dl.pi_se; -- queue_flag |= ENQUEUE_REPLENISH; -- } else { -- p->dl.pi_se = &p->dl; -- } -- } else if (rt_prio(prio)) { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- if (oldprio < prio) -- queue_flag |= ENQUEUE_HEAD; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flag); -+ if (running) -+ put_prev_task(rq, p); -+ -+ /* -+ * Boosting condition are: -+ * 1. -rt task is running and holds mutex A -+ * --> -dl task blocks on mutex A -+ * -+ * 2. -dl task is running and holds mutex A -+ * --> -dl task blocks on mutex A and could preempt the -+ * running task -+ */ -+ if (dl_prio(prio)) { -+ if (!dl_prio(p->normal_prio) || -+ (pi_task && dl_prio(pi_task->prio) && -+ dl_entity_preempt(&pi_task->dl, &p->dl))) { -+ p->dl.pi_se = pi_task->dl.pi_se; -+ queue_flag |= ENQUEUE_REPLENISH; - } else { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- else if (rt_prio(oldprio)) -- p->rt.timeout = 0; -- else if (!task_has_idle_policy(p)) -- reweight_task(p, prio - MAX_RT_PRIO); -+ p->dl.pi_se = &p->dl; - } -- -- __setscheduler_prio(p, prio); -+ } else if (rt_prio(prio)) { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ if (oldprio < prio) -+ queue_flag |= ENQUEUE_HEAD; -+ } else { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ else if (rt_prio(oldprio)) -+ p->rt.timeout = 0; -+ else if (!task_has_idle_policy(p)) -+ reweight_task(p, prio - MAX_RT_PRIO); - } - -+ __setscheduler_prio(p, prio); -+ -+ if (queued) -+ enqueue_task(rq, p, queue_flag); -+ if (running) -+ set_next_task(rq, p); -+ - check_class_changed(rq, p, prev_class, oldprio); - out_unlock: - /* Avoid rq from going away on us: */ -@@ -7403,7 +7494,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - - void set_user_nice(struct task_struct *p, long nice) - { -- bool allowed = false; -+ bool queued, running, allowed = false; - int old_prio; - struct rq_flags rf; - struct rq *rq; -@@ -7432,13 +7523,22 @@ void set_user_nice(struct task_struct *p, long nice) - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); -+ if (running) -+ put_prev_task(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->static_prio = NICE_TO_PRIO(nice); -- set_load_weight(p, true); -- old_prio = p->prio; -- p->prio = effective_prio(p); -- } -+ p->static_prio = NICE_TO_PRIO(nice); -+ set_load_weight(p, true); -+ old_prio = p->prio; -+ p->prio = effective_prio(p); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - - /* - * If the task increased its priority or is running and -@@ -7847,7 +7947,7 @@ static int __sched_setscheduler(struct task_struct *p, - bool user, bool pi) - { - int oldpolicy = -1, policy = attr->sched_policy; -- int retval, oldprio, newprio; -+ int retval, oldprio, newprio, queued, running; - const struct sched_class *prev_class; - struct balance_callback *head; - struct rq_flags rf; -@@ -7855,6 +7955,9 @@ static int __sched_setscheduler(struct task_struct *p, - int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct rq *rq; - bool cpuset_locked = false; -+#ifdef CONFIG_HMBIRD_SCHED -+ unsigned long flags; -+#endif - - - /* The pi code expects interrupts enabled */ -@@ -7921,7 +8024,9 @@ static int __sched_setscheduler(struct task_struct *p, - * To be able to change p->policy safely, the appropriate - * runqueue lock must be held. - */ -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+#endif - rq = task_rq_lock(p, &rf); - update_rq_clock(rq); - -@@ -7933,10 +8038,11 @@ static int __sched_setscheduler(struct task_struct *p, - goto unlock; - } - -- retval = scx_check_setscheduler(p, policy); -+#ifdef CONFIG_HMBIRD_SCHED -+ retval = hmbird_check_setscheduler(p, policy); - if (retval) - goto unlock; -- -+#endif - /* - * If not changing anything there's no need to proceed further, - * but store a possible modification of reset_on_fork. -@@ -7993,7 +8099,9 @@ static int __sched_setscheduler(struct task_struct *p, - if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { - policy = oldpolicy = -1; - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - goto recheck; -@@ -8026,16 +8134,23 @@ static int __sched_setscheduler(struct task_struct *p, - queue_flags &= ~DEQUEUE_MOVE; - } - -- SCHED_CHANGE_BLOCK(rq, p, queue_flags) { -- prev_class = p->sched_class; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flags); -+ if (running) -+ put_prev_task(rq, p); -+ -+ prev_class = p->sched_class; - -- if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -- __setscheduler_params(p, attr); -- __setscheduler_prio(p, newprio); -- trace_android_rvh_setscheduler(p); -- } -- __setscheduler_uclamp(p, attr); -+ if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -+ __setscheduler_params(p, attr); -+ __setscheduler_prio(p, newprio); -+ trace_android_rvh_setscheduler(p); -+ } -+ __setscheduler_uclamp(p, attr); - -+ if (queued) { - /* - * We enqueue to tail when the priority of a task is - * increased (user space view). -@@ -8043,7 +8158,10 @@ static int __sched_setscheduler(struct task_struct *p, - if (oldprio < p->prio) - queue_flags |= ENQUEUE_HEAD; - -+ enqueue_task(rq, p, queue_flags); - } -+ if (running) -+ set_next_task(rq, p); - - check_class_changed(rq, p, prev_class, oldprio); - -@@ -8051,7 +8169,9 @@ static int __sched_setscheduler(struct task_struct *p, - preempt_disable(); - head = splice_balance_callbacks(rq); - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (pi) { - if (cpuset_locked) - cpuset_unlock(); -@@ -8066,7 +8186,9 @@ static int __sched_setscheduler(struct task_struct *p, - - unlock: - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - return retval; -@@ -9288,7 +9410,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - break; - } -@@ -9316,7 +9440,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - } - return ret; -@@ -9621,15 +9747,25 @@ int migrate_task_to(struct task_struct *p, int target_cpu) - */ - void sched_setnuma(struct task_struct *p, int nid) - { -+ bool queued, running; - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) { -- p->numa_preferred_nid = nid; -- } -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE); -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->numa_preferred_nid = nid; - -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - task_rq_unlock(rq, p, &rf); - } - #endif /* CONFIG_NUMA_BALANCING */ -@@ -10195,15 +10331,22 @@ void __init sched_init(void) - int i; - - /* Make sure the linker didn't screw up */ -+#ifdef CONFIG_HMBIRD_SCHED -+#ifdef CONFIG_SMP -+ WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+#endif -+ WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class)); -+#else -+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 || -+ &fair_sched_class != &rt_sched_class + 1 || -+ &rt_sched_class != &dl_sched_class + 1); - #ifdef CONFIG_SMP -- BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+ BUG_ON(&dl_sched_class != &stop_sched_class + 1); - #endif -- BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); -- BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); -- BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); --#ifdef CONFIG_SCHED_CLASS_EXT -- BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); -- BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); - #endif - - wait_bit_init(); -@@ -10374,8 +10517,9 @@ void __init sched_init(void) - balance_push_set(smp_processor_id(), false); - #endif - init_sched_fair_class(); -- init_sched_ext_class(); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ init_sched_hmbird_class(); -+#endif - psi_init(); - - init_uclamp(); -@@ -10785,6 +10929,8 @@ static void sched_change_group(struct task_struct *tsk, struct task_group *group - */ - void sched_move_task(struct task_struct *tsk) - { -+ int queued, running, queue_flags = -+ DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct task_group *group; - struct rq_flags rf; - struct rq *rq; -@@ -10801,18 +10947,27 @@ void sched_move_task(struct task_struct *tsk) - - update_rq_clock(rq); - -- SCHED_CHANGE_BLOCK(rq, tsk, -- DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { -- sched_change_group(tsk, group); -- } -+ running = task_current(rq, tsk); -+ queued = task_on_rq_queued(tsk); - -- /* -- * After changing group, the running task may have joined a throttled -- * one but it's still the running task. Trigger a resched to make sure -- * that task can still run. -- */ -- if (task_current(rq, tsk)) -+ if (queued) -+ dequeue_task(rq, tsk, queue_flags); -+ if (running) -+ put_prev_task(rq, tsk); -+ -+ sched_change_group(tsk, group); -+ -+ if (queued) -+ enqueue_task(rq, tsk, queue_flags); -+ if (running) { -+ set_next_task(rq, tsk); -+ /* -+ * After changing group, the running task may have joined a -+ * throttled one but it's still the running task. Trigger a -+ * resched to make sure that task can still run. -+ */ - resched_curr(rq); -+ } - - unlock: - task_rq_unlock(rq, tsk, &rf); -@@ -10850,6 +11005,10 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) - struct task_group *tg = css_tg(css); - struct task_group *parent = css_tg(css->parent); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_tg_online(tg); -+#endif -+ - if (parent) - sched_online_group(tg, parent); - -@@ -10899,13 +11058,77 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) - } - #endif - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 dl) -+{ -+ int i; -+ -+ for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) { -+ if (dl < HMBIRD_BPF_DSQS_DEADLINE[i]) -+ break; -+ } -+ -+ i = max_t(int, i-1, MIN_CGROUP_DL_IDX); -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return 0; -+ update_cgroup_ids_table(css->cgroup->kn->id, i); -+ -+ return 0; -+} -+ -+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ u8 i; -+ -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX]; -+ i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1); -+ if (i < 0) { -+ pr_err(" <%s> i is %d, less than 0, name is %s\n", -+ __func__, i, css->cgroup->kn->name); -+ i = DEFAULT_CGROUP_DL_IDX; -+ } -+ -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[i]; -+} -+ -+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css, -+ struct task_struct *p, int prio) -+{ -+ if (!css || !rt_prio(prio)) -+ return; -+ -+ if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) { -+ if (!strcmp(css->cgroup->kn->name, "display")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3); -+ else if (!strcmp(css->cgroup->kn->name, "touch")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2); -+ else if (!strcmp(css->cgroup->kn->name, "multimedia")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+ } -+} -+#endif -+ - static void cpu_cgroup_attach(struct cgroup_taskset *tset) - { - struct task_struct *task; - struct cgroup_subsys_state *css; - - cgroup_taskset_for_each(task, css, tset) { -- -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_change_rt_sched_prop(css, task, task->prio); -+#endif - sched_move_task(task); - } - -@@ -11587,7 +11810,13 @@ static struct cftype cpu_legacy_files[] = { - .write_u64 = cpu_uclamp_ls_write_u64, - }, - #endif -- -+#ifdef CONFIG_HMBIRD_SCHED -+ { -+ .name = "hmbird.deadline", -+ .read_u64 = cgroup_read_hmbird_deadline, -+ .write_u64 = cgroup_write_hmbird_deadline, -+ }, -+#endif - { } /* Terminate */ - }; - -@@ -11636,7 +11865,6 @@ static int cpu_local_stat_show(struct seq_file *sf, - } - - #ifdef CONFIG_FAIR_GROUP_SCHED -- - static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, - struct cftype *cft) - { -diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c -index 423404100c9a..2fe1a734e640 100755 ---- a/kernel/sched/debug.c -+++ b/kernel/sched/debug.c -@@ -376,8 +376,8 @@ static __init int sched_init_debug(void) - - debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); - --#ifdef CONFIG_SCHED_CLASS_EXT -- debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops); -+#ifdef CONFIG_HMBIRD_SCHED -+ debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops); - #endif - return 0; - } -@@ -1006,9 +1006,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - SEQ_printf(m, - "---------------------------------------------------------" - "----------\n"); --#ifdef CONFIG_SLIM_SCHED -- SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop); --#endif - - #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) - #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) -@@ -1102,8 +1099,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - P(dl.runtime); - P(dl.deadline); - } --#ifdef CONFIG_SCHED_CLASS_EXT -- __PS("ext.enabled", p->sched_class == &ext_sched_class); -+#ifdef CONFIG_HMBIRD_SCHED -+ __PS("hmbird.enabled", p->sched_class == &hmbird_sched_class); -+ __PS("sched_prop", get_hmbird_ts(p)->sched_prop); -+ __PS("top_task_prop", get_hmbird_ts(p)->top_task_prop); - #endif - #undef PN_SCHEDSTAT - #undef P_SCHEDSTAT -diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c -deleted file mode 100755 -index 4845d441df2b..000000000000 ---- a/kernel/sched/ext.c -+++ /dev/null -@@ -1,3978 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -- --enum scx_internal_consts { -- SCX_NR_ONLINE_OPS = SCX_OP_IDX(init), -- SCX_DSP_DFL_MAX_BATCH = 32, -- SCX_DSP_MAX_LOOPS = 32, -- SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, --}; -- --enum scx_ops_enable_state { -- SCX_OPS_PREPPING, -- SCX_OPS_ENABLING, -- SCX_OPS_ENABLED, -- SCX_OPS_DISABLING, -- SCX_OPS_DISABLED, --}; -- --static inline struct task_group *css_tg(struct cgroup_subsys_state *css) --{ -- return css ? container_of(css, struct task_group, css) : NULL; --} -- --/* -- * sched_ext_entity->ops_state -- * -- * Used to track the task ownership between the SCX core and the BPF scheduler. -- * State transitions look as follows: -- * -- * NONE -> QUEUEING -> QUEUED -> DISPATCHING -- * ^ | | -- * | v v -- * \-------------------------------/ -- * -- * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -- * sites for explanations on the conditions being waited upon and why they are -- * safe. Transitions out of them into NONE or QUEUED must store_release and the -- * waiters should load_acquire. -- * -- * Tracking scx_ops_state enables sched_ext core to reliably determine whether -- * any given task can be dispatched by the BPF scheduler at all times and thus -- * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -- * to try to dispatch any task anytime regardless of its state as the SCX core -- * can safely reject invalid dispatches. -- */ --enum scx_ops_state { -- SCX_OPSS_NONE, /* owned by the SCX core */ -- SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -- SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ -- SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ -- -- /* -- * QSEQ brands each QUEUED instance so that, when dispatch races -- * dequeue/requeue, the dispatcher can tell whether it still has a claim -- * on the task being dispatched. -- */ -- SCX_OPSS_QSEQ_SHIFT = 2, -- SCX_OPSS_STATE_MASK = (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1, -- SCX_OPSS_QSEQ_MASK = ~SCX_OPSS_STATE_MASK, --}; -- --/* -- * During exit, a task may schedule after losing its PIDs. When disabling the -- * BPF scheduler, we need to be able to iterate tasks in every state to -- * guarantee system safety. Maintain a dedicated task list which contains every -- * task between its fork and eventual free. -- */ --static DEFINE_SPINLOCK(scx_tasks_lock); --static LIST_HEAD(scx_tasks); -- --/* ops enable/disable */ --static struct kthread_worker *scx_ops_helper; --static DEFINE_MUTEX(scx_ops_enable_mutex); --DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); --DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); --static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); --static bool scx_switch_all_req; --static bool scx_switching_all; --DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -- --static struct sched_ext_ops scx_ops; --static bool scx_warned_zero_slice; -- --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); --DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); --static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); -- --struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] = -- { [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT }; -- --static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE); --static struct scx_exit_info scx_exit_info; -- --static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0); -- --/* -- * The maximum amount of time in jiffies that a task may be runnable without -- * being scheduled on a CPU. If this timeout is exceeded, it will trigger -- * scx_ops_error(). -- */ --unsigned long scx_watchdog_timeout; -- --/* -- * The last time the delayed work was run. This delayed work relies on -- * ksoftirqd being able to run to service timer interrupts, so it's possible -- * that this work itself could get wedged. To account for this, we check that -- * it's not stalled in the timer tick, and trigger an error if it is. -- */ --unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; -- --static struct delayed_work scx_watchdog_work; -- --/* idle tracking */ --#ifdef CONFIG_SMP --#ifdef CONFIG_CPUMASK_OFFSTACK --#define CL_ALIGNED_IF_ONSTACK --#else --#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp --#endif -- --static struct { -- cpumask_var_t cpu; -- cpumask_var_t smt; --} idle_masks CL_ALIGNED_IF_ONSTACK; -- --static bool __cacheline_aligned_in_smp scx_has_idle_cpus; --#endif /* CONFIG_SMP */ -- --/* for %SCX_KICK_WAIT */ --static u64 __percpu *scx_kick_cpus_pnt_seqs; -- --/* -- * Direct dispatch marker. -- * -- * Non-NULL values are used for direct dispatch from enqueue path. A valid -- * pointer points to the task currently being enqueued. An ERR_PTR value is used -- * to indicate that direct dispatch has already happened. -- */ --static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); -- --/* dispatch queues */ --static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; -- --static LLIST_HEAD(dsqs_to_free); -- --/* dispatch buf */ --struct scx_dsp_buf_ent { -- struct task_struct *task; -- u64 qseq; -- u64 dsq_id; -- u64 enq_flags; --}; -- --static u32 scx_dsp_max_batch; --static struct scx_dsp_buf_ent __percpu *scx_dsp_buf; -- --struct scx_dsp_ctx { -- struct rq *rq; -- struct rq_flags *rf; -- u32 buf_cursor; -- u32 nr_tasks; --}; -- --static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx); -- --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags); --void scx_bpf_kick_cpu(s32 cpu, u64 flags); -- --struct scx_task_iter { -- struct sched_ext_entity cursor; -- struct task_struct *locked; -- struct rq *rq; -- struct rq_flags rf; --}; -- --#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) -- --/* if the highest set bit is N, return a mask with bits [N+1, 31] set */ --static u32 higher_bits(u32 flags) --{ -- return ~((1 << fls(flags)) - 1); --} -- --/* return the mask with only the highest bit set */ --static u32 highest_bit(u32 flags) --{ -- int bit = fls(flags); -- return bit ? 1 << (bit - 1) : 0; --} -- --/* -- * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX -- * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate -- * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check -- * whether it's running from an allowed context. -- * -- * @mask is constant, always inline to cull the mask calculations. -- */ --static __always_inline void scx_kf_allow(u32 mask) --{ -- /* nesting is allowed only in increasing scx_kf_mask order */ -- WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask, -- "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n", -- current->scx->kf_mask, mask); -- current->scx->kf_mask |= mask; --} -- --static void scx_kf_disallow(u32 mask) --{ -- current->scx->kf_mask &= ~mask; --} -- --#define SCX_CALL_OP(mask, op, args...) \ --do { \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- scx_ops.op(args); \ -- } \ --} while (0) -- --#define SCX_CALL_OP_RET(mask, op, args...) \ --({ \ -- __typeof__(scx_ops.op(args)) __ret; \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- __ret = scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- __ret = scx_ops.op(args); \ -- } \ -- __ret; \ --}) -- --/* -- * Some kfuncs are allowed only on the tasks that are subjects of the -- * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such -- * restrictions, the following SCX_CALL_OP_*() variants should be used when -- * invoking scx_ops operations that take task arguments. These can only be used -- * for non-nesting operations due to the way the tasks are tracked. -- * -- * kfuncs which can only operate on such tasks can in turn use -- * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on -- * the specific task. -- */ --#define SCX_CALL_OP_TASK(mask, op, task, args...) \ --do { \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- SCX_CALL_OP(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ --} while (0) -- --#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ --({ \ -- __typeof__(scx_ops.op(task, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- __ret; \ --}) -- --#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ --({ \ -- __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task0; \ -- current->scx->kf_tasks[1] = task1; \ -- __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- current->scx->kf_tasks[1] = NULL; \ -- __ret; \ --}) -- --//#include "./slim_walt.c" -- --/* @mask is constant, always inline to cull unnecessary branches */ --static __always_inline bool scx_kf_allowed(u32 mask) --{ -- if (unlikely(!(current->scx->kf_mask & mask))) { -- scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", -- mask, current->scx->kf_mask); -- return false; -- } -- -- if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) && -- in_interrupt())) { -- scx_ops_error("sleepable kfunc called from non-sleepable context"); -- return false; -- } -- -- /* -- * Enforce nesting boundaries. e.g. A kfunc which can be called from -- * DISPATCH must not be called if we're running DEQUEUE which is nested -- * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE -- * boundary thanks to the above in_interrupt() check. -- */ -- if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && -- (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { -- scx_ops_error("cpu_release kfunc called from a nested operation"); -- return false; -- } -- -- if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && -- (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) { -- scx_ops_error("dispatch kfunc called from a nested operation"); -- return false; -- } -- -- return true; --} -- --/* see SCX_CALL_OP_TASK() */ --static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, -- struct task_struct *p) --{ -- if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED)) -- return false; -- -- if (unlikely((p != current->scx->kf_tasks[0] && -- p != current->scx->kf_tasks[1]))) { -- scx_ops_error("called on a task not being operated on"); -- return false; -- } -- -- return true; --} -- --/** -- * scx_task_iter_init - Initialize a task iterator -- * @iter: iterator to init -- * -- * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized, -- * @iter must eventually be exited with scx_task_iter_exit(). -- * -- * scx_tasks_lock may be released between this and the first next() call or -- * between any two next() calls. If scx_tasks_lock is released between two -- * next() calls, the caller is responsible for ensuring that the task being -- * iterated remains accessible either through RCU read lock or obtaining a -- * reference count. -- * -- * All tasks which existed when the iteration started are guaranteed to be -- * visited as long as they still exist. -- */ --static void scx_task_iter_init(struct scx_task_iter *iter) --{ -- lockdep_assert_held(&scx_tasks_lock); -- -- iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; -- list_add(&iter->cursor.tasks_node, &scx_tasks); -- iter->locked = NULL; --} -- --/** -- * scx_task_iter_exit - Exit a task iterator -- * @iter: iterator to exit -- * -- * Exit a previously initialized @iter. Must be called with scx_tasks_lock held. -- * If the iterator holds a task's rq lock, that rq lock is released. See -- * scx_task_iter_init() for details. -- */ --static void scx_task_iter_exit(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- if (list_empty(cursor)) -- return; -- -- list_del_init(cursor); --} -- --/** -- * scx_task_iter_next - Next task -- * @iter: iterator to walk -- * -- * Visit the next task. See scx_task_iter_init() for details. -- */ --static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- struct sched_ext_entity *pos; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- list_for_each_entry(pos, cursor, tasks_node) { -- if (&pos->tasks_node == &scx_tasks) -- return NULL; -- if (!(pos->flags & SCX_TASK_CURSOR)) { -- list_move(cursor, &pos->tasks_node); -- return pos->task; -- } -- } -- -- /* can't happen, should always terminate at scx_tasks above */ -- BUG(); --} -- --/** -- * scx_task_iter_next_filtered - Next non-idle task -- * @iter: iterator to walk -- * -- * Visit the next non-idle task. See scx_task_iter_init() for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- while ((p = scx_task_iter_next(iter))) { -- if (!is_idle_task(p)) -- return p; -- } -- return NULL; --} -- --/** -- * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked -- * @iter: iterator to walk -- * -- * Visit the next non-idle task with its rq lock held. See scx_task_iter_init() -- * for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered_locked(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- p = scx_task_iter_next_filtered(iter); -- if (!p) -- return NULL; -- -- iter->rq = task_rq_lock(p, &iter->rf); -- iter->locked = p; -- return p; --} -- --static enum scx_ops_enable_state scx_ops_enable_state(void) --{ -- return atomic_read(&scx_ops_enable_state_var); --} -- --static enum scx_ops_enable_state --scx_ops_set_enable_state(enum scx_ops_enable_state to) --{ -- return atomic_xchg(&scx_ops_enable_state_var, to); --} -- --static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, -- enum scx_ops_enable_state from) --{ -- int from_v = from; -- -- return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); --} -- --static bool scx_ops_disabling(void) --{ -- return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING); --} -- --/** -- * wait_ops_state - Busy-wait the specified ops state to end -- * @p: target task -- * @opss: state to wait the end of -- * -- * Busy-wait for @p to transition out of @opss. This can only be used when the -- * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also -- * has load_acquire semantics to ensure that the caller can see the updates made -- * in the enqueueing and dispatching paths. -- */ --static void wait_ops_state(struct task_struct *p, u64 opss) --{ -- do { -- cpu_relax(); -- } while (atomic64_read_acquire(&p->scx->ops_state) == opss); --} -- --/** -- * ops_cpu_valid - Verify a cpu number -- * @cpu: cpu number which came from a BPF ops -- * -- * @cpu is a cpu number which came from the BPF scheduler and can be any value. -- * Verify that it is in range and one of the possible cpus. -- */ --static bool ops_cpu_valid(s32 cpu) --{ -- return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); --} -- --/** -- * ops_sanitize_err - Sanitize a -errno value -- * @ops_name: operation to blame on failure -- * @err: -errno value to sanitize -- * -- * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return -- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can -- * cause misbehaviors. For an example, a large negative return from -- * ops.prep_enable() triggers an oops when passed up the call chain because the -- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is -- * handled as a pointer. -- */ --static int ops_sanitize_err(const char *ops_name, s32 err) --{ -- if (err < 0 && err >= -MAX_ERRNO) -- return err; -- -- scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); -- return -EPROTO; --} -- --/** -- * touch_core_sched - Update timestamp used for core-sched task ordering -- * @rq: rq to read clock from, must be locked -- * @p: task to update the timestamp for -- * -- * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to -- * implement global or local-DSQ FIFO ordering for core-sched. Should be called -- * when a task becomes runnable and its turn on the CPU ends (e.g. slice -- * exhaustion). -- */ --static void touch_core_sched(struct rq *rq, struct task_struct *p) --{ --#ifdef CONFIG_SCHED_CORE -- /* -- * It's okay to update the timestamp spuriously. Use -- * sched_core_disabled() which is cheaper than enabled(). -- */ -- if (!sched_core_disabled()) -- p->scx->core_sched_at = rq_clock_task(rq); --#endif --} -- --/** -- * touch_core_sched_dispatch - Update core-sched timestamp on dispatch -- * @rq: rq to read clock from, must be locked -- * @p: task being dispatched -- * -- * If the BPF scheduler implements custom core-sched ordering via -- * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO -- * ordering within each local DSQ. This function is called from dispatch paths -- * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect. -- */ --static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- assert_clock_updated(rq); -- --#ifdef CONFIG_SCHED_CORE -- if (SCX_HAS_OP(core_sched_before)) -- touch_core_sched(rq, p); --#endif --} -- --static void update_curr_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- u64 now = rq_clock_task(rq); -- u64 delta_exec; -- -- if (time_before_eq64(now, curr->se.exec_start)) -- return; -- -- delta_exec = now - curr->se.exec_start; -- curr->se.exec_start = now; -- curr->se.sum_exec_runtime += delta_exec; -- account_group_exec_runtime(curr, delta_exec); -- cgroup_account_cputime(curr, delta_exec); -- -- if (curr->scx->slice != SCX_SLICE_INF) { -- curr->scx->slice -= min(curr->scx->slice, delta_exec); -- if (!curr->scx->slice) -- touch_core_sched(rq, curr); -- } --} -- --static bool scx_dsq_priq_less(struct rb_node *node_a, -- const struct rb_node *node_b) --{ -- const struct sched_ext_entity *a = -- container_of(node_a, struct sched_ext_entity, dsq_node.priq); -- const struct sched_ext_entity *b = -- container_of(node_b, struct sched_ext_entity, dsq_node.priq); -- -- return time_before64(a->dsq_vtime, b->dsq_vtime); --} -- --static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, -- u64 enq_flags) --{ -- bool is_local = dsq->id == SCX_DSQ_LOCAL; -- -- WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo)); -- WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq)); -- -- if (!is_local) { -- raw_spin_lock(&dsq->lock); -- if (unlikely(dsq->id == SCX_DSQ_INVALID)) { -- scx_ops_error("attempting to dispatch to a destroyed dsq"); -- /* fall back to the global dsq */ -- raw_spin_unlock(&dsq->lock); -- dsq = &scx_dsq_global; -- raw_spin_lock(&dsq->lock); -- } -- } -- -- if (enq_flags & SCX_ENQ_DSQ_PRIQ) { -- p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; -- rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq, -- scx_dsq_priq_less); -- } else { -- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) -- list_add(&p->scx->dsq_node.fifo, &dsq->fifo); -- else -- list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo); -- } -- dsq->nr++; -- p->scx->dsq = dsq; -- -- /* -- * We're transitioning out of QUEUEING or DISPATCHING. store_release to -- * match waiters' load_acquire. -- */ -- if (enq_flags & SCX_ENQ_CLEAR_OPSS) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- if (is_local) { -- struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq); -- struct rq *rq = scx->rq; -- bool preempt = false; -- -- if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && -- rq->curr->sched_class == &ext_sched_class) { -- rq->curr->scx->slice = 0; -- preempt = true; -- } -- -- if (preempt || sched_class_above(&ext_sched_class, -- rq->curr->sched_class)) -- resched_curr(rq); -- } else { -- raw_spin_unlock(&dsq->lock); -- } --} -- --static void task_unlink_from_dsq(struct task_struct *p, -- struct scx_dispatch_q *dsq) --{ -- if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { -- rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; -- } else { -- list_del_init(&p->scx->dsq_node.fifo); -- } -- --} -- --static bool task_linked_on_dsq(struct task_struct *p) --{ -- return !list_empty(&p->scx->dsq_node.fifo) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq); --} -- --static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p) --{ -- struct scx_dispatch_q *dsq = p->scx->dsq; -- bool is_local = dsq == &scx_rq->local_dsq; -- -- if (!dsq) { -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- /* -- * When dispatching directly from the BPF scheduler to a local -- * DSQ, the task isn't associated with any DSQ but -- * @p->scx->holding_cpu may be set under the protection of -- * %SCX_OPSS_DISPATCHING. -- */ -- if (p->scx->holding_cpu >= 0) -- p->scx->holding_cpu = -1; -- return; -- } -- -- if (!is_local) -- raw_spin_lock(&dsq->lock); -- -- /* -- * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node -- * can't change underneath us. -- */ -- if (p->scx->holding_cpu < 0) { -- /* @p must still be on @dsq, dequeue */ -- WARN_ON_ONCE(!task_linked_on_dsq(p)); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- } else { -- /* -- * We're racing against dispatch_to_local_dsq() which already -- * removed @p from @dsq and set @p->scx->holding_cpu. Clear the -- * holding_cpu which tells dispatch_to_local_dsq() that it lost -- * the race. -- */ -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- p->scx->holding_cpu = -1; -- } -- p->scx->dsq = NULL; -- -- if (!is_local) -- raw_spin_unlock(&dsq->lock); --} -- --static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) --{ -- lockdep_assert(rcu_read_lock_any_held()); -- -- return &scx_dsq_global; --} -- --static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, -- struct task_struct *p) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id == SCX_DSQ_LOCAL) -- return &rq->scx->local_dsq; -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", -- dsq_id, p->comm, p->pid); -- return &scx_dsq_global; -- } -- -- return dsq; --} -- --static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- -- /* @p must match the task which is being enqueued */ -- if (unlikely(p != ddsp_task)) { -- if (IS_ERR(ddsp_task)) -- scx_ops_error("%s[%d] already direct-dispatched", -- p->comm, p->pid); -- else -- scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]", -- ddsp_task->comm, ddsp_task->pid, -- p->comm, p->pid); -- return; -- } -- -- /* -- * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because -- * dispatching to the local DSQ of a different CPU requires unlocking -- * the current rq which isn't allowed in the enqueue path. Use -- * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL. -- */ -- if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) { -- scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch"); -- return; -- } -- -- touch_core_sched_dispatch(task_rq(p), p); -- -- dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- -- /* -- * Mark that dispatch already happened by spoiling direct_dispatch_task -- * with a non-NULL value which can never match a valid task pointer. -- */ -- __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); --} -- --static bool test_rq_online(struct rq *rq) --{ --#ifdef CONFIG_SMP -- return rq->online; --#else -- return true; --#endif --} -- --static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- int sticky_cpu) --{ -- struct task_struct **ddsp_taskp; -- u64 qseq; -- -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- if (p->scx->flags & SCX_TASK_ENQ_LOCAL) { -- enq_flags |= SCX_ENQ_LOCAL; -- p->scx->flags &= ~SCX_TASK_ENQ_LOCAL; -- } -- -- /* rq migration */ -- if (sticky_cpu == cpu_of(rq)) -- goto local_norefill; -- -- /* -- * If !rq->online, we already told the BPF scheduler that the CPU is -- * offline. We're just trying to on/offline the CPU. Don't bother the -- * BPF scheduler. -- */ -- if (unlikely(!test_rq_online(rq))) -- goto local; -- -- /* see %SCX_OPS_ENQ_EXITING */ -- if (!static_branch_unlikely(&scx_ops_enq_exiting) && -- unlikely(p->flags & PF_EXITING)) -- goto local; -- -- /* see %SCX_OPS_ENQ_LAST */ -- if (!static_branch_unlikely(&scx_ops_enq_last) && -- (enq_flags & SCX_ENQ_LAST)) -- goto local; -- -- if (!SCX_HAS_OP(enqueue)) { -- if (enq_flags & SCX_ENQ_LOCAL) -- goto local; -- else -- goto global; -- } -- -- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ -- qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; -- -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq); -- -- ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); -- WARN_ON_ONCE(*ddsp_taskp); -- *ddsp_taskp = p; -- -- SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); -- -- /* -- * If not directly dispatched, QUEUEING isn't clear yet and dispatch or -- * dequeue may be waiting. The store_release matches their load_acquire. -- */ -- if (*ddsp_taskp == p) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq); -- *ddsp_taskp = NULL; -- return; -- --local: -- /* -- * For task-ordering, slice refill must be treated as implying the end -- * of the current slice. Otherwise, the longer @p stays on the CPU, the -- * higher priority it becomes from scx_prio_less()'s POV. -- */ -- touch_core_sched(rq, p); -- p->scx->slice = SCX_SLICE_DFL; --local_norefill: -- dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags); -- return; -- --global: -- touch_core_sched(rq, p); /* see the comment in local: */ -- p->scx->slice = SCX_SLICE_DFL; -- dispatch_enqueue(&scx_dsq_global, p, enq_flags); --} -- --static bool watchdog_task_watched(const struct task_struct *p) --{ -- return !list_empty(&p->scx->watchdog_node); --} -- --static void watchdog_watch_task(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- if (p->scx->flags & SCX_TASK_WATCHDOG_RESET) -- p->scx->runnable_at = jiffies; -- p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET; -- list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list); --} -- --static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) --{ -- list_del_init(&p->scx->watchdog_node); -- if (reset_timeout) -- p->scx->flags |= SCX_TASK_WATCHDOG_RESET; --} -- --static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) --{ -- int sticky_cpu = p->scx->sticky_cpu; -- -- enq_flags |= rq->scx->extra_enq_flags; -- -- if (sticky_cpu >= 0) -- p->scx->sticky_cpu = -1; -- -- /* -- * Restoring a running task will be immediately followed by -- * set_next_task_scx() which expects the task to not be on the BPF -- * scheduler as tasks can only start running through local DSQs. Force -- * direct-dispatch into the local DSQ by setting the sticky_cpu. -- */ -- if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -- sticky_cpu = cpu_of(rq); -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- WARN_ON_ONCE(!watchdog_task_watched(p)); -- return; -- } -- -- watchdog_watch_task(rq, p); -- p->scx->flags |= SCX_TASK_QUEUED; -- rq->scx->nr_running++; -- add_nr_running(rq, 1); -- -- if (SCX_HAS_OP(runnable)) -- SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); -- -- if (enq_flags & SCX_ENQ_WAKEUP) -- touch_core_sched(rq, p); -- -- do_enqueue_task(rq, p, enq_flags, sticky_cpu); --} -- --static void ops_dequeue(struct task_struct *p, u64 deq_flags) --{ -- u64 opss; -- -- watchdog_unwatch_task(p, false); -- -- /* acquire ensures that we see the preceding updates on QUEUED */ -- opss = atomic64_read_acquire(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_NONE: -- break; -- case SCX_OPSS_QUEUEING: -- /* -- * QUEUEING is started and finished while holding @p's rq lock. -- * As we're holding the rq lock now, we shouldn't see QUEUEING. -- */ -- BUG(); -- case SCX_OPSS_QUEUED: -- if (SCX_HAS_OP(dequeue)) -- SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); -- -- if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_NONE)) -- break; -- fallthrough; -- case SCX_OPSS_DISPATCHING: -- /* -- * If @p is being dispatched from the BPF scheduler to a DSQ, -- * wait for the transfer to complete so that @p doesn't get -- * added to its DSQ after dequeueing is complete. -- * -- * As we're waiting on DISPATCHING with the rq locked, the -- * dispatching side shouldn't try to lock the rq while -- * DISPATCHING is set. See dispatch_to_local_dsq(). -- * -- * DISPATCHING shouldn't have qseq set and control can reach -- * here with NONE @opss from the above QUEUED case block. -- * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. -- */ -- wait_ops_state(p, SCX_OPSS_DISPATCHING); -- BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- break; -- } --} -- --static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) --{ -- struct scx_rq *scx_rq = rq->scx; -- -- if (!(p->scx->flags & SCX_TASK_QUEUED)) { -- WARN_ON_ONCE(watchdog_task_watched(p)); -- return; -- } -- -- ops_dequeue(p, deq_flags); -- -- /* -- * A currently running task which is going off @rq first gets dequeued -- * and then stops running. As we want running <-> stopping transitions -- * to be contained within runnable <-> quiescent transitions, trigger -- * ->stopping() early here instead of in put_prev_task_scx(). -- * -- * @p may go through multiple stopping <-> running transitions between -- * here and put_prev_task_scx() if task attribute changes occur while -- * balance_scx() leaves @rq unlocked. However, they don't contain any -- * information meaningful to the BPF scheduler and can be suppressed by -- * skipping the callbacks if the task is !QUEUED. -- */ -- if (SCX_HAS_OP(stopping) && task_current(rq, p)) { -- update_curr_scx(rq); -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); -- } -- -- //if(task_current(rq, p)) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- if (SCX_HAS_OP(quiescent)) -- SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); -- -- if (deq_flags & SCX_DEQ_SLEEP) -- p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP; -- else -- p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP; -- -- p->scx->flags &= ~SCX_TASK_QUEUED; -- BUG_ON(!scx_rq->nr_running); -- scx_rq->nr_running--; -- sub_nr_running(rq, 1); -- -- dispatch_dequeue(scx_rq, p); --} -- --static void yield_task_scx(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); -- else -- p->scx->slice = 0; --} -- --static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) --{ -- struct task_struct *from = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); -- else -- return false; --} -- --#ifdef CONFIG_SMP --/** -- * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -- * @rq: rq to move the task into, currently locked -- * @p: task to move -- * @enq_flags: %SCX_ENQ_* -- * -- * Move @p which is currently on a different rq to @rq's local DSQ. The caller -- * must: -- * -- * 1. Start with exclusive access to @p either through its DSQ lock or -- * %SCX_OPSS_DISPATCHING flag. -- * -- * 2. Set @p->scx->holding_cpu to raw_smp_processor_id(). -- * -- * 3. Remember task_rq(@p). Release the exclusive access so that we don't -- * deadlock with dequeue. -- * -- * 4. Lock @rq and the task_rq from #3. -- * -- * 5. Call this function. -- * -- * Returns %true if @p was successfully moved. %false after racing dequeue and -- * losing. -- */ --static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -- u64 enq_flags) --{ -- struct rq *task_rq; -- -- lockdep_assert_rq_held(rq); -- -- /* -- * If dequeue got to @p while we were trying to lock both rq's, it'd -- * have cleared @p->scx->holding_cpu to -1. While other cpus may have -- * updated it to different values afterwards, as this operation can't be -- * preempted or recurse, @p->scx->holding_cpu can never become -- * raw_smp_processor_id() again before we're done. Thus, we can tell -- * whether we lost to dequeue by testing whether @p->scx->holding_cpu is -- * still raw_smp_processor_id(). -- * -- * See dispatch_dequeue() for the counterpart. -- */ -- if (unlikely(p->scx->holding_cpu != raw_smp_processor_id())) -- return false; -- -- /* @p->rq couldn't have changed if we're still the holding cpu */ -- task_rq = task_rq(p); -- lockdep_assert_rq_held(task_rq); -- -- WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)); -- deactivate_task(task_rq, p, 0); -- set_task_cpu(p, cpu_of(rq)); -- p->scx->sticky_cpu = cpu_of(rq); -- -- /* -- * We want to pass scx-specific enq_flags but activate_task() will -- * truncate the upper 32 bit. As we own @rq, we can pass them through -- * @rq->scx->extra_enq_flags instead. -- */ -- WARN_ON_ONCE(rq->scx->extra_enq_flags); -- rq->scx->extra_enq_flags = enq_flags; -- activate_task(rq, p, 0); -- rq->scx->extra_enq_flags = 0; -- -- return true; --} -- --/** -- * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * We're holding @rq lock and trying to dispatch a task from @src_rq to -- * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether -- * @rq stays locked isn't important as long as the state is restored after -- * dispatch_to_local_dsq_unlock(). -- */ --static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- rq_unpin_lock(rq, rf); -- -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(rq); -- raw_spin_rq_lock(dst_rq); -- } else if (rq == src_rq) { -- double_lock_balance(rq, dst_rq); -- rq_repin_lock(rq, rf); -- } else if (rq == dst_rq) { -- double_lock_balance(rq, src_rq); -- rq_repin_lock(rq, rf); -- } else { -- raw_spin_rq_unlock(rq); -- double_rq_lock(src_rq, dst_rq); -- } --} -- --/** -- * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock() -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return. -- */ --static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } else if (rq == src_rq) { -- double_unlock_balance(rq, dst_rq); -- } else if (rq == dst_rq) { -- double_unlock_balance(rq, src_rq); -- } else { -- double_rq_unlock(src_rq, dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } --} --#endif /* CONFIG_SMP */ -- -- --static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq) --{ -- return likely(test_rq_online(rq)) && !is_migration_disabled(p) && -- cpumask_test_cpu(cpu_of(rq), p->cpus_ptr); --} -- --static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -- struct scx_dispatch_q *dsq) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rb_node *rb_node; -- struct rq *task_rq; -- bool moved = false; --retry: -- if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -- return false; -- -- raw_spin_lock(&dsq->lock); -- -- list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- for (rb_node = rb_first_cached(&dsq->priq); rb_node; -- rb_node = rb_next(rb_node)) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- raw_spin_unlock(&dsq->lock); -- return false; -- --this_rq: -- /* @dsq is locked and @p is on this rq */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo); -- dsq->nr--; -- scx_rq->local_dsq.nr++; -- p->scx->dsq = &scx_rq->local_dsq; -- raw_spin_unlock(&dsq->lock); -- return true; -- --remote_rq: --#ifdef CONFIG_SMP -- /* -- * @dsq is locked and @p is on a remote rq. @p is currently protected by -- * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -- * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -- * rq lock or fail, do a little dancing from our side. See -- * move_task_to_local_dsq(). -- */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- p->scx->holding_cpu = raw_smp_processor_id(); -- raw_spin_unlock(&dsq->lock); -- -- rq_unpin_lock(rq, rf); -- double_lock_balance(rq, task_rq); -- rq_repin_lock(rq, rf); -- -- moved = move_task_to_local_dsq(rq, p, 0); -- -- double_unlock_balance(rq, task_rq); --#endif /* CONFIG_SMP */ -- if (likely(moved)) -- return true; -- goto retry; --} -- --enum dispatch_to_local_dsq_ret { -- DTL_DISPATCHED, /* successfully dispatched */ -- DTL_LOST, /* lost race to dequeue */ -- DTL_NOT_LOCAL, /* destination is not a local DSQ */ -- DTL_INVALID, /* invalid local dsq_id */ --}; -- --/** -- * dispatch_to_local_dsq - Dispatch a task to a local dsq -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @dsq_id: destination dsq ID -- * @p: task to dispatch -- * @enq_flags: %SCX_ENQ_* -- * -- * We're holding @rq lock and want to dispatch @p to the local DSQ identified by -- * @dsq_id. This function performs all the synchronization dancing needed -- * because local DSQs are protected with rq locks. -- * -- * The caller must have exclusive ownership of @p (e.g. through -- * %SCX_OPSS_DISPATCHING). -- */ --static enum dispatch_to_local_dsq_ret --dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id, -- struct task_struct *p, u64 enq_flags) --{ -- struct rq *src_rq = task_rq(p); -- struct rq *dst_rq; -- -- /* -- * We're synchronized against dequeue through DISPATCHING. As @p can't -- * be dequeued, its task_rq and cpus_allowed are stable too. -- */ -- if (dsq_id == SCX_DSQ_LOCAL) { -- dst_rq = rq; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]", -- cpu, p->comm, p->pid); -- return DTL_INVALID; -- } -- dst_rq = cpu_rq(cpu); -- } else { -- return DTL_NOT_LOCAL; -- } -- -- /* if dispatching to @rq that @p is already on, no lock dancing needed */ -- if (rq == src_rq && rq == dst_rq) { -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags | SCX_ENQ_CLEAR_OPSS); -- return DTL_DISPATCHED; -- } -- --#ifdef CONFIG_SMP -- if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { -- struct rq *locked_dst_rq = dst_rq; -- bool dsp; -- -- /* -- * @p is on a possibly remote @src_rq which we need to lock to -- * move the task. If dequeue is in progress, it'd be locking -- * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq -- * lock while holding DISPATCHING. -- * -- * As DISPATCHING guarantees that @p is wholly ours, we can -- * pretend that we're moving from a DSQ and use the same -- * mechanism - mark the task under transfer with holding_cpu, -- * release DISPATCHING and then follow the same protocol. -- */ -- p->scx->holding_cpu = raw_smp_processor_id(); -- -- /* store_release ensures that dequeue sees the above */ -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq); -- -- /* -- * We don't require the BPF scheduler to avoid dispatching to -- * offline CPUs mostly for convenience but also because CPUs can -- * go offline between scx_bpf_dispatch() calls and here. If @p -- * is destined to an offline CPU, queue it on its current CPU -- * instead, which should always be safe. As this is an allowed -- * behavior, don't trigger an ops error. -- */ -- if (unlikely(!test_rq_online(dst_rq))) -- dst_rq = src_rq; -- -- if (src_rq == dst_rq) { -- /* -- * As @p is staying on the same rq, there's no need to -- * go through the full deactivate/activate cycle. -- * Optimize by abbreviating the operations in -- * move_task_to_local_dsq(). -- */ -- dsp = p->scx->holding_cpu == raw_smp_processor_id(); -- if (likely(dsp)) { -- p->scx->holding_cpu = -1; -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags); -- } -- } else { -- dsp = move_task_to_local_dsq(dst_rq, p, enq_flags); -- } -- -- /* if the destination CPU is idle, wake it up */ -- if (dsp && p->sched_class > dst_rq->curr->sched_class) -- resched_curr(dst_rq); -- -- dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq); -- -- return dsp ? DTL_DISPATCHED : DTL_LOST; -- } --#endif /* CONFIG_SMP */ -- -- scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", -- cpu_of(dst_rq), p->comm, p->pid); -- return DTL_INVALID; --} -- --/** -- * finish_dispatch - Asynchronously finish dispatching a task -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @p: task to finish dispatching -- * @qseq_at_dispatch: qseq when @p started getting dispatched -- * @dsq_id: destination DSQ ID -- * @enq_flags: %SCX_ENQ_* -- * -- * Dispatching to local DSQs may need to wait for queueing to complete or -- * require rq lock dancing. As we don't wanna do either while inside -- * ops.dispatch() to avoid locking order inversion, we split dispatching into -- * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the -- * task and its qseq. Once ops.dispatch() returns, this function is called to -- * finish up. -- * -- * There is no guarantee that @p is still valid for dispatching or even that it -- * was valid in the first place. Make sure that the task is still owned by the -- * BPF scheduler and claim the ownership before dispatching. -- */ --static void finish_dispatch(struct rq *rq, struct rq_flags *rf, -- struct task_struct *p, u64 qseq_at_dispatch, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- u64 opss; -- -- touch_core_sched_dispatch(rq, p); --retry: -- /* -- * No need for _acquire here. @p is accessed only after a successful -- * try_cmpxchg to DISPATCHING. -- */ -- opss = atomic64_read(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_DISPATCHING: -- case SCX_OPSS_NONE: -- /* someone else already got to it */ -- return; -- case SCX_OPSS_QUEUED: -- /* -- * If qseq doesn't match, @p has gone through at least one -- * dispatch/dequeue and re-enqueue cycle between -- * scx_bpf_dispatch() and here and we have no claim on it. -- */ -- if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) -- return; -- -- /* -- * While we know @p is accessible, we don't yet have a claim on -- * it - the BPF scheduler is allowed to dispatch tasks -- * spuriously and there can be a racing dequeue attempt. Let's -- * claim @p by atomically transitioning it from QUEUED to -- * DISPATCHING. -- */ -- if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_DISPATCHING))) -- break; -- goto retry; -- case SCX_OPSS_QUEUEING: -- /* -- * do_enqueue_task() is in the process of transferring the task -- * to the BPF scheduler while holding @p's rq lock. As we aren't -- * holding any kernel or BPF resource that the enqueue path may -- * depend upon, it's safe to wait. -- */ -- wait_ops_state(p, opss); -- goto retry; -- } -- -- BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) { -- case DTL_DISPATCHED: -- break; -- case DTL_LOST: -- break; -- case DTL_INVALID: -- dsq_id = SCX_DSQ_GLOBAL; -- fallthrough; -- case DTL_NOT_LOCAL: -- dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()), -- dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- break; -- } --} -- --static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- u32 u; -- -- for (u = 0; u < dspc->buf_cursor; u++) { -- struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u]; -- -- finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id, -- ent->enq_flags); -- } -- -- dspc->nr_tasks += dspc->buf_cursor; -- dspc->buf_cursor = 0; --} -- --static int balance_one(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf, bool local) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- bool prev_on_scx = prev->sched_class == &ext_sched_class; -- int nr_loops = SCX_DSP_MAX_LOOPS; -- -- lockdep_assert_rq_held(rq); -- -- if (static_branch_unlikely(&scx_ops_cpu_preempt) && -- unlikely(rq->scx->cpu_released)) { -- /* -- * If the previous sched_class for the current CPU was not SCX, -- * notify the BPF scheduler that it again has control of the -- * core. This callback complements ->cpu_release(), which is -- * emitted in scx_notify_pick_next_task(). -- */ -- if (SCX_HAS_OP(cpu_acquire)) -- SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq), -- NULL); -- rq->scx->cpu_released = false; -- } -- -- if (prev_on_scx) { -- WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP)); -- update_curr_scx(rq); -- -- /* -- * If @prev is runnable & has slice left, it has priority and -- * fetching more just increases latency for the fetched tasks. -- * Tell put_prev_task_scx() to put @prev on local_dsq. If the -- * BPF scheduler wants to handle this explicitly, it should -- * implement ->cpu_released(). -- * -- * See scx_ops_disable_workfn() for the explanation on the -- * disabling() test. -- * -- * When balancing a remote CPU for core-sched, there won't be a -- * following put_prev_task_scx() call and we don't own -- * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the -- * same conditions later and pick @rq->curr accordingly. -- */ -- if ((prev->scx->flags & SCX_TASK_QUEUED) && -- prev->scx->slice && !scx_ops_disabling()) { -- if (local) -- prev->scx->flags |= SCX_TASK_BAL_KEEP; -- return 1; -- } -- } -- -- /* if there already are tasks to run, nothing to do */ -- if (scx_rq->local_dsq.nr) -- return 1; -- -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- if (!SCX_HAS_OP(dispatch)) -- return 0; -- -- dspc->rq = rq; -- dspc->rf = rf; -- -- /* -- * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, -- * the local DSQ might still end up empty after a successful -- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() -- * produced some tasks, retry. The BPF scheduler may depend on this -- * looping behavior to simplify its implementation. -- */ -- do { -- dspc->nr_tasks = 0; -- -- SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), -- prev_on_scx ? prev : NULL); -- -- flush_dispatch_buf(rq, rf); -- -- if (scx_rq->local_dsq.nr) -- return 1; -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- /* -- * ops.dispatch() can trap us in this loop by repeatedly -- * dispatching ineligible tasks. Break out once in a while to -- * allow the watchdog to run. As IRQ can't be enabled in -- * balance(), we want to complete this scheduling cycle and then -- * start a new one. IOW, we want to call resched_curr() on the -- * next, most likely idle, task, not the current one. Use -- * scx_bpf_kick_cpu() for deferred kicking. -- */ -- if (unlikely(!--nr_loops)) { -- scx_bpf_kick_cpu(cpu_of(rq), 0); -- break; -- } -- } while (dspc->nr_tasks); -- -- return 0; --} -- --static int balance_scx(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf) --{ -- int ret; -- -- ret = balance_one(rq, prev, rf, true); --#ifdef CONFIG_SCHED_SMT -- /* -- * When core-sched is enabled, this ops.balance() call will be followed -- * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx() -- * on the SMT siblings. Balance the siblings too. -- */ -- if (sched_core_enabled(rq)) { -- const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); -- int scpu; -- -- for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { -- struct rq *srq = cpu_rq(scpu); -- struct rq_flags srf; -- struct task_struct *sprev = srq->curr; -- -- /* -- * While core-scheduling, rq lock is shared among -- * siblings but the debug annotations and rq clock -- * aren't. Do pinning dance to transfer the ownership. -- */ -- WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); -- rq_unpin_lock(rq, rf); -- rq_pin_lock(srq, &srf); -- -- update_rq_clock(srq); -- balance_one(srq, sprev, &srf, false); -- -- rq_unpin_lock(srq, &srf); -- rq_repin_lock(rq, rf); -- } -- } --#endif -- return ret; --} -- --static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) --{ -- if (p->scx->flags & SCX_TASK_QUEUED) { -- /* -- * Core-sched might decide to execute @p before it is -- * dispatched. Call ops_dequeue() to notify the BPF scheduler. -- */ -- ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); -- dispatch_dequeue(rq->scx, p); -- } -- -- p->se.exec_start = rq_clock_task(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, running, p); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock); -- -- watchdog_unwatch_task(p, true); -- -- /* -- * @p is getting newly scheduled or got kicked after someone updated its -- * slice. Refresh whether tick can be stopped. See can_stop_tick_scx(). -- */ -- if ((p->scx->slice == SCX_SLICE_INF) != -- (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) { -- if (p->scx->slice == SCX_SLICE_INF) -- rq->scx->flags |= SCX_RQ_CAN_STOP_TICK; -- else -- rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK; -- -- sched_update_tick_dependency(rq); -- } --} -- --static void put_prev_task_scx(struct rq *rq, struct task_struct *p) --{ --#ifndef CONFIG_SMP -- /* -- * UP workaround. -- * -- * Because SCX may transfer tasks across CPUs during dispatch, dispatch -- * is performed from its balance operation which isn't called in UP. -- * Let's work around by calling it from the operations which come right -- * after. -- * -- * 1. If the prev task is on SCX, pick_next_task() calls -- * .put_prev_task() right after. As .put_prev_task() is also called -- * from other places, we need to distinguish the calls which can be -- * done by looking at the previous task's state - if still queued or -- * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task(). -- * This case is handled here. -- * -- * 2. If the prev task is not on SCX, the first following call into SCX -- * will be .pick_next_task(), which is covered by calling -- * balance_scx() from pick_next_task_scx(). -- * -- * Note that we can't merge the first case into the second as -- * balance_scx() must be called before the previous SCX task goes -- * through put_prev_task_scx(). -- * -- * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf. -- * Pass in %NULL. -- */ -- if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP)) -- balance_scx(rq, p, NULL); --#endif -- -- update_curr_scx(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- -- /* -- * If we're being called from put_prev_task_balance(), balance_scx() may -- * have decided that @p should keep running. -- */ -- if (p->scx->flags & SCX_TASK_BAL_KEEP) { -- p->scx->flags &= ~SCX_TASK_BAL_KEEP; -- watchdog_watch_task(rq, p); -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- watchdog_watch_task(rq, p); -- -- /* -- * If @p has slice left and balance_scx() didn't tag it for -- * keeping, @p is getting preempted by a higher priority -- * scheduler class or core-sched forcing a different task. Leave -- * it at the head of the local DSQ. -- */ -- if (p->scx->slice && !scx_ops_disabling()) { -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- /* -- * If we're in the pick_next_task path, balance_scx() should -- * have already populated the local DSQ if there are any other -- * available tasks. If empty, tell ops.enqueue() that @p is the -- * only one available for this cpu. ops.enqueue() should put it -- * on the local DSQ so that the subsequent pick_next_task_scx() -- * can find the task unless it wants to trigger a separate -- * follow-up scheduling event. -- */ -- if (list_empty(&rq->scx->local_dsq.fifo)) -- do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1); -- else -- do_enqueue_task(rq, p, 0, -1); -- } --} -- --static struct task_struct *first_local_task(struct rq *rq) --{ -- struct rb_node *rb_node; -- struct sched_ext_entity *entity; -- -- if (!list_empty(&rq->scx->local_dsq.fifo)) { -- entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo); -- return entity->task; -- } -- -- rb_node = rb_first_cached(&rq->scx->local_dsq.priq); -- if (rb_node) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- return entity->task; -- } -- -- return NULL; --} -- --static struct task_struct *pick_next_task_scx(struct rq *rq) --{ -- struct task_struct *p; -- --#ifndef CONFIG_SMP -- /* UP workaround - see the comment at the head of put_prev_task_scx() */ -- if (unlikely(rq->curr->sched_class != &ext_sched_class)) -- balance_scx(rq, rq->curr, NULL); --#endif -- -- p = first_local_task(rq); -- if (!p) -- return NULL; -- -- if (unlikely(!p->scx->slice)) { -- if (!scx_ops_disabling() && !scx_warned_zero_slice) { -- printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n", -- p->comm, p->pid); -- scx_warned_zero_slice = true; -- } -- p->scx->slice = SCX_SLICE_DFL; -- } -- -- set_next_task_scx(rq, p, true); -- -- return p; --} -- --#ifdef CONFIG_SCHED_CORE --/** -- * scx_prio_less - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Core-sched is implemented as an additional scheduling layer on top of the -- * usual sched_class'es and needs to find out the expected task ordering. For -- * SCX, core-sched calls this function to interrogate the task ordering. -- * -- * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used -- * to implement the default task ordering. The older the timestamp, the higher -- * prority the task - the global FIFO ordering matching the default scheduling -- * behavior. -- * -- * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to -- * implement FIFO ordering within each local DSQ. See pick_task_scx(). -- */ --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi) --{ -- /* -- * The const qualifiers are dropped from task_struct pointers when -- * calling ops.core_sched_before(). Accesses are controlled by the -- * verifier. -- */ -- if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling()) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, -- (struct task_struct *)a, -- (struct task_struct *)b); -- else -- return time_after64(a->scx->core_sched_at, b->scx->core_sched_at); --} -- --/** -- * pick_task_scx - Pick a candidate task for core-sched -- * @rq: rq to pick the candidate task from -- * -- * Core-sched calls this function on each SMT sibling to determine the next -- * tasks to run on the SMT siblings. balance_one() has been called on all -- * siblings and put_prev_task_scx() has been called only for the current CPU. -- * -- * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look -- * at the first task in the local dsq. @rq->curr has to be considered explicitly -- * to mimic %SCX_TASK_BAL_KEEP. -- */ --static struct task_struct *pick_task_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- struct task_struct *first = first_local_task(rq); -- -- if (curr->scx->flags & SCX_TASK_QUEUED) { -- /* is curr the only runnable task? */ -- if (!first) -- return curr; -- -- /* -- * Does curr trump first? We can always go by core_sched_at for -- * this comparison as it represents global FIFO ordering when -- * the default core-sched ordering is used and local-DSQ FIFO -- * ordering otherwise. -- * -- * We can have a task with an earlier timestamp on the DSQ. For -- * example, when a current task is preempted by a sibling -- * picking a different cookie, the task would be requeued at the -- * head of the local DSQ with an earlier timestamp than the -- * core-sched picked next task. Besides, the BPF scheduler may -- * dispatch any tasks to the local DSQ anytime. -- */ -- if (curr->scx->slice && time_before64(curr->scx->core_sched_at, -- first->scx->core_sched_at)) -- return curr; -- } -- -- return first; /* this may be %NULL */ --} --#endif /* CONFIG_SCHED_CORE */ -- --static enum scx_cpu_preempt_reason --preempt_reason_from_class(const struct sched_class *class) --{ --#ifdef CONFIG_SMP -- if (class == &stop_sched_class) -- return SCX_CPU_PREEMPT_STOP; --#endif -- if (class == &dl_sched_class) -- return SCX_CPU_PREEMPT_DL; -- if (class == &rt_sched_class) -- return SCX_CPU_PREEMPT_RT; -- return SCX_CPU_PREEMPT_UNKNOWN; --} -- --void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task, -- const struct sched_class *active) --{ -- lockdep_assert_rq_held(rq); -- -- /* -- * The callback is conceptually meant to convey that the CPU is no -- * longer under the control of SCX. Therefore, don't invoke the -- * callback if the CPU is is staying on SCX, or going idle (in which -- * case the SCX scheduler has actively decided not to schedule any -- * tasks on the CPU). -- */ -- if (likely(active >= &ext_sched_class)) -- return; -- -- /* -- * At this point we know that SCX was preempted by a higher priority -- * sched_class, so invoke the ->cpu_release() callback if we have not -- * done so already. We only send the callback once between SCX being -- * preempted, and it regaining control of the CPU. -- * -- * ->cpu_release() complements ->cpu_acquire(), which is emitted the -- * next time that balance_scx() is invoked. -- */ -- if (!rq->scx->cpu_released) { -- if (SCX_HAS_OP(cpu_release)) { -- struct scx_cpu_release_args args = { -- .reason = preempt_reason_from_class(active), -- .task = task, -- }; -- -- SCX_CALL_OP(SCX_KF_CPU_RELEASE, -- cpu_release, cpu_of(rq), &args); -- } -- rq->scx->cpu_released = true; -- } --} -- --#ifdef CONFIG_SMP -- --static bool test_and_clear_cpu_idle(int cpu) --{ -- if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -- if (cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- return true; -- } else { -- return false; -- } --} -- --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- int cpu; -- -- do { -- cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); -- if (cpu < nr_cpu_ids) { -- const struct cpumask *sbm = topology_sibling_cpumask(cpu); -- -- /* -- * If offline, @cpu is not its own sibling and we can -- * get caught in an infinite loop as @cpu is never -- * cleared from idle_masks.smt. Clear @cpu directly in -- * such cases. -- */ -- if (likely(cpumask_test_cpu(cpu, sbm))) -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm); -- else -- cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu)); -- } else { -- cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -- if (cpu >= nr_cpu_ids) -- return -EBUSY; -- } -- } while (!test_and_clear_cpu_idle(cpu)); -- -- return cpu; --} -- --static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) --{ -- s32 cpu; -- -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return prev_cpu; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local DSQ of the waker. -- */ -- if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 && -- scx_has_idle_cpus && !(current->flags & PF_EXITING)) { -- cpu = smp_processor_id(); -- if (cpumask_test_cpu(cpu, p->cpus_ptr)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (test_and_clear_cpu_idle(prev_cpu)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return prev_cpu; -- } -- -- if (p->nr_cpus_allowed == 1) -- return prev_cpu; -- -- cpu = scx_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- -- return prev_cpu; --} -- --static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) --{ -- if (SCX_HAS_OP(select_cpu)) { -- s32 cpu; -- -- cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu, -- wake_flags); -- if (ops_cpu_valid(cpu)) { -- return cpu; -- } else { -- scx_ops_error("select_cpu returned invalid cpu %d", cpu); -- return prev_cpu; -- } -- } else { -- return scx_select_cpu_dfl(p, prev_cpu, wake_flags); -- } --} -- --static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx) --{ -- set_cpus_allowed_common(p, ctx); -- -- /* -- * The effective cpumask is stored in @p->cpus_ptr which may temporarily -- * differ from the configured one in @p->cpus_mask. Always tell the bpf -- * scheduler the effective one. -- * -- * Fine-grained memory write control is enforced by BPF making the const -- * designation pointless. Cast it away when calling the operation. -- */ -- if (SCX_HAS_OP(set_cpumask)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, -- (struct cpumask *)p->cpus_ptr); --} -- --static void reset_idle_masks(void) --{ -- /* consider all cpus idle, should converge to the actual state quickly */ -- cpumask_setall(idle_masks.cpu); -- cpumask_setall(idle_masks.smt); -- scx_has_idle_cpus = true; --} -- --void __scx_update_idle(struct rq *rq, bool idle) --{ -- int cpu = cpu_of(rq); -- struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -- -- if (SCX_HAS_OP(update_idle)) { -- SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); -- if (!static_branch_unlikely(&scx_builtin_idle_enabled)) -- return; -- } -- -- if (idle) { -- cpumask_set_cpu(cpu, idle_masks.cpu); -- if (!scx_has_idle_cpus) -- scx_has_idle_cpus = true; -- -- /* -- * idle_masks.smt handling is racy but that's fine as it's only -- * for optimization and self-correcting. -- */ -- for_each_cpu(cpu, sib_mask) { -- if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -- return; -- } -- cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -- } else { -- cpumask_clear_cpu(cpu, idle_masks.cpu); -- if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -- } --} -- --#else /* !CONFIG_SMP */ -- --static bool test_and_clear_cpu_idle(int cpu) { return false; } --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } --static void reset_idle_masks(void) {} -- --#endif /* CONFIG_SMP */ -- --static bool check_rq_for_timeouts(struct rq *rq) --{ -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rq_flags rf; -- bool timed_out = false; -- -- rq_lock_irqsave(rq, &rf); -- list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) { -- unsigned long last_runnable; -- -- p = entity->task; -- last_runnable = p->scx->runnable_at; -- -- if (unlikely(time_after(jiffies, -- last_runnable + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "%s[%d] failed to run for %u.%03us", -- p->comm, p->pid, -- dur_ms / 1000, dur_ms % 1000); -- timed_out = true; -- break; -- } -- } -- rq_unlock_irqrestore(rq, &rf); -- -- return timed_out; --} -- --static void scx_watchdog_workfn(struct work_struct *work) --{ -- int cpu; -- -- scx_watchdog_timestamp = jiffies; -- -- for_each_online_cpu(cpu) { -- if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) -- break; -- -- cond_resched(); -- } -- queue_delayed_work(system_unbound_wq, to_delayed_work(work), -- scx_watchdog_timeout / 2); --} -- --static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) --{ -- update_curr_scx(rq); -- //scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock); -- /* -- * While disabling, always resched and refresh core-sched timestamp as -- * we can't trust the slice management or ops.core_sched_before(). -- */ -- if (scx_ops_disabling()) { -- curr->scx->slice = 0; -- touch_core_sched(rq, curr); -- } -- -- if (!curr->scx->slice) -- resched_curr(rq); --} -- --#define SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- --static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg) --{ -- int ret; -- -- WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED); -- -- p->scx->disallow = false; -- -- if (SCX_HAS_OP(prep_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- }; -- -- ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args); -- if (unlikely(ret)) { -- ret = ops_sanitize_err("prep_enable", ret); -- return ret; -- } -- } -- //scx_sched_init_task(p); -- -- if (p->scx->disallow) { -- struct rq *rq; -- struct rq_flags rf; -- -- rq = task_rq_lock(p, &rf); -- -- /* -- * We're either in fork or load path and @p->policy will be -- * applied right after. Reverting @p->policy here and rejecting -- * %SCHED_EXT transitions from scx_check_setscheduler() -- * guarantees that if ops.prep_enable() sets @p->disallow, @p -- * can never be in SCX. -- */ -- if (p->policy == SCHED_EXT) { -- p->policy = SCHED_NORMAL; -- atomic64_inc(&scx_nr_rejected); -- } -- -- task_rq_unlock(rq, p, &rf); -- } -- -- p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET); -- return 0; --} -- --static void scx_ops_enable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED)); -- -- if (SCX_HAS_OP(enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- p->scx->flags |= SCX_TASK_OPS_ENABLED; --} -- --static void scx_ops_disable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- if (p->scx->flags & SCX_TASK_OPS_PREPPED) { -- if (SCX_HAS_OP(cancel_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- } else if (p->scx->flags & SCX_TASK_OPS_ENABLED) { -- if (SCX_HAS_OP(disable)) -- SCX_CALL_OP(SCX_KF_REST, disable, p); -- p->scx->flags &= ~SCX_TASK_OPS_ENABLED; -- } --} -- --static void set_task_scx_weight(struct task_struct *p) --{ -- u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -- -- p->scx->weight = sched_weight_to_cgroup(weight); --} -- --/** -- * refresh_scx_weight - Refresh a task's ext weight -- * @p: task to refresh ext weight for -- * -- * @p->scx->weight carries the task's static priority in cgroup weight scale to -- * enable easy access from the BPF scheduler. To keep it synchronized with the -- * current task priority, this function should be called when a new task is -- * created, priority is changed for a task on sched_ext, and a task is switched -- * to sched_ext from other classes. -- */ --static void refresh_scx_weight(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- set_task_scx_weight(p); -- if (SCX_HAS_OP(set_weight)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight); --} -- --void scx_pre_fork(struct task_struct *p) --{ -- p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL); -- if (!p->scx) { -- goto lock; -- } -- -- p->scx->dsq = NULL; -- INIT_LIST_HEAD(&p->scx->dsq_node.fifo); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- INIT_LIST_HEAD(&p->scx->watchdog_node); -- p->scx->flags = 0; -- p->scx->weight = 0; -- p->scx->sticky_cpu = -1; -- p->scx->holding_cpu = -1; -- p->scx->kf_mask = 0; -- atomic64_set(&p->scx->ops_state, 0); -- p->scx->runnable_at = INITIAL_JIFFIES; -- p->scx->slice = SCX_SLICE_DFL; -- p->scx->task = p; -- p->sched_prop = 0; -- -- /* -- * BPF scheduler enable/disable paths want to be able to iterate and -- * update all tasks which can become complex when racing forks. As -- * enable/disable are very cold paths, let's use a percpu_rwsem to -- * exclude forks. -- */ --lock: -- percpu_down_read(&scx_fork_rwsem); --} -- --int scx_fork(struct task_struct *p) --{ -- percpu_rwsem_assert_held(&scx_fork_rwsem); -- -- if (scx_enabled()) -- return scx_ops_prepare_task(p, task_group(p)); -- else -- return 0; --} -- --void scx_post_fork(struct task_struct *p) --{ -- if (scx_enabled()) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- /* -- * Set the weight manually before calling ops.enable() so that -- * the scheduler doesn't see a stale value if they inspect the -- * task struct. We'll invoke ops.set_weight() afterwards, as it -- * would be odd to receive a callback on the task before we -- * tell the scheduler that it's been fully enabled. -- */ -- set_task_scx_weight(p); -- scx_ops_enable_task(p); -- refresh_scx_weight(p); -- task_rq_unlock(rq, p, &rf); -- } -- -- spin_lock_irq(&scx_tasks_lock); -- list_add_tail(&p->scx->tasks_node, &scx_tasks); -- spin_unlock_irq(&scx_tasks_lock); -- -- percpu_up_read(&scx_fork_rwsem); --} -- --void scx_cancel_fork(struct task_struct *p) --{ -- if (scx_enabled()) -- scx_ops_disable_task(p); -- percpu_up_read(&scx_fork_rwsem); --} -- --void sched_ext_free(struct task_struct *p) --{ -- unsigned long flags; -- -- spin_lock_irqsave(&scx_tasks_lock, flags); -- list_del_init(&p->scx->tasks_node); -- spin_unlock_irqrestore(&scx_tasks_lock, flags); -- -- /* -- * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED -> -- * ENABLED transitions can't race us. Disable ops for @p. -- */ -- if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- scx_ops_disable_task(p); -- task_rq_unlock(rq, p, &rf); -- } --} -- --static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) --{ --} -- --static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} --static void switched_to_scx(struct rq *rq, struct task_struct *p) {} -- --int scx_check_setscheduler(struct task_struct *p, int policy) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- /* if disallow, reject transitioning into SCX */ -- if (scx_enabled() && READ_ONCE(p->scx->disallow) && -- p->policy != policy && policy == SCHED_EXT) -- return -EACCES; -- -- return 0; --} -- --#ifdef CONFIG_NO_HZ_FULL --bool scx_can_stop_tick(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (scx_ops_disabling()) -- return false; -- -- if (p->sched_class != &ext_sched_class) -- return true; -- -- /* -- * @rq can dispatch from different DSQs, so we can't tell whether it -- * needs the tick or not by looking at nr_running. Allow stopping ticks -- * iff the BPF scheduler indicated so. See set_next_task_scx(). -- */ -- return rq->scx->flags & SCX_RQ_CAN_STOP_TICK; --} --#endif -- --static inline void scx_cgroup_lock(void) {} --static inline void scx_cgroup_unlock(void) {} -- --/* -- * Omitted operations: -- * -- * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -- * task isn't tied to the CPU at that point. Preemption is implemented by -- * resetting the victim task's slice to 0 and triggering reschedule on the -- * target CPU. -- * -- * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -- * -- * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -- * their current sched_class. Call them directly from sched core instead. -- * -- * - task_woken, switched_from: Unnecessary. -- */ --DEFINE_SCHED_CLASS(ext) = { -- .enqueue_task = enqueue_task_scx, -- .dequeue_task = dequeue_task_scx, -- .yield_task = yield_task_scx, -- .yield_to_task = yield_to_task_scx, -- -- .check_preempt_curr = check_preempt_curr_scx, -- -- .pick_next_task = pick_next_task_scx, -- -- .put_prev_task = put_prev_task_scx, -- .set_next_task = set_next_task_scx, -- --#ifdef CONFIG_SMP -- .balance = balance_scx, -- .select_task_rq = select_task_rq_scx, -- .set_cpus_allowed = set_cpus_allowed_scx, --#endif -- --#ifdef CONFIG_SCHED_CORE -- .pick_task = pick_task_scx, --#endif -- -- .task_tick = task_tick_scx, -- -- .switched_to = switched_to_scx, -- .prio_changed = prio_changed_scx, -- -- .update_curr = update_curr_scx, -- --#ifdef CONFIG_UCLAMP_TASK -- .uclamp_enabled = 0, --#endif --}; -- --static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) --{ -- memset(dsq, 0, sizeof(*dsq)); -- -- raw_spin_lock_init(&dsq->lock); -- INIT_LIST_HEAD(&dsq->fifo); -- dsq->id = dsq_id; --} -- --static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id & SCX_DSQ_FLAG_BUILTIN) -- return ERR_PTR(-EINVAL); -- -- dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node); -- if (!dsq) -- return ERR_PTR(-ENOMEM); -- -- init_dsq(dsq, dsq_id); -- -- return dsq; --} -- --static void free_dsq_irq_workfn(struct irq_work *irq_work) --{ -- struct llist_node *to_free = llist_del_all(&dsqs_to_free); -- struct scx_dispatch_q *dsq, *tmp_dsq; -- -- llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) -- kfree_rcu(dsq, rcu); --} -- --static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); -- --static void destroy_dsq(u64 dsq_id) --{ --} -- --static void scx_cgroup_exit(void) {} --static int scx_cgroup_init(void) { return 0; } --static void scx_cgroup_config_knobs(void) {} -- --/* -- * Used by sched_fork() and __setscheduler_prio() to pick the matching -- * sched_class. dl/rt are already handled. -- */ --bool task_on_scx(struct task_struct *p) --{ -- if (!scx_enabled() || scx_ops_disabling()) -- return false; -- if (READ_ONCE(scx_switching_all)) -- return true; -- return p->policy == SCHED_EXT; --} -- --static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags) --{ -- if (enq_flags & SCX_ENQ_LAST) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); --} -- --static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {} -- --static void scx_ops_disable_workfn(struct kthread_work *work) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- struct scx_task_iter sti; -- struct task_struct *p; -- const char *reason; -- int i, cpu, type; -- -- type = atomic_read(&scx_exit_type); -- while (true) { -- /* -- * NONE indicates that a new scx_ops has been registered since -- * disable was scheduled - don't kill the new ops. DONE -- * indicates that the ops has already been disabled. -- */ -- if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE) -- return; -- if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE)) -- break; -- } -- -- cancel_delayed_work_sync(&scx_watchdog_work); -- -- switch (type) { -- case SCX_EXIT_UNREG: -- reason = "BPF scheduler unregistered"; -- break; -- case SCX_EXIT_SYSRQ: -- reason = "disabled by sysrq-S"; -- break; -- case SCX_EXIT_ERROR: -- reason = "runtime error"; -- break; -- case SCX_EXIT_ERROR_BPF: -- reason = "scx_bpf_error"; -- break; -- case SCX_EXIT_ERROR_STALL: -- reason = "runnable task stall"; -- break; -- default: -- reason = ""; -- } -- -- ei->type = type; -- strlcpy(ei->reason, reason, sizeof(ei->reason)); -- -- switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { -- case SCX_OPS_DISABLED: -- pr_warn("sched_ext: ops error detected without ops (%s)\n", -- scx_exit_info.msg); -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- return; -- case SCX_OPS_PREPPING: -- goto forward_progress_guaranteed; -- case SCX_OPS_DISABLING: -- /* shouldn't happen but handle it like ENABLING if it does */ -- WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); -- fallthrough; -- case SCX_OPS_ENABLING: -- case SCX_OPS_ENABLED: -- break; -- } -- -- /* -- * DISABLING is set and ops was either ENABLING or ENABLED indicating -- * that the ops and static branches are set. -- * -- * We must guarantee that all runnable tasks make forward progress -- * without trusting the BPF scheduler. We can't grab any mutexes or -- * rwsems as they might be held by tasks that the BPF scheduler is -- * forgetting to run, which unfortunately also excludes toggling the -- * static branches. -- * -- * Let's work around by overriding a couple ops and modifying behaviors -- * based on the DISABLING state and then cycling the tasks through -- * dequeue/enqueue to force global FIFO scheduling. -- * -- * a. ops.enqueue() and .dispatch() are overridden for simple global -- * FIFO scheduling. -- * -- * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value -- * can't be trusted. Whenever a tick triggers, the running task is -- * rotated to the tail of the queue with core_sched_at touched. -- * -- * c. pick_next_task() suppresses zero slice warning. -- * -- * d. scx_prio_less() reverts to the default core_sched_at order. -- */ -- scx_ops.enqueue = scx_ops_fallback_enqueue; -- scx_ops.dispatch = scx_ops_fallback_dispatch; -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- SCHED_CHANGE_BLOCK(task_rq(p), p, -- DEQUEUE_SAVE | DEQUEUE_MOVE) { -- /* cycling deq/enq is enough, see above */ -- } -- } -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* kick all CPUs to restore ticks */ -- for_each_possible_cpu(cpu) -- resched_cpu(cpu); -- --forward_progress_guaranteed: -- /* -- * Here, every runnable task is guaranteed to make forward progress and -- * we can safely use blocking synchronization constructs. Actually -- * disable ops. -- */ -- mutex_lock(&scx_ops_enable_mutex); -- -- static_branch_disable(&__scx_switched_all); -- WRITE_ONCE(scx_switching_all, false); -- -- /* avoid racing against fork and cgroup changes */ -- cpus_read_lock(); -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- bool alive = READ_ONCE(p->__state) != TASK_DEAD; -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL); -- -- __setscheduler_prio(p, p->prio); -- } -- -- if (alive) -- check_class_changed(task_rq(p), p, old_class, p->prio); -- -- scx_ops_disable_task(p); -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* no task is on scx, turn off all the switches and flush in-progress calls */ -- static_branch_disable_cpuslocked(&__scx_ops_enabled); -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- static_branch_disable_cpuslocked(&scx_has_op[i]); -- static_branch_disable_cpuslocked(&scx_ops_enq_last); -- static_branch_disable_cpuslocked(&scx_ops_enq_exiting); -- static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- synchronize_rcu(); -- -- scx_cgroup_exit(); -- -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- cpus_read_unlock(); -- -- if (ei->type >= SCX_EXIT_ERROR) { -- printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name); -- -- if (ei->msg[0] == '\0') -- printk(KERN_ERR "sched_ext: %s\n", ei->reason); -- else -- printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg); -- -- stack_trace_print(ei->bt, ei->bt_len, 2); -- } -- -- if (scx_ops.exit) -- SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); -- -- memset(&scx_ops, 0, sizeof(scx_ops)); -- -- free_percpu(scx_dsp_buf); -- scx_dsp_buf = NULL; -- scx_dsp_max_batch = 0; -- -- mutex_unlock(&scx_ops_enable_mutex); -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- -- scx_cgroup_config_knobs(); --} -- --static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); -- --static void schedule_scx_ops_disable_work(void) --{ -- struct kthread_worker *helper = READ_ONCE(scx_ops_helper); -- -- /* -- * We may be called spuriously before the first bpf_sched_ext_reg(). If -- * scx_ops_helper isn't set up yet, there's nothing to do. -- */ -- if (helper) -- kthread_queue_work(helper, &scx_ops_disable_work); --} -- --static void scx_ops_disable(enum scx_exit_type type) --{ -- int none = SCX_EXIT_NONE; -- -- if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)) -- type = SCX_EXIT_ERROR; -- -- atomic_try_cmpxchg(&scx_exit_type, &none, type); -- -- schedule_scx_ops_disable_work(); --} -- --static void scx_ops_error_irq_workfn(struct irq_work *irq_work) --{ -- schedule_scx_ops_disable_work(); --} -- --static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- int none = SCX_EXIT_NONE; -- va_list args; -- -- if (!atomic_try_cmpxchg(&scx_exit_type, &none, type)) -- return; -- -- ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1); -- -- va_start(args, fmt); -- vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args); -- va_end(args); -- -- irq_work_queue(&scx_ops_error_irq_work); --} -- --static struct kthread_worker *scx_create_rt_helper(const char *name) --{ -- struct kthread_worker *helper; -- -- helper = kthread_create_worker(0, name); -- if (helper) -- sched_set_fifo(helper->task); -- return helper; --} -- --static int scx_ops_enable(struct sched_ext_ops *ops) --{ -- struct scx_task_iter sti; -- struct task_struct *p; -- int i, ret; -- int tcnt = 0; -- unsigned long long start = 0; -- unsigned long long total_start = sched_clock(); -- -- mutex_lock(&scx_ops_enable_mutex); -- -- if (!scx_ops_helper) { -- WRITE_ONCE(scx_ops_helper, -- scx_create_rt_helper("sched_ext_ops_helper")); -- if (!scx_ops_helper) { -- ret = -ENOMEM; -- goto err_unlock; -- } -- } -- -- if (scx_ops_enable_state() != SCX_OPS_DISABLED) { -- ret = -EBUSY; -- goto err_unlock; -- } -- -- //slim_walt_enable(true); -- -- /* -- * Set scx_ops, transition to PREPPING and clear exit info to arm the -- * disable path. Failure triggers full disabling from here on. -- */ -- scx_ops = *ops; -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != -- SCX_OPS_DISABLED); -- -- memset(&scx_exit_info, 0, sizeof(scx_exit_info)); -- atomic_set(&scx_exit_type, SCX_EXIT_NONE); -- scx_warned_zero_slice = false; -- -- atomic64_set(&scx_nr_rejected, 0); -- -- /* -- * Keep CPUs stable during enable so that the BPF scheduler can track -- * online CPUs by watching ->on/offline_cpu() after ->init(). -- */ -- cpus_read_lock(); -- -- scx_switch_all_req = true; -- if (scx_ops.init) { -- ret = SCX_CALL_OP_RET(SCX_KF_INIT, init); -- if (ret) { -- ret = ops_sanitize_err("init", ret); -- goto err_disable; -- } -- -- /* -- * Exit early if ops.init() triggered scx_bpf_error(). Not -- * strictly necessary as we'll fail transitioning into ENABLING -- * later but that'd be after calling ops.prep_enable() on all -- * tasks and with -EBUSY which isn't very intuitive. Let's exit -- * early with success so that the condition is notified through -- * ops.exit() like other scx_bpf_error() invocations. -- */ -- if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE) -- goto err_disable; -- } -- -- WARN_ON_ONCE(scx_dsp_buf); -- scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; -- scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch, -- __alignof__(scx_dsp_buf[0])); -- if (!scx_dsp_buf) { -- ret = -ENOMEM; -- goto err_disable; -- } -- -- scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; -- if (ops->timeout_ms) -- scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); -- -- scx_watchdog_timestamp = jiffies; -- queue_delayed_work(system_unbound_wq, &scx_watchdog_work, -- scx_watchdog_timeout / 2); -- -- /* -- * Lock out forks, cgroup on/offlining and moves before opening the -- * floodgate so that they don't wander into the operations prematurely. -- */ -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- if (((void (**)(void))ops)[i]) -- static_branch_enable_cpuslocked(&scx_has_op[i]); -- -- if (ops->flags & SCX_OPS_ENQ_LAST) -- static_branch_enable_cpuslocked(&scx_ops_enq_last); -- -- if (ops->flags & SCX_OPS_ENQ_EXITING) -- static_branch_enable_cpuslocked(&scx_ops_enq_exiting); -- if (scx_ops.cpu_acquire || scx_ops.cpu_release) -- static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); -- -- if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { -- reset_idle_masks(); -- static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); -- } else { -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- } -- -- /* -- * All cgroups should be initialized before letting in tasks. cgroup -- * on/offlining and task migrations are already locked out. -- */ -- ret = scx_cgroup_init(); -- if (ret) -- goto err_disable_unlock; -- -- static_branch_enable_cpuslocked(&__scx_ops_enabled); -- -- /* -- * Enable ops for every task. Fork is excluded by scx_fork_rwsem -- * preventing new tasks from being added. No need to exclude tasks -- * leaving as sched_ext_free() can handle both prepped and enabled -- * tasks. Prep all tasks first and then enable them with preemption -- * disabled. -- */ -- spin_lock_irq(&scx_tasks_lock); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered(&sti))) { -- get_task_struct(p); -- spin_unlock_irq(&scx_tasks_lock); -- -- ret = scx_ops_prepare_task(p, task_group(p)); -- if (ret) { -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n", -- ret, p->comm, p->pid); -- goto err_disable_unlock; -- } -- -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- } -- scx_task_iter_exit(&sti); -- -- /* -- * All tasks are prepped but are still ops-disabled. Ensure that -- * %current can't be scheduled out and switch everyone. -- * preempt_disable() is necessary because we can't guarantee that -- * %current won't be starved if scheduled out while switching. -- */ -- preempt_disable(); -- -- /* -- * From here on, the disable path must assume that tasks have ops -- * enabled and need to be recovered. -- */ -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { -- preempt_enable(); -- spin_unlock_irq(&scx_tasks_lock); -- ret = -EBUSY; -- goto err_disable_unlock; -- } -- -- /* -- * We're fully committed and can't fail. The PREPPED -> ENABLED -- * transitions here are synchronized against sched_ext_free() through -- * scx_tasks_lock. -- */ -- WRITE_ONCE(scx_switching_all, scx_switch_all_req); -- start = sched_clock(); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- tcnt++; -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- scx_ops_enable_task(p); -- __setscheduler_prio(p, p->prio); -- } -- -- check_class_changed(task_rq(p), p, old_class, p->prio); -- } else { -- scx_ops_disable_task(p); -- } -- } -- printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id()); -- scx_task_iter_exit(&sti); -- -- spin_unlock_irq(&scx_tasks_lock); -- preempt_enable(); -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { -- ret = -EBUSY; -- goto err_disable; -- } -- -- if (scx_switch_all_req) -- static_branch_enable_cpuslocked(&__scx_switched_all); -- -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- -- scx_cgroup_config_knobs(); -- printk("\n total duration = %llu , current cpu = %d\n", sched_clock() - total_start, smp_processor_id()); -- -- return 0; -- --err_unlock: -- mutex_unlock(&scx_ops_enable_mutex); -- return ret; -- --err_disable_unlock: -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); --err_disable: -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- /* must be fully disabled before returning */ -- scx_ops_disable(SCX_EXIT_ERROR); -- kthread_flush_work(&scx_ops_disable_work); -- return ret; --} -- --#ifdef CONFIG_SCHED_DEBUG --static const char *scx_ops_enable_state_str[] = { -- [SCX_OPS_PREPPING] = "prepping", -- [SCX_OPS_ENABLING] = "enabling", -- [SCX_OPS_ENABLED] = "enabled", -- [SCX_OPS_DISABLING] = "disabling", -- [SCX_OPS_DISABLED] = "disabled", --}; -- --static int scx_debug_show(struct seq_file *m, void *v) --{ -- mutex_lock(&scx_ops_enable_mutex); -- seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name); -- seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled()); -- seq_printf(m, "%-30s: %d\n", "switching_all", -- READ_ONCE(scx_switching_all)); -- seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all()); -- seq_printf(m, "%-30s: %s\n", "enable_state", -- scx_ops_enable_state_str[scx_ops_enable_state()]); -- seq_printf(m, "%-30s: %llu\n", "nr_rejected", -- atomic64_read(&scx_nr_rejected)); -- mutex_unlock(&scx_ops_enable_mutex); -- return 0; --} -- --static int scx_debug_open(struct inode *inode, struct file *file) --{ -- return single_open(file, scx_debug_show, NULL); --} -- --const struct file_operations sched_ext_fops = { -- .open = scx_debug_open, -- .read = seq_read, -- .llseek = seq_lseek, -- .release = single_release, --}; --#endif -- --/******************************************************************************** -- * bpf_struct_ops plumbing. -- */ --#include --#include --#include -- --extern struct btf *btf_vmlinux; --static const struct btf_type *task_struct_type; -- --static bool bpf_scx_is_valid_access(int off, int size, -- enum bpf_access_type type, -- const struct bpf_prog *prog, -- struct bpf_insn_access_aux *info) --{ -- if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) -- return false; -- if (type != BPF_READ) -- return false; -- if (off % size != 0) -- return false; -- -- return btf_ctx_access(off, size, type, prog, info); --} -- --static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, -- const struct bpf_reg_state *reg, -- int off, int size) --{ -- return 0; --} -- --static const struct bpf_func_proto * --bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) --{ -- switch (func_id) { -- case BPF_FUNC_task_storage_get: -- return &bpf_task_storage_get_proto; -- case BPF_FUNC_task_storage_delete: -- return &bpf_task_storage_delete_proto; -- default: -- return bpf_base_func_proto(func_id); -- } --} -- --const struct bpf_verifier_ops bpf_scx_verifier_ops = { -- .get_func_proto = bpf_scx_get_func_proto, -- .is_valid_access = bpf_scx_is_valid_access, -- .btf_struct_access = bpf_scx_btf_struct_access, --}; -- --static int bpf_scx_init_member(const struct btf_type *t, -- const struct btf_member *member, -- void *kdata, const void *udata) --{ -- const struct sched_ext_ops *uops = udata; -- struct sched_ext_ops *ops = kdata; -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- int ret; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, dispatch_max_batch): -- if (*(u32 *)(udata + moff) > INT_MAX) -- return -E2BIG; -- ops->dispatch_max_batch = *(u32 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, flags): -- if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) -- return -EINVAL; -- ops->flags = *(u64 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, name): -- ret = bpf_obj_name_cpy(ops->name, uops->name, -- sizeof(ops->name)); -- if (ret < 0) -- return ret; -- if (ret == 0) -- return -EINVAL; -- return 1; -- case offsetof(struct sched_ext_ops, timeout_ms): -- if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT) -- return -E2BIG; -- ops->timeout_ms = *(u32 *)(udata + moff); -- return 1; -- } -- -- return 0; --} -- --static int bpf_scx_check_member(const struct btf_type *t, -- const struct btf_member *member, -- const struct bpf_prog *prog) --{ -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, prep_enable): -- case offsetof(struct sched_ext_ops, init): -- case offsetof(struct sched_ext_ops, exit): -- break; -- default: -- /* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */ -- /* if (prog->aux->sleepable) -- return -EINVAL; */ -- break; -- } -- -- return 0; --} -- --static int bpf_scx_reg(void *kdata) --{ -- return scx_ops_enable(kdata); --} -- --static void bpf_scx_unreg(void *kdata) --{ -- scx_ops_disable(SCX_EXIT_UNREG); -- kthread_flush_work(&scx_ops_disable_work); --} -- --static int bpf_scx_init(struct btf *btf) --{ -- u32 type_id; -- -- type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); -- if (type_id < 0) -- return -EINVAL; -- task_struct_type = btf_type_by_id(btf, type_id); -- -- return 0; --} -- --/* "extern" to avoid sparse warning, only used in this file */ --extern struct bpf_struct_ops bpf_sched_ext_ops; -- --struct bpf_struct_ops bpf_sched_ext_ops = { -- .verifier_ops = &bpf_scx_verifier_ops, -- .reg = bpf_scx_reg, -- .unreg = bpf_scx_unreg, -- .check_member = bpf_scx_check_member, -- .init_member = bpf_scx_init_member, -- .init = bpf_scx_init, -- .name = "sched_ext_ops", --}; -- --static void sysrq_handle_sched_ext_reset(u8 key) --{ -- if (scx_ops_helper) -- scx_ops_disable(SCX_EXIT_SYSRQ); -- else -- pr_info("sched_ext: BPF scheduler not yet used\n"); --} -- --static const struct sysrq_key_op sysrq_sched_ext_reset_op = { -- .handler = sysrq_handle_sched_ext_reset, -- .help_msg = "reset-sched-ext(S)", -- .action_msg = "Disable sched_ext and revert all tasks to CFS", -- .enable_mask = SYSRQ_ENABLE_RTNICE, --}; -- --static void kick_cpus_irq_workfn(struct irq_work *irq_work) --{ -- struct rq *this_rq = this_rq(); -- u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); -- int this_cpu = cpu_of(this_rq); -- int cpu; -- -- for_each_cpu(cpu, this_rq->scx->cpus_to_kick) { -- struct rq *rq = cpu_rq(cpu); -- unsigned long flags; -- -- raw_spin_rq_lock_irqsave(rq, flags); -- -- if (cpu_online(cpu) || cpu == this_cpu) { -- if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) && -- rq->curr->sched_class == &ext_sched_class) -- rq->curr->scx->slice = 0; -- pseqs[cpu] = rq->scx->pnt_seq; -- resched_curr(rq); -- } else { -- cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait); -- } -- -- raw_spin_rq_unlock_irqrestore(rq, flags); -- } -- -- for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait, -- cpumask_of(this_cpu)) { -- /* -- * Pairs with smp_store_release() issued by this CPU in -- * scx_notify_pick_next_task() on the resched path. -- * -- * We busy-wait here to guarantee that no other task can be -- * scheduled on our core before the target CPU has entered the -- * resched path. -- */ -- while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu]) -- cpu_relax(); -- } -- -- cpumask_clear(this_rq->scx->cpus_to_kick); -- cpumask_clear(this_rq->scx->cpus_to_preempt); -- cpumask_clear(this_rq->scx->cpus_to_wait); --} -- --void __init init_sched_ext_class(void) --{ -- int cpu; -- u32 v; -- -- /* -- * The following is to prevent the compiler from optimizing out the enum -- * definitions so that BPF scheduler implementations can use them -- * through the generated vmlinux.h. -- */ -- WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | -- SCX_TG_ONLINE | SCX_KICK_PREEMPT); -- -- init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); --#ifdef CONFIG_SMP -- BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -- BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); --#endif -- scx_kick_cpus_pnt_seqs = -- __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * -- num_possible_cpus(), -- __alignof__(scx_kick_cpus_pnt_seqs[0])); -- BUG_ON(!scx_kick_cpus_pnt_seqs); -- -- for_each_possible_cpu(cpu) { -- struct rq *rq = cpu_rq(cpu); -- -- rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL); -- if (rq->scx) -- rq->scx->rq = rq; -- else -- pr_err("fatal error : alloc rq->scx failed!!!\n"); -- -- init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL); -- INIT_LIST_HEAD(&rq->scx->watchdog_list); -- -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL)); -- init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn); -- } -- -- register_sysrq_key('S', &sysrq_sched_ext_reset_op); -- INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); -- scx_cgroup_config_knobs(); --} -- -- --/******************************************************************************** -- * Helpers that can be called from the BPF scheduler. -- */ --#include -- --/* Disables missing prototype warnings for kfuncs */ --__diag_push(); --__diag_ignore_all("-Wmissing-prototypes", -- "Global functions as their definitions will be in vmlinux BTF"); -- --/** -- * scx_bpf_switch_all - Switch all tasks into SCX -- * @into_scx: switch direction -- * -- * If @into_scx is %true, all existing and future non-dl/rt tasks are switched -- * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on -- * SCX. The actual switching is asynchronous. Can be called from ops.init(). -- */ --void scx_bpf_switch_all(void) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return; -- -- scx_switch_all_req = true; --} -- --BTF_SET8_START(scx_kfunc_ids_init) --BTF_ID_FLAGS(func, scx_bpf_switch_all) --BTF_SET8_END(scx_kfunc_ids_init) -- --static const struct btf_kfunc_id_set scx_kfunc_set_init = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_init, --}; -- --/** -- * scx_bpf_create_dsq - Create a custom DSQ -- * @dsq_id: DSQ to create -- * @node: NUMA node to allocate from -- * -- * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(), -- * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move(). -- */ --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return -EINVAL; -- -- if (unlikely(node >= (int)nr_node_ids || -- (node < 0 && node != NUMA_NO_NODE))) -- return -EINVAL; -- return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); --} -- --BTF_SET8_START(scx_kfunc_ids_sleepable) --BTF_ID_FLAGS(func, scx_bpf_create_dsq) --BTF_SET8_END(scx_kfunc_ids_sleepable) -- --static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_sleepable, --}; -- --static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) --{ -- if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) -- return false; -- -- lockdep_assert_irqs_disabled(); -- -- if (unlikely(!p)) { -- scx_ops_error("called with NULL task"); -- return false; -- } -- -- if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { -- scx_ops_error("invalid enq_flags 0x%llx", enq_flags); -- return false; -- } -- -- return true; --} -- --static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) --{ -- struct task_struct *ddsp_task; -- int idx; -- -- ddsp_task = __this_cpu_read(direct_dispatch_task); -- if (ddsp_task) { -- direct_dispatch(ddsp_task, p, dsq_id, enq_flags); -- return; -- } -- -- idx = __this_cpu_read(scx_dsp_ctx.buf_cursor); -- if (unlikely(idx >= scx_dsp_max_batch)) { -- scx_ops_error("dispatch buffer overflow"); -- return; -- } -- -- this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){ -- .task = p, -- .qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK, -- .dsq_id = dsq_id, -- .enq_flags = enq_flags, -- }; -- __this_cpu_inc(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe -- * to call this function spuriously. Can be called from ops.enqueue() and -- * ops.dispatch(). -- * -- * When called from ops.enqueue(), it's for direct dispatch and @p must match -- * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the -- * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be -- * on the target CPU in the first place. -- * -- * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id -- * and this function can be called upto ops.dispatch_max_batch times to dispatch -- * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the -- * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. -- * -- * This function doesn't have any locking restrictions and may be called under -- * BPF locks (in the future when BPF introduces more flexible locking). -- * -- * @p is allowed to run for @slice. The scheduling path is triggered on slice -- * exhaustion. If zero, the current residual slice is maintained. If -- * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with -- * scx_bpf_kick_cpu() to trigger scheduling. -- */ --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- scx_dispatch_commit(p, dsq_id, enq_flags); --} -- --/** -- * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. -- * Tasks queued into the priority queue are ordered by @vtime and always -- * consumed after the tasks in the FIFO queue. All other aspects are identical -- * to scx_bpf_dispatch(). -- * -- * @vtime ordering is according to time_before64() which considers wrapping. A -- * numerically larger vtime may indicate an earlier position in the ordering and -- * vice-versa. -- */ --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 vtime, u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- p->scx->dsq_vtime = vtime; -- -- scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); --} -- --BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime) --BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_enqueue_dispatch, --}; -- --/** -- * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots -- * -- * Can only be called from ops.dispatch(). -- */ --u32 scx_bpf_dispatch_nr_slots(void) --{ -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return 0; -- -- return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ -- * @dsq_id: DSQ to consume -- * -- * Consume a task from the non-local DSQ identified by @dsq_id and transfer it -- * to the current CPU's local DSQ for execution. Can only be called from -- * ops.dispatch(). -- * -- * This function flushes the in-flight dispatches from scx_bpf_dispatch() before -- * trying to consume the specified DSQ. It may also grab rq locks and thus can't -- * be called under any BPF locks. -- * -- * Returns %true if a task has been consumed, %false if there isn't any task to -- * consume. -- */ --bool scx_bpf_consume(u64 dsq_id) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- struct scx_dispatch_q *dsq; -- -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return false; -- -- flush_dispatch_buf(dspc->rq, dspc->rf); -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); -- return false; -- } -- -- if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) { -- /* -- * A successfully consumed task can be dequeued before it starts -- * running while the CPU is trying to migrate other dispatched -- * tasks. Bump nr_tasks to tell balance_scx() to retry on empty -- * local DSQ. -- */ -- dspc->nr_tasks++; -- return true; -- } else { -- return false; -- } --} -- --BTF_SET8_START(scx_kfunc_ids_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) --BTF_ID_FLAGS(func, scx_bpf_consume) --BTF_SET8_END(scx_kfunc_ids_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_dispatch, --}; -- --/** -- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ -- * -- * Iterate over all of the tasks currently enqueued on the local DSQ of the -- * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of -- * processed tasks. Can only be called from ops.cpu_release(). -- */ --u32 scx_bpf_reenqueue_local(void) --{ -- u32 nr_enqueued, i; -- struct rq *rq; -- struct scx_rq *scx_rq; -- -- if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) -- return 0; -- -- rq = cpu_rq(smp_processor_id()); -- lockdep_assert_rq_held(rq); -- scx_rq = rq->scx; -- -- /* -- * Get the number of tasks on the local DSQ before iterating over it to -- * pull off tasks. The enqueue callback below can signal that it wants -- * the task to stay on the local DSQ, and we want to prevent the BPF -- * scheduler from causing us to loop indefinitely. -- */ -- nr_enqueued = scx_rq->local_dsq.nr; -- for (i = 0; i < nr_enqueued; i++) { -- struct task_struct *p; -- -- p = first_local_task(rq); -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- WARN_ON_ONCE(p->scx->holding_cpu != -1); -- dispatch_dequeue(scx_rq, p); -- do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); -- } -- -- return nr_enqueued; --} -- --BTF_SET8_START(scx_kfunc_ids_cpu_release) --BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) --BTF_SET8_END(scx_kfunc_ids_cpu_release) -- --static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_cpu_release, --}; -- --/** -- * scx_bpf_kick_cpu - Trigger reschedule on a CPU -- * @cpu: cpu to kick -- * @flags: SCX_KICK_* flags -- * -- * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or -- * trigger rescheduling on a busy CPU. This can be called from any online -- * scx_ops operation and the actual kicking is performed asynchronously through -- * an irq work. -- */ --void scx_bpf_kick_cpu(s32 cpu, u64 flags) --{ -- struct rq *rq; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d", cpu); -- return; -- } -- -- preempt_disable(); -- rq = this_rq(); -- -- /* -- * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting -- * rq locks. We can probably be smarter and avoid bouncing if called -- * from ops which don't hold a rq lock. -- */ -- cpumask_set_cpu(cpu, rq->scx->cpus_to_kick); -- if (flags & SCX_KICK_PREEMPT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt); -- if (flags & SCX_KICK_WAIT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_wait); -- -- irq_work_queue(&rq->scx->kick_cpus_irq_work); -- preempt_enable(); --} -- --/** -- * scx_bpf_dsq_nr_queued - Return the number of queued tasks -- * @dsq_id: id of the DSQ -- * -- * Return the number of tasks in the DSQ matching @dsq_id. If not found, -- * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops -- * operations. -- */ --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) --{ -- struct scx_dispatch_q *dsq; -- -- lockdep_assert(rcu_read_lock_any_held()); -- -- if (dsq_id == SCX_DSQ_LOCAL) { -- return this_rq()->scx->local_dsq.nr; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (ops_cpu_valid(cpu)) -- return cpu_rq(cpu)->scx->local_dsq.nr; -- } else { -- dsq = find_non_local_dsq(dsq_id); -- if (dsq) -- return dsq->nr; -- } -- return -ENOENT; --} -- --/** -- * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state -- * @cpu: cpu to test and clear idle for -- * -- * Returns %true if @cpu was idle and its idle state was successfully cleared. -- * %false otherwise. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return false; -- } -- -- if (ops_cpu_valid(cpu)) -- return test_and_clear_cpu_idle(cpu); -- else -- return false; --} -- --/** -- * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu -- * @cpus_allowed: Allowed cpumask -- * -- * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked -- * idle cpu number on success. -%EBUSY if no matching cpu was found. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return -EBUSY; -- } -- -- return scx_pick_idle_cpu(cpus_allowed); --} -- --/** -- * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking -- * per-CPU cpumask. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_cpumask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.cpu; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, -- * per-physical-core cpumask. Can be used to determine if an entire physical -- * core is free. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_smtmask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.smt; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to -- * either the percpu, or SMT idle-tracking cpumask. -- */ --void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) --{ -- /* -- * Empty function body because we aren't actually acquiring or -- * releasing a reference to a global idle cpumask, which is read-only -- * in the caller and is never released. The acquire / release semantics -- * here are just used to make the cpumask is a trusted pointer in the -- * caller. -- */ --} -- --struct scx_bpf_error_bstr_bufs { -- u64 data[MAX_BPRINTF_VARARGS]; -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs); -- --/** -- * scx_bpf_error_bstr - Indicate fatal error -- * @fmt: error message format string -- * @data: format string parameters packaged using ___bpf_fill() macro -- * @data__sz: @data len, must end in '__sz' for the verifier -- * -- * Indicate that the BPF scheduler encountered a fatal error and initiate ops -- * disabling. -- */ --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) --{ -- struct scx_bpf_error_bstr_bufs *bufs; -- unsigned long flags; -- struct bpf_bprintf_data args = {}; -- int ret; -- -- local_irq_save(flags); -- bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs); -- -- if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || -- (data__sz && !data)) { -- scx_ops_error("invalid data=%p and data__sz=%u", -- (void *)data, data__sz); -- goto out_restore; -- } -- -- ret = copy_from_kernel_nofault(bufs->data, data, data__sz); -- if (ret) { -- scx_ops_error("failed to read data fields (%d)", ret); -- goto out_restore; -- } -- -- ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args); -- if (ret < 0) { -- scx_ops_error("failed to format prepration (%d)", ret); -- goto out_restore; -- } -- -- ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt, -- args.bin_args); -- bpf_bprintf_cleanup(&args); -- if (ret < 0) { -- scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format", -- fmt, data, data__sz); -- goto out_restore; -- } -- -- scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg); --out_restore: -- local_irq_restore(flags); --} -- --/** -- * scx_bpf_destroy_dsq - Destroy a custom DSQ -- * @dsq_id: DSQ to destroy -- * -- * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with -- * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is -- * empty and no further tasks are dispatched to it. Ignored if called on a DSQ -- * which doesn't exist. Can be called from any online scx_ops operations. -- */ --void scx_bpf_destroy_dsq(u64 dsq_id) --{ -- destroy_dsq(dsq_id); --} -- --/** -- * scx_bpf_task_running - Is task currently running? -- * @p: task of interest -- */ --bool scx_bpf_task_running(const struct task_struct *p) --{ -- return task_rq(p)->curr == p; --} -- --/** -- * scx_bpf_task_cpu - CPU a task is currently associated with -- * @p: task of interest -- */ --s32 scx_bpf_task_cpu(const struct task_struct *p) --{ -- return task_cpu(p); --} -- --/** -- * scx_bpf_task_cgroup - Return the sched cgroup of a task -- * @p: task of interest -- * -- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with -- * from the scheduler's POV. SCX operations should use this function to -- * determine @p's current cgroup as, unlike following @p->cgroups, -- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all -- * rq-locked operations. Can be called on the parameter tasks of rq-locked -- * operations. The restriction guarantees that @p's rq is locked by the caller. -- */ --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) --{ -- struct task_group *tg = p->sched_task_group; -- struct cgroup *cgrp = &cgrp_dfl_root.cgrp; -- -- if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) -- goto out; -- -- /* -- * A task_group may either be a cgroup or an autogroup. In the latter -- * case, @tg->css.cgroup is %NULL. A task_group can't become the other -- * kind once created. -- */ -- if (tg && tg->css.cgroup) -- cgrp = tg->css.cgroup; -- else -- cgrp = &cgrp_dfl_root.cgrp; --out: -- cgroup_get(cgrp); -- return cgrp; --} -- --BTF_SET8_START(scx_kfunc_ids_any) --BTF_ID_FLAGS(func, scx_bpf_kick_cpu) --BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) --BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) --BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu) --BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) --BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) --BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) --BTF_ID_FLAGS(func, scx_bpf_task_running) --BTF_ID_FLAGS(func, scx_bpf_task_cpu) --BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE) --BTF_SET8_END(scx_kfunc_ids_any) -- --static const struct btf_kfunc_id_set scx_kfunc_set_any = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_any, --}; -- --__diag_pop(); -- --/* -- * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set() -- * needs most of the system to be up. -- */ --static int __init register_ext_kfuncs(void) --{ -- int ret; -- -- /* -- * Some kfuncs are context-sensitive and can only be called from -- * specific SCX ops. They are grouped into BTF sets accordingly. -- * Unfortunately, BPF currently doesn't have a way of enforcing such -- * restrictions. Eventually, the verifier should be able to enforce -- * them. For now, register them the same and make each kfunc explicitly -- * check using scx_kf_allowed(). -- */ -- if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_init)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_sleepable)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_enqueue_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_cpu_release)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_any))) { -- pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret); -- return ret; -- } -- -- return 0; --} --__initcall(register_ext_kfuncs); -diff --git a/kernel/sched/ext.h b/kernel/sched/ext.h -deleted file mode 100755 -index fba8ed845f99..000000000000 ---- a/kernel/sched/ext.h -+++ /dev/null -@@ -1,257 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ -- -- --/* -- * Tag marking a kernel function as a kfunc. This is meant to minimize the -- * amount of copy-paste that kfunc authors have to include for correctness so -- * as to avoid issues such as the compiler inlining or eliding either a static -- * kfunc, or a global kfunc in an LTO build. -- */ --#define __bpf_kfunc __used noinline -- --enum scx_wake_flags { -- /* expose select WF_* flags as enums */ -- SCX_WAKE_EXEC = WF_EXEC, -- SCX_WAKE_FORK = WF_FORK, -- SCX_WAKE_TTWU = WF_TTWU, -- SCX_WAKE_SYNC = WF_SYNC, --}; -- --enum scx_enq_flags { -- /* expose select ENQUEUE_* flags as enums */ -- SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, -- SCX_ENQ_HEAD = ENQUEUE_HEAD, -- -- /* high 32bits are SCX specific */ -- -- /* -- * Set the following to trigger preemption when calling -- * scx_bpf_dispatch() with a local dsq as the target. The slice of the -- * current task is cleared to zero and the CPU is kicked into the -- * scheduling path. Implies %SCX_ENQ_HEAD. -- */ -- SCX_ENQ_PREEMPT = 1LLU << 32, -- -- /* -- * The task being enqueued was previously enqueued on the current CPU's -- * %SCX_DSQ_LOCAL, but was removed from it in a call to the -- * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was -- * invoked in a ->cpu_release() callback, and the task is again -- * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the -- * task will not be scheduled on the CPU until at least the next invocation -- * of the ->cpu_acquire() callback. -- */ -- SCX_ENQ_REENQ = 1LLU << 40, -- -- /* -- * The task being enqueued is the only task available for the cpu. By -- * default, ext core keeps executing such tasks but when -- * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with -- * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set. -- * -- * If the BPF scheduler wants to continue executing the task, -- * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately. -- * If the task gets queued on a different dsq or the BPF side, the BPF -- * scheduler is responsible for triggering a follow-up scheduling event. -- * Otherwise, Execution may stall. -- */ -- SCX_ENQ_LAST = 1LLU << 41, -- -- /* -- * A hint indicating that it's advisable to enqueue the task on the -- * local dsq of the currently selected CPU. Currently used by -- * select_cpu_dfl() and together with %SCX_ENQ_LAST. -- */ -- SCX_ENQ_LOCAL = 1LLU << 42, -- -- /* high 8 bits are internal */ -- __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, -- -- SCX_ENQ_CLEAR_OPSS = 1LLU << 56, -- SCX_ENQ_DSQ_PRIQ = 1LLU << 57, --}; -- --enum scx_deq_flags { -- /* expose select DEQUEUE_* flags as enums */ -- SCX_DEQ_SLEEP = DEQUEUE_SLEEP, -- -- /* high 32bits are SCX specific */ -- -- /* -- * The generic core-sched layer decided to execute the task even though -- * it hasn't been dispatched yet. Dequeue from the BPF side. -- */ -- SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, --}; -- --enum scx_tg_flags { -- SCX_TG_ONLINE = 1U << 0, -- SCX_TG_INITED = 1U << 1, --}; -- --enum scx_kick_flags { -- SCX_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -- SCX_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ --}; -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --extern const struct sched_class ext_sched_class; --extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops; --extern const struct file_operations sched_ext_fops; --extern unsigned long scx_watchdog_timeout; --extern unsigned long scx_watchdog_timestamp; -- --DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); --DECLARE_STATIC_KEY_FALSE(__scx_switched_all); --#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) --#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) -- --DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); -- --bool task_on_scx(struct task_struct *p); --void scx_pre_fork(struct task_struct *p); --int scx_fork(struct task_struct *p); --void scx_post_fork(struct task_struct *p); --void scx_cancel_fork(struct task_struct *p); --int scx_check_setscheduler(struct task_struct *p, int policy); --bool scx_can_stop_tick(struct rq *rq); --void init_sched_ext_class(void); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...); --#define scx_ops_error(fmt, args...) \ -- scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args) -- --void __scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active); -- --static inline void scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active) --{ -- if (!scx_enabled()) -- return; --#ifdef CONFIG_SMP -- /* -- * Pairs with the smp_load_acquire() issued by a CPU in -- * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -- * resched. -- */ -- smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1); --#endif -- if (!static_branch_unlikely(&scx_ops_cpu_preempt)) -- return; -- __scx_notify_pick_next_task(rq, p, active); --} -- --//extern void scx_scheduler_tick(void); --static inline void scx_notify_sched_tick(void) --{ -- unsigned long last_check; -- -- if (!scx_enabled()) -- return; -- //scx_scheduler_tick(); -- -- last_check = scx_watchdog_timestamp; -- if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "watchdog failed to check in for %u.%03us", -- dur_ms / 1000, dur_ms % 1000); -- } --} -- --static inline const struct sched_class *next_active_class(const struct sched_class *class) --{ -- class++; -- if (scx_switched_all() && class == &fair_sched_class) -- class++; -- if (!scx_enabled() && class == &ext_sched_class) -- class++; -- return class; --} -- --#define for_active_class_range(class, _from, _to) \ -- for (class = (_from); class != (_to); class = next_active_class(class)) -- --#define for_each_active_class(class) \ -- for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -- --/* -- * SCX requires a balance() call before every pick_next_task() call including -- * when waking up from idle. -- */ --#define for_balance_class_range(class, prev_class, end_class) \ -- for_active_class_range(class, (prev_class) > &ext_sched_class ? \ -- &ext_sched_class : (prev_class), (end_class)) -- --#ifdef CONFIG_SCHED_CORE --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi); --#endif -- --#else /* CONFIG_SCHED_CLASS_EXT */ -- --#define scx_enabled() false --#define scx_switched_all() false -- --static inline void scx_pre_fork(struct task_struct *p) {} --static inline int scx_fork(struct task_struct *p) { return 0; } --static inline void scx_post_fork(struct task_struct *p) {} --static inline void scx_cancel_fork(struct task_struct *p) {} --static inline int scx_check_setscheduler(struct task_struct *p, -- int policy) { return 0; } --static inline bool scx_can_stop_tick(struct rq *rq) { return true; } --static inline void init_sched_ext_class(void) {} --static inline void scx_notify_pick_next_task(struct rq *rq, -- const struct task_struct *p, -- const struct sched_class *active) {} --static inline void scx_notify_sched_tick(void) {} -- --#define for_each_active_class for_each_class --#define for_balance_class_range for_class_range -- --#endif /* CONFIG_SCHED_CLASS_EXT */ -- --#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) --void __scx_update_idle(struct rq *rq, bool idle); -- --static inline void scx_update_idle(struct rq *rq, bool idle) --{ -- if (scx_enabled()) -- __scx_update_idle(rq, idle); --} --#else --static inline void scx_update_idle(struct rq *rq, bool idle) {} --#endif -- --#ifdef CONFIG_CGROUP_SCHED --#ifdef CONFIG_EXT_GROUP_SCHED --int scx_tg_online(struct task_group *tg); --void scx_tg_offline(struct task_group *tg); --int scx_cgroup_can_attach(struct cgroup_taskset *tset); --void scx_move_task(struct task_struct *p); --void scx_cgroup_finish_attach(void); --void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); --void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); --#else /* CONFIG_EXT_GROUP_SCHED */ --static inline int scx_tg_online(struct task_group *tg) { return 0; } --static inline void scx_tg_offline(struct task_group *tg) {} --static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } --static inline void scx_move_task(struct task_struct *p) {} --static inline void scx_cgroup_finish_attach(void) {} --static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} --static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} --#endif /* CONFIG_EXT_GROUP_SCHED */ --#endif /* CONFIG_CGROUP_SCHED */ -diff --git a/kernel/sched/hmbird.h b/kernel/sched/hmbird.h -new file mode 100755 -index 000000000000..d0d84ddee13d ---- /dev/null -+++ b/kernel/sched/hmbird.h -@@ -0,0 +1,343 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#ifndef _HMBIRD_H_ -+#define _HMBIRD_H_ -+ -+/* -+ * Tag marking a kernel function as a kfunc. This is meant to minimize the -+ * amount of copy-paste that kfunc authors have to include for correctness so -+ * as to avoid issues such as the compiler inlining or eliding either a static -+ * kfunc, or a global kfunc in an LTO build. -+ */ -+ -+#define DEBUG_INTERNAL (1 << 0) -+#define DEBUG_INFO_TRACE (1 << 1) -+#define DEBUG_INFO_SYSTRACE (1 << 2) -+ -+#define NUMS_CGROUP_KINDS (512) -+#define SLIM_FOR_SGAME (1 << 0) -+#define SLIM_FOR_GENSHIN (1 << 1) -+ -+#ifdef MAX_TASK_NR -+#define MAX_KEY_THREAD_RECORD ((MAX_TASK_NR + 1) >> 1) -+#else -+#define MAX_KEY_THREAD_RECORD (1 << 3) -+#endif /* MAX_TASK_NR */ -+ -+#define TOP_TASK_SHIFT (8) -+#define TOP_TASK_MAX (1 << TOP_TASK_SHIFT) -+#ifndef TOP_TASK_BITS_MASK -+#define TOP_TASK_BITS_MASK (TOP_TASK_MAX - 1) -+#endif /* TOP_TASK_BITS_MASK */ -+ -+extern struct md_info_t *md_info; -+ -+#define debug_enabled() \ -+ (unlikely(hmbirdcore_debug)) -+ -+#define hmbird_debug(fmt, ...) \ -+ pr_info(":"fmt, ##__VA_ARGS__) -+ -+#define hmbird_err(id, fmt, ...) \ -+do { \ -+ pr_err(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_deferred_err(id, fmt, ...) \ -+do { \ -+ printk_deferred(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \ -+do { \ -+ if (unlikely(cond)) { \ -+ hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+ } \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_info_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_info_trace(fmt, ...) -+#endif -+ -+#define hmbird_info_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+ } \ -+} while (0) -+ -+#define hmbird_output_systrace(fmt, ...) \ -+do { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_internal_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_internal_trace(fmt, ...) -+#endif -+ -+#define hmbird_internal_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) { \ -+ hmbird_output_systrace(fmt, ##__VA_ARGS__); \ -+ } \ -+} while (0) -+ -+enum hmbird_wake_flags { -+ /* expose select WF_* flags as enums */ -+ HMBIRD_WAKE_EXEC = WF_EXEC, -+ HMBIRD_WAKE_FORK = WF_FORK, -+ HMBIRD_WAKE_TTWU = WF_TTWU, -+ HMBIRD_WAKE_SYNC = WF_SYNC, -+}; -+ -+enum hmbird_enq_flags { -+ /* expose select ENQUEUE_* flags as enums */ -+ HMBIRD_ENQ_WAKEUP = ENQUEUE_WAKEUP, -+ HMBIRD_ENQ_HEAD = ENQUEUE_HEAD, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * Set the following to trigger preemption when calling -+ * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the -+ * current task is cleared to zero and the CPU is kicked into the -+ * scheduling path. Implies %HMBIRD_ENQ_HEAD. -+ */ -+ HMBIRD_ENQ_PREEMPT = 1LLU << 32, -+ HMBIRD_ENQ_REENQ = 1LLU << 40, -+ HMBIRD_ENQ_LAST = 1LLU << 41, -+ -+ /* -+ * A hint indicating that it's advisable to enqueue the task on the -+ * local dsq of the currently selected CPU. Currently used by -+ * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST. -+ */ -+ HMBIRD_ENQ_LOCAL = 1LLU << 42, -+ -+ /* high 8 bits are internal */ -+ __HMBIRD_ENQ_INTERNAL_MASK = 0xffLLU << 56, -+ -+ HMBIRD_ENQ_CLEAR_OPSS = 1LLU << 56, -+ HMBIRD_ENQ_DSQ_PRIQ = 1LLU << 57, -+}; -+ -+enum hmbird_deq_flags { -+ /* expose select DEQUEUE_* flags as enums */ -+ HMBIRD_DEQ_SLEEP = DEQUEUE_SLEEP, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * The generic core-sched layer decided to execute the task even though -+ * it hasn't been dispatched yet. Dequeue from the BPF side. -+ */ -+ HMBIRD_DEQ_CORE_SCHED_EXEC = 1LLU << 32, -+}; -+ -+enum hmbird_kick_flags { -+ HMBIRD_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -+ HMBIRD_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ -+}; -+ -+enum hmbird_task_prop_type { -+ HMBIRD_TASK_PROP_COMMON, -+ HMBIRD_TASK_PROP_PIPELINE, -+ HMBIRD_TASK_PROP_DEBUG_OR_LOG, -+ HMBIRD_TASK_PROP_HIGH_LOAD, -+ HMBIRD_TASK_PROP_IO, -+ HMBIRD_TASK_PROP_NETWORK, -+ HMBIRD_TASK_PROP_PERIODIC, -+ HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL, -+ HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL, -+ HMBIRD_TASK_PROP_ISOLATE, -+ HMBIRD_TASK_PROP_MAX, -+}; -+ -+enum hmbird_preempt_policy_id { -+ HMBIRD_PREEMPT_POLICY_NONE, -+ HMBIRD_PREEMPT_POLICY_PRIO_BASED, -+}; -+ -+extern const struct sched_class hmbird_sched_class; -+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops; -+extern const struct file_operations sched_hmbird_fops; -+extern unsigned long hmbird_watchdog_timeout; -+extern unsigned long hmbird_watchdog_timestamp; -+ -+enum scx_rq_flags { -+ HMBIRD_RQ_CAN_STOP_TICK = 1 << 0, -+}; -+ -+struct hmbird_rq { -+ struct hmbird_dispatch_q local_dsq; -+ struct list_head watchdog_list; -+ u64 ops_qseq; -+ u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -+ u32 nr_running; -+ u32 flags; -+ bool cpu_released; -+ cpumask_var_t cpus_to_kick; -+ cpumask_var_t cpus_to_preempt; -+ cpumask_var_t cpus_to_wait; -+ u64 pnt_seq; -+ u64 *prev_runnable_sum_fixed; -+ u32 *prev_window_size; -+ struct irq_work kick_cpus_irq_work; -+ bool pipeline; -+ bool exclusive; -+ bool period_disallow; -+ bool nonperiod_disallow; -+ struct rq *rq; -+ struct hmbird_sched_rq_stats *srq; -+}; -+ -+struct hmbird_sched_change_guard { -+ struct task_struct *p; -+ struct rq *rq; -+ bool queued; -+ bool running; -+ bool done; -+}; -+ -+extern struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -+ -+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags); -+ -+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -+ for (struct hmbird_sched_change_guard __cg = \ -+ hmbird_sched_change_guard_init(__rq, __p, __flags); \ -+ !__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags)) -+ -+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+bool task_on_hmbird(struct task_struct *p); -+int hmbird_pre_fork(struct task_struct *p); -+int hmbird_fork(struct task_struct *p); -+void hmbird_post_fork(struct task_struct *p); -+void hmbird_cancel_fork(struct task_struct *p); -+int hmbird_check_setscheduler(struct task_struct *p, int policy); -+bool hmbird_can_stop_tick(struct rq *rq); -+void init_sched_hmbird_class(void); -+ -+void hmbird_ops_exit(void); -+#define hmbird_ops_error(fmt, ...) \ -+do { \ -+ hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__); \ -+ hmbird_ops_exit(); \ -+} while (0) -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active); -+ -+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight) -+{ -+ return clamp_t(unsigned long, -+ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -+ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); -+} -+ -+static inline void hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active) -+{ -+ if (!hmbird_enabled()) -+ return; -+#ifdef CONFIG_SMP -+ /* -+ * Pairs with the smp_load_acquire() issued by a CPU in -+ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -+ * resched. -+ */ -+ smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1); -+#endif -+ if (!static_branch_unlikely(&hmbird_ops_cpu_preempt)) -+ return; -+ __hmbird_notify_pick_next_task(rq, p, active); -+} -+ -+extern void hmbird_scheduler_tick(void); -+void scan_timeout(struct rq *rq); -+void hmbird_notify_sched_tick(void); -+ -+static inline const struct sched_class *next_active_class(const struct sched_class *class) -+{ -+ class++; -+ -+ if (hmbird_enabled() && class == &fair_sched_class) -+ class++; -+ if (!hmbird_enabled() && class == &hmbird_sched_class) -+ class++; -+ return class; -+} -+ -+#define for_active_class_range(class, _from, _to) \ -+ for (class = (_from); class != (_to); class = next_active_class(class)) -+ -+#define for_each_active_class(class) \ -+ for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -+ -+/* -+ * HMBIRD requires a balance() call before every pick_next_task() call including -+ * when waking up from idle. -+ */ -+#define for_balance_class_range(class, prev_class, end_class) \ -+ for_active_class_range(class, (prev_class) > &hmbird_sched_class ? \ -+ &hmbird_sched_class : (prev_class), (end_class)) -+ -+#define MIN_CGROUP_DL_IDX (5) /* 8ms */ -+#define DEFAULT_CGROUP_DL_IDX (8) /* 64ms */ -+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS]; -+ -+ -+void __hmbird_update_idle(struct rq *rq, bool idle); -+ -+static inline void hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ if (hmbird_enabled()) -+ __hmbird_update_idle(rq, idle); -+} -+ -+int hmbird_ctrl(bool enable); -+ -+int hmbird_tg_online(struct task_group *tg); -+ -+ -+static inline u16 hmbird_task_util(struct task_struct *p) -+{ -+ return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0; -+} -+u16 hmbird_cpu_util(int cpu); -+ -+bool get_hmbird_ops_enabled(void); -+bool get_non_hmbird_task(void); -+void set_cpu_cluster(u64 cpu_cluster); -+#endif /*_EXT_H_*/ -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -new file mode 100755 -index 000000000000..ce05928392bf ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird.c -@@ -0,0 +1,4313 @@ -+// SPDX-License-Identifier: GPL-2.0 -+ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#include -+#include -+ -+#include "slim.h" -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+#include -+ -+#define CLUSTER_SEPARATE -+ -+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0); -+atomic_t non_hmbird_task; -+atomic_t hmbird_module_loaded = ATOMIC_INIT(0); -+int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0}; -+ -+enum hmbird_internal_consts { -+ HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ, -+}; -+ -+enum hmbird_ops_enable_state { -+ HMBIRD_OPS_PREPPING, -+ HMBIRD_OPS_ENABLING, -+ HMBIRD_OPS_ENABLED, -+ HMBIRD_OPS_DISABLING, -+ HMBIRD_OPS_DISABLED, -+}; -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+/* -+ * hmbird_entity->ops_state -+ * -+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler. -+ * State transitions look as follows: -+ * -+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING -+ * ^ | | -+ * | v v -+ * \-------------------------------/ -+ * -+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -+ * sites for explanations on the conditions being waited upon and why they are -+ * safe. Transitions out of them into NONE or QUEUED must store_release and the -+ * waiters should load_acquire. -+ * -+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether -+ * any given task can be dispatched by the BPF scheduler at all times and thus -+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core -+ * can safely reject invalid dispatches. -+ */ -+enum hmbird_ops_state { -+ HMBIRD_OPSS_NONE, /* owned by the HMBIRD core */ -+ HMBIRD_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -+ HMBIRD_OPSS_QUEUED, /* owned by the BPF scheduler */ -+ HMBIRD_OPSS_DISPATCHING, /* in transit back to the HMBIRD core */ -+ -+ /* -+ * QSEQ brands each QUEUED instance so that, when dispatch races -+ * dequeue/requeue, the dispatcher can tell whether it still has a claim -+ * on the task being dispatched. -+ */ -+ HMBIRD_OPSS_QSEQ_SHIFT = 2, -+ HMBIRD_OPSS_STATE_MASK = (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1, -+ HMBIRD_OPSS_QSEQ_MASK = ~HMBIRD_OPSS_STATE_MASK, -+}; -+ -+enum switch_stat { -+ HMBIRD_DISABLED, -+ HMBIRD_SWITCH_PREP, -+ HMBIRD_RQ_SWITCH_BEGIN, -+ HMBIRD_RQ_SWITCH_DONE, -+ HMBIRD_ENABLED, -+}; -+enum switch_stat curr_ss; -+ -+/* -+ * During exit, a task may schedule after losing its PIDs. When disabling the -+ * BPF scheduler, we need to be able to iterate tasks in every state to -+ * guarantee system safety. Maintain a dedicated task list which contains every -+ * task between its fork and eventual free. -+ */ -+DEFINE_SPINLOCK(hmbird_tasks_lock); -+static LIST_HEAD(hmbird_tasks); -+ -+/* ops enable/disable */ -+static struct kthread_worker *hmbird_ops_helper; -+static DEFINE_MUTEX(hmbird_ops_enable_mutex); -+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem); -+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED); -+ -+static bool hmbird_warned_zero_slice; -+enum hmbird_switch_type sw_type; -+static int skip_num[MAX_GLOBAL_DSQS]; -+static int big_distribute_mask_prev; -+static int little_distribute_mask_prev; -+ -+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0); -+ -+/* -+ * The maximum amount of time in jiffies that a task may be runnable without -+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger -+ * hmbird_ops_error(). -+ */ -+unsigned long hmbird_watchdog_timeout; -+ -+/* -+ * The last time the delayed work was run. This delayed work relies on -+ * ksoftirqd being able to run to service timer interrupts, so it's possible -+ * that this work itself could get wedged. To account for this, we check that -+ * it's not stalled in the timer tick, and trigger an error if it is. -+ */ -+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES; -+ -+static struct delayed_work hmbird_watchdog_work; -+static struct work_struct hmbird_err_exit_work; -+ -+/* idle tracking */ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_CPUMASK_OFFSTACK -+#define CL_ALIGNED_IF_ONSTACK -+#else -+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp -+#endif -+ -+static struct { -+ cpumask_var_t cpu; -+ cpumask_var_t smt; -+} idle_masks CL_ALIGNED_IF_ONSTACK; -+ -+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus; -+#endif /* CONFIG_SMP */ -+ -+/* dispatch queues */ -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global; -+ -+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128}; -+u32 pcp_dsq_deadline = 20; -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS]; -+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq); -+ -+static u64 max_hmbird_dsq_internal_id; -+ -+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/ -+#define CLUSTER_SEPARATE_IDX (8) -+#define GDSQS_ID_BASE (3) -+#define UX_COMPATIBLE_IDX (4) -+#define NON_PERIOD_START (5) -+#define NON_PERIOD_END (MAX_GLOBAL_DSQS) -+#define CREATE_DSQ_LEVEL_WITHIN (1) -+ -+struct hmbird_sched_info { -+ spinlock_t lock; -+ int curr_idx[2]; -+ int rtime[MAX_GLOBAL_DSQS]; -+}; -+ -+struct pcp_sched_info { -+ s64 pcp_seq; -+ int rtime; -+ bool pcp_round; -+}; -+ -+/* -+ * pcp_info may rw by another cpu. -+ * protected by rq->lock. -+ */ -+atomic64_t pcp_dsq_round; -+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info); -+ -+struct md_info_t *md_info; -+ -+static int b_rescue_l, l_rescue_b; -+static struct hmbird_sched_info sinfo; -+ -+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC; -+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = { -+ 0, 0, 0, 0, 0, -+ 32 * NSEC_PER_MSEC, -+ 20 * NSEC_PER_MSEC, -+ 14 * NSEC_PER_MSEC, -+ 8 * NSEC_PER_MSEC, -+ 6 * NSEC_PER_MSEC -+}; -+ -+struct cluster_ctx { -+ /* cpu-dsq map must within [lower, upper) */ -+ int upper; -+ int lower; -+ int tidx; -+}; -+ -+enum stat_items { -+ GLOBAL_STAT, -+ CPU_ALLOW_FAIL, -+ RT_CNT, -+ KEY_TASK_CNT, -+ SWITCH_IDX, -+ TIMEOUT_CNT, -+ -+ TOTAL_DSP_CNT, -+ MOVE_RQ_CNT, -+ SELECT_CPU, -+ -+ DWORD_STAT_END = SELECT_CPU, -+ -+ GDSQ_CNT, -+ ERR_IDX, -+ PCP_TIMEOUT_CNT, -+ PCP_LDSQ_CNT, -+ PCP_ENQL_CNT, -+ -+ MAX_ITEMS, -+}; -+static DEFINE_SPINLOCK(stats_lock); -+static char *stats_str[MAX_ITEMS] = { -+ "global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt", -+ "switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt", -+ "select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt", -+ "pcp_ldsq_cnt", "pcp_enql_cnt" -+}; -+ -+ -+struct stats_struct { -+ u64 global_stat[2]; -+ u64 cpu_allow_fail[2]; -+ u64 rt_cnt[2]; -+ u64 key_task_cnt[2]; -+ u64 switch_idx[2]; -+ u64 timeout_cnt[2]; -+ -+ u64 total_dsp_cnt[2]; -+ u64 move_rq_cnt[2]; -+ u64 select_cpu[2]; -+ -+ u64 gdsq_count[MAX_GLOBAL_DSQS][2]; -+ u64 err_idx[5]; -+ u64 pcp_timeout_cnt[NR_CPUS]; -+ u64 pcp_ldsq_count[NR_CPUS][2]; -+ u64 pcp_enql_cnt[NR_CPUS]; -+} stats_data; -+ -+static struct { -+ cpumask_var_t ex_free; -+ cpumask_var_t exclusive; -+ cpumask_var_t partial; -+ cpumask_var_t big; -+ cpumask_var_t little; -+} iso_masks __read_mostly; -+ -+/* -+ * Need more synchronization for these two variables? -+ * I choose not to. -+ */ -+static int l_need_rescue, b_need_rescue; -+ -+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...) \ -+{ \ -+ char buf[MAX_FATAL_INFO]; \ -+ \ -+ scnprintf(buf, MAX_FATAL_INFO, fmt, ##args); \ -+ hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf); \ -+ trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable), \ -+ READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf); \ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); \ -+} \ -+ -+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2) -+{ -+ int c1, c2; -+ struct cpumask mask = {.bits = {0}}; -+ struct cpumask tmp = {.bits = {0}}; -+ -+ if (!slim_stats) -+ return false; -+ -+ if (!rq1 || !rq2) -+ return false; -+ -+ c1 = cpu_of(rq1); -+ c2 = cpu_of(rq2); -+ cpumask_set_cpu(c1, &mask); -+ cpumask_set_cpu(c2, &mask); -+ -+ if (cpumask_and(&tmp, iso_masks.little, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.big, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.partial, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ return false; -+} -+ -+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu) -+{ -+ unsigned long flags; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ if (!slim_stats) -+ return; -+ -+ switch (item) { -+ case GLOBAL_STAT: -+ fallthrough; -+ case CPU_ALLOW_FAIL: -+ fallthrough; -+ case RT_CNT: -+ fallthrough; -+ case KEY_TASK_CNT: -+ fallthrough; -+ case SWITCH_IDX: -+ fallthrough; -+ case TIMEOUT_CNT: -+ fallthrough; -+ case TOTAL_DSP_CNT: -+ fallthrough; -+ case MOVE_RQ_CNT: -+ fallthrough; -+ case SELECT_CPU: -+ pval = pbase + item * 2 + idx; -+ break; -+ case GDSQ_CNT: -+ pval = &stats_data.gdsq_count[dsq_id][idx]; -+ break; -+ case ERR_IDX: -+ pval = &stats_data.err_idx[idx]; -+ break; -+ case PCP_TIMEOUT_CNT: -+ pval = &stats_data.pcp_timeout_cnt[cpu]; -+ break; -+ case PCP_LDSQ_CNT: -+ pval = &stats_data.pcp_ldsq_count[cpu][idx]; -+ break; -+ case PCP_ENQL_CNT: -+ pval = &stats_data.pcp_enql_cnt[cpu]; -+ break; -+ default: -+ return; -+ } -+ -+ spin_lock_irqsave(&stats_lock, flags); -+ *pval += 1; -+ spin_unlock_irqrestore(&stats_lock, flags); -+} -+ -+static inline bool handle_ret(int ret, int *idx, int len) -+{ -+ if (ret < 0 || ret >= len - *idx) -+ return true; -+ *idx += ret; -+ return false; -+} -+ -+#define PRINT_INTV (5 * HZ) -+void stats_print(char *buf, int len) -+{ -+ int idx = 0, i, j, ret; -+ int item = 0; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ for (item = 0; item < MAX_ITEMS; item++) { -+ if (item <= DWORD_STAT_END) { -+ pval = pbase + item * 2; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n", -+ stats_str[item], pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == GDSQ_CNT) { -+ for (j = 0; j < MAX_GLOBAL_DSQS; j++) { -+ pval = (u64 *)&stats_data.gdsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == ERR_IDX) { -+ pval = (u64 *)&stats_data.err_idx; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n", -+ stats_str[item], pval[0], -+ pval[1], pval[2], pval[3], pval[4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == PCP_TIMEOUT_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_timeout_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_LDSQ_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_ldsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_ENQL_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_enql_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } -+ } -+ -+ if (!md_info) { -+ buf[idx] = '\0'; -+ return; -+ } -+ -+ ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_SWITCHS; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "sw_rec[%d] = %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.sw_rec[i].switch_at, -+ md_info->kern_dump.sw_rec[i].is_success, -+ md_info->kern_dump.sw_rec[i].end_state, -+ md_info->kern_dump.sw_rec[i].switch_reason); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_EXCEP_ID; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "excep[%d] = %llu %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.excep_rec[i][0], -+ md_info->kern_dump.excep_rec[i][1], -+ md_info->kern_dump.excep_rec[i][2], -+ md_info->kern_dump.excep_rec[i][3], -+ md_info->kern_dump.excep_rec[i][4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n", -+ i, md_info->kern_dump.excep_idx[i]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ -+ buf[idx] = '\0'; -+} -+ -+enum cpu_type { -+ LITTLE, -+ BIG, -+ PARTIAL, -+ EXCLUSIVE, -+ EX_FREE, -+ INVALID -+}; -+ -+enum dsq_type { -+ GLOBAL_DSQ, -+ PCP_DSQ, -+ OTHER, -+ MAX_DSQ_TYPE, -+}; -+ -+static enum cpu_type cpu_cluster(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (get_hmbird_rq(rq)->exclusive) { -+ return EXCLUSIVE; -+ } else { -+ if (cpumask_test_cpu(cpu, iso_masks.little)) -+ return LITTLE; -+ else if (cpumask_test_cpu(cpu, iso_masks.big)) -+ return BIG; -+ else if (cpumask_test_cpu(cpu, iso_masks.partial)) -+ return PARTIAL; -+ else if (cpumask_test_cpu(cpu, iso_masks.exclusive)) -+ return EXCLUSIVE; -+ else if (cpumask_test_cpu(cpu, iso_masks.ex_free)) -+ return EX_FREE; -+ } -+ return INVALID; -+} -+ -+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq) -+{ -+ if (!dsq) -+ return OTHER; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= GDSQS_ID_BASE) && -+ ((dsq->id & 0xff) < MAX_GLOBAL_DSQS)) -+ return GLOBAL_DSQ; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) && -+ ((dsq->id & 0xff) < max_hmbird_dsq_internal_id)) -+ return PCP_DSQ; -+ -+ return OTHER; -+} -+ -+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq) -+{ -+ enum dsq_type type; -+ -+ type = get_dsq_type(dsq); -+ switch (type) { -+ case GLOBAL_DSQ: -+ case PCP_DSQ: -+ return (dsq->id & 0xff) - GDSQS_ID_BASE; -+ default: -+ return -1; -+ } -+ return -1; -+} -+ -+static void update_cpus_idle(bool set, struct cpumask *mask) -+{ -+ int cpu; -+ struct rq *rq; -+ -+ if (set) { -+ for_each_cpu(cpu, mask) { -+ rq = cpu_rq(cpu); -+ if (is_idle_task(rq->curr)) -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ } -+ } else -+ cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask); -+} -+ -+static void set_partial_status(bool enable, bool little, bool big) -+{ -+ WRITE_ONCE(partial_enable, enable); -+ WRITE_ONCE(l_need_rescue, little); -+ WRITE_ONCE(b_need_rescue, big); -+} -+ -+static bool is_little_need_rescue(void) -+{ -+ return READ_ONCE(l_need_rescue); -+} -+ -+static bool is_big_need_rescue(void) -+{ -+ return READ_ONCE(b_need_rescue); -+} -+ -+static bool is_partial_enabled(void) -+{ -+ return READ_ONCE(partial_enable); -+} -+ -+static bool is_partial_cpu(int cpu) -+{ -+ return cpumask_test_cpu(cpu, iso_masks.partial); -+} -+ -+static void set_iso_par_free(bool enable) -+{ -+ WRITE_ONCE(isolate_ctrl, enable); -+} -+ -+static bool is_iso_par_free(void) -+{ -+ return READ_ONCE(isolate_ctrl); -+} -+ -+static bool skip_update_idle(void) -+{ -+ int cpu = smp_processor_id(); -+ enum cpu_type type = cpu_cluster(cpu); -+ -+ if ((type == EXCLUSIVE && !is_iso_par_free()) || -+ /* partial enable may changed during idle, it doesn't matter. */ -+ (!is_partial_enabled() && type == PARTIAL)) -+ return true; -+ -+ return false; -+} -+ -+static void init_isolate_cpus(void) -+{ -+ WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -+ cpumask_set_cpu(0, iso_masks.big); -+ cpumask_set_cpu(1, iso_masks.big); -+ cpumask_set_cpu(2, iso_masks.big); -+ cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(4, iso_masks.big); -+ cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(6, iso_masks.partial); -+ cpumask_set_cpu(7, iso_masks.ex_free); -+} -+ -+extern spinlock_t css_set_lock; -+ -+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp) -+{ -+ int i; -+ struct cgroup *anc; -+ -+ spin_lock_irq(&css_set_lock); -+ for (i = 0; i < cgrp->level; i++) { -+ anc = cgrp->ancestors[i]; -+ if (anc->level != CREATE_DSQ_LEVEL_WITHIN) -+ continue; -+ cgroup_get(anc); -+ spin_unlock_irq(&css_set_lock); -+ return anc; -+ } -+ spin_unlock_irq(&css_set_lock); -+ hmbird_err(NO_CGROUP_L1, ":error cgroup = %s\n", cgrp->kn->name); -+ return NULL; -+} -+ -+#define PCP_IDX_BIT (1 << 31) -+ -+static bool is_pcp_rt(struct task_struct *p) -+{ -+ return rt_prio(p->prio) && (p->nr_cpus_allowed == 1); -+} -+ -+static bool is_pcp_idx(int idx) -+{ -+ return idx >= MAX_GLOBAL_DSQS; -+} -+ -+static bool is_critical_system_task(struct task_struct *p) -+{ -+ int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK; -+ -+ return (p->prio < (MAX_RT_PRIO >> 1) && -+ (sp_dl < SCHED_PROP_DEADLINE_LEVEL3)); -+} -+ -+#define ISOLATE_TASK_TOP_BIT (1 << 17) -+#define PIPELINE_TASK_TOP_BIT (1 << 9) -+static bool is_pipeline_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT); -+} -+ -+static bool is_isolate_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT); -+} -+ -+static bool is_critical_app_task_without_isolate(struct task_struct *p) -+{ -+ return task_is_top_task(p) && !is_isolate_task(p); -+} -+ -+static int find_idx_from_task(struct task_struct *p) -+{ -+ int idx, cpu; -+ int sp_dl; -+ struct task_group *tg = p->sched_task_group; -+ -+ if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) { -+ cpu = cpumask_any(p->cpus_ptr); -+ idx = cpu + MAX_GLOBAL_DSQS; -+ return idx; -+ } -+ -+ if (is_critical_system_task(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL0; -+ goto done; -+ } -+ -+ if (is_critical_app_task_without_isolate(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL1; -+ goto done; -+ } -+ -+ sp_dl = hmbird_get_dsq_id(p); -+ if (sp_dl) { -+ idx = sp_dl; -+ goto done; -+ } -+ -+ if (rt_prio(p->prio)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL3; -+ goto done; -+ } -+ -+ if (tg && tg->css.cgroup && tg->css.cgroup->kn) { -+ if (likely(tg->css.cgroup->kn->id >= 0 && -+ tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS)) -+ idx = cgroup_ids_table[tg->css.cgroup->kn->id]; -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ -+done: -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) { -+ hmbird_err(DSQ_ID_ERR, " : idx error, idx = %d-----\n", idx); -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } -+ return idx; -+} -+ -+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p) -+{ -+ int idx; -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq; -+ -+ if (!p) -+ return NULL; -+ -+ idx = find_idx_from_task(p); -+ if (is_pcp_idx(idx)) { -+ idx -= MAX_GLOBAL_DSQS; -+ dsq = &per_cpu(pcp_ldsq, idx); -+ get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq); -+ slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx); -+ } else { -+ dsq = &gdsqs[idx]; -+ get_hmbird_ts(p)->gdsq_idx = idx; -+ slim_stats_record(GDSQ_CNT, 0, idx, 0); -+ } -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ dsq->last_consume_at = jiffies; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ return dsq; -+} -+ -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq); -+ -+static void set_partial_rescue(bool p_state, bool l_over, bool b_over) -+{ -+ set_partial_status(p_state, l_over, b_over); -+ update_cpus_idle(p_state, iso_masks.partial); -+ hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled()); -+ hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue()); -+ hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue()); -+} -+ -+static void free_isocpu(bool enable) -+{ -+ set_iso_par_free(enable); -+ update_cpus_idle(enable, iso_masks.ex_free); -+ hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free()); -+} -+ -+inline u64 get_hmbird_cpu_util(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed) -+ return 0; -+ u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed); -+ u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size); -+ -+ do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT); -+ -+ return prev_runnable_sum_fixed; -+} -+ -+static inline unsigned int get_scaling_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->max; -+} -+ -+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static u64 get_cpus_max_util(struct cpumask *mask) -+{ -+ int cpu; -+ u64 max = 0; -+ u64 util, ratio; -+ unsigned long effective_cap = 0; -+ for_each_cpu(cpu, mask) { -+ if (slim_walt_ctrl) -+ slim_get_cpu_util(cpu, &util); -+ else -+ util = get_hmbird_cpu_util(cpu); -+ -+ /* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/ -+ if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu))) -+ effective_cap = arch_scale_cpu_capacity(cpu); -+ else -+ effective_cap = arch_scale_cpu_capacity(cpu) * -+ get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu); -+ -+ ratio = util * 100 / effective_cap; -+ hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util); -+ hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n", -+ cpu, (u64)arch_scale_cpu_capacity(cpu)); -+ hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n", -+ cpu, (u64)effective_cap); -+ -+ if (ratio > 100) -+ ratio = 100; -+ -+ if (ratio > max) -+ max = ratio; -+ } -+ return max; -+} -+ -+static bool cluster_need_rescue(struct cpumask *mask, int hres) -+{ -+ return get_cpus_max_util(mask) > hres; -+} -+ -+void partial_dynamic_ctrl(void) -+{ -+ u64 lmax = 0, bmax = 0; -+ bool l_over, l_under, b_over, b_under; -+ static bool last_l_over, last_b_over; -+ static unsigned long last_check; -+ -+ /* Check partial every jiffies. need lock? */ -+ if (time_before_eq(jiffies, READ_ONCE(last_check))) -+ return; -+ -+ WRITE_ONCE(last_check, jiffies); -+ -+ lmax = get_cpus_max_util(iso_masks.little); -+ l_over = lmax > parctrl_high_ratio_l; -+ l_under = lmax < parctrl_low_ratio_l; -+ -+ bmax = get_cpus_max_util(iso_masks.big); -+ b_over = bmax > parctrl_high_ratio; -+ b_under = bmax < parctrl_low_ratio; -+ -+ if (is_partial_enabled() && (l_over || b_over)) { -+ if (last_l_over != l_over || last_b_over != b_over) -+ set_partial_rescue(true, l_over, b_over); -+ } else if (!is_partial_enabled() && (l_over || b_over)) { -+ set_partial_rescue(true, l_over, b_over); -+ } else if (is_partial_enabled() && l_under && b_under) { -+ set_partial_rescue(false, false, false); -+ } -+ last_l_over = l_over; -+ last_b_over = b_over; -+ -+ if (is_partial_enabled()) { -+ bmax = get_cpus_max_util(iso_masks.partial); -+ if (!is_iso_par_free() && bmax > isoctrl_high_ratio) { -+ hmbird_info_trace("partial max = %llu\n", bmax); -+ free_isocpu(true); -+ } else if (is_iso_par_free() && bmax < isoctrl_low_ratio) -+ free_isocpu(false); -+ } else if (is_iso_par_free()) { -+ free_isocpu(false); -+ } -+} -+ -+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx) -+{ -+ hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx); -+} -+ -+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx) -+{ -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[idx])) { -+ slim_stats_record(GDSQ_CNT, 1, idx, 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ int i; -+ -+ for (i = 0; i < UX_COMPATIBLE_IDX; i++) { -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ slim_stats_record(GDSQ_CNT, 1, i, 0); -+ return 1; -+ } -+ } -+ return 0; -+} -+ -+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) { -+ slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ goto clear_timeout; -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) -+ goto clear_timeout; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ hmbird_info_trace( -+ "dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n", -+ dsq_id_to_internal(dsq), entity->task->comm, -+ jiffies, msecs_to_jiffies(deadline), entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1); -+ return; -+ -+clear_timeout: -+ hmbird_info_trace("dsq[%d] clear timeout\n", -+ dsq_id_to_internal(dsq)); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0); -+ dsq->is_timeout = false; -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq)); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime) -+{ -+ hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime); -+} -+ -+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any) -+{ -+ bool is_timeout; -+ int cpu = cpu_of(rq); -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu); -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ is_timeout = dsq->is_timeout; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ /* -+ * dsq->is_timeout may change here, let it be. -+ * it won't cause serious problems. -+ * the same for consume_dispatch_q later. -+ */ -+ if (!is_timeout && !any) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, dsq)) { -+ if (is_timeout) { -+ hmbird_info_trace("dsq[%d] consume a pcp timeout task\n", -+ dsq_id_to_internal(dsq)); -+ update_timeout_stats(rq, dsq, pcp_dsq_deadline); -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu); -+ } -+ slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu); -+ return 1; -+ } -+ /* -+ * No pcp task, clear quota. -+ */ -+ if (any) { -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+ } -+ return 0; -+} -+ -+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf) -+{ -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ } -+ return 0; -+} -+ -+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf, -+ int tmp, int cidx, int tidx) -+{ -+ unsigned long flags; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) { -+ if (tmp != cidx) { -+ spin_lock(&sinfo.lock); -+ sinfo.curr_idx[tidx] = tmp; -+ spin_unlock(&sinfo.lock); -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]); -+ slim_stats_record(SWITCH_IDX, 0, 0, 0); -+ } -+ slim_stats_record(GDSQ_CNT, 1, tmp, 0); -+ -+ raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags); -+ gdsqs[tmp].last_consume_at = jiffies; -+ raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags); -+ return 1; -+ } -+ return 0; -+ -+} -+ -+static int get_cidx(struct cluster_ctx *ctx) -+{ -+ int cidx; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ if (cidx < ctx->lower || cidx >= ctx->upper) { -+ sinfo.curr_idx[ctx->tidx] = ctx->lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]); -+ slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ } -+ spin_unlock(&sinfo.lock); -+ -+ return cidx; -+} -+ -+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = CLUSTER_SEPARATE_IDX; -+ if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) { -+ pr_debug(" %s iso_masks.little first is %d\n", -+ __func__, cpumask_first(iso_masks.little)); -+ ctx->upper = NON_PERIOD_END; -+ } -+ ctx->tidx = 0; -+ break; -+ case LITTLE: -+ ctx->lower = CLUSTER_SEPARATE_IDX; -+ ctx->upper = NON_PERIOD_END; -+ if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) { -+ pr_debug(" %s iso_masks.big first is %d", -+ __func__, cpumask_first(iso_masks.big)); -+ ctx->lower = NON_PERIOD_START; -+ } -+ ctx->tidx = 1; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ case LITTLE: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = NON_PERIOD_END; -+ ctx->tidx = 0; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ if (cluster_separate) { -+ return gen_cluster_ctx_separate(ctx, type); -+ } else { -+ return gen_cluster_ctx_common(ctx, type); -+ } -+} -+ -+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ int i; -+ bool is_timeout; -+ unsigned long flags; -+ struct cluster_ctx ctx; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ /* Third param means only consume timeout pcp dsq. */ -+ if (consume_pcp_dsq(rq, rf, false)) -+ return 1; -+ -+ for (i = ctx.lower; i < ctx.upper; i++) { -+ raw_spin_lock_irqsave(&gdsqs[i].lock, flags); -+ is_timeout = gdsqs[i].is_timeout; -+ raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags); -+ /* gdsqs[i].is_timeout may change here, let it be... */ -+ if (is_timeout) { -+ /* -+ * consume_dispatch_q will acquire dsq-lock, -+ * So cannot keep lock here, annoy enough. -+ * may rewrite a consume_dispatch_q_locked, TODO. -+ */ -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ hmbird_info_trace("dsq[%d] consume a timeout task\n", i); -+ slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0); -+ update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]); -+ return 1; -+ } -+ } -+ } -+ return 0; -+} -+ -+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ struct cluster_ctx ctx; -+ int cidx; -+ int tmp; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ cidx = get_cidx(&ctx); -+ tmp = cidx; -+ do { -+ if (check_pcp_dsq_round(rq, rf)) -+ return 1; -+ if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx)) -+ return 1; -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[tmp] = 0; -+ systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]); -+ tmp++; -+ if (tmp >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ tmp = ctx.lower; -+ } -+ spin_unlock(&sinfo.lock); -+ } while (tmp != cidx); -+ -+ return consume_pcp_dsq(rq, rf, true); -+} -+ -+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ enum cpu_type type = cpu_cluster(cpu_of(rq)); -+ bool period_allowed = !get_hmbird_rq(rq)->period_disallow; -+ bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow; -+ -+ switch (type) { -+ case EX_FREE: -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ break; -+ case EXCLUSIVE: -+ if (!is_iso_par_free()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ } -+ /* Only non-period task can run on isolate cpus */ -+ if (!READ_ONCE(iso_free_rescue)) { -+ if (is_big_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ } else { -+ /* Free run, can run on any task. */ -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ case PARTIAL: -+ if (!is_partial_enabled()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ return 0; -+ } -+ if (is_big_need_rescue()) { -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ -+ case BIG: -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ if (is_iso_par_free() || (is_little_need_rescue() && -+ !cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) { -+ hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++); -+ return 1; -+ } -+ } -+ break; -+ -+ case LITTLE: -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ -+ if (is_iso_par_free() || (is_big_need_rescue() && -+ !cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) { -+ hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++); -+ return 1; -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return 0; -+} -+ -+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf) -+{ -+ return consume_hmbird_global_dsq(rq, rf); -+} -+ -+ -+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time) -+{ -+ int idx; -+ -+ /* which dsq belongs to while task enqueue, task will consume its running time. */ -+ idx = get_hmbird_ts(p)->gdsq_idx; -+ /* Only non-period dsq share running time between each other. */ -+ if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id) -+ return; -+ -+ if (idx >= MAX_GLOBAL_DSQS) { -+ per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } else { -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[idx] += exec_time; -+ spin_unlock(&sinfo.lock); -+ systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]); -+ } -+} -+ -+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type) -+{ -+ int cidx; -+ struct cluster_ctx ctx; -+ int cpu = cpu_of(rq); -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ if (cidx < ctx.lower || cidx >= ctx.upper) { -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ slim_stats_record(ERR_IDX, ctx.tidx, 0, 0); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ } -+ -+ while (1) { -+ if (per_cpu(pcp_info, cpu).pcp_round) { -+ if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) { -+ hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n", -+ cpu, per_cpu(pcp_info, cpu).rtime); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } -+ } -+ if (sinfo.rtime[cidx] < dsq_quota[cidx]) -+ break; -+ -+ /* clear current dsq rtime */ -+ sinfo.rtime[cidx] = 0; -+ systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]); -+ -+ sinfo.curr_idx[ctx.tidx]++; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", -+ ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ } -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ slim_stats_record(SWITCH_IDX, 1, 0, 0); -+ } -+ spin_unlock(&sinfo.lock); -+} -+ -+ -+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p) -+{ -+ enum cpu_type type; -+ -+ if (!rq || !p) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ switch (type) { -+ case PARTIAL: -+ return; -+ case EXCLUSIVE: -+ return; -+ case EX_FREE: -+ return; -+ default: -+ break; -+ } -+ update_dsq_idx(rq, p, type); -+} -+ -+ -+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ int dsq_id; -+ -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo) || dsq->is_timeout) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ hmbird_deferred_err(SCAN_ENTITY_NULL, -+ " : entity is NULL, dsq->id = %llu\n", dsq->id); -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ dsq->is_timeout = true; -+ dsq_id = dsq_id_to_internal(dsq); -+ hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n", -+ dsq_id, entity->task->comm, jiffies, entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1); -+ raw_spin_unlock(&dsq->lock); -+ -+ return true; -+} -+ -+void scan_timeout(struct rq *rq) -+{ -+ int i; -+ int cpu = cpu_of(rq); -+ struct hmbird_dispatch_q *dsq; -+ static u64 last_scan_at; -+ static DEFINE_PER_CPU(u64, pcp_last_scan_at); -+ -+ if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu))) -+ return; -+ per_cpu(pcp_last_scan_at, cpu) = jiffies; -+ -+ dsq = &per_cpu(pcp_ldsq, cpu); -+ scan_dsq_timeout(rq, dsq, pcp_dsq_deadline); -+ -+ if (time_before_eq(jiffies, (unsigned long)last_scan_at)) -+ return; -+ last_scan_at = jiffies; -+ -+ for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) { -+ dsq = &gdsqs[i]; -+ scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]); -+ } -+} -+ -+/*******************************Initialize***********************************/ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id); -+static void init_dsq_at_boot(void) -+{ -+ int i, cpu; -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i)); -+ } -+ for_each_possible_cpu(cpu) -+ init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i + cpu)); -+ -+ max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu; -+ spin_lock_init(&sinfo.lock); -+} -+ -+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgrp_name_to_idx(struct cgroup *cgrp) -+{ -+ int idx; -+ -+ if (!cgrp) -+ return -1; -+ -+ if (!strcmp(cgrp->kn->name, "display") -+ || !strcmp(cgrp->kn->name, "multimedia")) -+ idx = 5; /* 8ms */ -+ else if (!strcmp(cgrp->kn->name, "top-app") -+ || !strcmp(cgrp->kn->name, "ss-top")) -+ idx = 6; /* 16ms */ -+ else if (!strcmp(cgrp->kn->name, "ssfg") -+ || !strcmp(cgrp->kn->name, "foreground")) -+ idx = 7; /* 32ms */ -+ else if (!strcmp(cgrp->kn->name, "bg") -+ || !strcmp(cgrp->kn->name, "log") -+ || !strcmp(cgrp->kn->name, "dex2oat") -+ || !strcmp(cgrp->kn->name, "background")) -+ idx = 9; /* 128ms */ -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */ -+ -+ return idx; -+} -+ -+static void init_root_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX); -+} -+ -+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) { -+ pr_err("%s idx err!\n", __func__); -+ return; -+ } -+ -+ if (cgroup_ids_table[cgrp->kn->id] == -1) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp)); -+} -+ -+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ struct cgroup *l1cgrp; -+ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ l1cgrp = cgroup_ancestor_l1(cgrp); -+ if (l1cgrp) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp)); -+ cgroup_put(l1cgrp); -+} -+ -+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg) -+{ -+ switch (cgrp->level) { -+ case 0: -+ init_root_tg(cgrp, tg); -+ break; -+ case 1: -+ init_level1_tg(cgrp, tg); -+ break; -+ default: -+ init_child_tg(cgrp, tg); -+ break; -+ } -+} -+ -+/**************************************************************************/ -+ -+struct hmbird_task_iter { -+ struct hmbird_entity cursor; -+ struct task_struct *locked; -+ struct rq *rq; -+ struct rq_flags rf; -+}; -+ -+/** -+ * hmbird_task_iter_init - Initialize a task iterator -+ * @iter: iterator to init -+ * -+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized, -+ * @iter must eventually be exited with hmbird_task_iter_exit(). -+ * -+ * hmbird_tasks_lock may be released between this and the first next() call or -+ * between any two next() calls. If hmbird_tasks_lock is released between two -+ * next() calls, the caller is responsible for ensuring that the task being -+ * iterated remains accessible either through RCU read lock or obtaining a -+ * reference count. -+ * -+ * All tasks which existed when the iteration started are guaranteed to be -+ * visited as long as they still exist. -+ */ -+static void hmbird_task_iter_init(struct hmbird_task_iter *iter) -+{ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR }; -+ list_add(&iter->cursor.tasks_node, &hmbird_tasks); -+ iter->locked = NULL; -+} -+ -+/** -+ * hmbird_task_iter_exit - Exit a task iterator -+ * @iter: iterator to exit -+ * -+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held. -+ * If the iterator holds a task's rq lock, that rq lock is released. See -+ * hmbird_task_iter_init() for details. -+ */ -+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ if (list_empty(cursor)) -+ return; -+ -+ list_del_init(cursor); -+} -+ -+/** -+ * hmbird_task_iter_next - Next task -+ * @iter: iterator to walk -+ * -+ * Visit the next task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ struct hmbird_entity *pos; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ list_for_each_entry(pos, cursor, tasks_node) { -+ if (&pos->tasks_node == &hmbird_tasks) -+ return NULL; -+ if (!(pos->flags & HMBIRD_TASK_CURSOR)) { -+ list_move(cursor, &pos->tasks_node); -+ return pos->task; -+ } -+ } -+ -+ /* can't happen, should always terminate at hmbird_tasks above */ -+ hmbird_deferred_err(ITER_RET_NULL, " : unreachable path in scx_task_iter_next\n"); -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered - Next non-idle task -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ while ((p = hmbird_task_iter_next(iter))) { -+ if (!is_idle_task(p)) -+ return p; -+ } -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init() -+ * for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ p = hmbird_task_iter_next_filtered(iter); -+ if (!p) -+ return NULL; -+ -+ iter->rq = task_rq_lock(p, &iter->rf); -+ iter->locked = p; -+ return p; -+} -+ -+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void) -+{ -+ return atomic_read(&hmbird_ops_enable_state_var); -+} -+ -+static enum hmbird_ops_enable_state -+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to) -+{ -+ return atomic_xchg(&hmbird_ops_enable_state_var, to); -+} -+ -+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to, -+ enum hmbird_ops_enable_state from) -+{ -+ int from_v = from; -+ -+ return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to); -+} -+ -+static bool hmbird_ops_disabling(void) -+{ -+ return false; -+} -+ -+/** -+ * wait_ops_state - Busy-wait the specified ops state to end -+ * @p: target task -+ * @opss: state to wait the end of -+ * -+ * Busy-wait for @p to transition out of @opss. This can only be used when the -+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also -+ * has load_acquire semantics to ensure that the caller can see the updates made -+ * in the enqueueing and dispatching paths. -+ */ -+static void wait_ops_state(struct task_struct *p, u64 opss) -+{ -+ do { -+ cpu_relax(); -+ } while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss); -+} -+ -+ -+static void update_curr_hmbird(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ u64 now = rq_clock_task(rq); -+ u64 delta_exec; -+ -+ if (time_before_eq64(now, curr->se.exec_start)) -+ return; -+ -+ delta_exec = now - curr->se.exec_start; -+ curr->se.exec_start = now; -+ update_runningtime(rq, curr, delta_exec); -+ curr->se.sum_exec_runtime += delta_exec; -+ account_group_exec_runtime(curr, delta_exec); -+ cgroup_account_cputime(curr, delta_exec); -+ -+ if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF) -+ get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec); -+ -+ trace_sched_stat_runtime(curr, delta_exec, 0); -+} -+ -+static bool hmbird_dsq_priq_less(struct rb_node *node_a, -+ const struct rb_node *node_b) -+{ -+ const struct hmbird_entity *a = -+ container_of(node_a, struct hmbird_entity, dsq_node.priq); -+ const struct hmbird_entity *b = -+ container_of(node_b, struct hmbird_entity, dsq_node.priq); -+ -+ return time_before64(a->dsq_vtime, b->dsq_vtime); -+} -+ -+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ bool is_local = dsq->id == HMBIRD_DSQ_LOCAL; -+ unsigned long flags; -+ -+ hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq || -+ !list_empty(&get_hmbird_ts(p)->dsq_node.fifo), -+ "task = %s, dsq->id = %llu\n", p->comm, dsq->id); -+ hmbird_cond_deferred_err(ENQ_EXIST2, -+ (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq), -+ "task = %s\n", p->comm); -+ -+ if (!is_local) { -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ); -+ hmbird_ops_error(": %s\n", -+ "attempting to dispatch to a destroyed dsq"); -+ /* fall back to the global dsq */ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ dsq = &hmbird_dsq_global; -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ } -+ } -+ -+ if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) { -+ get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ; -+ rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq, -+ hmbird_dsq_priq_less); -+ } else { -+ if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT)) -+ list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ else -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ } -+ dsq->nr++; -+ get_hmbird_ts(p)->dsq = dsq; -+ -+ /* -+ * We're transitioning out of QUEUEING or DISPATCHING. store_release to -+ * match waiters' load_acquire. -+ */ -+ if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS) -+ atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE); -+ -+ if (is_local) { -+ struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq); -+ struct rq *rq = hmbird->rq; -+ bool preempt = false; -+ -+ if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr && -+ rq->curr->sched_class == &hmbird_sched_class) { -+ get_hmbird_ts(rq->curr)->slice = 0; -+ preempt = true; -+ } -+ -+ if (preempt || sched_class_above(&hmbird_sched_class, -+ rq->curr->sched_class)) -+ resched_curr(rq); -+ } else { -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ } -+} -+ -+static void task_unlink_from_dsq(struct task_struct *p, -+ struct hmbird_dispatch_q *dsq) -+{ -+ if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) { -+ rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ; -+ } else { -+ list_del_init(&get_hmbird_ts(p)->dsq_node.fifo); -+ } -+} -+ -+static bool task_linked_on_dsq(struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+} -+ -+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq; -+ bool is_local = dsq == &hmbird_rq->local_dsq; -+ -+ if (!dsq) { -+ hmbird_cond_deferred_err(TASK_LINKED1, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ /* -+ * When dispatching directly from the BPF scheduler to a local -+ * DSQ, the task isn't associated with any DSQ but -+ * @get_hmbird_ts(p)->holding_cpu may be set under the protection of -+ * %HMBIRD_OPSS_DISPATCHING. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu >= 0) -+ get_hmbird_ts(p)->holding_cpu = -1; -+ return; -+ } -+ -+ if (!is_local) -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ /* -+ * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node -+ * can't change underneath us. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu < 0) { -+ /* @p must still be on @dsq, dequeue */ -+ hmbird_cond_deferred_err(TASK_UNLINKED, -+ !task_linked_on_dsq(p), "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ } else { -+ /* -+ * We're racing against dispatch_to_local_dsq() which already -+ * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the -+ * holding_cpu which tells dispatch_to_local_dsq() that it lost -+ * the race. -+ */ -+ hmbird_cond_deferred_err(TASK_LINKED2, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ get_hmbird_ts(p)->holding_cpu = -1; -+ } -+ get_hmbird_ts(p)->dsq = NULL; -+ -+ if (!is_local) -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+ -+static bool test_rq_online(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ return rq->online; -+#else -+ return true; -+#endif -+} -+ -+static void refill_task_slice(struct task_struct *p) -+{ -+ if (is_isolate_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO; -+ else if (is_pipeline_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2; -+ else -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+} -+ -+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -+ int sticky_cpu) -+{ -+ struct hmbird_dispatch_q *d; -+ s32 cpu = -1; -+ -+ hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), -+ "task = %s\n", p->comm); -+ -+ if (is_pcp_rt(p)) { -+ /* Enqueue percpu rt task to local directly. */ -+ /* Or cause a bug when disable dispatch. */ -+ if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ } -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (cpu >= 0) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ -+ if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ /* rq migration */ -+ if (sticky_cpu == cpu_of(rq)) -+ goto local_norefill; -+ /* -+ * If !rq->online, we already told the BPF scheduler that the CPU is -+ * offline. We're just trying to on/offline the CPU. Don't bother the -+ * BPF scheduler. -+ */ -+ if (unlikely(!test_rq_online(rq))) -+ goto local; -+ -+ /* see %HMBIRD_OPS_ENQ_LAST */ -+ if (enq_flags & HMBIRD_ENQ_LAST) -+ goto local; -+ -+ if (enq_flags & HMBIRD_ENQ_LOCAL) -+ goto local; -+ else -+ goto global; -+local: -+ /* -+ * For task-ordering, slice refill must be treated as implying the end -+ * of the current slice. Otherwise, the longer @p stays on the CPU, the -+ * higher priority it becomes from hmbird_prio_less()'s POV. -+ */ -+ refill_task_slice(p); -+local_norefill: -+ dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags); -+ slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq)); -+ return; -+ -+global: -+ d = find_dsq_from_task(p); -+ if (d) { -+ refill_task_slice(p); -+ dispatch_enqueue(d, p, enq_flags); -+ return; -+ } -+ slim_stats_record(GLOBAL_STAT, 0, 0, 0); -+ refill_task_slice(p); -+ dispatch_enqueue(&hmbird_dsq_global, p, enq_flags); -+} -+ -+static bool watchdog_task_watched(const struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->watchdog_node); -+} -+ -+static void watchdog_watch_task(struct rq *rq, struct task_struct *p) -+{ -+ lockdep_assert_rq_held(rq); -+ if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ get_hmbird_ts(p)->runnable_at = jiffies; -+ clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list); -+} -+ -+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) -+{ -+ list_del_init(&get_hmbird_ts(p)->watchdog_node); -+ if (reset_timeout) -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags) -+{ -+ int sticky_cpu = get_hmbird_ts(p)->sticky_cpu; -+ -+ enq_flags |= get_hmbird_rq(rq)->extra_enq_flags; -+ -+ if (sticky_cpu >= 0) -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ -+ /* -+ * Restoring a running task will be immediately followed by -+ * set_next_task_hmbird() which expects the task to not be on the BPF -+ * scheduler as tasks can only start running through local DSQs. Force -+ * direct-dispatch into the local DSQ by setting the sticky_cpu. -+ */ -+ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -+ sticky_cpu = cpu_of(rq); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_UNWATCHED, -+ !watchdog_task_watched(p), "task = %s\n", p->comm); -+ return; -+ } -+ -+ watchdog_watch_task(rq, p); -+ set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ get_hmbird_rq(rq)->nr_running++; -+ add_nr_running(rq, 1); -+ -+ do_enqueue_task(rq, p, enq_flags, sticky_cpu); -+} -+ -+static void ops_dequeue(struct task_struct *p, u64 deq_flags) -+{ -+ u64 opss; -+ -+ watchdog_unwatch_task(p, false); -+ -+ /* acquire ensures that we see the preceding updates on QUEUED */ -+ opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state); -+ -+ switch (opss & HMBIRD_OPSS_STATE_MASK) { -+ case HMBIRD_OPSS_NONE: -+ break; -+ case HMBIRD_OPSS_QUEUEING: -+ /* -+ * QUEUEING is started and finished while holding @p's rq lock. -+ * As we're holding the rq lock now, we shouldn't see QUEUEING. -+ */ -+ hmbird_deferred_err(DEQ_DEQING, " : unreachable path in %s\n", __func__); -+ break; -+ case HMBIRD_OPSS_QUEUED: -+ if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss, -+ HMBIRD_OPSS_NONE)) -+ break; -+ fallthrough; -+ case HMBIRD_OPSS_DISPATCHING: -+ /* -+ * If @p is being dispatched from the BPF scheduler to a DSQ, -+ * wait for the transfer to complete so that @p doesn't get -+ * added to its DSQ after dequeueing is complete. -+ * -+ * As we're waiting on DISPATCHING with the rq locked, the -+ * dispatching side shouldn't try to lock the rq while -+ * DISPATCHING is set. See dispatch_to_local_dsq(). -+ * -+ * DISPATCHING shouldn't have qseq set and control can reach -+ * here with NONE @opss from the above QUEUED case block. -+ * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss. -+ */ -+ wait_ops_state(p, HMBIRD_OPSS_DISPATCHING); -+ hmbird_cond_deferred_err(HMBIRD_OPN, -+ atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE, -+ "task = %s\n", p->comm); -+ break; -+ } -+} -+ -+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ -+ if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p), -+ "task = %s\n", p->comm); -+ return; -+ } -+ -+ ops_dequeue(p, deq_flags); -+ -+ if (slim_walt_ctrl) { -+ if (task_current(rq, p)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ if (deq_flags & HMBIRD_DEQ_SLEEP) -+ set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else -+ clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), -+ (unsigned long *)&get_hmbird_ts(p)->flags); -+ -+ clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm); -+ hmbird_rq->nr_running--; -+ sub_nr_running(rq, 1); -+ dispatch_dequeue(hmbird_rq, p); -+} -+ -+static void yield_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ get_hmbird_ts(p)->slice = 0; -+} -+ -+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to) -+{ -+ return false; -+} -+ -+#ifdef CONFIG_SMP -+/** -+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -+ * @rq: rq to move the task into, currently locked -+ * @p: task to move -+ * @enq_flags: %HMBIRD_ENQ_* -+ * -+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller -+ * must: -+ * -+ * 1. Start with exclusive access to @p either through its DSQ lock or -+ * %HMBIRD_OPSS_DISPATCHING flag. -+ * -+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id(). -+ * -+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't -+ * deadlock with dequeue. -+ * -+ * 4. Lock @rq and the task_rq from #3. -+ * -+ * 5. Call this function. -+ * -+ * Returns %true if @p was successfully moved. %false after racing dequeue and -+ * losing. -+ */ -+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ struct rq *task_rq; -+ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * If dequeue got to @p while we were trying to lock both rq's, it'd -+ * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have -+ * updated it to different values afterwards, as this operation can't be -+ * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become -+ * raw_smp_processor_id() again before we're done. Thus, we can tell -+ * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is -+ * still raw_smp_processor_id(). -+ * -+ * See dispatch_dequeue() for the counterpart. -+ */ -+ if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id())) -+ return false; -+ -+ /* @p->rq couldn't have changed if we're still the holding cpu */ -+ task_rq = task_rq(p); -+ lockdep_assert_rq_held(task_rq); -+ deactivate_task(task_rq, p, 0); -+ set_task_cpu(p, cpu_of(rq)); -+ get_hmbird_ts(p)->sticky_cpu = cpu_of(rq); -+ -+ /* -+ * We want to pass hmbird-specific enq_flags but activate_task() will -+ * truncate the upper 32 bit. As we own @rq, we can pass them through -+ * @get_hmbird_rq(rq)->extra_enq_flags instead. -+ */ -+ hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags, -+ "task = %s\n", p->comm); -+ get_hmbird_rq(rq)->extra_enq_flags = enq_flags; -+ activate_task(rq, p, 0); -+ get_hmbird_rq(rq)->extra_enq_flags = 0; -+ -+ return true; -+} -+ -+#endif /* CONFIG_SMP */ -+ -+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu) -+{ -+ int fitable = 1; -+ -+ return fitable; -+} -+ -+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq, -+ struct hmbird_dispatch_q *dsq) -+{ -+ bool dsq_misfit; -+ int cpu = cpu_of(rq); -+ u64 task_util = 0; -+ struct cluster_ctx ctx; -+ int dsq_int = dsq_id_to_internal(dsq); -+ -+ if (!cpumask_test_cpu(cpu, iso_masks.little)) -+ return false; -+ -+ gen_cluster_ctx(&ctx, BIG); -+ dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 && -+ dsq_int <= SCHED_PROP_DEADLINE_LEVEL4); -+#ifdef CLUSTER_SEPARATE -+ /* In rescue mode, little will consume big cluster's dsq.*/ -+ dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper); -+#endif -+ if (!dsq_misfit) -+ return false; -+ -+ if (p) { -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &task_util); -+ else -+ task_util = get_hmbird_ts(p)->demand_scaled; -+ } -+ -+ if (task_util <= misfit_ds) -+ return false; -+ -+ hmbird_info_trace(":task %s can't run on cpu%d, util = %llu\n", -+ p->comm, cpu, task_util); -+ return true; -+} -+ -+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq) -+{ -+ if (!task_fits_cpu_hmbird(p, cpu_of(rq))) -+ return false; -+ -+ if (check_misfit_task_on_little(p, rq, dsq)) -+ return false; -+ -+ return likely(test_rq_online(rq)); -+} -+ -+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return; -+ -+ if (add) -+ skipn[idx]++; -+ else -+ skipn[idx] = 0; -+} -+ -+static bool skip_too_much(struct hmbird_dispatch_q *dsq) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return false; -+ -+ if (skip_num[idx] > 3) { -+ skip_num[idx] = 0; -+ return true; -+ } -+ -+ return false; -+} -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rb_node *rb_node; -+ struct rq *task_rq; -+ unsigned long flags; -+ bool moved = false; -+ struct task_struct *may_fit = NULL; -+ int skip = 0; -+ -+retry: -+ if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -+ return false; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) { -+ set_skip_num(dsq, skip_num, (bool)may_fit); -+ goto this_rq; -+ } -+ if (skip_too_much(dsq)) -+ goto remote_rq; -+ if (!may_fit) -+ may_fit = p; -+ if (++skip <= 3) -+ continue; -+ /* -+ * If the recent 3 tasks not fit, use the first one. -+ * and clear the skip, because the first one is consumed. -+ */ -+ set_skip_num(dsq, skip_num, false); -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ /* No more task, use the first may fit task.*/ -+ if (may_fit) { -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ -+ for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) -+ goto this_rq; -+ goto remote_rq; -+ } -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ return false; -+ -+this_rq: -+ /* @dsq is locked and @p is on this rq */ -+ hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo); -+ dsq->nr--; -+ hmbird_rq->local_dsq.nr++; -+ get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ -+remote_rq: -+#ifdef CONFIG_SMP -+ if (cpu_same_cluster_stat(p, rq, task_rq)) -+ slim_stats_record(MOVE_RQ_CNT, 0, 0, 0); -+ else -+ slim_stats_record(MOVE_RQ_CNT, 1, 0, 0); -+ /* -+ * @dsq is locked and @p is on a remote rq. @p is currently protected by -+ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -+ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -+ * rq lock or fail, do a little dancing from our side. See -+ * move_task_to_local_dsq(). -+ */ -+ hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id(); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ rq_unpin_lock(rq, rf); -+ double_lock_balance(rq, task_rq); -+ rq_repin_lock(rq, rf); -+ -+ moved = move_task_to_local_dsq(rq, p, 0); -+ -+ double_unlock_balance(rq, task_rq); -+#endif /* CONFIG_SMP */ -+ if (likely(moved)) { -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ } -+ may_fit = NULL; -+ goto retry; -+} -+ -+ -+static int balance_one(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf, bool local) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class; -+ -+ if (!hmbird_rq) -+ return 1; -+ -+ lockdep_assert_rq_held(rq); -+ -+ if (static_branch_unlikely(&hmbird_ops_cpu_preempt) && -+ unlikely(get_hmbird_rq(rq)->cpu_released)) { -+ /* -+ * If the previous sched_class for the current CPU was not HMBIRD, -+ * notify the BPF scheduler that it again has control of the -+ * core. This callback complements ->cpu_release(), which is -+ * emitted in hmbird_notify_pick_next_task(). -+ */ -+ get_hmbird_rq(rq)->cpu_released = false; -+ } -+ -+ if (prev_on_hmbird) -+ update_curr_hmbird(rq); -+ /* if there already are tasks to run, nothing to do */ -+ if (hmbird_rq->local_dsq.nr) -+ return 1; -+ -+ if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) { -+ slim_stats_record(GLOBAL_STAT, 1, 0, 0); -+ return 1; -+ } -+ -+ if (consume_dispatch_global(rq, rf)) -+ return 1; -+ -+ return 0; -+} -+ -+static int balance_hmbird(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf) -+{ -+ return balance_one(rq, prev, rf, true); -+} -+ -+/* -+ * output task util to systrace, only for debug mode. -+ * we can not output too many logs to systrace buffer even in debug mode -+ * only output debug-info while it exceed misfit_ds. -+ */ -+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p) -+{ -+ static DEFINE_PER_CPU(int, is_last_exceed); -+ int cpu = cpu_of(rq); -+ u64 util = 0; -+ -+ if (likely(!debug_enabled())) -+ return; -+ -+ if (!p) -+ return; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ util = uclamp_rq_util_with(rq, util, p); -+ -+ if (util >= misfit_ds) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util); -+ per_cpu(is_last_exceed, cpu) = true; -+ } else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0); -+ per_cpu(is_last_exceed, cpu) = false; -+ } else { -+ } -+} -+ -+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first) -+{ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ /* -+ * Core-sched might decide to execute @p before it is -+ * dispatched. Call ops_dequeue() to notify the BPF scheduler. -+ */ -+ ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC); -+ dispatch_dequeue(get_hmbird_rq(rq), p); -+ } -+ -+ p->se.exec_start = rq_clock_task(rq); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK); -+ } -+ -+ watchdog_unwatch_task(p, true); -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx); -+ systrace_output_cpu_ds(rq, p); -+ /* -+ * @p is getting newly scheduled or got kicked after someone updated its -+ * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird(). -+ */ -+ if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) != -+ (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) { -+ if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) -+ get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK; -+ else -+ get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK; -+ -+ sched_update_tick_dependency(rq); -+ } -+ -+ p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime; -+} -+ -+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p) -+{ -+ update_curr_hmbird(rq); -+ -+ update_dispatch_dsq_info(rq, p); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ watchdog_watch_task(rq, p); -+ -+ if (is_pipeline_task(p)) { -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1); -+ return; -+ } -+ -+ /* -+ * If we're in the pick_next_task path, balance_hmbird() should -+ * have already populated the local DSQ if there are any other -+ * available tasks. If empty, tell ops.enqueue() that @p is the -+ * only one available for this cpu. ops.enqueue() should put it -+ * on the local DSQ so that the subsequent pick_next_task_hmbird() -+ * can find the task unless it wants to trigger a separate -+ * follow-up scheduling event. -+ */ -+ if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1); -+ else -+ do_enqueue_task(rq, p, 0, -1); -+ } -+} -+ -+static struct task_struct *first_local_task(struct rq *rq) -+{ -+ struct rb_node *rb_node; -+ struct hmbird_entity *entity; -+ -+ if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) { -+ entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo, -+ struct hmbird_entity, dsq_node.fifo); -+ return entity->task; -+ } -+ -+ rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq); -+ if (rb_node) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ return entity->task; -+ } -+ return NULL; -+} -+ -+static struct task_struct *pick_next_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p; -+ -+ p = first_local_task(rq); -+ if (!p) -+ return NULL; -+ -+ if (unlikely(!get_hmbird_ts(p)->slice)) { -+ if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice) -+ hmbird_warned_zero_slice = true; -+ -+ refill_task_slice(p); -+ } -+ -+ set_next_task_hmbird(rq, p, true); -+ -+ return p; -+} -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task, -+ const struct sched_class *active) -+{ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * The callback is conceptually meant to convey that the CPU is no -+ * longer under the control of HMBIRD. Therefore, don't invoke the -+ * callback if the CPU is staying on HMBIRD, or going idle (in which -+ * case the HMBIRD scheduler has actively decided not to schedule any -+ * tasks on the CPU). -+ */ -+ if (likely(active >= &hmbird_sched_class)) -+ return; -+ -+ /* -+ * At this point we know that HMBIRD was preempted by a higher priority -+ * sched_class, so invoke the ->cpu_release() callback if we have not -+ * done so already. We only send the callback once between HMBIRD being -+ * preempted, and it regaining control of the CPU. -+ * -+ * ->cpu_release() complements ->cpu_acquire(), which is emitted the -+ * next time that balance_hmbird() is invoked. -+ */ -+ if (!get_hmbird_rq(rq)->cpu_released) -+ get_hmbird_rq(rq)->cpu_released = true; -+} -+ -+#ifdef CONFIG_SMP -+ -+static bool test_and_clear_cpu_idle(int cpu) -+{ -+ if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -+ if (cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ return true; -+ } else { -+ return false; -+ } -+} -+ -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) -+{ -+ int cpu; -+ -+ do { -+ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -+ if (cpu >= nr_cpu_ids) -+ return -EBUSY; -+ } while (!test_and_clear_cpu_idle(cpu)); -+ -+ return cpu; -+} -+ -+ -+static bool prev_cpu_misfit(int prev) -+{ -+ if (!is_partial_enabled() && is_partial_cpu(prev)) -+ return true; -+ -+ return false; -+} -+ -+static int heavy_rt_placement(struct task_struct *p, int prev) -+{ -+ struct cpumask tmp = {.bits = {0}}; -+ int cpu; -+ u64 util = 0; -+ -+ if (!rt_prio(p->prio)) -+ return -EFAULT; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ -+ if (util < misfit_ds) -+ return -EFAULT; -+ -+ if (is_partial_enabled()) -+ cpumask_or(&tmp, iso_masks.big, iso_masks.partial); -+ else -+ cpumask_copy(&tmp, iso_masks.big); -+ -+ cpu = hmbird_pick_idle_cpu(&tmp); -+ if (cpu >= 0) -+ return cpu; -+ -+ if (cpumask_test_cpu(prev, iso_masks.big) || -+ (is_partial_enabled() && is_partial_cpu(prev))) -+ return prev; -+ -+ return cpumask_first(&tmp); -+} -+ -+static int spec_task_before_pick_idle(struct task_struct *p, int prev) -+{ -+ int cpu; -+ -+ cpu = heavy_rt_placement(p, prev); -+ if (cpu >= 0) -+ return cpu; -+ return -EFAULT; -+} -+ -+static int cpumask_distribute_next(struct cpumask *mask, int *prev) -+{ -+ int p = *prev, n; -+ -+ n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1); -+ if (n < nr_cpu_ids) -+ WRITE_ONCE(*prev, n); -+ return n; -+} -+ -+static int repick_fallback_cpu(void) -+{ -+ /* -+ * partial cpu follow big cluster's Scheduling policy, -+ * simply return first bit cpu. -+ */ -+ return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev); -+} -+ -+/* -+ * Must return a valid cpu num, as this task's cpu. -+ */ -+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu, -+ struct cpumask *mask, int *prev_mask) -+{ -+ int cpu; -+ -+ cpu = hmbird_pick_idle_cpu(mask); -+ if (cpu >= 0) -+ return cpu; -+ return cpumask_distribute_next(mask, prev_mask); -+} -+ -+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type) -+{ -+ int idx; -+ struct cluster_ctx ctx; -+ -+ idx = find_idx_from_task(p); -+ if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS) -+ return false; -+ -+ gen_cluster_ctx(&ctx, type); -+ if (idx >= ctx.lower && idx < ctx.upper) -+ return true; -+ return false; -+} -+ -+static bool is_valid_cpu(int cpu) -+{ -+ return (cpu >= 0) && (cpu < nr_cpu_ids); -+} -+ -+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) -+{ -+ s32 cpu = -1; -+ struct cpumask mask = {.bits = {0}}; -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (is_valid_cpu(cpu)) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ partial_dynamic_ctrl(); -+ -+ if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ iso_masks.big, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ if (p->nr_cpus_allowed == 1) { -+ cpu = cpumask_any(p->cpus_ptr); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* For non-period global dsq, not contain pcp task. */ -+ if (task_only_blongs_to_cluster(p, LITTLE)) { -+ cpumask_copy(&mask, iso_masks.little); -+ if (unlikely(l_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &little_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ if (task_only_blongs_to_cluster(p, BIG)) { -+ cpumask_copy(&mask, iso_masks.big); -+ if (unlikely(b_need_rescue)) -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ cpu = spec_task_before_pick_idle(p, prev_cpu); -+ if (is_valid_cpu(cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* if the previous CPU is idle, dispatch directly to it */ -+ if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+ } -+ -+ cpu = hmbird_pick_idle_cpu(cpu_possible_mask); -+ if (is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ if (prev_cpu_misfit(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ cpu = repick_fallback_cpu(); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+} -+ -+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags) -+{ -+ return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags); -+} -+ -+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx) -+{ -+ set_cpus_allowed_common(p, ctx); -+} -+ -+static void reset_idle_masks(void) -+{ -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little); -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big); -+ if (is_partial_enabled()) -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial); -+ hmbird_has_idle_cpus = true; -+} -+ -+void __hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ int cpu = cpu_of(rq); -+ struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -+ -+ if (skip_update_idle()) -+ return; -+ -+ if (idle) { -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ if (!hmbird_has_idle_cpus) -+ hmbird_has_idle_cpus = true; -+ -+ /* -+ * idle_masks.smt handling is racy but that's fine as it's only -+ * for optimization and self-correcting. -+ */ -+ for_each_cpu(cpu, sib_mask) { -+ if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -+ return; -+ } -+ cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -+ } else { -+ cpumask_clear_cpu(cpu, idle_masks.cpu); -+ if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ -+ cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -+ } -+} -+ -+#else /* !CONFIG_SMP */ -+ -+static bool test_and_clear_cpu_idle(int cpu) { return false; } -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } -+static void reset_idle_masks(void) {} -+ -+#endif /* CONFIG_SMP */ -+ -+static bool check_rq_for_timeouts(struct rq *rq) -+{ -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rq_flags rf; -+ -+ rq_lock_irqsave(rq, &rf); -+ list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) { -+ unsigned long last_runnable; -+ -+ p = entity->task; -+ last_runnable = get_hmbird_ts(p)->runnable_at; -+ -+ if (unlikely(time_after(jiffies, -+ last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -+ -+ rq_unlock_irqrestore(rq, &rf); -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb", -+ p->comm, p->pid, -+ dur_ms / 1000, dur_ms % 1000, -+ get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0, -+ cpumask_pr_args(&p->cpus_mask)); -+ return 1; -+ } -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ return 0; -+} -+ -+static void hmbird_watchdog_workfn(struct work_struct *work) -+{ -+ int cpu; -+ bool timeout; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ -+ for_each_online_cpu(cpu) { -+ timeout = check_rq_for_timeouts(cpu_rq(cpu)); -+ if (unlikely(timeout)) -+ break; -+ -+ cond_resched(); -+ } -+ if (!timeout) -+ queue_delayed_work(system_unbound_wq, to_delayed_work(work), -+ hmbird_watchdog_timeout / 2); -+} -+ -+static void set_pcp_round(struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) { -+ per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round); -+ per_cpu(pcp_info, cpu).pcp_round = true; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+} -+ -+ -+/* -+ * Just for debug: output hmbird on/off state per 10s. -+ */ -+#define OUTPUT_INTVAL (msecs_to_jiffies(10 * 1000)) -+static void inform_hmbird_onoff_from_systrace(void) -+{ -+ static unsigned long __read_mostly next_print; -+ -+ if (time_before(jiffies, READ_ONCE(next_print))) -+ return; -+ -+ WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL); -+ hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio); -+} -+ -+void hmbird_notify_sched_tick(void) -+{ -+ unsigned long last_check; -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ hmbird_scheduler_tick(); -+ -+ if (!hmbird_enabled()) -+ return; -+ -+ last_check = hmbird_watchdog_timestamp; -+ if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -+ -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "watchdog failed to check in for %u.%03us", -+ dur_ms / 1000, dur_ms % 1000); -+ } -+ scan_timeout(rq); -+} -+ -+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued) -+{ -+ update_curr_hmbird(rq); -+ -+ set_pcp_round(rq); -+ -+ if (slim_walt_ctrl) -+ hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE); -+ /* -+ * While disabling, always resched and refresh core-sched timestamp as -+ * we can't trust the slice management or ops.core_sched_before(). -+ */ -+ if (hmbird_ops_disabling()) -+ get_hmbird_ts(curr)->slice = 0; -+ -+ if (!get_hmbird_ts(curr)->slice) -+ resched_curr(rq); -+ -+ inform_hmbird_onoff_from_systrace(); -+} -+ -+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg) -+{ -+ hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ get_hmbird_ts(p)->disallow = false; -+ -+ hmbird_sched_init_task(p); -+ -+ set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ return 0; -+} -+ -+static void hmbird_ops_enable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void hmbird_ops_disable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void set_task_hmbird_weight(struct task_struct *p) -+{ -+ u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -+ -+ get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight); -+} -+ -+/** -+ * refresh_hmbird_weight - Refresh a task's hmbird weight -+ * @p: task to refresh hmbird weight for -+ * -+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to -+ * enable easy access from the BPF scheduler. To keep it synchronized with the -+ * current task priority, this function should be called when a new task is -+ * created, priority is changed for a task on hmbird, and a task is switched -+ * to hmbird from other classes. -+ */ -+static void refresh_hmbird_weight(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ set_task_hmbird_weight(p); -+} -+ -+int hmbird_pre_fork(struct task_struct *p) -+{ -+ int ret = 0; -+ -+ p->android_oem_data1[HMBIRD_TS_IDX] = -+ (u64)(kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL)); -+ if (!get_hmbird_ts(p)) { -+ ret = -1; -+ goto lock; -+ } -+ -+ get_hmbird_ts(p)->dsq = NULL; -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node); -+ get_hmbird_ts(p)->flags = 0; -+ get_hmbird_ts(p)->weight = 0; -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ get_hmbird_ts(p)->holding_cpu = -1; -+ get_hmbird_ts(p)->kf_mask = 0; -+ atomic64_set(&get_hmbird_ts(p)->ops_state, 0); -+ get_hmbird_ts(p)->runnable_at = INITIAL_JIFFIES; -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+ get_hmbird_ts(p)->task = p; -+ hmbird_set_sched_prop(p, 0); -+ -+ get_hmbird_ts(p)->critical_affinity_cpu = -1; -+ get_hmbird_ts(p)->sched_class = &hmbird_sched_class; -+ -+ /* -+ * BPF scheduler enable/disable paths want to be able to iterate and -+ * update all tasks which can become complex when racing forks. As -+ * enable/disable are very cold paths, let's use a percpu_rwsem to -+ * exclude forks. -+ */ -+lock: -+ percpu_down_read(&hmbird_fork_rwsem); -+ -+ return ret; -+} -+ -+int hmbird_fork(struct task_struct *p) -+{ -+ percpu_rwsem_assert_held(&hmbird_fork_rwsem); -+ -+ if (hmbird_enabled()) -+ return hmbird_ops_prepare_task(p, task_group(p)); -+ else -+ return 0; -+} -+ -+void hmbird_post_fork(struct task_struct *p) -+{ -+ if (hmbird_enabled()) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ /* -+ * Set the weight manually before calling ops.enable() so that -+ * the scheduler doesn't see a stale value if they inspect the -+ * task struct. We'll invoke ops.set_weight() afterwards, as it -+ * would be odd to receive a callback on the task before we -+ * tell the scheduler that it's been fully enabled. -+ */ -+ set_task_hmbird_weight(p); -+ hmbird_ops_enable_task(p); -+ refresh_hmbird_weight(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ -+ spin_lock_irq(&hmbird_tasks_lock); -+ list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_cancel_fork(struct task_struct *p) -+{ -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ if (hmbird_enabled()) -+ hmbird_ops_disable_task(p); -+ -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_free(struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_entity *see = get_hmbird_ts(p); -+ -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+ list_del_init(&get_hmbird_ts(p)->tasks_node); -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+ -+ /* -+ * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED -> -+ * ENABLED transitions can't race us. Disable ops for @p. -+ */ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) || -+ test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ hmbird_ops_disable_task(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ kfree(get_hmbird_ts(p)); -+ see = NULL; -+} -+ -+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio) -+{ -+} -+ -+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type) -+{ -+ return (prop >> TOP_TASK_SHIFT) & (1 << type); -+} -+ -+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p) -+{ -+ uint32_t prop = get_top_task_prop(p); -+ -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE)) -+ return HMBIRD_TASK_PROP_PIPELINE; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) || -+ !task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) { -+ return HMBIRD_TASK_PROP_COMMON; -+ } -+ return HMBIRD_TASK_PROP_DEBUG_OR_LOG; -+} -+ -+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b) -+{ -+ int type_a = hmbird_get_task_type(a); -+ int type_b = hmbird_get_task_type(b); -+ -+ return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b]; -+} -+ -+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ enum cpu_type type; -+ int sp_dl; -+ struct task_struct *curr = NULL; -+ -+ switch (hmbird_preempt_policy) { -+ case HMBIRD_PREEMPT_POLICY_PRIO_BASED: -+ curr = rq->curr; -+ if (curr && hmbird_prio_higher(p, curr)) -+ goto preempt; -+ break; -+ default: -+ break; -+ } -+ if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) || -+ (is_critical_system_task(p) && !is_critical_system_task(rq->curr))) -+ goto preempt; -+ -+ sp_dl = find_idx_from_task(p); -+ if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled())) -+ return; -+ -+ if (rq->curr->prio > p->prio) -+ goto preempt; -+ -+ return; -+ -+preempt: -+ resched_curr(rq); -+} -+ -+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {} -+ -+int hmbird_check_setscheduler(struct task_struct *p, int policy) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ /* if disallow, reject transitioning into HMBIRD */ -+ if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) && -+ p->policy != policy && policy == SCHED_HMBIRD) -+ return -EACCES; -+ -+ return 0; -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool hmbird_can_stop_tick(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ if (hmbird_ops_disabling()) -+ return false; -+ -+ if (p->sched_class != &hmbird_sched_class) -+ return true; -+ -+ return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK; -+} -+#endif -+ -+int hmbird_tg_online(struct task_group *tg) -+{ -+ struct cgroup *cgrp; -+ -+ if (!tg) -+ return 0; -+ cgrp = tg->css.cgroup; -+ if (!cgrp || !(cgrp->kn)) -+ return 0; -+ update_cgroup_ids_table(cgrp->kn->id, -1); -+ return 0; -+} -+ -+/* -+ * Omitted operations: -+ * -+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -+ * task isn't tied to the CPU at that point. Preemption is implemented by -+ * resetting the victim task's slice to 0 and triggering reschedule on the -+ * target CPU. -+ * -+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -+ * -+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -+ * their current sched_class. Call them directly from sched core instead. -+ * -+ * - task_woken, switched_from: Unnecessary. -+ */ -+DEFINE_SCHED_CLASS(hmbird) = { -+ .enqueue_task = enqueue_task_hmbird, -+ .dequeue_task = dequeue_task_hmbird, -+ .yield_task = yield_task_hmbird, -+ .yield_to_task = yield_to_task_hmbird, -+ -+ .check_preempt_curr = check_preempt_curr_hmbird, -+ -+ .pick_next_task = pick_next_task_hmbird, -+ -+ .put_prev_task = put_prev_task_hmbird, -+ .set_next_task = set_next_task_hmbird, -+ -+#ifdef CONFIG_SMP -+ .balance = balance_hmbird, -+ .select_task_rq = select_task_rq_hmbird, -+ .set_cpus_allowed = set_cpus_allowed_hmbird, -+#endif -+ .task_tick = task_tick_hmbird, -+ -+ .switched_to = switched_to_hmbird, -+ .prio_changed = prio_changed_hmbird, -+ -+ .update_curr = update_curr_hmbird, -+ -+#ifdef CONFIG_UCLAMP_TASK -+ .uclamp_enabled = 0, -+#endif -+}; -+ -+/* -+ * Must with rq lock held. -+ */ -+bool task_is_hmbird(struct task_struct *p) -+{ -+ return p->sched_class == &hmbird_sched_class; -+} -+ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id) -+{ -+ memset(dsq, 0, sizeof(*dsq)); -+ -+ raw_spin_lock_init(&dsq->lock); -+ INIT_LIST_HEAD(&dsq->fifo); -+ dsq->id = dsq_id; -+} -+ -+static int hmbird_cgroup_init(void) -+{ -+ struct cgroup_subsys_state *css; -+ -+ css_for_each_descendant_pre(css, &root_task_group.css) { -+ struct task_group *tg = css_tg(css); -+ -+ cgrp_dsq_idx_init(css->cgroup, tg); -+ } -+ return 0; -+} -+ -+/* -+ * Used by sched_fork() and __setscheduler_prio() to pick the matching -+ * sched_class. dl/rt are already handled. -+ */ -+bool task_on_hmbird(struct task_struct *p) -+{ -+ return hmbird_enabled(); -+} -+ -+static void __setscheduler_prio(struct task_struct *p, int prio) -+{ -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) { -+ p->prio = prio; -+ return; -+ } else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (on_hmbird && rt_prio(prio)) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ -+ p->prio = prio; -+} -+ -+/* -+ * Heartbeat, avoid humbird keep running while APP already exit. -+ * Check whether APP send alive-signal periodly. -+ */ -+#define HEARTBEAT_TIMEOUT (msecs_to_jiffies(2500)) -+#define HEARTBEAT_CHECK_INTERVAL (msecs_to_jiffies(1000)) -+static struct timer_list hb_timer; -+static unsigned long next_hb; -+ -+void hb_timer_handler(struct timer_list *timer) -+{ -+ if (!heartbeat_enable) -+ goto refill; -+ -+ pr_err(": enter timer.\n"); -+ if (heartbeat) { -+ heartbeat = 0; -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ } -+ -+ /* can't detect heartbeat, disable ext. */ -+ if (time_after(jiffies, READ_ONCE(next_hb))) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT, -+ "can't detect heartbeat, disable ext\n"); -+ } -+refill: -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_start(void) -+{ -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_init(void) -+{ -+ timer_setup(&hb_timer, hb_timer_handler, 0); -+} -+ -+static void hb_timer_exit(void) -+{ -+ del_timer(&hb_timer); -+} -+ -+static bool check_and_disable_cpuhp(void) -+{ -+ struct rq *rq; -+ struct rq_flags rf; -+ int cpu; -+ -+ cpu_hotplug_disable(); -+ cpus_read_lock(); -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ rq_lock_irqsave(rq, &rf); -+ if (!rq->online) { -+ rq_unlock_irqrestore(rq, &rf); -+ goto err_offline; -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ cpus_read_unlock(); -+ return true; -+ -+err_offline: -+ cpus_read_unlock(); -+ cpu_hotplug_enable(); -+ return false; -+} -+ -+static void reenable_cpuhp(void) -+{ -+ cpu_hotplug_enable(); -+} -+ -+static void scheduler_switch_done(bool final_state) -+{ -+ /* set scx_enable again, switch may fail. */ -+ scx_enable = final_state; -+ /* reset heartbeat*/ -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ -+ if (final_state) -+ hb_timer_start(); -+ else -+ hb_timer_exit(); -+} -+ -+/** -+ * ss : curr switch state -+ * finish : success or fail -+ * enable : curr operation is diabling or enabling? -+ * fail_reason : string output when failed, empty string when success. -+ */ -+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason) -+{ -+ char *s1 = finish ? "finished" : "failed"; -+ char *s2 = enable ? "enabled" : "disabled"; -+ -+ hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss); -+ if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) { -+ sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type)); -+ hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n", -+ s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason); -+ } -+ curr_ss = ss; -+} -+ -+bool get_hmbird_ops_enabled(void) -+{ -+ return atomic_read(&__hmbird_ops_enabled); -+} -+ -+bool get_non_hmbird_task(void) -+{ -+ return atomic_read(&non_hmbird_task); -+} -+ -+static void hmbird_ops_disable_workfn(struct kthread_work *work) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int cpu; -+ -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, ""); -+ cancel_delayed_work_sync(&hmbird_watchdog_work); -+ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) { -+ case HMBIRD_OPS_DISABLED: -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled"); -+ scheduler_switch_done(false); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return; -+ case HMBIRD_OPS_PREPPING: -+ fallthrough; -+ case HMBIRD_OPS_DISABLING: -+ /* shouldn't happen but handle it like ENABLING if it does */ -+ WARN_ONCE(true, "hmbird: duplicate disabling instance?"); -+ fallthrough; -+ case HMBIRD_OPS_ENABLING: -+ case HMBIRD_OPS_ENABLED: -+ break; -+ } -+ -+ /* kick all CPUs to restore ticks */ -+ for_each_possible_cpu(cpu) -+ resched_cpu(cpu); -+ -+ /* avoid racing against fork and cgroup changes */ -+ cpus_read_lock(); -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, ""); -+ spin_lock_irq(&hmbird_tasks_lock); -+ atomic_set(&__hmbird_ops_enabled, false); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ bool alive = READ_ONCE(p->__state) != TASK_DEAD; -+ -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ get_hmbird_ts(p)->slice = min_t(u64, -+ get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL); -+ -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ if (alive) -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ -+ hmbird_ops_disable_task(p); -+ } -+ hmbird_task_iter_exit(&sti); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, ""); -+ -+ atomic_set(&non_hmbird_task, true); -+ /* no task is on hmbird, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt); -+ synchronize_rcu(); -+ -+ percpu_up_write(&hmbird_fork_rwsem); -+ cpus_read_unlock(); -+ -+ if (slim_walt_ctrl) -+ slim_walt_enable(false); -+ -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 1, 0, ""); -+ scheduler_switch_done(false); -+ reenable_cpuhp(); -+} -+ -+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn); -+ -+static void schedule_hmbird_ops_disable_work(void) -+{ -+ struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper); -+ -+ /* -+ * We may be called spuriously before the first bpf_hmbird_reg(). If -+ * hmbird_ops_helper isn't set up yet, there's nothing to do. -+ */ -+ if (helper) -+ kthread_queue_work(helper, &hmbird_ops_disable_work); -+} -+ -+static void hmbird_ops_disable(void) -+{ -+ schedule_hmbird_ops_disable_work(); -+} -+ -+static void hmbird_err_exit_workfn(struct work_struct *work) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ hmbird_ctrl(false); -+ -+ for_each_present_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy) -+ continue; -+ if (cpu != policy->cpu) -+ goto put; -+ down_write(&policy->rwsem); -+ WARN_ON(store_scaling_governor(policy, -+ saved_gov[cpu], strlen(saved_gov[cpu])) <= 0); -+ up_write(&policy->rwsem); -+ hmbird_info_trace(":restore origin gov : %s\n", saved_gov[cpu]); -+put: -+ cpufreq_cpu_put(policy); -+ } -+ memset((char *)saved_gov, 0, sizeof(saved_gov)); -+} -+ -+void hmbird_ops_exit(void) -+{ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); -+} -+ -+static struct kthread_worker *hmbird_create_rt_helper(const char *name) -+{ -+ struct kthread_worker *helper; -+ -+ helper = kthread_create_worker(0, name); -+ if (helper) -+ sched_set_fifo(helper->task); -+ return helper; -+} -+ -+static inline void set_audio_thread_sched_prop(struct task_struct *p) -+{ -+ struct cgroup_subsys_state *css; -+ -+ if (likely(p->prio >= MAX_RT_PRIO)) -+ return; -+ -+ rcu_read_lock(); -+ css = task_css(p, cpuset_cgrp_id); -+ if (!css) { -+ rcu_read_unlock(); -+ return; -+ } -+ rcu_read_unlock(); -+ -+ if (!strcmp(css->cgroup->kn->name, "audio-app")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+} -+ -+static int hmbird_ops_enable(void *unused) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int ret; -+ int tcnt = 0; -+ unsigned long long start = 0; -+ -+ if (!check_and_disable_cpuhp()) { -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline"); -+ return -EBUSY; -+ } -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, ""); -+ mutex_lock(&hmbird_ops_enable_mutex); -+ -+ if (!hmbird_ops_helper) { -+ WRITE_ONCE(hmbird_ops_helper, -+ hmbird_create_rt_helper("hmbird_ops_helper")); -+ if (!hmbird_ops_helper) { -+ ret = -ENOMEM; -+ goto err_unlock; -+ } -+ } -+ -+ if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) { -+ ret = -EBUSY; -+ goto err_unlock; -+ } -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) != -+ HMBIRD_OPS_DISABLED); -+ -+ hmbird_warned_zero_slice = false; -+ -+ atomic64_set(&hmbird_nr_rejected, 0); -+ -+ /* -+ * Keep CPUs stable during enable so that the BPF scheduler can track -+ * online CPUs by watching ->on/offline_cpu() after ->init(). -+ */ -+ cpus_read_lock(); -+ -+ hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work, -+ hmbird_watchdog_timeout / 2); -+ -+ /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ */ -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ reset_idle_masks(); -+ -+ /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. -+ */ -+ ret = hmbird_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock; -+ -+ /* -+ * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem -+ * preventing new tasks from being added. No need to exclude tasks -+ * leaving as hmbird_free() can handle both prepped and enabled -+ * tasks. Prep all tasks first and then enable them with preemption -+ * disabled. -+ */ -+ spin_lock_irq(&hmbird_tasks_lock); -+ -+ atomic_set(&non_hmbird_task, false); -+ atomic_set(&__hmbird_ops_enabled, true); -+ -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered(&sti))) -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ hmbird_task_iter_exit(&sti); -+ -+ /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. -+ */ -+ preempt_disable(); -+ -+ /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ */ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) { -+ atomic_set(&non_hmbird_task, true); -+ atomic_set(&__hmbird_ops_enabled, false); -+ preempt_enable(); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ ret = -EBUSY; -+ goto err_disable_unlock; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED -+ * transitions here are synchronized against hmbird_free() through -+ * hmbird_tasks_lock. -+ */ -+ start = sched_clock(); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, ""); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ tcnt++; -+ if (READ_ONCE(p->__state) != TASK_DEAD) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ -+ set_audio_thread_sched_prop(p); -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ hmbird_ops_enable_task(p); -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ } else { -+ hmbird_ops_disable_task(p); -+ } -+ } -+ hmbird_task_iter_exit(&sti); -+ -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, ""); -+ preempt_enable(); -+ percpu_up_write(&hmbird_fork_rwsem); -+ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) { -+ ret = -EBUSY; -+ goto err_disable; -+ } -+ -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_ENABLED, 1, 1, ""); -+ scheduler_switch_done(true); -+ -+ return 0; -+ -+err_unlock: -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock"); -+ scheduler_switch_done(false); -+ return ret; -+ -+err_disable_unlock: -+ percpu_up_write(&hmbird_fork_rwsem); -+err_disable: -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ /* must be fully disabled before returning */ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable"); -+ scheduler_switch_done(false); -+ return ret; -+} -+ -+#ifdef CONFIG_SCHED_DEBUG -+static const char *hmbird_ops_enable_state_str[] = { -+ [HMBIRD_OPS_PREPPING] = "prepping", -+ [HMBIRD_OPS_ENABLING] = "enabling", -+ [HMBIRD_OPS_ENABLED] = "enabled", -+ [HMBIRD_OPS_DISABLING] = "disabling", -+ [HMBIRD_OPS_DISABLED] = "disabled", -+}; -+ -+static int hmbird_debug_show(struct seq_file *m, void *v) -+{ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled()); -+ seq_printf(m, "%-30s: %s\n", "enable_state", -+ hmbird_ops_enable_state_str[hmbird_ops_enable_state()]); -+ seq_printf(m, "%-30s: %llu\n", "nr_rejected", -+ atomic64_read(&hmbird_nr_rejected)); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return 0; -+} -+ -+static int hmbird_debug_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_debug_show, NULL); -+} -+ -+const struct file_operations sched_hmbird_fops = { -+ .open = hmbird_debug_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif -+ -+ -+static int bpf_hmbird_reg(void *kdata) -+{ -+ return hmbird_ops_enable(kdata); -+} -+ -+static int bpf_hmbird_unreg(void *kdata) -+{ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ return 0; -+} -+ -+void set_hmbird_module_loaded(int is_loaded) -+{ -+ atomic_set(&hmbird_module_loaded, is_loaded); -+} -+ -+/* -+ * MUST load hmbird module before enable hmbird scheduler -+ * load track & hmbird gover implemented in hmbird module -+ */ -+int hmbird_ctrl(bool enable) -+{ -+ if (!atomic_read(&hmbird_module_loaded)) { -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "ext module unloaded\n"); -+ return -EINVAL; -+ } -+ -+ if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED -+ || hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING -+ || hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already enabled(ing)\n"); -+ hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING || -+ hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already disabled(ing)\n"); -+ hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (enable) -+ return bpf_hmbird_reg(NULL); -+ else -+ return bpf_hmbird_unreg(NULL); -+} -+ -+void set_cpu_isomask(int cpu, cpumask_var_t *mask) -+{ -+ cpumask_clear_cpu(cpu, iso_masks.ex_free); -+ cpumask_clear_cpu(cpu, iso_masks.exclusive); -+ cpumask_clear_cpu(cpu, iso_masks.partial); -+ cpumask_clear_cpu(cpu, iso_masks.big); -+ cpumask_clear_cpu(cpu, iso_masks.little); -+ cpumask_set_cpu(cpu, *mask); -+} -+ -+void set_cpu_cluster(u64 cpu_cluster) -+{ -+ int cpu; -+ -+ for_each_present_cpu(cpu) { -+ u64 pos = 1 << cpu; -+ -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.ex_free)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.exclusive)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.partial)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.big)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) -+ set_cpu_isomask(cpu, &(iso_masks.little)); -+ } -+} -+ -+static void get_hmbird_snapshot(struct panic_snapshot_t *p) -+{ -+ struct hmbird_dispatch_q *dsq; -+ struct rq *rq; -+ int cpu, i; -+ -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ p->rq_nr[cpu] = rq->nr_running; -+ p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running; -+ } -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ struct hmbird_entity *entity; -+ -+ dsq = &gdsqs[i]; -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo)) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ p->runnable_at[i] = entity->runnable_at; -+ raw_spin_unlock(&dsq->lock); -+ -+ } -+ -+ p->snap_misc.hmbird_enabled = hmbird_enabled(); -+ p->snap_misc.curr_ss = curr_ss; -+ p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state(); -+ p->snap_misc.parctrl_high_ratio = parctrl_high_ratio; -+ p->snap_misc.parctrl_low_ratio = parctrl_low_ratio; -+ p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l; -+ p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l; -+ p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio; -+ p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio; -+ p->snap_misc.misfit_ds = misfit_ds; -+ p->snap_misc.partial_enable = partial_enable; -+ p->snap_misc.iso_free_rescue = iso_free_rescue; -+ p->snap_misc.isolate_ctrl = isolate_ctrl; -+ p->snap_misc.snap_jiffies = jiffies; -+ p->snap_misc.snap_time = local_clock(); -+} -+ -+// MTK minidump begin -+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3) -+{ -+ strscpy(m->desc_str, str, DESC_STR_LEN); -+ m->len = d1 * d2 * d3; -+ m->parse[0] = d1; -+ m->parse[1] = d2; -+ m->parse[2] = d3; -+} -+ -+static void init_desc_metas(struct md_info_t *m) -+{ -+ init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :", -+ 1, MAX_SWITCHS, SWITCH_ITEMS); -+ -+ init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1); -+ -+ init_desc_meta(&m->kern_dump.excep_rec_meta, -+ "excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS); -+ -+ init_desc_meta(&m->kern_dump.excep_idx_meta, -+ "excep idx :", 1, 1, MAX_EXCEP_ID); -+ -+ init_desc_meta(&m->kern_dump.snap.runnable_at_meta, -+ "each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS); -+ -+ init_desc_meta(&m->kern_dump.snap.rq_nr_meta, -+ "rq runnable task nr:", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta, -+ "scxrq runnable task nr :", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.snap_misc_meta, -+ "misc snap params :", 1, 1, SNAP_ITEMS); -+} -+ -+static void init_md_meta(struct md_info_t *m) -+{ -+ m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64); -+ m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64); -+ m->meta.unit_size = sizeof(u64); -+ m->meta.switches = MAX_SWITCHS; -+ m->meta.exceps = MAX_EXCEPS; -+ m->meta.global_dsqs = MAX_GLOBAL_DSQS; -+ m->meta.parse_dimens = PARSE_DIMENS; -+ m->meta.nr_cpus = num_possible_cpus(); -+ m->meta.real_cpus = nr_cpu_ids; -+ m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64); -+ m->meta.nr_meta_desc = 8; -+ m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64); -+ -+ init_desc_metas(m); -+} -+ -+#define MINIDUMP_DFL_SIZE (4 * 1024) -+ -+struct notifier_block hmbird_panic_blk; -+static int hmbird_panic_handler(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ if (!md_info) -+ return NOTIFY_DONE; -+ -+ get_hmbird_snapshot(&md_info->kern_dump.snap); -+ -+ return NOTIFY_DONE; -+} -+ -+static void panic_blk_init(void) -+{ -+ int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE); -+ -+ md_info = kzalloc(dump_size, GFP_KERNEL); -+ if (!md_info) -+ return; -+ init_md_meta(md_info); -+ -+ hmbird_panic_blk.notifier_call = hmbird_panic_handler; -+ /* make sure to execute before minidump. */ -+ hmbird_panic_blk.priority = INT_MAX; -+ atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk); -+ -+ hmbird_debug("register minidump.\n"); -+} -+ -+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size) -+{ -+ *vaddr = (unsigned long)md_info; -+ *size = sizeof(struct md_info_t); -+} -+// MTK minidump end -+ -+static inline void init_sched_prop_to_preempt_prio(void) -+{ -+ for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) { -+ switch (i) { -+ case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 5; -+ break; -+ -+ case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 4; -+ break; -+ -+ case HMBIRD_TASK_PROP_PIPELINE: -+ case HMBIRD_TASK_PROP_ISOLATE: -+ sched_prop_to_preempt_prio[i] = 3; -+ break; -+ -+ case HMBIRD_TASK_PROP_COMMON: -+ sched_prop_to_preempt_prio[i] = 1; -+ break; -+ -+ case HMBIRD_TASK_PROP_DEBUG_OR_LOG: -+ sched_prop_to_preempt_prio[i] = 0; -+ break; -+ -+ default: -+ sched_prop_to_preempt_prio[i] = 2; -+ break; -+ } -+ } -+} -+ -+void __init init_sched_hmbird_class(void) -+{ -+ int cpu; -+ u32 v; -+ struct hmbird_entity *init_hmbird; -+ -+ /* -+ * The following is to prevent the compiler from optimizing out the enum -+ * definitions so that BPF scheduler implementations can use them -+ * through the generated vmlinux.h. -+ */ -+ WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP | -+ HMBIRD_KICK_PREEMPT); -+ -+ init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL); -+ init_dsq_at_boot(); -+ init_isolate_cpus(); -+ hb_timer_init(); -+ init_sched_prop_to_preempt_prio(); -+#ifdef CONFIG_SMP -+ WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); -+#endif -+ -+ /* -+ * we can't static init init_task's hmbird struct, init here. -+ * init_task->hmbird would not use during boot. -+ */ -+ init_hmbird = kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL); -+ init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird; -+ if (init_hmbird) { -+ INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo); -+ INIT_LIST_HEAD(&init_hmbird->watchdog_node); -+ init_hmbird->sticky_cpu = -1; -+ init_hmbird->holding_cpu = -1; -+ atomic64_set(&init_hmbird->ops_state, 0); -+ init_hmbird->runnable_at = jiffies; -+ init_hmbird->slice = HMBIRD_SLICE_DFL; -+ init_hmbird->task = &init_task; -+ hmbird_set_sched_prop(&init_task, 0); -+ } else { -+ hmbird_err(INIT_TASK_FAIL, ":alloc init_task.scx failed!!!\n"); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ /* -+ * exec during boot phase, no need to care about alloc failed. -+ * lifecycle same to rq, no need to free. -+ */ -+ rq->android_oem_data1[HMBIRD_RQ_IDX] = -+ (u64)kmalloc(sizeof(struct hmbird_rq), GFP_KERNEL); -+ if (get_hmbird_rq(rq)) { -+ get_hmbird_rq(rq)->rq = rq; -+ get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window; -+ } else { -+ hmbird_err(ALLOC_RQSCX_FAIL, ":alloc rq->scx failed!!!\n"); -+ } -+ -+ rq->android_oem_data1[HMBIRD_OPS_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL); -+ if (!get_hmbird_ops(rq)) -+ pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n"); -+ init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL); -+ INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list); -+ -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL)); -+ -+ hmbird_ops_init(get_hmbird_ops(rq)); -+ } -+ -+ INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn); -+ INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn); -+ hmbird_misc_init(); -+ -+ panic_blk_init(); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_misc.c b/kernel/sched/hmbird/hmbird_misc.c -new file mode 100755 -index 000000000000..52582c22052c ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_misc.c -@@ -0,0 +1,124 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * @Description: -+ * @Version: -+ * @Author: wangrui8 -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include "hmbird_sched.h" -+#include -+#include -+#include "slim.h" -+ -+noinline int tracing_mark_write(const char *buf) -+{ -+ trace_printk(buf); -+ return 0; -+} -+ -+struct yield_opt_params yield_opt_params = { -+ .enable = 0, -+ .frame_per_sec = 120, -+ .frame_time_ns = NSEC_PER_SEC / 120, -+ .yield_headroom = 10, -+}; -+ -+DEFINE_PER_CPU(struct sched_yield_state, ystate); -+ -+static inline void hmbird_yield_state_update(struct sched_yield_state *ys) -+{ -+ if (!raw_spin_is_locked(&ys->lock)) -+ return; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ -+ if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1 -+ || ys->yield_cnt_after_sleep > yield_headroom) { -+ ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP); -+ } else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) { -+ ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP); -+ } -+ ys->yield_cnt = 0; -+ ys->sleep_times = 0; -+ ys->yield_cnt_after_sleep = 0; -+} -+ -+void hmbird_skip_yield(long *skip) -+{ -+ if (!get_hmbird_ops_enabled() || !yield_opt_params.enable) -+ return; -+ unsigned long flags, sleep_now = 0; -+ struct sched_yield_state *ys; -+ int cpu = raw_smp_processor_id(), cont_yield, new_frame; -+ int frame_time_ns = yield_opt_params.frame_time_ns; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ u64 wc; -+ -+ if (!(*skip)) { -+ wc = sched_clock(); -+ ys = &per_cpu(ystate, cpu); -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ -+ cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP; -+ new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1); -+ -+ if (!cont_yield && new_frame) { -+ hmbird_yield_state_update(ys); -+ ys->last_update_time = wc; -+ ys->sleep_end = ys->last_yield_time + frame_time_ns -+ - yield_headroom * YIELD_DURATION; -+ } -+ -+ if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) { -+ *skip = true; -+ -+ sleep_now = ys->sleep_times ? -+ max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep; -+ if (wc + sleep_now > ys->sleep_end) { -+ u64 delta = ys->sleep_end - wc; -+ -+ if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION) -+ sleep_now = delta; -+ else -+ sleep_now = 0; -+ } -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ if (sleep_now) { -+ sleep_now = div64_u64(sleep_now, 1000); -+ usleep_range_state(sleep_now, sleep_now, TASK_IDLE); -+ } -+ ys->sleep_times++; -+ ys->last_yield_time = sched_clock(); -+ return; -+ } -+ if (ys->sleep_times) -+ ys->yield_cnt_after_sleep++; -+ else -+ (ys->yield_cnt)++; -+ ys->last_yield_time = wc; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+} -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops) -+{ -+ hmbird_ops->scx_enable = get_hmbird_ops_enabled; -+ hmbird_ops->check_non_task = get_non_hmbird_task; -+ hmbird_ops->hmbird_get_md_info = hmbird_get_md_info; -+} -+ -+void hmbird_misc_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_init(&ys->lock); -+ } -+ -+ set_hmbird_module_loaded(1); -+} -+ -diff --git a/kernel/sched/hmbird/hmbird_sched.h b/kernel/sched/hmbird/hmbird_sched.h -new file mode 100755 -index 000000000000..6b5ef43cb827 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef __HMBIRD_SCHED__ -+#define __HMBIRD_SCHED__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include "../../time/tick-sched.h" -+#include "../../sched/sched.h" -+#include -+#include -+#include -+ -+#define REGISTER_TRACE_VH(vender_hook, handler) \ -+{ \ -+ ret = register_trace_##vender_hook(handler, NULL); \ -+ if (ret) { \ -+ pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \ -+ } \ -+} -+#define REGISTER_TRACE(vendor_hook, handler, data, err) \ -+do { \ -+ ret = register_trace_##vendor_hook(handler, data); \ -+ if (ret) { \ -+ pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret); \ -+ goto err; \ -+ } \ -+} while (0) -+ -+#define UNREGISTER_TRACE(vendor_hook, handler, data) \ -+ unregister_trace_##vendor_hook(handler, data) \ -+ -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+ -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int sched_ravg_window_frame_per_sec; -+extern int slim_gov_debug; -+extern int cpu7_tl; -+extern int scx_gov_ctrl; -+extern spinlock_t new_sched_ravg_window_lock; -+extern int cluster_separate; -+ -+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER BIT(31) -+#define MAX_YIELD_SLEEP (2000000ULL) -+#define MIN_YIELD_SLEEP (200000ULL) -+#define YIELD_DURATION (5000ULL) -+#define DEFAULT_YIELD_SLEEP_TH (10) -+ -+struct sched_yield_state { -+ raw_spinlock_t lock; -+ u64 last_yield_time; -+ u64 last_update_time; -+ u64 sleep_end; -+ unsigned long yield_cnt; -+ unsigned long yield_cnt_after_sleep; -+ unsigned long sleep; -+ int sleep_times; -+}; -+ -+DECLARE_PER_CPU(struct sched_yield_state, ystate); -+ -+void hmbird_window_rollover_run_once(struct rq *rq); -+void hmbird_yield_state_update_per_frame(void); -+void hmbird_misc_init(void); -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops); -+#endif /*__HMBIRD_SCHED__*/ -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.c b/kernel/sched/hmbird/hmbird_sched_proc.c -new file mode 100755 -index 000000000000..234768fc767a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.c -@@ -0,0 +1,527 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "hmbird_sched_proc.h" -+#include -+ -+#include "hmbird_util_track.h" -+#include "slim.h" -+ -+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched" -+#define SLIM_FREQ_GOV_DIR "slim_freq_gov" -+#define LOAD_TRACK_DIR "slim_walt" -+#define HMBIRD_PROC_PERMISSION 0666 -+ -+int scx_enable; -+int partial_enable; -+int cpuctrl_high_ratio = 55; -+int cpuctrl_low_ratio = 40; -+int slim_stats; -+int hmbirdcore_debug; -+int slim_for_app; -+int misfit_ds = 90; -+unsigned int highres_tick_ctrl; -+unsigned int highres_tick_ctrl_dbg; -+int cpu7_tl = 70; -+int slim_walt_ctrl; -+int slim_walt_dump; -+int slim_walt_policy; -+int slim_gov_debug; -+int scx_gov_ctrl = 1; -+int sched_ravg_window_frame_per_sec = 125; -+int parctrl_high_ratio = 55; -+int parctrl_low_ratio = 40; -+int parctrl_high_ratio_l = 65; -+int parctrl_low_ratio_l = 50; -+int isoctrl_high_ratio = 75; -+int isoctrl_low_ratio = 60; -+int isolate_ctrl; -+int iso_free_rescue; -+int heartbeat; -+int heartbeat_enable = 1; -+int watchdog_enable; -+int save_gov; -+u64 cpu_cluster_masks; -+int hmbird_preempt_policy; -+int cluster_separate; -+ -+char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= 32) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtoint(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoint\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+static int set_proc_buf_val_u64(struct file *file, const char __user *buf, -+ size_t count, u64 *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= sizeof(kbuf)) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtou64(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoul\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/* common ops begin */ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ return count; -+} -+ -+static int hmbird_common_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%d\n", *(int *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_show, pde_data(inode)); -+} -+ -+static int hmbird_common_ul_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%lu\n", *(unsigned long *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_ul_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_ul_show, pde_data(inode)); -+} -+ -+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write); -+/* common ops end */ -+ -+/* scx_enable ops begin */ -+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC); -+ if (hmbird_ctrl(*pval)) -+ return -EFAULT; -+ -+ return count; -+} -+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write); -+/* scx_enable ops end */ -+ -+/* hmbird_stats ops begin */ -+#define MAX_STATS_BUF (4096) -+static int hmbird_stats_proc_show(struct seq_file *m, void *v) -+{ -+ char *buf; -+ -+ buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ stats_print(buf, MAX_STATS_BUF); -+ -+ seq_printf(m, "%s\n", buf); -+ -+ kfree(buf); -+ return 0; -+} -+ -+static int hmbird_stats_proc_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_stats_proc_show, inode); -+} -+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL); -+/* hmbird_stats ops end */ -+ -+/* sched_ravg_window_frame_per_sec ops begin */ -+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file, -+ const char __user *buf, size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ sched_ravg_window_change(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open, -+ sched_ravg_window_frame_per_sec_proc_write); -+/* sched_ravg_window_frame_per_sec ops end */ -+ -+static ssize_t save_gov_str(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ for_each_possible_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy || (cpu != policy->cpu)) -+ continue; -+ WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0); -+ hmbird_info_systrace(":save origin gov : %s\n", saved_gov[cpu]); -+ } -+ return count; -+} -+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str); -+ -+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ u64 *pval = (u64 *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val_u64(file, buf, count, pval)) -+ return -EFAULT; -+ -+ if (scx_enable == 0) -+ set_cpu_cluster(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write); -+ -+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ int tmp_val; -+ -+ if (set_proc_buf_val(file, buf, count, &tmp_val)) -+ return -EFAULT; -+ -+ slim_walt_enable(tmp_val); -+ *pval = tmp_val; -+ -+ return count; -+} -+ -+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write); -+ -+/* yield_opt ops begin */ -+static int yield_opt_show(struct seq_file *m, void *v) -+{ -+ struct yield_opt_params *data = m->private; -+ -+ seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n", -+ data->enable, data->frame_per_sec, data->yield_headroom); -+ return 0; -+} -+ -+static int yield_opt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, yield_opt_show, pde_data(inode)); -+} -+ -+static ssize_t yield_opt_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu; -+ unsigned long flags; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if (copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp -+ != 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) || -+ (yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp; -+ yield_opt_params.frame_per_sec = frame_per_sec_tmp; -+ yield_opt_params.yield_headroom = yield_headroom_tmp; -+ yield_opt_params.enable = enable_tmp; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ ys->last_yield_time = 0; -+ ys->last_update_time = 0; -+ ys->sleep_end = 0; -+ ys->yield_cnt = 0; -+ ys->yield_cnt_after_sleep = 0; -+ ys->sleep = 0; -+ ys->sleep_times = 0; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write); -+ -+ -+static int __init hmbird_proc_init(void) -+{ -+ struct proc_dir_entry *hmbird_dir; -+ struct proc_dir_entry *load_track_dir; -+ struct proc_dir_entry *freq_gov_dir; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ -+ /* mkdir /proc/hmbird_sched */ -+ hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL); -+ if (!hmbird_dir) { -+ pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &scx_enable_proc_ops, -+ &scx_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &partial_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_stats); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbirdcore_debug); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_for_app); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &misfit_ds); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl_dbg); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpu7_tl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &cpu_cluster_masks_proc_ops, -+ &cpu_cluster_masks); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &save_gov_proc_ops, -+ &save_gov); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &watchdog_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isolate_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &iso_free_rescue); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_stats_proc_ops); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &yield_opt_proc_ops, -+ &yield_opt_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbird_preempt_policy); -+ /* /proc/hmbird_sched--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_walt */ -+ load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir); -+ if (!load_track_dir) { -+ pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_walt--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &slim_walt_ctrl_proc_ops, -+ &slim_walt_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_dump); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_policy); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &sched_ravg_window_frame_per_sec_proc_ops, -+ &sched_ravg_window_frame_per_sec); -+ /* /proc/hmbird_sched/slim_walt--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_freq_gov */ -+ freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir); -+ if (!freq_gov_dir) { -+ pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_freq_gov--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &slim_gov_debug); -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &scx_gov_ctrl); -+ /* /proc/hmbird_sched/slim_freq_gov--end */ -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cluster_separate); -+ -+ return 0; -+} -+ -+device_initcall(hmbird_proc_init); -diff --git a/kernel/sched/hmbird/hmbird_sched_proc.h b/kernel/sched/hmbird/hmbird_sched_proc.h -new file mode 100755 -index 000000000000..e22bc75f1dd7 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_sched_proc.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SCHED_PROC_H__ -+#define __HMBIRD_SCHED_PROC_H__ -+ -+#include -+#include -+ -+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \ -+ do { \ -+ if (!proc_create(name, mode, parent, proc_ops)) { \ -+ pr_err("Error creating proc entry %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \ -+ do { \ -+ if (!proc_create_data(name, mode, parent, proc_ops, data)) { \ -+ pr_err("Error creating proc entry with data %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_PROC_OPS(name, open_func, write_func) \ -+ static const struct proc_ops name##_proc_ops = { \ -+ .proc_open = open_func, \ -+ .proc_write = write_func, \ -+ .proc_read = seq_read, \ -+ .proc_lseek = seq_lseek, \ -+ .proc_release = single_release, \ -+ } -+ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos); -+static int hmbird_common_show(struct seq_file *m, void *v); -+static int hmbird_common_open(struct inode *inode, struct file *file); -+ -+#endif -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.c b/kernel/sched/hmbird/hmbird_shadow_tick.c -new file mode 100755 -index 000000000000..21de551c5132 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.c -@@ -0,0 +1,159 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include -+#include "../../time/tick-sched.h" -+#include -+ -+#include "hmbird_shadow_tick.h" -+ -+#define HIGHRES_WATCH_CPU 0 -+ -+#include -+static bool shadow_tick_enable(void) {return highres_tick_ctrl; } -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; } -+#endif -+static bool shadow_tick_timer_init_flag; -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define shadow_tick_printk(fmt, args...) \ -+do { \ -+ int cpu = smp_processor_id(); \ -+ if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU) \ -+ trace_printk("hmbird shadow tick :"fmt, args); \ -+} while (0) -+#else -+#define shadow_tick_printk(fmt, args...) -+#endif -+ -+DEFINE_PER_CPU(struct hrtimer, stt); -+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu))) -+#define STOP_IDLE_TRIGGER (1) -+#define PERIODIC_TICK_TRIGGER (2) -+#define TICK_INTVAL (1000000ULL) -+/* -+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms, -+ * so we can't restart hrtimer in scheduler tick. -+ */ -+static DEFINE_PER_CPU(u8, trigger_event); -+ -+/* -+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick. -+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler. -+ */ -+static DEFINE_PER_CPU(u8, tick_phase); -+ -+static inline void highres_timer_ctrl(bool enable, int cpu) -+{ -+ if (enable && hmbird_enabled()) { -+ if (!hrtimer_active(shadow_tick_timer(cpu))) -+ hrtimer_start(shadow_tick_timer(cpu), -+ ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED); -+ } else { -+ if (!enable) -+ hrtimer_cancel(shadow_tick_timer(cpu)); -+ } -+} -+ -+static inline void high_res_clear_phase(int cpu) -+{ -+ per_cpu(tick_phase, cpu) = 0; -+} -+ -+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer) -+{ -+ per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3; -+ if (per_cpu(tick_phase, cpu)) { -+ hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) { -+ per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ struct rq_flags rf; -+ -+ rq_lock(rq, &rf); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ rq_unlock(rq, &rf); -+ -+ return highres_next_phase(cpu, timer); -+} -+ -+void shadow_tick_timer_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance; -+ } -+} -+ -+void start_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable()) { -+ if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER) -+ highres_timer_ctrl(false, cpu); -+ per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static void stop_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ per_cpu(trigger_event, cpu) = 0; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(false, cpu); -+} -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ stop_shadow_tick_timer(); -+} -+ -+void scheduler_tick_handler(void *unused, struct rq *rq) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ start_shadow_tick_timer(); -+} -+ -+static int __init hmbird_shadow_tick_init(void) -+{ -+ int ret = 0; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ shadow_tick_timer_init(); -+ shadow_tick_timer_init_flag = true; -+ return ret; -+} -+ -+device_initcall(hmbird_shadow_tick_init); -diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.h b/kernel/sched/hmbird/hmbird_shadow_tick.h -new file mode 100755 -index 000000000000..0c26fd984f0a ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_shadow_tick.h -@@ -0,0 +1,11 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SHADOW_TICK_H__ -+#define __HMBIRD_SHADOW_TICK_H__ -+ -+#include -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+void scheduler_tick_handler(void *unused, struct rq *rq); -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state); -+#endif -diff --git a/kernel/sched/hmbird/hmbird_trace.h b/kernel/sched/hmbird/hmbird_trace.h -new file mode 100755 -index 000000000000..8468b406f347 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_trace.h -@@ -0,0 +1,92 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hmbird -+ -+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HMBIRD_H -+ -+#include -+#include -+#include -+ -+#define MAX_FATAL_INFO (64) -+ -+TRACE_EVENT(hmbird_fatal_info, -+ -+ TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info), -+ -+ TP_ARGS(type, pe, lnr, bnr, info), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, type) -+ __field(int, pe) -+ __field(int, lnr) -+ __field(int, bnr) -+ __array(char, info, MAX_FATAL_INFO)), -+ -+ TP_fast_assign( -+ __entry->type = type; -+ __entry->pe = pe; -+ __entry->lnr = lnr; -+ __entry->bnr = bnr; -+ memcpy(__entry->info, info, MAX_FATAL_INFO);), -+ -+ TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s", -+ __entry->type, __entry->pe, __entry->lnr, __entry->bnr, -+ __entry->info) -+); -+ -+TRACE_EVENT(hmbird_update_history, -+ -+ TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq, -+ struct task_struct *p, u32 runtime, int samples, int event), -+ -+ TP_ARGS(hmbird, rq, p, runtime, samples, event), -+ -+ TP_STRUCT__entry( -+ __array(char, comm, TASK_COMM_LEN) -+ __field(pid_t, pid) -+ __field(unsigned int, runtime) -+ __field(int, samples) -+ __field(int, event) -+ __field(unsigned int, demand) -+ __array(u32, hist, RAVG_HIST_SIZE) -+ __field(u16, task_util) -+ __field(int, cpu)), -+ -+ TP_fast_assign( -+ memcpy(__entry->comm, p->comm, TASK_COMM_LEN); -+ __entry->pid = p->pid; -+ __entry->runtime = runtime; -+ __entry->samples = samples; -+ __entry->event = event; -+ __entry->demand = hmbird->sts.demand; -+ memcpy(__entry->hist, hmbird->sts.sum_history, -+ RAVG_HIST_SIZE * sizeof(u32)); -+ __entry->task_util = hmbird->sts.demand_scaled; -+ __entry->cpu = rq->cpu;), -+ -+ TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d", -+ __entry->comm, __entry->pid, -+ __entry->runtime, __entry->samples, -+ __entry->event, -+ __entry->demand, -+ __entry->hist[0], __entry->hist[1], -+ __entry->hist[2], __entry->hist[3], -+ __entry->hist[4], -+ __entry->task_util, -+ __entry->cpu) -+); -+ -+#endif /*_TRACE_HMBIRD_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird -+ -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE hmbird_trace -+/* This part must be outside protection */ -+#include -diff --git a/kernel/sched/hmbird/hmbird_util_track.c b/kernel/sched/hmbird/hmbird_util_track.c -new file mode 100755 -index 000000000000..d520a8403401 ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.c -@@ -0,0 +1,626 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+ -+#define CREATE_TRACE_POINTS -+#include "hmbird_trace.h" -+#undef CREATE_TRACE_POINTS -+ -+#define HMBIRD_DEBUG_PANIC (1 << 3) -+static bool init_irq_work_inited; -+ -+extern noinline int tracing_mark_write(const char *buf); -+ -+int hmbird_sched_ravg_window = 8000000; -+int new_hmbird_sched_ravg_window = 8000000; -+DEFINE_SPINLOCK(new_sched_ravg_window_lock); -+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+ -+static struct irq_work hmbird_slim_walt_irq_work; -+ -+/*Sysctl related interface*/ -+#define WINDOW_STATS_RECENT 0 -+#define WINDOW_STATS_MAX 1 -+#define WINDOW_STATS_MAX_RECENT_AVG 2 -+#define WINDOW_STATS_AVG 3 -+#define WINDOW_STATS_INVALID_POLICY 4 -+ -+ -+#define SCX_SCHED_CAPACITY_SHIFT 10 -+#define SCHED_ACCOUNT_WAIT_TIME 0 -+ -+atomic64_t hmbird_irq_work_lastq_ws; -+static u64 tick_sched_clock; -+ -+static inline u64 scale_exec_time(u64 delta, struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT; -+} -+ -+static inline u64 scale_time_to_util(u64 d) -+{ -+ do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT); -+ return d; -+} -+ -+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ delta = scale_exec_time(delta, rq); -+ sts->sum += delta; -+ if (unlikely(sts->sum > hmbird_sched_ravg_window)) -+ sts->sum = hmbird_sched_ravg_window; -+ -+ return delta; -+} -+ -+ -+static int -+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) -+{ -+ /* -+ * No need to bother updating task demand for the idle task. -+ */ -+ if (is_idle_task(p)) -+ return 0; -+ -+ /* -+ * When a task is waking up it is completing a segment of non-busy -+ * time. Likewise, if wait time is not treated as busy time, then -+ * when a task begins to run or is migrated, it is not running and -+ * is completing a segment of non-busy time. -+ */ -+ if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && -+ (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) -+ return 0; -+ -+ /* -+ * The idle exit time is not accounted for the first task _picked_ up to -+ * run on the idle CPU. -+ */ -+ if (event == PICK_NEXT_TASK && rq->curr == rq->idle) -+ return 0; -+ -+ /* -+ * TASK_UPDATE can be called on sleeping task, when its moved between -+ * related groups -+ */ -+ if (event == TASK_UPDATE) { -+ if (rq->curr == p) -+ return 1; -+ -+ return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; -+ } -+ -+ return 1; -+} -+ -+static void rollover_cpu_window(struct rq *rq, bool full_window) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 curr_sum = srq->curr_runnable_sum; -+ -+ if (unlikely(full_window)) -+ curr_sum = 0; -+ -+ srq->prev_runnable_sum = curr_sum; -+ srq->curr_runnable_sum = 0; -+} -+ -+static u64 -+update_window_start(struct rq *rq, u64 wallclock, int event) -+{ -+ s64 delta; -+ int nr_windows; -+ bool full_window; -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start = srq->window_start; -+ -+ if (wallclock < srq->latest_clock) -+ wallclock = srq->latest_clock; -+ delta = wallclock - srq->window_start; -+ if (delta < 0) { -+ delta = 0; -+ wallclock = srq->window_start; -+ } -+ srq->latest_clock = wallclock; -+ if (delta < hmbird_sched_ravg_window) -+ return old_window_start; -+ -+ nr_windows = div64_u64(delta, hmbird_sched_ravg_window); -+ srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window; -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ full_window = nr_windows > 1; -+ rollover_cpu_window(rq, full_window); -+ -+ return old_window_start; -+} -+ -+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -+ -+static inline unsigned int get_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static inline unsigned int cpu_cur_freq(int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cur; -+} -+ -+static void -+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, -+ u64 wallclock) -+{ -+ int cpu = cpu_of(rq); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * -+ arch_scale_cpu_capacity(cpu), get_max_freq(cpu)); -+} -+ -+/* -+ * Called when new window is starting for a task, to record cpu usage over -+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 -+ * when, say, a real-time task runs without preemption for several windows at a -+ * stretch. -+ */ -+static void update_history(struct rq *rq, struct task_struct *p, -+ u32 runtime, int samples, int event) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u32 *hist = &sts->sum_history[0]; -+ int i; -+ u32 max = 0, avg, demand; -+ u64 sum = 0; -+ u16 demand_scaled; -+ -+ /* Ignore windows where task had no activity */ -+ if (!runtime || is_idle_task(p) || !samples) -+ goto done; -+ -+ /* Push new 'runtime' value onto stack */ -+ for (; samples > 0; samples--) { -+ hist[sts->cidx] = runtime; -+ sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE; -+ } -+ -+ for (i = 0; i < RAVG_HIST_SIZE; i++) { -+ sum += hist[i]; -+ if (hist[i] > max) -+ max = hist[i]; -+ } -+ -+ sts->sum = 0; -+ avg = div64_u64(sum, RAVG_HIST_SIZE); -+ -+ switch (slim_walt_policy) { -+ case WINDOW_STATS_RECENT: -+ demand = runtime; -+ break; -+ case WINDOW_STATS_MAX: -+ demand = max; -+ break; -+ case WINDOW_STATS_AVG: -+ demand = avg; -+ break; -+ default: -+ demand = max(avg, runtime); -+ } -+ -+ demand_scaled = scale_time_to_util(demand); -+ -+ sts->demand = demand; -+ sts->demand_scaled = demand_scaled; -+ -+done: -+ return; -+} -+ -+ -+static u64 update_task_demand(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ -+ u64 delta, window_start = srq->window_start; -+ int new_window, nr_full_windows; -+ u32 window_size = hmbird_sched_ravg_window; -+ u64 runtime; -+ -+ new_window = mark_start < window_start; -+ if (!account_busy_for_task_demand(rq, p, event)) { -+ if (new_window) -+ /* -+ * If the time accounted isn't being accounted as -+ * busy time, and a new window started, only the -+ * previous window need be closed out with the -+ * pre-existing demand. Multiple windows may have -+ * elapsed, but since empty windows are dropped, -+ * it is not necessary to account those. -+ */ -+ update_history(rq, p, sts->sum, 1, event); -+ return 0; -+ } -+ -+ if (!new_window) { -+ /* -+ * The simple case - busy time contained within the existing -+ * window. -+ */ -+ return add_to_task_demand(rq, p, wallclock - mark_start); -+ } -+ -+ /* -+ * Busy time spans at least two windows. Temporarily rewind -+ * window_start to first window boundary after mark_start. -+ */ -+ delta = window_start - mark_start; -+ nr_full_windows = div64_u64(delta, window_size); -+ window_start -= (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (window_start - mark_start) first */ -+ runtime = add_to_task_demand(rq, p, window_start - mark_start); -+ -+ /* Push new sample(s) into task's demand history */ -+ update_history(rq, p, sts->sum, 1, event); -+ if (nr_full_windows) { -+ u64 scaled_window = scale_exec_time(window_size, rq); -+ -+ update_history(rq, p, scaled_window, nr_full_windows, event); -+ runtime += nr_full_windows * scaled_window; -+ } -+ -+ /* -+ * Roll window_start back to current to process any remainder -+ * in current window. -+ */ -+ window_start += (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (wallclock - window_start) next */ -+ mark_start = window_start; -+ runtime += add_to_task_demand(rq, p, wallclock - mark_start); -+ -+ return runtime; -+} -+ -+u16 slim_walt_cpu_util(int cpu) -+{ -+ u64 prev_runnable_sum; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ prev_runnable_sum = srq->prev_runnable_sum; -+ do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT); -+ -+ return (u16)prev_runnable_sum; -+} -+ -+static DEFINE_PER_CPU(u16, prev_cpu_util); -+static inline void cpu_util_update_systrace_c(int cpu) -+{ -+ char buf[256]; -+ u16 cpu_util = slim_walt_cpu_util(cpu); -+ -+ if (cpu_util != per_cpu(prev_cpu_util, cpu)) { -+ snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n", -+ cpu, cpu_util); -+ tracing_mark_write(buf); -+ per_cpu(prev_cpu_util, cpu) = cpu_util; -+ } -+} -+ -+ -+static inline int account_busy_for_cpu_time(struct rq *rq, -+ struct task_struct *p, -+ int event) -+{ -+ return !is_idle_task(p) && (event == PUT_PREV_TASK -+ || event == TASK_UPDATE); -+} -+ -+ -+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ int new_window, full_window = 0; -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 window_start = srq->window_start; -+ u32 window_size = srq->prev_window_size; -+ u64 delta; -+ u64 *curr_runnable_sum = &srq->curr_runnable_sum; -+ u64 *prev_runnable_sum = &srq->prev_runnable_sum; -+ -+ new_window = mark_start < window_start; -+ if (new_window) -+ full_window = (window_start - mark_start) >= window_size; -+ -+ -+ if (!account_busy_for_cpu_time(rq, p, event)) -+ goto done; -+ -+ -+ if (!new_window) { -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. No rollover -+ * since we didn't start a new window. An example of this is -+ * when a task starts execution and then sleeps within the -+ * same window. -+ */ -+ delta = wallclock - mark_start; -+ -+ delta = scale_exec_time(delta, rq); -+ *curr_runnable_sum += delta; -+ -+ goto done; -+ } -+ -+ /* -+ * situations below this need window rollover, -+ * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done -+ * in update_window_start() -+ * -+ * For task counters curr/prev_window[_cpu] are rolled over in the early part of -+ * this function. If full_window(s) have expired and time since last update needs -+ * to be accounted as busy time, set the prev to a complete window size time, else -+ * add the prev window portion. -+ * -+ * For task curr counters a new window has begun, always assign -+ */ -+ -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. A new window -+ * must have been started in udpate_window_start() -+ * If any of these three above conditions are true -+ * then this busy time can't be accounted as irqtime. -+ * -+ * Busy time for the idle task need not be accounted. -+ * -+ * An example of this would be a task that starts execution -+ * and then sleeps once a new window has begun. -+ */ -+ -+ /* -+ * A full window hasn't elapsed, account partial -+ * contribution to previous completed window. -+ */ -+ -+ -+ delta = full_window ? scale_exec_time(window_size, rq) : -+ scale_exec_time(window_start - mark_start, rq); -+ -+ *prev_runnable_sum += delta; -+ -+ /* Account piece of busy time in the current window. */ -+ delta = scale_exec_time(wallclock - window_start, rq); -+ *curr_runnable_sum += delta; -+ -+done: -+ if (slim_walt_dump && new_window) -+ cpu_util_update_systrace_c(rq->cpu); -+} -+ -+ -+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq) -+{ -+ u64 result; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 new_window_start = srq->window_start; -+ -+ if (old_window_start == new_window_start) -+ return; -+ -+ result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws, -+ old_window_start, new_window_start); -+ if (result != old_window_start) -+ return; -+ -+ if (likely(cpu_online(raw_smp_processor_id()))) -+ irq_work_queue(&hmbird_slim_walt_irq_work); -+ else -+ irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask)); -+} -+ -+/* -+ * In the core scheduler, most of the load update points update the rq_clock after -+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of -+ * obtaining the time, but to prevent the subsequent migration of the load update -+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock -+ * update here. -+ */ -+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p, -+ struct rq *rq, int event) -+{ -+ if (!(rq->clock_update_flags & RQCF_UPDATED)) -+ update_rq_clock(rq); -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock)); -+} -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start; -+ -+ if (!slim_walt_ctrl) -+ return; -+ -+ if (!srq->window_start || sts->mark_start == wallclock) -+ return; -+ -+ old_window_start = update_window_start(rq, wallclock, event); -+ -+ if (!sts->window_start) -+ sts->window_start = srq->window_start; -+ -+ if (!sts->mark_start) -+ goto done; -+ -+ update_task_rq_cpu_cycles(p, rq, event, wallclock); -+ update_task_demand(p, rq, event, wallclock); -+ update_cpu_busy_time(p, rq, event, wallclock); -+ -+ sts->window_start = srq->window_start; -+ -+done: -+ sts->mark_start = wallclock; -+ slim_walt_window_rollover_run_once(old_window_start, rq); -+} -+ -+static void slim_walt_irq_work(struct irq_work *irq_work) -+{ -+ cpumask_t lock_cpus; -+ struct hmbird_sched_rq_stats *srq; -+ struct rq *rq; -+ int cpu; -+ int level = 0; -+ u64 wc; -+ unsigned long flags; -+ -+ cpumask_copy(&lock_cpus, cpu_possible_mask); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ if (level == 0) -+ raw_spin_lock(&cpu_rq(cpu)->__lock); -+ else -+ raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); -+ level++; -+ } -+ -+ wc = sched_clock(); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ rq = cpu_rq(cpu); -+ hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc); -+ } -+ -+ cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER); -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) { -+ srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id()); -+ if (wc < srq->window_start + new_hmbird_sched_ravg_window) -+ hmbird_sched_ravg_window = new_hmbird_sched_ravg_window; -+ } -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ raw_spin_unlock(&cpu_rq(cpu)->__lock); -+ } -+} -+static void hmbird_sched_init_rq(struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ srq->task_exec_scale = 1024; -+ srq->window_start = 0; -+} -+ -+void hmbird_sched_init_task(struct task_struct *p) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ memset(sts, 0, sizeof(struct hmbird_sched_task_stats)); -+} -+ -+static void hmbird_sched_stats_init(void) -+{ -+ unsigned long flags; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ raw_spin_lock_irqsave(&rq->__lock, flags); -+ hmbird_sched_init_rq(rq); -+ raw_spin_unlock_irqrestore(&rq->__lock, flags); -+ } -+ slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG; -+ -+ if (false == init_irq_work_inited) { -+ init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work); -+ init_irq_work_inited = true; -+ } -+} -+ -+void hmbird_scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (unlikely(!tick_sched_clock)) { -+ /* -+ * Let the window begin 20us prior to the tick, -+ * that way we are guaranteed a rollover when the tick occurs. -+ * Use rq->clock directly instead of rq_clock() since -+ * we do not have the rq lock and -+ * rq->clock was updated in the tick callpath. -+ */ -+ if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000)) -+ return; -+ for_each_possible_cpu(cpu) { -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ srq->window_start = tick_sched_clock; -+ } -+ atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock); -+ } -+} -+ -+void slim_walt_enable(int enable) -+{ -+ if (1 == !!enable) { -+ hmbird_sched_stats_init(); -+ WRITE_ONCE(tick_sched_clock, 0); -+ } else -+ slim_walt_ctrl = 0; -+} -+ -+void slim_get_cpu_util(int cpu, u64 *util) -+{ -+ if (cpu < 0) -+ return; -+ -+ *util = slim_walt_cpu_util(cpu); -+} -+ -+void slim_get_task_util(struct task_struct *p, u64 *util) -+{ -+ if (p == NULL) -+ return; -+ -+ *util = get_hmbird_ts(p)->sts.demand_scaled; -+} -+ -+void sched_ravg_window_change(int frame_per_sec) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec; -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+} -diff --git a/kernel/sched/hmbird/hmbird_util_track.h b/kernel/sched/hmbird/hmbird_util_track.h -new file mode 100755 -index 000000000000..17b85df7f83b ---- /dev/null -+++ b/kernel/sched/hmbird/hmbird_util_track.h -@@ -0,0 +1,33 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef __HMBIRD_UTIL_TRACK_H__ -+#define __HMBIRD_UTIL_TRACK_H__ -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock); -+void hmbird_sched_init_task(struct task_struct *p); -+void slim_walt_enable(int enable); -+void slim_get_cpu_util(int cpu, u64 *util); -+void slim_get_task_util(struct task_struct *p, u64 *util); -+ -+extern atomic64_t hmbird_irq_work_lastq_ws; -+ -+enum task_event { -+ PUT_PREV_TASK = 0, -+ PICK_NEXT_TASK = 1, -+ TASK_WAKE = 2, -+ TASK_MIGRATE = 3, -+ TASK_UPDATE = 4, -+ IRQ_UPDATE = 5, -+}; -+ -+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+#endif /* __HMBIRD_UTIL_TRACK_H__ */ -diff --git a/kernel/sched/hmbird/slim.h b/kernel/sched/hmbird/slim.h -new file mode 100755 -index 000000000000..eb386260e950 ---- /dev/null -+++ b/kernel/sched/hmbird/slim.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+ -+#ifndef __SLIM_H -+#define __SLIM_H -+ -+extern atomic_t __hmbird_ops_enabled; -+extern atomic_t non_hmbird_task; -+extern int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+extern int heartbeat; -+extern int heartbeat_enable; -+extern int watchdog_enable; -+extern int isolate_ctrl; -+extern int parctrl_high_ratio; -+extern int parctrl_low_ratio; -+extern int isoctrl_high_ratio; -+extern int isoctrl_low_ratio; -+extern int iso_free_rescue; -+extern int yield_opt; -+ -+extern enum hmbird_switch_type sw_type; -+extern noinline int tracing_mark_write(const char *buf); -+int task_top_id(struct task_struct *p); -+void stats_print(char *buf, int len); -+void hmbird_skip_yield(long *skip); -+extern spinlock_t hmbird_tasks_lock; -+ -+struct yield_opt_params { -+ int enable; -+ int frame_per_sec; -+ u64 frame_time_ns; -+ int yield_headroom; -+}; -+ -+extern struct yield_opt_params yield_opt_params; -+ -+#define MAX_GOV_LEN (16) -+extern char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+#endif -diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c -index b33cefeb4188..487255ac6832 100755 ---- a/kernel/sched/idle.c -+++ b/kernel/sched/idle.c -@@ -408,13 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl - - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) - { -- scx_update_idle(rq, false); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, false); -+#endif - } - - static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) - { - update_idle_core(rq); -- scx_update_idle(rq, true); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, true); -+#endif - schedstat_inc(rq->sched_goidle); - } - -diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h -index 509e8dc616ab..ab546ffc244b 100755 ---- a/kernel/sched/sched.h -+++ b/kernel/sched/sched.h -@@ -188,18 +188,13 @@ static inline int idle_policy(int policy) - return policy == SCHED_IDLE; - } - --static inline int normal_policy(int policy) --{ --#ifdef CONFIG_SCHED_CLASS_EXT -- if (policy == SCHED_EXT) -- return true; --#endif -- return policy == SCHED_NORMAL; --} -- - static inline int fair_policy(int policy) - { -- return normal_policy(policy) || policy == SCHED_BATCH; -+#ifdef CONFIG_HMBIRD_SCHED -+ return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#else -+ return policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#endif - } - - static inline int rt_policy(int policy) -@@ -247,24 +242,6 @@ static inline void update_avg(u64 *avg, u64 sample) - #define shr_bound(val, shift) \ - (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) - --/* -- * cgroup weight knobs should use the common MIN, DFL and MAX values which are -- * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it -- * maps pretty well onto the shares value used by scheduler and the round-trip -- * conversions preserve the original value over the entire range. -- */ --static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) --{ -- return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); --} -- --static inline unsigned long sched_weight_to_cgroup(unsigned long weight) --{ -- return clamp_t(unsigned long, -- DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -- CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); --} -- - /* - * !! For sched_setattr_nocheck() (kernel) only !! - * -@@ -532,10 +509,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) { } - #endif /* CONFIG_SMP */ - #else /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_HMBIRD_SCHED - static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) - { - return 0; - } -+#endif - #endif /* CONFIG_FAIR_GROUP_SCHED */ - - #else /* CONFIG_CGROUP_SCHED */ -@@ -700,29 +679,6 @@ struct cfs_rq { - #endif /* CONFIG_FAIR_GROUP_SCHED */ - }; - --#ifdef CONFIG_SCHED_CLASS_EXT --/* scx_rq->flags, protected by the rq lock */ --enum scx_rq_flags { -- SCX_RQ_CAN_STOP_TICK = 1 << 0, --}; -- --struct scx_rq { -- struct scx_dispatch_q local_dsq; -- struct list_head watchdog_list; -- u64 ops_qseq; -- u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -- u32 nr_running; -- u32 flags; -- bool cpu_released; -- cpumask_var_t cpus_to_kick; -- cpumask_var_t cpus_to_preempt; -- cpumask_var_t cpus_to_wait; -- u64 pnt_seq; -- struct irq_work kick_cpus_irq_work; -- struct rq *rq; --}; --#endif /* CONFIG_SCHED_CLASS_EXT */ -- - static inline int rt_bandwidth_enabled(void) - { - return sysctl_sched_rt_runtime >= 0; -@@ -1258,11 +1214,7 @@ struct rq { - - ANDROID_OEM_DATA_ARRAY(1, 16); - --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, struct scx_rq *scx); --#else - ANDROID_KABI_RESERVE(1); --#endif - ANDROID_KABI_RESERVE(2); - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); -@@ -2340,11 +2292,6 @@ struct affinity_context { - unsigned int flags; - }; - --enum rq_onoff_reason { -- RQ_ONOFF_HOTPLUG, /* CPU is going on/offline */ -- RQ_ONOFF_TOPOLOGY, /* sched domain topology update */ --}; -- - struct sched_class { - - #ifdef CONFIG_UCLAMP_TASK -@@ -2551,7 +2498,6 @@ extern void init_sched_rt_class(void); - extern void init_sched_fair_class(void); - - extern void reweight_task(struct task_struct *p, int prio); --extern void __setscheduler_prio(struct task_struct *p, int prio); - - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); -@@ -2632,53 +2578,6 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) - extern void activate_task(struct rq *rq, struct task_struct *p, int flags); - extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); - --struct sched_change_guard { -- struct task_struct *p; -- struct rq *rq; -- bool queued; -- bool running; -- bool done; --}; -- --extern struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -- --extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags); -- --/** -- * SCHED_CHANGE_BLOCK - Nested block for task attribute updates -- * @__rq: Runqueue the target task belongs to -- * @__p: Target task -- * @__flags: DEQUEUE/ENQUEUE_* flags -- * -- * A task may need to be dequeued and put_prev_task'd for attribute updates and -- * set_next_task'd and re-enqueued afterwards. This helper defines a nested -- * block which automatically handles these preparation and cleanup operations. -- * -- * SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- * update_attribute(p); -- * ... -- * } -- * -- * If @__flags is a variable, the variable may be updated in the block body and -- * the updated value will be used when re-enqueueing @p. -- * -- * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling -- * update_rq_clock() beforehand. Otherwise, the rq clock is automatically -- * updated iff the task needs to be dequeued and re-enqueued. Only the former -- * case guarantees that the rq clock is up-to-date inside and after the block. -- */ --#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -- for (struct sched_change_guard __cg = \ -- sched_change_guard_init(__rq, __p, __flags); \ -- !__cg.done; sched_change_guard_fini(&__cg, __flags)) -- --extern void check_class_changing(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class); --extern void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio); -- - extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); - - #ifdef CONFIG_PREEMPT_RT -@@ -3710,6 +3609,8 @@ static inline bool cpu_busy_with_softirqs(int cpu) - } - #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ - --#include "ext.h" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird.h" -+#endif - - #endif /* _KERNEL_SCHED_SCHED_H */ -diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c -index 998c2eefe173..2a3f819b9e88 100755 ---- a/kernel/time/tick-sched.c -+++ b/kernel/time/tick-sched.c -@@ -34,6 +34,10 @@ - - #include - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "../sched/hmbird/hmbird_shadow_tick.h" -+#endif -+ - /* - * Per-CPU nohz control structure - */ -@@ -1112,6 +1116,9 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) - return true; - } - -+#ifdef CONFIG_HMBIRD_SCHED -+extern void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+#endif - /** - * tick_nohz_idle_stop_tick - stop the idle tick from the idle task - * -@@ -1124,6 +1131,9 @@ void tick_nohz_idle_stop_tick(void) - ktime_t expires; - - trace_android_vh_tick_nohz_idle_stop_tick(NULL); -+#ifdef CONFIG_HMBIRD_SCHED -+ android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL); -+#endif - /* - * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the - * tick timer expiration time is known already. -diff --git a/tools/sched_ext/.gitignore b/tools/sched_ext/.gitignore -deleted file mode 100644 -index a3240f9f7eba..000000000000 ---- a/tools/sched_ext/.gitignore -+++ /dev/null -@@ -1,9 +0,0 @@ --scx_example_simple --scx_example_qmap --scx_example_central --scx_example_pair --scx_example_flatcg --scx_example_userland --*.skel.h --*.subskel.h --/tools/ -diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile -deleted file mode 100644 -index 73c43782837d..000000000000 ---- a/tools/sched_ext/Makefile -+++ /dev/null -@@ -1,216 +0,0 @@ --# SPDX-License-Identifier: GPL-2.0 --# Copyright (c) 2022 Meta Platforms, Inc. and affiliates. --include ../build/Build.include --include ../scripts/Makefile.arch --include ../scripts/Makefile.include -- --ifneq ($(LLVM),) --ifneq ($(filter %/,$(LLVM)),) --LLVM_PREFIX := $(LLVM) --else ifneq ($(filter -%,$(LLVM)),) --LLVM_SUFFIX := $(LLVM) --endif -- --CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi --CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu --CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl --CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu --CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu --CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu --CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu --CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu --CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu --CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) -- --ifeq ($(CROSS_COMPILE),) --ifeq ($(CLANG_TARGET_FLAGS),) --$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk --else --CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) --endif # CLANG_TARGET_FLAGS --else --CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) --endif # CROSS_COMPILE -- --CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as --else --CC := $(CROSS_COMPILE)gcc --endif # LLVM -- --CURDIR := $(abspath .) --TOOLSDIR := $(abspath ..) --LIBDIR := $(TOOLSDIR)/lib --BPFDIR := $(LIBDIR)/bpf --TOOLSINCDIR := $(TOOLSDIR)/include --BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool --APIDIR := $(TOOLSINCDIR)/uapi --GENDIR := $(abspath ../../include/generated) --GENHDR := $(GENDIR)/autoconf.h -- --SCRATCH_DIR := $(CURDIR)/tools --BUILD_DIR := $(SCRATCH_DIR)/build --INCLUDE_DIR := $(SCRATCH_DIR)/include --BPFOBJ_DIR := $(BUILD_DIR)/libbpf --BPFOBJ := $(BPFOBJ_DIR)/libbpf.a --ifneq ($(CROSS_COMPILE),) --HOST_BUILD_DIR := $(BUILD_DIR)/host --HOST_SCRATCH_DIR := host-tools --HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include --else --HOST_BUILD_DIR := $(BUILD_DIR) --HOST_SCRATCH_DIR := $(SCRATCH_DIR) --HOST_INCLUDE_DIR := $(INCLUDE_DIR) --endif --HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a --RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids --DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -- --VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ -- $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ -- ../../vmlinux \ -- /sys/kernel/btf/vmlinux \ -- /boot/vmlinux-$(shell uname -r) --VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) --ifeq ($(VMLINUX_BTF),) --$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") --endif -- --BPFTOOL ?= $(DEFAULT_BPFTOOL) -- --ifneq ($(wildcard $(GENHDR)),) -- GENFLAGS := -DHAVE_GENHDR --endif -- --CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ -- -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -- -I$(TOOLSINCDIR) -I$(APIDIR) -- --CARGOFLAGS := --release -- --# Silence some warnings when compiled with clang --ifneq ($(LLVM),) --CFLAGS += -Wno-unused-command-line-argument --endif -- --LDFLAGS = -lelf -lz -lpthread -- --IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - &1 \ -- | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ --$(shell $(1) -dM -E - $@ --else -- $(call msg,CP,,$@) -- $(Q)cp "$(VMLINUX_H)" $@ --endif -- --%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \ -- | $(BPFOBJ) -- $(call msg,CLNG-BPF,,$@) -- $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ -- --%.skel.h: %.bpf.o $(BPFTOOL) -- $(call msg,GEN-SKEL,,$@) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) -- $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) -- $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@ -- $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h) -- --scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_userland: scx_example_userland.c scx_example_userland.skel.h \ -- scx_example_userland_common.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR) --atropos: export ATROPOS_CLANG = $(CLANG) --atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS) --atropos: $(INCLUDE_DIR)/vmlinux.h -- cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS) -- --clean: -- cargo clean --manifest-path=atropos/Cargo.toml -- rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) -- rm -f *.o *.bpf.o *.skel.h *.subskel.h -- rm -f scx_example_simple scx_example_qmap scx_example_central \ -- scx_example_pair scx_example_flatcg scx_example_userland -- --.PHONY: all atropos clean -- --# delete failed targets --.DELETE_ON_ERROR: -- --# keep intermediate (.skel.h, .bpf.o, etc) targets --.SECONDARY: -diff --git a/tools/sched_ext/atropos/.gitignore b/tools/sched_ext/atropos/.gitignore -deleted file mode 100644 -index 186dba259ec2..000000000000 ---- a/tools/sched_ext/atropos/.gitignore -+++ /dev/null -@@ -1,3 +0,0 @@ --src/bpf/.output --Cargo.lock --target -diff --git a/tools/sched_ext/atropos/Cargo.toml b/tools/sched_ext/atropos/Cargo.toml -deleted file mode 100644 -index 7462a836d53d..000000000000 ---- a/tools/sched_ext/atropos/Cargo.toml -+++ /dev/null -@@ -1,28 +0,0 @@ --[package] --name = "scx_atropos" --version = "0.5.0" --authors = ["Dan Schatzberg ", "Meta"] --edition = "2021" --description = "Userspace scheduling with BPF" --license = "GPL-2.0-only" -- --[dependencies] --anyhow = "1.0.65" --bitvec = { version = "1.0", features = ["serde"] } --clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } --ctrlc = { version = "3.1", features = ["termination"] } --fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"} --hex = "0.4.3" --libbpf-rs = "0.19.1" --libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] } --libc = "0.2.137" --log = "0.4.17" --ordered-float = "3.4.0" --simplelog = "0.12.0" -- --[build-dependencies] --bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false } --libbpf-cargo = "0.13.0" -- --[features] --enable_backtrace = [] -diff --git a/tools/sched_ext/atropos/build.rs b/tools/sched_ext/atropos/build.rs -deleted file mode 100644 -index 26e792c5e17e..000000000000 ---- a/tools/sched_ext/atropos/build.rs -+++ /dev/null -@@ -1,70 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --extern crate bindgen; -- --use std::env; --use std::fs::create_dir_all; --use std::path::Path; --use std::path::PathBuf; -- --use libbpf_cargo::SkeletonBuilder; -- --const HEADER_PATH: &str = "src/bpf/atropos.h"; -- --fn bindgen_atropos() { -- // Tell cargo to invalidate the built crate whenever the wrapper changes -- println!("cargo:rerun-if-changed={}", HEADER_PATH); -- -- // The bindgen::Builder is the main entry point -- // to bindgen, and lets you build up options for -- // the resulting bindings. -- let bindings = bindgen::Builder::default() -- // The input header we would like to generate -- // bindings for. -- .header(HEADER_PATH) -- // Tell cargo to invalidate the built crate whenever any of the -- // included header files changed. -- .parse_callbacks(Box::new(bindgen::CargoCallbacks)) -- // Finish the builder and generate the bindings. -- .generate() -- // Unwrap the Result and panic on failure. -- .expect("Unable to generate bindings"); -- -- // Write the bindings to the $OUT_DIR/bindings.rs file. -- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); -- bindings -- .write_to_file(out_path.join("atropos-sys.rs")) -- .expect("Couldn't write bindings!"); --} -- --fn gen_bpf_sched(name: &str) { -- let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap(); -- let clang = env::var("ATROPOS_CLANG").unwrap(); -- eprintln!("{}", clang); -- let outpath = format!("./src/bpf/.output/{}.skel.rs", name); -- let skel = Path::new(&outpath); -- let src = format!("./src/bpf/{}.bpf.c", name); -- SkeletonBuilder::new() -- .source(src.clone()) -- .clang(clang) -- .clang_args(bpf_cflags) -- .build_and_generate(&skel) -- .unwrap(); -- println!("cargo:rerun-if-changed={}", src); --} -- --fn main() { -- bindgen_atropos(); -- // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton. -- // Reasons are because the generated skeleton contains compiler attributes -- // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]` -- // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside -- // the path attribute either (see https://github.com/rust-lang/rust/pull/83366). -- // -- // However, there is hope! When the above feature stabilizes we can clean this -- // all up. -- create_dir_all("./src/bpf/.output").unwrap(); -- gen_bpf_sched("atropos"); --} -diff --git a/tools/sched_ext/atropos/rustfmt.toml b/tools/sched_ext/atropos/rustfmt.toml -deleted file mode 100644 -index b7258ed0a8d8..000000000000 ---- a/tools/sched_ext/atropos/rustfmt.toml -+++ /dev/null -@@ -1,8 +0,0 @@ --# Get help on options with `rustfmt --help=config` --# Please keep these in alphabetical order. --edition = "2021" --group_imports = "StdExternalCrate" --imports_granularity = "Item" --merge_derives = false --use_field_init_shorthand = true --version = "Two" -diff --git a/tools/sched_ext/atropos/src/atropos_sys.rs b/tools/sched_ext/atropos/src/atropos_sys.rs -deleted file mode 100644 -index bbeaf856d40e..000000000000 ---- a/tools/sched_ext/atropos/src/atropos_sys.rs -+++ /dev/null -@@ -1,10 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#![allow(non_upper_case_globals)] --#![allow(non_camel_case_types)] --#![allow(non_snake_case)] --#![allow(dead_code)] -- --include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs")); -diff --git a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -deleted file mode 100644 -index 3905a403e940..000000000000 ---- a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -+++ /dev/null -@@ -1,743 +0,0 @@ --/* Copyright (c) Meta Platforms, Inc. and affiliates. */ --/* -- * This software may be used and distributed according to the terms of the -- * GNU General Public License version 2. -- * -- * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF -- * part does simple round robin in each domain and the userspace part -- * calculates the load factor of each domain and tells the BPF part how to load -- * balance the domains. -- * -- * Every task has an entry in the task_data map which lists which domain the -- * task belongs to. When a task first enters the system (atropos_prep_enable), -- * they are round-robined to a domain. -- * -- * atropos_select_cpu is the primary scheduling logic, invoked when a task -- * becomes runnable. The lb_data map is populated by userspace to inform the BPF -- * scheduler that a task should be migrated to a new domain. Otherwise, the task -- * is scheduled in priority order as follows: -- * * The current core if the task was woken up synchronously and there are idle -- * cpus in the system -- * * The previous core, if idle -- * * The pinned-to core if the task is pinned to a specific core -- * * Any idle cpu in the domain -- * -- * If none of the above conditions are met, then the task is enqueued to a -- * dispatch queue corresponding to the domain (atropos_enqueue). -- * -- * atropos_dispatch will attempt to consume a task from its domain's -- * corresponding dispatch queue (this occurs after scheduling any tasks directly -- * assigned to it due to the logic in atropos_select_cpu). If no task is found, -- * then greedy load stealing will attempt to find a task on another dispatch -- * queue to run. -- * -- * Load balancing is almost entirely handled by userspace. BPF populates the -- * task weight, dom mask and current dom in the task_data map and executes the -- * load balance based on userspace populating the lb_data map. -- */ --#include "../../../scx_common.bpf.h" --#include "atropos.h" -- --#include --#include --#include --#include --#include --#include -- --char _license[] SEC("license") = "GPL"; -- --/* -- * const volatiles are set during initialization and treated as consts by the -- * jit compiler. -- */ -- --/* -- * Domains and cpus -- */ --const volatile __u32 nr_doms = 32; /* !0 for veristat, set during init */ --const volatile __u32 nr_cpus = 64; /* !0 for veristat, set during init */ --const volatile __u32 cpu_dom_id_map[MAX_CPUS]; --const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; -- --const volatile bool kthreads_local; --const volatile bool fifo_sched; --const volatile bool switch_partial; --const volatile __u32 greedy_threshold; -- --/* base slice duration */ --const volatile __u64 slice_us = 20000; -- --/* -- * Exit info -- */ --int exit_type = SCX_EXIT_NONE; --char exit_msg[SCX_EXIT_MSG_LEN]; -- --struct pcpu_ctx { -- __u32 dom_rr_cur; /* used when scanning other doms */ -- -- /* libbpf-rs does not respect the alignment, so pad out the struct explicitly */ -- __u8 _padding[CACHELINE_SIZE - sizeof(u64)]; --} __attribute__((aligned(CACHELINE_SIZE))); -- --struct pcpu_ctx pcpu_ctx[MAX_CPUS]; -- --/* -- * Domain context -- */ --struct dom_ctx { -- struct bpf_cpumask __kptr *cpumask; -- u64 vtime_now; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __type(key, u32); -- __type(value, struct dom_ctx); -- __uint(max_entries, MAX_DOMS); -- __uint(map_flags, 0); --} dom_ctx SEC(".maps"); -- --/* -- * Statistics -- */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, ATROPOS_NR_STATS); --} stats SEC(".maps"); -- --static inline void stat_add(enum stat_idx idx, u64 addend) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p) += addend; --} -- --/* Map pid -> task_ctx */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, struct task_ctx); -- __uint(max_entries, 1000000); -- __uint(map_flags, 0); --} task_data SEC(".maps"); -- --/* -- * This is populated from userspace to indicate which pids should be reassigned -- * to new doms. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, u32); -- __uint(max_entries, 1000); -- __uint(map_flags, 0); --} lb_data SEC(".maps"); -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p, -- u32 new_dom_id) --{ -- struct dom_ctx *old_domc, *new_domc; -- struct bpf_cpumask *d_cpumask, *t_cpumask; -- u32 old_dom_id = task_ctx->dom_id; -- s64 vtime_delta; -- -- old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id); -- if (!old_domc) { -- scx_bpf_error("No dom%u", old_dom_id); -- return false; -- } -- -- vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now; -- -- new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id); -- if (!new_domc) { -- scx_bpf_error("No dom%u", new_dom_id); -- return false; -- } -- -- d_cpumask = new_domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to get domain %u cpumask kptr", -- new_dom_id); -- return false; -- } -- -- t_cpumask = task_ctx->cpumask; -- if (!t_cpumask) { -- scx_bpf_error("Failed to look up task cpumask"); -- return false; -- } -- -- /* -- * set_cpumask might have happened between userspace requesting LB and -- * here and @p might not be able to run in @dom_id anymore. Verify. -- */ -- if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- p->cpus_ptr)) { -- p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta; -- task_ctx->dom_id = new_dom_id; -- bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask, -- p->cpus_ptr); -- } -- -- return task_ctx->dom_id == new_dom_id; --} -- --s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu, -- u32 wake_flags) --{ -- s32 cpu; -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- struct bpf_cpumask *p_cpumask; -- -- if (!task_ctx) -- return -ENOENT; -- -- if (kthreads_local && -- (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- cpu = prev_cpu; -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local dsq of the waker. -- */ -- if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) { -- struct task_struct *current = (void *)bpf_get_current_task(); -- -- if (!(BPF_CORE_READ(current, flags) & PF_EXITING) && -- task_ctx->dom_id < MAX_DOMS) { -- struct dom_ctx *domc; -- struct bpf_cpumask *d_cpumask; -- const struct cpumask *idle_cpumask; -- bool has_idle; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id); -- if (!domc) { -- scx_bpf_error("Failed to find dom%u", -- task_ctx->dom_id); -- return prev_cpu; -- } -- d_cpumask = domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to acquire domain %u cpumask kptr", -- task_ctx->dom_id); -- return prev_cpu; -- } -- -- idle_cpumask = scx_bpf_get_idle_cpumask(); -- -- has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- idle_cpumask); -- -- scx_bpf_put_idle_cpumask(idle_cpumask); -- -- if (has_idle) { -- cpu = bpf_get_smp_processor_id(); -- if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- stat_add(ATROPOS_STAT_WAKE_SYNC, 1); -- goto local; -- } -- } -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- stat_add(ATROPOS_STAT_PREV_IDLE, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- /* If only one core is allowed, dispatch */ -- if (p->nr_cpus_allowed == 1) { -- stat_add(ATROPOS_STAT_PINNED, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) -- return -ENOENT; -- -- /* If there is an eligible idle CPU, dispatch directly */ -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- if (cpu >= 0) { -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * @prev_cpu may be in a different domain. Returning an out-of-domain -- * CPU can lead to stalls as all in-domain CPUs may be idle by the time -- * @p gets enqueued. -- */ -- if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask)) -- cpu = prev_cpu; -- else -- cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask); -- -- return cpu; -- --local: -- task_ctx->dispatch_local = true; -- return cpu; --} -- --void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- u32 *new_dom; -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- new_dom = bpf_map_lookup_elem(&lb_data, &pid); -- if (new_dom && *new_dom != task_ctx->dom_id && -- task_set_dsq(task_ctx, p, *new_dom)) { -- struct bpf_cpumask *p_cpumask; -- s32 cpu; -- -- stat_add(ATROPOS_STAT_LOAD_BALANCE, 1); -- -- /* -- * If dispatch_local is set, We own @p's idle state but we are -- * not gonna put the task in the associated local dsq which can -- * cause the CPU to stall. Kick it. -- */ -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0); -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) { -- scx_bpf_error("Failed to get task_ctx->cpumask"); -- return; -- } -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- } -- -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags); -- return; -- } -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000, -- enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- u32 dom_id = task_ctx->dom_id; -- struct dom_ctx *domc; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, domc->vtime_now - slice_us * 1000)) -- vtime = domc->vtime_now - slice_us * 1000; -- -- scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --static u32 cpu_to_dom_id(s32 cpu) --{ -- const volatile u32 *dom_idp; -- -- if (nr_doms <= 1) -- return 0; -- -- dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]); -- if (!dom_idp) -- return MAX_DOMS; -- -- return *dom_idp; --} -- --static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) --{ -- s32 cpu; -- -- if (dom_id >= MAX_DOMS) -- return false; -- -- bpf_for(cpu, 0, nr_cpus) { -- if (bpf_cpumask_test_cpu(cpu, cpumask) && -- (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64)))) -- return true; -- } -- return false; --} -- --static u32 dom_rr_next(s32 cpu) --{ -- struct pcpu_ctx *pcpuc; -- u32 dom_id; -- -- pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]); -- if (!pcpuc) -- return 0; -- -- dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms; -- -- if (dom_id == cpu_to_dom_id(cpu)) -- dom_id = (dom_id + 1) % nr_doms; -- -- pcpuc->dom_rr_cur = dom_id; -- return dom_id; --} -- --void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 dom = cpu_to_dom_id(cpu); -- -- if (scx_bpf_consume(dom)) { -- stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1); -- return; -- } -- -- if (!greedy_threshold) -- return; -- -- bpf_repeat(nr_doms - 1) { -- u32 dom_id = dom_rr_next(cpu); -- -- if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold && -- scx_bpf_consume(dom_id)) { -- stat_add(ATROPOS_STAT_GREEDY, 1); -- break; -- } -- } --} -- --void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_at = bpf_ktime_get_ns(); --} -- --void BPF_STRUCT_OPS(atropos_running, struct task_struct *p) --{ -- struct task_ctx *taskc; -- struct dom_ctx *domc; -- pid_t pid = p->pid; -- u32 dom_id; -- -- if (fifo_sched) -- return; -- -- taskc = bpf_map_lookup_elem(&task_data, &pid); -- if (!taskc) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- dom_id = taskc->dom_id; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(domc->vtime_now, p->scx.dsq_vtime)) -- domc->vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at; -- task_ctx->runnable_at = 0; --} -- --void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->weight = weight; --} -- --struct pick_task_domain_loop_ctx { -- struct task_struct *p; -- const struct cpumask *cpumask; -- u64 dom_mask; -- u32 dom_rr_base; -- u32 dom_id; --}; -- --static int pick_task_domain_loopfn(u32 idx, void *data) --{ -- struct pick_task_domain_loop_ctx *lctx = data; -- u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms; -- -- if (dom_id >= MAX_DOMS) -- return 1; -- -- if (cpumask_intersects_domain(lctx->cpumask, dom_id)) { -- lctx->dom_mask |= 1LLU << dom_id; -- if (lctx->dom_id == MAX_DOMS) -- lctx->dom_id = dom_id; -- } -- return 0; --} -- --static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- struct pick_task_domain_loop_ctx lctx = { -- .p = p, -- .cpumask = cpumask, -- .dom_id = MAX_DOMS, -- }; -- s32 cpu = bpf_get_smp_processor_id(); -- -- if (cpu < 0 || cpu >= MAX_CPUS) -- return MAX_DOMS; -- -- lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur); -- -- bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0); -- task_ctx->dom_mask = lctx.dom_mask; -- -- return lctx.dom_id; --} -- --static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- u32 dom_id = 0; -- -- if (nr_doms > 1) -- dom_id = pick_task_domain(task_ctx, p, cpumask); -- -- if (!task_set_dsq(task_ctx, p, dom_id)) -- scx_bpf_error("Failed to set domain %d for %s[%d]", -- dom_id, p->comm, p->pid); --} -- --void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_set_domain(task_ctx, p, cpumask); --} -- --s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct bpf_cpumask *cpumask; -- struct task_ctx task_ctx, *map_value; -- long ret; -- pid_t pid; -- -- memset(&task_ctx, 0, sizeof(task_ctx)); -- -- pid = p->pid; -- ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return ret; -- } -- -- /* -- * Read the entry from the map immediately so we can add the cpumask -- * with bpf_kptr_xchg(). -- */ -- map_value = bpf_map_lookup_elem(&task_data, &pid); -- if (!map_value) -- /* Should never happen -- it was just inserted above. */ -- return -EINVAL; -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- bpf_map_delete_elem(&task_data, &pid); -- return -ENOMEM; -- } -- -- cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask); -- if (cpumask) { -- /* Should never happen as we just inserted it above. */ -- bpf_cpumask_release(cpumask); -- bpf_map_delete_elem(&task_data, &pid); -- return -EINVAL; -- } -- -- task_set_domain(map_value, p, p->cpus_ptr); -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p) --{ -- pid_t pid = p->pid; -- long ret = bpf_map_delete_elem(&task_data, &pid); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return; -- } --} -- --static int create_dom_dsq(u32 idx, void *data) --{ -- struct dom_ctx domc_init = {}, *domc; -- struct bpf_cpumask *cpumask; -- u32 cpu, dom_id = idx; -- s32 ret; -- -- ret = scx_bpf_create_dsq(dom_id, -1); -- if (ret < 0) { -- scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret); -- return 1; -- } -- -- ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0); -- if (ret) { -- scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret); -- return 1; -- } -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- /* Should never happen, we just inserted it above. */ -- scx_bpf_error("No dom%u", dom_id); -- return 1; -- } -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id); -- return 1; -- } -- -- for (cpu = 0; cpu < MAX_CPUS; cpu++) { -- const volatile __u64 *dmask; -- -- dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]); -- if (!dmask) { -- scx_bpf_error("array index error"); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- if (*dmask & (1LLU << (cpu % 64))) -- bpf_cpumask_set_cpu(cpu, cpumask); -- } -- -- cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask); -- if (cpumask) { -- scx_bpf_error("Domain %u was already present", dom_id); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(atropos_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- bpf_loop(nr_doms, create_dom_dsq, NULL, 0); -- -- for (u32 i = 0; i < nr_cpus; i++) -- pcpu_ctx[i].dom_rr_cur = i; -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg); -- exit_type = ei->type; --} -- --SEC(".struct_ops") --struct sched_ext_ops atropos = { -- .select_cpu = (void *)atropos_select_cpu, -- .enqueue = (void *)atropos_enqueue, -- .dispatch = (void *)atropos_dispatch, -- .runnable = (void *)atropos_runnable, -- .running = (void *)atropos_running, -- .stopping = (void *)atropos_stopping, -- .quiescent = (void *)atropos_quiescent, -- .set_weight = (void *)atropos_set_weight, -- .set_cpumask = (void *)atropos_set_cpumask, -- .prep_enable = (void *)atropos_prep_enable, -- .disable = (void *)atropos_disable, -- .init = (void *)atropos_init, -- .exit = (void *)atropos_exit, -- .flags = 0, -- .name = "atropos", --}; -diff --git a/tools/sched_ext/atropos/src/bpf/atropos.h b/tools/sched_ext/atropos/src/bpf/atropos.h -deleted file mode 100644 -index addf29ca104a..000000000000 ---- a/tools/sched_ext/atropos/src/bpf/atropos.h -+++ /dev/null -@@ -1,44 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#ifndef __ATROPOS_H --#define __ATROPOS_H -- --#include --#ifndef __kptr --#ifdef __KERNEL__ --#error "__kptr_ref not defined in the kernel" --#endif --#define __kptr --#endif -- --#define MAX_CPUS 512 --#define MAX_DOMS 64 /* limited to avoid complex bitmask ops */ --#define CACHELINE_SIZE 64 -- --/* Statistics */ --enum stat_idx { -- ATROPOS_STAT_TASK_GET_ERR, -- ATROPOS_STAT_WAKE_SYNC, -- ATROPOS_STAT_PREV_IDLE, -- ATROPOS_STAT_PINNED, -- ATROPOS_STAT_DIRECT_DISPATCH, -- ATROPOS_STAT_DSQ_DISPATCH, -- ATROPOS_STAT_GREEDY, -- ATROPOS_STAT_LOAD_BALANCE, -- ATROPOS_STAT_LAST_TASK, -- ATROPOS_NR_STATS, --}; -- --struct task_ctx { -- unsigned long long dom_mask; /* the domains this task can run on */ -- struct bpf_cpumask __kptr *cpumask; -- unsigned int dom_id; -- unsigned int weight; -- unsigned long long runnable_at; -- unsigned long long runnable_for; -- bool dispatch_local; --}; -- --#endif /* __ATROPOS_H */ -diff --git a/tools/sched_ext/atropos/src/main.rs b/tools/sched_ext/atropos/src/main.rs -deleted file mode 100644 -index 0d313662f713..000000000000 ---- a/tools/sched_ext/atropos/src/main.rs -+++ /dev/null -@@ -1,942 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#[path = "bpf/.output/atropos.skel.rs"] --mod atropos; --pub use atropos::*; --pub mod atropos_sys; -- --use std::cell::Cell; --use std::collections::{BTreeMap, BTreeSet}; --use std::ffi::CStr; --use std::ops::Bound::{Included, Unbounded}; --use std::sync::atomic::{AtomicBool, Ordering}; --use std::sync::Arc; --use std::time::{Duration, SystemTime}; -- --use ::fb_procfs as procfs; --use anyhow::{anyhow, bail, Context, Result}; --use bitvec::prelude::*; --use clap::Parser; --use log::{info, trace, warn}; --use ordered_float::OrderedFloat; -- --/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF --/// part does simple round robin in each domain and the userspace part --/// calculates the load factor of each domain and tells the BPF part how to load --/// balance the domains. -- --/// This scheduler demonstrates dividing scheduling logic between BPF and --/// userspace and using rust to build the userspace part. An earlier variant of --/// this scheduler was used to balance across six domains, each representing a --/// chiplet in a six-chiplet AMD processor, and could match the performance of --/// production setup using CFS. --#[derive(Debug, Parser)] --struct Opts { -- /// Scheduling slice duration in microseconds. -- #[clap(short, long, default_value = "20000")] -- slice_us: u64, -- -- /// Monitoring and load balance interval in seconds. -- #[clap(short, long, default_value = "2.0")] -- interval: f64, -- -- /// Build domains according to how CPUs are grouped at this cache level -- /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id. -- #[clap(short = 'c', long, default_value = "3")] -- cache_level: u32, -- -- /// Instead of using cache locality, set the cpumask for each domain -- /// manually, provide multiple --cpumasks, one for each domain. E.g. -- /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with -- /// the corresponding CPUs belonging to each domain. Each CPU must -- /// belong to precisely one domain. -- #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")] -- cpumasks: Vec, -- -- /// When non-zero, enable greedy task stealing. When a domain is idle, a -- /// cpu will attempt to steal tasks from a domain with at least -- /// greedy_threshold tasks enqueued. These tasks aren't permanently -- /// stolen from the domain. -- #[clap(short, long, default_value = "4")] -- greedy_threshold: u32, -- -- /// The load decay factor. Every interval, the existing load is decayed -- /// by this factor and new load is added. Must be in the range [0.0, -- /// 0.99]. The smaller the value, the more sensitive load calculation -- /// is to recent changes. When 0.0, history is ignored and the load -- /// value from the latest period is used directly. -- #[clap(short, long, default_value = "0.5")] -- load_decay_factor: f64, -- -- /// Disable load balancing. Unless disabled, periodically userspace will -- /// calculate the load factor of each domain and instruct BPF which -- /// processes to move. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- no_load_balance: bool, -- -- /// Put per-cpu kthreads directly into local dsq's. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- kthreads_local: bool, -- -- /// Use FIFO scheduling instead of weighted vtime scheduling. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- fifo_sched: bool, -- -- /// If specified, only tasks which have their scheduling policy set to -- /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all -- /// tasks are switched. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- partial: bool, -- -- /// Enable verbose output including libbpf details. Specify multiple -- /// times to increase verbosity. -- #[clap(short, long, action = clap::ArgAction::Count)] -- verbose: u8, --} -- --fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result { -- Ok(reader -- .read_stat() -- .context("Failed to read procfs")? -- .total_cpu -- .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?) --} -- --fn now_monotonic() -> u64 { -- let mut time = libc::timespec { -- tv_sec: 0, -- tv_nsec: 0, -- }; -- let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) }; -- assert!(ret == 0); -- time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 --} -- --fn clear_map(map: &mut libbpf_rs::Map) { -- // XXX: libbpf_rs has some design flaw that make it impossible to -- // delete while iterating despite it being safe so we alias it here -- let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) }; -- for key in map.keys() { -- let _ = deleter.delete(&key); -- } --} -- --#[derive(Debug)] --struct TaskLoad { -- runnable_for: u64, -- load: f64, --} -- --#[derive(Debug)] --struct TaskInfo { -- pid: i32, -- dom_mask: u64, -- migrated: Cell, --} -- --struct LoadBalancer<'a, 'b, 'c> { -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- -- tasks_by_load: Vec, TaskInfo>>, -- load_avg: f64, -- dom_loads: Vec, -- -- imbal: Vec, -- doms_to_push: BTreeMap, u32>, -- doms_to_pull: BTreeMap, u32>, -- -- nr_lb_data_errors: &'c mut u64, --} -- --impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> { -- const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10; -- const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1; -- const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50; -- -- fn new( -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- nr_lb_data_errors: &'c mut u64, -- ) -> Self { -- Self { -- maps, -- task_loads, -- nr_doms, -- load_decay_factor, -- -- tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(), -- load_avg: 0f64, -- dom_loads: vec![0.0; nr_doms], -- -- imbal: vec![0.0; nr_doms], -- doms_to_pull: BTreeMap::new(), -- doms_to_push: BTreeMap::new(), -- -- nr_lb_data_errors, -- } -- } -- -- fn read_task_loads(&mut self, period: Duration) -> Result<()> { -- let now_mono = now_monotonic(); -- let task_data = self.maps.task_data(); -- let mut this_task_loads = BTreeMap::::new(); -- let mut load_sum = 0.0f64; -- self.dom_loads = vec![0f64; self.nr_doms]; -- -- for key in task_data.keys() { -- if let Some(task_ctx_vec) = task_data -- .lookup(&key, libbpf_rs::MapFlags::ANY) -- .context("Failed to lookup task_data")? -- { -- let task_ctx = -- unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) }; -- let pid = i32::from_ne_bytes( -- key.as_slice() -- .try_into() -- .context("Invalid key length in task_data map")?, -- ); -- -- let (this_at, this_for, weight) = unsafe { -- ( -- std::ptr::read_volatile(&task_ctx.runnable_at as *const u64), -- std::ptr::read_volatile(&task_ctx.runnable_for as *const u64), -- std::ptr::read_volatile(&task_ctx.weight as *const u32), -- ) -- }; -- -- let (mut delta, prev_load) = match self.task_loads.get(&pid) { -- Some(prev) => (this_for - prev.runnable_for, Some(prev.load)), -- None => (this_for, None), -- }; -- -- // Non-zero this_at indicates that the task is currently -- // runnable. Note that we read runnable_at and runnable_for -- // without any synchronization and there is a small window -- // where we end up misaccounting. While this can cause -- // temporary error, it's unlikely to cause any noticeable -- // misbehavior especially given the load value clamping. -- if this_at > 0 && this_at < now_mono { -- delta += now_mono - this_at; -- } -- -- delta = delta.min(period.as_nanos() as u64); -- let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64) -- .clamp(0.0, weight as f64); -- -- let this_load = match prev_load { -- Some(prev_load) => { -- prev_load * self.load_decay_factor -- + this_load * (1.0 - self.load_decay_factor) -- } -- None => this_load, -- }; -- -- this_task_loads.insert( -- pid, -- TaskLoad { -- runnable_for: this_for, -- load: this_load, -- }, -- ); -- -- load_sum += this_load; -- self.dom_loads[task_ctx.dom_id as usize] += this_load; -- // Only record pids that are eligible for load balancing -- if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) { -- continue; -- } -- self.tasks_by_load[task_ctx.dom_id as usize].insert( -- OrderedFloat(this_load), -- TaskInfo { -- pid, -- dom_mask: task_ctx.dom_mask, -- migrated: Cell::new(false), -- }, -- ); -- } -- } -- -- self.load_avg = load_sum / self.nr_doms as f64; -- *self.task_loads = this_task_loads; -- Ok(()) -- } -- -- // To balance dom loads we identify doms with lower and higher load than average -- fn calculate_dom_load_balance(&mut self) -> Result<()> { -- for (dom, dom_load) in self.dom_loads.iter().enumerate() { -- let imbal = dom_load - self.load_avg; -- if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO { -- if imbal > 0f64 { -- self.doms_to_push.insert(OrderedFloat(imbal), dom as u32); -- } else { -- self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32); -- } -- self.imbal[dom] = imbal; -- } -- } -- Ok(()) -- } -- -- // Find the first candidate pid which hasn't already been migrated and -- // can run in @pull_dom. -- fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)> -- where -- I: IntoIterator, &'d TaskInfo)>, -- { -- match tasks_by_load -- .into_iter() -- .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0) -- .next() -- { -- Some((OrderedFloat(load), task)) => Some((*load, task)), -- None => None, -- } -- } -- -- fn pick_victim( -- &self, -- (push_dom, to_push): (u32, f64), -- (pull_dom, to_pull): (u32, f64), -- ) -> Option<(&TaskInfo, f64)> { -- let to_xfer = to_pull.min(to_push); -- -- trace!( -- "considering dom {}@{:.2} -> {}@{:.2}", -- push_dom, -- to_push, -- pull_dom, -- to_pull -- ); -- -- let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs(); -- -- trace!( -- "to_xfer={:.2} tasks_by_load={:?}", -- to_xfer, -- &self.tasks_by_load[push_dom as usize] -- ); -- -- // We want to pick a task to transfer from push_dom to pull_dom to -- // maximize the reduction of load imbalance between the two. IOW, -- // pick a task which has the closest load value to $to_xfer that can -- // be migrated. Find such task by locating the first migratable task -- // while scanning left from $to_xfer and the counterpart while -- // scanning right and picking the better of the two. -- let (load, task, new_imbal) = match ( -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Unbounded, Included(&OrderedFloat(to_xfer)))) -- .rev(), -- pull_dom, -- ), -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Included(&OrderedFloat(to_xfer)), Unbounded)), -- pull_dom, -- ), -- ) { -- (None, None) => return None, -- (Some((load, task)), None) | (None, Some((load, task))) => { -- (load, task, calc_new_imbal(load)) -- } -- (Some((load0, task0)), Some((load1, task1))) => { -- let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1)); -- if new_imbal0 <= new_imbal1 { -- (load0, task0, new_imbal0) -- } else { -- (load1, task1, new_imbal1) -- } -- } -- }; -- -- // If the best candidate can't reduce the imbalance, there's nothing -- // to do for this pair. -- let old_imbal = to_push + to_pull; -- if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal { -- trace!( -- "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal -- ); -- return None; -- } -- -- trace!( -- "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal, -- ); -- -- Some((task, load)) -- } -- -- // Actually execute the load balancing. Concretely this writes pid -> dom -- // entries into the lb_data map for bpf side to consume. -- fn load_balance(&mut self) -> Result<()> { -- clear_map(self.maps.lb_data()); -- -- trace!("imbal={:?}", &self.imbal); -- trace!("doms_to_push={:?}", &self.doms_to_push); -- trace!("doms_to_pull={:?}", &self.doms_to_pull); -- -- // Push from the most imbalanced to least. -- while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() { -- let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO; -- let mut pushed = 0f64; -- -- // Transfer tasks from push_dom to reduce imbalance. -- loop { -- let last_pushed = pushed; -- -- // Pull from the most imbalaned to least. -- let mut doms_to_pull = BTreeMap::<_, _>::new(); -- std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull); -- let mut pull_doms = doms_to_pull.into_iter().rev().collect::>(); -- -- for (to_pull, pull_dom) in pull_doms.iter_mut() { -- if let Some((task, load)) = -- self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull))) -- { -- // Execute migration. -- task.migrated.set(true); -- to_push -= load; -- *to_pull -= load; -- pushed += load; -- -- // Ask BPF code to execute the migration. -- let pid = task.pid; -- let cpid = (pid as libc::pid_t).to_ne_bytes(); -- if let Err(e) = self.maps.lb_data().update( -- &cpid, -- &pull_dom.to_ne_bytes(), -- libbpf_rs::MapFlags::NO_EXIST, -- ) { -- warn!( -- "Failed to update lb_data map for pid={} error={:?}", -- pid, &e -- ); -- *self.nr_lb_data_errors += 1; -- } -- -- // Always break after a successful migration so that -- // the pulling domains are always considered in the -- // descending imbalance order. -- break; -- } -- } -- -- pull_doms -- .into_iter() -- .map(|(k, v)| self.doms_to_pull.insert(k, v)) -- .count(); -- -- // Stop repeating if nothing got transferred or pushed enough. -- if pushed == last_pushed || pushed >= push_max { -- break; -- } -- } -- } -- Ok(()) -- } --} -- --struct Scheduler<'a> { -- skel: AtroposSkel<'a>, -- struct_ops: Option, -- -- nr_cpus: usize, -- nr_doms: usize, -- load_decay_factor: f64, -- balance_load: bool, -- -- proc_reader: procfs::ProcReader, -- -- prev_at: SystemTime, -- prev_total_cpu: procfs::CpuStat, -- task_loads: BTreeMap, -- -- nr_lb_data_errors: u64, --} -- --impl<'a> Scheduler<'a> { -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn parse_cpusets( -- cpumasks: &[String], -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- if cpumasks.len() > atropos_sys::MAX_DOMS as usize { -- bail!( -- "Number of requested DSQs ({}) is greater than MAX_DOMS ({})", -- cpumasks.len(), -- atropos_sys::MAX_DOMS -- ); -- } -- let mut cpus = vec![-1i32; nr_cpus]; -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()]; -- for (dq, cpumask) in cpumasks.iter().enumerate() { -- let hex_str = { -- let mut tmp_str = cpumask -- .strip_prefix("0x") -- .unwrap_or(cpumask) -- .replace('_', ""); -- if tmp_str.len() % 2 != 0 { -- tmp_str = "0".to_string() + &tmp_str; -- } -- tmp_str -- }; -- let byte_vec = hex::decode(&hex_str) -- .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?; -- -- for (index, &val) in byte_vec.iter().rev().enumerate() { -- let mut v = val; -- while v != 0 { -- let lsb = v.trailing_zeros() as usize; -- v &= !(1 << lsb); -- let cpu = index * 8 + lsb; -- if cpu > nr_cpus { -- bail!( -- concat!( -- "Found cpu ({}) in cpumask ({}) which is larger", -- " than the number of cpus on the machine ({})" -- ), -- cpu, -- cpumask, -- nr_cpus -- ); -- } -- if cpus[cpu] != -1 { -- bail!( -- "Found cpu ({}) with dq ({}) but also in cpumask ({})", -- cpu, -- cpus[cpu], -- cpumask -- ); -- } -- cpus[cpu] = dq as i32; -- cpusets[dq].set(cpu, true); -- } -- } -- cpusets[dq].set_uninitialized(false); -- } -- -- for (cpu, &dq) in cpus.iter().enumerate() { -- if dq < 0 { -- bail!( -- "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.", -- cpu -- ); -- } -- } -- -- Ok((cpusets, cpus)) -- } -- -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn cpusets_from_cache( -- level: u32, -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- let mut cpu_to_cache = vec![]; // (cpu_id, cache_id) -- let mut cache_ids = BTreeSet::::new(); -- let mut nr_not_found = 0; -- -- // Build cpu -> cache ID mapping. -- for cpu in 0..nr_cpus { -- let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level); -- let id = match std::fs::read_to_string(&path) { -- Ok(val) => val -- .trim() -- .parse::() -- .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?, -- Err(e) if e.kind() == std::io::ErrorKind::NotFound => { -- nr_not_found += 1; -- 0 -- } -- Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)), -- }; -- -- cpu_to_cache.push(id); -- cache_ids.insert(id); -- } -- -- if nr_not_found > 1 { -- warn!( -- "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0", -- level, nr_not_found, nr_cpus -- ); -- } -- -- // Cache IDs may have holes. Assign consecutive domain IDs to -- // existing cache IDs. -- let mut cache_to_dom = BTreeMap::::new(); -- let mut nr_doms = 0; -- for cache_id in cache_ids.iter() { -- cache_to_dom.insert(*cache_id, nr_doms); -- nr_doms += 1; -- } -- -- if nr_doms > atropos_sys::MAX_DOMS { -- bail!( -- "Total number of doms {} is greater than MAX_DOMS ({})", -- nr_doms, -- atropos_sys::MAX_DOMS -- ); -- } -- -- // Build and return dom -> cpumask and cpu -> dom mappings. -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize]; -- let mut cpu_to_dom = vec![]; -- -- for cpu in 0..nr_cpus { -- let dom_id = cache_to_dom[&cpu_to_cache[cpu]]; -- cpusets[dom_id as usize].set(cpu, true); -- cpu_to_dom.push(dom_id as i32); -- } -- -- Ok((cpusets, cpu_to_dom)) -- } -- -- fn init(opts: &Opts) -> Result { -- // Open the BPF prog first for verification. -- let mut skel_builder = AtroposSkelBuilder::default(); -- skel_builder.obj_builder.debug(opts.verbose > 0); -- let mut skel = skel_builder.open().context("Failed to open BPF program")?; -- -- let nr_cpus = libbpf_rs::num_possible_cpus().unwrap(); -- if nr_cpus > atropos_sys::MAX_CPUS as usize { -- bail!( -- "nr_cpus ({}) is greater than MAX_CPUS ({})", -- nr_cpus, -- atropos_sys::MAX_CPUS -- ); -- } -- -- // Initialize skel according to @opts. -- let (cpusets, cpus) = if opts.cpumasks.len() > 0 { -- Self::parse_cpusets(&opts.cpumasks, nr_cpus)? -- } else { -- Self::cpusets_from_cache(opts.cache_level, nr_cpus)? -- }; -- let nr_doms = cpusets.len(); -- skel.rodata().nr_doms = nr_doms as u32; -- skel.rodata().nr_cpus = nr_cpus as u32; -- -- for (cpu, dom) in cpus.iter().enumerate() { -- skel.rodata().cpu_dom_id_map[cpu] = *dom as u32; -- } -- -- for (dom, cpuset) in cpusets.iter().enumerate() { -- let raw_cpuset_slice = cpuset.as_raw_slice(); -- let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom]; -- let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len()); -- left.clone_from_slice(cpuset.as_raw_slice()); -- let cpumask_str = dom_cpumask_slice -- .iter() -- .take((nr_cpus + 63) / 64) -- .rev() -- .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x)); -- info!( -- "DOM[{:02}] cpumask{} ({} cpus)", -- dom, -- &cpumask_str, -- cpuset.count_ones() -- ); -- } -- -- skel.rodata().slice_us = opts.slice_us; -- skel.rodata().kthreads_local = opts.kthreads_local; -- skel.rodata().fifo_sched = opts.fifo_sched; -- skel.rodata().switch_partial = opts.partial; -- skel.rodata().greedy_threshold = opts.greedy_threshold; -- -- // Attach. -- let mut skel = skel.load().context("Failed to load BPF program")?; -- skel.attach().context("Failed to attach BPF program")?; -- let struct_ops = Some( -- skel.maps_mut() -- .atropos() -- .attach_struct_ops() -- .context("Failed to attach atropos struct ops")?, -- ); -- info!("Atropos Scheduler Attached"); -- -- // Other stuff. -- let mut proc_reader = procfs::ProcReader::new(); -- let prev_total_cpu = read_total_cpu(&mut proc_reader)?; -- -- Ok(Self { -- skel, -- struct_ops, // should be held to keep it attached -- -- nr_cpus, -- nr_doms, -- load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99), -- balance_load: !opts.no_load_balance, -- -- proc_reader, -- -- prev_at: SystemTime::now(), -- prev_total_cpu, -- task_loads: BTreeMap::new(), -- -- nr_lb_data_errors: 0, -- }) -- } -- -- fn get_cpu_busy(&mut self) -> Result { -- let total_cpu = read_total_cpu(&mut self.proc_reader)?; -- let busy = match (&self.prev_total_cpu, &total_cpu) { -- ( -- procfs::CpuStat { -- user_usec: Some(prev_user), -- nice_usec: Some(prev_nice), -- system_usec: Some(prev_system), -- idle_usec: Some(prev_idle), -- iowait_usec: Some(prev_iowait), -- irq_usec: Some(prev_irq), -- softirq_usec: Some(prev_softirq), -- stolen_usec: Some(prev_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- procfs::CpuStat { -- user_usec: Some(curr_user), -- nice_usec: Some(curr_nice), -- system_usec: Some(curr_system), -- idle_usec: Some(curr_idle), -- iowait_usec: Some(curr_iowait), -- irq_usec: Some(curr_irq), -- softirq_usec: Some(curr_softirq), -- stolen_usec: Some(curr_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- ) => { -- let idle_usec = curr_idle - prev_idle; -- let iowait_usec = curr_iowait - prev_iowait; -- let user_usec = curr_user - prev_user; -- let system_usec = curr_system - prev_system; -- let nice_usec = curr_nice - prev_nice; -- let irq_usec = curr_irq - prev_irq; -- let softirq_usec = curr_softirq - prev_softirq; -- let stolen_usec = curr_stolen - prev_stolen; -- -- let busy_usec = -- user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec; -- let total_usec = idle_usec + busy_usec + iowait_usec; -- busy_usec as f64 / total_usec as f64 -- } -- _ => { -- bail!("Some procfs stats are not populated!"); -- } -- }; -- -- self.prev_total_cpu = total_cpu; -- Ok(busy) -- } -- -- fn read_bpf_stats(&mut self) -> Result> { -- let mut maps = self.skel.maps_mut(); -- let stats_map = maps.stats(); -- let mut stats: Vec = Vec::new(); -- let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus]; -- -- for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS { -- let cpu_stat_vec = stats_map -- .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY) -- .with_context(|| format!("Failed to lookup stat {}", stat))? -- .expect("per-cpu stat should exist"); -- let sum = cpu_stat_vec -- .iter() -- .map(|val| { -- u64::from_ne_bytes( -- val.as_slice() -- .try_into() -- .expect("Invalid value length in stat map"), -- ) -- }) -- .sum(); -- stats_map -- .update_percpu( -- &(stat as u32).to_ne_bytes(), -- &zero_vec, -- libbpf_rs::MapFlags::ANY, -- ) -- .context("Failed to zero stat")?; -- stats.push(sum); -- } -- Ok(stats) -- } -- -- fn report( -- &self, -- stats: &Vec, -- cpu_busy: f64, -- processing_dur: Duration, -- load_avg: f64, -- dom_loads: &Vec, -- imbal: &Vec, -- ) { -- let stat = |idx| stats[idx as usize]; -- let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK); -- -- info!( -- "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms", -- cpu_busy * 100.0, -- load_avg, -- stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize], -- stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize], -- self.nr_lb_data_errors, -- processing_dur.as_millis(), -- ); -- -- let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0; -- -- info!( -- "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}", -- total, -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY), -- ); -- -- for i in 0..self.nr_doms { -- info!( -- "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}", -- i, -- dom_loads[i], -- if imbal[i] < 0.0 { -imbal[i] } else { 0.0 }, -- if imbal[i] > 0.0 { imbal[i] } else { 0.0 }, -- ); -- } -- } -- -- fn step(&mut self) -> Result<()> { -- let started_at = std::time::SystemTime::now(); -- let bpf_stats = self.read_bpf_stats()?; -- let cpu_busy = self.get_cpu_busy()?; -- -- let mut lb = LoadBalancer::new( -- self.skel.maps_mut(), -- &mut self.task_loads, -- self.nr_doms, -- self.load_decay_factor, -- &mut self.nr_lb_data_errors, -- ); -- -- lb.read_task_loads(started_at.duration_since(self.prev_at)?)?; -- lb.calculate_dom_load_balance()?; -- -- if self.balance_load { -- lb.load_balance()?; -- } -- -- // Extract fields needed for reporting and drop lb to release -- // mutable borrows. -- let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal); -- -- self.report( -- &bpf_stats, -- cpu_busy, -- std::time::SystemTime::now().duration_since(started_at)?, -- load_avg, -- &dom_loads, -- &imbal, -- ); -- -- self.prev_at = started_at; -- Ok(()) -- } -- -- fn read_bpf_exit_type(&mut self) -> i32 { -- unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) } -- } -- -- fn report_bpf_exit_type(&mut self) -> Result<()> { -- // Report msg if EXT_OPS_EXIT_ERROR. -- match self.read_bpf_exit_type() { -- 0 => Ok(()), -- etype if etype == 2 => { -- let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) }; -- let msg = cstr -- .to_str() -- .context("Failed to convert exit msg to string") -- .unwrap(); -- bail!("BPF exit_type={} msg={}", etype, msg); -- } -- etype => { -- info!("BPF exit_type={}", etype); -- Ok(()) -- } -- } -- } --} -- --impl<'a> Drop for Scheduler<'a> { -- fn drop(&mut self) { -- if let Some(struct_ops) = self.struct_ops.take() { -- drop(struct_ops); -- } -- } --} -- --fn main() -> Result<()> { -- let opts = Opts::parse(); -- -- let llv = match opts.verbose { -- 0 => simplelog::LevelFilter::Info, -- 1 => simplelog::LevelFilter::Debug, -- _ => simplelog::LevelFilter::Trace, -- }; -- let mut lcfg = simplelog::ConfigBuilder::new(); -- lcfg.set_time_level(simplelog::LevelFilter::Error) -- .set_location_level(simplelog::LevelFilter::Off) -- .set_target_level(simplelog::LevelFilter::Off) -- .set_thread_level(simplelog::LevelFilter::Off); -- simplelog::TermLogger::init( -- llv, -- lcfg.build(), -- simplelog::TerminalMode::Stderr, -- simplelog::ColorChoice::Auto, -- )?; -- -- let shutdown = Arc::new(AtomicBool::new(false)); -- let shutdown_clone = shutdown.clone(); -- ctrlc::set_handler(move || { -- shutdown_clone.store(true, Ordering::Relaxed); -- }) -- .context("Error setting Ctrl-C handler")?; -- -- let mut sched = Scheduler::init(&opts)?; -- -- while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 { -- std::thread::sleep(Duration::from_secs_f64(opts.interval)); -- sched.step()?; -- } -- -- sched.report_bpf_exit_type() --} -diff --git a/tools/sched_ext/gnu/stubs.h b/tools/sched_ext/gnu/stubs.h -deleted file mode 100644 -index 719225b16626..000000000000 ---- a/tools/sched_ext/gnu/stubs.h -+++ /dev/null -@@ -1 +0,0 @@ --/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ -diff --git a/tools/sched_ext/scx_common.bpf.h b/tools/sched_ext/scx_common.bpf.h -deleted file mode 100644 -index e56de9dc86f2..000000000000 ---- a/tools/sched_ext/scx_common.bpf.h -+++ /dev/null -@@ -1,288 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __SCHED_EXT_COMMON_BPF_H --#define __SCHED_EXT_COMMON_BPF_H -- --#include "vmlinux.h" --#include --#include --#include --#include "user_exit_info.h" -- --#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ --#define PF_EXITING 0x00000004 --#define CLOCK_MONOTONIC 1 -- --/* -- * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can -- * lead to really confusing misbehaviors. Let's trigger a build failure. -- */ --static inline void ___vmlinux_h_sanity_check___(void) --{ -- _Static_assert(SCX_DSQ_FLAG_BUILTIN, -- "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); --} -- --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; -- --static inline __attribute__((format(printf, 1, 2))) --void ___scx_bpf_error_format_checker(const char *fmt, ...) {} -- --/* -- * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments -- * instead of an array of u64. Note that __param[] must have at least one -- * element to keep the verifier happy. -- */ --#define scx_bpf_error(fmt, args...) \ --({ \ -- static char ___fmt[] = fmt; \ -- unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ -- \ -- _Pragma("GCC diagnostic push") \ -- _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ -- ___bpf_fill(___param, args); \ -- _Pragma("GCC diagnostic pop") \ -- \ -- scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ -- \ -- ___scx_bpf_error_format_checker(fmt, ##args); \ --}) -- --void scx_bpf_switch_all(void) __ksym; --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; --bool scx_bpf_consume(u64 dsq_id) __ksym; --u32 scx_bpf_dispatch_nr_slots(void) __ksym; --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym; --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym; --void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; --s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym; --const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; --const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; --void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; --void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; --bool scx_bpf_task_running(const struct task_struct *p) __ksym; --s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym; --u32 scx_bpf_reenqueue_local(void) __ksym; -- --#define BPF_STRUCT_OPS(name, args...) \ --SEC("struct_ops/"#name) \ --BPF_PROG(name, ##args) -- --#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ --SEC("struct_ops.s/"#name) \ --BPF_PROG(name, ##args) -- --/** -- * MEMBER_VPTR - Obtain the verified pointer to a struct or array member -- * @base: struct or array to index -- * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...) -- * -- * The verifier often gets confused by the instruction sequence the compiler -- * generates for indexing struct fields or arrays. This macro forces the -- * compiler to generate a code sequence which first calculates the byte offset, -- * checks it against the struct or array size and add that byte offset to -- * generate the pointer to the member to help the verifier. -- * -- * Ideally, we want to abort if the calculated offset is out-of-bounds. However, -- * BPF currently doesn't support abort, so evaluate to NULL instead. The caller -- * must check for NULL and take appropriate action to appease the verifier. To -- * avoid confusing the verifier, it's best to check for NULL and dereference -- * immediately. -- * -- * vptr = MEMBER_VPTR(my_array, [i][j]); -- * if (!vptr) -- * return error; -- * *vptr = new_value; -- */ --#define MEMBER_VPTR(base, member) (typeof(base member) *)({ \ -- u64 __base = (u64)base; \ -- u64 __addr = (u64)&(base member) - __base; \ -- asm volatile ( \ -- "if %0 <= %[max] goto +2\n" \ -- "%0 = 0\n" \ -- "goto +1\n" \ -- "%0 += %1\n" \ -- : "+r"(__addr) \ -- : "r"(__base), \ -- [max]"i"(sizeof(base) - sizeof(base member))); \ -- __addr; \ --}) -- --/* -- * BPF core and other generic helpers -- */ -- --/* list and rbtree */ --#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) --#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) -- --void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; --void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; -- --#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) --#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) -- --void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; --struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; --struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, -- struct bpf_rb_node *node) __ksym; --void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, -- bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym; --struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; -- --/* task */ --struct task_struct *bpf_task_from_pid(s32 pid) __ksym; --struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; --void bpf_task_release(struct task_struct *p) __ksym; -- --/* cgroup */ --struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; --void bpf_cgroup_release(struct cgroup *cgrp) __ksym; --struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; -- --/* cpumask */ --struct bpf_cpumask *bpf_cpumask_create(void) __ksym; --struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; --u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; --u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym; -- --/* rcu */ --void bpf_rcu_read_lock(void) __ksym; --void bpf_rcu_read_unlock(void) __ksym; -- --/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */ --struct bpf_iter_num; -- --extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym; --extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym; --extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym; -- --#ifndef bpf_for_each --/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for -- * using BPF open-coded iterators without having to write mundane explicit -- * low-level loop logic. Instead, it provides for()-like generic construct -- * that can be used pretty naturally. E.g., for some hypothetical cgroup -- * iterator, you'd write: -- * -- * struct cgroup *cg, *parent_cg = <...>; -- * -- * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { -- * bpf_printk("Child cgroup id = %d", cg->cgroup_id); -- * if (cg->cgroup_id == 123) -- * break; -- * } -- * -- * I.e., it looks almost like high-level for each loop in other languages, -- * supports continue/break, and is verifiable by BPF verifier. -- * -- * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) -- * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to -- * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int -- * *`, not just `int`. So for integers bpf_for() is more convenient. -- * -- * Note: this macro relies on C99 feature of allowing to declare variables -- * inside for() loop, bound to for() loop lifetime. It also utilizes GCC -- * extension: __attribute__((cleanup())), supported by both GCC and -- * Clang. -- */ --#define bpf_for_each(type, cur, args...) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ -- cleanup(bpf_iter_##type##_destroy))), \ -- /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_##type##_new(&___it, ##args), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_##type##_destroy, (void *)0); \ -- /* iteration and termination check */ \ -- (((cur) = bpf_iter_##type##_next(&___it))); \ --) --#endif /* bpf_for_each */ -- --#ifndef bpf_for --/* bpf_for(i, start, end) implements a for()-like looping construct that sets -- * provided integer variable *i* to values starting from *start* through, -- * but not including, *end*. It also proves to BPF verifier that *i* belongs -- * to range [start, end), so this can be used for accessing arrays without -- * extra checks. -- * -- * Note: *start* and *end* are assumed to be expressions with no side effects -- * and whose values do not change throughout bpf_for() loop execution. They do -- * not have to be statically known or constant, though. -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_for(i, start, end) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, (start), (end)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- ({ \ -- /* iteration step */ \ -- int *___t = bpf_iter_num_next(&___it); \ -- /* termination and bounds check */ \ -- (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ -- }); \ --) --#endif /* bpf_for */ -- --#ifndef bpf_repeat --/* bpf_repeat(N) performs N iterations without exposing iteration number -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_repeat(N) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, 0, (N)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- bpf_iter_num_next(&___it); \ -- /* nothing here */ \ --) --#endif /* bpf_repeat */ -- --#endif /* __SCHED_EXT_COMMON_BPF_H */ -diff --git a/tools/sched_ext/scx_example_central.bpf.c b/tools/sched_ext/scx_example_central.bpf.c -deleted file mode 100644 -index 4cec04b4c2ed..000000000000 ---- a/tools/sched_ext/scx_example_central.bpf.c -+++ /dev/null -@@ -1,334 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A central FIFO sched_ext scheduler which demonstrates the followings: -- * -- * a. Making all scheduling decisions from one CPU: -- * -- * The central CPU is the only one making scheduling decisions. All other -- * CPUs kick the central CPU when they run out of tasks to run. -- * -- * There is one global BPF queue and the central CPU schedules all CPUs by -- * dispatching from the global queue to each CPU's local dsq from dispatch(). -- * This isn't the most straightforward. e.g. It'd be easier to bounce -- * through per-CPU BPF queues. The current design is chosen to maximally -- * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching. -- * -- * b. Tickless operation -- * -- * All tasks are dispatched with the infinite slice which allows stopping the -- * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full -- * parameter. The tickless operation can be observed through -- * /proc/interrupts. -- * -- * Periodic switching is enforced by a periodic timer checking all CPUs and -- * preempting them as necessary. Unfortunately, BPF timer currently doesn't -- * have a way to pin to a specific CPU, so the periodic timer isn't pinned to -- * the central CPU. -- * -- * c. Preemption -- * -- * Kthreads are unconditionally queued to the head of a matching local dsq -- * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always -- * prioritized over user threads, which is required for ensuring forward -- * progress as e.g. the periodic timer may run on a ksoftirqd and if the -- * ksoftirqd gets starved by a user thread, there may not be anything else to -- * vacate that user thread. -- * -- * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the -- * next tasks. -- * -- * This scheduler is designed to maximize usage of various SCX mechanisms. A -- * more practical implementation would likely put the scheduling loop outside -- * the central CPU's dispatch() path and add some form of priority mechanism. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --enum { -- FALLBACK_DSQ_ID = 0, -- MAX_CPUS = 4096, -- MS_TO_NS = 1000LLU * 1000, -- TIMER_INTERVAL_NS = 1 * MS_TO_NS, --}; -- --const volatile bool switch_partial; --const volatile s32 central_cpu; --const volatile u32 nr_cpu_ids = 64; /* !0 for veristat, set during init */ -- --u64 nr_total, nr_locals, nr_queued, nr_lost_pids; --u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries; --u64 nr_overflows; -- --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, s32); --} central_q SEC(".maps"); -- --/* can't use percpu map due to bad lookups */ --static bool cpu_gimme_task[MAX_CPUS]; --static u64 cpu_started_at[MAX_CPUS]; -- --struct central_timer { -- struct bpf_timer timer; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, 1); -- __type(key, u32); -- __type(value, struct central_timer); --} central_timer SEC(".maps"); -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- /* -- * Steer wakeups to the central CPU as much as possible to avoid -- * disturbing other CPUs. It's safe to blindly return the central cpu as -- * select_cpu() is a hint and if @p can't be on it, the kernel will -- * automatically pick a fallback CPU. -- */ -- return central_cpu; --} -- --void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags) --{ -- s32 pid = p->pid; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- /* -- * Push per-cpu kthreads at the head of local dsq's and preempt the -- * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked -- * behind other threads which is necessary for forward progress -- * guarantee as we depend on the BPF timer which may run from ksoftirqd. -- */ -- if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- __sync_fetch_and_add(&nr_locals, 1); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF, -- enq_flags | SCX_ENQ_PREEMPT); -- return; -- } -- -- if (bpf_map_push_elem(¢ral_q, &pid, 0)) { -- __sync_fetch_and_add(&nr_overflows, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_queued, 1); -- -- if (!scx_bpf_task_running(p)) -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); --} -- --static bool dispatch_to_cpu(s32 cpu) --{ -- struct task_struct *p; -- s32 pid; -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (bpf_map_pop_elem(¢ral_q, &pid)) -- break; -- -- __sync_fetch_and_sub(&nr_queued, 1); -- -- p = bpf_task_from_pid(pid); -- if (!p) { -- __sync_fetch_and_add(&nr_lost_pids, 1); -- continue; -- } -- -- /* -- * If we can't run the task at the top, do the dumb thing and -- * bounce it to the fallback dsq. -- */ -- if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- __sync_fetch_and_add(&nr_mismatches, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0); -- bpf_task_release(p); -- continue; -- } -- -- /* dispatch to local and mark that @cpu doesn't need more */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0); -- -- if (cpu != central_cpu) -- scx_bpf_kick_cpu(cpu, 0); -- -- bpf_task_release(p); -- return true; -- } -- -- return false; --} -- --void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (cpu == central_cpu) { -- /* dispatch for all other CPUs first */ -- __sync_fetch_and_add(&nr_dispatches, 1); -- -- bpf_for(cpu, 0, nr_cpu_ids) { -- bool *gimme; -- -- if (!scx_bpf_dispatch_nr_slots()) -- break; -- -- /* central's gimme is never set */ -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme && !*gimme) -- continue; -- -- if (dispatch_to_cpu(cpu)) -- *gimme = false; -- } -- -- /* -- * Retry if we ran out of dispatch buffer slots as we might have -- * skipped some CPUs and also need to dispatch for self. The ext -- * core automatically retries if the local dsq is empty but we -- * can't rely on that as we're dispatching for other CPUs too. -- * Kick self explicitly to retry. -- */ -- if (!scx_bpf_dispatch_nr_slots()) { -- __sync_fetch_and_add(&nr_retries, 1); -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- return; -- } -- -- /* look for a task to run on the central CPU */ -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- dispatch_to_cpu(central_cpu); -- } else { -- bool *gimme; -- -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme) -- *gimme = true; -- -- /* -- * Force dispatch on the scheduling CPU so that it finds a task -- * to run for us. -- */ -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- } --} -- --void BPF_STRUCT_OPS(central_running, struct task_struct *p) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ --} -- --void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = 0; --} -- --static int central_timerfn(void *map, int *key, struct bpf_timer *timer) --{ -- u64 now = bpf_ktime_get_ns(); -- u64 nr_to_kick = nr_queued; -- s32 i; -- -- bpf_for(i, 0, nr_cpu_ids) { -- s32 cpu = (nr_timers + i) % nr_cpu_ids; -- u64 *started_at; -- -- if (cpu == central_cpu) -- continue; -- -- /* kick iff the current one exhausted its slice */ -- started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at && *started_at && -- vtime_before(now, *started_at + SCX_SLICE_DFL)) -- continue; -- -- /* and there's something pending */ -- if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) || -- scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu)) -- ; -- else if (nr_to_kick) -- nr_to_kick--; -- else -- continue; -- -- scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); -- } -- -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- -- bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- __sync_fetch_and_add(&nr_timers, 1); -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(central_init) --{ -- u32 key = 0; -- struct bpf_timer *timer; -- int ret; -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1); -- if (ret) -- return ret; -- -- timer = bpf_map_lookup_elem(¢ral_timer, &key); -- if (!timer) -- return -ESRCH; -- -- bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); -- bpf_timer_set_callback(timer, central_timerfn); -- ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- return ret; --} -- --void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops central_ops = { -- /* -- * We are offloading all scheduling decisions to the central CPU and -- * thus being the last task on a given CPU doesn't mean anything -- * special. Enqueue the last tasks like any other tasks. -- */ -- .flags = SCX_OPS_ENQ_LAST, -- -- .select_cpu = (void *)central_select_cpu, -- .enqueue = (void *)central_enqueue, -- .dispatch = (void *)central_dispatch, -- .running = (void *)central_running, -- .stopping = (void *)central_stopping, -- .init = (void *)central_init, -- .exit = (void *)central_exit, -- .name = "central", --}; -diff --git a/tools/sched_ext/scx_example_central.c b/tools/sched_ext/scx_example_central.c -deleted file mode 100644 -index 7ad591cbdc65..000000000000 ---- a/tools/sched_ext/scx_example_central.c -+++ /dev/null -@@ -1,94 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_central.skel.h" -- --const char help_fmt[] = --"A central FIFO sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-c CPU] [-p]\n" --"\n" --" -c CPU Override the central CPU (default: 0)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_central *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_central__open(); -- assert(skel); -- -- skel->rodata->central_cpu = 0; -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "c:ph")) != -1) { -- switch (opt) { -- case 'c': -- skel->rodata->central_cpu = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_central__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.central_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf("total :%10lu local:%10lu queued:%10lu lost:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_locals, -- skel->bss->nr_queued, -- skel->bss->nr_lost_pids); -- printf("timer :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n", -- skel->bss->nr_timers, -- skel->bss->nr_dispatches, -- skel->bss->nr_mismatches, -- skel->bss->nr_retries); -- printf("overflow:%10lu\n", -- skel->bss->nr_overflows); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_central__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_flatcg.bpf.c b/tools/sched_ext/scx_example_flatcg.bpf.c -deleted file mode 100644 -index f6078b9a681f..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.bpf.c -+++ /dev/null -@@ -1,872 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext flattened cgroup hierarchy scheduler. It implements -- * hierarchical weight-based cgroup CPU control by flattening the cgroup -- * hierarchy into a single layer by compounding the active weight share at each -- * level. Consider the following hierarchy with weights in parentheses: -- * -- * R + A (100) + B (100) -- * | \ C (100) -- * \ D (200) -- * -- * Ignoring the root and threaded cgroups, only B, C and D can contain tasks. -- * Let's say all three have runnable tasks. The total share that each of these -- * three cgroups is entitled to can be calculated by compounding its share at -- * each level. -- * -- * For example, B is competing against C and in that competition its share is -- * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's -- * share in that competition is 200/(200+100) == 1/3. B's eventual share in the -- * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's -- * eventual shaer is the same at 1/6. D is only competing at the top level and -- * its share is 200/(100+200) == 2/3. -- * -- * So, instead of hierarchically scheduling level-by-level, we can consider it -- * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3 -- * and keep updating the eventual shares as the cgroups' runnable states change. -- * -- * This flattening of hierarchy can bring a substantial performance gain when -- * the cgroup hierarchy is nested multiple levels. in a simple benchmark using -- * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it -- * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two -- * apache instances competing with 2:1 weight ratio nested four level deep. -- * -- * However, the gain comes at the cost of not being able to properly handle -- * thundering herd of cgroups. For example, if many cgroups which are nested -- * behind a low priority parent cgroup wake up around the same time, they may be -- * able to consume more CPU cycles than they are entitled to. In many use cases, -- * this isn't a real concern especially given the performance gain. Also, there -- * are ways to mitigate the problem further by e.g. introducing an extra -- * scheduling layer on cgroup delegation boundaries. -- * -- * The scheduler first picks the cgroup to run and then schedule the tasks -- * within by using nested weighted vtime scheduling by default. The -- * cgroup-internal scheduling can be switched to FIFO with the -f option. -- */ --#include "scx_common.bpf.h" --#include "user_exit_info.h" --#include "scx_example_flatcg.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */ --const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL; --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --u64 cvtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, u64); -- __uint(max_entries, FCG_NR_STATS); --} stats SEC(".maps"); -- --static void stat_inc(enum fcg_stat_idx idx) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p)++; --} -- --struct fcg_cpu_ctx { -- u64 cur_cgid; -- u64 cur_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, struct fcg_cpu_ctx); -- __uint(max_entries, 1); --} cpu_ctx SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_cgrp_ctx); --} cgrp_ctx SEC(".maps"); -- --struct cgv_node { -- struct bpf_rb_node rb_node; -- __u64 cvtime; -- __u64 cgid; --}; -- --private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock; --private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node); -- --struct cgv_node_stash { -- struct cgv_node __kptr *node; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, 16384); -- __type(key, __u64); -- __type(value, struct cgv_node_stash); --} cgv_node_stash SEC(".maps"); -- --struct fcg_task_ctx { -- u64 bypassed_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_task_ctx); --} task_ctx SEC(".maps"); -- --/* gets inc'd on weight tree changes to expire the cached hweights */ --unsigned long hweight_gen = 1; -- --static u64 div_round_up(u64 dividend, u64 divisor) --{ -- return (dividend + divisor - 1) / divisor; --} -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) --{ -- struct cgv_node *cgc_a, *cgc_b; -- -- cgc_a = container_of(a, struct cgv_node, rb_node); -- cgc_b = container_of(b, struct cgv_node, rb_node); -- -- return cgc_a->cvtime < cgc_b->cvtime; --} -- --static struct fcg_cpu_ctx *find_cpu_ctx(void) --{ -- struct fcg_cpu_ctx *cpuc; -- u32 idx = 0; -- -- cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx); -- if (!cpuc) { -- scx_bpf_error("cpu_ctx lookup failed"); -- return NULL; -- } -- return cpuc; --} -- --static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id); -- return NULL; -- } -- return cgc; --} -- --static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgrp = bpf_cgroup_ancestor(cgrp, level); -- if (!cgrp) { -- scx_bpf_error("ancestor cgroup lookup failed"); -- return NULL; -- } -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- scx_bpf_error("ancestor cgrp_ctx lookup failed"); -- bpf_cgroup_release(cgrp); -- return cgc; --} -- --static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- int level; -- -- if (!cgc->nr_active) { -- stat_inc(FCG_STAT_HWT_SKIP); -- return; -- } -- -- if (cgc->hweight_gen == hweight_gen) { -- stat_inc(FCG_STAT_HWT_CACHE); -- return; -- } -- -- stat_inc(FCG_STAT_HWT_UPDATES); -- bpf_for(level, 0, cgrp->level + 1) { -- struct fcg_cgrp_ctx *cgc; -- bool is_active; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- -- if (!level) { -- cgc->hweight = FCG_HWEIGHT_ONE; -- cgc->hweight_gen = hweight_gen; -- } else { -- struct fcg_cgrp_ctx *pcgc; -- -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- -- /* -- * We can be oppotunistic here and not grab the -- * cgv_tree_lock and deal with the occasional races. -- * However, hweight updates are already cached and -- * relatively low-frequency. Let's just do the -- * straightforward thing. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- is_active = cgc->nr_active; -- if (is_active) { -- cgc->hweight_gen = pcgc->hweight_gen; -- cgc->hweight = -- div_round_up(pcgc->hweight * cgc->weight, -- pcgc->child_weight_sum); -- } -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!is_active) { -- stat_inc(FCG_STAT_HWT_RACE); -- break; -- } -- } -- } --} -- --static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) --{ -- u64 delta, cvtime, max_budget; -- -- /* -- * A node which is on the rbtree can't be pointed to from elsewhere yet -- * and thus can't be updated and repositioned. Instead, we collect the -- * vtime deltas separately and apply it asynchronously here. -- */ -- delta = cgc->cvtime_delta; -- __sync_fetch_and_sub(&cgc->cvtime_delta, delta); -- cvtime = cgv_node->cvtime + delta; -- -- /* -- * Allow a cgroup to carry the maximum budget proportional to its -- * hweight such that a full-hweight cgroup can immediately take up half -- * of the CPUs at the most while staying at the front of the rbtree. -- */ -- max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / -- (2 * FCG_HWEIGHT_ONE); -- if (vtime_before(cvtime, cvtime_now - max_budget)) -- cvtime = cvtime_now - max_budget; -- -- cgv_node->cvtime = cvtime; --} -- --static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- u64 cgid = cgrp->kn->id; -- -- /* paired with cmpxchg in try_pick_next_cgroup() */ -- if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { -- stat_inc(FCG_STAT_ENQ_SKIP); -- return; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid); -- return; -- } -- -- /* NULL if the node is already on the rbtree */ -- cgv_node = bpf_kptr_xchg(&stash->node, NULL); -- if (!cgv_node) { -- stat_inc(FCG_STAT_ENQ_RACE); -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); --} -- --void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * If select_cpu_dfl() is recommending local enqueue, the target CPU is -- * idle. Follow it and charge the cgroup later in fcg_stopping() after -- * the fact. Use the same mechanism to deal with tasks with custom -- * affinities so that we don't have to worry about per-cgroup dq's -- * containing tasks that can't be executed from some CPUs. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) { -- /* -- * Tell fcg_stopping() that this bypassed the regular scheduling -- * path and should be force charged to the cgroup. 0 is used to -- * indicate that the task isn't bypassing, so if the current -- * runtime is 0, go back by one nanosecond. -- */ -- taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1; -- -- /* -- * The global dq is deprioritized as we don't want to let tasks -- * to boost themselves by constraining its cpumask. The -- * deprioritization is rather severe, so let's not apply that to -- * per-cpu kernel threads. This is ham-fisted. We probably wanna -- * implement per-cgroup fallback dq's instead so that we have -- * more control over when tasks with custom cpumask get issued. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || -- (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) { -- stat_inc(FCG_STAT_LOCAL); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- } else { -- stat_inc(FCG_STAT_GLOBAL); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } -- return; -- } -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- goto out_release; -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 tvtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) -- tvtime = cgc->tvtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, -- tvtime, enq_flags); -- } -- -- cgrp_enqueued(cgrp, cgc); --out_release: -- bpf_cgroup_release(cgrp); --} -- --/* -- * Walk the cgroup tree to update the active weight sums as tasks wake up and -- * sleep. The weight sums are used as the base when calculating the proportion a -- * given cgroup or task is entitled to at each level. -- */ --static void update_active_weight_sums(struct cgroup *cgrp, bool runnable) --{ -- struct fcg_cgrp_ctx *cgc; -- bool updated = false; -- int idx; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- /* -- * In most cases, a hot cgroup would have multiple threads going to -- * sleep and waking up while the whole cgroup stays active. In leaf -- * cgroups, ->nr_runnable which is updated with __sync operations gates -- * ->nr_active updates, so that we don't have to grab the cgv_tree_lock -- * repeatedly for a busy cgroup which is staying active. -- */ -- if (runnable) { -- if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_ACT); -- } else { -- if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_DEACT); -- } -- -- /* -- * If @cgrp is becoming runnable, its hweight should be refreshed after -- * it's added to the weight tree so that enqueue has the up-to-date -- * value. If @cgrp is becoming quiescent, the hweight should be -- * refreshed before it's removed from the weight tree so that the usage -- * charging which happens afterwards has access to the latest value. -- */ -- if (!runnable) -- cgrp_refresh_hweight(cgrp, cgc); -- -- /* propagate upwards */ -- bpf_for(idx, 0, cgrp->level) { -- int level = cgrp->level - idx; -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- bool propagate = false; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- if (level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- } -- -- /* -- * We need the propagation protected by a lock to synchronize -- * against weight changes. There's no reason to drop the lock at -- * each level but bpf_spin_lock() doesn't want any function -- * calls while locked. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- -- if (runnable) { -- if (!cgc->nr_active++) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum += cgc->weight; -- } -- } -- } else { -- if (!--cgc->nr_active) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum -= cgc->weight; -- } -- } -- } -- -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!propagate) -- break; -- } -- -- if (updated) -- __sync_fetch_and_add(&hweight_gen, 1); -- -- if (runnable) -- cgrp_refresh_hweight(cgrp, cgc); --} -- --void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, true); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) --{ -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- if (fifo_sched) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- /* -- * @cgc->tvtime_now always progresses forward as tasks start -- * executing. The test and update can be performed concurrently -- * from multiple CPUs and thus racy. Any error should be -- * contained and temporary. Let's just live with it. -- */ -- if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) -- cgc->tvtime_now = p->scx.dsq_vtime; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- /* scale the execution time by the inverse of the weight and charge */ -- if (!fifo_sched) -- p->scx.dsq_vtime += -- (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- if (!taskc->bypassed_at) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- __sync_fetch_and_add(&cgc->cvtime_delta, -- p->se.sum_exec_runtime - taskc->bypassed_at); -- taskc->bypassed_at = 0; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, false); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) --{ -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- if (cgrp->level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1); -- if (!pcgc) -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- if (pcgc && cgc->nr_active) -- pcgc->child_weight_sum += (s64)weight - cgc->weight; -- cgc->weight = weight; -- bpf_spin_unlock(&cgv_tree_lock); --} -- --static bool try_pick_next_cgroup(u64 *cgidp) --{ -- struct bpf_rb_node *rb_node; -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 cgid; -- -- /* pop the front cgroup and wind cvtime_now accordingly */ -- bpf_spin_lock(&cgv_tree_lock); -- -- rb_node = bpf_rbtree_first(&cgv_tree); -- if (!rb_node) { -- bpf_spin_unlock(&cgv_tree_lock); -- stat_inc(FCG_STAT_PNC_NO_CGRP); -- *cgidp = 0; -- return true; -- } -- -- rb_node = bpf_rbtree_remove(&cgv_tree, rb_node); -- bpf_spin_unlock(&cgv_tree_lock); -- -- cgv_node = container_of(rb_node, struct cgv_node, rb_node); -- cgid = cgv_node->cgid; -- -- if (vtime_before(cvtime_now, cgv_node->cvtime)) -- cvtime_now = cgv_node->cvtime; -- -- /* -- * If lookup fails, the cgroup's gone. Free and move on. See -- * fcg_cgroup_exit(). -- */ -- cgrp = bpf_cgroup_from_id(cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- if (!scx_bpf_consume(cgid)) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_EMPTY); -- goto out_stash; -- } -- -- /* -- * Successfully consumed from the cgroup. This will be our current -- * cgroup for the new slice. Refresh its hweight. -- */ -- cgrp_refresh_hweight(cgrp, cgc); -- -- bpf_cgroup_release(cgrp); -- -- /* -- * As the cgroup may have more tasks, add it back to the rbtree. Note -- * that here we charge the full slice upfront and then exact later -- * according to the actual consumption. This prevents lowpri thundering -- * herd from saturating the machine. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- -- *cgidp = cgid; -- stat_inc(FCG_STAT_PNC_NEXT); -- return true; -- --out_stash: -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- /* -- * Paired with cmpxchg in cgrp_enqueued(). If they see the following -- * transition, they'll enqueue the cgroup. If they are earlier, we'll -- * see their task in the dq below and requeue the cgroup. -- */ -- __sync_val_compare_and_swap(&cgc->queued, 1, 0); -- -- if (scx_bpf_dsq_nr_queued(cgid)) { -- bpf_spin_lock(&cgv_tree_lock); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- goto out_free; -- } -- } -- -- return false; -- --out_free: -- bpf_obj_drop(cgv_node); -- return false; --} -- --void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) --{ -- struct fcg_cpu_ctx *cpuc; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 now = bpf_ktime_get_ns(); -- -- cpuc = find_cpu_ctx(); -- if (!cpuc) -- return; -- -- if (!cpuc->cur_cgid) -- goto pick_next_cgroup; -- -- if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { -- if (scx_bpf_consume(cpuc->cur_cgid)) { -- stat_inc(FCG_STAT_CNS_KEEP); -- return; -- } -- stat_inc(FCG_STAT_CNS_EMPTY); -- } else { -- stat_inc(FCG_STAT_CNS_EXPIRE); -- } -- -- /* -- * The current cgroup is expiring. It was already charged a full slice. -- * Calculate the actual usage and accumulate the delta. -- */ -- cgrp = bpf_cgroup_from_id(cpuc->cur_cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_CNS_GONE); -- goto pick_next_cgroup; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (cgc) { -- /* -- * We want to update the vtime delta and then look for the next -- * cgroup to execute but the latter needs to be done in a loop -- * and we can't keep the lock held. Oh well... -- */ -- bpf_spin_lock(&cgv_tree_lock); -- __sync_fetch_and_add(&cgc->cvtime_delta, -- (cpuc->cur_at + cgrp_slice_ns - now) * -- FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- stat_inc(FCG_STAT_CNS_GONE); -- } -- -- bpf_cgroup_release(cgrp); -- --pick_next_cgroup: -- cpuc->cur_at = now; -- -- if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { -- cpuc->cur_cgid = 0; -- return; -- } -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_pick_next_cgroup(&cpuc->cur_cgid)) -- break; -- } --} -- --s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct fcg_task_ctx *taskc; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- taskc = bpf_task_storage_get(&task_ctx, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!taskc) -- return -ENOMEM; -- -- taskc->bypassed_at = 0; -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp, -- struct scx_cgroup_init_args *args) --{ -- struct fcg_cgrp_ctx *cgc; -- struct cgv_node *cgv_node; -- struct cgv_node_stash empty_stash = {}, *stash; -- u64 cgid = cgrp->kn->id; -- int ret; -- -- /* -- * Technically incorrect as cgroup ID is full 64bit while dq ID is -- * 63bit. Should not be a problem in practice and easy to spot in the -- * unlikely case that it breaks. -- */ -- ret = scx_bpf_create_dsq(cgid, -1); -- if (ret) -- return ret; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!cgc) { -- ret = -ENOMEM; -- goto err_destroy_dsq; -- } -- -- cgc->weight = args->weight; -- cgc->hweight = FCG_HWEIGHT_ONE; -- -- ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash, -- BPF_NOEXIST); -- if (ret) { -- if (ret != -ENOMEM) -- scx_bpf_error("unexpected stash creation error (%d)", -- ret); -- goto err_destroy_dsq; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("unexpected cgv_node stash lookup failure"); -- ret = -ENOENT; -- goto err_destroy_dsq; -- } -- -- cgv_node = bpf_obj_new(struct cgv_node); -- if (!cgv_node) { -- ret = -ENOMEM; -- goto err_del_cgv_node; -- } -- -- cgv_node->cgid = cgid; -- cgv_node->cvtime = cvtime_now; -- -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- ret = -EBUSY; -- goto err_drop; -- } -- -- return 0; -- --err_drop: -- bpf_obj_drop(cgv_node); --err_del_cgv_node: -- bpf_map_delete_elem(&cgv_node_stash, &cgid); --err_destroy_dsq: -- scx_bpf_destroy_dsq(cgid); -- return ret; --} -- --void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- -- /* -- * For now, there's no way find and remove the cgv_node if it's on the -- * cgv_tree. Let's drain them in the dispatch path as they get popped -- * off the front of the tree. -- */ -- bpf_map_delete_elem(&cgv_node_stash, &cgid); -- scx_bpf_destroy_dsq(cgid); --} -- --s32 BPF_STRUCT_OPS(fcg_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops flatcg_ops = { -- .enqueue = (void *)fcg_enqueue, -- .dispatch = (void *)fcg_dispatch, -- .runnable = (void *)fcg_runnable, -- .running = (void *)fcg_running, -- .stopping = (void *)fcg_stopping, -- .quiescent = (void *)fcg_quiescent, -- .prep_enable = (void *)fcg_prep_enable, -- .cgroup_set_weight = (void *)fcg_cgroup_set_weight, -- .cgroup_init = (void *)fcg_cgroup_init, -- .cgroup_exit = (void *)fcg_cgroup_exit, -- .init = (void *)fcg_init, -- .exit = (void *)fcg_exit, -- .flags = SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING, -- .name = "flatcg", --}; -diff --git a/tools/sched_ext/scx_example_flatcg.c b/tools/sched_ext/scx_example_flatcg.c -deleted file mode 100644 -index f9c8a5b84a70..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.c -+++ /dev/null -@@ -1,232 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2023 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2023 Tejun Heo -- * Copyright (c) 2023 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_flatcg.h" --#include "scx_example_flatcg.skel.h" -- --#ifndef FILEID_KERNFS --#define FILEID_KERNFS 0xfe --#endif -- --const char help_fmt[] = --"A flattened cgroup hierarchy sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -i INTERVAL Report interval\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --static float read_cpu_util(__u64 *last_sum, __u64 *last_idle) --{ -- FILE *fp; -- char buf[4096]; -- char *line, *cur = NULL, *tok; -- __u64 sum = 0, idle = 0; -- __u64 delta_sum, delta_idle; -- int idx; -- -- fp = fopen("/proc/stat", "r"); -- if (!fp) { -- perror("fopen(\"/proc/stat\")"); -- return 0.0; -- } -- -- if (!fgets(buf, sizeof(buf), fp)) { -- perror("fgets(\"/proc/stat\")"); -- fclose(fp); -- return 0.0; -- } -- fclose(fp); -- -- line = buf; -- for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) { -- char *endp = NULL; -- __u64 v; -- -- if (idx == 0) { -- line = NULL; -- continue; -- } -- v = strtoull(tok, &endp, 0); -- if (!endp || *endp != '\0') { -- fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n", -- idx, tok); -- continue; -- } -- sum += v; -- if (idx == 4) -- idle = v; -- } -- -- delta_sum = sum - *last_sum; -- delta_idle = idle - *last_idle; -- *last_sum = sum; -- *last_idle = idle; -- -- return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0; --} -- --static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats) --{ -- __u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus]; -- __u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); -- -- for (idx = 0; idx < FCG_NR_STATS; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_flatcg *skel; -- struct bpf_link *link; -- struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 }; -- bool dump_cgrps = false; -- __u64 last_cpu_sum = 0, last_cpu_idle = 0; -- __u64 last_stats[FCG_NR_STATS] = {}; -- unsigned long seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_flatcg__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open: %s\n", strerror(errno)); -- return 1; -- } -- -- skel->rodata->nr_cpus = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) { -- double v; -- -- switch (opt) { -- case 's': -- v = strtod(optarg, NULL); -- skel->rodata->cgrp_slice_ns = v * 1000; -- break; -- case 'i': -- v = strtod(optarg, NULL); -- intv_ts.tv_sec = v; -- intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000; -- break; -- case 'd': -- dump_cgrps = true; -- break; -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- case 'h': -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d", -- (double)skel->rodata->cgrp_slice_ns / 1000000.0, -- (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0, -- dump_cgrps); -- -- if (scx_example_flatcg__load(skel)) { -- fprintf(stderr, "Failed to load: %s\n", strerror(errno)); -- return 1; -- } -- -- link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops); -- if (!link) { -- fprintf(stderr, "Failed to attach_struct_ops: %s\n", -- strerror(errno)); -- return 1; -- } -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- __u64 acc_stats[FCG_NR_STATS]; -- __u64 stats[FCG_NR_STATS]; -- float cpu_util; -- int i; -- -- cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle); -- -- fcg_read_stats(skel, acc_stats); -- for (i = 0; i < FCG_NR_STATS; i++) -- stats[i] = acc_stats[i] - last_stats[i]; -- -- memcpy(last_stats, acc_stats, sizeof(acc_stats)); -- -- printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n", -- seq++, cpu_util * 100.0, skel->data->hweight_gen); -- printf(" act:%6llu deact:%6llu local:%6llu global:%6llu\n", -- stats[FCG_STAT_ACT], -- stats[FCG_STAT_DEACT], -- stats[FCG_STAT_LOCAL], -- stats[FCG_STAT_GLOBAL]); -- printf("HWT skip:%6llu race:%6llu cache:%6llu update:%6llu\n", -- stats[FCG_STAT_HWT_SKIP], -- stats[FCG_STAT_HWT_RACE], -- stats[FCG_STAT_HWT_CACHE], -- stats[FCG_STAT_HWT_UPDATES]); -- printf("ENQ skip:%6llu race:%6llu\n", -- stats[FCG_STAT_ENQ_SKIP], -- stats[FCG_STAT_ENQ_RACE]); -- printf("CNS keep:%6llu expire:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_CNS_KEEP], -- stats[FCG_STAT_CNS_EXPIRE], -- stats[FCG_STAT_CNS_EMPTY], -- stats[FCG_STAT_CNS_GONE]); -- printf("PNC nocgrp:%6llu next:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_PNC_NO_CGRP], -- stats[FCG_STAT_PNC_NEXT], -- stats[FCG_STAT_PNC_EMPTY], -- stats[FCG_STAT_PNC_GONE]); -- printf("BAD remove:%6llu\n", -- acc_stats[FCG_STAT_BAD_REMOVAL]); -- -- nanosleep(&intv_ts, NULL); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_flatcg__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_flatcg.h b/tools/sched_ext/scx_example_flatcg.h -deleted file mode 100644 -index 490758ed41f0..000000000000 ---- a/tools/sched_ext/scx_example_flatcg.h -+++ /dev/null -@@ -1,49 +0,0 @@ --#ifndef __SCX_EXAMPLE_FLATCG_H --#define __SCX_EXAMPLE_FLATCG_H -- --enum { -- FCG_HWEIGHT_ONE = 1LLU << 16, --}; -- --enum fcg_stat_idx { -- FCG_STAT_ACT, -- FCG_STAT_DEACT, -- FCG_STAT_LOCAL, -- FCG_STAT_GLOBAL, -- -- FCG_STAT_HWT_UPDATES, -- FCG_STAT_HWT_CACHE, -- FCG_STAT_HWT_SKIP, -- FCG_STAT_HWT_RACE, -- -- FCG_STAT_ENQ_SKIP, -- FCG_STAT_ENQ_RACE, -- -- FCG_STAT_CNS_KEEP, -- FCG_STAT_CNS_EXPIRE, -- FCG_STAT_CNS_EMPTY, -- FCG_STAT_CNS_GONE, -- -- FCG_STAT_PNC_NO_CGRP, -- FCG_STAT_PNC_NEXT, -- FCG_STAT_PNC_EMPTY, -- FCG_STAT_PNC_GONE, -- -- FCG_STAT_BAD_REMOVAL, -- -- FCG_NR_STATS, --}; -- --struct fcg_cgrp_ctx { -- u32 nr_active; -- u32 nr_runnable; -- u32 queued; -- u32 weight; -- u32 hweight; -- u64 child_weight_sum; -- u64 hweight_gen; -- s64 cvtime_delta; -- u64 tvtime_now; --}; -- --#endif /* __SCX_EXAMPLE_FLATCG_H */ -diff --git a/tools/sched_ext/scx_example_pair.bpf.c b/tools/sched_ext/scx_example_pair.bpf.c -deleted file mode 100644 -index 279efe58b777..000000000000 ---- a/tools/sched_ext/scx_example_pair.bpf.c -+++ /dev/null -@@ -1,627 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext core-scheduler which always makes every sibling CPU pair -- * execute from the same CPU cgroup. -- * -- * This scheduler is a minimal implementation and would need some form of -- * priority handling both inside each cgroup and across the cgroups to be -- * practically useful. -- * -- * Each CPU in the system is paired with exactly one other CPU, according to a -- * "stride" value that can be specified when the BPF scheduler program is first -- * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee -- * that they will only ever schedule tasks that belong to the same CPU cgroup. -- * -- * Scheduler Initialization -- * ------------------------ -- * -- * The scheduler BPF program is first initialized from user space, before it is -- * enabled. During this initialization process, each CPU on the system is -- * assigned several values that are constant throughout its runtime: -- * -- * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling -- * decisions. Paired CPUs always schedule tasks from the same -- * CPU cgroup, and synchronize with each other to guarantee -- * that this constraint is not violated. -- * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access -- * a struct pair_ctx object that is shared between the pair. -- * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the -- * pair. Each struct pair_ctx has an active_mask field, -- * which is a bitmap used to indicate whether each core -- * in the pair currently has an actively running task. -- * This index specifies which entry in the bitmap corresponds -- * to each CPU in the pair. -- * -- * During this initialization, the CPUs are paired according to a "stride" that -- * may be specified when invoking the user space program that initializes and -- * loads the scheduler. By default, the stride is 1/2 the total number of CPUs. -- * -- * Tasks and cgroups -- * ----------------- -- * -- * Every cgroup in the system is registered with the scheduler using the -- * pair_cgroup_init() callback, and every task in the system is associated with -- * exactly one cgroup. At a high level, the idea with the pair scheduler is to -- * always schedule tasks from the same cgroup within a given CPU pair. When a -- * task is enqueued (i.e. passed to the pair_enqueue() callback function), its -- * cgroup ID is read from its task struct, and then a corresponding queue map -- * is used to FIFO-enqueue the task for that cgroup. -- * -- * If you look through the implementation of the scheduler, you'll notice that -- * there is quite a bit of complexity involved with looking up the per-cgroup -- * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash -- * BPF hash map that is used to map a cgroup ID to a globally unique ID that's -- * allocated in the BPF program. This is done because we use separate maps to -- * store the FIFO queue of tasks, and the length of that map, per cgroup. This -- * complexity is only present because of current deficiencies in BPF that will -- * soon be addressed. The main point to keep in mind is that newly enqueued -- * tasks are added to their cgroup's FIFO queue. -- * -- * Dispatching tasks -- * ----------------- -- * -- * This section will describe how enqueued tasks are dispatched and scheduled. -- * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is -- * as follows: -- * -- * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is -- * the structure that's used to synchronize amongst the two pair CPUs in their -- * scheduling decisions. After any of the following events have occurred: -- * -- * - The cgroup's slice run has expired, or -- * - The cgroup becomes empty, or -- * - Either CPU in the pair is preempted by a higher priority scheduling class -- * -- * The cgroup transitions to the draining state and stops executing new tasks -- * from the cgroup. -- * -- * 2. If the pair is still executing a task, mark the pair_ctx as draining, and -- * wait for the pair CPU to be preempted. -- * -- * 3. Otherwise, if the pair CPU is not running a task, we can move onto -- * scheduling new tasks. Pop the next cgroup id from the top_q queue. -- * -- * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it. -- * -- * Note again that this scheduling behavior is simple, but the implementation -- * is complex mostly because this it hits several BPF shortcomings and has to -- * work around in often awkward ways. Most of the shortcomings are expected to -- * be resolved in the near future which should allow greatly simplifying this -- * scheduler. -- * -- * Dealing with preemption -- * ----------------------- -- * -- * SCX is the lowest priority sched_class, and could be preempted by them at -- * any time. To address this, the scheduler implements pair_cpu_release() and -- * pair_cpu_acquire() callbacks which are invoked by the core scheduler when -- * the scheduler loses and gains control of the CPU respectively. -- * -- * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and -- * then invoke: -- * -- * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- * -- * This preempts the pair CPU, and waits until it has re-entered the scheduler -- * before returning. This is necessary to ensure that the higher priority -- * sched_class that preempted our scheduler does not schedule a task -- * concurrently with our pair CPU. -- * -- * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption -- * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable -- * pair scheduling. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include "scx_example_pair.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; -- --/* !0 for veristat, set during init */ --const volatile u32 nr_cpu_ids = 64; -- --/* a pair of CPUs stay on a cgroup for this duration */ --const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL; -- --/* cpu ID -> pair cpu ID */ --const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 }; -- --/* cpu ID -> pair_id */ --const volatile u32 pair_id[MAX_CPUS]; -- --/* CPU ID -> CPU # in the pair (0 or 1) */ --const volatile u32 in_pair_idx[MAX_CPUS]; -- --struct pair_ctx { -- struct bpf_spin_lock lock; -- -- /* the cgroup the pair is currently executing */ -- u64 cgid; -- -- /* the pair started executing the current cgroup at */ -- u64 started_at; -- -- /* whether the current cgroup is draining */ -- bool draining; -- -- /* the CPUs that are currently active on the cgroup */ -- u32 active_mask; -- -- /* -- * the CPUs that are currently preempted and running tasks in a -- * different scheduler. -- */ -- u32 preempted_mask; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, MAX_CPUS / 2); -- __type(key, u32); -- __type(value, struct pair_ctx); --} pair_ctx SEC(".maps"); -- --/* queue of cgrp_q's possibly with tasks on them */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- /* -- * Because it's difficult to build strong synchronization encompassing -- * multiple non-trivial operations in BPF, this queue is managed in an -- * opportunistic way so that we guarantee that a cgroup w/ active tasks -- * is always on it but possibly multiple times. Once we have more robust -- * synchronization constructs and e.g. linked list, we should be able to -- * do this in a prettier way but for now just size it big enough. -- */ -- __uint(max_entries, 4 * MAX_CGRPS); -- __type(value, u64); --} top_q SEC(".maps"); -- --/* per-cgroup q which FIFOs the tasks from the cgroup */ --struct cgrp_q { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, MAX_QUEUED); -- __type(value, u32); --}; -- --/* -- * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local -- * storage; however, a cgroup local storage can only be accessed from the BPF -- * progs attached to the cgroup. For now, work around by allocating array of -- * cgrp_q's and then allocating per-cgroup indices. -- * -- * Another caveat: It's difficult to populate a large array of maps statically -- * or from BPF. Initialize it from userland. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, MAX_CGRPS); -- __type(key, s32); -- __array(values, struct cgrp_q); --} cgrp_q_arr SEC(".maps"); -- --static u64 cgrp_q_len[MAX_CGRPS]; -- --/* -- * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be -- * useful to have as a map type. -- */ --static u32 cgrp_q_idx_cursor; --static u64 cgrp_q_idx_busy[MAX_CGRPS]; -- --/* -- * All added up, the following is what we do: -- * -- * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking -- * for a free ID. If not found, fail cgroup creation with -EBUSY. -- * -- * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following -- * cgrp_q_idx_hash. -- * -- * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from -- * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr. -- * -- * This is sadly complicated for something pretty simple. Hopefully, we should -- * be able to simplify in the future. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, MAX_CGRPS); -- __uint(key_size, sizeof(u64)); /* cgrp ID */ -- __uint(value_size, sizeof(s32)); /* cgrp_q idx */ --} cgrp_q_idx_hash SEC(".maps"); -- --/* statistics */ --u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions; --u64 nr_exps, nr_exp_waits, nr_exp_empty; --u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty; -- --struct user_exit_info uei; -- --static bool time_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- struct cgrp_q *cgq; -- s32 pid = p->pid; -- u64 cgid; -- u32 *q_idx; -- u64 *cgq_len; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- cgrp = scx_bpf_task_cgroup(p); -- cgid = cgrp->kn->id; -- bpf_cgroup_release(cgrp); -- -- /* find the cgroup's q and push @p into it */ -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!q_idx) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return; -- } -- -- cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx); -- if (!cgq) { -- scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]", -- cgid, *q_idx); -- return; -- } -- -- if (bpf_map_push_elem(cgq, &pid, 0)) { -- scx_bpf_error("cgroup[%llu] queue overflow", cgid); -- return; -- } -- -- /* bump q len, if going 0 -> 1, queue cgroup into the top_q */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len) { -- scx_bpf_error("MEMBER_VTPR malfunction"); -- return; -- } -- -- if (!__sync_fetch_and_add(cgq_len, 1) && -- bpf_map_push_elem(&top_q, &cgid, 0)) { -- scx_bpf_error("top_q overflow"); -- return; -- } --} -- --static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) --{ -- u32 *vptr; -- -- vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *pairc = bpf_map_lookup_elem(&pair_ctx, vptr); -- if (!(*pairc)) -- return -EINVAL; -- -- vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *mask = 1U << *vptr; -- -- return 0; --} -- --static int try_dispatch(s32 cpu) --{ -- struct pair_ctx *pairc; -- struct bpf_map *cgq_map; -- struct task_struct *p; -- u64 now = bpf_ktime_get_ns(); -- bool kick_pair = false; -- bool expired, pair_preempted; -- u32 *vptr, in_pair_mask; -- s32 pid, q_idx; -- u64 cgid; -- int ret; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) { -- scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]", -- cpu); -- return -ENOENT; -- } -- -- bpf_spin_lock(&pairc->lock); -- pairc->active_mask &= ~in_pair_mask; -- -- expired = time_before(pairc->started_at + pair_batch_dur_ns, now); -- if (expired || pairc->draining) { -- u64 new_cgid = 0; -- -- __sync_fetch_and_add(&nr_exps, 1); -- -- /* -- * We're done with the current cgid. An obvious optimization -- * would be not draining if the next cgroup is the current one. -- * For now, be dumb and always expire. -- */ -- pairc->draining = true; -- -- pair_preempted = pairc->preempted_mask; -- if (pairc->active_mask || pair_preempted) { -- /* -- * The other CPU is still active, or is no longer under -- * our control due to e.g. being preempted by a higher -- * priority sched_class. We want to wait until this -- * cgroup expires, or until control of our pair CPU has -- * been returned to us. -- * -- * If the pair controls its CPU, and the time already -- * expired, kick. When the other CPU arrives at -- * dispatch and clears its active mask, it'll push the -- * pair to the next cgroup and kick this CPU. -- */ -- __sync_fetch_and_add(&nr_exp_waits, 1); -- bpf_spin_unlock(&pairc->lock); -- if (expired && !pair_preempted) -- kick_pair = true; -- goto out_maybe_kick; -- } -- -- bpf_spin_unlock(&pairc->lock); -- -- /* -- * Pick the next cgroup. It'd be easier / cleaner to not drop -- * pairc->lock and use stronger synchronization here especially -- * given that we'll be switching cgroups significantly less -- * frequently than tasks. Unfortunately, bpf_spin_lock can't -- * really protect anything non-trivial. Let's do opportunistic -- * operations instead. -- */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u32 *q_idx; -- u64 *cgq_len; -- -- if (bpf_map_pop_elem(&top_q, &new_cgid)) { -- /* no active cgroup, go idle */ -- __sync_fetch_and_add(&nr_exp_empty, 1); -- return 0; -- } -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid); -- if (!q_idx) -- continue; -- -- /* -- * This is the only place where empty cgroups are taken -- * off the top_q. -- */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len || !*cgq_len) -- continue; -- -- /* -- * If it has any tasks, requeue as we may race and not -- * execute it. -- */ -- bpf_map_push_elem(&top_q, &new_cgid, 0); -- break; -- } -- -- bpf_spin_lock(&pairc->lock); -- -- /* -- * The other CPU may already have started on a new cgroup while -- * we dropped the lock. Make sure that we're still draining and -- * start on the new cgroup. -- */ -- if (pairc->draining && !pairc->active_mask) { -- __sync_fetch_and_add(&nr_cgrp_next, 1); -- pairc->cgid = new_cgid; -- pairc->started_at = now; -- pairc->draining = false; -- kick_pair = true; -- } else { -- __sync_fetch_and_add(&nr_cgrp_coll, 1); -- } -- } -- -- cgid = pairc->cgid; -- pairc->active_mask |= in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- -- /* again, it'd be better to do all these with the lock held, oh well */ -- vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!vptr) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return -ENOENT; -- } -- q_idx = *vptr; -- -- /* claim one task from cgrp_q w/ q_idx */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u64 *cgq_len, len; -- -- cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]); -- if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) { -- /* the cgroup must be empty, expire and repeat */ -- __sync_fetch_and_add(&nr_cgrp_empty, 1); -- bpf_spin_lock(&pairc->lock); -- pairc->draining = true; -- pairc->active_mask &= ~in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- return -EAGAIN; -- } -- -- if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len) -- continue; -- -- break; -- } -- -- cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx); -- if (!cgq_map) { -- scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- if (bpf_map_pop_elem(cgq_map, &pid)) { -- scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- p = bpf_task_from_pid(pid); -- if (p) { -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } else { -- /* we don't handle dequeues, retry on lost tasks */ -- __sync_fetch_and_add(&nr_missing, 1); -- return -EAGAIN; -- } -- --out_maybe_kick: -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } -- return 0; --} -- --void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) --{ -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_dispatch(cpu) != -EAGAIN) -- break; -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask &= ~in_pair_mask; -- /* Kick the pair CPU, unless it was also preempted. */ -- kick_pair = !pairc->preempted_mask; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask |= in_pair_mask; -- pairc->active_mask &= ~in_pair_mask; -- /* Kick the pair CPU if it's still running. */ -- kick_pair = pairc->active_mask; -- pairc->draining = true; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- } -- } -- __sync_fetch_and_add(&nr_preemptions, 1); --} -- --s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 i, q_idx; -- -- bpf_for(i, 0, MAX_CGRPS) { -- q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS; -- if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1)) -- break; -- } -- if (i == MAX_CGRPS) -- return -EBUSY; -- -- if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]); -- if (busy) -- *busy = 0; -- return -EBUSY; -- } -- -- return 0; --} -- --void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 *q_idx; -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (q_idx) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]); -- if (busy) -- *busy = 0; -- bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid); -- } --} -- --s32 BPF_STRUCT_OPS(pair_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops pair_ops = { -- .enqueue = (void *)pair_enqueue, -- .dispatch = (void *)pair_dispatch, -- .cpu_acquire = (void *)pair_cpu_acquire, -- .cpu_release = (void *)pair_cpu_release, -- .cgroup_init = (void *)pair_cgroup_init, -- .cgroup_exit = (void *)pair_cgroup_exit, -- .init = (void *)pair_init, -- .exit = (void *)pair_exit, -- .name = "pair", --}; -diff --git a/tools/sched_ext/scx_example_pair.c b/tools/sched_ext/scx_example_pair.c -deleted file mode 100644 -index 18e032bbc173..000000000000 ---- a/tools/sched_ext/scx_example_pair.c -+++ /dev/null -@@ -1,143 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_pair.h" --#include "scx_example_pair.skel.h" -- --const char help_fmt[] = --"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n" --"execute from the same CPU cgroup.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-S STRIDE] [-p]\n" --"\n" --" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_pair *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 stride, i, opt, outer_fd; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_pair__open(); -- assert(skel); -- -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- /* pair up the earlier half to the latter by default, override with -s */ -- stride = skel->rodata->nr_cpu_ids / 2; -- -- while ((opt = getopt(argc, argv, "S:ph")) != -1) { -- switch (opt) { -- case 'S': -- stride = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- for (i = 0; i < skel->rodata->nr_cpu_ids; i++) { -- if (skel->rodata->pair_cpu[i] < 0) { -- skel->rodata->pair_cpu[i] = i + stride; -- skel->rodata->pair_cpu[i + stride] = i; -- skel->rodata->pair_id[i] = i; -- skel->rodata->pair_id[i + stride] = i; -- skel->rodata->in_pair_idx[i] = 0; -- skel->rodata->in_pair_idx[i + stride] = 1; -- } -- } -- -- assert(!scx_example_pair__load(skel)); -- -- /* -- * Populate the cgrp_q_arr map which is an array containing per-cgroup -- * queues. It'd probably be better to do this from BPF but there are too -- * many to initialize statically and there's no way to dynamically -- * populate from BPF. -- */ -- outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr); -- assert(outer_fd >= 0); -- -- printf("Initializing"); -- for (i = 0; i < MAX_CGRPS; i++) { -- s32 inner_fd; -- -- if (exit_req) -- break; -- -- inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, -- sizeof(u32), MAX_QUEUED, NULL); -- assert(inner_fd >= 0); -- assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY)); -- close(inner_fd); -- -- if (!(i % 10)) -- printf("."); -- fflush(stdout); -- } -- printf("\n"); -- -- /* -- * Fully initialized, attach and run. -- */ -- link = bpf_map__attach_struct_ops(skel->maps.pair_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf(" total:%10lu dispatch:%10lu missing:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_dispatched, -- skel->bss->nr_missing); -- printf(" kicks:%10lu preemptions:%7lu\n", -- skel->bss->nr_kicks, -- skel->bss->nr_preemptions); -- printf(" exp:%10lu exp_wait:%10lu exp_empty:%10lu\n", -- skel->bss->nr_exps, -- skel->bss->nr_exp_waits, -- skel->bss->nr_exp_empty); -- printf("cgnext:%10lu cgcoll:%10lu cgempty:%10lu\n", -- skel->bss->nr_cgrp_next, -- skel->bss->nr_cgrp_coll, -- skel->bss->nr_cgrp_empty); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_pair__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_pair.h b/tools/sched_ext/scx_example_pair.h -deleted file mode 100644 -index f60b824272f7..000000000000 ---- a/tools/sched_ext/scx_example_pair.h -+++ /dev/null -@@ -1,10 +0,0 @@ --#ifndef __SCX_EXAMPLE_PAIR_H --#define __SCX_EXAMPLE_PAIR_H -- --enum { -- MAX_CPUS = 4096, -- MAX_QUEUED = 4096, -- MAX_CGRPS = 4096, --}; -- --#endif /* __SCX_EXAMPLE_PAIR_H */ -diff --git a/tools/sched_ext/scx_example_qmap.bpf.c b/tools/sched_ext/scx_example_qmap.bpf.c -deleted file mode 100644 -index 579ab21ae403..000000000000 ---- a/tools/sched_ext/scx_example_qmap.bpf.c -+++ /dev/null -@@ -1,401 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple five-level FIFO queue scheduler. -- * -- * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets -- * assigned to one depending on its compound weight. Each CPU round robins -- * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from -- * queue0, 2 from queue1, 4 from queue2 and so on. -- * -- * This scheduler demonstrates: -- * -- * - BPF-side queueing using PIDs. -- * - Sleepable per-task storage allocation using ops.prep_enable(). -- * - Using ops.cpu_release() to handle a higher priority scheduling class taking -- * the CPU away. -- * - Core-sched support. -- * -- * This scheduler is primarily for demonstration and testing of sched_ext -- * features and unlikely to be useful for actual workloads. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include -- --char _license[] SEC("license") = "GPL"; -- --const volatile u64 slice_ns = SCX_SLICE_DFL; --const volatile bool switch_partial; --const volatile u32 stall_user_nth; --const volatile u32 stall_kernel_nth; --const volatile u32 dsp_inf_loop_after; --const volatile s32 disallow_tgid; -- --u32 test_error_cnt; -- --struct user_exit_info uei; -- --struct qmap { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, u32); --} queue0 SEC(".maps"), -- queue1 SEC(".maps"), -- queue2 SEC(".maps"), -- queue3 SEC(".maps"), -- queue4 SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, 5); -- __type(key, int); -- __array(values, struct qmap); --} queue_arr SEC(".maps") = { -- .values = { -- [0] = &queue0, -- [1] = &queue1, -- [2] = &queue2, -- [3] = &queue3, -- [4] = &queue4, -- }, --}; -- --/* -- * Per-queue sequence numbers to implement core-sched ordering. -- * -- * Tail seq is assigned to each queued task and incremented. Head seq tracks the -- * sequence number of the latest dispatched task. The distance between the a -- * task's seq and the associated queue's head seq is called the queue distance -- * and used when comparing two tasks for ordering. See qmap_core_sched_before(). -- */ --static u64 core_sched_head_seqs[5]; --static u64 core_sched_tail_seqs[5]; -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local_dsq */ -- u64 core_sched_seq; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --/* Per-cpu dispatch index and remaining count */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(max_entries, 2); -- __type(key, u32); -- __type(value, u64); --} dispatch_idx_cnt SEC(".maps"); -- --/* Statistics */ --unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued; --unsigned long nr_core_sched_execed; -- --s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- struct task_ctx *tctx; -- s32 cpu; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- return cpu; -- -- return prev_cpu; --} -- --static int weight_to_idx(u32 weight) --{ -- /* Coarsely map the compound weight to a FIFO. */ -- if (weight <= 25) -- return 0; -- else if (weight <= 50) -- return 1; -- else if (weight < 200) -- return 2; -- else if (weight < 400) -- return 3; -- else -- return 4; --} -- --void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) --{ -- static u32 user_cnt, kernel_cnt; -- struct task_ctx *tctx; -- u32 pid = p->pid; -- int idx = weight_to_idx(p->scx.weight); -- void *ring; -- -- if (p->flags & PF_KTHREAD) { -- if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth)) -- return; -- } else { -- if (stall_user_nth && !(++user_cnt % stall_user_nth)) -- return; -- } -- -- if (test_error_cnt && !--test_error_cnt) -- scx_bpf_error("test triggering error"); -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * All enqueued tasks must have their core_sched_seq updated for correct -- * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in -- * qmap_ops.flags. -- */ -- tctx->core_sched_seq = core_sched_tail_seqs[idx]++; -- -- /* -- * If qmap_select_cpu() is telling us to or this is the last runnable -- * task on the CPU, enqueue locally. -- */ -- if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) { -- tctx->force_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); -- return; -- } -- -- /* -- * If the task was re-enqueued due to the CPU being preempted by a -- * higher priority scheduling class, just re-enqueue the task directly -- * on the global DSQ. As we want another CPU to pick it up, find and -- * kick an idle CPU. -- */ -- if (enq_flags & SCX_ENQ_REENQ) { -- s32 cpu; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags); -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- return; -- } -- -- ring = bpf_map_lookup_elem(&queue_arr, &idx); -- if (!ring) { -- scx_bpf_error("failed to find ring %d", idx); -- return; -- } -- -- /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */ -- if (bpf_map_push_elem(ring, &pid, 0)) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_enqueued, 1); --} -- --/* -- * The BPF queue map doesn't support removal and sched_ext can handle spurious -- * dispatches. qmap_dequeue() is only used to collect statistics. -- */ --void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags) --{ -- __sync_fetch_and_add(&nr_dequeued, 1); -- if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) -- __sync_fetch_and_add(&nr_core_sched_execed, 1); --} -- --static void update_core_sched_head_seq(struct task_struct *p) --{ -- struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- int idx = weight_to_idx(p->scx.weight); -- -- if (tctx) -- core_sched_head_seqs[idx] = tctx->core_sched_seq; -- else -- scx_bpf_error("task_ctx lookup failed"); --} -- --void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 zero = 0, one = 1; -- u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero); -- u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one); -- void *fifo; -- s32 pid; -- int i; -- -- if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) { -- struct task_struct *p; -- -- /* -- * PID 2 should be kthreadd which should mostly be idle and off -- * the scheduler. Let's keep dispatching it to force the kernel -- * to call this function over and over again. -- */ -- p = bpf_task_from_pid(2); -- if (p) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- if (!idx || !cnt) { -- scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt); -- return; -- } -- -- for (i = 0; i < 5; i++) { -- /* Advance the dispatch cursor and pick the fifo. */ -- if (!*cnt) { -- *idx = (*idx + 1) % 5; -- *cnt = 1 << *idx; -- } -- (*cnt)--; -- -- fifo = bpf_map_lookup_elem(&queue_arr, idx); -- if (!fifo) { -- scx_bpf_error("failed to find ring %llu", *idx); -- return; -- } -- -- /* Dispatch or advance. */ -- if (!bpf_map_pop_elem(fifo, &pid)) { -- struct task_struct *p; -- -- p = bpf_task_from_pid(pid); -- if (p) { -- update_core_sched_head_seq(p); -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- *cnt = 0; -- } --} -- --/* -- * The distance from the head of the queue scaled by the weight of the queue. -- * The lower the number, the older the task and the higher the priority. -- */ --static s64 task_qdist(struct task_struct *p) --{ -- int idx = weight_to_idx(p->scx.weight); -- struct task_ctx *tctx; -- s64 qdist; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return 0; -- } -- -- qdist = tctx->core_sched_seq - core_sched_head_seqs[idx]; -- -- /* -- * As queue index increments, the priority doubles. The queue w/ index 3 -- * is dispatched twice more frequently than 2. Reflect the difference by -- * scaling qdists accordingly. Note that the shift amount needs to be -- * flipped depending on the sign to avoid flipping priority direction. -- */ -- if (qdist >= 0) -- return qdist << (4 - idx); -- else -- return qdist << idx; --} -- --/* -- * This is called to determine the task ordering when core-sched is picking -- * tasks to execute on SMT siblings and should encode about the same ordering as -- * the regular scheduling path. Use the priority-scaled distances from the head -- * of the queues to compare the two tasks which should be consistent with the -- * dispatch path behavior. -- */ --bool BPF_STRUCT_OPS(qmap_core_sched_before, -- struct task_struct *a, struct task_struct *b) --{ -- return task_qdist(a) > task_qdist(b); --} -- --void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- u32 cnt; -- -- /* -- * Called when @cpu is taken by a higher priority scheduling class. This -- * makes @cpu no longer available for executing sched_ext tasks. As we -- * don't want the tasks in @cpu's local dsq to sit there until @cpu -- * becomes available again, re-enqueue them into the global dsq. See -- * %SCX_ENQ_REENQ handling in qmap_enqueue(). -- */ -- cnt = scx_bpf_reenqueue_local(); -- if (cnt) -- __sync_fetch_and_add(&nr_reenqueued, cnt); --} -- --s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (p->tgid == disallow_tgid) -- p->scx.disallow = true; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(qmap_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops qmap_ops = { -- .select_cpu = (void *)qmap_select_cpu, -- .enqueue = (void *)qmap_enqueue, -- .dequeue = (void *)qmap_dequeue, -- .dispatch = (void *)qmap_dispatch, -- .core_sched_before = (void *)qmap_core_sched_before, -- .cpu_release = (void *)qmap_cpu_release, -- .prep_enable = (void *)qmap_prep_enable, -- .init = (void *)qmap_init, -- .exit = (void *)qmap_exit, -- .flags = SCX_OPS_ENQ_LAST, -- .timeout_ms = 5000U, -- .name = "qmap", --}; -diff --git a/tools/sched_ext/scx_example_qmap.c b/tools/sched_ext/scx_example_qmap.c -deleted file mode 100644 -index ccb4814ee61b..000000000000 ---- a/tools/sched_ext/scx_example_qmap.c -+++ /dev/null -@@ -1,107 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_qmap.skel.h" -- --const char help_fmt[] = --"A simple five-level FIFO queue sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" --" -t COUNT Stall every COUNT'th user thread\n" --" -T COUNT Stall every COUNT'th kernel thread\n" --" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n" --" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_qmap *skel; -- struct bpf_link *link; -- int opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_qmap__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) { -- switch (opt) { -- case 's': -- skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; -- break; -- case 'e': -- skel->bss->test_error_cnt = strtoul(optarg, NULL, 0); -- break; -- case 't': -- skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0); -- break; -- case 'T': -- skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0); -- break; -- case 'l': -- skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0); -- break; -- case 'd': -- skel->rodata->disallow_tgid = strtol(optarg, NULL, 0); -- if (skel->rodata->disallow_tgid < 0) -- skel->rodata->disallow_tgid = getpid(); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_qmap__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.qmap_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- long nr_enqueued = skel->bss->nr_enqueued; -- long nr_dispatched = skel->bss->nr_dispatched; -- -- printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n", -- nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, -- skel->bss->nr_reenqueued, skel->bss->nr_dequeued, -- skel->bss->nr_core_sched_execed); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_qmap__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_simple.bpf.c b/tools/sched_ext/scx_example_simple.bpf.c -deleted file mode 100644 -index 4bccca3e2047..000000000000 ---- a/tools/sched_ext/scx_example_simple.bpf.c -+++ /dev/null -@@ -1,128 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple scheduler. -- * -- * By default, it operates as a simple global weighted vtime scheduler and can -- * be switched to FIFO scheduling. It also demonstrates the following niceties. -- * -- * - Statistics tracking how many tasks are queued to local and global dsq's. -- * - Termination notification for userspace. -- * -- * While very simple, this scheduler should work reasonably well on CPUs with a -- * uniform L3 cache topology. While preemption is not implemented, the fact that -- * the scheduling queue is shared across all CPUs means that whatever is at the -- * front of the queue is likely to be executed fairly quickly given enough -- * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads -- * but comes with the usual problems with FIFO scheduling where saturating -- * threads can easily drown out interactive ones. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --static u64 vtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, 2); /* [local, global] */ --} stats SEC(".maps"); -- --static void stat_inc(u32 idx) --{ -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); -- if (cnt_p) -- (*cnt_p)++; --} -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) --{ -- /* -- * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that -- * running @p on its CPU directly shouldn't affect fairness. Just queue -- * it on the local FIFO. -- */ -- if (enq_flags & SCX_ENQ_LOCAL) { -- stat_inc(0); /* count local queueing */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- return; -- } -- -- stat_inc(1); /* count global queueing */ -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) -- vtime = vtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --void BPF_STRUCT_OPS(simple_running, struct task_struct *p) --{ -- if (fifo_sched) -- return; -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(vtime_now, p->scx.dsq_vtime)) -- vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --s32 BPF_STRUCT_OPS(simple_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .running = (void *)simple_running, -- .stopping = (void *)simple_stopping, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", --}; -diff --git a/tools/sched_ext/scx_example_simple.c b/tools/sched_ext/scx_example_simple.c -deleted file mode 100644 -index 486b401f7c95..000000000000 ---- a/tools/sched_ext/scx_example_simple.c -+++ /dev/null -@@ -1,101 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_simple.skel.h" -- --const char help_fmt[] = --"A simple sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-f] [-p]\n" --"\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int simple) --{ -- exit_req = 1; --} -- --static void read_stats(struct scx_example_simple *skel, u64 *stats) --{ -- int nr_cpus = libbpf_num_possible_cpus(); -- u64 cnts[2][nr_cpus]; -- u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * 2); -- -- for (idx = 0; idx < 2; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_simple *skel; -- struct bpf_link *link; -- u32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_simple__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "fph")) != -1) { -- switch (opt) { -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_simple__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.simple_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- u64 stats[2]; -- -- read_stats(skel, stats); -- printf("local=%lu global=%lu\n", stats[0], stats[1]); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_simple__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_userland.bpf.c b/tools/sched_ext/scx_example_userland.bpf.c -deleted file mode 100644 -index a089bc6bbe86..000000000000 ---- a/tools/sched_ext/scx_example_userland.bpf.c -+++ /dev/null -@@ -1,269 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A minimal userland scheduler. -- * -- * In terms of scheduling, this provides two different types of behaviors: -- * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity. -- * All such tasks are direct-dispatched from the kernel, and are never -- * enqueued in user space. -- * 2. A primitive vruntime scheduler that is implemented in user space, for all -- * other tasks. -- * -- * Some parts of this example user space scheduler could be implemented more -- * efficiently using more complex and sophisticated data structures. For -- * example, rather than using BPF_MAP_TYPE_QUEUE's, -- * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between -- * user space and kernel space. Similarly, we use a simple vruntime-sorted list -- * in user space, but an rbtree could be used instead. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include --#include "scx_common.bpf.h" --#include "scx_example_userland_common.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; --const volatile s32 usersched_pid; -- --/* !0 for veristat, set during init */ --const volatile u32 num_possible_cpus = 64; -- --/* Stats that are printed by user space. */ --u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues; -- --struct user_exit_info uei; -- --/* -- * Whether the user space scheduler needs to be scheduled due to a task being -- * enqueued in user space. -- */ --static bool usersched_needed; -- --/* -- * The map containing tasks that are enqueued in user space from the kernel. -- * -- * This map is drained by the user space scheduler. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, struct scx_userland_enqueued_task); --} enqueued SEC(".maps"); -- --/* -- * The map containing tasks that are dispatched to the kernel from user space. -- * -- * Drained by the kernel in userland_dispatch(). -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, s32); --} dispatched SEC(".maps"); -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local DSQ */ --}; -- --/* Map that contains task-local storage. */ --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --static bool is_usersched_task(const struct task_struct *p) --{ -- return p->pid == usersched_pid; --} -- --static bool keep_in_kernel(const struct task_struct *p) --{ -- return p->nr_cpus_allowed < num_possible_cpus; --} -- --static struct task_struct *usersched_task(void) --{ -- struct task_struct *p; -- -- p = bpf_task_from_pid(usersched_pid); -- /* -- * Should never happen -- the usersched task should always be managed -- * by sched_ext. -- */ -- if (!p) { -- scx_bpf_error("Failed to find usersched task %d", usersched_pid); -- /* -- * We should never hit this path, and we error out of the -- * scheduler above just in case, so the scheduler will soon be -- * be evicted regardless. So as to simplify the logic in the -- * caller to not have to check for NULL, return an acquired -- * reference to the current task here rather than NULL. -- */ -- return bpf_task_acquire(bpf_get_current_task_btf()); -- } -- -- return p; --} -- --s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- if (keep_in_kernel(p)) { -- s32 cpu; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to look up task-local storage for %s", p->comm); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- tctx->force_local = true; -- return cpu; -- } -- } -- -- return prev_cpu; --} -- --static void dispatch_user_scheduler(void) --{ -- struct task_struct *p; -- -- usersched_needed = false; -- p = usersched_task(); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); --} -- --static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags) --{ -- struct scx_userland_enqueued_task task; -- -- memset(&task, 0, sizeof(task)); -- task.pid = p->pid; -- task.sum_exec_runtime = p->se.sum_exec_runtime; -- task.weight = p->scx.weight; -- -- if (bpf_map_push_elem(&enqueued, &task, 0)) { -- /* -- * If we fail to enqueue the task in user space, put it -- * directly on the global DSQ. -- */ -- __sync_fetch_and_add(&nr_failed_enqueues, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- __sync_fetch_and_add(&nr_user_enqueues, 1); -- usersched_needed = true; -- } --} -- --void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags) --{ -- if (keep_in_kernel(p)) { -- u64 dsq_id = SCX_DSQ_GLOBAL; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to lookup task ctx for %s", p->comm); -- return; -- } -- -- if (tctx->force_local) -- dsq_id = SCX_DSQ_LOCAL; -- tctx->force_local = false; -- scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags); -- __sync_fetch_and_add(&nr_kernel_enqueues, 1); -- return; -- } else if (!is_usersched_task(p)) { -- enqueue_task_in_user_space(p, enq_flags); -- } --} -- --void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (usersched_needed) -- dispatch_user_scheduler(); -- -- bpf_repeat(4096) { -- s32 pid; -- struct task_struct *p; -- -- if (bpf_map_pop_elem(&dispatched, &pid)) -- break; -- -- /* -- * The task could have exited by the time we get around to -- * dispatching it. Treat this as a normal occurrence, and simply -- * move onto the next iteration. -- */ -- p = bpf_task_from_pid(pid); -- if (!p) -- continue; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } --} -- --s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(userland_init) --{ -- if (num_possible_cpus == 0) { -- scx_bpf_error("User scheduler # CPUs uninitialized (%d)", -- num_possible_cpus); -- return -EINVAL; -- } -- -- if (usersched_pid <= 0) { -- scx_bpf_error("User scheduler pid uninitialized (%d)", -- usersched_pid); -- return -EINVAL; -- } -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops userland_ops = { -- .select_cpu = (void *)userland_select_cpu, -- .enqueue = (void *)userland_enqueue, -- .dispatch = (void *)userland_dispatch, -- .prep_enable = (void *)userland_prep_enable, -- .init = (void *)userland_init, -- .exit = (void *)userland_exit, -- .timeout_ms = 3000, -- .name = "userland", --}; -diff --git a/tools/sched_ext/scx_example_userland.c b/tools/sched_ext/scx_example_userland.c -deleted file mode 100644 -index 4152b1e65fe1..000000000000 ---- a/tools/sched_ext/scx_example_userland.c -+++ /dev/null -@@ -1,402 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext user space scheduler which provides vruntime semantics -- * using a simple ordered-list implementation. -- * -- * Each CPU in the system resides in a single, global domain. This precludes -- * the need to do any load balancing between domains. The scheduler could -- * easily be extended to support multiple domains, with load balancing -- * happening in user space. -- * -- * Any task which has any CPU affinity is scheduled entirely in BPF. This -- * program only schedules tasks which may run on any CPU. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include "user_exit_info.h" --#include "scx_example_userland_common.h" --#include "scx_example_userland.skel.h" -- --const char help_fmt[] = --"A minimal userland sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-b BATCH] [-p]\n" --"\n" --" -b BATCH The number of tasks to batch when dispatching (default: 8)\n" --" -p Don't switch all, switch only tasks on SCHED_EXT policy\n" --" -h Display this help and exit\n"; -- --/* Defined in UAPI */ --#define SCHED_EXT 7 -- --/* Number of tasks to batch when dispatching to user space. */ --static __u32 batch_size = 8; -- --static volatile int exit_req; --static int enqueued_fd, dispatched_fd; -- --static struct scx_example_userland *skel; --static struct bpf_link *ops_link; -- --/* Stats collected in user space. */ --static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches; -- --/* The data structure containing tasks that are enqueued in user space. */ --struct enqueued_task { -- LIST_ENTRY(enqueued_task) entries; -- __u64 sum_exec_runtime; -- double vruntime; --}; -- --/* -- * Use a vruntime-sorted list to store tasks. This could easily be extended to -- * a more optimal data structure, such as an rbtree as is done in CFS. We -- * currently elect to use a sorted list to simplify the example for -- * illustrative purposes. -- */ --LIST_HEAD(listhead, enqueued_task); -- --/* -- * A vruntime-sorted list of tasks. The head of the list contains the task with -- * the lowest vruntime. That is, the task that has the "highest" claim to be -- * scheduled. -- */ --static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head); -- --/* -- * The statically allocated array of tasks. We use a statically allocated list -- * here to avoid having to allocate on the enqueue path, which could cause a -- * deadlock. A more substantive user space scheduler could e.g. provide a hook -- * for newly enabled tasks that are passed to the scheduler from the -- * .prep_enable() callback to allows the scheduler to allocate on safe paths. -- */ --struct enqueued_task tasks[USERLAND_MAX_TASKS]; -- --static double min_vruntime; -- --static void sigint_handler(int userland) --{ -- exit_req = 1; --} -- --static __u32 task_pid(const struct enqueued_task *task) --{ -- return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task); --} -- --static int dispatch_task(s32 pid) --{ -- int err; -- -- err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d\n", pid); -- exit_req = 1; -- } else { -- nr_vruntime_dispatches++; -- } -- -- return err; --} -- --static struct enqueued_task *get_enqueued_task(__s32 pid) --{ -- if (pid >= USERLAND_MAX_TASKS) -- return NULL; -- -- return &tasks[pid]; --} -- --static double calc_vruntime_delta(__u64 weight, __u64 delta) --{ -- double weight_f = (double)weight / 100.0; -- double delta_f = (double)delta; -- -- return delta_f / weight_f; --} -- --static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task) --{ -- __u64 delta; -- -- delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime; -- -- enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta); -- if (min_vruntime > enqueued->vruntime) -- enqueued->vruntime = min_vruntime; -- enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime; --} -- --static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task) --{ -- struct enqueued_task *curr, *enqueued, *prev; -- -- curr = get_enqueued_task(bpf_task->pid); -- if (!curr) -- return ENOENT; -- -- update_enqueued(curr, bpf_task); -- nr_vruntime_enqueues++; -- -- /* -- * Enqueue the task in a vruntime-sorted list. A more optimal data -- * structure such as an rbtree could easily be used as well. We elect -- * to use a list here simply because it's less code, and thus the -- * example is less convoluted and better serves to illustrate what a -- * user space scheduler could look like. -- */ -- -- if (LIST_EMPTY(&vruntime_head)) { -- LIST_INSERT_HEAD(&vruntime_head, curr, entries); -- return 0; -- } -- -- LIST_FOREACH(enqueued, &vruntime_head, entries) { -- if (curr->vruntime <= enqueued->vruntime) { -- LIST_INSERT_BEFORE(enqueued, curr, entries); -- return 0; -- } -- prev = enqueued; -- } -- -- LIST_INSERT_AFTER(prev, curr, entries); -- -- return 0; --} -- --static void drain_enqueued_map(void) --{ -- while (1) { -- struct scx_userland_enqueued_task task; -- int err; -- -- if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) -- return; -- -- err = vruntime_enqueue(&task); -- if (err) { -- fprintf(stderr, "Failed to enqueue task %d: %s\n", -- task.pid, strerror(err)); -- exit_req = 1; -- return; -- } -- } --} -- --static void dispatch_batch(void) --{ -- __u32 i; -- -- for (i = 0; i < batch_size; i++) { -- struct enqueued_task *task; -- int err; -- __s32 pid; -- -- task = LIST_FIRST(&vruntime_head); -- if (!task) -- return; -- -- min_vruntime = task->vruntime; -- pid = task_pid(task); -- LIST_REMOVE(task, entries); -- err = dispatch_task(pid); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d in %u\n", -- pid, i); -- return; -- } -- } --} -- --static void *run_stats_printer(void *arg) --{ -- while (!exit_req) { -- __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; -- -- nr_failed_enqueues = skel->bss->nr_failed_enqueues; -- nr_kernel_enqueues = skel->bss->nr_kernel_enqueues; -- nr_user_enqueues = skel->bss->nr_user_enqueues; -- total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues; -- -- printf("o-----------------------o\n"); -- printf("| BPF ENQUEUES |\n"); -- printf("|-----------------------|\n"); -- printf("| kern: %10llu |\n", nr_kernel_enqueues); -- printf("| user: %10llu |\n", nr_user_enqueues); -- printf("| failed: %10llu |\n", nr_failed_enqueues); -- printf("| -------------------- |\n"); -- printf("| total: %10llu |\n", total); -- printf("| |\n"); -- printf("|-----------------------|\n"); -- printf("| VRUNTIME / USER |\n"); -- printf("|-----------------------|\n"); -- printf("| enq: %10llu |\n", nr_vruntime_enqueues); -- printf("| disp: %10llu |\n", nr_vruntime_dispatches); -- printf("o-----------------------o\n"); -- printf("\n\n"); -- sleep(1); -- } -- -- return NULL; --} -- --static int spawn_stats_thread(void) --{ -- pthread_t stats_printer; -- -- return pthread_create(&stats_printer, NULL, run_stats_printer, NULL); --} -- --static int bootstrap(int argc, char **argv) --{ -- int err; -- __u32 opt; -- struct sched_param sched_param = { -- .sched_priority = sched_get_priority_max(SCHED_EXT), -- }; -- bool switch_partial = false; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- /* -- * Enforce that the user scheduler task is managed by sched_ext. The -- * task eagerly drains the list of enqueued tasks in its main work -- * loop, and then yields the CPU. The BPF scheduler only schedules the -- * user space scheduler task when at least one other task in the system -- * needs to be scheduled. -- */ -- err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param); -- if (err) { -- fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err)); -- return err; -- } -- -- while ((opt = getopt(argc, argv, "b:ph")) != -1) { -- switch (opt) { -- case 'b': -- batch_size = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- exit(opt != 'h'); -- } -- } -- -- /* -- * It's not always safe to allocate in a user space scheduler, as an -- * enqueued task could hold a lock that we require in order to be able -- * to allocate. -- */ -- err = mlockall(MCL_CURRENT | MCL_FUTURE); -- if (err) { -- fprintf(stderr, "Failed to prefault and lock address space: %s\n", -- strerror(err)); -- return err; -- } -- -- skel = scx_example_userland__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno)); -- return errno; -- } -- skel->rodata->num_possible_cpus = libbpf_num_possible_cpus(); -- assert(skel->rodata->num_possible_cpus > 0); -- skel->rodata->usersched_pid = getpid(); -- assert(skel->rodata->usersched_pid > 0); -- skel->rodata->switch_partial = switch_partial; -- -- err = scx_example_userland__load(skel); -- if (err) { -- fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- enqueued_fd = bpf_map__fd(skel->maps.enqueued); -- dispatched_fd = bpf_map__fd(skel->maps.dispatched); -- assert(enqueued_fd > 0); -- assert(dispatched_fd > 0); -- -- err = spawn_stats_thread(); -- if (err) { -- fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops); -- if (!ops_link) { -- fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno)); -- err = errno; -- goto destroy_skel; -- } -- -- return 0; -- --destroy_skel: -- scx_example_userland__destroy(skel); -- exit_req = 1; -- return err; --} -- --static void sched_main_loop(void) --{ -- while (!exit_req) { -- /* -- * Perform the following work in the main user space scheduler -- * loop: -- * -- * 1. Drain all tasks from the enqueued map, and enqueue them -- * to the vruntime sorted list. -- * -- * 2. Dispatch a batch of tasks from the vruntime sorted list -- * down to the kernel. -- * -- * 3. Yield the CPU back to the system. The BPF scheduler will -- * reschedule the user space scheduler once another task has -- * been enqueued to user space. -- */ -- drain_enqueued_map(); -- dispatch_batch(); -- sched_yield(); -- } --} -- --int main(int argc, char **argv) --{ -- int err; -- -- err = bootstrap(argc, argv); -- if (err) { -- fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err)); -- return err; -- } -- -- sched_main_loop(); -- -- exit_req = 1; -- bpf_link__destroy(ops_link); -- uei_print(&skel->bss->uei); -- scx_example_userland__destroy(skel); -- return 0; --} -diff --git a/tools/sched_ext/scx_example_userland_common.h b/tools/sched_ext/scx_example_userland_common.h -deleted file mode 100644 -index 639c6809c5ff..000000000000 ---- a/tools/sched_ext/scx_example_userland_common.h -+++ /dev/null -@@ -1,19 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* Copyright (c) 2022 Meta, Inc */ -- --#ifndef __SCX_USERLAND_COMMON_H --#define __SCX_USERLAND_COMMON_H -- --#define USERLAND_MAX_TASKS 8192 -- --/* -- * An instance of a task that has been enqueued by the kernel for consumption -- * by a user space global scheduler thread. -- */ --struct scx_userland_enqueued_task { -- __s32 pid; -- u64 sum_exec_runtime; -- u64 weight; --}; -- --#endif // __SCX_USERLAND_COMMON_H -diff --git a/tools/sched_ext/user_exit_info.h b/tools/sched_ext/user_exit_info.h -deleted file mode 100644 -index e701ef0e0b86..000000000000 ---- a/tools/sched_ext/user_exit_info.h -+++ /dev/null -@@ -1,50 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Define struct user_exit_info which is shared between BPF and userspace parts -- * to communicate exit status and other information. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __USER_EXIT_INFO_H --#define __USER_EXIT_INFO_H -- --struct user_exit_info { -- int type; -- char reason[128]; -- char msg[1024]; --}; -- --#ifdef __bpf__ -- --#include "vmlinux.h" --#include -- --static inline void uei_record(struct user_exit_info *uei, -- const struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason); -- bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg); -- /* use __sync to force memory barrier */ -- __sync_val_compare_and_swap(&uei->type, uei->type, ei->type); --} -- --#else /* !__bpf__ */ -- --static inline bool uei_exited(struct user_exit_info *uei) --{ -- /* use __sync to force memory barrier */ -- return __sync_val_compare_and_swap(&uei->type, -1, -1); --} -- --static inline void uei_print(const struct user_exit_info *uei) --{ -- fprintf(stderr, "EXIT: %s", uei->reason); -- if (uei->msg[0] != '\0') -- fprintf(stderr, " (%s)", uei->msg); -- fputs("\n", stderr); --} -- --#endif /* __bpf__ */ --#endif /* __USER_EXIT_INFO_H */ -diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c -index ce05928392bf..f47f79079297 100755 ---- a/kernel/sched/hmbird/hmbird.c -+++ b/kernel/sched/hmbird/hmbird.c -@@ -633,14 +633,14 @@ static void init_isolate_cpus(void) - WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); - WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -- cpumask_set_cpu(0, iso_masks.big); -- cpumask_set_cpu(1, iso_masks.big); -- cpumask_set_cpu(2, iso_masks.big); -- cpumask_set_cpu(3, iso_masks.big); -- cpumask_set_cpu(4, iso_masks.big); -- cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(0, iso_masks.little); -+ cpumask_set_cpu(1, iso_masks.little); -+ cpumask_set_cpu(2, iso_masks.little); -+ cpumask_set_cpu(3, iso_masks.little); -+ cpumask_set_cpu(4, iso_masks.little); -+ cpumask_set_cpu(5, iso_masks.little); - cpumask_set_cpu(6, iso_masks.partial); -- cpumask_set_cpu(7, iso_masks.ex_free); -+ cpumask_set_cpu(7, iso_masks.big); - } - - extern spinlock_t css_set_lock; diff --git a/oneplus/hmbird/fengchi_OP13T_A16.patch b/oneplus/hmbird/fengchi_OP13T_A16.patch deleted file mode 100644 index 81dfa588..00000000 --- a/oneplus/hmbird/fengchi_OP13T_A16.patch +++ /dev/null @@ -1,19895 +0,0 @@ -diff --git b/Documentation/scheduler/index.rst a/Documentation/scheduler/index.rst -index 0b650bb550e6..3170747226f6 100644 ---- b/Documentation/scheduler/index.rst -+++ a/Documentation/scheduler/index.rst -@@ -19,7 +19,6 @@ Scheduler - sched-nice-design - sched-rt-group - sched-stats -- sched-ext - sched-debug - - text_files -diff --git b/Documentation/scheduler/sched-ext.rst a/Documentation/scheduler/sched-ext.rst -deleted file mode 100644 -index 84c30b44f104..000000000000 ---- b/Documentation/scheduler/sched-ext.rst -+++ /dev/null -@@ -1,230 +0,0 @@ --========================== --Extensible Scheduler Class --========================== -- --sched_ext is a scheduler class whose behavior can be defined by a set of BPF --programs - the BPF scheduler. -- --* sched_ext exports a full scheduling interface so that any scheduling -- algorithm can be implemented on top. -- --* The BPF scheduler can group CPUs however it sees fit and schedule them -- together, as tasks aren't tied to specific CPUs at the time of wakeup. -- --* The BPF scheduler can be turned on and off dynamically anytime. -- --* The system integrity is maintained no matter what the BPF scheduler does. -- The default scheduling behavior is restored anytime an error is detected, -- a runnable task stalls, or on invoking the SysRq key sequence -- :kbd:`SysRq-S`. -- --Switching to and from sched_ext --=============================== -- --``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and --``tools/sched_ext`` contains the example schedulers. -- --sched_ext is used only when the BPF scheduler is loaded and running. -- --If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be --treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is --loaded. On load, such tasks will be switched to and scheduled by sched_ext. -- --The BPF scheduler can choose to schedule all normal and lower class tasks by --calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this --case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and --``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers, --this mode can be selected with the ``-a`` option. -- --Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or --detection of any internal error including stalled runnable tasks aborts the --BPF scheduler and reverts all tasks back to CFS. -- --.. code-block:: none -- -- # make -j16 -C tools/sched_ext -- # tools/sched_ext/scx_example_simple -- local=0 global=3 -- local=5 global=24 -- local=9 global=44 -- local=13 global=56 -- local=17 global=72 -- ^CEXIT: BPF scheduler unregistered -- --If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler --and whether a given task is on sched_ext can be determined as follows: -- --.. code-block:: none -- -- # cat /sys/kernel/debug/sched/ext -- ops : simple -- enabled : 1 -- switching_all : 1 -- switched_all : 1 -- enable_state : enabled -- -- # grep ext /proc/self/sched -- ext.enabled : 1 -- --The Basics --========== -- --Userspace can implement an arbitrary BPF scheduler by loading a set of BPF --programs that implement ``struct sched_ext_ops``. The only mandatory field --is ``ops.name`` which must be a valid BPF object name. All operations are --optional. The following modified excerpt is from --``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO --scheduler. -- --.. code-block:: c -- -- s32 BPF_STRUCT_OPS(simple_init) -- { -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; -- } -- -- void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) -- { -- if (enq_flags & SCX_ENQ_LOCAL) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags); -- } -- -- void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) -- { -- exit_type = ei->type; -- } -- -- SEC(".struct_ops") -- struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", -- }; -- --Dispatch Queues ----------------- -- --To match the impedance between the scheduler core and the BPF scheduler, --sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a --priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``), --and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage --an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and --``scx_bpf_destroy_dsq()``. -- --A CPU always executes a task from its local DSQ. A task is "dispatched" to a --DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's --local DSQ. -- --When a CPU is looking for the next task to run, if the local DSQ is not --empty, the first task is picked. Otherwise, the CPU tries to consume the --global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()`` --is invoked. -- --Scheduling Cycle ------------------ -- --The following briefly shows how a waking task is scheduled and executed. -- --1. When a task is waking up, ``ops.select_cpu()`` is the first operation -- invoked. This serves two purposes. First, CPU selection optimization -- hint. Second, waking up the selected CPU if idle. -- -- The CPU selected by ``ops.select_cpu()`` is an optimization hint and not -- binding. The actual decision is made at the last step of scheduling. -- However, there is a small performance gain if the CPU -- ``ops.select_cpu()`` returns matches the CPU the task eventually runs on. -- -- A side-effect of selecting a CPU is waking it up from idle. While a BPF -- scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper, -- using ``ops.select_cpu()`` judiciously can be simpler and more efficient. -- -- Note that the scheduler core will ignore an invalid CPU selection, for -- example, if it's outside the allowed cpumask of the task. -- --2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can -- make one of the following decisions: -- -- * Immediately dispatch the task to either the global or local DSQ by -- calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or -- ``SCX_DSQ_LOCAL``, respectively. -- -- * Immediately dispatch the task to a custom DSQ by calling -- ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63. -- -- * Queue the task on the BPF side. -- --3. When a CPU is ready to schedule, it first looks at its local DSQ. If -- empty, it then looks at the global DSQ. If there still isn't a task to -- run, ``ops.dispatch()`` is invoked which can use the following two -- functions to populate the local DSQ. -- -- * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can -- be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``, -- ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()`` -- currently can't be called with BPF locks held, this is being worked on -- and will be supported. ``scx_bpf_dispatch()`` schedules dispatching -- rather than performing them immediately. There can be up to -- ``ops.dispatch_max_batch`` pending tasks. -- -- * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ -- to the dispatching DSQ. This function cannot be called with any BPF -- locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks -- before trying to consume the specified DSQ. -- --4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ, -- the CPU runs the first one. If empty, the following steps are taken: -- -- * Try to consume the global DSQ. If successful, run the task. -- -- * If ``ops.dispatch()`` has dispatched any tasks, retry #3. -- -- * If the previous task is an SCX task and still runnable, keep executing -- it (see ``SCX_OPS_ENQ_LAST``). -- -- * Go idle. -- --Note that the BPF scheduler can always choose to dispatch tasks immediately --in ``ops.enqueue()`` as illustrated in the above simple example. If only the --built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as --a task is never queued on the BPF scheduler and both the local and global --DSQs are consumed automatically. -- --``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use --``scx_bpf_dispatch_vtime()`` for the priority queue. See the function --documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for --more information. -- --Where to Look --============= -- --* ``include/linux/sched/ext.h`` defines the core data structures, ops table -- and constants. -- --* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. -- The functions prefixed with ``scx_bpf_`` can be called from the BPF -- scheduler. -- --* ``tools/sched_ext/`` hosts example BPF scheduler implementations. -- -- * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example -- using a custom DSQ. -- -- * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting -- five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. -- --ABI Instability --=============== -- --The APIs provided by sched_ext to BPF schedulers programs have no stability --guarantees. This includes the ops table callbacks and constants defined in --``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in --``kernel/sched/ext.c``. -- --While we will attempt to provide a relatively stable API surface when --possible, they are subject to change without warning between kernel --versions. -diff --git b/MAINTAINERS a/MAINTAINERS -index 9fc6108503c3..2384b55682ee 100644 ---- b/MAINTAINERS -+++ a/MAINTAINERS -@@ -19132,8 +19132,6 @@ R: Ben Segall (CONFIG_CFS_BANDWIDTH) - R: Mel Gorman (CONFIG_NUMA_BALANCING) - R: Daniel Bristot de Oliveira (SCHED_DEADLINE) - R: Valentin Schneider (TOPOLOGY) --R: Tejun Heo (SCHED_EXT) --R: David Vernet (SCHED_EXT) - L: linux-kernel@vger.kernel.org - S: Maintained - T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core -@@ -19142,7 +19140,6 @@ F: include/linux/sched.h - F: include/linux/wait.h - F: include/uapi/linux/sched.h - F: kernel/sched/ --F: tools/sched_ext/ - - SCSI LIBSAS SUBSYSTEM - R: John Garry -diff --git b/arch/arm64/configs/defconfig a/arch/arm64/configs/defconfig -index f93980c09d7f..c2d530535980 100644 ---- b/arch/arm64/configs/defconfig -+++ a/arch/arm64/configs/defconfig -@@ -6,8 +6,6 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_BPF_SYSCALL=y - CONFIG_BPF_JIT=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_BSD_PROCESS_ACCT=y - CONFIG_BSD_PROCESS_ACCT_V3=y -@@ -1587,3 +1585,4 @@ CONFIG_CORESIGHT_STM=m - CONFIG_CORESIGHT_CPU_DEBUG=m - CONFIG_CORESIGHT_CTI=m - CONFIG_MEMTEST=y -+CONFIG_HMBIRD_SCHED=y -diff --git b/arch/arm64/configs/gki_defconfig a/arch/arm64/configs/gki_defconfig -index 4f756dca5e05..6c1bb94f875d 100644 ---- b/arch/arm64/configs/gki_defconfig -+++ a/arch/arm64/configs/gki_defconfig -@@ -10,8 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y - # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set - CONFIG_BPF_LSM=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_TASKSTATS=y - CONFIG_TASK_DELAY_ACCT=y -diff --git b/drivers/gpu/drm/msm/msm_drv.c a/drivers/gpu/drm/msm/msm_drv.c -index 4bd028fa7500..5825ce9d6d7b 100755 ---- b/drivers/gpu/drm/msm/msm_drv.c -+++ a/drivers/gpu/drm/msm/msm_drv.c -@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) - } - - sched_set_fifo(ev_thread->worker->task); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3); -+#endif - } - - ret = drm_vblank_init(ddev, priv->num_crtcs); -diff --git b/drivers/tty/sysrq.c a/drivers/tty/sysrq.c -index bd001445815e..dcf2e1ebf9c5 100644 ---- b/drivers/tty/sysrq.c -+++ a/drivers/tty/sysrq.c -@@ -524,7 +524,6 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { - NULL, /* P */ - NULL, /* Q */ - NULL, /* R */ -- /* S: May be registered by sched_ext for resetting */ - NULL, /* S */ - NULL, /* T */ - NULL, /* U */ -diff --git b/include/asm-generic/vmlinux.lds.h a/include/asm-generic/vmlinux.lds.h -index c45078a445c8..6c15659f874e 100755 ---- b/include/asm-generic/vmlinux.lds.h -+++ a/include/asm-generic/vmlinux.lds.h -@@ -131,7 +131,7 @@ - *(__dl_sched_class) \ - *(__rt_sched_class) \ - *(__fair_sched_class) \ -- *(__ext_sched_class) \ -+ *(__hmbird_sched_class) \ - *(__idle_sched_class) \ - __sched_class_lowest = .; - -diff --git b/include/linux/sched.h a/include/linux/sched.h -index ba49d7bd2b67..dd6b1d58af01 100755 ---- b/include/linux/sched.h -+++ a/include/linux/sched.h -@@ -73,7 +73,9 @@ struct task_delay_info; - struct task_group; - struct user_event_mm; - --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - - /* - * Task state bitmask. NOTE! These bits are also -@@ -298,11 +300,6 @@ enum { - - extern void scheduler_tick(void); - --#ifdef CONFIG_SLIM_SCHED --extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer); --extern void stop_shadow_tick_timer(void); --#endif -- - #define MAX_SCHEDULE_TIMEOUT LONG_MAX - - extern long schedule_timeout(long timeout); -@@ -1933,6 +1925,91 @@ static inline int task_nice(const struct task_struct *p) - return PRIO_TO_NICE((p)->static_prio); - } - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline bool task_is_top_task(struct task_struct *p) -+{ -+ return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ ->top_task_prop & TOP_TASK_BITS_MASK); -+} -+ -+static inline int get_top_task_prop(struct task_struct *p) -+{ -+ return get_hmbird_ts(p)->top_task_prop; -+} -+ -+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear) -+{ -+ if (set) -+ get_hmbird_ts(p)->top_task_prop |= set; -+ if (clear) -+ get_hmbird_ts(p)->top_task_prop &= ~clear; -+ return 0; -+} -+ -+static inline void reset_top_task_prop(struct task_struct *p) -+{ -+ get_hmbird_ts(p)->top_task_prop = 0; -+} -+ -+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ entity->sched_prop = sp; -+ } -+ return 0; -+} -+ -+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->sched_prop; -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq) -+{ -+ unsigned long new_dsq; -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq; -+ entity->sched_prop = new_dsq; -+ } -+} -+ -+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK); -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ entity->dsq_sync_ux = val; -+} -+ -+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->dsq_sync_ux; -+ else -+ return 0; -+} -+#endif - - extern int can_nice(const struct task_struct *p, const int nice); - extern int task_curr(const struct task_struct *p); -diff --git b/include/linux/sched/ext.h a/include/linux/sched/ext.h -deleted file mode 100755 -index b737a00c1373..000000000000 ---- b/include/linux/sched/ext.h -+++ /dev/null -@@ -1,647 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef _LINUX_SCHED_EXT_H --#define _LINUX_SCHED_EXT_H -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --#include -- --enum scx_consts { -- SCX_OPS_NAME_LEN = 128, -- SCX_EXIT_REASON_LEN = 128, -- SCX_EXIT_BT_LEN = 64, -- SCX_EXIT_MSG_LEN = 1024, -- -- SCX_SLICE_DFL = 20 * NSEC_PER_MSEC, -- SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */ --}; -- --/* -- * DSQ (dispatch queue) IDs are 64bit of the format: -- * -- * Bits: [63] [62 .. 0] -- * [ B] [ ID ] -- * -- * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -- * ID: 63 bit ID -- * -- * Built-in IDs: -- * -- * Bits: [63] [62] [61..32] [31 .. 0] -- * [ 1] [ L] [ R ] [ V ] -- * -- * 1: 1 for built-in DSQs. -- * L: 1 for LOCAL_ON DSQ IDs, 0 for others -- * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -- */ --enum scx_dsq_id_flags { -- SCX_DSQ_FLAG_BUILTIN = 1LLU << 63, -- SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -- -- SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0, -- SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, -- SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, -- SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, -- SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, --}; -- --enum scx_exit_type { -- SCX_EXIT_NONE, -- SCX_EXIT_DONE, -- -- SCX_EXIT_UNREG = 64, /* BPF unregistration */ -- SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -- SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ -- SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ --}; -- --/* -- * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is -- * being disabled. -- */ --struct scx_exit_info { -- /* %SCX_EXIT_* - broad category of the exit reason */ -- enum scx_exit_type type; -- /* textual representation of the above */ -- char reason[SCX_EXIT_REASON_LEN]; -- /* number of entries in the backtrace */ -- u32 bt_len; -- /* backtrace if exiting due to an error */ -- unsigned long bt[SCX_EXIT_BT_LEN]; -- /* extra message */ -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --/* sched_ext_ops.flags */ --enum scx_ops_flags { -- /* -- * Keep built-in idle tracking even if ops.update_idle() is implemented. -- */ -- SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, -- -- /* -- * By default, if there are no other task to run on the CPU, ext core -- * keeps running the current task even after its slice expires. If this -- * flag is specified, such tasks are passed to ops.enqueue() with -- * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. -- */ -- SCX_OPS_ENQ_LAST = 1LLU << 1, -- -- /* -- * An exiting task may schedule after PF_EXITING is set. In such cases, -- * bpf_task_from_pid() may not be able to find the task and if the BPF -- * scheduler depends on pid lookup for dispatching, the task will be -- * lost leading to various issues including RCU grace period stalls. -- * -- * To mask this problem, by default, unhashed tasks are automatically -- * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't -- * depend on pid lookups and wants to handle these tasks directly, the -- * following flag can be used. -- */ -- SCX_OPS_ENQ_EXITING = 1LLU << 2, -- -- /* -- * CPU cgroup knob enable flags -- */ -- SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16, /* cpu.weight */ -- -- SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | -- SCX_OPS_ENQ_LAST | -- SCX_OPS_ENQ_EXITING | -- SCX_OPS_CGROUP_KNOB_WEIGHT, --}; -- --/* argument container for ops.enable() and friends */ --struct scx_enable_args { --}; -- --/* argument container for ops->cgroup_init() */ --struct scx_cgroup_init_args { -- /* the weight of the cgroup [1..10000] */ -- u32 weight; --}; -- --enum scx_cpu_preempt_reason { -- /* next task is being scheduled by &sched_class_rt */ -- SCX_CPU_PREEMPT_RT, -- /* next task is being scheduled by &sched_class_dl */ -- SCX_CPU_PREEMPT_DL, -- /* next task is being scheduled by &sched_class_stop */ -- SCX_CPU_PREEMPT_STOP, -- /* unknown reason for SCX being preempted */ -- SCX_CPU_PREEMPT_UNKNOWN, --}; -- --/* -- * Argument container for ops->cpu_acquire(). Currently empty, but may be -- * expanded in the future. -- */ --struct scx_cpu_acquire_args {}; -- --/* argument container for ops->cpu_release() */ --struct scx_cpu_release_args { -- /* the reason the CPU was preempted */ -- enum scx_cpu_preempt_reason reason; -- -- /* the task that's going to be scheduled on the CPU */ -- struct task_struct *task; --}; -- --/** -- * struct sched_ext_ops - Operation table for BPF scheduler implementation -- * -- * Userland can implement an arbitrary scheduling policy by implementing and -- * loading operations in this table. -- */ --struct sched_ext_ops { -- /** -- * select_cpu - Pick the target CPU for a task which is being woken up -- * @p: task being woken up -- * @prev_cpu: the cpu @p was on before sleeping -- * @wake_flags: SCX_WAKE_* -- * -- * Decision made here isn't final. @p may be moved to any CPU while it -- * is getting dispatched for execution later. However, as @p is not on -- * the rq at this point, getting the eventual execution CPU right here -- * saves a small bit of overhead down the line. -- * -- * If an idle CPU is returned, the CPU is kicked and will try to -- * dispatch. While an explicit custom mechanism can be added, -- * select_cpu() serves as the default way to wake up idle CPUs. -- */ -- s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); -- -- /** -- * enqueue - Enqueue a task on the BPF scheduler -- * @p: task being enqueued -- * @enq_flags: %SCX_ENQ_* -- * -- * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() -- * or enqueue on the BPF scheduler. If not directly dispatched, the bpf -- * scheduler owns @p and if it fails to dispatch @p, the task will -- * stall. -- */ -- void (*enqueue)(struct task_struct *p, u64 enq_flags); -- -- /** -- * dequeue - Remove a task from the BPF scheduler -- * @p: task being dequeued -- * @deq_flags: %SCX_DEQ_* -- * -- * Remove @p from the BPF scheduler. This is usually called to isolate -- * the task while updating its scheduling properties (e.g. priority). -- * -- * The ext core keeps track of whether the BPF side owns a given task or -- * not and can gracefully ignore spurious dispatches from BPF side, -- * which makes it safe to not implement this method. However, depending -- * on the scheduling logic, this can lead to confusing behaviors - e.g. -- * scheduling position not being updated across a priority change. -- */ -- void (*dequeue)(struct task_struct *p, u64 deq_flags); -- -- /** -- * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs -- * @cpu: CPU to dispatch tasks for -- * @prev: previous task being switched out -- * -- * Called when a CPU's local dsq is empty. The operation should dispatch -- * one or more tasks from the BPF scheduler into the DSQs using -- * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using -- * scx_bpf_consume(). -- * -- * The maximum number of times scx_bpf_dispatch() can be called without -- * an intervening scx_bpf_consume() is specified by -- * ops.dispatch_max_batch. See the comments on top of the two functions -- * for more details. -- * -- * When not %NULL, @prev is an SCX task with its slice depleted. If -- * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in -- * @prev->scx.flags, it is not enqueued yet and will be enqueued after -- * ops.dispatch() returns. To keep executing @prev, return without -- * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. -- */ -- void (*dispatch)(s32 cpu, struct task_struct *prev); -- -- /** -- * runnable - A task is becoming runnable on its associated CPU -- * @p: task becoming runnable -- * @enq_flags: %SCX_ENQ_* -- * -- * This and the following three functions can be used to track a task's -- * execution state transitions. A task becomes ->runnable() on a CPU, -- * and then goes through one or more ->running() and ->stopping() pairs -- * as it runs on the CPU, and eventually becomes ->quiescent() when it's -- * done running on the CPU. -- * -- * @p is becoming runnable on the CPU because it's -- * -- * - waking up (%SCX_ENQ_WAKEUP) -- * - being moved from another CPU -- * - being restored after temporarily taken off the queue for an -- * attribute change. -- * -- * This and ->enqueue() are related but not coupled. This operation -- * notifies @p's state transition and may not be followed by ->enqueue() -- * e.g. when @p is being dispatched to a remote CPU. Likewise, a task -- * may be ->enqueue()'d without being preceded by this operation e.g. -- * after exhausting its slice. -- */ -- void (*runnable)(struct task_struct *p, u64 enq_flags); -- -- /** -- * running - A task is starting to run on its associated CPU -- * @p: task starting to run -- * -- * See ->runnable() for explanation on the task state notifiers. -- */ -- void (*running)(struct task_struct *p); -- -- /** -- * stopping - A task is stopping execution -- * @p: task stopping to run -- * @runnable: is task @p still runnable? -- * -- * See ->runnable() for explanation on the task state notifiers. If -- * !@runnable, ->quiescent() will be invoked after this operation -- * returns. -- */ -- void (*stopping)(struct task_struct *p, bool runnable); -- -- /** -- * quiescent - A task is becoming not runnable on its associated CPU -- * @p: task becoming not runnable -- * @deq_flags: %SCX_DEQ_* -- * -- * See ->runnable() for explanation on the task state notifiers. -- * -- * @p is becoming quiescent on the CPU because it's -- * -- * - sleeping (%SCX_DEQ_SLEEP) -- * - being moved to another CPU -- * - being temporarily taken off the queue for an attribute change -- * (%SCX_DEQ_SAVE) -- * -- * This and ->dequeue() are related but not coupled. This operation -- * notifies @p's state transition and may not be preceded by ->dequeue() -- * e.g. when @p is being dispatched to a remote CPU. -- */ -- void (*quiescent)(struct task_struct *p, u64 deq_flags); -- -- /** -- * yield - Yield CPU -- * @from: yielding task -- * @to: optional yield target task -- * -- * If @to is NULL, @from is yielding the CPU to other runnable tasks. -- * The BPF scheduler should ensure that other available tasks are -- * dispatched before the yielding task. Return value is ignored in this -- * case. -- * -- * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf -- * scheduler can implement the request, return %true; otherwise, %false. -- */ -- bool (*yield)(struct task_struct *from, struct task_struct *to); -- -- /** -- * core_sched_before - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Used by core-sched to determine the ordering between two tasks. See -- * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on -- * core-sched. -- * -- * Both @a and @b are runnable and may or may not currently be queued on -- * the BPF scheduler. Should return %true if @a should run before @b. -- * %false if there's no required ordering or @b should run before @a. -- * -- * If not specified, the default is ordering them according to when they -- * became runnable. -- */ -- bool (*core_sched_before)(struct task_struct *a,struct task_struct *b); -- -- /** -- * set_weight - Set task weight -- * @p: task to set weight for -- * @weight: new eight [1..10000] -- * -- * Update @p's weight to @weight. -- */ -- void (*set_weight)(struct task_struct *p, u32 weight); -- -- /** -- * set_cpumask - Set CPU affinity -- * @p: task to set CPU affinity for -- * @cpumask: cpumask of cpus that @p can run on -- * -- * Update @p's CPU affinity to @cpumask. -- */ -- void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask); -- -- /** -- * update_idle - Update the idle state of a CPU -- * @cpu: CPU to udpate the idle state for -- * @idle: whether entering or exiting the idle state -- * -- * This operation is called when @rq's CPU goes or leaves the idle -- * state. By default, implementing this operation disables the built-in -- * idle CPU tracking and the following helpers become unavailable: -- * -- * - scx_bpf_select_cpu_dfl() -- * - scx_bpf_test_and_clear_cpu_idle() -- * - scx_bpf_pick_idle_cpu() -- * - scx_bpf_any_idle_cpu() -- * -- * The user also must implement ops.select_cpu() as the default -- * implementation relies on scx_bpf_select_cpu_dfl(). -- * -- * If you keep the built-in idle tracking, specify the -- * %SCX_OPS_KEEP_BUILTIN_IDLE flag. -- */ -- void (*update_idle)(s32 cpu, bool idle); -- -- /** -- * cpu_acquire - A CPU is becoming available to the BPF scheduler -- * @cpu: The CPU being acquired by the BPF scheduler. -- * @args: Acquire arguments, see the struct definition. -- * -- * A CPU that was previously released from the BPF scheduler is now once -- * again under its control. -- */ -- void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); -- -- /** -- * cpu_release - A CPU is taken away from the BPF scheduler -- * @cpu: The CPU being released by the BPF scheduler. -- * @args: Release arguments, see the struct definition. -- * -- * The specified CPU is no longer under the control of the BPF -- * scheduler. This could be because it was preempted by a higher -- * priority sched_class, though there may be other reasons as well. The -- * caller should consult @args->reason to determine the cause. -- */ -- void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -- -- /** -- * cpu_online - A CPU became online -- * @cpu: CPU which just came up -- * -- * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs beforehand. -- */ -- void (*cpu_online)(s32 cpu); -- -- /** -- * cpu_offline - A CPU is going offline -- * @cpu: CPU which is going offline -- * -- * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs afterwards. -- */ -- void (*cpu_offline)(s32 cpu); -- -- /** -- * prep_enable - Prepare to enable BPF scheduling for a task -- * @p: task to prepare BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Either we're loading a BPF scheduler or a new task is being forked. -- * Prepare BPF scheduling for @p. This operation may block and can be -- * used for allocations. -- * -- * Return 0 for success, -errno for failure. An error return while -- * loading will abort loading of the BPF scheduler. During a fork, will -- * abort the specific fork. -- */ -- s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * enable - Enable BPF scheduling for a task -- * @p: task to enable BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Enable @p for BPF scheduling. @p is now in the cgroup specified for -- * the preceding prep_enable() and will start running soon. -- */ -- void (*enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * cancel_enable - Cancel prep_enable() -- * @p: task being canceled -- * @args: enable arguments, see the struct definition -- * -- * @p was prep_enable()'d but failed before reaching enable(). Undo the -- * preparation. -- */ -- void (*cancel_enable)(struct task_struct *p, -- struct scx_enable_args *args); -- -- /** -- * disable - Disable BPF scheduling for a task -- * @p: task to disable BPF scheduling for -- * -- * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. -- * Disable BPF scheduling for @p. -- */ -- void (*disable)(struct task_struct *p); -- -- /* -- * All online ops must come before ops.init(). -- */ -- -- /** -- * init - Initialize the BPF scheduler -- */ -- s32 (*init)(void); -- -- /** -- * exit - Clean up after the BPF scheduler -- * @info: Exit info -- */ -- void (*exit)(struct scx_exit_info *info); -- -- /** -- * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch -- */ -- u32 dispatch_max_batch; -- -- /** -- * flags - %SCX_OPS_* flags -- */ -- u64 flags; -- -- /** -- * timeout_ms - The maximum amount of time, in milliseconds, that a -- * runnable task should be able to wait before being scheduled. The -- * maximum timeout may not exceed the default timeout of 30 seconds. -- * -- * Defaults to the maximum allowed timeout value of 30 seconds. -- */ -- u32 timeout_ms; -- -- /** -- * name - BPF scheduler's name -- * -- * Must be a non-zero valid BPF object name including only isalnum(), -- * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the -- * BPF scheduler is enabled. -- */ -- char name[SCX_OPS_NAME_LEN]; --}; -- --/* -- * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -- * scheduler core and the BPF scheduler. See the documentation for more details. -- */ --struct scx_dispatch_q { -- raw_spinlock_t lock; -- struct list_head fifo; /* processed in dispatching order */ -- struct rb_root_cached priq; /* processed in p->scx.dsq_vtime order */ -- u32 nr; -- u64 id; -- struct llist_node free_node; -- struct rcu_head rcu; --}; -- --/* scx_entity.flags */ --enum scx_ent_flags { -- SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ -- SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -- SCX_TASK_ENQ_LOCAL = 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */ -- -- SCX_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -- SCX_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -- -- SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -- SCX_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -- -- SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ --}; -- --/* scx_entity.dsq_flags */ --enum scx_ent_dsq_flags { -- SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ --}; -- --/* -- * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from -- * everywhere and the following bits track which kfunc sets are currently -- * allowed for %current. This simple per-task tracking works because SCX ops -- * nest in a limited way. BPF will likely implement a way to allow and disallow -- * kfuncs depending on the calling context which will replace this manual -- * mechanism. See scx_kf_allow(). -- */ --enum scx_kf_mask { -- SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ -- /* all non-sleepables may be nested inside INIT and SLEEPABLE */ -- SCX_KF_INIT = 1 << 0, /* running ops.init() */ -- SCX_KF_SLEEPABLE = 1 << 1, /* other sleepable init operations */ -- /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ -- SCX_KF_CPU_RELEASE = 1 << 2, /* ops.cpu_release() */ -- /* ops.dequeue (in REST) may be nested inside DISPATCH */ -- SCX_KF_DISPATCH = 1 << 3, /* ops.dispatch() */ -- SCX_KF_ENQUEUE = 1 << 4, /* ops.enqueue() */ -- SCX_KF_REST = 1 << 5, /* other rq-locked operations */ -- -- __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | -- SCX_KF_ENQUEUE | SCX_KF_REST, -- __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_REST, --}; -- --#define RAVG_HIST_SIZE 5 --struct scx_sched_task_stats { -- u64 mark_start; -- u64 window_start; -- u32 sum; -- u32 sum_history[RAVG_HIST_SIZE]; -- int cidx; -- -- u32 demand; -- u16 demand_scaled; -- void *sdsq; --}; -- -- -- --/* -- * The following is embedded in task_struct and contains all fields necessary -- * for a task to be scheduled by SCX. -- */ --struct sched_ext_entity { -- struct scx_dispatch_q *dsq; -- struct { -- struct list_head fifo; /* dispatch order */ -- struct rb_node priq; /* p->scx.dsq_vtime order */ -- } dsq_node; -- struct list_head watchdog_node; -- u32 flags; /* protected by rq lock */ -- u32 dsq_flags; /* protected by dsq lock */ -- u32 weight; -- s32 sticky_cpu; -- s32 holding_cpu; -- u32 kf_mask; /* see scx_kf_mask above */ -- struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ -- atomic64_t ops_state; -- unsigned long runnable_at; --#ifdef CONFIG_SCHED_CORE -- u64 core_sched_at; /* see scx_prio_less() */ --#endif -- -- /* BPF scheduler modifiable fields */ -- -- /* -- * Runtime budget in nsecs. This is usually set through -- * scx_bpf_dispatch() but can also be modified directly by the BPF -- * scheduler. Automatically decreased by SCX as the task executes. On -- * depletion, a scheduling event is triggered. -- * -- * This value is cleared to zero if the task is preempted by -- * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the -- * task ran. Use p->se.sum_exec_runtime instead. -- */ -- u64 slice; -- -- /* -- * Used to order tasks when dispatching to the vtime-ordered priority -- * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime() -- * but can also be modified directly by the BPF scheduler. Modifying it -- * while a task is queued on a dsq may mangle the ordering and is not -- * recommended. -- */ -- u64 dsq_vtime; -- -- /* -- * If set, reject future sched_setscheduler(2) calls updating the policy -- * to %SCHED_EXT with -%EACCES. -- * -- * If set from ops.prep_enable() and the task's policy is already -- * %SCHED_EXT, which can happen while the BPF scheduler is being loaded -- * or by inhering the parent's policy during fork, the task's policy is -- * rejected and forcefully reverted to %SCHED_NORMAL. The number of such -- * events are reported through /sys/kernel/debug/sched_ext::nr_rejected. -- */ -- bool disallow; /* reject switching into SCX */ -- -- /* cold fields */ -- struct list_head tasks_node; -- struct task_struct *task; -- struct scx_sched_task_stats sts; --}; -- --void sched_ext_free(struct task_struct *p); -- --#else /* !CONFIG_SCHED_CLASS_EXT */ -- --static inline void sched_ext_free(struct task_struct *p) {} -- --#endif /* CONFIG_SCHED_CLASS_EXT */ --#endif /* _LINUX_SCHED_EXT_H */ -diff --git b/include/linux/sched/hmbird_proc_val.h a/include/linux/sched/hmbird_proc_val.h -new file mode 100755 -index 000000000000..e59708b16ea3 ---- /dev/null -+++ a/include/linux/sched/hmbird_proc_val.h -@@ -0,0 +1,22 @@ -+#ifndef __HMBORD_PROC_VAL_H__ -+#define __HMBORD_PROC_VAL_H__ -+ -+extern int scx_enable; -+extern int partial_enable; -+extern int cpuctrl_high_ratio; -+extern int cpuctrl_low_ratio; -+extern int slim_stats; -+extern int hmbirdcore_debug; -+extern int slim_for_app; -+extern int misfit_ds; -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+extern int cpu7_tl; -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int slim_gov_debug; -+extern int scx_gov_ctrl; -+extern int sched_ravg_window_frame_per_sec; -+ -+#endif -\ No newline at end of file -diff --git b/include/linux/sched/hmbird_version.h a/include/linux/sched/hmbird_version.h -new file mode 100755 -index 000000000000..b2843881c26f ---- /dev/null -+++ a/include/linux/sched/hmbird_version.h -@@ -0,0 +1,52 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#ifndef _OPLUS_HMBIRD_VERSION_H_ -+#define _OPLUS_HMBIRD_VERSION_H_ -+#include -+#include -+#include -+ -+enum hmbird_version { -+ HMBIRD_UNINIT, -+ HMBIRD_GKI_VERSION, -+ HMBIRD_OGKI_VERSION, -+ HMBIRD_UNKNOW_VERSION, -+}; -+ -+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT; -+ -+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type" -+ -+static inline enum hmbird_version get_hmbird_version_type(void) -+{ -+ struct device_node *np = NULL; -+ const char *hmbird_version_str = NULL; -+ if (HMBIRD_UNINIT != hmbird_version_type) -+ return hmbird_version_type; -+ np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH); -+ if (np) { -+ of_property_read_string(np, "type", &hmbird_version_str); -+ if (NULL != hmbird_version_str) { -+ if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) { -+ hmbird_version_type = HMBIRD_OGKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi"); -+ } else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) { -+ hmbird_version_type = HMBIRD_GKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi"); -+ } else { -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi"); -+ } -+ return hmbird_version_type; -+ } -+ } -+ -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION"); -+ return hmbird_version_type; -+} -+ -+#endif /*_OPLUS_HMBIRD_VERSION_H_ */ -diff --git b/include/linux/sched/task.h a/include/linux/sched/task.h -index 03d35e3eda3c..6a5f0c0d74bd 100644 ---- b/include/linux/sched/task.h -+++ a/include/linux/sched/task.h -@@ -61,8 +61,10 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); - extern void init_idle(struct task_struct *idle, int cpu); - - extern int sched_fork(unsigned long clone_flags, struct task_struct *p); --extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); --extern void sched_cancel_fork(struct task_struct *p); -+extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); -+#ifdef CONFIG_HMBIRD_SCHED -+extern void hmbird_cancel_fork(struct task_struct *p); -+#endif - extern void sched_post_fork(struct task_struct *p); - extern void sched_dead(struct task_struct *p); - -diff --git b/include/uapi/linux/sched.h a/include/uapi/linux/sched.h -index 359a14cc76a4..969716ab4320 100755 ---- b/include/uapi/linux/sched.h -+++ a/include/uapi/linux/sched.h -@@ -118,7 +118,7 @@ struct clone_args { - /* SCHED_ISO: reserved but not implemented yet */ - #define SCHED_IDLE 5 - #define SCHED_DEADLINE 6 --#define SCHED_EXT 7 -+#define SCHED_HMBIRD 7 - - /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ - #define SCHED_RESET_ON_FORK 0x40000000 -diff --git b/init/Kconfig a/init/Kconfig -index 337276cbc7f8..bf052ca532b4 100644 ---- b/init/Kconfig -+++ a/init/Kconfig -@@ -1030,10 +1030,6 @@ config RT_GROUP_SCHED - realtime bandwidth for them. - See Documentation/scheduler/sched-rt-group.rst for more information. - --config EXT_GROUP_SCHED -- bool -- depends on SCHED_CLASS_EXT && CGROUP_SCHED -- default y - endif #CGROUP_SCHED - - config SCHED_MM_CID -diff --git b/init/init_task.c a/init/init_task.c -index 69a046388995..31ceb0e469f7 100755 ---- b/init/init_task.c -+++ a/init/init_task.c -@@ -6,7 +6,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -102,9 +101,6 @@ struct task_struct init_task - #endif - #ifdef CONFIG_CGROUP_SCHED - .sched_task_group = &root_task_group, --#endif --#ifdef CONFIG_SLIM_SCHED -- .scx = NULL, - #endif - .ptraced = LIST_HEAD_INIT(init_task.ptraced), - .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), -diff --git b/kernel/Kconfig.preempt a/kernel/Kconfig.preempt -index cf22ef6107b8..ca8de6eb1e16 100755 ---- b/kernel/Kconfig.preempt -+++ a/kernel/Kconfig.preempt -@@ -133,12 +133,8 @@ config SCHED_CORE - which is the likely usage by Linux distributions, there should - be no measurable impact on performance. - --config SLIM_SCHED -- bool "slim sched" -- default n --config SCHED_CLASS_EXT -- bool "Extensible Scheduling Class" -- depends on BPF_SYSCALL && BPF_JIT -+config HMBIRD_SCHED -+ bool "hmbird Scheduling Class(base on ext scheduler)" - help - This option enables a new scheduler class sched_ext (SCX), which - allows scheduling policies to be implemented as BPF programs to -diff --git b/kernel/bpf/bpf_struct_ops_types.h a/kernel/bpf/bpf_struct_ops_types.h -index 3618769d853d..5678a9ddf817 100644 ---- b/kernel/bpf/bpf_struct_ops_types.h -+++ a/kernel/bpf/bpf_struct_ops_types.h -@@ -9,8 +9,4 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) - #include - BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) - #endif --#ifdef CONFIG_SCHED_CLASS_EXT --#include --BPF_STRUCT_OPS_TYPE(sched_ext_ops) --#endif - #endif -diff --git b/kernel/fork.c a/kernel/fork.c -index a43f97302332..7a91505b9378 100755 ---- b/kernel/fork.c -+++ a/kernel/fork.c -@@ -23,7 +23,9 @@ - #include - #include - #include --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - #include - #include - #include -@@ -999,8 +1001,9 @@ void __put_task_struct(struct task_struct *tsk) - WARN_ON(!tsk->exit_state); - WARN_ON(refcount_read(&tsk->usage)); - WARN_ON(tsk == current); -- -- sched_ext_free(tsk); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_free(tsk); -+#endif - io_uring_free(tsk); - cgroup_free(tsk); - task_numa_free(tsk, true); -@@ -2517,7 +2520,11 @@ __latent_entropy struct task_struct *copy_process( - - retval = perf_event_init_task(p, clone_flags); - if (retval) -- goto bad_fork_sched_cancel_fork; -+#ifdef CONFIG_HMBIRD_SCHED -+ goto bad_fork_hmbird_cancel_fork; -+#else -+ goto bad_fork_cleanup_policy; -+#endif - retval = audit_alloc(p); - if (retval) - goto bad_fork_cleanup_perf; -@@ -2649,9 +2656,7 @@ __latent_entropy struct task_struct *copy_process( - * cgroup specific, it unconditionally needs to place the task on a - * runqueue. - */ -- retval = sched_cgroup_fork(p, args); -- if (retval) -- goto bad_fork_cancel_cgroup; -+ sched_cgroup_fork(p, args); - - /* - * From this point on we must avoid any synchronous user-space -@@ -2697,13 +2702,13 @@ __latent_entropy struct task_struct *copy_process( - /* Don't start children in a dying pid namespace */ - if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { - retval = -ENOMEM; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* Let kill terminate clone/fork in the middle */ - if (fatal_signal_pending(current)) { - retval = -EINTR; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* No more failure paths after this point. */ -@@ -2780,11 +2785,10 @@ __latent_entropy struct task_struct *copy_process( - - return p; - --bad_fork_core_free: -+bad_fork_cancel_cgroup: - sched_core_free(p); - spin_unlock(¤t->sighand->siglock); - write_unlock_irq(&tasklist_lock); --bad_fork_cancel_cgroup: - cgroup_cancel_fork(p, args); - bad_fork_put_pidfd: - if (clone_flags & CLONE_PIDFD) { -@@ -2823,8 +2827,10 @@ __latent_entropy struct task_struct *copy_process( - audit_free(p); - bad_fork_cleanup_perf: - perf_event_free_task(p); --bad_fork_sched_cancel_fork: -- sched_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+bad_fork_hmbird_cancel_fork: -+ hmbird_cancel_fork(p); -+#endif - bad_fork_cleanup_policy: - lockdep_free_task(p); - #ifdef CONFIG_NUMA -diff --git b/kernel/sched/build_policy.c a/kernel/sched/build_policy.c -index 005025f55bea..c091539a0f60 100755 ---- b/kernel/sched/build_policy.c -+++ a/kernel/sched/build_policy.c -@@ -28,8 +28,10 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED - #include - #include -+#endif - - #include - -@@ -54,6 +56,10 @@ - #include "cputime.c" - #include "deadline.c" - --#ifdef CONFIG_SCHED_CLASS_EXT --# include "ext.c" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/hmbird_util_track.c" -+#include "hmbird/hmbird_sched_proc.c" -+#include "hmbird/hmbird_shadow_tick.c" -+#include "hmbird/hmbird.c" -+#include "hmbird/hmbird_misc.c" - #endif -diff --git b/kernel/sched/core.c a/kernel/sched/core.c -index 25d5703df992..b894417ce7df 100755 ---- b/kernel/sched/core.c -+++ a/kernel/sched/core.c -@@ -96,6 +96,12 @@ - #include "../../io_uring/io-wq.h" - #include "../smpboot.h" - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/slim.h" -+#include "hmbird/hmbird_shadow_tick.h" -+#include "hmbird.h" -+#endif -+ - #include - #include - #include -@@ -170,7 +176,6 @@ __read_mostly int scheduler_running; - - DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); - -- - /* kernel prio, less is more */ - static inline int __task_prio(const struct task_struct *p) - { -@@ -183,11 +188,10 @@ static inline int __task_prio(const struct task_struct *p) - if (p->sched_class == &idle_sched_class) - return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - --#ifdef CONFIG_SCHED_CLASS_EXT -- if (p->sched_class == &ext_sched_class) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (p->sched_class == &hmbird_sched_class) - return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ - #endif -- - return MAX_RT_PRIO + MAX_NICE; /* 119, squash fair */ - } - -@@ -217,9 +221,9 @@ static inline bool prio_less(const struct task_struct *a, - if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ - return cfs_prio_less(a, b, in_fi); - --#ifdef CONFIG_SCHED_CLASS_EXT -+#ifdef CONFIG_HMBIRD_SCHED - if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ -- return scx_prio_less(a, b, in_fi); -+ return hmbird_prio_less(a, b, in_fi); - #endif - - return false; -@@ -1279,17 +1283,26 @@ bool sched_can_stop_tick(struct rq *rq) - if (fifo_nr_running) - return true; - -+#ifdef CONFIG_HMBIRD_SCHED - /* -- * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks -+ * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks - * left. For CFS, if there's more than one we need the tick for -- * involuntary preemption. For SCX, ask. -+ * involuntary preemption. For HMBIRD, ask. - */ -- if (!scx_switched_all() && rq->nr_running > 1) -+ if (!hmbird_enabled() && rq->nr_running > 1) - return false; - -- if (scx_enabled() && !scx_can_stop_tick(rq)) -+ if (hmbird_enabled() && !hmbird_can_stop_tick(rq)) - return false; -- -+#else -+ /* -+ * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; -+ * if there's more than one we need the tick for involuntary -+ * preemption. -+ */ -+ if (rq->nr_running > 1) -+ return false; -+#endif - /* - * If there is one task and it has CFS runtime bandwidth constraints - * and it's on the cpu now we don't want to stop the tick. -@@ -2222,10 +2235,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) - } - EXPORT_SYMBOL_GPL(deactivate_task); - --struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) -+#ifdef CONFIG_HMBIRD_SCHED -+struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - { -- struct sched_change_guard cg = { -+ struct hmbird_sched_change_guard cg = { - .rq = rq, - .p = p, - .queued = task_on_rq_queued(p), -@@ -2247,7 +2261,7 @@ sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - return cg; - } - --void sched_change_guard_fini(struct sched_change_guard *cg, int flags) -+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags) - { - if (cg->queued) - enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); -@@ -2255,6 +2269,7 @@ void sched_change_guard_fini(struct sched_change_guard *cg, int flags) - set_next_task(cg->rq, cg->p); - cg->done = true; - } -+#endif - - static inline int __normal_prio(int policy, int rt_prio, int nice) - { -@@ -2320,9 +2335,9 @@ inline int task_curr(const struct task_struct *p) - * this means any call to check_class_changed() must be followed by a call to - * balance_callback(). - */ --void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio) -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) - { - if (prev_class != p->sched_class) { - if (prev_class->switched_from) -@@ -2885,6 +2900,7 @@ static void - __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - { - struct rq *rq = task_rq(p); -+ bool queued, running; - - /* - * This here violates the locking rules for affinity, since we're only -@@ -2903,9 +2919,26 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - else - lockdep_assert_held(&p->pi_lock); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->sched_class->set_cpus_allowed(p, ctx); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ -+ if (queued) { -+ /* -+ * Because __kthread_bind() calls this on blocked tasks without -+ * holding rq->lock. -+ */ -+ lockdep_assert_rq_held(rq); -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); - } -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->sched_class->set_cpus_allowed(p, ctx); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - } - - /* -@@ -3772,7 +3805,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class)) -+#else - if (unlikely(!is_cpu_allowed(p, cpu))) -+#endif - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -@@ -4891,8 +4928,9 @@ late_initcall(sched_core_sysctl_init); - */ - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { -- -+#ifdef CONFIG_HMBIRD_SCHED - int ret; -+#endif - - trace_android_rvh_sched_fork(p); - -@@ -4933,21 +4971,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_reset_on_fork = 0; - } - -- scx_pre_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ ret = hmbird_pre_fork(p); -+ if (ret) -+ goto out_cancel; - - if (dl_prio(p->prio)) { - ret = -EAGAIN; - goto out_cancel; --#ifdef CONFIG_SCHED_CLASS_EXT -- } else if (task_on_scx(p)) { -- p->sched_class = &ext_sched_class; --#endif -+ } else if (task_on_hmbird(p)) { -+ p->sched_class = &hmbird_sched_class; - } else if (rt_prio(p->prio)) { - p->sched_class = &rt_sched_class; - } else { - p->sched_class = &fair_sched_class; - } -- -+#else -+ if (dl_prio(p->prio)) -+ return -EAGAIN; -+ else if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - init_entity_runnable_average(&p->se); - trace_android_rvh_finish_prio_fork(p); - -@@ -4966,12 +5012,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - #endif - return 0; - -+#ifdef CONFIG_HMBIRD_SCHED - out_cancel: -- scx_cancel_fork(p); -+ hmbird_cancel_fork(p); - return ret; -+#endif - } - --int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) -+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - { - unsigned long flags; - -@@ -4998,20 +5046,14 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -- -- return scx_fork(p); --} -- --void sched_cancel_fork(struct task_struct *p) --{ -- scx_cancel_fork(p); - } - - void sched_post_fork(struct task_struct *p) - { - uclamp_post_fork(p); -- -- scx_post_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_post_fork(p); -+#endif - } - - unsigned long to_ratio(u64 period, u64 runtime) -@@ -5875,21 +5917,28 @@ void scheduler_tick(void) - - if (sched_feat(LATENCY_WARN) && resched_latency) - resched_latency_warn(cpu, resched_latency); -- -- scx_notify_sched_tick(); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_notify_sched_tick(); -+#endif - perf_event_task_tick(); - - if (curr->flags & PF_WQ_WORKER) - wq_worker_tick(curr); - - #ifdef CONFIG_SMP -- if (!scx_switched_all()) { -+#ifdef CONFIG_HMBIRD_SCHED -+ if (!hmbird_enabled()) { -+#endif - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); -+#ifdef CONFIG_HMBIRD_SCHED - } -+#endif - #endif - trace_android_vh_scheduler_tick(rq); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ scheduler_tick_handler(NULL, NULL); -+#endif - } - - #ifdef CONFIG_NO_HZ_FULL -@@ -6191,7 +6240,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ -+#ifdef CONFIG_HMBIRD_SCHED - for_balance_class_range(class, prev->sched_class, &idle_sched_class) { -+#else -+ for_class_range(class, prev->sched_class, &idle_sched_class) { -+#endif - if (class->balance(rq, prev, rf)) - break; - } -@@ -6209,9 +6262,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - const struct sched_class *class; - struct task_struct *p; - -- if (scx_enabled()) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (hmbird_enabled()) - goto restart; -- -+#endif - /* - * Optimization: we know that if all tasks are in the fair class we can - * call that function directly, but only if the @prev task wasn't of a -@@ -6237,13 +6291,21 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - restart: - put_prev_task_balance(rq, prev, rf); - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { - p = class->pick_next_task(rq); - if (p) { -- scx_notify_pick_next_task(rq, p, class); -+ hmbird_notify_pick_next_task(rq, p, class); - return p; - } - } -+#else -+ for_each_class(class) { -+ p = class->pick_next_task(rq); -+ if (p) -+ return p; -+ } -+#endif - - BUG(); /* The idle class should always have a runnable task. */ - } -@@ -6272,7 +6334,11 @@ static inline struct task_struct *pick_task(struct rq *rq) - const struct sched_class *class; - struct task_struct *p; - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { -+#else -+ for_each_class(class) { -+#endif - p = class->pick_task(rq); - if (p) - return p; -@@ -6916,7 +6982,9 @@ static void __sched notrace __schedule(unsigned int sched_mode) - psi_sched_switch(prev, next, !task_on_rq_queued(prev)); - - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#endif - /* Also unlocks the rq: */ - rq = context_switch(rq, prev, next, &rf); - } else { -@@ -7244,24 +7312,31 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag - } - EXPORT_SYMBOL(default_wake_function); - --void __setscheduler_prio(struct task_struct *p, int prio) -+static void __setscheduler_prio(struct task_struct *p, int prio) - { -- /* -- * After switching all rt and fair class to ext, -- * stop class is switched to ext class too. Just -- * skip it. */ -+#ifdef CONFIG_HMBIRD_SCHED -+ bool on_hmbird = task_on_hmbird(p); -+ - if (p->sched_class == &stop_sched_class) -- ;/* do nothing */ -+ ; - else if (dl_prio(prio)) - p->sched_class = &dl_sched_class; --#ifdef CONFIG_SCHED_CLASS_EXT -- else if (task_on_scx(p)) -- p->sched_class = &ext_sched_class; --#endif -+ else if (rt_prio(prio) && on_hmbird) -+ p->sched_class = &hmbird_sched_class; - else if (rt_prio(prio)) - p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; - else - p->sched_class = &fair_sched_class; -+#else -+ if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - - p->prio = prio; - trace_android_rvh_setscheduler_prio(p); -@@ -7297,7 +7372,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - */ - void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - { -- int prio, oldprio, queue_flag = -+ int prio, oldprio, queued, running, queue_flag = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *prev_class; - struct rq_flags rf; -@@ -7360,42 +7435,52 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - queue_flag &= ~DEQUEUE_MOVE; - - prev_class = p->sched_class; -- SCHED_CHANGE_BLOCK(rq, p, queue_flag) { -- /* -- * Boosting condition are: -- * 1. -rt task is running and holds mutex A -- * --> -dl task blocks on mutex A -- * -- * 2. -dl task is running and holds mutex A -- * --> -dl task blocks on mutex A and could preempt the -- * running task -- */ -- if (dl_prio(prio)) { -- if (!dl_prio(p->normal_prio) || -- (pi_task && dl_prio(pi_task->prio) && -- dl_entity_preempt(&pi_task->dl, &p->dl))) { -- p->dl.pi_se = pi_task->dl.pi_se; -- queue_flag |= ENQUEUE_REPLENISH; -- } else { -- p->dl.pi_se = &p->dl; -- } -- } else if (rt_prio(prio)) { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- if (oldprio < prio) -- queue_flag |= ENQUEUE_HEAD; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flag); -+ if (running) -+ put_prev_task(rq, p); -+ -+ /* -+ * Boosting condition are: -+ * 1. -rt task is running and holds mutex A -+ * --> -dl task blocks on mutex A -+ * -+ * 2. -dl task is running and holds mutex A -+ * --> -dl task blocks on mutex A and could preempt the -+ * running task -+ */ -+ if (dl_prio(prio)) { -+ if (!dl_prio(p->normal_prio) || -+ (pi_task && dl_prio(pi_task->prio) && -+ dl_entity_preempt(&pi_task->dl, &p->dl))) { -+ p->dl.pi_se = pi_task->dl.pi_se; -+ queue_flag |= ENQUEUE_REPLENISH; - } else { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- else if (rt_prio(oldprio)) -- p->rt.timeout = 0; -- else if (!task_has_idle_policy(p)) -- reweight_task(p, prio - MAX_RT_PRIO); -+ p->dl.pi_se = &p->dl; - } -- -- __setscheduler_prio(p, prio); -+ } else if (rt_prio(prio)) { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ if (oldprio < prio) -+ queue_flag |= ENQUEUE_HEAD; -+ } else { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ else if (rt_prio(oldprio)) -+ p->rt.timeout = 0; -+ else if (!task_has_idle_policy(p)) -+ reweight_task(p, prio - MAX_RT_PRIO); - } - -+ __setscheduler_prio(p, prio); -+ -+ if (queued) -+ enqueue_task(rq, p, queue_flag); -+ if (running) -+ set_next_task(rq, p); -+ - check_class_changed(rq, p, prev_class, oldprio); - out_unlock: - /* Avoid rq from going away on us: */ -@@ -7416,7 +7501,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - - void set_user_nice(struct task_struct *p, long nice) - { -- bool allowed = false; -+ bool queued, running, allowed = false; - int old_prio; - struct rq_flags rf; - struct rq *rq; -@@ -7445,13 +7530,22 @@ void set_user_nice(struct task_struct *p, long nice) - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); -+ if (running) -+ put_prev_task(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->static_prio = NICE_TO_PRIO(nice); -- set_load_weight(p, true); -- old_prio = p->prio; -- p->prio = effective_prio(p); -- } -+ p->static_prio = NICE_TO_PRIO(nice); -+ set_load_weight(p, true); -+ old_prio = p->prio; -+ p->prio = effective_prio(p); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - - /* - * If the task increased its priority or is running and -@@ -7868,7 +7962,7 @@ static int __sched_setscheduler(struct task_struct *p, - bool user, bool pi) - { - int oldpolicy = -1, policy = attr->sched_policy; -- int retval, oldprio, newprio; -+ int retval, oldprio, newprio, queued, running; - const struct sched_class *prev_class; - struct balance_callback *head; - struct rq_flags rf; -@@ -7876,7 +7970,9 @@ static int __sched_setscheduler(struct task_struct *p, - int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct rq *rq; - bool cpuset_locked = false; -- -+#ifdef CONFIG_HMBIRD_SCHED -+ unsigned long flags; -+#endif - - /* The pi code expects interrupts enabled */ - BUG_ON(pi && in_interrupt()); -@@ -7942,7 +8038,9 @@ static int __sched_setscheduler(struct task_struct *p, - * To be able to change p->policy safely, the appropriate - * runqueue lock must be held. - */ -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+#endif - rq = task_rq_lock(p, &rf); - update_rq_clock(rq); - -@@ -7954,10 +8052,11 @@ static int __sched_setscheduler(struct task_struct *p, - goto unlock; - } - -- retval = scx_check_setscheduler(p, policy); -+#ifdef CONFIG_HMBIRD_SCHED -+ retval = hmbird_check_setscheduler(p, policy); - if (retval) - goto unlock; -- -+#endif - /* - * If not changing anything there's no need to proceed further, - * but store a possible modification of reset_on_fork. -@@ -8014,7 +8113,9 @@ static int __sched_setscheduler(struct task_struct *p, - if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { - policy = oldpolicy = -1; - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - goto recheck; -@@ -8047,16 +8148,23 @@ static int __sched_setscheduler(struct task_struct *p, - queue_flags &= ~DEQUEUE_MOVE; - } - -- SCHED_CHANGE_BLOCK(rq, p, queue_flags) { -- prev_class = p->sched_class; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flags); -+ if (running) -+ put_prev_task(rq, p); -+ -+ prev_class = p->sched_class; - -- if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -- __setscheduler_params(p, attr); -- __setscheduler_prio(p, newprio); -- trace_android_rvh_setscheduler(p); -- } -- __setscheduler_uclamp(p, attr); -+ if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -+ __setscheduler_params(p, attr); -+ __setscheduler_prio(p, newprio); -+ trace_android_rvh_setscheduler(p); -+ } -+ __setscheduler_uclamp(p, attr); - -+ if (queued) { - /* - * We enqueue to tail when the priority of a task is - * increased (user space view). -@@ -8064,7 +8172,10 @@ static int __sched_setscheduler(struct task_struct *p, - if (oldprio < p->prio) - queue_flags |= ENQUEUE_HEAD; - -+ enqueue_task(rq, p, queue_flags); - } -+ if (running) -+ set_next_task(rq, p); - - check_class_changed(rq, p, prev_class, oldprio); - -@@ -8072,7 +8183,9 @@ static int __sched_setscheduler(struct task_struct *p, - preempt_disable(); - head = splice_balance_callbacks(rq); - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (pi) { - if (cpuset_locked) - cpuset_unlock(); -@@ -8087,7 +8200,9 @@ static int __sched_setscheduler(struct task_struct *p, - - unlock: - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - return retval; -@@ -8785,6 +8900,9 @@ static void do_sched_yield(void) - long skip = 0; - - trace_android_rvh_before_do_sched_yield(&skip); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_skip_yield(&skip); -+#endif - if (skip) - return; - -@@ -9309,7 +9427,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - break; - } -@@ -9337,7 +9457,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - } - return ret; -@@ -9638,15 +9760,25 @@ int migrate_task_to(struct task_struct *p, int target_cpu) - */ - void sched_setnuma(struct task_struct *p, int nid) - { -+ bool queued, running; - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) { -- p->numa_preferred_nid = nid; -- } -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE); -+ if (running) -+ put_prev_task(rq, p); - -+ p->numa_preferred_nid = nid; -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - task_rq_unlock(rq, p, &rf); - } - #endif /* CONFIG_NUMA_BALANCING */ -@@ -10212,17 +10344,23 @@ void __init sched_init(void) - int i; - - /* Make sure the linker didn't screw up */ -+#ifdef CONFIG_HMBIRD_SCHED - #ifdef CONFIG_SMP -- BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+#endif -+ WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class)); -+#else -+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 || -+ &fair_sched_class != &rt_sched_class + 1 || -+ &rt_sched_class != &dl_sched_class + 1); -+#ifdef CONFIG_SMP -+ BUG_ON(&dl_sched_class != &stop_sched_class + 1); - #endif -- BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); -- BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); -- BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); --#ifdef CONFIG_SCHED_CLASS_EXT -- BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); -- BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); - #endif -- - wait_bit_init(); - - #ifdef CONFIG_FAIR_GROUP_SCHED -@@ -10392,8 +10530,9 @@ void __init sched_init(void) - balance_push_set(smp_processor_id(), false); - #endif - init_sched_fair_class(); -- init_sched_ext_class(); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ init_sched_hmbird_class(); -+#endif - psi_init(); - - init_uclamp(); -@@ -10863,6 +11002,11 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) - struct task_group *tg = css_tg(css); - struct task_group *parent = css_tg(css->parent); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_tg_online(tg); -+#endif -+ -+ - if (parent) - sched_online_group(tg, parent); - -@@ -10912,13 +11056,77 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) - } - #endif - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 dl) -+{ -+ int i; -+ -+ for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) { -+ if (dl < HMBIRD_BPF_DSQS_DEADLINE[i]) -+ break; -+ } -+ -+ i = max_t(int, i-1, MIN_CGROUP_DL_IDX); -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return 0; -+ update_cgroup_ids_table(css->cgroup->kn->id, i); -+ -+ return 0; -+} -+ -+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ u8 i; -+ -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX]; -+ i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1); -+ if (i < 0) { -+ pr_err(" <%s> i is %d, less than 0, name is %s\n", -+ __func__, i, css->cgroup->kn->name); -+ i = DEFAULT_CGROUP_DL_IDX; -+ } -+ -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[i]; -+} -+ -+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css, -+ struct task_struct *p, int prio) -+{ -+ if (!css || !rt_prio(prio)) -+ return; -+ -+ if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) { -+ if (!strcmp(css->cgroup->kn->name, "display")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3); -+ else if (!strcmp(css->cgroup->kn->name, "touch")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2); -+ else if (!strcmp(css->cgroup->kn->name, "multimedia")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+ } -+} -+#endif -+ - static void cpu_cgroup_attach(struct cgroup_taskset *tset) - { - struct task_struct *task; - struct cgroup_subsys_state *css; - - cgroup_taskset_for_each(task, css, tset) { -- -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_change_rt_sched_prop(css, task, task->prio); -+#endif - sched_move_task(task); - } - -@@ -11600,7 +11808,13 @@ static struct cftype cpu_legacy_files[] = { - .write_u64 = cpu_uclamp_ls_write_u64, - }, - #endif -- -+#ifdef CONFIG_HMBIRD_SCHED -+ { -+ .name = "hmbird.deadline", -+ .read_u64 = cgroup_read_hmbird_deadline, -+ .write_u64 = cgroup_write_hmbird_deadline, -+ }, -+#endif - { } /* Terminate */ - }; - -@@ -11649,7 +11863,6 @@ static int cpu_local_stat_show(struct seq_file *sf, - } - - #ifdef CONFIG_FAIR_GROUP_SCHED -- - static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, - struct cftype *cft) - { -diff --git b/kernel/sched/debug.c a/kernel/sched/debug.c -index a6c99df9edf9..e09904f4d6e5 100755 ---- b/kernel/sched/debug.c -+++ a/kernel/sched/debug.c -@@ -375,9 +375,8 @@ static __init int sched_init_debug(void) - #endif - - debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); -- --#ifdef CONFIG_SCHED_CLASS_EXT -- debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops); -+#ifdef CONFIG_HMBIRD_SCHED -+ debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops); - #endif - return 0; - } -@@ -1006,9 +1005,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - SEQ_printf(m, - "---------------------------------------------------------" - "----------\n"); --#ifdef CONFIG_SLIM_SCHED -- SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop); --#endif - - #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) - #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) -@@ -1104,8 +1100,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - } else if (fair_policy(p->policy)) { - P(se.slice); - } --#ifdef CONFIG_SCHED_CLASS_EXT -- __PS("ext.enabled", p->sched_class == &ext_sched_class); -+#ifdef CONFIG_HMBIRD_SCHED -+ __PS("hmbird.enabled", p->sched_class == &hmbird_sched_class); -+ __PS("sched_prop", get_hmbird_ts(p)->sched_prop); -+ __PS("top_task_prop", get_hmbird_ts(p)->top_task_prop); - #endif - #undef PN_SCHEDSTAT - #undef P_SCHEDSTAT -diff --git b/kernel/sched/ext.c a/kernel/sched/ext.c -deleted file mode 100755 -index 4845d441df2b..000000000000 ---- b/kernel/sched/ext.c -+++ /dev/null -@@ -1,3978 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -- --enum scx_internal_consts { -- SCX_NR_ONLINE_OPS = SCX_OP_IDX(init), -- SCX_DSP_DFL_MAX_BATCH = 32, -- SCX_DSP_MAX_LOOPS = 32, -- SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, --}; -- --enum scx_ops_enable_state { -- SCX_OPS_PREPPING, -- SCX_OPS_ENABLING, -- SCX_OPS_ENABLED, -- SCX_OPS_DISABLING, -- SCX_OPS_DISABLED, --}; -- --static inline struct task_group *css_tg(struct cgroup_subsys_state *css) --{ -- return css ? container_of(css, struct task_group, css) : NULL; --} -- --/* -- * sched_ext_entity->ops_state -- * -- * Used to track the task ownership between the SCX core and the BPF scheduler. -- * State transitions look as follows: -- * -- * NONE -> QUEUEING -> QUEUED -> DISPATCHING -- * ^ | | -- * | v v -- * \-------------------------------/ -- * -- * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -- * sites for explanations on the conditions being waited upon and why they are -- * safe. Transitions out of them into NONE or QUEUED must store_release and the -- * waiters should load_acquire. -- * -- * Tracking scx_ops_state enables sched_ext core to reliably determine whether -- * any given task can be dispatched by the BPF scheduler at all times and thus -- * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -- * to try to dispatch any task anytime regardless of its state as the SCX core -- * can safely reject invalid dispatches. -- */ --enum scx_ops_state { -- SCX_OPSS_NONE, /* owned by the SCX core */ -- SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -- SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ -- SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ -- -- /* -- * QSEQ brands each QUEUED instance so that, when dispatch races -- * dequeue/requeue, the dispatcher can tell whether it still has a claim -- * on the task being dispatched. -- */ -- SCX_OPSS_QSEQ_SHIFT = 2, -- SCX_OPSS_STATE_MASK = (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1, -- SCX_OPSS_QSEQ_MASK = ~SCX_OPSS_STATE_MASK, --}; -- --/* -- * During exit, a task may schedule after losing its PIDs. When disabling the -- * BPF scheduler, we need to be able to iterate tasks in every state to -- * guarantee system safety. Maintain a dedicated task list which contains every -- * task between its fork and eventual free. -- */ --static DEFINE_SPINLOCK(scx_tasks_lock); --static LIST_HEAD(scx_tasks); -- --/* ops enable/disable */ --static struct kthread_worker *scx_ops_helper; --static DEFINE_MUTEX(scx_ops_enable_mutex); --DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); --DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); --static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); --static bool scx_switch_all_req; --static bool scx_switching_all; --DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -- --static struct sched_ext_ops scx_ops; --static bool scx_warned_zero_slice; -- --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); --DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); --static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); -- --struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] = -- { [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT }; -- --static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE); --static struct scx_exit_info scx_exit_info; -- --static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0); -- --/* -- * The maximum amount of time in jiffies that a task may be runnable without -- * being scheduled on a CPU. If this timeout is exceeded, it will trigger -- * scx_ops_error(). -- */ --unsigned long scx_watchdog_timeout; -- --/* -- * The last time the delayed work was run. This delayed work relies on -- * ksoftirqd being able to run to service timer interrupts, so it's possible -- * that this work itself could get wedged. To account for this, we check that -- * it's not stalled in the timer tick, and trigger an error if it is. -- */ --unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; -- --static struct delayed_work scx_watchdog_work; -- --/* idle tracking */ --#ifdef CONFIG_SMP --#ifdef CONFIG_CPUMASK_OFFSTACK --#define CL_ALIGNED_IF_ONSTACK --#else --#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp --#endif -- --static struct { -- cpumask_var_t cpu; -- cpumask_var_t smt; --} idle_masks CL_ALIGNED_IF_ONSTACK; -- --static bool __cacheline_aligned_in_smp scx_has_idle_cpus; --#endif /* CONFIG_SMP */ -- --/* for %SCX_KICK_WAIT */ --static u64 __percpu *scx_kick_cpus_pnt_seqs; -- --/* -- * Direct dispatch marker. -- * -- * Non-NULL values are used for direct dispatch from enqueue path. A valid -- * pointer points to the task currently being enqueued. An ERR_PTR value is used -- * to indicate that direct dispatch has already happened. -- */ --static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); -- --/* dispatch queues */ --static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; -- --static LLIST_HEAD(dsqs_to_free); -- --/* dispatch buf */ --struct scx_dsp_buf_ent { -- struct task_struct *task; -- u64 qseq; -- u64 dsq_id; -- u64 enq_flags; --}; -- --static u32 scx_dsp_max_batch; --static struct scx_dsp_buf_ent __percpu *scx_dsp_buf; -- --struct scx_dsp_ctx { -- struct rq *rq; -- struct rq_flags *rf; -- u32 buf_cursor; -- u32 nr_tasks; --}; -- --static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx); -- --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags); --void scx_bpf_kick_cpu(s32 cpu, u64 flags); -- --struct scx_task_iter { -- struct sched_ext_entity cursor; -- struct task_struct *locked; -- struct rq *rq; -- struct rq_flags rf; --}; -- --#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) -- --/* if the highest set bit is N, return a mask with bits [N+1, 31] set */ --static u32 higher_bits(u32 flags) --{ -- return ~((1 << fls(flags)) - 1); --} -- --/* return the mask with only the highest bit set */ --static u32 highest_bit(u32 flags) --{ -- int bit = fls(flags); -- return bit ? 1 << (bit - 1) : 0; --} -- --/* -- * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX -- * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate -- * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check -- * whether it's running from an allowed context. -- * -- * @mask is constant, always inline to cull the mask calculations. -- */ --static __always_inline void scx_kf_allow(u32 mask) --{ -- /* nesting is allowed only in increasing scx_kf_mask order */ -- WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask, -- "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n", -- current->scx->kf_mask, mask); -- current->scx->kf_mask |= mask; --} -- --static void scx_kf_disallow(u32 mask) --{ -- current->scx->kf_mask &= ~mask; --} -- --#define SCX_CALL_OP(mask, op, args...) \ --do { \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- scx_ops.op(args); \ -- } \ --} while (0) -- --#define SCX_CALL_OP_RET(mask, op, args...) \ --({ \ -- __typeof__(scx_ops.op(args)) __ret; \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- __ret = scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- __ret = scx_ops.op(args); \ -- } \ -- __ret; \ --}) -- --/* -- * Some kfuncs are allowed only on the tasks that are subjects of the -- * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such -- * restrictions, the following SCX_CALL_OP_*() variants should be used when -- * invoking scx_ops operations that take task arguments. These can only be used -- * for non-nesting operations due to the way the tasks are tracked. -- * -- * kfuncs which can only operate on such tasks can in turn use -- * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on -- * the specific task. -- */ --#define SCX_CALL_OP_TASK(mask, op, task, args...) \ --do { \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- SCX_CALL_OP(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ --} while (0) -- --#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ --({ \ -- __typeof__(scx_ops.op(task, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- __ret; \ --}) -- --#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ --({ \ -- __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task0; \ -- current->scx->kf_tasks[1] = task1; \ -- __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- current->scx->kf_tasks[1] = NULL; \ -- __ret; \ --}) -- --//#include "./slim_walt.c" -- --/* @mask is constant, always inline to cull unnecessary branches */ --static __always_inline bool scx_kf_allowed(u32 mask) --{ -- if (unlikely(!(current->scx->kf_mask & mask))) { -- scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", -- mask, current->scx->kf_mask); -- return false; -- } -- -- if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) && -- in_interrupt())) { -- scx_ops_error("sleepable kfunc called from non-sleepable context"); -- return false; -- } -- -- /* -- * Enforce nesting boundaries. e.g. A kfunc which can be called from -- * DISPATCH must not be called if we're running DEQUEUE which is nested -- * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE -- * boundary thanks to the above in_interrupt() check. -- */ -- if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && -- (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { -- scx_ops_error("cpu_release kfunc called from a nested operation"); -- return false; -- } -- -- if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && -- (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) { -- scx_ops_error("dispatch kfunc called from a nested operation"); -- return false; -- } -- -- return true; --} -- --/* see SCX_CALL_OP_TASK() */ --static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, -- struct task_struct *p) --{ -- if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED)) -- return false; -- -- if (unlikely((p != current->scx->kf_tasks[0] && -- p != current->scx->kf_tasks[1]))) { -- scx_ops_error("called on a task not being operated on"); -- return false; -- } -- -- return true; --} -- --/** -- * scx_task_iter_init - Initialize a task iterator -- * @iter: iterator to init -- * -- * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized, -- * @iter must eventually be exited with scx_task_iter_exit(). -- * -- * scx_tasks_lock may be released between this and the first next() call or -- * between any two next() calls. If scx_tasks_lock is released between two -- * next() calls, the caller is responsible for ensuring that the task being -- * iterated remains accessible either through RCU read lock or obtaining a -- * reference count. -- * -- * All tasks which existed when the iteration started are guaranteed to be -- * visited as long as they still exist. -- */ --static void scx_task_iter_init(struct scx_task_iter *iter) --{ -- lockdep_assert_held(&scx_tasks_lock); -- -- iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; -- list_add(&iter->cursor.tasks_node, &scx_tasks); -- iter->locked = NULL; --} -- --/** -- * scx_task_iter_exit - Exit a task iterator -- * @iter: iterator to exit -- * -- * Exit a previously initialized @iter. Must be called with scx_tasks_lock held. -- * If the iterator holds a task's rq lock, that rq lock is released. See -- * scx_task_iter_init() for details. -- */ --static void scx_task_iter_exit(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- if (list_empty(cursor)) -- return; -- -- list_del_init(cursor); --} -- --/** -- * scx_task_iter_next - Next task -- * @iter: iterator to walk -- * -- * Visit the next task. See scx_task_iter_init() for details. -- */ --static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- struct sched_ext_entity *pos; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- list_for_each_entry(pos, cursor, tasks_node) { -- if (&pos->tasks_node == &scx_tasks) -- return NULL; -- if (!(pos->flags & SCX_TASK_CURSOR)) { -- list_move(cursor, &pos->tasks_node); -- return pos->task; -- } -- } -- -- /* can't happen, should always terminate at scx_tasks above */ -- BUG(); --} -- --/** -- * scx_task_iter_next_filtered - Next non-idle task -- * @iter: iterator to walk -- * -- * Visit the next non-idle task. See scx_task_iter_init() for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- while ((p = scx_task_iter_next(iter))) { -- if (!is_idle_task(p)) -- return p; -- } -- return NULL; --} -- --/** -- * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked -- * @iter: iterator to walk -- * -- * Visit the next non-idle task with its rq lock held. See scx_task_iter_init() -- * for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered_locked(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- p = scx_task_iter_next_filtered(iter); -- if (!p) -- return NULL; -- -- iter->rq = task_rq_lock(p, &iter->rf); -- iter->locked = p; -- return p; --} -- --static enum scx_ops_enable_state scx_ops_enable_state(void) --{ -- return atomic_read(&scx_ops_enable_state_var); --} -- --static enum scx_ops_enable_state --scx_ops_set_enable_state(enum scx_ops_enable_state to) --{ -- return atomic_xchg(&scx_ops_enable_state_var, to); --} -- --static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, -- enum scx_ops_enable_state from) --{ -- int from_v = from; -- -- return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); --} -- --static bool scx_ops_disabling(void) --{ -- return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING); --} -- --/** -- * wait_ops_state - Busy-wait the specified ops state to end -- * @p: target task -- * @opss: state to wait the end of -- * -- * Busy-wait for @p to transition out of @opss. This can only be used when the -- * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also -- * has load_acquire semantics to ensure that the caller can see the updates made -- * in the enqueueing and dispatching paths. -- */ --static void wait_ops_state(struct task_struct *p, u64 opss) --{ -- do { -- cpu_relax(); -- } while (atomic64_read_acquire(&p->scx->ops_state) == opss); --} -- --/** -- * ops_cpu_valid - Verify a cpu number -- * @cpu: cpu number which came from a BPF ops -- * -- * @cpu is a cpu number which came from the BPF scheduler and can be any value. -- * Verify that it is in range and one of the possible cpus. -- */ --static bool ops_cpu_valid(s32 cpu) --{ -- return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); --} -- --/** -- * ops_sanitize_err - Sanitize a -errno value -- * @ops_name: operation to blame on failure -- * @err: -errno value to sanitize -- * -- * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return -- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can -- * cause misbehaviors. For an example, a large negative return from -- * ops.prep_enable() triggers an oops when passed up the call chain because the -- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is -- * handled as a pointer. -- */ --static int ops_sanitize_err(const char *ops_name, s32 err) --{ -- if (err < 0 && err >= -MAX_ERRNO) -- return err; -- -- scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); -- return -EPROTO; --} -- --/** -- * touch_core_sched - Update timestamp used for core-sched task ordering -- * @rq: rq to read clock from, must be locked -- * @p: task to update the timestamp for -- * -- * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to -- * implement global or local-DSQ FIFO ordering for core-sched. Should be called -- * when a task becomes runnable and its turn on the CPU ends (e.g. slice -- * exhaustion). -- */ --static void touch_core_sched(struct rq *rq, struct task_struct *p) --{ --#ifdef CONFIG_SCHED_CORE -- /* -- * It's okay to update the timestamp spuriously. Use -- * sched_core_disabled() which is cheaper than enabled(). -- */ -- if (!sched_core_disabled()) -- p->scx->core_sched_at = rq_clock_task(rq); --#endif --} -- --/** -- * touch_core_sched_dispatch - Update core-sched timestamp on dispatch -- * @rq: rq to read clock from, must be locked -- * @p: task being dispatched -- * -- * If the BPF scheduler implements custom core-sched ordering via -- * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO -- * ordering within each local DSQ. This function is called from dispatch paths -- * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect. -- */ --static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- assert_clock_updated(rq); -- --#ifdef CONFIG_SCHED_CORE -- if (SCX_HAS_OP(core_sched_before)) -- touch_core_sched(rq, p); --#endif --} -- --static void update_curr_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- u64 now = rq_clock_task(rq); -- u64 delta_exec; -- -- if (time_before_eq64(now, curr->se.exec_start)) -- return; -- -- delta_exec = now - curr->se.exec_start; -- curr->se.exec_start = now; -- curr->se.sum_exec_runtime += delta_exec; -- account_group_exec_runtime(curr, delta_exec); -- cgroup_account_cputime(curr, delta_exec); -- -- if (curr->scx->slice != SCX_SLICE_INF) { -- curr->scx->slice -= min(curr->scx->slice, delta_exec); -- if (!curr->scx->slice) -- touch_core_sched(rq, curr); -- } --} -- --static bool scx_dsq_priq_less(struct rb_node *node_a, -- const struct rb_node *node_b) --{ -- const struct sched_ext_entity *a = -- container_of(node_a, struct sched_ext_entity, dsq_node.priq); -- const struct sched_ext_entity *b = -- container_of(node_b, struct sched_ext_entity, dsq_node.priq); -- -- return time_before64(a->dsq_vtime, b->dsq_vtime); --} -- --static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, -- u64 enq_flags) --{ -- bool is_local = dsq->id == SCX_DSQ_LOCAL; -- -- WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo)); -- WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq)); -- -- if (!is_local) { -- raw_spin_lock(&dsq->lock); -- if (unlikely(dsq->id == SCX_DSQ_INVALID)) { -- scx_ops_error("attempting to dispatch to a destroyed dsq"); -- /* fall back to the global dsq */ -- raw_spin_unlock(&dsq->lock); -- dsq = &scx_dsq_global; -- raw_spin_lock(&dsq->lock); -- } -- } -- -- if (enq_flags & SCX_ENQ_DSQ_PRIQ) { -- p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; -- rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq, -- scx_dsq_priq_less); -- } else { -- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) -- list_add(&p->scx->dsq_node.fifo, &dsq->fifo); -- else -- list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo); -- } -- dsq->nr++; -- p->scx->dsq = dsq; -- -- /* -- * We're transitioning out of QUEUEING or DISPATCHING. store_release to -- * match waiters' load_acquire. -- */ -- if (enq_flags & SCX_ENQ_CLEAR_OPSS) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- if (is_local) { -- struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq); -- struct rq *rq = scx->rq; -- bool preempt = false; -- -- if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && -- rq->curr->sched_class == &ext_sched_class) { -- rq->curr->scx->slice = 0; -- preempt = true; -- } -- -- if (preempt || sched_class_above(&ext_sched_class, -- rq->curr->sched_class)) -- resched_curr(rq); -- } else { -- raw_spin_unlock(&dsq->lock); -- } --} -- --static void task_unlink_from_dsq(struct task_struct *p, -- struct scx_dispatch_q *dsq) --{ -- if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { -- rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; -- } else { -- list_del_init(&p->scx->dsq_node.fifo); -- } -- --} -- --static bool task_linked_on_dsq(struct task_struct *p) --{ -- return !list_empty(&p->scx->dsq_node.fifo) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq); --} -- --static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p) --{ -- struct scx_dispatch_q *dsq = p->scx->dsq; -- bool is_local = dsq == &scx_rq->local_dsq; -- -- if (!dsq) { -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- /* -- * When dispatching directly from the BPF scheduler to a local -- * DSQ, the task isn't associated with any DSQ but -- * @p->scx->holding_cpu may be set under the protection of -- * %SCX_OPSS_DISPATCHING. -- */ -- if (p->scx->holding_cpu >= 0) -- p->scx->holding_cpu = -1; -- return; -- } -- -- if (!is_local) -- raw_spin_lock(&dsq->lock); -- -- /* -- * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node -- * can't change underneath us. -- */ -- if (p->scx->holding_cpu < 0) { -- /* @p must still be on @dsq, dequeue */ -- WARN_ON_ONCE(!task_linked_on_dsq(p)); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- } else { -- /* -- * We're racing against dispatch_to_local_dsq() which already -- * removed @p from @dsq and set @p->scx->holding_cpu. Clear the -- * holding_cpu which tells dispatch_to_local_dsq() that it lost -- * the race. -- */ -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- p->scx->holding_cpu = -1; -- } -- p->scx->dsq = NULL; -- -- if (!is_local) -- raw_spin_unlock(&dsq->lock); --} -- --static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) --{ -- lockdep_assert(rcu_read_lock_any_held()); -- -- return &scx_dsq_global; --} -- --static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, -- struct task_struct *p) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id == SCX_DSQ_LOCAL) -- return &rq->scx->local_dsq; -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", -- dsq_id, p->comm, p->pid); -- return &scx_dsq_global; -- } -- -- return dsq; --} -- --static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- -- /* @p must match the task which is being enqueued */ -- if (unlikely(p != ddsp_task)) { -- if (IS_ERR(ddsp_task)) -- scx_ops_error("%s[%d] already direct-dispatched", -- p->comm, p->pid); -- else -- scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]", -- ddsp_task->comm, ddsp_task->pid, -- p->comm, p->pid); -- return; -- } -- -- /* -- * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because -- * dispatching to the local DSQ of a different CPU requires unlocking -- * the current rq which isn't allowed in the enqueue path. Use -- * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL. -- */ -- if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) { -- scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch"); -- return; -- } -- -- touch_core_sched_dispatch(task_rq(p), p); -- -- dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- -- /* -- * Mark that dispatch already happened by spoiling direct_dispatch_task -- * with a non-NULL value which can never match a valid task pointer. -- */ -- __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); --} -- --static bool test_rq_online(struct rq *rq) --{ --#ifdef CONFIG_SMP -- return rq->online; --#else -- return true; --#endif --} -- --static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- int sticky_cpu) --{ -- struct task_struct **ddsp_taskp; -- u64 qseq; -- -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- if (p->scx->flags & SCX_TASK_ENQ_LOCAL) { -- enq_flags |= SCX_ENQ_LOCAL; -- p->scx->flags &= ~SCX_TASK_ENQ_LOCAL; -- } -- -- /* rq migration */ -- if (sticky_cpu == cpu_of(rq)) -- goto local_norefill; -- -- /* -- * If !rq->online, we already told the BPF scheduler that the CPU is -- * offline. We're just trying to on/offline the CPU. Don't bother the -- * BPF scheduler. -- */ -- if (unlikely(!test_rq_online(rq))) -- goto local; -- -- /* see %SCX_OPS_ENQ_EXITING */ -- if (!static_branch_unlikely(&scx_ops_enq_exiting) && -- unlikely(p->flags & PF_EXITING)) -- goto local; -- -- /* see %SCX_OPS_ENQ_LAST */ -- if (!static_branch_unlikely(&scx_ops_enq_last) && -- (enq_flags & SCX_ENQ_LAST)) -- goto local; -- -- if (!SCX_HAS_OP(enqueue)) { -- if (enq_flags & SCX_ENQ_LOCAL) -- goto local; -- else -- goto global; -- } -- -- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ -- qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; -- -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq); -- -- ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); -- WARN_ON_ONCE(*ddsp_taskp); -- *ddsp_taskp = p; -- -- SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); -- -- /* -- * If not directly dispatched, QUEUEING isn't clear yet and dispatch or -- * dequeue may be waiting. The store_release matches their load_acquire. -- */ -- if (*ddsp_taskp == p) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq); -- *ddsp_taskp = NULL; -- return; -- --local: -- /* -- * For task-ordering, slice refill must be treated as implying the end -- * of the current slice. Otherwise, the longer @p stays on the CPU, the -- * higher priority it becomes from scx_prio_less()'s POV. -- */ -- touch_core_sched(rq, p); -- p->scx->slice = SCX_SLICE_DFL; --local_norefill: -- dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags); -- return; -- --global: -- touch_core_sched(rq, p); /* see the comment in local: */ -- p->scx->slice = SCX_SLICE_DFL; -- dispatch_enqueue(&scx_dsq_global, p, enq_flags); --} -- --static bool watchdog_task_watched(const struct task_struct *p) --{ -- return !list_empty(&p->scx->watchdog_node); --} -- --static void watchdog_watch_task(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- if (p->scx->flags & SCX_TASK_WATCHDOG_RESET) -- p->scx->runnable_at = jiffies; -- p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET; -- list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list); --} -- --static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) --{ -- list_del_init(&p->scx->watchdog_node); -- if (reset_timeout) -- p->scx->flags |= SCX_TASK_WATCHDOG_RESET; --} -- --static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) --{ -- int sticky_cpu = p->scx->sticky_cpu; -- -- enq_flags |= rq->scx->extra_enq_flags; -- -- if (sticky_cpu >= 0) -- p->scx->sticky_cpu = -1; -- -- /* -- * Restoring a running task will be immediately followed by -- * set_next_task_scx() which expects the task to not be on the BPF -- * scheduler as tasks can only start running through local DSQs. Force -- * direct-dispatch into the local DSQ by setting the sticky_cpu. -- */ -- if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -- sticky_cpu = cpu_of(rq); -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- WARN_ON_ONCE(!watchdog_task_watched(p)); -- return; -- } -- -- watchdog_watch_task(rq, p); -- p->scx->flags |= SCX_TASK_QUEUED; -- rq->scx->nr_running++; -- add_nr_running(rq, 1); -- -- if (SCX_HAS_OP(runnable)) -- SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); -- -- if (enq_flags & SCX_ENQ_WAKEUP) -- touch_core_sched(rq, p); -- -- do_enqueue_task(rq, p, enq_flags, sticky_cpu); --} -- --static void ops_dequeue(struct task_struct *p, u64 deq_flags) --{ -- u64 opss; -- -- watchdog_unwatch_task(p, false); -- -- /* acquire ensures that we see the preceding updates on QUEUED */ -- opss = atomic64_read_acquire(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_NONE: -- break; -- case SCX_OPSS_QUEUEING: -- /* -- * QUEUEING is started and finished while holding @p's rq lock. -- * As we're holding the rq lock now, we shouldn't see QUEUEING. -- */ -- BUG(); -- case SCX_OPSS_QUEUED: -- if (SCX_HAS_OP(dequeue)) -- SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); -- -- if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_NONE)) -- break; -- fallthrough; -- case SCX_OPSS_DISPATCHING: -- /* -- * If @p is being dispatched from the BPF scheduler to a DSQ, -- * wait for the transfer to complete so that @p doesn't get -- * added to its DSQ after dequeueing is complete. -- * -- * As we're waiting on DISPATCHING with the rq locked, the -- * dispatching side shouldn't try to lock the rq while -- * DISPATCHING is set. See dispatch_to_local_dsq(). -- * -- * DISPATCHING shouldn't have qseq set and control can reach -- * here with NONE @opss from the above QUEUED case block. -- * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. -- */ -- wait_ops_state(p, SCX_OPSS_DISPATCHING); -- BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- break; -- } --} -- --static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) --{ -- struct scx_rq *scx_rq = rq->scx; -- -- if (!(p->scx->flags & SCX_TASK_QUEUED)) { -- WARN_ON_ONCE(watchdog_task_watched(p)); -- return; -- } -- -- ops_dequeue(p, deq_flags); -- -- /* -- * A currently running task which is going off @rq first gets dequeued -- * and then stops running. As we want running <-> stopping transitions -- * to be contained within runnable <-> quiescent transitions, trigger -- * ->stopping() early here instead of in put_prev_task_scx(). -- * -- * @p may go through multiple stopping <-> running transitions between -- * here and put_prev_task_scx() if task attribute changes occur while -- * balance_scx() leaves @rq unlocked. However, they don't contain any -- * information meaningful to the BPF scheduler and can be suppressed by -- * skipping the callbacks if the task is !QUEUED. -- */ -- if (SCX_HAS_OP(stopping) && task_current(rq, p)) { -- update_curr_scx(rq); -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); -- } -- -- //if(task_current(rq, p)) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- if (SCX_HAS_OP(quiescent)) -- SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); -- -- if (deq_flags & SCX_DEQ_SLEEP) -- p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP; -- else -- p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP; -- -- p->scx->flags &= ~SCX_TASK_QUEUED; -- BUG_ON(!scx_rq->nr_running); -- scx_rq->nr_running--; -- sub_nr_running(rq, 1); -- -- dispatch_dequeue(scx_rq, p); --} -- --static void yield_task_scx(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); -- else -- p->scx->slice = 0; --} -- --static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) --{ -- struct task_struct *from = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); -- else -- return false; --} -- --#ifdef CONFIG_SMP --/** -- * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -- * @rq: rq to move the task into, currently locked -- * @p: task to move -- * @enq_flags: %SCX_ENQ_* -- * -- * Move @p which is currently on a different rq to @rq's local DSQ. The caller -- * must: -- * -- * 1. Start with exclusive access to @p either through its DSQ lock or -- * %SCX_OPSS_DISPATCHING flag. -- * -- * 2. Set @p->scx->holding_cpu to raw_smp_processor_id(). -- * -- * 3. Remember task_rq(@p). Release the exclusive access so that we don't -- * deadlock with dequeue. -- * -- * 4. Lock @rq and the task_rq from #3. -- * -- * 5. Call this function. -- * -- * Returns %true if @p was successfully moved. %false after racing dequeue and -- * losing. -- */ --static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -- u64 enq_flags) --{ -- struct rq *task_rq; -- -- lockdep_assert_rq_held(rq); -- -- /* -- * If dequeue got to @p while we were trying to lock both rq's, it'd -- * have cleared @p->scx->holding_cpu to -1. While other cpus may have -- * updated it to different values afterwards, as this operation can't be -- * preempted or recurse, @p->scx->holding_cpu can never become -- * raw_smp_processor_id() again before we're done. Thus, we can tell -- * whether we lost to dequeue by testing whether @p->scx->holding_cpu is -- * still raw_smp_processor_id(). -- * -- * See dispatch_dequeue() for the counterpart. -- */ -- if (unlikely(p->scx->holding_cpu != raw_smp_processor_id())) -- return false; -- -- /* @p->rq couldn't have changed if we're still the holding cpu */ -- task_rq = task_rq(p); -- lockdep_assert_rq_held(task_rq); -- -- WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)); -- deactivate_task(task_rq, p, 0); -- set_task_cpu(p, cpu_of(rq)); -- p->scx->sticky_cpu = cpu_of(rq); -- -- /* -- * We want to pass scx-specific enq_flags but activate_task() will -- * truncate the upper 32 bit. As we own @rq, we can pass them through -- * @rq->scx->extra_enq_flags instead. -- */ -- WARN_ON_ONCE(rq->scx->extra_enq_flags); -- rq->scx->extra_enq_flags = enq_flags; -- activate_task(rq, p, 0); -- rq->scx->extra_enq_flags = 0; -- -- return true; --} -- --/** -- * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * We're holding @rq lock and trying to dispatch a task from @src_rq to -- * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether -- * @rq stays locked isn't important as long as the state is restored after -- * dispatch_to_local_dsq_unlock(). -- */ --static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- rq_unpin_lock(rq, rf); -- -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(rq); -- raw_spin_rq_lock(dst_rq); -- } else if (rq == src_rq) { -- double_lock_balance(rq, dst_rq); -- rq_repin_lock(rq, rf); -- } else if (rq == dst_rq) { -- double_lock_balance(rq, src_rq); -- rq_repin_lock(rq, rf); -- } else { -- raw_spin_rq_unlock(rq); -- double_rq_lock(src_rq, dst_rq); -- } --} -- --/** -- * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock() -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return. -- */ --static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } else if (rq == src_rq) { -- double_unlock_balance(rq, dst_rq); -- } else if (rq == dst_rq) { -- double_unlock_balance(rq, src_rq); -- } else { -- double_rq_unlock(src_rq, dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } --} --#endif /* CONFIG_SMP */ -- -- --static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq) --{ -- return likely(test_rq_online(rq)) && !is_migration_disabled(p) && -- cpumask_test_cpu(cpu_of(rq), p->cpus_ptr); --} -- --static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -- struct scx_dispatch_q *dsq) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rb_node *rb_node; -- struct rq *task_rq; -- bool moved = false; --retry: -- if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -- return false; -- -- raw_spin_lock(&dsq->lock); -- -- list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- for (rb_node = rb_first_cached(&dsq->priq); rb_node; -- rb_node = rb_next(rb_node)) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- raw_spin_unlock(&dsq->lock); -- return false; -- --this_rq: -- /* @dsq is locked and @p is on this rq */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo); -- dsq->nr--; -- scx_rq->local_dsq.nr++; -- p->scx->dsq = &scx_rq->local_dsq; -- raw_spin_unlock(&dsq->lock); -- return true; -- --remote_rq: --#ifdef CONFIG_SMP -- /* -- * @dsq is locked and @p is on a remote rq. @p is currently protected by -- * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -- * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -- * rq lock or fail, do a little dancing from our side. See -- * move_task_to_local_dsq(). -- */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- p->scx->holding_cpu = raw_smp_processor_id(); -- raw_spin_unlock(&dsq->lock); -- -- rq_unpin_lock(rq, rf); -- double_lock_balance(rq, task_rq); -- rq_repin_lock(rq, rf); -- -- moved = move_task_to_local_dsq(rq, p, 0); -- -- double_unlock_balance(rq, task_rq); --#endif /* CONFIG_SMP */ -- if (likely(moved)) -- return true; -- goto retry; --} -- --enum dispatch_to_local_dsq_ret { -- DTL_DISPATCHED, /* successfully dispatched */ -- DTL_LOST, /* lost race to dequeue */ -- DTL_NOT_LOCAL, /* destination is not a local DSQ */ -- DTL_INVALID, /* invalid local dsq_id */ --}; -- --/** -- * dispatch_to_local_dsq - Dispatch a task to a local dsq -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @dsq_id: destination dsq ID -- * @p: task to dispatch -- * @enq_flags: %SCX_ENQ_* -- * -- * We're holding @rq lock and want to dispatch @p to the local DSQ identified by -- * @dsq_id. This function performs all the synchronization dancing needed -- * because local DSQs are protected with rq locks. -- * -- * The caller must have exclusive ownership of @p (e.g. through -- * %SCX_OPSS_DISPATCHING). -- */ --static enum dispatch_to_local_dsq_ret --dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id, -- struct task_struct *p, u64 enq_flags) --{ -- struct rq *src_rq = task_rq(p); -- struct rq *dst_rq; -- -- /* -- * We're synchronized against dequeue through DISPATCHING. As @p can't -- * be dequeued, its task_rq and cpus_allowed are stable too. -- */ -- if (dsq_id == SCX_DSQ_LOCAL) { -- dst_rq = rq; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]", -- cpu, p->comm, p->pid); -- return DTL_INVALID; -- } -- dst_rq = cpu_rq(cpu); -- } else { -- return DTL_NOT_LOCAL; -- } -- -- /* if dispatching to @rq that @p is already on, no lock dancing needed */ -- if (rq == src_rq && rq == dst_rq) { -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags | SCX_ENQ_CLEAR_OPSS); -- return DTL_DISPATCHED; -- } -- --#ifdef CONFIG_SMP -- if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { -- struct rq *locked_dst_rq = dst_rq; -- bool dsp; -- -- /* -- * @p is on a possibly remote @src_rq which we need to lock to -- * move the task. If dequeue is in progress, it'd be locking -- * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq -- * lock while holding DISPATCHING. -- * -- * As DISPATCHING guarantees that @p is wholly ours, we can -- * pretend that we're moving from a DSQ and use the same -- * mechanism - mark the task under transfer with holding_cpu, -- * release DISPATCHING and then follow the same protocol. -- */ -- p->scx->holding_cpu = raw_smp_processor_id(); -- -- /* store_release ensures that dequeue sees the above */ -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq); -- -- /* -- * We don't require the BPF scheduler to avoid dispatching to -- * offline CPUs mostly for convenience but also because CPUs can -- * go offline between scx_bpf_dispatch() calls and here. If @p -- * is destined to an offline CPU, queue it on its current CPU -- * instead, which should always be safe. As this is an allowed -- * behavior, don't trigger an ops error. -- */ -- if (unlikely(!test_rq_online(dst_rq))) -- dst_rq = src_rq; -- -- if (src_rq == dst_rq) { -- /* -- * As @p is staying on the same rq, there's no need to -- * go through the full deactivate/activate cycle. -- * Optimize by abbreviating the operations in -- * move_task_to_local_dsq(). -- */ -- dsp = p->scx->holding_cpu == raw_smp_processor_id(); -- if (likely(dsp)) { -- p->scx->holding_cpu = -1; -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags); -- } -- } else { -- dsp = move_task_to_local_dsq(dst_rq, p, enq_flags); -- } -- -- /* if the destination CPU is idle, wake it up */ -- if (dsp && p->sched_class > dst_rq->curr->sched_class) -- resched_curr(dst_rq); -- -- dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq); -- -- return dsp ? DTL_DISPATCHED : DTL_LOST; -- } --#endif /* CONFIG_SMP */ -- -- scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", -- cpu_of(dst_rq), p->comm, p->pid); -- return DTL_INVALID; --} -- --/** -- * finish_dispatch - Asynchronously finish dispatching a task -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @p: task to finish dispatching -- * @qseq_at_dispatch: qseq when @p started getting dispatched -- * @dsq_id: destination DSQ ID -- * @enq_flags: %SCX_ENQ_* -- * -- * Dispatching to local DSQs may need to wait for queueing to complete or -- * require rq lock dancing. As we don't wanna do either while inside -- * ops.dispatch() to avoid locking order inversion, we split dispatching into -- * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the -- * task and its qseq. Once ops.dispatch() returns, this function is called to -- * finish up. -- * -- * There is no guarantee that @p is still valid for dispatching or even that it -- * was valid in the first place. Make sure that the task is still owned by the -- * BPF scheduler and claim the ownership before dispatching. -- */ --static void finish_dispatch(struct rq *rq, struct rq_flags *rf, -- struct task_struct *p, u64 qseq_at_dispatch, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- u64 opss; -- -- touch_core_sched_dispatch(rq, p); --retry: -- /* -- * No need for _acquire here. @p is accessed only after a successful -- * try_cmpxchg to DISPATCHING. -- */ -- opss = atomic64_read(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_DISPATCHING: -- case SCX_OPSS_NONE: -- /* someone else already got to it */ -- return; -- case SCX_OPSS_QUEUED: -- /* -- * If qseq doesn't match, @p has gone through at least one -- * dispatch/dequeue and re-enqueue cycle between -- * scx_bpf_dispatch() and here and we have no claim on it. -- */ -- if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) -- return; -- -- /* -- * While we know @p is accessible, we don't yet have a claim on -- * it - the BPF scheduler is allowed to dispatch tasks -- * spuriously and there can be a racing dequeue attempt. Let's -- * claim @p by atomically transitioning it from QUEUED to -- * DISPATCHING. -- */ -- if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_DISPATCHING))) -- break; -- goto retry; -- case SCX_OPSS_QUEUEING: -- /* -- * do_enqueue_task() is in the process of transferring the task -- * to the BPF scheduler while holding @p's rq lock. As we aren't -- * holding any kernel or BPF resource that the enqueue path may -- * depend upon, it's safe to wait. -- */ -- wait_ops_state(p, opss); -- goto retry; -- } -- -- BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) { -- case DTL_DISPATCHED: -- break; -- case DTL_LOST: -- break; -- case DTL_INVALID: -- dsq_id = SCX_DSQ_GLOBAL; -- fallthrough; -- case DTL_NOT_LOCAL: -- dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()), -- dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- break; -- } --} -- --static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- u32 u; -- -- for (u = 0; u < dspc->buf_cursor; u++) { -- struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u]; -- -- finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id, -- ent->enq_flags); -- } -- -- dspc->nr_tasks += dspc->buf_cursor; -- dspc->buf_cursor = 0; --} -- --static int balance_one(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf, bool local) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- bool prev_on_scx = prev->sched_class == &ext_sched_class; -- int nr_loops = SCX_DSP_MAX_LOOPS; -- -- lockdep_assert_rq_held(rq); -- -- if (static_branch_unlikely(&scx_ops_cpu_preempt) && -- unlikely(rq->scx->cpu_released)) { -- /* -- * If the previous sched_class for the current CPU was not SCX, -- * notify the BPF scheduler that it again has control of the -- * core. This callback complements ->cpu_release(), which is -- * emitted in scx_notify_pick_next_task(). -- */ -- if (SCX_HAS_OP(cpu_acquire)) -- SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq), -- NULL); -- rq->scx->cpu_released = false; -- } -- -- if (prev_on_scx) { -- WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP)); -- update_curr_scx(rq); -- -- /* -- * If @prev is runnable & has slice left, it has priority and -- * fetching more just increases latency for the fetched tasks. -- * Tell put_prev_task_scx() to put @prev on local_dsq. If the -- * BPF scheduler wants to handle this explicitly, it should -- * implement ->cpu_released(). -- * -- * See scx_ops_disable_workfn() for the explanation on the -- * disabling() test. -- * -- * When balancing a remote CPU for core-sched, there won't be a -- * following put_prev_task_scx() call and we don't own -- * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the -- * same conditions later and pick @rq->curr accordingly. -- */ -- if ((prev->scx->flags & SCX_TASK_QUEUED) && -- prev->scx->slice && !scx_ops_disabling()) { -- if (local) -- prev->scx->flags |= SCX_TASK_BAL_KEEP; -- return 1; -- } -- } -- -- /* if there already are tasks to run, nothing to do */ -- if (scx_rq->local_dsq.nr) -- return 1; -- -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- if (!SCX_HAS_OP(dispatch)) -- return 0; -- -- dspc->rq = rq; -- dspc->rf = rf; -- -- /* -- * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, -- * the local DSQ might still end up empty after a successful -- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() -- * produced some tasks, retry. The BPF scheduler may depend on this -- * looping behavior to simplify its implementation. -- */ -- do { -- dspc->nr_tasks = 0; -- -- SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), -- prev_on_scx ? prev : NULL); -- -- flush_dispatch_buf(rq, rf); -- -- if (scx_rq->local_dsq.nr) -- return 1; -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- /* -- * ops.dispatch() can trap us in this loop by repeatedly -- * dispatching ineligible tasks. Break out once in a while to -- * allow the watchdog to run. As IRQ can't be enabled in -- * balance(), we want to complete this scheduling cycle and then -- * start a new one. IOW, we want to call resched_curr() on the -- * next, most likely idle, task, not the current one. Use -- * scx_bpf_kick_cpu() for deferred kicking. -- */ -- if (unlikely(!--nr_loops)) { -- scx_bpf_kick_cpu(cpu_of(rq), 0); -- break; -- } -- } while (dspc->nr_tasks); -- -- return 0; --} -- --static int balance_scx(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf) --{ -- int ret; -- -- ret = balance_one(rq, prev, rf, true); --#ifdef CONFIG_SCHED_SMT -- /* -- * When core-sched is enabled, this ops.balance() call will be followed -- * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx() -- * on the SMT siblings. Balance the siblings too. -- */ -- if (sched_core_enabled(rq)) { -- const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); -- int scpu; -- -- for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { -- struct rq *srq = cpu_rq(scpu); -- struct rq_flags srf; -- struct task_struct *sprev = srq->curr; -- -- /* -- * While core-scheduling, rq lock is shared among -- * siblings but the debug annotations and rq clock -- * aren't. Do pinning dance to transfer the ownership. -- */ -- WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); -- rq_unpin_lock(rq, rf); -- rq_pin_lock(srq, &srf); -- -- update_rq_clock(srq); -- balance_one(srq, sprev, &srf, false); -- -- rq_unpin_lock(srq, &srf); -- rq_repin_lock(rq, rf); -- } -- } --#endif -- return ret; --} -- --static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) --{ -- if (p->scx->flags & SCX_TASK_QUEUED) { -- /* -- * Core-sched might decide to execute @p before it is -- * dispatched. Call ops_dequeue() to notify the BPF scheduler. -- */ -- ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); -- dispatch_dequeue(rq->scx, p); -- } -- -- p->se.exec_start = rq_clock_task(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, running, p); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock); -- -- watchdog_unwatch_task(p, true); -- -- /* -- * @p is getting newly scheduled or got kicked after someone updated its -- * slice. Refresh whether tick can be stopped. See can_stop_tick_scx(). -- */ -- if ((p->scx->slice == SCX_SLICE_INF) != -- (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) { -- if (p->scx->slice == SCX_SLICE_INF) -- rq->scx->flags |= SCX_RQ_CAN_STOP_TICK; -- else -- rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK; -- -- sched_update_tick_dependency(rq); -- } --} -- --static void put_prev_task_scx(struct rq *rq, struct task_struct *p) --{ --#ifndef CONFIG_SMP -- /* -- * UP workaround. -- * -- * Because SCX may transfer tasks across CPUs during dispatch, dispatch -- * is performed from its balance operation which isn't called in UP. -- * Let's work around by calling it from the operations which come right -- * after. -- * -- * 1. If the prev task is on SCX, pick_next_task() calls -- * .put_prev_task() right after. As .put_prev_task() is also called -- * from other places, we need to distinguish the calls which can be -- * done by looking at the previous task's state - if still queued or -- * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task(). -- * This case is handled here. -- * -- * 2. If the prev task is not on SCX, the first following call into SCX -- * will be .pick_next_task(), which is covered by calling -- * balance_scx() from pick_next_task_scx(). -- * -- * Note that we can't merge the first case into the second as -- * balance_scx() must be called before the previous SCX task goes -- * through put_prev_task_scx(). -- * -- * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf. -- * Pass in %NULL. -- */ -- if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP)) -- balance_scx(rq, p, NULL); --#endif -- -- update_curr_scx(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- -- /* -- * If we're being called from put_prev_task_balance(), balance_scx() may -- * have decided that @p should keep running. -- */ -- if (p->scx->flags & SCX_TASK_BAL_KEEP) { -- p->scx->flags &= ~SCX_TASK_BAL_KEEP; -- watchdog_watch_task(rq, p); -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- watchdog_watch_task(rq, p); -- -- /* -- * If @p has slice left and balance_scx() didn't tag it for -- * keeping, @p is getting preempted by a higher priority -- * scheduler class or core-sched forcing a different task. Leave -- * it at the head of the local DSQ. -- */ -- if (p->scx->slice && !scx_ops_disabling()) { -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- /* -- * If we're in the pick_next_task path, balance_scx() should -- * have already populated the local DSQ if there are any other -- * available tasks. If empty, tell ops.enqueue() that @p is the -- * only one available for this cpu. ops.enqueue() should put it -- * on the local DSQ so that the subsequent pick_next_task_scx() -- * can find the task unless it wants to trigger a separate -- * follow-up scheduling event. -- */ -- if (list_empty(&rq->scx->local_dsq.fifo)) -- do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1); -- else -- do_enqueue_task(rq, p, 0, -1); -- } --} -- --static struct task_struct *first_local_task(struct rq *rq) --{ -- struct rb_node *rb_node; -- struct sched_ext_entity *entity; -- -- if (!list_empty(&rq->scx->local_dsq.fifo)) { -- entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo); -- return entity->task; -- } -- -- rb_node = rb_first_cached(&rq->scx->local_dsq.priq); -- if (rb_node) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- return entity->task; -- } -- -- return NULL; --} -- --static struct task_struct *pick_next_task_scx(struct rq *rq) --{ -- struct task_struct *p; -- --#ifndef CONFIG_SMP -- /* UP workaround - see the comment at the head of put_prev_task_scx() */ -- if (unlikely(rq->curr->sched_class != &ext_sched_class)) -- balance_scx(rq, rq->curr, NULL); --#endif -- -- p = first_local_task(rq); -- if (!p) -- return NULL; -- -- if (unlikely(!p->scx->slice)) { -- if (!scx_ops_disabling() && !scx_warned_zero_slice) { -- printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n", -- p->comm, p->pid); -- scx_warned_zero_slice = true; -- } -- p->scx->slice = SCX_SLICE_DFL; -- } -- -- set_next_task_scx(rq, p, true); -- -- return p; --} -- --#ifdef CONFIG_SCHED_CORE --/** -- * scx_prio_less - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Core-sched is implemented as an additional scheduling layer on top of the -- * usual sched_class'es and needs to find out the expected task ordering. For -- * SCX, core-sched calls this function to interrogate the task ordering. -- * -- * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used -- * to implement the default task ordering. The older the timestamp, the higher -- * prority the task - the global FIFO ordering matching the default scheduling -- * behavior. -- * -- * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to -- * implement FIFO ordering within each local DSQ. See pick_task_scx(). -- */ --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi) --{ -- /* -- * The const qualifiers are dropped from task_struct pointers when -- * calling ops.core_sched_before(). Accesses are controlled by the -- * verifier. -- */ -- if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling()) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, -- (struct task_struct *)a, -- (struct task_struct *)b); -- else -- return time_after64(a->scx->core_sched_at, b->scx->core_sched_at); --} -- --/** -- * pick_task_scx - Pick a candidate task for core-sched -- * @rq: rq to pick the candidate task from -- * -- * Core-sched calls this function on each SMT sibling to determine the next -- * tasks to run on the SMT siblings. balance_one() has been called on all -- * siblings and put_prev_task_scx() has been called only for the current CPU. -- * -- * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look -- * at the first task in the local dsq. @rq->curr has to be considered explicitly -- * to mimic %SCX_TASK_BAL_KEEP. -- */ --static struct task_struct *pick_task_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- struct task_struct *first = first_local_task(rq); -- -- if (curr->scx->flags & SCX_TASK_QUEUED) { -- /* is curr the only runnable task? */ -- if (!first) -- return curr; -- -- /* -- * Does curr trump first? We can always go by core_sched_at for -- * this comparison as it represents global FIFO ordering when -- * the default core-sched ordering is used and local-DSQ FIFO -- * ordering otherwise. -- * -- * We can have a task with an earlier timestamp on the DSQ. For -- * example, when a current task is preempted by a sibling -- * picking a different cookie, the task would be requeued at the -- * head of the local DSQ with an earlier timestamp than the -- * core-sched picked next task. Besides, the BPF scheduler may -- * dispatch any tasks to the local DSQ anytime. -- */ -- if (curr->scx->slice && time_before64(curr->scx->core_sched_at, -- first->scx->core_sched_at)) -- return curr; -- } -- -- return first; /* this may be %NULL */ --} --#endif /* CONFIG_SCHED_CORE */ -- --static enum scx_cpu_preempt_reason --preempt_reason_from_class(const struct sched_class *class) --{ --#ifdef CONFIG_SMP -- if (class == &stop_sched_class) -- return SCX_CPU_PREEMPT_STOP; --#endif -- if (class == &dl_sched_class) -- return SCX_CPU_PREEMPT_DL; -- if (class == &rt_sched_class) -- return SCX_CPU_PREEMPT_RT; -- return SCX_CPU_PREEMPT_UNKNOWN; --} -- --void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task, -- const struct sched_class *active) --{ -- lockdep_assert_rq_held(rq); -- -- /* -- * The callback is conceptually meant to convey that the CPU is no -- * longer under the control of SCX. Therefore, don't invoke the -- * callback if the CPU is is staying on SCX, or going idle (in which -- * case the SCX scheduler has actively decided not to schedule any -- * tasks on the CPU). -- */ -- if (likely(active >= &ext_sched_class)) -- return; -- -- /* -- * At this point we know that SCX was preempted by a higher priority -- * sched_class, so invoke the ->cpu_release() callback if we have not -- * done so already. We only send the callback once between SCX being -- * preempted, and it regaining control of the CPU. -- * -- * ->cpu_release() complements ->cpu_acquire(), which is emitted the -- * next time that balance_scx() is invoked. -- */ -- if (!rq->scx->cpu_released) { -- if (SCX_HAS_OP(cpu_release)) { -- struct scx_cpu_release_args args = { -- .reason = preempt_reason_from_class(active), -- .task = task, -- }; -- -- SCX_CALL_OP(SCX_KF_CPU_RELEASE, -- cpu_release, cpu_of(rq), &args); -- } -- rq->scx->cpu_released = true; -- } --} -- --#ifdef CONFIG_SMP -- --static bool test_and_clear_cpu_idle(int cpu) --{ -- if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -- if (cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- return true; -- } else { -- return false; -- } --} -- --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- int cpu; -- -- do { -- cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); -- if (cpu < nr_cpu_ids) { -- const struct cpumask *sbm = topology_sibling_cpumask(cpu); -- -- /* -- * If offline, @cpu is not its own sibling and we can -- * get caught in an infinite loop as @cpu is never -- * cleared from idle_masks.smt. Clear @cpu directly in -- * such cases. -- */ -- if (likely(cpumask_test_cpu(cpu, sbm))) -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm); -- else -- cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu)); -- } else { -- cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -- if (cpu >= nr_cpu_ids) -- return -EBUSY; -- } -- } while (!test_and_clear_cpu_idle(cpu)); -- -- return cpu; --} -- --static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) --{ -- s32 cpu; -- -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return prev_cpu; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local DSQ of the waker. -- */ -- if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 && -- scx_has_idle_cpus && !(current->flags & PF_EXITING)) { -- cpu = smp_processor_id(); -- if (cpumask_test_cpu(cpu, p->cpus_ptr)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (test_and_clear_cpu_idle(prev_cpu)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return prev_cpu; -- } -- -- if (p->nr_cpus_allowed == 1) -- return prev_cpu; -- -- cpu = scx_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- -- return prev_cpu; --} -- --static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) --{ -- if (SCX_HAS_OP(select_cpu)) { -- s32 cpu; -- -- cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu, -- wake_flags); -- if (ops_cpu_valid(cpu)) { -- return cpu; -- } else { -- scx_ops_error("select_cpu returned invalid cpu %d", cpu); -- return prev_cpu; -- } -- } else { -- return scx_select_cpu_dfl(p, prev_cpu, wake_flags); -- } --} -- --static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx) --{ -- set_cpus_allowed_common(p, ctx); -- -- /* -- * The effective cpumask is stored in @p->cpus_ptr which may temporarily -- * differ from the configured one in @p->cpus_mask. Always tell the bpf -- * scheduler the effective one. -- * -- * Fine-grained memory write control is enforced by BPF making the const -- * designation pointless. Cast it away when calling the operation. -- */ -- if (SCX_HAS_OP(set_cpumask)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, -- (struct cpumask *)p->cpus_ptr); --} -- --static void reset_idle_masks(void) --{ -- /* consider all cpus idle, should converge to the actual state quickly */ -- cpumask_setall(idle_masks.cpu); -- cpumask_setall(idle_masks.smt); -- scx_has_idle_cpus = true; --} -- --void __scx_update_idle(struct rq *rq, bool idle) --{ -- int cpu = cpu_of(rq); -- struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -- -- if (SCX_HAS_OP(update_idle)) { -- SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); -- if (!static_branch_unlikely(&scx_builtin_idle_enabled)) -- return; -- } -- -- if (idle) { -- cpumask_set_cpu(cpu, idle_masks.cpu); -- if (!scx_has_idle_cpus) -- scx_has_idle_cpus = true; -- -- /* -- * idle_masks.smt handling is racy but that's fine as it's only -- * for optimization and self-correcting. -- */ -- for_each_cpu(cpu, sib_mask) { -- if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -- return; -- } -- cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -- } else { -- cpumask_clear_cpu(cpu, idle_masks.cpu); -- if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -- } --} -- --#else /* !CONFIG_SMP */ -- --static bool test_and_clear_cpu_idle(int cpu) { return false; } --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } --static void reset_idle_masks(void) {} -- --#endif /* CONFIG_SMP */ -- --static bool check_rq_for_timeouts(struct rq *rq) --{ -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rq_flags rf; -- bool timed_out = false; -- -- rq_lock_irqsave(rq, &rf); -- list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) { -- unsigned long last_runnable; -- -- p = entity->task; -- last_runnable = p->scx->runnable_at; -- -- if (unlikely(time_after(jiffies, -- last_runnable + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "%s[%d] failed to run for %u.%03us", -- p->comm, p->pid, -- dur_ms / 1000, dur_ms % 1000); -- timed_out = true; -- break; -- } -- } -- rq_unlock_irqrestore(rq, &rf); -- -- return timed_out; --} -- --static void scx_watchdog_workfn(struct work_struct *work) --{ -- int cpu; -- -- scx_watchdog_timestamp = jiffies; -- -- for_each_online_cpu(cpu) { -- if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) -- break; -- -- cond_resched(); -- } -- queue_delayed_work(system_unbound_wq, to_delayed_work(work), -- scx_watchdog_timeout / 2); --} -- --static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) --{ -- update_curr_scx(rq); -- //scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock); -- /* -- * While disabling, always resched and refresh core-sched timestamp as -- * we can't trust the slice management or ops.core_sched_before(). -- */ -- if (scx_ops_disabling()) { -- curr->scx->slice = 0; -- touch_core_sched(rq, curr); -- } -- -- if (!curr->scx->slice) -- resched_curr(rq); --} -- --#define SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- --static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg) --{ -- int ret; -- -- WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED); -- -- p->scx->disallow = false; -- -- if (SCX_HAS_OP(prep_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- }; -- -- ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args); -- if (unlikely(ret)) { -- ret = ops_sanitize_err("prep_enable", ret); -- return ret; -- } -- } -- //scx_sched_init_task(p); -- -- if (p->scx->disallow) { -- struct rq *rq; -- struct rq_flags rf; -- -- rq = task_rq_lock(p, &rf); -- -- /* -- * We're either in fork or load path and @p->policy will be -- * applied right after. Reverting @p->policy here and rejecting -- * %SCHED_EXT transitions from scx_check_setscheduler() -- * guarantees that if ops.prep_enable() sets @p->disallow, @p -- * can never be in SCX. -- */ -- if (p->policy == SCHED_EXT) { -- p->policy = SCHED_NORMAL; -- atomic64_inc(&scx_nr_rejected); -- } -- -- task_rq_unlock(rq, p, &rf); -- } -- -- p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET); -- return 0; --} -- --static void scx_ops_enable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED)); -- -- if (SCX_HAS_OP(enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- p->scx->flags |= SCX_TASK_OPS_ENABLED; --} -- --static void scx_ops_disable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- if (p->scx->flags & SCX_TASK_OPS_PREPPED) { -- if (SCX_HAS_OP(cancel_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- } else if (p->scx->flags & SCX_TASK_OPS_ENABLED) { -- if (SCX_HAS_OP(disable)) -- SCX_CALL_OP(SCX_KF_REST, disable, p); -- p->scx->flags &= ~SCX_TASK_OPS_ENABLED; -- } --} -- --static void set_task_scx_weight(struct task_struct *p) --{ -- u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -- -- p->scx->weight = sched_weight_to_cgroup(weight); --} -- --/** -- * refresh_scx_weight - Refresh a task's ext weight -- * @p: task to refresh ext weight for -- * -- * @p->scx->weight carries the task's static priority in cgroup weight scale to -- * enable easy access from the BPF scheduler. To keep it synchronized with the -- * current task priority, this function should be called when a new task is -- * created, priority is changed for a task on sched_ext, and a task is switched -- * to sched_ext from other classes. -- */ --static void refresh_scx_weight(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- set_task_scx_weight(p); -- if (SCX_HAS_OP(set_weight)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight); --} -- --void scx_pre_fork(struct task_struct *p) --{ -- p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL); -- if (!p->scx) { -- goto lock; -- } -- -- p->scx->dsq = NULL; -- INIT_LIST_HEAD(&p->scx->dsq_node.fifo); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- INIT_LIST_HEAD(&p->scx->watchdog_node); -- p->scx->flags = 0; -- p->scx->weight = 0; -- p->scx->sticky_cpu = -1; -- p->scx->holding_cpu = -1; -- p->scx->kf_mask = 0; -- atomic64_set(&p->scx->ops_state, 0); -- p->scx->runnable_at = INITIAL_JIFFIES; -- p->scx->slice = SCX_SLICE_DFL; -- p->scx->task = p; -- p->sched_prop = 0; -- -- /* -- * BPF scheduler enable/disable paths want to be able to iterate and -- * update all tasks which can become complex when racing forks. As -- * enable/disable are very cold paths, let's use a percpu_rwsem to -- * exclude forks. -- */ --lock: -- percpu_down_read(&scx_fork_rwsem); --} -- --int scx_fork(struct task_struct *p) --{ -- percpu_rwsem_assert_held(&scx_fork_rwsem); -- -- if (scx_enabled()) -- return scx_ops_prepare_task(p, task_group(p)); -- else -- return 0; --} -- --void scx_post_fork(struct task_struct *p) --{ -- if (scx_enabled()) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- /* -- * Set the weight manually before calling ops.enable() so that -- * the scheduler doesn't see a stale value if they inspect the -- * task struct. We'll invoke ops.set_weight() afterwards, as it -- * would be odd to receive a callback on the task before we -- * tell the scheduler that it's been fully enabled. -- */ -- set_task_scx_weight(p); -- scx_ops_enable_task(p); -- refresh_scx_weight(p); -- task_rq_unlock(rq, p, &rf); -- } -- -- spin_lock_irq(&scx_tasks_lock); -- list_add_tail(&p->scx->tasks_node, &scx_tasks); -- spin_unlock_irq(&scx_tasks_lock); -- -- percpu_up_read(&scx_fork_rwsem); --} -- --void scx_cancel_fork(struct task_struct *p) --{ -- if (scx_enabled()) -- scx_ops_disable_task(p); -- percpu_up_read(&scx_fork_rwsem); --} -- --void sched_ext_free(struct task_struct *p) --{ -- unsigned long flags; -- -- spin_lock_irqsave(&scx_tasks_lock, flags); -- list_del_init(&p->scx->tasks_node); -- spin_unlock_irqrestore(&scx_tasks_lock, flags); -- -- /* -- * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED -> -- * ENABLED transitions can't race us. Disable ops for @p. -- */ -- if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- scx_ops_disable_task(p); -- task_rq_unlock(rq, p, &rf); -- } --} -- --static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) --{ --} -- --static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} --static void switched_to_scx(struct rq *rq, struct task_struct *p) {} -- --int scx_check_setscheduler(struct task_struct *p, int policy) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- /* if disallow, reject transitioning into SCX */ -- if (scx_enabled() && READ_ONCE(p->scx->disallow) && -- p->policy != policy && policy == SCHED_EXT) -- return -EACCES; -- -- return 0; --} -- --#ifdef CONFIG_NO_HZ_FULL --bool scx_can_stop_tick(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (scx_ops_disabling()) -- return false; -- -- if (p->sched_class != &ext_sched_class) -- return true; -- -- /* -- * @rq can dispatch from different DSQs, so we can't tell whether it -- * needs the tick or not by looking at nr_running. Allow stopping ticks -- * iff the BPF scheduler indicated so. See set_next_task_scx(). -- */ -- return rq->scx->flags & SCX_RQ_CAN_STOP_TICK; --} --#endif -- --static inline void scx_cgroup_lock(void) {} --static inline void scx_cgroup_unlock(void) {} -- --/* -- * Omitted operations: -- * -- * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -- * task isn't tied to the CPU at that point. Preemption is implemented by -- * resetting the victim task's slice to 0 and triggering reschedule on the -- * target CPU. -- * -- * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -- * -- * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -- * their current sched_class. Call them directly from sched core instead. -- * -- * - task_woken, switched_from: Unnecessary. -- */ --DEFINE_SCHED_CLASS(ext) = { -- .enqueue_task = enqueue_task_scx, -- .dequeue_task = dequeue_task_scx, -- .yield_task = yield_task_scx, -- .yield_to_task = yield_to_task_scx, -- -- .check_preempt_curr = check_preempt_curr_scx, -- -- .pick_next_task = pick_next_task_scx, -- -- .put_prev_task = put_prev_task_scx, -- .set_next_task = set_next_task_scx, -- --#ifdef CONFIG_SMP -- .balance = balance_scx, -- .select_task_rq = select_task_rq_scx, -- .set_cpus_allowed = set_cpus_allowed_scx, --#endif -- --#ifdef CONFIG_SCHED_CORE -- .pick_task = pick_task_scx, --#endif -- -- .task_tick = task_tick_scx, -- -- .switched_to = switched_to_scx, -- .prio_changed = prio_changed_scx, -- -- .update_curr = update_curr_scx, -- --#ifdef CONFIG_UCLAMP_TASK -- .uclamp_enabled = 0, --#endif --}; -- --static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) --{ -- memset(dsq, 0, sizeof(*dsq)); -- -- raw_spin_lock_init(&dsq->lock); -- INIT_LIST_HEAD(&dsq->fifo); -- dsq->id = dsq_id; --} -- --static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id & SCX_DSQ_FLAG_BUILTIN) -- return ERR_PTR(-EINVAL); -- -- dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node); -- if (!dsq) -- return ERR_PTR(-ENOMEM); -- -- init_dsq(dsq, dsq_id); -- -- return dsq; --} -- --static void free_dsq_irq_workfn(struct irq_work *irq_work) --{ -- struct llist_node *to_free = llist_del_all(&dsqs_to_free); -- struct scx_dispatch_q *dsq, *tmp_dsq; -- -- llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) -- kfree_rcu(dsq, rcu); --} -- --static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); -- --static void destroy_dsq(u64 dsq_id) --{ --} -- --static void scx_cgroup_exit(void) {} --static int scx_cgroup_init(void) { return 0; } --static void scx_cgroup_config_knobs(void) {} -- --/* -- * Used by sched_fork() and __setscheduler_prio() to pick the matching -- * sched_class. dl/rt are already handled. -- */ --bool task_on_scx(struct task_struct *p) --{ -- if (!scx_enabled() || scx_ops_disabling()) -- return false; -- if (READ_ONCE(scx_switching_all)) -- return true; -- return p->policy == SCHED_EXT; --} -- --static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags) --{ -- if (enq_flags & SCX_ENQ_LAST) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); --} -- --static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {} -- --static void scx_ops_disable_workfn(struct kthread_work *work) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- struct scx_task_iter sti; -- struct task_struct *p; -- const char *reason; -- int i, cpu, type; -- -- type = atomic_read(&scx_exit_type); -- while (true) { -- /* -- * NONE indicates that a new scx_ops has been registered since -- * disable was scheduled - don't kill the new ops. DONE -- * indicates that the ops has already been disabled. -- */ -- if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE) -- return; -- if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE)) -- break; -- } -- -- cancel_delayed_work_sync(&scx_watchdog_work); -- -- switch (type) { -- case SCX_EXIT_UNREG: -- reason = "BPF scheduler unregistered"; -- break; -- case SCX_EXIT_SYSRQ: -- reason = "disabled by sysrq-S"; -- break; -- case SCX_EXIT_ERROR: -- reason = "runtime error"; -- break; -- case SCX_EXIT_ERROR_BPF: -- reason = "scx_bpf_error"; -- break; -- case SCX_EXIT_ERROR_STALL: -- reason = "runnable task stall"; -- break; -- default: -- reason = ""; -- } -- -- ei->type = type; -- strlcpy(ei->reason, reason, sizeof(ei->reason)); -- -- switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { -- case SCX_OPS_DISABLED: -- pr_warn("sched_ext: ops error detected without ops (%s)\n", -- scx_exit_info.msg); -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- return; -- case SCX_OPS_PREPPING: -- goto forward_progress_guaranteed; -- case SCX_OPS_DISABLING: -- /* shouldn't happen but handle it like ENABLING if it does */ -- WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); -- fallthrough; -- case SCX_OPS_ENABLING: -- case SCX_OPS_ENABLED: -- break; -- } -- -- /* -- * DISABLING is set and ops was either ENABLING or ENABLED indicating -- * that the ops and static branches are set. -- * -- * We must guarantee that all runnable tasks make forward progress -- * without trusting the BPF scheduler. We can't grab any mutexes or -- * rwsems as they might be held by tasks that the BPF scheduler is -- * forgetting to run, which unfortunately also excludes toggling the -- * static branches. -- * -- * Let's work around by overriding a couple ops and modifying behaviors -- * based on the DISABLING state and then cycling the tasks through -- * dequeue/enqueue to force global FIFO scheduling. -- * -- * a. ops.enqueue() and .dispatch() are overridden for simple global -- * FIFO scheduling. -- * -- * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value -- * can't be trusted. Whenever a tick triggers, the running task is -- * rotated to the tail of the queue with core_sched_at touched. -- * -- * c. pick_next_task() suppresses zero slice warning. -- * -- * d. scx_prio_less() reverts to the default core_sched_at order. -- */ -- scx_ops.enqueue = scx_ops_fallback_enqueue; -- scx_ops.dispatch = scx_ops_fallback_dispatch; -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- SCHED_CHANGE_BLOCK(task_rq(p), p, -- DEQUEUE_SAVE | DEQUEUE_MOVE) { -- /* cycling deq/enq is enough, see above */ -- } -- } -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* kick all CPUs to restore ticks */ -- for_each_possible_cpu(cpu) -- resched_cpu(cpu); -- --forward_progress_guaranteed: -- /* -- * Here, every runnable task is guaranteed to make forward progress and -- * we can safely use blocking synchronization constructs. Actually -- * disable ops. -- */ -- mutex_lock(&scx_ops_enable_mutex); -- -- static_branch_disable(&__scx_switched_all); -- WRITE_ONCE(scx_switching_all, false); -- -- /* avoid racing against fork and cgroup changes */ -- cpus_read_lock(); -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- bool alive = READ_ONCE(p->__state) != TASK_DEAD; -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL); -- -- __setscheduler_prio(p, p->prio); -- } -- -- if (alive) -- check_class_changed(task_rq(p), p, old_class, p->prio); -- -- scx_ops_disable_task(p); -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* no task is on scx, turn off all the switches and flush in-progress calls */ -- static_branch_disable_cpuslocked(&__scx_ops_enabled); -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- static_branch_disable_cpuslocked(&scx_has_op[i]); -- static_branch_disable_cpuslocked(&scx_ops_enq_last); -- static_branch_disable_cpuslocked(&scx_ops_enq_exiting); -- static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- synchronize_rcu(); -- -- scx_cgroup_exit(); -- -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- cpus_read_unlock(); -- -- if (ei->type >= SCX_EXIT_ERROR) { -- printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name); -- -- if (ei->msg[0] == '\0') -- printk(KERN_ERR "sched_ext: %s\n", ei->reason); -- else -- printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg); -- -- stack_trace_print(ei->bt, ei->bt_len, 2); -- } -- -- if (scx_ops.exit) -- SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); -- -- memset(&scx_ops, 0, sizeof(scx_ops)); -- -- free_percpu(scx_dsp_buf); -- scx_dsp_buf = NULL; -- scx_dsp_max_batch = 0; -- -- mutex_unlock(&scx_ops_enable_mutex); -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- -- scx_cgroup_config_knobs(); --} -- --static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); -- --static void schedule_scx_ops_disable_work(void) --{ -- struct kthread_worker *helper = READ_ONCE(scx_ops_helper); -- -- /* -- * We may be called spuriously before the first bpf_sched_ext_reg(). If -- * scx_ops_helper isn't set up yet, there's nothing to do. -- */ -- if (helper) -- kthread_queue_work(helper, &scx_ops_disable_work); --} -- --static void scx_ops_disable(enum scx_exit_type type) --{ -- int none = SCX_EXIT_NONE; -- -- if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)) -- type = SCX_EXIT_ERROR; -- -- atomic_try_cmpxchg(&scx_exit_type, &none, type); -- -- schedule_scx_ops_disable_work(); --} -- --static void scx_ops_error_irq_workfn(struct irq_work *irq_work) --{ -- schedule_scx_ops_disable_work(); --} -- --static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- int none = SCX_EXIT_NONE; -- va_list args; -- -- if (!atomic_try_cmpxchg(&scx_exit_type, &none, type)) -- return; -- -- ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1); -- -- va_start(args, fmt); -- vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args); -- va_end(args); -- -- irq_work_queue(&scx_ops_error_irq_work); --} -- --static struct kthread_worker *scx_create_rt_helper(const char *name) --{ -- struct kthread_worker *helper; -- -- helper = kthread_create_worker(0, name); -- if (helper) -- sched_set_fifo(helper->task); -- return helper; --} -- --static int scx_ops_enable(struct sched_ext_ops *ops) --{ -- struct scx_task_iter sti; -- struct task_struct *p; -- int i, ret; -- int tcnt = 0; -- unsigned long long start = 0; -- unsigned long long total_start = sched_clock(); -- -- mutex_lock(&scx_ops_enable_mutex); -- -- if (!scx_ops_helper) { -- WRITE_ONCE(scx_ops_helper, -- scx_create_rt_helper("sched_ext_ops_helper")); -- if (!scx_ops_helper) { -- ret = -ENOMEM; -- goto err_unlock; -- } -- } -- -- if (scx_ops_enable_state() != SCX_OPS_DISABLED) { -- ret = -EBUSY; -- goto err_unlock; -- } -- -- //slim_walt_enable(true); -- -- /* -- * Set scx_ops, transition to PREPPING and clear exit info to arm the -- * disable path. Failure triggers full disabling from here on. -- */ -- scx_ops = *ops; -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != -- SCX_OPS_DISABLED); -- -- memset(&scx_exit_info, 0, sizeof(scx_exit_info)); -- atomic_set(&scx_exit_type, SCX_EXIT_NONE); -- scx_warned_zero_slice = false; -- -- atomic64_set(&scx_nr_rejected, 0); -- -- /* -- * Keep CPUs stable during enable so that the BPF scheduler can track -- * online CPUs by watching ->on/offline_cpu() after ->init(). -- */ -- cpus_read_lock(); -- -- scx_switch_all_req = true; -- if (scx_ops.init) { -- ret = SCX_CALL_OP_RET(SCX_KF_INIT, init); -- if (ret) { -- ret = ops_sanitize_err("init", ret); -- goto err_disable; -- } -- -- /* -- * Exit early if ops.init() triggered scx_bpf_error(). Not -- * strictly necessary as we'll fail transitioning into ENABLING -- * later but that'd be after calling ops.prep_enable() on all -- * tasks and with -EBUSY which isn't very intuitive. Let's exit -- * early with success so that the condition is notified through -- * ops.exit() like other scx_bpf_error() invocations. -- */ -- if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE) -- goto err_disable; -- } -- -- WARN_ON_ONCE(scx_dsp_buf); -- scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; -- scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch, -- __alignof__(scx_dsp_buf[0])); -- if (!scx_dsp_buf) { -- ret = -ENOMEM; -- goto err_disable; -- } -- -- scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; -- if (ops->timeout_ms) -- scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); -- -- scx_watchdog_timestamp = jiffies; -- queue_delayed_work(system_unbound_wq, &scx_watchdog_work, -- scx_watchdog_timeout / 2); -- -- /* -- * Lock out forks, cgroup on/offlining and moves before opening the -- * floodgate so that they don't wander into the operations prematurely. -- */ -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- if (((void (**)(void))ops)[i]) -- static_branch_enable_cpuslocked(&scx_has_op[i]); -- -- if (ops->flags & SCX_OPS_ENQ_LAST) -- static_branch_enable_cpuslocked(&scx_ops_enq_last); -- -- if (ops->flags & SCX_OPS_ENQ_EXITING) -- static_branch_enable_cpuslocked(&scx_ops_enq_exiting); -- if (scx_ops.cpu_acquire || scx_ops.cpu_release) -- static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); -- -- if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { -- reset_idle_masks(); -- static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); -- } else { -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- } -- -- /* -- * All cgroups should be initialized before letting in tasks. cgroup -- * on/offlining and task migrations are already locked out. -- */ -- ret = scx_cgroup_init(); -- if (ret) -- goto err_disable_unlock; -- -- static_branch_enable_cpuslocked(&__scx_ops_enabled); -- -- /* -- * Enable ops for every task. Fork is excluded by scx_fork_rwsem -- * preventing new tasks from being added. No need to exclude tasks -- * leaving as sched_ext_free() can handle both prepped and enabled -- * tasks. Prep all tasks first and then enable them with preemption -- * disabled. -- */ -- spin_lock_irq(&scx_tasks_lock); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered(&sti))) { -- get_task_struct(p); -- spin_unlock_irq(&scx_tasks_lock); -- -- ret = scx_ops_prepare_task(p, task_group(p)); -- if (ret) { -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n", -- ret, p->comm, p->pid); -- goto err_disable_unlock; -- } -- -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- } -- scx_task_iter_exit(&sti); -- -- /* -- * All tasks are prepped but are still ops-disabled. Ensure that -- * %current can't be scheduled out and switch everyone. -- * preempt_disable() is necessary because we can't guarantee that -- * %current won't be starved if scheduled out while switching. -- */ -- preempt_disable(); -- -- /* -- * From here on, the disable path must assume that tasks have ops -- * enabled and need to be recovered. -- */ -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { -- preempt_enable(); -- spin_unlock_irq(&scx_tasks_lock); -- ret = -EBUSY; -- goto err_disable_unlock; -- } -- -- /* -- * We're fully committed and can't fail. The PREPPED -> ENABLED -- * transitions here are synchronized against sched_ext_free() through -- * scx_tasks_lock. -- */ -- WRITE_ONCE(scx_switching_all, scx_switch_all_req); -- start = sched_clock(); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- tcnt++; -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- scx_ops_enable_task(p); -- __setscheduler_prio(p, p->prio); -- } -- -- check_class_changed(task_rq(p), p, old_class, p->prio); -- } else { -- scx_ops_disable_task(p); -- } -- } -- printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id()); -- scx_task_iter_exit(&sti); -- -- spin_unlock_irq(&scx_tasks_lock); -- preempt_enable(); -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { -- ret = -EBUSY; -- goto err_disable; -- } -- -- if (scx_switch_all_req) -- static_branch_enable_cpuslocked(&__scx_switched_all); -- -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- -- scx_cgroup_config_knobs(); -- printk("\n total duration = %llu , current cpu = %d\n", sched_clock() - total_start, smp_processor_id()); -- -- return 0; -- --err_unlock: -- mutex_unlock(&scx_ops_enable_mutex); -- return ret; -- --err_disable_unlock: -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); --err_disable: -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- /* must be fully disabled before returning */ -- scx_ops_disable(SCX_EXIT_ERROR); -- kthread_flush_work(&scx_ops_disable_work); -- return ret; --} -- --#ifdef CONFIG_SCHED_DEBUG --static const char *scx_ops_enable_state_str[] = { -- [SCX_OPS_PREPPING] = "prepping", -- [SCX_OPS_ENABLING] = "enabling", -- [SCX_OPS_ENABLED] = "enabled", -- [SCX_OPS_DISABLING] = "disabling", -- [SCX_OPS_DISABLED] = "disabled", --}; -- --static int scx_debug_show(struct seq_file *m, void *v) --{ -- mutex_lock(&scx_ops_enable_mutex); -- seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name); -- seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled()); -- seq_printf(m, "%-30s: %d\n", "switching_all", -- READ_ONCE(scx_switching_all)); -- seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all()); -- seq_printf(m, "%-30s: %s\n", "enable_state", -- scx_ops_enable_state_str[scx_ops_enable_state()]); -- seq_printf(m, "%-30s: %llu\n", "nr_rejected", -- atomic64_read(&scx_nr_rejected)); -- mutex_unlock(&scx_ops_enable_mutex); -- return 0; --} -- --static int scx_debug_open(struct inode *inode, struct file *file) --{ -- return single_open(file, scx_debug_show, NULL); --} -- --const struct file_operations sched_ext_fops = { -- .open = scx_debug_open, -- .read = seq_read, -- .llseek = seq_lseek, -- .release = single_release, --}; --#endif -- --/******************************************************************************** -- * bpf_struct_ops plumbing. -- */ --#include --#include --#include -- --extern struct btf *btf_vmlinux; --static const struct btf_type *task_struct_type; -- --static bool bpf_scx_is_valid_access(int off, int size, -- enum bpf_access_type type, -- const struct bpf_prog *prog, -- struct bpf_insn_access_aux *info) --{ -- if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) -- return false; -- if (type != BPF_READ) -- return false; -- if (off % size != 0) -- return false; -- -- return btf_ctx_access(off, size, type, prog, info); --} -- --static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, -- const struct bpf_reg_state *reg, -- int off, int size) --{ -- return 0; --} -- --static const struct bpf_func_proto * --bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) --{ -- switch (func_id) { -- case BPF_FUNC_task_storage_get: -- return &bpf_task_storage_get_proto; -- case BPF_FUNC_task_storage_delete: -- return &bpf_task_storage_delete_proto; -- default: -- return bpf_base_func_proto(func_id); -- } --} -- --const struct bpf_verifier_ops bpf_scx_verifier_ops = { -- .get_func_proto = bpf_scx_get_func_proto, -- .is_valid_access = bpf_scx_is_valid_access, -- .btf_struct_access = bpf_scx_btf_struct_access, --}; -- --static int bpf_scx_init_member(const struct btf_type *t, -- const struct btf_member *member, -- void *kdata, const void *udata) --{ -- const struct sched_ext_ops *uops = udata; -- struct sched_ext_ops *ops = kdata; -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- int ret; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, dispatch_max_batch): -- if (*(u32 *)(udata + moff) > INT_MAX) -- return -E2BIG; -- ops->dispatch_max_batch = *(u32 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, flags): -- if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) -- return -EINVAL; -- ops->flags = *(u64 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, name): -- ret = bpf_obj_name_cpy(ops->name, uops->name, -- sizeof(ops->name)); -- if (ret < 0) -- return ret; -- if (ret == 0) -- return -EINVAL; -- return 1; -- case offsetof(struct sched_ext_ops, timeout_ms): -- if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT) -- return -E2BIG; -- ops->timeout_ms = *(u32 *)(udata + moff); -- return 1; -- } -- -- return 0; --} -- --static int bpf_scx_check_member(const struct btf_type *t, -- const struct btf_member *member, -- const struct bpf_prog *prog) --{ -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, prep_enable): -- case offsetof(struct sched_ext_ops, init): -- case offsetof(struct sched_ext_ops, exit): -- break; -- default: -- /* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */ -- /* if (prog->aux->sleepable) -- return -EINVAL; */ -- break; -- } -- -- return 0; --} -- --static int bpf_scx_reg(void *kdata) --{ -- return scx_ops_enable(kdata); --} -- --static void bpf_scx_unreg(void *kdata) --{ -- scx_ops_disable(SCX_EXIT_UNREG); -- kthread_flush_work(&scx_ops_disable_work); --} -- --static int bpf_scx_init(struct btf *btf) --{ -- u32 type_id; -- -- type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); -- if (type_id < 0) -- return -EINVAL; -- task_struct_type = btf_type_by_id(btf, type_id); -- -- return 0; --} -- --/* "extern" to avoid sparse warning, only used in this file */ --extern struct bpf_struct_ops bpf_sched_ext_ops; -- --struct bpf_struct_ops bpf_sched_ext_ops = { -- .verifier_ops = &bpf_scx_verifier_ops, -- .reg = bpf_scx_reg, -- .unreg = bpf_scx_unreg, -- .check_member = bpf_scx_check_member, -- .init_member = bpf_scx_init_member, -- .init = bpf_scx_init, -- .name = "sched_ext_ops", --}; -- --static void sysrq_handle_sched_ext_reset(u8 key) --{ -- if (scx_ops_helper) -- scx_ops_disable(SCX_EXIT_SYSRQ); -- else -- pr_info("sched_ext: BPF scheduler not yet used\n"); --} -- --static const struct sysrq_key_op sysrq_sched_ext_reset_op = { -- .handler = sysrq_handle_sched_ext_reset, -- .help_msg = "reset-sched-ext(S)", -- .action_msg = "Disable sched_ext and revert all tasks to CFS", -- .enable_mask = SYSRQ_ENABLE_RTNICE, --}; -- --static void kick_cpus_irq_workfn(struct irq_work *irq_work) --{ -- struct rq *this_rq = this_rq(); -- u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); -- int this_cpu = cpu_of(this_rq); -- int cpu; -- -- for_each_cpu(cpu, this_rq->scx->cpus_to_kick) { -- struct rq *rq = cpu_rq(cpu); -- unsigned long flags; -- -- raw_spin_rq_lock_irqsave(rq, flags); -- -- if (cpu_online(cpu) || cpu == this_cpu) { -- if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) && -- rq->curr->sched_class == &ext_sched_class) -- rq->curr->scx->slice = 0; -- pseqs[cpu] = rq->scx->pnt_seq; -- resched_curr(rq); -- } else { -- cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait); -- } -- -- raw_spin_rq_unlock_irqrestore(rq, flags); -- } -- -- for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait, -- cpumask_of(this_cpu)) { -- /* -- * Pairs with smp_store_release() issued by this CPU in -- * scx_notify_pick_next_task() on the resched path. -- * -- * We busy-wait here to guarantee that no other task can be -- * scheduled on our core before the target CPU has entered the -- * resched path. -- */ -- while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu]) -- cpu_relax(); -- } -- -- cpumask_clear(this_rq->scx->cpus_to_kick); -- cpumask_clear(this_rq->scx->cpus_to_preempt); -- cpumask_clear(this_rq->scx->cpus_to_wait); --} -- --void __init init_sched_ext_class(void) --{ -- int cpu; -- u32 v; -- -- /* -- * The following is to prevent the compiler from optimizing out the enum -- * definitions so that BPF scheduler implementations can use them -- * through the generated vmlinux.h. -- */ -- WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | -- SCX_TG_ONLINE | SCX_KICK_PREEMPT); -- -- init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); --#ifdef CONFIG_SMP -- BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -- BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); --#endif -- scx_kick_cpus_pnt_seqs = -- __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * -- num_possible_cpus(), -- __alignof__(scx_kick_cpus_pnt_seqs[0])); -- BUG_ON(!scx_kick_cpus_pnt_seqs); -- -- for_each_possible_cpu(cpu) { -- struct rq *rq = cpu_rq(cpu); -- -- rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL); -- if (rq->scx) -- rq->scx->rq = rq; -- else -- pr_err("fatal error : alloc rq->scx failed!!!\n"); -- -- init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL); -- INIT_LIST_HEAD(&rq->scx->watchdog_list); -- -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL)); -- init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn); -- } -- -- register_sysrq_key('S', &sysrq_sched_ext_reset_op); -- INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); -- scx_cgroup_config_knobs(); --} -- -- --/******************************************************************************** -- * Helpers that can be called from the BPF scheduler. -- */ --#include -- --/* Disables missing prototype warnings for kfuncs */ --__diag_push(); --__diag_ignore_all("-Wmissing-prototypes", -- "Global functions as their definitions will be in vmlinux BTF"); -- --/** -- * scx_bpf_switch_all - Switch all tasks into SCX -- * @into_scx: switch direction -- * -- * If @into_scx is %true, all existing and future non-dl/rt tasks are switched -- * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on -- * SCX. The actual switching is asynchronous. Can be called from ops.init(). -- */ --void scx_bpf_switch_all(void) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return; -- -- scx_switch_all_req = true; --} -- --BTF_SET8_START(scx_kfunc_ids_init) --BTF_ID_FLAGS(func, scx_bpf_switch_all) --BTF_SET8_END(scx_kfunc_ids_init) -- --static const struct btf_kfunc_id_set scx_kfunc_set_init = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_init, --}; -- --/** -- * scx_bpf_create_dsq - Create a custom DSQ -- * @dsq_id: DSQ to create -- * @node: NUMA node to allocate from -- * -- * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(), -- * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move(). -- */ --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return -EINVAL; -- -- if (unlikely(node >= (int)nr_node_ids || -- (node < 0 && node != NUMA_NO_NODE))) -- return -EINVAL; -- return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); --} -- --BTF_SET8_START(scx_kfunc_ids_sleepable) --BTF_ID_FLAGS(func, scx_bpf_create_dsq) --BTF_SET8_END(scx_kfunc_ids_sleepable) -- --static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_sleepable, --}; -- --static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) --{ -- if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) -- return false; -- -- lockdep_assert_irqs_disabled(); -- -- if (unlikely(!p)) { -- scx_ops_error("called with NULL task"); -- return false; -- } -- -- if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { -- scx_ops_error("invalid enq_flags 0x%llx", enq_flags); -- return false; -- } -- -- return true; --} -- --static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) --{ -- struct task_struct *ddsp_task; -- int idx; -- -- ddsp_task = __this_cpu_read(direct_dispatch_task); -- if (ddsp_task) { -- direct_dispatch(ddsp_task, p, dsq_id, enq_flags); -- return; -- } -- -- idx = __this_cpu_read(scx_dsp_ctx.buf_cursor); -- if (unlikely(idx >= scx_dsp_max_batch)) { -- scx_ops_error("dispatch buffer overflow"); -- return; -- } -- -- this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){ -- .task = p, -- .qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK, -- .dsq_id = dsq_id, -- .enq_flags = enq_flags, -- }; -- __this_cpu_inc(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe -- * to call this function spuriously. Can be called from ops.enqueue() and -- * ops.dispatch(). -- * -- * When called from ops.enqueue(), it's for direct dispatch and @p must match -- * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the -- * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be -- * on the target CPU in the first place. -- * -- * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id -- * and this function can be called upto ops.dispatch_max_batch times to dispatch -- * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the -- * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. -- * -- * This function doesn't have any locking restrictions and may be called under -- * BPF locks (in the future when BPF introduces more flexible locking). -- * -- * @p is allowed to run for @slice. The scheduling path is triggered on slice -- * exhaustion. If zero, the current residual slice is maintained. If -- * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with -- * scx_bpf_kick_cpu() to trigger scheduling. -- */ --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- scx_dispatch_commit(p, dsq_id, enq_flags); --} -- --/** -- * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. -- * Tasks queued into the priority queue are ordered by @vtime and always -- * consumed after the tasks in the FIFO queue. All other aspects are identical -- * to scx_bpf_dispatch(). -- * -- * @vtime ordering is according to time_before64() which considers wrapping. A -- * numerically larger vtime may indicate an earlier position in the ordering and -- * vice-versa. -- */ --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 vtime, u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- p->scx->dsq_vtime = vtime; -- -- scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); --} -- --BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime) --BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_enqueue_dispatch, --}; -- --/** -- * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots -- * -- * Can only be called from ops.dispatch(). -- */ --u32 scx_bpf_dispatch_nr_slots(void) --{ -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return 0; -- -- return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ -- * @dsq_id: DSQ to consume -- * -- * Consume a task from the non-local DSQ identified by @dsq_id and transfer it -- * to the current CPU's local DSQ for execution. Can only be called from -- * ops.dispatch(). -- * -- * This function flushes the in-flight dispatches from scx_bpf_dispatch() before -- * trying to consume the specified DSQ. It may also grab rq locks and thus can't -- * be called under any BPF locks. -- * -- * Returns %true if a task has been consumed, %false if there isn't any task to -- * consume. -- */ --bool scx_bpf_consume(u64 dsq_id) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- struct scx_dispatch_q *dsq; -- -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return false; -- -- flush_dispatch_buf(dspc->rq, dspc->rf); -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); -- return false; -- } -- -- if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) { -- /* -- * A successfully consumed task can be dequeued before it starts -- * running while the CPU is trying to migrate other dispatched -- * tasks. Bump nr_tasks to tell balance_scx() to retry on empty -- * local DSQ. -- */ -- dspc->nr_tasks++; -- return true; -- } else { -- return false; -- } --} -- --BTF_SET8_START(scx_kfunc_ids_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) --BTF_ID_FLAGS(func, scx_bpf_consume) --BTF_SET8_END(scx_kfunc_ids_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_dispatch, --}; -- --/** -- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ -- * -- * Iterate over all of the tasks currently enqueued on the local DSQ of the -- * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of -- * processed tasks. Can only be called from ops.cpu_release(). -- */ --u32 scx_bpf_reenqueue_local(void) --{ -- u32 nr_enqueued, i; -- struct rq *rq; -- struct scx_rq *scx_rq; -- -- if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) -- return 0; -- -- rq = cpu_rq(smp_processor_id()); -- lockdep_assert_rq_held(rq); -- scx_rq = rq->scx; -- -- /* -- * Get the number of tasks on the local DSQ before iterating over it to -- * pull off tasks. The enqueue callback below can signal that it wants -- * the task to stay on the local DSQ, and we want to prevent the BPF -- * scheduler from causing us to loop indefinitely. -- */ -- nr_enqueued = scx_rq->local_dsq.nr; -- for (i = 0; i < nr_enqueued; i++) { -- struct task_struct *p; -- -- p = first_local_task(rq); -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- WARN_ON_ONCE(p->scx->holding_cpu != -1); -- dispatch_dequeue(scx_rq, p); -- do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); -- } -- -- return nr_enqueued; --} -- --BTF_SET8_START(scx_kfunc_ids_cpu_release) --BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) --BTF_SET8_END(scx_kfunc_ids_cpu_release) -- --static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_cpu_release, --}; -- --/** -- * scx_bpf_kick_cpu - Trigger reschedule on a CPU -- * @cpu: cpu to kick -- * @flags: SCX_KICK_* flags -- * -- * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or -- * trigger rescheduling on a busy CPU. This can be called from any online -- * scx_ops operation and the actual kicking is performed asynchronously through -- * an irq work. -- */ --void scx_bpf_kick_cpu(s32 cpu, u64 flags) --{ -- struct rq *rq; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d", cpu); -- return; -- } -- -- preempt_disable(); -- rq = this_rq(); -- -- /* -- * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting -- * rq locks. We can probably be smarter and avoid bouncing if called -- * from ops which don't hold a rq lock. -- */ -- cpumask_set_cpu(cpu, rq->scx->cpus_to_kick); -- if (flags & SCX_KICK_PREEMPT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt); -- if (flags & SCX_KICK_WAIT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_wait); -- -- irq_work_queue(&rq->scx->kick_cpus_irq_work); -- preempt_enable(); --} -- --/** -- * scx_bpf_dsq_nr_queued - Return the number of queued tasks -- * @dsq_id: id of the DSQ -- * -- * Return the number of tasks in the DSQ matching @dsq_id. If not found, -- * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops -- * operations. -- */ --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) --{ -- struct scx_dispatch_q *dsq; -- -- lockdep_assert(rcu_read_lock_any_held()); -- -- if (dsq_id == SCX_DSQ_LOCAL) { -- return this_rq()->scx->local_dsq.nr; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (ops_cpu_valid(cpu)) -- return cpu_rq(cpu)->scx->local_dsq.nr; -- } else { -- dsq = find_non_local_dsq(dsq_id); -- if (dsq) -- return dsq->nr; -- } -- return -ENOENT; --} -- --/** -- * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state -- * @cpu: cpu to test and clear idle for -- * -- * Returns %true if @cpu was idle and its idle state was successfully cleared. -- * %false otherwise. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return false; -- } -- -- if (ops_cpu_valid(cpu)) -- return test_and_clear_cpu_idle(cpu); -- else -- return false; --} -- --/** -- * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu -- * @cpus_allowed: Allowed cpumask -- * -- * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked -- * idle cpu number on success. -%EBUSY if no matching cpu was found. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return -EBUSY; -- } -- -- return scx_pick_idle_cpu(cpus_allowed); --} -- --/** -- * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking -- * per-CPU cpumask. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_cpumask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.cpu; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, -- * per-physical-core cpumask. Can be used to determine if an entire physical -- * core is free. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_smtmask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.smt; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to -- * either the percpu, or SMT idle-tracking cpumask. -- */ --void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) --{ -- /* -- * Empty function body because we aren't actually acquiring or -- * releasing a reference to a global idle cpumask, which is read-only -- * in the caller and is never released. The acquire / release semantics -- * here are just used to make the cpumask is a trusted pointer in the -- * caller. -- */ --} -- --struct scx_bpf_error_bstr_bufs { -- u64 data[MAX_BPRINTF_VARARGS]; -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs); -- --/** -- * scx_bpf_error_bstr - Indicate fatal error -- * @fmt: error message format string -- * @data: format string parameters packaged using ___bpf_fill() macro -- * @data__sz: @data len, must end in '__sz' for the verifier -- * -- * Indicate that the BPF scheduler encountered a fatal error and initiate ops -- * disabling. -- */ --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) --{ -- struct scx_bpf_error_bstr_bufs *bufs; -- unsigned long flags; -- struct bpf_bprintf_data args = {}; -- int ret; -- -- local_irq_save(flags); -- bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs); -- -- if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || -- (data__sz && !data)) { -- scx_ops_error("invalid data=%p and data__sz=%u", -- (void *)data, data__sz); -- goto out_restore; -- } -- -- ret = copy_from_kernel_nofault(bufs->data, data, data__sz); -- if (ret) { -- scx_ops_error("failed to read data fields (%d)", ret); -- goto out_restore; -- } -- -- ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args); -- if (ret < 0) { -- scx_ops_error("failed to format prepration (%d)", ret); -- goto out_restore; -- } -- -- ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt, -- args.bin_args); -- bpf_bprintf_cleanup(&args); -- if (ret < 0) { -- scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format", -- fmt, data, data__sz); -- goto out_restore; -- } -- -- scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg); --out_restore: -- local_irq_restore(flags); --} -- --/** -- * scx_bpf_destroy_dsq - Destroy a custom DSQ -- * @dsq_id: DSQ to destroy -- * -- * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with -- * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is -- * empty and no further tasks are dispatched to it. Ignored if called on a DSQ -- * which doesn't exist. Can be called from any online scx_ops operations. -- */ --void scx_bpf_destroy_dsq(u64 dsq_id) --{ -- destroy_dsq(dsq_id); --} -- --/** -- * scx_bpf_task_running - Is task currently running? -- * @p: task of interest -- */ --bool scx_bpf_task_running(const struct task_struct *p) --{ -- return task_rq(p)->curr == p; --} -- --/** -- * scx_bpf_task_cpu - CPU a task is currently associated with -- * @p: task of interest -- */ --s32 scx_bpf_task_cpu(const struct task_struct *p) --{ -- return task_cpu(p); --} -- --/** -- * scx_bpf_task_cgroup - Return the sched cgroup of a task -- * @p: task of interest -- * -- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with -- * from the scheduler's POV. SCX operations should use this function to -- * determine @p's current cgroup as, unlike following @p->cgroups, -- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all -- * rq-locked operations. Can be called on the parameter tasks of rq-locked -- * operations. The restriction guarantees that @p's rq is locked by the caller. -- */ --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) --{ -- struct task_group *tg = p->sched_task_group; -- struct cgroup *cgrp = &cgrp_dfl_root.cgrp; -- -- if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) -- goto out; -- -- /* -- * A task_group may either be a cgroup or an autogroup. In the latter -- * case, @tg->css.cgroup is %NULL. A task_group can't become the other -- * kind once created. -- */ -- if (tg && tg->css.cgroup) -- cgrp = tg->css.cgroup; -- else -- cgrp = &cgrp_dfl_root.cgrp; --out: -- cgroup_get(cgrp); -- return cgrp; --} -- --BTF_SET8_START(scx_kfunc_ids_any) --BTF_ID_FLAGS(func, scx_bpf_kick_cpu) --BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) --BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) --BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu) --BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) --BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) --BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) --BTF_ID_FLAGS(func, scx_bpf_task_running) --BTF_ID_FLAGS(func, scx_bpf_task_cpu) --BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE) --BTF_SET8_END(scx_kfunc_ids_any) -- --static const struct btf_kfunc_id_set scx_kfunc_set_any = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_any, --}; -- --__diag_pop(); -- --/* -- * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set() -- * needs most of the system to be up. -- */ --static int __init register_ext_kfuncs(void) --{ -- int ret; -- -- /* -- * Some kfuncs are context-sensitive and can only be called from -- * specific SCX ops. They are grouped into BTF sets accordingly. -- * Unfortunately, BPF currently doesn't have a way of enforcing such -- * restrictions. Eventually, the verifier should be able to enforce -- * them. For now, register them the same and make each kfunc explicitly -- * check using scx_kf_allowed(). -- */ -- if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_init)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_sleepable)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_enqueue_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_cpu_release)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_any))) { -- pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret); -- return ret; -- } -- -- return 0; --} --__initcall(register_ext_kfuncs); -diff --git b/kernel/sched/ext.h a/kernel/sched/ext.h -deleted file mode 100755 -index fba8ed845f99..000000000000 ---- b/kernel/sched/ext.h -+++ /dev/null -@@ -1,257 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ -- -- --/* -- * Tag marking a kernel function as a kfunc. This is meant to minimize the -- * amount of copy-paste that kfunc authors have to include for correctness so -- * as to avoid issues such as the compiler inlining or eliding either a static -- * kfunc, or a global kfunc in an LTO build. -- */ --#define __bpf_kfunc __used noinline -- --enum scx_wake_flags { -- /* expose select WF_* flags as enums */ -- SCX_WAKE_EXEC = WF_EXEC, -- SCX_WAKE_FORK = WF_FORK, -- SCX_WAKE_TTWU = WF_TTWU, -- SCX_WAKE_SYNC = WF_SYNC, --}; -- --enum scx_enq_flags { -- /* expose select ENQUEUE_* flags as enums */ -- SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, -- SCX_ENQ_HEAD = ENQUEUE_HEAD, -- -- /* high 32bits are SCX specific */ -- -- /* -- * Set the following to trigger preemption when calling -- * scx_bpf_dispatch() with a local dsq as the target. The slice of the -- * current task is cleared to zero and the CPU is kicked into the -- * scheduling path. Implies %SCX_ENQ_HEAD. -- */ -- SCX_ENQ_PREEMPT = 1LLU << 32, -- -- /* -- * The task being enqueued was previously enqueued on the current CPU's -- * %SCX_DSQ_LOCAL, but was removed from it in a call to the -- * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was -- * invoked in a ->cpu_release() callback, and the task is again -- * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the -- * task will not be scheduled on the CPU until at least the next invocation -- * of the ->cpu_acquire() callback. -- */ -- SCX_ENQ_REENQ = 1LLU << 40, -- -- /* -- * The task being enqueued is the only task available for the cpu. By -- * default, ext core keeps executing such tasks but when -- * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with -- * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set. -- * -- * If the BPF scheduler wants to continue executing the task, -- * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately. -- * If the task gets queued on a different dsq or the BPF side, the BPF -- * scheduler is responsible for triggering a follow-up scheduling event. -- * Otherwise, Execution may stall. -- */ -- SCX_ENQ_LAST = 1LLU << 41, -- -- /* -- * A hint indicating that it's advisable to enqueue the task on the -- * local dsq of the currently selected CPU. Currently used by -- * select_cpu_dfl() and together with %SCX_ENQ_LAST. -- */ -- SCX_ENQ_LOCAL = 1LLU << 42, -- -- /* high 8 bits are internal */ -- __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, -- -- SCX_ENQ_CLEAR_OPSS = 1LLU << 56, -- SCX_ENQ_DSQ_PRIQ = 1LLU << 57, --}; -- --enum scx_deq_flags { -- /* expose select DEQUEUE_* flags as enums */ -- SCX_DEQ_SLEEP = DEQUEUE_SLEEP, -- -- /* high 32bits are SCX specific */ -- -- /* -- * The generic core-sched layer decided to execute the task even though -- * it hasn't been dispatched yet. Dequeue from the BPF side. -- */ -- SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, --}; -- --enum scx_tg_flags { -- SCX_TG_ONLINE = 1U << 0, -- SCX_TG_INITED = 1U << 1, --}; -- --enum scx_kick_flags { -- SCX_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -- SCX_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ --}; -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --extern const struct sched_class ext_sched_class; --extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops; --extern const struct file_operations sched_ext_fops; --extern unsigned long scx_watchdog_timeout; --extern unsigned long scx_watchdog_timestamp; -- --DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); --DECLARE_STATIC_KEY_FALSE(__scx_switched_all); --#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) --#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) -- --DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); -- --bool task_on_scx(struct task_struct *p); --void scx_pre_fork(struct task_struct *p); --int scx_fork(struct task_struct *p); --void scx_post_fork(struct task_struct *p); --void scx_cancel_fork(struct task_struct *p); --int scx_check_setscheduler(struct task_struct *p, int policy); --bool scx_can_stop_tick(struct rq *rq); --void init_sched_ext_class(void); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...); --#define scx_ops_error(fmt, args...) \ -- scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args) -- --void __scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active); -- --static inline void scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active) --{ -- if (!scx_enabled()) -- return; --#ifdef CONFIG_SMP -- /* -- * Pairs with the smp_load_acquire() issued by a CPU in -- * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -- * resched. -- */ -- smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1); --#endif -- if (!static_branch_unlikely(&scx_ops_cpu_preempt)) -- return; -- __scx_notify_pick_next_task(rq, p, active); --} -- --//extern void scx_scheduler_tick(void); --static inline void scx_notify_sched_tick(void) --{ -- unsigned long last_check; -- -- if (!scx_enabled()) -- return; -- //scx_scheduler_tick(); -- -- last_check = scx_watchdog_timestamp; -- if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "watchdog failed to check in for %u.%03us", -- dur_ms / 1000, dur_ms % 1000); -- } --} -- --static inline const struct sched_class *next_active_class(const struct sched_class *class) --{ -- class++; -- if (scx_switched_all() && class == &fair_sched_class) -- class++; -- if (!scx_enabled() && class == &ext_sched_class) -- class++; -- return class; --} -- --#define for_active_class_range(class, _from, _to) \ -- for (class = (_from); class != (_to); class = next_active_class(class)) -- --#define for_each_active_class(class) \ -- for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -- --/* -- * SCX requires a balance() call before every pick_next_task() call including -- * when waking up from idle. -- */ --#define for_balance_class_range(class, prev_class, end_class) \ -- for_active_class_range(class, (prev_class) > &ext_sched_class ? \ -- &ext_sched_class : (prev_class), (end_class)) -- --#ifdef CONFIG_SCHED_CORE --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi); --#endif -- --#else /* CONFIG_SCHED_CLASS_EXT */ -- --#define scx_enabled() false --#define scx_switched_all() false -- --static inline void scx_pre_fork(struct task_struct *p) {} --static inline int scx_fork(struct task_struct *p) { return 0; } --static inline void scx_post_fork(struct task_struct *p) {} --static inline void scx_cancel_fork(struct task_struct *p) {} --static inline int scx_check_setscheduler(struct task_struct *p, -- int policy) { return 0; } --static inline bool scx_can_stop_tick(struct rq *rq) { return true; } --static inline void init_sched_ext_class(void) {} --static inline void scx_notify_pick_next_task(struct rq *rq, -- const struct task_struct *p, -- const struct sched_class *active) {} --static inline void scx_notify_sched_tick(void) {} -- --#define for_each_active_class for_each_class --#define for_balance_class_range for_class_range -- --#endif /* CONFIG_SCHED_CLASS_EXT */ -- --#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) --void __scx_update_idle(struct rq *rq, bool idle); -- --static inline void scx_update_idle(struct rq *rq, bool idle) --{ -- if (scx_enabled()) -- __scx_update_idle(rq, idle); --} --#else --static inline void scx_update_idle(struct rq *rq, bool idle) {} --#endif -- --#ifdef CONFIG_CGROUP_SCHED --#ifdef CONFIG_EXT_GROUP_SCHED --int scx_tg_online(struct task_group *tg); --void scx_tg_offline(struct task_group *tg); --int scx_cgroup_can_attach(struct cgroup_taskset *tset); --void scx_move_task(struct task_struct *p); --void scx_cgroup_finish_attach(void); --void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); --void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); --#else /* CONFIG_EXT_GROUP_SCHED */ --static inline int scx_tg_online(struct task_group *tg) { return 0; } --static inline void scx_tg_offline(struct task_group *tg) {} --static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } --static inline void scx_move_task(struct task_struct *p) {} --static inline void scx_cgroup_finish_attach(void) {} --static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} --static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} --#endif /* CONFIG_EXT_GROUP_SCHED */ --#endif /* CONFIG_CGROUP_SCHED */ -diff --git b/kernel/sched/hmbird.h a/kernel/sched/hmbird.h -new file mode 100755 -index 000000000000..fa76c7f09977 ---- /dev/null -+++ a/kernel/sched/hmbird.h -@@ -0,0 +1,342 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#ifndef _HMBIRD_H_ -+#define _HMBIRD_H_ -+ -+/* -+ * Tag marking a kernel function as a kfunc. This is meant to minimize the -+ * amount of copy-paste that kfunc authors have to include for correctness so -+ * as to avoid issues such as the compiler inlining or eliding either a static -+ * kfunc, or a global kfunc in an LTO build. -+ */ -+ -+#define DEBUG_INTERNAL (1 << 0) -+#define DEBUG_INFO_TRACE (1 << 1) -+#define DEBUG_INFO_SYSTRACE (1 << 2) -+ -+#define NUMS_CGROUP_KINDS (512) -+#define SLIM_FOR_SGAME (1 << 0) -+#define SLIM_FOR_GENSHIN (1 << 1) -+ -+#ifdef MAX_TASK_NR -+#define MAX_KEY_THREAD_RECORD ((MAX_TASK_NR + 1) >> 1) -+#else -+#define MAX_KEY_THREAD_RECORD (1 << 3) -+#endif /* MAX_TASK_NR */ -+ -+#define TOP_TASK_SHIFT (8) -+#define TOP_TASK_MAX (1 << TOP_TASK_SHIFT) -+#ifndef TOP_TASK_BITS_MASK -+#define TOP_TASK_BITS_MASK (TOP_TASK_MAX - 1) -+#endif /* TOP_TASK_BITS_MASK */ -+ -+extern struct md_info_t *md_info; -+ -+#define debug_enabled() \ -+ (unlikely(hmbirdcore_debug)) -+ -+#define hmbird_debug(fmt, ...) \ -+ pr_info(":"fmt, ##__VA_ARGS__) -+ -+#define hmbird_err(id, fmt, ...) \ -+do { \ -+ pr_err(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_deferred_err(id, fmt, ...) \ -+do { \ -+ printk_deferred(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \ -+do { \ -+ if (unlikely(cond)) { \ -+ hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+ } \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_info_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_info_trace(fmt, ...) -+#endif -+ -+#define hmbird_info_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+ } \ -+} while (0) -+ -+#define hmbird_output_systrace(fmt, ...) \ -+do { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_internal_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_internal_trace(fmt, ...) -+#endif -+ -+#define hmbird_internal_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) { \ -+ hmbird_output_systrace(fmt, ##__VA_ARGS__); \ -+ } \ -+} while (0) -+ -+enum hmbird_wake_flags { -+ /* expose select WF_* flags as enums */ -+ HMBIRD_WAKE_EXEC = WF_EXEC, -+ HMBIRD_WAKE_FORK = WF_FORK, -+ HMBIRD_WAKE_TTWU = WF_TTWU, -+ HMBIRD_WAKE_SYNC = WF_SYNC, -+}; -+ -+enum hmbird_enq_flags { -+ /* expose select ENQUEUE_* flags as enums */ -+ HMBIRD_ENQ_WAKEUP = ENQUEUE_WAKEUP, -+ HMBIRD_ENQ_HEAD = ENQUEUE_HEAD, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * Set the following to trigger preemption when calling -+ * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the -+ * current task is cleared to zero and the CPU is kicked into the -+ * scheduling path. Implies %HMBIRD_ENQ_HEAD. -+ */ -+ HMBIRD_ENQ_PREEMPT = 1LLU << 32, -+ HMBIRD_ENQ_REENQ = 1LLU << 40, -+ HMBIRD_ENQ_LAST = 1LLU << 41, -+ -+ /* -+ * A hint indicating that it's advisable to enqueue the task on the -+ * local dsq of the currently selected CPU. Currently used by -+ * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST. -+ */ -+ HMBIRD_ENQ_LOCAL = 1LLU << 42, -+ -+ /* high 8 bits are internal */ -+ __HMBIRD_ENQ_INTERNAL_MASK = 0xffLLU << 56, -+ -+ HMBIRD_ENQ_CLEAR_OPSS = 1LLU << 56, -+ HMBIRD_ENQ_DSQ_PRIQ = 1LLU << 57, -+}; -+ -+enum hmbird_deq_flags { -+ /* expose select DEQUEUE_* flags as enums */ -+ HMBIRD_DEQ_SLEEP = DEQUEUE_SLEEP, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * The generic core-sched layer decided to execute the task even though -+ * it hasn't been dispatched yet. Dequeue from the BPF side. -+ */ -+ HMBIRD_DEQ_CORE_SCHED_EXEC = 1LLU << 32, -+}; -+ -+enum hmbird_kick_flags { -+ HMBIRD_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -+ HMBIRD_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ -+}; -+ -+enum hmbird_task_prop_type { -+ HMBIRD_TASK_PROP_COMMON, -+ HMBIRD_TASK_PROP_PIPELINE, -+ HMBIRD_TASK_PROP_DEBUG_OR_LOG, -+ HMBIRD_TASK_PROP_HIGH_LOAD, -+ HMBIRD_TASK_PROP_IO, -+ HMBIRD_TASK_PROP_NETWORK, -+ HMBIRD_TASK_PROP_PERIODIC, -+ HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL, -+ HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL, -+ HMBIRD_TASK_PROP_ISOLATE, -+ HMBIRD_TASK_PROP_MAX, -+}; -+ -+enum hmbird_preempt_policy_id { -+ HMBIRD_PREEMPT_POLICY_NONE, -+ HMBIRD_PREEMPT_POLICY_PRIO_BASED, -+}; -+ -+extern const struct sched_class hmbird_sched_class; -+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops; -+extern const struct file_operations sched_hmbird_fops; -+extern unsigned long hmbird_watchdog_timeout; -+extern unsigned long hmbird_watchdog_timestamp; -+ -+enum scx_rq_flags { -+ HMBIRD_RQ_CAN_STOP_TICK = 1 << 0, -+}; -+ -+struct hmbird_rq { -+ struct hmbird_dispatch_q local_dsq; -+ struct list_head watchdog_list; -+ u64 ops_qseq; -+ u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -+ u32 nr_running; -+ u32 flags; -+ bool cpu_released; -+ cpumask_var_t cpus_to_kick; -+ cpumask_var_t cpus_to_preempt; -+ cpumask_var_t cpus_to_wait; -+ u64 pnt_seq; -+ u64 *prev_runnable_sum_fixed; -+ u32 *prev_window_size; -+ struct irq_work kick_cpus_irq_work; -+ bool pipeline; -+ bool exclusive; -+ bool period_disallow; -+ bool nonperiod_disallow; -+ struct rq *rq; -+ struct hmbird_sched_rq_stats *srq; -+}; -+ -+struct hmbird_sched_change_guard { -+ struct task_struct *p; -+ struct rq *rq; -+ bool queued; -+ bool running; -+ bool done; -+}; -+ -+extern struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -+ -+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags); -+ -+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -+ for (struct hmbird_sched_change_guard __cg = \ -+ hmbird_sched_change_guard_init(__rq, __p, __flags); \ -+ !__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags)) -+ -+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+bool task_on_hmbird(struct task_struct *p); -+int hmbird_pre_fork(struct task_struct *p); -+void hmbird_post_fork(struct task_struct *p); -+void hmbird_cancel_fork(struct task_struct *p); -+int hmbird_check_setscheduler(struct task_struct *p, int policy); -+bool hmbird_can_stop_tick(struct rq *rq); -+void init_sched_hmbird_class(void); -+ -+void hmbird_ops_exit(void); -+#define hmbird_ops_error(fmt, ...) \ -+do { \ -+ hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__); \ -+ hmbird_ops_exit(); \ -+} while (0) -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active); -+ -+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight) -+{ -+ return clamp_t(unsigned long, -+ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -+ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); -+} -+ -+static inline void hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active) -+{ -+ if (!hmbird_enabled()) -+ return; -+#ifdef CONFIG_SMP -+ /* -+ * Pairs with the smp_load_acquire() issued by a CPU in -+ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -+ * resched. -+ */ -+ smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1); -+#endif -+ if (!static_branch_unlikely(&hmbird_ops_cpu_preempt)) -+ return; -+ __hmbird_notify_pick_next_task(rq, p, active); -+} -+ -+extern void hmbird_scheduler_tick(void); -+void scan_timeout(struct rq *rq); -+void hmbird_notify_sched_tick(void); -+ -+static inline const struct sched_class *next_active_class(const struct sched_class *class) -+{ -+ class++; -+ -+ if (hmbird_enabled() && class == &fair_sched_class) -+ class++; -+ if (!hmbird_enabled() && class == &hmbird_sched_class) -+ class++; -+ return class; -+} -+ -+#define for_active_class_range(class, _from, _to) \ -+ for (class = (_from); class != (_to); class = next_active_class(class)) -+ -+#define for_each_active_class(class) \ -+ for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -+ -+/* -+ * HMBIRD requires a balance() call before every pick_next_task() call including -+ * when waking up from idle. -+ */ -+#define for_balance_class_range(class, prev_class, end_class) \ -+ for_active_class_range(class, (prev_class) > &hmbird_sched_class ? \ -+ &hmbird_sched_class : (prev_class), (end_class)) -+ -+#define MIN_CGROUP_DL_IDX (5) /* 8ms */ -+#define DEFAULT_CGROUP_DL_IDX (8) /* 64ms */ -+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS]; -+ -+ -+void __hmbird_update_idle(struct rq *rq, bool idle); -+ -+static inline void hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ if (hmbird_enabled()) -+ __hmbird_update_idle(rq, idle); -+} -+ -+int hmbird_ctrl(bool enable); -+ -+int hmbird_tg_online(struct task_group *tg); -+ -+ -+static inline u16 hmbird_task_util(struct task_struct *p) -+{ -+ return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0; -+} -+u16 hmbird_cpu_util(int cpu); -+ -+bool get_hmbird_ops_enabled(void); -+bool get_non_hmbird_task(void); -+void set_cpu_cluster(u64 cpu_cluster); -+#endif /*_EXT_H_*/ -diff --git b/kernel/sched/hmbird/hmbird.c a/kernel/sched/hmbird/hmbird.c -new file mode 100755 -index 000000000000..86f9fd092424 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird.c -@@ -0,0 +1,4354 @@ -+// SPDX-License-Identifier: GPL-2.0 -+ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#include -+#include -+ -+#include "slim.h" -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+#include -+ -+#define CLUSTER_SEPARATE -+ -+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0); -+atomic_t non_hmbird_task; -+atomic_t hmbird_module_loaded = ATOMIC_INIT(0); -+int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0}; -+ -+enum hmbird_internal_consts { -+ HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ, -+}; -+ -+enum hmbird_ops_enable_state { -+ HMBIRD_OPS_PREPPING, -+ HMBIRD_OPS_ENABLING, -+ HMBIRD_OPS_ENABLED, -+ HMBIRD_OPS_DISABLING, -+ HMBIRD_OPS_DISABLED, -+}; -+ -+static inline void put_hmbird_ts(struct task_struct *p) -+{ -+ kfree((void *)p->android_oem_data1[HMBIRD_TS_IDX]); -+ p->android_oem_data1[HMBIRD_TS_IDX] = 0; -+} -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+/* -+ * hmbird_entity->ops_state -+ * -+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler. -+ * State transitions look as follows: -+ * -+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING -+ * ^ | | -+ * | v v -+ * \-------------------------------/ -+ * -+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -+ * sites for explanations on the conditions being waited upon and why they are -+ * safe. Transitions out of them into NONE or QUEUED must store_release and the -+ * waiters should load_acquire. -+ * -+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether -+ * any given task can be dispatched by the BPF scheduler at all times and thus -+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core -+ * can safely reject invalid dispatches. -+ */ -+enum hmbird_ops_state { -+ HMBIRD_OPSS_NONE, /* owned by the HMBIRD core */ -+ HMBIRD_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -+ HMBIRD_OPSS_QUEUED, /* owned by the BPF scheduler */ -+ HMBIRD_OPSS_DISPATCHING, /* in transit back to the HMBIRD core */ -+ -+ /* -+ * QSEQ brands each QUEUED instance so that, when dispatch races -+ * dequeue/requeue, the dispatcher can tell whether it still has a claim -+ * on the task being dispatched. -+ */ -+ HMBIRD_OPSS_QSEQ_SHIFT = 2, -+ HMBIRD_OPSS_STATE_MASK = (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1, -+ HMBIRD_OPSS_QSEQ_MASK = ~HMBIRD_OPSS_STATE_MASK, -+}; -+ -+enum switch_stat { -+ HMBIRD_DISABLED, -+ HMBIRD_SWITCH_PREP, -+ HMBIRD_RQ_SWITCH_BEGIN, -+ HMBIRD_RQ_SWITCH_DONE, -+ HMBIRD_ENABLED, -+}; -+enum switch_stat curr_ss; -+ -+/* -+ * During exit, a task may schedule after losing its PIDs. When disabling the -+ * BPF scheduler, we need to be able to iterate tasks in every state to -+ * guarantee system safety. Maintain a dedicated task list which contains every -+ * task between its fork and eventual free. -+ */ -+DEFINE_SPINLOCK(hmbird_tasks_lock); -+static LIST_HEAD(hmbird_tasks); -+ -+/* ops enable/disable */ -+static struct kthread_worker *hmbird_ops_helper; -+static DEFINE_MUTEX(hmbird_ops_enable_mutex); -+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem); -+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED); -+ -+static bool hmbird_warned_zero_slice; -+enum hmbird_switch_type sw_type; -+static int skip_num[MAX_GLOBAL_DSQS]; -+static int big_distribute_mask_prev; -+static int little_distribute_mask_prev; -+ -+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0); -+ -+/* -+ * The maximum amount of time in jiffies that a task may be runnable without -+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger -+ * hmbird_ops_error(). -+ */ -+unsigned long hmbird_watchdog_timeout; -+ -+/* -+ * The last time the delayed work was run. This delayed work relies on -+ * ksoftirqd being able to run to service timer interrupts, so it's possible -+ * that this work itself could get wedged. To account for this, we check that -+ * it's not stalled in the timer tick, and trigger an error if it is. -+ */ -+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES; -+ -+static struct delayed_work hmbird_watchdog_work; -+static struct work_struct hmbird_err_exit_work; -+ -+/* idle tracking */ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_CPUMASK_OFFSTACK -+#define CL_ALIGNED_IF_ONSTACK -+#else -+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp -+#endif -+ -+static struct { -+ cpumask_var_t cpu; -+ cpumask_var_t smt; -+} idle_masks CL_ALIGNED_IF_ONSTACK; -+ -+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus; -+#endif /* CONFIG_SMP */ -+ -+/* dispatch queues */ -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global; -+ -+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128}; -+u32 pcp_dsq_deadline = 20; -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS]; -+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq); -+ -+static u64 max_hmbird_dsq_internal_id; -+ -+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/ -+#define CLUSTER_SEPARATE_IDX (8) -+#define GDSQS_ID_BASE (3) -+#define UX_COMPATIBLE_IDX (4) -+#define NON_PERIOD_START (5) -+#define NON_PERIOD_END (MAX_GLOBAL_DSQS) -+#define CREATE_DSQ_LEVEL_WITHIN (1) -+ -+struct hmbird_sched_info { -+ spinlock_t lock; -+ int curr_idx[2]; -+ int rtime[MAX_GLOBAL_DSQS]; -+}; -+ -+struct pcp_sched_info { -+ s64 pcp_seq; -+ int rtime; -+ bool pcp_round; -+}; -+ -+/* -+ * pcp_info may rw by another cpu. -+ * protected by rq->lock. -+ */ -+atomic64_t pcp_dsq_round; -+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info); -+ -+struct md_info_t *md_info; -+ -+static int b_rescue_l, l_rescue_b; -+static struct hmbird_sched_info sinfo; -+ -+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC; -+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = { -+ 0, 0, 0, 0, 0, -+ 32 * NSEC_PER_MSEC, -+ 20 * NSEC_PER_MSEC, -+ 14 * NSEC_PER_MSEC, -+ 8 * NSEC_PER_MSEC, -+ 6 * NSEC_PER_MSEC -+}; -+ -+struct cluster_ctx { -+ /* cpu-dsq map must within [lower, upper) */ -+ int upper; -+ int lower; -+ int tidx; -+}; -+ -+enum stat_items { -+ GLOBAL_STAT, -+ CPU_ALLOW_FAIL, -+ RT_CNT, -+ KEY_TASK_CNT, -+ SWITCH_IDX, -+ TIMEOUT_CNT, -+ -+ TOTAL_DSP_CNT, -+ MOVE_RQ_CNT, -+ SELECT_CPU, -+ -+ DWORD_STAT_END = SELECT_CPU, -+ -+ GDSQ_CNT, -+ ERR_IDX, -+ PCP_TIMEOUT_CNT, -+ PCP_LDSQ_CNT, -+ PCP_ENQL_CNT, -+ -+ MAX_ITEMS, -+}; -+static DEFINE_SPINLOCK(stats_lock); -+static char *stats_str[MAX_ITEMS] = { -+ "global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt", -+ "switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt", -+ "select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt", -+ "pcp_ldsq_cnt", "pcp_enql_cnt" -+}; -+ -+ -+struct stats_struct { -+ u64 global_stat[2]; -+ u64 cpu_allow_fail[2]; -+ u64 rt_cnt[2]; -+ u64 key_task_cnt[2]; -+ u64 switch_idx[2]; -+ u64 timeout_cnt[2]; -+ -+ u64 total_dsp_cnt[2]; -+ u64 move_rq_cnt[2]; -+ u64 select_cpu[2]; -+ -+ u64 gdsq_count[MAX_GLOBAL_DSQS][2]; -+ u64 err_idx[5]; -+ u64 pcp_timeout_cnt[NR_CPUS]; -+ u64 pcp_ldsq_count[NR_CPUS][2]; -+ u64 pcp_enql_cnt[NR_CPUS]; -+} stats_data; -+ -+static struct { -+ cpumask_var_t ex_free; -+ cpumask_var_t exclusive; -+ cpumask_var_t partial; -+ cpumask_var_t big; -+ cpumask_var_t little; -+} iso_masks __read_mostly; -+ -+/* -+ * Need more synchronization for these two variables? -+ * I choose not to. -+ */ -+static int l_need_rescue, b_need_rescue; -+ -+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...) \ -+{ \ -+ char buf[MAX_FATAL_INFO]; \ -+ \ -+ scnprintf(buf, MAX_FATAL_INFO, fmt, ##args); \ -+ hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf); \ -+ trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable), \ -+ READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf); \ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); \ -+} \ -+ -+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2) -+{ -+ int c1, c2; -+ struct cpumask mask = {.bits = {0}}; -+ struct cpumask tmp = {.bits = {0}}; -+ -+ if (!slim_stats) -+ return false; -+ -+ if (!rq1 || !rq2) -+ return false; -+ -+ c1 = cpu_of(rq1); -+ c2 = cpu_of(rq2); -+ cpumask_set_cpu(c1, &mask); -+ cpumask_set_cpu(c2, &mask); -+ -+ if (cpumask_and(&tmp, iso_masks.little, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.big, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.partial, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ return false; -+} -+ -+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu) -+{ -+ unsigned long flags; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ if (!slim_stats) -+ return; -+ -+ switch (item) { -+ case GLOBAL_STAT: -+ fallthrough; -+ case CPU_ALLOW_FAIL: -+ fallthrough; -+ case RT_CNT: -+ fallthrough; -+ case KEY_TASK_CNT: -+ fallthrough; -+ case SWITCH_IDX: -+ fallthrough; -+ case TIMEOUT_CNT: -+ fallthrough; -+ case TOTAL_DSP_CNT: -+ fallthrough; -+ case MOVE_RQ_CNT: -+ fallthrough; -+ case SELECT_CPU: -+ pval = pbase + item * 2 + idx; -+ break; -+ case GDSQ_CNT: -+ pval = &stats_data.gdsq_count[dsq_id][idx]; -+ break; -+ case ERR_IDX: -+ pval = &stats_data.err_idx[idx]; -+ break; -+ case PCP_TIMEOUT_CNT: -+ pval = &stats_data.pcp_timeout_cnt[cpu]; -+ break; -+ case PCP_LDSQ_CNT: -+ pval = &stats_data.pcp_ldsq_count[cpu][idx]; -+ break; -+ case PCP_ENQL_CNT: -+ pval = &stats_data.pcp_enql_cnt[cpu]; -+ break; -+ default: -+ return; -+ } -+ -+ spin_lock_irqsave(&stats_lock, flags); -+ *pval += 1; -+ spin_unlock_irqrestore(&stats_lock, flags); -+} -+ -+static inline bool handle_ret(int ret, int *idx, int len) -+{ -+ if (ret < 0 || ret >= len - *idx) -+ return true; -+ *idx += ret; -+ return false; -+} -+ -+#define PRINT_INTV (5 * HZ) -+void stats_print(char *buf, int len) -+{ -+ int idx = 0, i, j, ret; -+ int item = 0; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ for (item = 0; item < MAX_ITEMS; item++) { -+ if (item <= DWORD_STAT_END) { -+ pval = pbase + item * 2; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n", -+ stats_str[item], pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == GDSQ_CNT) { -+ for (j = 0; j < MAX_GLOBAL_DSQS; j++) { -+ pval = (u64 *)&stats_data.gdsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == ERR_IDX) { -+ pval = (u64 *)&stats_data.err_idx; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n", -+ stats_str[item], pval[0], -+ pval[1], pval[2], pval[3], pval[4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == PCP_TIMEOUT_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_timeout_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_LDSQ_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_ldsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_ENQL_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_enql_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } -+ } -+ -+ if (!md_info) { -+ buf[idx] = '\0'; -+ return; -+ } -+ -+ ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_SWITCHS; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "sw_rec[%d] = %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.sw_rec[i].switch_at, -+ md_info->kern_dump.sw_rec[i].is_success, -+ md_info->kern_dump.sw_rec[i].end_state, -+ md_info->kern_dump.sw_rec[i].switch_reason); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_EXCEP_ID; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "excep[%d] = %llu %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.excep_rec[i][0], -+ md_info->kern_dump.excep_rec[i][1], -+ md_info->kern_dump.excep_rec[i][2], -+ md_info->kern_dump.excep_rec[i][3], -+ md_info->kern_dump.excep_rec[i][4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n", -+ i, md_info->kern_dump.excep_idx[i]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ -+ buf[idx] = '\0'; -+} -+ -+enum cpu_type { -+ LITTLE, -+ BIG, -+ PARTIAL, -+ EXCLUSIVE, -+ EX_FREE, -+ INVALID -+}; -+ -+enum dsq_type { -+ GLOBAL_DSQ, -+ PCP_DSQ, -+ OTHER, -+ MAX_DSQ_TYPE, -+}; -+ -+static enum cpu_type cpu_cluster(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (get_hmbird_rq(rq)->exclusive) { -+ return EXCLUSIVE; -+ } else { -+ if (cpumask_test_cpu(cpu, iso_masks.little)) -+ return LITTLE; -+ else if (cpumask_test_cpu(cpu, iso_masks.big)) -+ return BIG; -+ else if (cpumask_test_cpu(cpu, iso_masks.partial)) -+ return PARTIAL; -+ else if (cpumask_test_cpu(cpu, iso_masks.exclusive)) -+ return EXCLUSIVE; -+ else if (cpumask_test_cpu(cpu, iso_masks.ex_free)) -+ return EX_FREE; -+ } -+ return INVALID; -+} -+ -+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq) -+{ -+ if (!dsq) -+ return OTHER; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= GDSQS_ID_BASE) && -+ ((dsq->id & 0xff) < MAX_GLOBAL_DSQS)) -+ return GLOBAL_DSQ; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) && -+ ((dsq->id & 0xff) < max_hmbird_dsq_internal_id)) -+ return PCP_DSQ; -+ -+ return OTHER; -+} -+ -+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq) -+{ -+ enum dsq_type type; -+ -+ type = get_dsq_type(dsq); -+ switch (type) { -+ case GLOBAL_DSQ: -+ case PCP_DSQ: -+ return (dsq->id & 0xff) - GDSQS_ID_BASE; -+ default: -+ return -1; -+ } -+ return -1; -+} -+ -+static void update_cpus_idle(bool set, struct cpumask *mask) -+{ -+ int cpu; -+ struct rq *rq; -+ -+ if (set) { -+ for_each_cpu(cpu, mask) { -+ rq = cpu_rq(cpu); -+ if (is_idle_task(rq->curr)) -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ } -+ } else -+ cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask); -+} -+ -+static void set_partial_status(bool enable, bool little, bool big) -+{ -+ WRITE_ONCE(partial_enable, enable); -+ WRITE_ONCE(l_need_rescue, little); -+ WRITE_ONCE(b_need_rescue, big); -+} -+ -+static bool is_little_need_rescue(void) -+{ -+ return READ_ONCE(l_need_rescue); -+} -+ -+static bool is_big_need_rescue(void) -+{ -+ return READ_ONCE(b_need_rescue); -+} -+ -+static bool is_partial_enabled(void) -+{ -+ return READ_ONCE(partial_enable); -+} -+ -+static bool is_partial_cpu(int cpu) -+{ -+ return cpumask_test_cpu(cpu, iso_masks.partial); -+} -+ -+static void set_iso_par_free(bool enable) -+{ -+ WRITE_ONCE(isolate_ctrl, enable); -+} -+ -+static bool is_iso_par_free(void) -+{ -+ return READ_ONCE(isolate_ctrl); -+} -+ -+static bool skip_update_idle(void) -+{ -+ int cpu = smp_processor_id(); -+ enum cpu_type type = cpu_cluster(cpu); -+ -+ if ((type == EXCLUSIVE && !is_iso_par_free()) || -+ /* partial enable may changed during idle, it doesn't matter. */ -+ (!is_partial_enabled() && type == PARTIAL)) -+ return true; -+ -+ return false; -+} -+ -+static void init_isolate_cpus(void) -+{ -+ WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -+ cpumask_set_cpu(0, iso_masks.big); -+ cpumask_set_cpu(1, iso_masks.big); -+ cpumask_set_cpu(2, iso_masks.big); -+ cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(4, iso_masks.big); -+ cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(6, iso_masks.partial); -+ cpumask_set_cpu(7, iso_masks.ex_free); -+} -+ -+extern spinlock_t css_set_lock; -+ -+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp) -+{ -+ int i; -+ struct cgroup *anc; -+ -+ spin_lock_irq(&css_set_lock); -+ for (i = 0; i < cgrp->level; i++) { -+ anc = cgrp->ancestors[i]; -+ if (anc->level != CREATE_DSQ_LEVEL_WITHIN) -+ continue; -+ cgroup_get(anc); -+ spin_unlock_irq(&css_set_lock); -+ return anc; -+ } -+ spin_unlock_irq(&css_set_lock); -+ hmbird_err(NO_CGROUP_L1, ":error cgroup = %s\n", cgrp->kn->name); -+ return NULL; -+} -+ -+#define PCP_IDX_BIT (1 << 31) -+ -+static bool is_pcp_rt(struct task_struct *p) -+{ -+ return rt_prio(p->prio) && (p->nr_cpus_allowed == 1); -+} -+ -+static bool is_pcp_idx(int idx) -+{ -+ return idx >= MAX_GLOBAL_DSQS; -+} -+ -+static bool is_critical_system_task(struct task_struct *p) -+{ -+ int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK; -+ -+ return (p->prio < (MAX_RT_PRIO >> 1) && -+ (sp_dl < SCHED_PROP_DEADLINE_LEVEL3)); -+} -+ -+#define ISOLATE_TASK_TOP_BIT (1 << 17) -+#define PIPELINE_TASK_TOP_BIT (1 << 9) -+static bool is_pipeline_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT); -+} -+ -+static bool is_isolate_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT); -+} -+ -+static bool is_critical_app_task_without_isolate(struct task_struct *p) -+{ -+ return task_is_top_task(p) && !is_isolate_task(p); -+} -+ -+static int find_idx_from_task(struct task_struct *p) -+{ -+ int idx, cpu; -+ int sp_dl; -+ struct task_group *tg = p->sched_task_group; -+ -+ if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) { -+ cpu = cpumask_any(p->cpus_ptr); -+ idx = cpu + MAX_GLOBAL_DSQS; -+ return idx; -+ } -+ -+ if (is_critical_system_task(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL0; -+ goto done; -+ } -+ -+ if (is_critical_app_task_without_isolate(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL1; -+ goto done; -+ } -+ -+ if (p->pid == scx_systemui_pid) { -+ idx = SCHED_PROP_DEADLINE_LEVEL4; -+ goto done; -+ } -+ -+ sp_dl = hmbird_get_dsq_id(p); -+ if (sp_dl) { -+ idx = sp_dl; -+ goto done; -+ } -+ -+ if (rt_prio(p->prio)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL3; -+ goto done; -+ } -+ -+ if (tg && tg->css.cgroup && tg->css.cgroup->kn) { -+ if (likely(tg->css.cgroup->kn->id >= 0 && -+ tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS)) -+ idx = cgroup_ids_table[tg->css.cgroup->kn->id]; -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ -+done: -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) { -+ hmbird_err(DSQ_ID_ERR, " : idx error, idx = %d-----\n", idx); -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } -+ return idx; -+} -+ -+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p) -+{ -+ int idx; -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq; -+ -+ if (!p) -+ return NULL; -+ -+ idx = find_idx_from_task(p); -+ if (is_pcp_idx(idx)) { -+ idx -= MAX_GLOBAL_DSQS; -+ dsq = &per_cpu(pcp_ldsq, idx); -+ get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq); -+ slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx); -+ } else { -+ dsq = &gdsqs[idx]; -+ get_hmbird_ts(p)->gdsq_idx = idx; -+ slim_stats_record(GDSQ_CNT, 0, idx, 0); -+ } -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ dsq->last_consume_at = jiffies; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ return dsq; -+} -+ -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq); -+ -+static void set_partial_rescue(bool p_state, bool l_over, bool b_over) -+{ -+ set_partial_status(p_state, l_over, b_over); -+ update_cpus_idle(p_state, iso_masks.partial); -+ hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled()); -+ hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue()); -+ hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue()); -+} -+ -+static void free_isocpu(bool enable) -+{ -+ set_iso_par_free(enable); -+ update_cpus_idle(enable, iso_masks.ex_free); -+ hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free()); -+} -+ -+inline u64 get_hmbird_cpu_util(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed) -+ return 0; -+ u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed); -+ u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size); -+ -+ do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT); -+ -+ return prev_runnable_sum_fixed; -+} -+ -+static inline unsigned int get_scaling_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->max; -+} -+ -+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static u64 get_cpus_max_util(struct cpumask *mask) -+{ -+ int cpu; -+ u64 max = 0; -+ u64 util, ratio; -+ unsigned long effective_cap = 0; -+ -+ for_each_cpu(cpu, mask) { -+ if (slim_walt_ctrl) -+ slim_get_cpu_util(cpu, &util); -+ else -+ util = get_hmbird_cpu_util(cpu); -+ -+ /* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/ -+ if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu))) -+ effective_cap = arch_scale_cpu_capacity(cpu); -+ else -+ effective_cap = arch_scale_cpu_capacity(cpu) * -+ get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu); -+ -+ ratio = util * 100 / effective_cap; -+ hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util); -+ hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n", -+ cpu, (u64)arch_scale_cpu_capacity(cpu)); -+ hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n", -+ cpu, (u64)effective_cap); -+ -+ if (ratio > 100) -+ ratio = 100; -+ -+ if (ratio > max) -+ max = ratio; -+ } -+ return max; -+} -+ -+static bool cluster_need_rescue(struct cpumask *mask, int hres) -+{ -+ return get_cpus_max_util(mask) > hres; -+} -+ -+void partial_dynamic_ctrl(void) -+{ -+ u64 lmax = 0, bmax = 0; -+ bool l_over, l_under, b_over, b_under; -+ static bool last_l_over, last_b_over; -+ static unsigned long last_check; -+ -+ /* Check partial every jiffies. need lock? */ -+ if (time_before_eq(jiffies, READ_ONCE(last_check))) -+ return; -+ -+ WRITE_ONCE(last_check, jiffies); -+ -+ lmax = get_cpus_max_util(iso_masks.little); -+ l_over = lmax > parctrl_high_ratio_l; -+ l_under = lmax < parctrl_low_ratio_l; -+ -+ bmax = get_cpus_max_util(iso_masks.big); -+ b_over = bmax > parctrl_high_ratio; -+ b_under = bmax < parctrl_low_ratio; -+ -+ if (is_partial_enabled() && (l_over || b_over)) { -+ if (last_l_over != l_over || last_b_over != b_over) -+ set_partial_rescue(true, l_over, b_over); -+ } else if (!is_partial_enabled() && (l_over || b_over)) { -+ set_partial_rescue(true, l_over, b_over); -+ } else if (is_partial_enabled() && l_under && b_under) { -+ set_partial_rescue(false, false, false); -+ } -+ last_l_over = l_over; -+ last_b_over = b_over; -+ -+ if (is_partial_enabled()) { -+ if (cpumask_empty(iso_masks.partial)) { -+ bmax = bmax > lmax ? bmax : lmax; -+ } else { -+ bmax = get_cpus_max_util(iso_masks.partial); -+ } -+ if (!is_iso_par_free() && bmax > isoctrl_high_ratio) { -+ hmbird_info_trace("partial max = %llu\n", bmax); -+ free_isocpu(true); -+ } else if (is_iso_par_free() && bmax < isoctrl_low_ratio) -+ free_isocpu(false); -+ } else if (is_iso_par_free()) { -+ free_isocpu(false); -+ } -+} -+ -+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx) -+{ -+ hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx); -+} -+ -+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx) -+{ -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[idx])) { -+ slim_stats_record(GDSQ_CNT, 1, idx, 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ int i; -+ -+ for (i = 0; i < UX_COMPATIBLE_IDX; i++) { -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ slim_stats_record(GDSQ_CNT, 1, i, 0); -+ return 1; -+ } -+ } -+ return 0; -+} -+ -+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) { -+ slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ goto clear_timeout; -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) -+ goto clear_timeout; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ hmbird_info_trace( -+ "dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n", -+ dsq_id_to_internal(dsq), entity->task->comm, -+ jiffies, msecs_to_jiffies(deadline), entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1); -+ return; -+ -+clear_timeout: -+ hmbird_info_trace("dsq[%d] clear timeout\n", -+ dsq_id_to_internal(dsq)); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0); -+ dsq->is_timeout = false; -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq)); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime) -+{ -+ hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime); -+} -+ -+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any) -+{ -+ bool is_timeout; -+ int cpu = cpu_of(rq); -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu); -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ is_timeout = dsq->is_timeout; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ /* -+ * dsq->is_timeout may change here, let it be. -+ * it won't cause serious problems. -+ * the same for consume_dispatch_q later. -+ */ -+ if (!is_timeout && !any) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, dsq)) { -+ if (is_timeout) { -+ hmbird_info_trace("dsq[%d] consume a pcp timeout task\n", -+ dsq_id_to_internal(dsq)); -+ update_timeout_stats(rq, dsq, pcp_dsq_deadline); -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu); -+ } -+ slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu); -+ return 1; -+ } -+ /* -+ * No pcp task, clear quota. -+ */ -+ if (any) { -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+ } -+ return 0; -+} -+ -+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf) -+{ -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ } -+ return 0; -+} -+ -+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf, -+ int tmp, int cidx, int tidx) -+{ -+ unsigned long flags; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) { -+ if (tmp != cidx) { -+ spin_lock(&sinfo.lock); -+ sinfo.curr_idx[tidx] = tmp; -+ spin_unlock(&sinfo.lock); -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]); -+ slim_stats_record(SWITCH_IDX, 0, 0, 0); -+ } -+ slim_stats_record(GDSQ_CNT, 1, tmp, 0); -+ -+ raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags); -+ gdsqs[tmp].last_consume_at = jiffies; -+ raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags); -+ return 1; -+ } -+ return 0; -+ -+} -+ -+static int get_cidx(struct cluster_ctx *ctx) -+{ -+ int cidx; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ if (cidx < ctx->lower || cidx >= ctx->upper) { -+ sinfo.curr_idx[ctx->tidx] = ctx->lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]); -+ slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ } -+ spin_unlock(&sinfo.lock); -+ -+ return cidx; -+} -+ -+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = CLUSTER_SEPARATE_IDX; -+ if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) { -+ pr_debug(" %s iso_masks.little first is %d\n", -+ __func__, cpumask_first(iso_masks.little)); -+ ctx->upper = NON_PERIOD_END; -+ } -+ ctx->tidx = 0; -+ break; -+ case LITTLE: -+ ctx->lower = CLUSTER_SEPARATE_IDX; -+ ctx->upper = NON_PERIOD_END; -+ if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) { -+ pr_debug(" %s iso_masks.big first is %d", -+ __func__, cpumask_first(iso_masks.big)); -+ ctx->lower = NON_PERIOD_START; -+ } -+ ctx->tidx = 1; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ case LITTLE: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = NON_PERIOD_END; -+ ctx->tidx = 0; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ if (cluster_separate) { -+ return gen_cluster_ctx_separate(ctx, type); -+ } else { -+ return gen_cluster_ctx_common(ctx, type); -+ } -+} -+ -+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ int i; -+ bool is_timeout; -+ unsigned long flags; -+ struct cluster_ctx ctx; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ /* Third param means only consume timeout pcp dsq. */ -+ if (consume_pcp_dsq(rq, rf, false)) -+ return 1; -+ -+ for (i = ctx.lower; i < ctx.upper; i++) { -+ raw_spin_lock_irqsave(&gdsqs[i].lock, flags); -+ is_timeout = gdsqs[i].is_timeout; -+ raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags); -+ /* gdsqs[i].is_timeout may change here, let it be... */ -+ if (is_timeout) { -+ /* -+ * consume_dispatch_q will acquire dsq-lock, -+ * So cannot keep lock here, annoy enough. -+ * may rewrite a consume_dispatch_q_locked, TODO. -+ */ -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ hmbird_info_trace("dsq[%d] consume a timeout task\n", i); -+ slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0); -+ update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]); -+ return 1; -+ } -+ } -+ } -+ return 0; -+} -+ -+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ struct cluster_ctx ctx; -+ int cidx; -+ int tmp; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ cidx = get_cidx(&ctx); -+ tmp = cidx; -+ do { -+ if (check_pcp_dsq_round(rq, rf)) -+ return 1; -+ if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx)) -+ return 1; -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[tmp] = 0; -+ systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]); -+ tmp++; -+ if (tmp >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ tmp = ctx.lower; -+ } -+ spin_unlock(&sinfo.lock); -+ } while (tmp != cidx); -+ -+ return consume_pcp_dsq(rq, rf, true); -+} -+ -+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ enum cpu_type type = cpu_cluster(cpu_of(rq)); -+ bool period_allowed = !get_hmbird_rq(rq)->period_disallow; -+ bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow; -+ -+ switch (type) { -+ case EX_FREE: -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ break; -+ case EXCLUSIVE: -+ if (!is_iso_par_free()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ } -+ /* Only non-period task can run on isolate cpus */ -+ if (!READ_ONCE(iso_free_rescue)) { -+ if (is_big_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ } else { -+ /* Free run, can run on any task. */ -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ case PARTIAL: -+ if (!is_partial_enabled()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ return 0; -+ } -+ if (is_big_need_rescue()) { -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ -+ case BIG: -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ if (is_iso_par_free() || (is_little_need_rescue() && -+ !cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) { -+ hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++); -+ return 1; -+ } -+ } -+ break; -+ -+ case LITTLE: -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ -+ if (is_iso_par_free() || (is_big_need_rescue() && -+ !cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) { -+ hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++); -+ return 1; -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return 0; -+} -+ -+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf) -+{ -+ return consume_hmbird_global_dsq(rq, rf); -+} -+ -+ -+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time) -+{ -+ int idx; -+ -+ /* which dsq belongs to while task enqueue, task will consume its running time. */ -+ idx = get_hmbird_ts(p)->gdsq_idx; -+ /* Only non-period dsq share running time between each other. */ -+ if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id) -+ return; -+ -+ if (idx >= MAX_GLOBAL_DSQS) { -+ per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } else { -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[idx] += exec_time; -+ spin_unlock(&sinfo.lock); -+ systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]); -+ } -+} -+ -+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type) -+{ -+ int cidx; -+ struct cluster_ctx ctx; -+ int cpu = cpu_of(rq); -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ if (cidx < ctx.lower || cidx >= ctx.upper) { -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ slim_stats_record(ERR_IDX, ctx.tidx, 0, 0); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ } -+ -+ while (1) { -+ if (per_cpu(pcp_info, cpu).pcp_round) { -+ if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) { -+ hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n", -+ cpu, per_cpu(pcp_info, cpu).rtime); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } -+ } -+ if (sinfo.rtime[cidx] < dsq_quota[cidx]) -+ break; -+ -+ /* clear current dsq rtime */ -+ sinfo.rtime[cidx] = 0; -+ systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]); -+ -+ sinfo.curr_idx[ctx.tidx]++; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", -+ ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ } -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ slim_stats_record(SWITCH_IDX, 1, 0, 0); -+ } -+ spin_unlock(&sinfo.lock); -+} -+ -+ -+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p) -+{ -+ enum cpu_type type; -+ -+ if (!rq || !p) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ switch (type) { -+ case PARTIAL: -+ return; -+ case EXCLUSIVE: -+ return; -+ case EX_FREE: -+ return; -+ default: -+ break; -+ } -+ update_dsq_idx(rq, p, type); -+} -+ -+ -+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ int dsq_id; -+ -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo) || dsq->is_timeout) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ hmbird_deferred_err(SCAN_ENTITY_NULL, -+ " : entity is NULL, dsq->id = %llu\n", dsq->id); -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ dsq->is_timeout = true; -+ dsq_id = dsq_id_to_internal(dsq); -+ hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n", -+ dsq_id, entity->task->comm, jiffies, entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1); -+ raw_spin_unlock(&dsq->lock); -+ -+ return true; -+} -+ -+void scan_timeout(struct rq *rq) -+{ -+ int i; -+ int cpu = cpu_of(rq); -+ struct hmbird_dispatch_q *dsq; -+ static u64 last_scan_at; -+ static DEFINE_PER_CPU(u64, pcp_last_scan_at); -+ -+ if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu))) -+ return; -+ per_cpu(pcp_last_scan_at, cpu) = jiffies; -+ -+ dsq = &per_cpu(pcp_ldsq, cpu); -+ scan_dsq_timeout(rq, dsq, pcp_dsq_deadline); -+ -+ if (time_before_eq(jiffies, (unsigned long)last_scan_at)) -+ return; -+ last_scan_at = jiffies; -+ -+ for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) { -+ dsq = &gdsqs[i]; -+ scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]); -+ } -+} -+ -+/*******************************Initialize***********************************/ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id); -+static void init_dsq_at_boot(void) -+{ -+ int i, cpu; -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i)); -+ } -+ for_each_possible_cpu(cpu) -+ init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i + cpu)); -+ -+ max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu; -+ spin_lock_init(&sinfo.lock); -+} -+ -+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgrp_name_to_idx(struct cgroup *cgrp) -+{ -+ int idx; -+ -+ if (!cgrp) -+ return -1; -+ -+ if (!strcmp(cgrp->kn->name, "display") -+ || !strcmp(cgrp->kn->name, "multimedia")) -+ idx = 5; /* 8ms */ -+ else if (!strcmp(cgrp->kn->name, "top-app") -+ || !strcmp(cgrp->kn->name, "ss-top")) -+ idx = 6; /* 16ms */ -+ else if (!strcmp(cgrp->kn->name, "ssfg") -+ || !strcmp(cgrp->kn->name, "foreground")) -+ idx = 7; /* 32ms */ -+ else if (!strcmp(cgrp->kn->name, "bg") -+ || !strcmp(cgrp->kn->name, "log") -+ || !strcmp(cgrp->kn->name, "dex2oat") -+ || !strcmp(cgrp->kn->name, "background")) -+ idx = 9; /* 128ms */ -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */ -+ -+ return idx; -+} -+ -+static void init_root_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX); -+} -+ -+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) { -+ pr_err("%s idx err!\n", __func__); -+ return; -+ } -+ -+ if (cgroup_ids_table[cgrp->kn->id] == -1) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp)); -+} -+ -+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ struct cgroup *l1cgrp; -+ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ l1cgrp = cgroup_ancestor_l1(cgrp); -+ if (l1cgrp) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp)); -+ cgroup_put(l1cgrp); -+} -+ -+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg) -+{ -+ switch (cgrp->level) { -+ case 0: -+ init_root_tg(cgrp, tg); -+ break; -+ case 1: -+ init_level1_tg(cgrp, tg); -+ break; -+ default: -+ init_child_tg(cgrp, tg); -+ break; -+ } -+} -+ -+/**************************************************************************/ -+ -+struct hmbird_task_iter { -+ struct hmbird_entity cursor; -+ struct task_struct *locked; -+ struct rq *rq; -+ struct rq_flags rf; -+}; -+ -+/** -+ * hmbird_task_iter_init - Initialize a task iterator -+ * @iter: iterator to init -+ * -+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized, -+ * @iter must eventually be exited with hmbird_task_iter_exit(). -+ * -+ * hmbird_tasks_lock may be released between this and the first next() call or -+ * between any two next() calls. If hmbird_tasks_lock is released between two -+ * next() calls, the caller is responsible for ensuring that the task being -+ * iterated remains accessible either through RCU read lock or obtaining a -+ * reference count. -+ * -+ * All tasks which existed when the iteration started are guaranteed to be -+ * visited as long as they still exist. -+ */ -+static void hmbird_task_iter_init(struct hmbird_task_iter *iter) -+{ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR }; -+ list_add(&iter->cursor.tasks_node, &hmbird_tasks); -+ iter->locked = NULL; -+} -+ -+/** -+ * hmbird_task_iter_exit - Exit a task iterator -+ * @iter: iterator to exit -+ * -+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held. -+ * If the iterator holds a task's rq lock, that rq lock is released. See -+ * hmbird_task_iter_init() for details. -+ */ -+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ if (list_empty(cursor)) -+ return; -+ -+ list_del_init(cursor); -+} -+ -+/** -+ * hmbird_task_iter_next - Next task -+ * @iter: iterator to walk -+ * -+ * Visit the next task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ struct hmbird_entity *pos; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ list_for_each_entry(pos, cursor, tasks_node) { -+ if (&pos->tasks_node == &hmbird_tasks) -+ return NULL; -+ if (!(pos->flags & HMBIRD_TASK_CURSOR)) { -+ list_move(cursor, &pos->tasks_node); -+ return pos->task; -+ } -+ } -+ -+ /* can't happen, should always terminate at hmbird_tasks above */ -+ hmbird_deferred_err(ITER_RET_NULL, " : unreachable path in scx_task_iter_next\n"); -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered - Next non-idle task -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ while ((p = hmbird_task_iter_next(iter))) { -+ if (!is_idle_task(p)) -+ return p; -+ } -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init() -+ * for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ p = hmbird_task_iter_next_filtered(iter); -+ if (!p) -+ return NULL; -+ -+ iter->rq = task_rq_lock(p, &iter->rf); -+ iter->locked = p; -+ return p; -+} -+ -+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void) -+{ -+ return atomic_read(&hmbird_ops_enable_state_var); -+} -+ -+static enum hmbird_ops_enable_state -+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to) -+{ -+ return atomic_xchg(&hmbird_ops_enable_state_var, to); -+} -+ -+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to, -+ enum hmbird_ops_enable_state from) -+{ -+ int from_v = from; -+ -+ return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to); -+} -+ -+static bool hmbird_ops_disabling(void) -+{ -+ return false; -+} -+ -+/** -+ * wait_ops_state - Busy-wait the specified ops state to end -+ * @p: target task -+ * @opss: state to wait the end of -+ * -+ * Busy-wait for @p to transition out of @opss. This can only be used when the -+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also -+ * has load_acquire semantics to ensure that the caller can see the updates made -+ * in the enqueueing and dispatching paths. -+ */ -+static void wait_ops_state(struct task_struct *p, u64 opss) -+{ -+ do { -+ cpu_relax(); -+ } while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss); -+} -+ -+ -+static void update_curr_hmbird(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ u64 now = rq_clock_task(rq); -+ u64 delta_exec; -+ -+ if (time_before_eq64(now, curr->se.exec_start)) -+ return; -+ -+ delta_exec = now - curr->se.exec_start; -+ curr->se.exec_start = now; -+ update_runningtime(rq, curr, delta_exec); -+ curr->se.sum_exec_runtime += delta_exec; -+ account_group_exec_runtime(curr, delta_exec); -+ cgroup_account_cputime(curr, delta_exec); -+ -+ if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF) -+ get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec); -+ -+ trace_sched_stat_runtime(curr, delta_exec, 0); -+} -+ -+static bool hmbird_dsq_priq_less(struct rb_node *node_a, -+ const struct rb_node *node_b) -+{ -+ const struct hmbird_entity *a = -+ container_of(node_a, struct hmbird_entity, dsq_node.priq); -+ const struct hmbird_entity *b = -+ container_of(node_b, struct hmbird_entity, dsq_node.priq); -+ -+ return time_before64(a->dsq_vtime, b->dsq_vtime); -+} -+ -+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ bool is_local = dsq->id == HMBIRD_DSQ_LOCAL; -+ unsigned long flags; -+ -+ hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq || -+ !list_empty(&get_hmbird_ts(p)->dsq_node.fifo), -+ "task = %s, dsq->id = %llu\n", p->comm, dsq->id); -+ hmbird_cond_deferred_err(ENQ_EXIST2, -+ (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq), -+ "task = %s\n", p->comm); -+ -+ if (!is_local) { -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ); -+ hmbird_ops_error(": %s\n", -+ "attempting to dispatch to a destroyed dsq"); -+ /* fall back to the global dsq */ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ dsq = &hmbird_dsq_global; -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ } -+ } -+ -+ if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) { -+ get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ; -+ rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq, -+ hmbird_dsq_priq_less); -+ } else { -+ if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT)) -+ list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ else -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ } -+ dsq->nr++; -+ get_hmbird_ts(p)->dsq = dsq; -+ -+ /* -+ * We're transitioning out of QUEUEING or DISPATCHING. store_release to -+ * match waiters' load_acquire. -+ */ -+ if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS) -+ atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE); -+ -+ if (is_local) { -+ struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq); -+ struct rq *rq = hmbird->rq; -+ bool preempt = false; -+ -+ if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr && -+ rq->curr->sched_class == &hmbird_sched_class) { -+ get_hmbird_ts(rq->curr)->slice = 0; -+ preempt = true; -+ } -+ -+ if (preempt || sched_class_above(&hmbird_sched_class, -+ rq->curr->sched_class)) -+ resched_curr(rq); -+ } else { -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ } -+} -+ -+static void task_unlink_from_dsq(struct task_struct *p, -+ struct hmbird_dispatch_q *dsq) -+{ -+ if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) { -+ rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ; -+ } else { -+ list_del_init(&get_hmbird_ts(p)->dsq_node.fifo); -+ } -+} -+ -+static bool task_linked_on_dsq(struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+} -+ -+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq; -+ bool is_local = dsq == &hmbird_rq->local_dsq; -+ -+ if (!dsq) { -+ hmbird_cond_deferred_err(TASK_LINKED1, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ /* -+ * When dispatching directly from the BPF scheduler to a local -+ * DSQ, the task isn't associated with any DSQ but -+ * @get_hmbird_ts(p)->holding_cpu may be set under the protection of -+ * %HMBIRD_OPSS_DISPATCHING. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu >= 0) -+ get_hmbird_ts(p)->holding_cpu = -1; -+ return; -+ } -+ -+ if (!is_local) -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ /* -+ * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node -+ * can't change underneath us. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu < 0) { -+ /* @p must still be on @dsq, dequeue */ -+ hmbird_cond_deferred_err(TASK_UNLINKED, -+ !task_linked_on_dsq(p), "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ } else { -+ /* -+ * We're racing against dispatch_to_local_dsq() which already -+ * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the -+ * holding_cpu which tells dispatch_to_local_dsq() that it lost -+ * the race. -+ */ -+ hmbird_cond_deferred_err(TASK_LINKED2, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ get_hmbird_ts(p)->holding_cpu = -1; -+ } -+ get_hmbird_ts(p)->dsq = NULL; -+ -+ if (!is_local) -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+ -+static bool test_rq_online(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ return rq->online; -+#else -+ return true; -+#endif -+} -+ -+static void refill_task_slice(struct task_struct *p) -+{ -+ if (is_isolate_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO; -+ else if (is_pipeline_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2; -+ else -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+} -+ -+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -+ int sticky_cpu) -+{ -+ struct hmbird_dispatch_q *d; -+ s32 cpu = -1; -+ -+ hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), -+ "task = %s\n", p->comm); -+ -+ if (is_pcp_rt(p)) { -+ /* Enqueue percpu rt task to local directly. */ -+ /* Or cause a bug when disable dispatch. */ -+ if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ } -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (cpu >= 0) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ -+ if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ /* rq migration */ -+ if (sticky_cpu == cpu_of(rq)) -+ goto local_norefill; -+ /* -+ * If !rq->online, we already told the BPF scheduler that the CPU is -+ * offline. We're just trying to on/offline the CPU. Don't bother the -+ * BPF scheduler. -+ */ -+ if (unlikely(!test_rq_online(rq))) -+ goto local; -+ -+ /* see %HMBIRD_OPS_ENQ_LAST */ -+ if (enq_flags & HMBIRD_ENQ_LAST) -+ goto local; -+ -+ if (enq_flags & HMBIRD_ENQ_LOCAL) -+ goto local; -+ else -+ goto global; -+local: -+ /* -+ * For task-ordering, slice refill must be treated as implying the end -+ * of the current slice. Otherwise, the longer @p stays on the CPU, the -+ * higher priority it becomes from hmbird_prio_less()'s POV. -+ */ -+ refill_task_slice(p); -+local_norefill: -+ dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags); -+ slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq)); -+ return; -+ -+global: -+ d = find_dsq_from_task(p); -+ if (d) { -+ refill_task_slice(p); -+ dispatch_enqueue(d, p, enq_flags); -+ return; -+ } -+ slim_stats_record(GLOBAL_STAT, 0, 0, 0); -+ refill_task_slice(p); -+ dispatch_enqueue(&hmbird_dsq_global, p, enq_flags); -+} -+ -+static bool watchdog_task_watched(const struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->watchdog_node); -+} -+ -+static void watchdog_watch_task(struct rq *rq, struct task_struct *p) -+{ -+ lockdep_assert_rq_held(rq); -+ if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ get_hmbird_ts(p)->runnable_at = jiffies; -+ clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list); -+} -+ -+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) -+{ -+ list_del_init(&get_hmbird_ts(p)->watchdog_node); -+ if (reset_timeout) -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags) -+{ -+ int sticky_cpu = get_hmbird_ts(p)->sticky_cpu; -+ -+ enq_flags |= get_hmbird_rq(rq)->extra_enq_flags; -+ -+ if (sticky_cpu >= 0) -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ -+ /* -+ * Restoring a running task will be immediately followed by -+ * set_next_task_hmbird() which expects the task to not be on the BPF -+ * scheduler as tasks can only start running through local DSQs. Force -+ * direct-dispatch into the local DSQ by setting the sticky_cpu. -+ */ -+ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -+ sticky_cpu = cpu_of(rq); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_UNWATCHED, -+ !watchdog_task_watched(p), "task = %s\n", p->comm); -+ return; -+ } -+ -+ watchdog_watch_task(rq, p); -+ set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ get_hmbird_rq(rq)->nr_running++; -+ add_nr_running(rq, 1); -+ -+ do_enqueue_task(rq, p, enq_flags, sticky_cpu); -+} -+ -+static void ops_dequeue(struct task_struct *p, u64 deq_flags) -+{ -+ u64 opss; -+ -+ watchdog_unwatch_task(p, false); -+ -+ /* acquire ensures that we see the preceding updates on QUEUED */ -+ opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state); -+ -+ switch (opss & HMBIRD_OPSS_STATE_MASK) { -+ case HMBIRD_OPSS_NONE: -+ break; -+ case HMBIRD_OPSS_QUEUEING: -+ /* -+ * QUEUEING is started and finished while holding @p's rq lock. -+ * As we're holding the rq lock now, we shouldn't see QUEUEING. -+ */ -+ hmbird_deferred_err(DEQ_DEQING, " : unreachable path in %s\n", __func__); -+ break; -+ case HMBIRD_OPSS_QUEUED: -+ if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss, -+ HMBIRD_OPSS_NONE)) -+ break; -+ fallthrough; -+ case HMBIRD_OPSS_DISPATCHING: -+ /* -+ * If @p is being dispatched from the BPF scheduler to a DSQ, -+ * wait for the transfer to complete so that @p doesn't get -+ * added to its DSQ after dequeueing is complete. -+ * -+ * As we're waiting on DISPATCHING with the rq locked, the -+ * dispatching side shouldn't try to lock the rq while -+ * DISPATCHING is set. See dispatch_to_local_dsq(). -+ * -+ * DISPATCHING shouldn't have qseq set and control can reach -+ * here with NONE @opss from the above QUEUED case block. -+ * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss. -+ */ -+ wait_ops_state(p, HMBIRD_OPSS_DISPATCHING); -+ hmbird_cond_deferred_err(HMBIRD_OPN, -+ atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE, -+ "task = %s\n", p->comm); -+ break; -+ } -+} -+ -+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ -+ if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p), -+ "task = %s\n", p->comm); -+ return; -+ } -+ -+ ops_dequeue(p, deq_flags); -+ -+ if (slim_walt_ctrl) { -+ if (task_current(rq, p)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ if (deq_flags & HMBIRD_DEQ_SLEEP) -+ set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else -+ clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), -+ (unsigned long *)&get_hmbird_ts(p)->flags); -+ -+ clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm); -+ hmbird_rq->nr_running--; -+ sub_nr_running(rq, 1); -+ dispatch_dequeue(hmbird_rq, p); -+} -+ -+static void yield_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ get_hmbird_ts(p)->slice = 0; -+} -+ -+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to) -+{ -+ return false; -+} -+ -+#ifdef CONFIG_SMP -+/** -+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -+ * @rq: rq to move the task into, currently locked -+ * @p: task to move -+ * @enq_flags: %HMBIRD_ENQ_* -+ * -+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller -+ * must: -+ * -+ * 1. Start with exclusive access to @p either through its DSQ lock or -+ * %HMBIRD_OPSS_DISPATCHING flag. -+ * -+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id(). -+ * -+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't -+ * deadlock with dequeue. -+ * -+ * 4. Lock @rq and the task_rq from #3. -+ * -+ * 5. Call this function. -+ * -+ * Returns %true if @p was successfully moved. %false after racing dequeue and -+ * losing. -+ */ -+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ struct rq *task_rq; -+ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * If dequeue got to @p while we were trying to lock both rq's, it'd -+ * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have -+ * updated it to different values afterwards, as this operation can't be -+ * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become -+ * raw_smp_processor_id() again before we're done. Thus, we can tell -+ * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is -+ * still raw_smp_processor_id(). -+ * -+ * See dispatch_dequeue() for the counterpart. -+ */ -+ if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id())) -+ return false; -+ -+ /* @p->rq couldn't have changed if we're still the holding cpu */ -+ task_rq = task_rq(p); -+ lockdep_assert_rq_held(task_rq); -+ deactivate_task(task_rq, p, 0); -+ set_task_cpu(p, cpu_of(rq)); -+ get_hmbird_ts(p)->sticky_cpu = cpu_of(rq); -+ -+ /* -+ * We want to pass hmbird-specific enq_flags but activate_task() will -+ * truncate the upper 32 bit. As we own @rq, we can pass them through -+ * @get_hmbird_rq(rq)->extra_enq_flags instead. -+ */ -+ hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags, -+ "task = %s\n", p->comm); -+ get_hmbird_rq(rq)->extra_enq_flags = enq_flags; -+ activate_task(rq, p, 0); -+ get_hmbird_rq(rq)->extra_enq_flags = 0; -+ -+ return true; -+} -+ -+#endif /* CONFIG_SMP */ -+ -+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu) -+{ -+ int fitable = 1; -+ -+ return fitable; -+} -+ -+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq, -+ struct hmbird_dispatch_q *dsq) -+{ -+ bool dsq_misfit; -+ int cpu = cpu_of(rq); -+ u64 task_util = 0; -+ struct cluster_ctx ctx; -+ int dsq_int = dsq_id_to_internal(dsq); -+ -+ if (!cpumask_test_cpu(cpu, iso_masks.little)) -+ return false; -+ if (p->pid == scx_systemui_pid) -+ return true; -+ -+ gen_cluster_ctx(&ctx, BIG); -+ dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 && -+ dsq_int <= SCHED_PROP_DEADLINE_LEVEL4); -+#ifdef CLUSTER_SEPARATE -+ /* In rescue mode, little will consume big cluster's dsq.*/ -+ dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper); -+#endif -+ if (!dsq_misfit) -+ return false; -+ -+ if (p) { -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &task_util); -+ else -+ task_util = get_hmbird_ts(p)->demand_scaled; -+ } -+ -+ if (task_util <= misfit_ds) -+ return false; -+ -+ hmbird_info_trace(":task %s can't run on cpu%d, util = %llu\n", -+ p->comm, cpu, task_util); -+ return true; -+} -+ -+static int check_misfit_task_on_fake_big(struct task_struct *p, struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (likely(p->pid != scx_systemui_pid)) -+ return false; -+ -+ if (topology_cluster_id(num_possible_cpus() - 1) > 1 && -+ arch_scale_cpu_capacity(cpu) == arch_scale_cpu_capacity(0) && -+ (parctrl_high_ratio <= 0 || parctrl_high_ratio_l <= 0)) -+ return true; -+ -+ return false; -+} -+ -+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq) -+{ -+ if (!cpumask_test_cpu(cpu_of(rq), task_cpu_possible_mask(p))) -+ return false; -+ -+ if (!task_fits_cpu_hmbird(p, cpu_of(rq))) -+ return false; -+ -+ if (check_misfit_task_on_little(p, rq, dsq)) -+ return false; -+ -+ if (check_misfit_task_on_fake_big(p, rq)) -+ return false; -+ -+ return likely(test_rq_online(rq)); -+} -+ -+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return; -+ -+ if (add) -+ skipn[idx]++; -+ else -+ skipn[idx] = 0; -+} -+ -+static bool skip_too_much(struct hmbird_dispatch_q *dsq) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return false; -+ -+ if (skip_num[idx] > 3) { -+ skip_num[idx] = 0; -+ return true; -+ } -+ -+ return false; -+} -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rb_node *rb_node; -+ struct rq *task_rq; -+ unsigned long flags; -+ bool moved = false; -+ struct task_struct *may_fit = NULL; -+ int skip = 0; -+ -+retry: -+ if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -+ return false; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) { -+ set_skip_num(dsq, skip_num, (bool)may_fit); -+ goto this_rq; -+ } -+ if (skip_too_much(dsq)) -+ goto remote_rq; -+ if (!may_fit) -+ may_fit = p; -+ if (++skip <= 3) -+ continue; -+ /* -+ * If the recent 3 tasks not fit, use the first one. -+ * and clear the skip, because the first one is consumed. -+ */ -+ set_skip_num(dsq, skip_num, false); -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ /* No more task, use the first may fit task.*/ -+ if (may_fit) { -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ -+ for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) -+ goto this_rq; -+ goto remote_rq; -+ } -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ return false; -+ -+this_rq: -+ /* @dsq is locked and @p is on this rq */ -+ hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo); -+ dsq->nr--; -+ hmbird_rq->local_dsq.nr++; -+ get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ -+remote_rq: -+#ifdef CONFIG_SMP -+ if (cpu_same_cluster_stat(p, rq, task_rq)) -+ slim_stats_record(MOVE_RQ_CNT, 0, 0, 0); -+ else -+ slim_stats_record(MOVE_RQ_CNT, 1, 0, 0); -+ /* -+ * @dsq is locked and @p is on a remote rq. @p is currently protected by -+ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -+ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -+ * rq lock or fail, do a little dancing from our side. See -+ * move_task_to_local_dsq(). -+ */ -+ hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id(); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ rq_unpin_lock(rq, rf); -+ double_lock_balance(rq, task_rq); -+ rq_repin_lock(rq, rf); -+ -+ moved = move_task_to_local_dsq(rq, p, 0); -+ -+ double_unlock_balance(rq, task_rq); -+#endif /* CONFIG_SMP */ -+ if (likely(moved)) { -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ } -+ may_fit = NULL; -+ goto retry; -+} -+ -+ -+static int balance_one(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf, bool local) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class; -+ -+ if (!hmbird_rq) -+ return 1; -+ -+ lockdep_assert_rq_held(rq); -+ -+ if (static_branch_unlikely(&hmbird_ops_cpu_preempt) && -+ unlikely(get_hmbird_rq(rq)->cpu_released)) { -+ /* -+ * If the previous sched_class for the current CPU was not HMBIRD, -+ * notify the BPF scheduler that it again has control of the -+ * core. This callback complements ->cpu_release(), which is -+ * emitted in hmbird_notify_pick_next_task(). -+ */ -+ get_hmbird_rq(rq)->cpu_released = false; -+ } -+ -+ if (prev_on_hmbird) -+ update_curr_hmbird(rq); -+ /* if there already are tasks to run, nothing to do */ -+ if (hmbird_rq->local_dsq.nr) -+ return 1; -+ -+ if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) { -+ slim_stats_record(GLOBAL_STAT, 1, 0, 0); -+ return 1; -+ } -+ -+ if (consume_dispatch_global(rq, rf)) -+ return 1; -+ -+ return 0; -+} -+ -+static int balance_hmbird(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf) -+{ -+ return balance_one(rq, prev, rf, true); -+} -+ -+/* -+ * output task util to systrace, only for debug mode. -+ * we can not output too many logs to systrace buffer even in debug mode -+ * only output debug-info while it exceed misfit_ds. -+ */ -+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p) -+{ -+ static DEFINE_PER_CPU(int, is_last_exceed); -+ int cpu = cpu_of(rq); -+ u64 util = 0; -+ -+ if (likely(!debug_enabled())) -+ return; -+ -+ if (!p) -+ return; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ util = uclamp_rq_util_with(rq, util, p); -+ -+ if (util >= misfit_ds) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util); -+ per_cpu(is_last_exceed, cpu) = true; -+ } else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0); -+ per_cpu(is_last_exceed, cpu) = false; -+ } else { -+ } -+} -+ -+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first) -+{ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ /* -+ * Core-sched might decide to execute @p before it is -+ * dispatched. Call ops_dequeue() to notify the BPF scheduler. -+ */ -+ ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC); -+ dispatch_dequeue(get_hmbird_rq(rq), p); -+ } -+ -+ p->se.exec_start = rq_clock_task(rq); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK); -+ } -+ -+ watchdog_unwatch_task(p, true); -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx); -+ systrace_output_cpu_ds(rq, p); -+ /* -+ * @p is getting newly scheduled or got kicked after someone updated its -+ * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird(). -+ */ -+ if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) != -+ (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) { -+ if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) -+ get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK; -+ else -+ get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK; -+ -+ sched_update_tick_dependency(rq); -+ } -+ -+ p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime; -+} -+ -+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p) -+{ -+ update_curr_hmbird(rq); -+ -+ update_dispatch_dsq_info(rq, p); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ watchdog_watch_task(rq, p); -+ -+ if (is_pipeline_task(p)) { -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1); -+ return; -+ } -+ -+ /* -+ * If we're in the pick_next_task path, balance_hmbird() should -+ * have already populated the local DSQ if there are any other -+ * available tasks. If empty, tell ops.enqueue() that @p is the -+ * only one available for this cpu. ops.enqueue() should put it -+ * on the local DSQ so that the subsequent pick_next_task_hmbird() -+ * can find the task unless it wants to trigger a separate -+ * follow-up scheduling event. -+ */ -+ if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1); -+ else -+ do_enqueue_task(rq, p, 0, -1); -+ } -+} -+ -+static struct task_struct *first_local_task(struct rq *rq) -+{ -+ struct rb_node *rb_node; -+ struct hmbird_entity *entity; -+ -+ if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) { -+ entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo, -+ struct hmbird_entity, dsq_node.fifo); -+ return entity->task; -+ } -+ -+ rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq); -+ if (rb_node) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ return entity->task; -+ } -+ return NULL; -+} -+ -+static struct task_struct *pick_next_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p; -+ -+ p = first_local_task(rq); -+ if (!p) -+ return NULL; -+ -+ if (unlikely(!get_hmbird_ts(p)->slice)) { -+ if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice) -+ hmbird_warned_zero_slice = true; -+ -+ refill_task_slice(p); -+ } -+ -+ set_next_task_hmbird(rq, p, true); -+ -+ return p; -+} -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task, -+ const struct sched_class *active) -+{ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * The callback is conceptually meant to convey that the CPU is no -+ * longer under the control of HMBIRD. Therefore, don't invoke the -+ * callback if the CPU is staying on HMBIRD, or going idle (in which -+ * case the HMBIRD scheduler has actively decided not to schedule any -+ * tasks on the CPU). -+ */ -+ if (likely(active >= &hmbird_sched_class)) -+ return; -+ -+ /* -+ * At this point we know that HMBIRD was preempted by a higher priority -+ * sched_class, so invoke the ->cpu_release() callback if we have not -+ * done so already. We only send the callback once between HMBIRD being -+ * preempted, and it regaining control of the CPU. -+ * -+ * ->cpu_release() complements ->cpu_acquire(), which is emitted the -+ * next time that balance_hmbird() is invoked. -+ */ -+ if (!get_hmbird_rq(rq)->cpu_released) -+ get_hmbird_rq(rq)->cpu_released = true; -+} -+ -+#ifdef CONFIG_SMP -+ -+static bool test_and_clear_cpu_idle(int cpu) -+{ -+ if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -+ if (cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ return true; -+ } else { -+ return false; -+ } -+} -+ -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) -+{ -+ int cpu; -+ -+ do { -+ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -+ if (cpu >= nr_cpu_ids) -+ return -EBUSY; -+ } while (!test_and_clear_cpu_idle(cpu)); -+ -+ return cpu; -+} -+ -+ -+static bool prev_cpu_misfit(int prev) -+{ -+ if (!is_partial_enabled() && is_partial_cpu(prev)) -+ return true; -+ -+ return false; -+} -+ -+static int heavy_rt_placement(struct task_struct *p, int prev) -+{ -+ struct cpumask tmp = {.bits = {0}}; -+ int cpu; -+ u64 util = 0; -+ -+ if (!rt_prio(p->prio)) -+ return -EFAULT; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ -+ if (util < misfit_ds) -+ return -EFAULT; -+ -+ if (is_partial_enabled()) -+ cpumask_or(&tmp, iso_masks.big, iso_masks.partial); -+ else -+ cpumask_copy(&tmp, iso_masks.big); -+ -+ cpu = hmbird_pick_idle_cpu(&tmp); -+ if (cpu >= 0) -+ return cpu; -+ -+ if (cpumask_test_cpu(prev, iso_masks.big) || -+ (is_partial_enabled() && is_partial_cpu(prev))) -+ return prev; -+ -+ return cpumask_first(&tmp); -+} -+ -+static int spec_task_before_pick_idle(struct task_struct *p, int prev) -+{ -+ int cpu; -+ -+ cpu = heavy_rt_placement(p, prev); -+ if (cpu >= 0) -+ return cpu; -+ return -EFAULT; -+} -+ -+static int cpumask_distribute_next(struct cpumask *mask, int *prev) -+{ -+ int p = *prev, n; -+ -+ n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1); -+ if (n < nr_cpu_ids) -+ WRITE_ONCE(*prev, n); -+ return n; -+} -+ -+static int repick_fallback_cpu(void) -+{ -+ /* -+ * partial cpu follow big cluster's Scheduling policy, -+ * simply return first bit cpu. -+ */ -+ return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev); -+} -+ -+/* -+ * Must return a valid cpu num, as this task's cpu. -+ */ -+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu, -+ struct cpumask *mask, int *prev_mask) -+{ -+ int cpu; -+ -+ cpu = hmbird_pick_idle_cpu(mask); -+ if (cpu >= 0) -+ return cpu; -+ return cpumask_distribute_next(mask, prev_mask); -+} -+ -+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type) -+{ -+ int idx; -+ struct cluster_ctx ctx; -+ -+ idx = find_idx_from_task(p); -+ if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS) -+ return false; -+ -+ gen_cluster_ctx(&ctx, type); -+ if (idx >= ctx.lower && idx < ctx.upper) -+ return true; -+ return false; -+} -+ -+static bool is_valid_cpu(int cpu) -+{ -+ return (cpu >= 0) && (cpu < nr_cpu_ids); -+} -+ -+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) -+{ -+ s32 cpu = -1; -+ struct cpumask mask = {.bits = {0}}; -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (is_valid_cpu(cpu)) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ partial_dynamic_ctrl(); -+ -+ if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ iso_masks.big, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ if (p->nr_cpus_allowed == 1) { -+ cpu = cpumask_any(p->cpus_ptr); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* For non-period global dsq, not contain pcp task. */ -+ if (task_only_blongs_to_cluster(p, LITTLE)) { -+ cpumask_copy(&mask, iso_masks.little); -+ if (unlikely(l_need_rescue)) { -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ cpumask_or(&mask, iso_masks.ex_free, &mask); -+ } -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &little_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ if (task_only_blongs_to_cluster(p, BIG)) { -+ cpumask_copy(&mask, iso_masks.big); -+ if (unlikely(b_need_rescue)) { -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ cpumask_or(&mask, iso_masks.ex_free, &mask); -+ } -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ cpu = spec_task_before_pick_idle(p, prev_cpu); -+ if (is_valid_cpu(cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* if the previous CPU is idle, dispatch directly to it */ -+ if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+ } -+ -+ cpu = hmbird_pick_idle_cpu(cpu_possible_mask); -+ if (is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ if (prev_cpu_misfit(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ cpu = repick_fallback_cpu(); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+} -+ -+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags) -+{ -+ return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags); -+} -+ -+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx) -+{ -+ set_cpus_allowed_common(p, ctx); -+} -+ -+static void reset_idle_masks(void) -+{ -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little); -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big); -+ if (is_partial_enabled()) -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial); -+ hmbird_has_idle_cpus = true; -+} -+ -+void __hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ int cpu = cpu_of(rq); -+ struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -+ -+ if (skip_update_idle()) -+ return; -+ -+ if (idle) { -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ if (!hmbird_has_idle_cpus) -+ hmbird_has_idle_cpus = true; -+ -+ /* -+ * idle_masks.smt handling is racy but that's fine as it's only -+ * for optimization and self-correcting. -+ */ -+ for_each_cpu(cpu, sib_mask) { -+ if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -+ return; -+ } -+ cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -+ } else { -+ cpumask_clear_cpu(cpu, idle_masks.cpu); -+ if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ -+ cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -+ } -+} -+ -+#else /* !CONFIG_SMP */ -+ -+static bool test_and_clear_cpu_idle(int cpu) { return false; } -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } -+static void reset_idle_masks(void) {} -+ -+#endif /* CONFIG_SMP */ -+ -+static bool check_rq_for_timeouts(struct rq *rq) -+{ -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rq_flags rf; -+ -+ rq_lock_irqsave(rq, &rf); -+ list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) { -+ unsigned long last_runnable; -+ -+ p = entity->task; -+ last_runnable = get_hmbird_ts(p)->runnable_at; -+ -+ if (unlikely(time_after(jiffies, -+ last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -+ -+ rq_unlock_irqrestore(rq, &rf); -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb", -+ p->comm, p->pid, -+ dur_ms / 1000, dur_ms % 1000, -+ get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0, -+ cpumask_pr_args(&p->cpus_mask)); -+ return 1; -+ } -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ return 0; -+} -+ -+static void hmbird_watchdog_workfn(struct work_struct *work) -+{ -+ int cpu; -+ bool timeout; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ -+ for_each_online_cpu(cpu) { -+ timeout = check_rq_for_timeouts(cpu_rq(cpu)); -+ if (unlikely(timeout)) -+ break; -+ -+ cond_resched(); -+ } -+ if (!timeout) -+ queue_delayed_work(system_unbound_wq, to_delayed_work(work), -+ hmbird_watchdog_timeout / 2); -+} -+ -+static void set_pcp_round(struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) { -+ per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round); -+ per_cpu(pcp_info, cpu).pcp_round = true; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+} -+ -+ -+/* -+ * Just for debug: output hmbird on/off state per 10s. -+ */ -+#define OUTPUT_INTVAL (msecs_to_jiffies(10 * 1000)) -+static void inform_hmbird_onoff_from_systrace(void) -+{ -+ static unsigned long __read_mostly next_print; -+ -+ if (time_before(jiffies, READ_ONCE(next_print))) -+ return; -+ -+ WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL); -+ hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio); -+} -+ -+void hmbird_notify_sched_tick(void) -+{ -+ unsigned long last_check; -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ hmbird_scheduler_tick(); -+ -+ if (!hmbird_enabled()) -+ return; -+ -+ last_check = hmbird_watchdog_timestamp; -+ if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -+ -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "watchdog failed to check in for %u.%03us", -+ dur_ms / 1000, dur_ms % 1000); -+ } -+ scan_timeout(rq); -+} -+ -+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued) -+{ -+ update_curr_hmbird(rq); -+ -+ set_pcp_round(rq); -+ -+ if (slim_walt_ctrl) -+ hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE); -+ /* -+ * While disabling, always resched and refresh core-sched timestamp as -+ * we can't trust the slice management or ops.core_sched_before(). -+ */ -+ if (hmbird_ops_disabling()) -+ get_hmbird_ts(curr)->slice = 0; -+ -+ if (!get_hmbird_ts(curr)->slice) -+ resched_curr(rq); -+ -+ inform_hmbird_onoff_from_systrace(); -+} -+ -+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg) -+{ -+ hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ get_hmbird_ts(p)->disallow = false; -+ -+ hmbird_sched_init_task(p); -+ -+ set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ return 0; -+} -+ -+static void hmbird_ops_enable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void hmbird_ops_disable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void set_task_hmbird_weight(struct task_struct *p) -+{ -+ u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -+ -+ get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight); -+} -+ -+/** -+ * refresh_hmbird_weight - Refresh a task's hmbird weight -+ * @p: task to refresh hmbird weight for -+ * -+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to -+ * enable easy access from the BPF scheduler. To keep it synchronized with the -+ * current task priority, this function should be called when a new task is -+ * created, priority is changed for a task on hmbird, and a task is switched -+ * to hmbird from other classes. -+ */ -+static void refresh_hmbird_weight(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ set_task_hmbird_weight(p); -+} -+ -+int hmbird_pre_fork(struct task_struct *p) -+{ -+ int ret = 0; -+ -+ p->android_oem_data1[HMBIRD_TS_IDX] = -+ (u64)(kzalloc(sizeof(struct hmbird_entity), GFP_KERNEL)); -+ if (!get_hmbird_ts(p)) -+ return -1; -+ -+ get_hmbird_ts(p)->dsq = NULL; -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node); -+ get_hmbird_ts(p)->flags = 0; -+ get_hmbird_ts(p)->weight = 0; -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ get_hmbird_ts(p)->holding_cpu = -1; -+ get_hmbird_ts(p)->kf_mask = 0; -+ atomic64_set(&get_hmbird_ts(p)->ops_state, 0); -+ get_hmbird_ts(p)->runnable_at = INITIAL_JIFFIES; -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+ get_hmbird_ts(p)->task = p; -+ hmbird_set_sched_prop(p, 0); -+ -+ get_hmbird_ts(p)->critical_affinity_cpu = -1; -+ get_hmbird_ts(p)->sched_class = &hmbird_sched_class; -+ get_hmbird_ts(p)->tick_hit_count = 0; -+ get_hmbird_ts(p)->start_jiffies = 0; -+ -+ /* -+ * BPF scheduler enable/disable paths want to be able to iterate and -+ * update all tasks which can become complex when racing forks. As -+ * enable/disable are very cold paths, let's use a percpu_rwsem to -+ * exclude forks. -+ */ -+ -+ return ret; -+} -+ -+void hmbird_post_fork(struct task_struct *p) -+{ -+ percpu_down_read(&hmbird_fork_rwsem); -+ -+ if (hmbird_enabled()) { -+ struct rq_flags rf; -+ struct rq *rq; -+ p->sched_class = &hmbird_sched_class; -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ rq = task_rq_lock(p, &rf); -+ /* -+ * Set the weight manually before calling ops.enable() so that -+ * the scheduler doesn't see a stale value if they inspect the -+ * task struct. We'll invoke ops.set_weight() afterwards, as it -+ * would be odd to receive a callback on the task before we -+ * tell the scheduler that it's been fully enabled. -+ */ -+ set_task_hmbird_weight(p); -+ hmbird_ops_enable_task(p); -+ refresh_hmbird_weight(p); -+ task_rq_unlock(rq, p, &rf); -+ } else { -+ if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ } -+ -+ spin_lock_irq(&hmbird_tasks_lock); -+ list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_cancel_fork(struct task_struct *p) -+{ -+ if (hmbird_enabled()) -+ hmbird_ops_disable_task(p); -+ put_hmbird_ts(p); -+} -+ -+void hmbird_free(struct task_struct *p) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+ list_del_init(&get_hmbird_ts(p)->tasks_node); -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+ -+ /* -+ * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED -> -+ * ENABLED transitions can't race us. Disable ops for @p. -+ */ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) || -+ test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ hmbird_ops_disable_task(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ put_hmbird_ts(p); -+} -+ -+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio) -+{ -+} -+ -+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type) -+{ -+ return (prop >> TOP_TASK_SHIFT) & (1 << type); -+} -+ -+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p) -+{ -+ uint32_t prop = get_top_task_prop(p); -+ -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE)) -+ return HMBIRD_TASK_PROP_PIPELINE; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) || -+ !task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) { -+ return HMBIRD_TASK_PROP_COMMON; -+ } -+ return HMBIRD_TASK_PROP_DEBUG_OR_LOG; -+} -+ -+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b) -+{ -+ int type_a = hmbird_get_task_type(a); -+ int type_b = hmbird_get_task_type(b); -+ -+ return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b]; -+} -+ -+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ enum cpu_type type; -+ int sp_dl; -+ struct task_struct *curr = NULL; -+ -+ switch (hmbird_preempt_policy) { -+ case HMBIRD_PREEMPT_POLICY_PRIO_BASED: -+ curr = rq->curr; -+ if (curr && hmbird_prio_higher(p, curr)) -+ goto preempt; -+ break; -+ default: -+ break; -+ } -+ if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) || -+ (is_critical_system_task(p) && !is_critical_system_task(rq->curr))) -+ goto preempt; -+ -+ sp_dl = find_idx_from_task(p); -+ if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled())) -+ return; -+ -+ if (rq->curr->prio > p->prio) -+ goto preempt; -+ -+ return; -+ -+preempt: -+ resched_curr(rq); -+} -+ -+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {} -+ -+int hmbird_check_setscheduler(struct task_struct *p, int policy) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ /* if disallow, reject transitioning into HMBIRD */ -+ if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) && -+ p->policy != policy && policy == SCHED_HMBIRD) -+ return -EACCES; -+ -+ return 0; -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool hmbird_can_stop_tick(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ if (hmbird_ops_disabling()) -+ return false; -+ -+ if (p->sched_class != &hmbird_sched_class) -+ return true; -+ -+ return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK; -+} -+#endif -+ -+int hmbird_tg_online(struct task_group *tg) -+{ -+ struct cgroup *cgrp; -+ -+ if (!tg) -+ return 0; -+ cgrp = tg->css.cgroup; -+ if (!cgrp || !(cgrp->kn)) -+ return 0; -+ update_cgroup_ids_table(cgrp->kn->id, -1); -+ return 0; -+} -+ -+/* -+ * Omitted operations: -+ * -+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -+ * task isn't tied to the CPU at that point. Preemption is implemented by -+ * resetting the victim task's slice to 0 and triggering reschedule on the -+ * target CPU. -+ * -+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -+ * -+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -+ * their current sched_class. Call them directly from sched core instead. -+ * -+ * - task_woken, switched_from: Unnecessary. -+ */ -+DEFINE_SCHED_CLASS(hmbird) = { -+ .enqueue_task = enqueue_task_hmbird, -+ .dequeue_task = dequeue_task_hmbird, -+ .yield_task = yield_task_hmbird, -+ .yield_to_task = yield_to_task_hmbird, -+ -+ .check_preempt_curr = check_preempt_curr_hmbird, -+ -+ .pick_next_task = pick_next_task_hmbird, -+ -+ .put_prev_task = put_prev_task_hmbird, -+ .set_next_task = set_next_task_hmbird, -+ -+#ifdef CONFIG_SMP -+ .balance = balance_hmbird, -+ .select_task_rq = select_task_rq_hmbird, -+ .set_cpus_allowed = set_cpus_allowed_hmbird, -+#endif -+ .task_tick = task_tick_hmbird, -+ -+ .switched_to = switched_to_hmbird, -+ .prio_changed = prio_changed_hmbird, -+ -+ .update_curr = update_curr_hmbird, -+ -+#ifdef CONFIG_UCLAMP_TASK -+ .uclamp_enabled = 0, -+#endif -+}; -+ -+/* -+ * Must with rq lock held. -+ */ -+bool task_is_hmbird(struct task_struct *p) -+{ -+ return p->sched_class == &hmbird_sched_class; -+} -+ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id) -+{ -+ memset(dsq, 0, sizeof(*dsq)); -+ -+ raw_spin_lock_init(&dsq->lock); -+ INIT_LIST_HEAD(&dsq->fifo); -+ dsq->id = dsq_id; -+} -+ -+static int hmbird_cgroup_init(void) -+{ -+ struct cgroup_subsys_state *css; -+ -+ css_for_each_descendant_pre(css, &root_task_group.css) { -+ struct task_group *tg = css_tg(css); -+ -+ cgrp_dsq_idx_init(css->cgroup, tg); -+ } -+ return 0; -+} -+ -+/* -+ * Used by sched_fork() and __setscheduler_prio() to pick the matching -+ * sched_class. dl/rt are already handled. -+ */ -+bool task_on_hmbird(struct task_struct *p) -+{ -+ return hmbird_enabled(); -+} -+ -+static void __setscheduler_prio(struct task_struct *p, int prio) -+{ -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) { -+ p->prio = prio; -+ return; -+ } else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (on_hmbird && rt_prio(prio)) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ -+ p->prio = prio; -+} -+ -+/* -+ * Heartbeat, avoid humbird keep running while APP already exit. -+ * Check whether APP send alive-signal periodly. -+ */ -+#define HEARTBEAT_TIMEOUT (msecs_to_jiffies(2500)) -+#define HEARTBEAT_CHECK_INTERVAL (msecs_to_jiffies(1000)) -+static struct timer_list hb_timer; -+static unsigned long next_hb; -+ -+void hb_timer_handler(struct timer_list *timer) -+{ -+ if (!heartbeat_enable) -+ goto refill; -+ -+ pr_info(": enter timer.\n"); -+ if (heartbeat) { -+ heartbeat = 0; -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ } -+ -+ /* can't detect heartbeat, disable ext. */ -+ if (time_after(jiffies, READ_ONCE(next_hb))) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT, -+ "can't detect heartbeat, disable ext\n"); -+ } -+refill: -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_start(void) -+{ -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_init(void) -+{ -+ timer_setup(&hb_timer, hb_timer_handler, 0); -+} -+ -+static void hb_timer_exit(void) -+{ -+ del_timer(&hb_timer); -+} -+ -+static bool check_and_disable_cpuhp(void) -+{ -+ struct rq *rq; -+ struct rq_flags rf; -+ int cpu; -+ -+ cpu_hotplug_disable(); -+ cpus_read_lock(); -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ rq_lock_irqsave(rq, &rf); -+ if (!rq->online) { -+ rq_unlock_irqrestore(rq, &rf); -+ goto err_offline; -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ cpus_read_unlock(); -+ return true; -+ -+err_offline: -+ cpus_read_unlock(); -+ cpu_hotplug_enable(); -+ return false; -+} -+ -+static void reenable_cpuhp(void) -+{ -+ cpu_hotplug_enable(); -+} -+ -+static void scheduler_switch_done(bool final_state) -+{ -+ /* set scx_enable again, switch may fail. */ -+ scx_enable = final_state; -+ /* reset heartbeat*/ -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ -+ if (final_state) -+ hb_timer_start(); -+ else -+ hb_timer_exit(); -+} -+ -+/** -+ * ss : curr switch state -+ * finish : success or fail -+ * enable : curr operation is diabling or enabling? -+ * fail_reason : string output when failed, empty string when success. -+ */ -+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason) -+{ -+ char *s1 = finish ? "finished" : "failed"; -+ char *s2 = enable ? "enabled" : "disabled"; -+ -+ hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss); -+ if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) { -+ sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type)); -+ hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n", -+ s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason); -+ } -+ curr_ss = ss; -+} -+ -+bool get_hmbird_ops_enabled(void) -+{ -+ return atomic_read(&__hmbird_ops_enabled); -+} -+ -+bool get_non_hmbird_task(void) -+{ -+ return atomic_read(&non_hmbird_task); -+} -+ -+static void hmbird_ops_disable_workfn(struct kthread_work *work) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int cpu; -+ -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, ""); -+ cancel_delayed_work_sync(&hmbird_watchdog_work); -+ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) { -+ case HMBIRD_OPS_DISABLED: -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled"); -+ scheduler_switch_done(false); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return; -+ case HMBIRD_OPS_PREPPING: -+ fallthrough; -+ case HMBIRD_OPS_DISABLING: -+ /* shouldn't happen but handle it like ENABLING if it does */ -+ WARN_ONCE(true, "hmbird: duplicate disabling instance?"); -+ fallthrough; -+ case HMBIRD_OPS_ENABLING: -+ case HMBIRD_OPS_ENABLED: -+ break; -+ } -+ -+ /* kick all CPUs to restore ticks */ -+ for_each_possible_cpu(cpu) -+ resched_cpu(cpu); -+ -+ /* avoid racing against fork and cgroup changes */ -+ cpus_read_lock(); -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, ""); -+ spin_lock_irq(&hmbird_tasks_lock); -+ atomic_set(&__hmbird_ops_enabled, false); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ bool alive = READ_ONCE(p->__state) != TASK_DEAD; -+ -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ get_hmbird_ts(p)->slice = min_t(u64, -+ get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL); -+ -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ if (alive) -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ -+ hmbird_ops_disable_task(p); -+ } -+ hmbird_task_iter_exit(&sti); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, ""); -+ -+ atomic_set(&non_hmbird_task, true); -+ /* no task is on hmbird, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt); -+ synchronize_rcu(); -+ -+ percpu_up_write(&hmbird_fork_rwsem); -+ cpus_read_unlock(); -+ -+ if (slim_walt_ctrl) -+ slim_walt_enable(false); -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ -+ hmbird_switch_log(HMBIRD_DISABLED, 1, 0, ""); -+ scheduler_switch_done(false); -+ reenable_cpuhp(); -+} -+ -+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn); -+ -+static void schedule_hmbird_ops_disable_work(void) -+{ -+ struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper); -+ -+ /* -+ * We may be called spuriously before the first bpf_hmbird_reg(). If -+ * hmbird_ops_helper isn't set up yet, there's nothing to do. -+ */ -+ if (helper) -+ kthread_queue_work(helper, &hmbird_ops_disable_work); -+} -+ -+static void hmbird_ops_disable(void) -+{ -+ schedule_hmbird_ops_disable_work(); -+} -+ -+static void hmbird_err_exit_workfn(struct work_struct *work) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ hmbird_ctrl(false); -+ -+ for_each_present_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy) -+ continue; -+ if (cpu != policy->cpu) -+ goto put; -+ down_write(&policy->rwsem); -+ WARN_ON(store_scaling_governor(policy, -+ saved_gov[cpu], strlen(saved_gov[cpu])) <= 0); -+ up_write(&policy->rwsem); -+ hmbird_info_trace(":restore origin gov : %s\n", saved_gov[cpu]); -+put: -+ cpufreq_cpu_put(policy); -+ } -+ memset((char *)saved_gov, 0, sizeof(saved_gov)); -+} -+ -+void hmbird_ops_exit(void) -+{ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); -+} -+ -+static struct kthread_worker *hmbird_create_rt_helper(const char *name) -+{ -+ struct kthread_worker *helper; -+ -+ helper = kthread_create_worker(KTW_FREEZABLE, name); -+ if (helper) -+ sched_set_fifo(helper->task); -+ return helper; -+} -+ -+static inline void set_audio_thread_sched_prop(struct task_struct *p) -+{ -+ struct cgroup_subsys_state *css; -+ -+ if (likely(p->prio >= MAX_RT_PRIO)) -+ return; -+ -+ rcu_read_lock(); -+ css = task_css(p, cpuset_cgrp_id); -+ if (!css) { -+ rcu_read_unlock(); -+ return; -+ } -+ rcu_read_unlock(); -+ -+ if (!strcmp(css->cgroup->kn->name, "audio-app")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+} -+ -+int scx_systemui_pid = -1; -+void set_systemui_thread_pid(struct task_struct *p) -+{ -+ if ((strcmp(p->comm, "ndroid.systemui") == 0) && (p->pid == p->tgid)) -+ scx_systemui_pid = p->pid; -+} -+ -+static int hmbird_ops_enable(void *unused) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int ret; -+ int tcnt = 0; -+ unsigned long long start = 0; -+ -+ if (!check_and_disable_cpuhp()) { -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline"); -+ return -EBUSY; -+ } -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, ""); -+ mutex_lock(&hmbird_ops_enable_mutex); -+ -+ if (!hmbird_ops_helper) { -+ WRITE_ONCE(hmbird_ops_helper, -+ hmbird_create_rt_helper("hmbird_ops_helper")); -+ if (!hmbird_ops_helper) { -+ ret = -ENOMEM; -+ goto err_unlock; -+ } -+ } -+ -+ if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) { -+ ret = -EBUSY; -+ goto err_unlock; -+ } -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) != -+ HMBIRD_OPS_DISABLED); -+ -+ hmbird_warned_zero_slice = false; -+ -+ atomic64_set(&hmbird_nr_rejected, 0); -+ -+ /* -+ * Keep CPUs stable during enable so that the BPF scheduler can track -+ * online CPUs by watching ->on/offline_cpu() after ->init(). -+ */ -+ cpus_read_lock(); -+ -+ hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work, -+ hmbird_watchdog_timeout / 2); -+ -+ /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ */ -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ reset_idle_masks(); -+ -+ /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. -+ */ -+ ret = hmbird_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock; -+ -+ /* -+ * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem -+ * preventing new tasks from being added. No need to exclude tasks -+ * leaving as hmbird_free() can handle both prepped and enabled -+ * tasks. Prep all tasks first and then enable them with preemption -+ * disabled. -+ */ -+ spin_lock_irq(&hmbird_tasks_lock); -+ -+ atomic_set(&non_hmbird_task, false); -+ atomic_set(&__hmbird_ops_enabled, true); -+ -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered(&sti))) -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ hmbird_task_iter_exit(&sti); -+ -+ /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. -+ */ -+ preempt_disable(); -+ -+ /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ */ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) { -+ atomic_set(&non_hmbird_task, true); -+ atomic_set(&__hmbird_ops_enabled, false); -+ preempt_enable(); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ ret = -EBUSY; -+ goto err_disable_unlock; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED -+ * transitions here are synchronized against hmbird_free() through -+ * hmbird_tasks_lock. -+ */ -+ start = sched_clock(); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, ""); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ tcnt++; -+ if (READ_ONCE(p->__state) != TASK_DEAD) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ -+ set_audio_thread_sched_prop(p); -+ set_systemui_thread_pid(p); -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ hmbird_ops_enable_task(p); -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ } else { -+ hmbird_ops_disable_task(p); -+ } -+ } -+ hmbird_task_iter_exit(&sti); -+ -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, ""); -+ preempt_enable(); -+ percpu_up_write(&hmbird_fork_rwsem); -+ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) { -+ ret = -EBUSY; -+ goto err_disable; -+ } -+ -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_ENABLED, 1, 1, ""); -+ scheduler_switch_done(true); -+ -+ return 0; -+ -+err_unlock: -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock"); -+ scheduler_switch_done(false); -+ return ret; -+ -+err_disable_unlock: -+ percpu_up_write(&hmbird_fork_rwsem); -+err_disable: -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ /* must be fully disabled before returning */ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable"); -+ scheduler_switch_done(false); -+ return ret; -+} -+ -+#ifdef CONFIG_SCHED_DEBUG -+static const char *hmbird_ops_enable_state_str[] = { -+ [HMBIRD_OPS_PREPPING] = "prepping", -+ [HMBIRD_OPS_ENABLING] = "enabling", -+ [HMBIRD_OPS_ENABLED] = "enabled", -+ [HMBIRD_OPS_DISABLING] = "disabling", -+ [HMBIRD_OPS_DISABLED] = "disabled", -+}; -+ -+static int hmbird_debug_show(struct seq_file *m, void *v) -+{ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled()); -+ seq_printf(m, "%-30s: %s\n", "enable_state", -+ hmbird_ops_enable_state_str[hmbird_ops_enable_state()]); -+ seq_printf(m, "%-30s: %llu\n", "nr_rejected", -+ atomic64_read(&hmbird_nr_rejected)); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return 0; -+} -+ -+static int hmbird_debug_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_debug_show, NULL); -+} -+ -+const struct file_operations sched_hmbird_fops = { -+ .open = hmbird_debug_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif -+ -+ -+static int bpf_hmbird_reg(void *kdata) -+{ -+ return hmbird_ops_enable(kdata); -+} -+ -+static int bpf_hmbird_unreg(void *kdata) -+{ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ return 0; -+} -+ -+void set_hmbird_module_loaded(int is_loaded) -+{ -+ atomic_set(&hmbird_module_loaded, is_loaded); -+} -+ -+/* -+ * MUST load hmbird module before enable hmbird scheduler -+ * load track & hmbird gover implemented in hmbird module -+ */ -+int hmbird_ctrl(bool enable) -+{ -+ if (!atomic_read(&hmbird_module_loaded)) { -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "ext module unloaded\n"); -+ return -EINVAL; -+ } -+ -+ if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED -+ || hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING -+ || hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already enabled(ing)\n"); -+ hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING || -+ hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already disabled(ing)\n"); -+ hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (enable) -+ return bpf_hmbird_reg(NULL); -+ else -+ return bpf_hmbird_unreg(NULL); -+} -+ -+void set_cpu_isomask(int cpu, cpumask_var_t *mask) -+{ -+ cpumask_clear_cpu(cpu, iso_masks.ex_free); -+ cpumask_clear_cpu(cpu, iso_masks.exclusive); -+ cpumask_clear_cpu(cpu, iso_masks.partial); -+ cpumask_clear_cpu(cpu, iso_masks.big); -+ cpumask_clear_cpu(cpu, iso_masks.little); -+ cpumask_set_cpu(cpu, *mask); -+} -+ -+void set_cpu_cluster(u64 cpu_cluster) -+{ -+ int cpu; -+ -+ for_each_present_cpu(cpu) { -+ u64 pos = 1 << cpu; -+ -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.ex_free)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.exclusive)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.partial)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.big)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) -+ set_cpu_isomask(cpu, &(iso_masks.little)); -+ } -+} -+ -+static void get_hmbird_snapshot(struct panic_snapshot_t *p) -+{ -+ struct hmbird_dispatch_q *dsq; -+ struct rq *rq; -+ int cpu, i; -+ -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ p->rq_nr[cpu] = rq->nr_running; -+ p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running; -+ } -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ struct hmbird_entity *entity; -+ -+ dsq = &gdsqs[i]; -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo)) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ p->runnable_at[i] = entity->runnable_at; -+ raw_spin_unlock(&dsq->lock); -+ -+ } -+ -+ p->snap_misc.hmbird_enabled = hmbird_enabled(); -+ p->snap_misc.curr_ss = curr_ss; -+ p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state(); -+ p->snap_misc.parctrl_high_ratio = parctrl_high_ratio; -+ p->snap_misc.parctrl_low_ratio = parctrl_low_ratio; -+ p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l; -+ p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l; -+ p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio; -+ p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio; -+ p->snap_misc.misfit_ds = misfit_ds; -+ p->snap_misc.partial_enable = partial_enable; -+ p->snap_misc.iso_free_rescue = iso_free_rescue; -+ p->snap_misc.isolate_ctrl = isolate_ctrl; -+ p->snap_misc.snap_jiffies = jiffies; -+ p->snap_misc.snap_time = local_clock(); -+} -+ -+// MTK minidump begin -+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3) -+{ -+ strscpy(m->desc_str, str, DESC_STR_LEN); -+ m->len = d1 * d2 * d3; -+ m->parse[0] = d1; -+ m->parse[1] = d2; -+ m->parse[2] = d3; -+} -+ -+static void init_desc_metas(struct md_info_t *m) -+{ -+ init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :", -+ 1, MAX_SWITCHS, SWITCH_ITEMS); -+ -+ init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1); -+ -+ init_desc_meta(&m->kern_dump.excep_rec_meta, -+ "excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS); -+ -+ init_desc_meta(&m->kern_dump.excep_idx_meta, -+ "excep idx :", 1, 1, MAX_EXCEP_ID); -+ -+ init_desc_meta(&m->kern_dump.snap.runnable_at_meta, -+ "each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS); -+ -+ init_desc_meta(&m->kern_dump.snap.rq_nr_meta, -+ "rq runnable task nr:", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta, -+ "scxrq runnable task nr :", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.snap_misc_meta, -+ "misc snap params :", 1, 1, SNAP_ITEMS); -+} -+ -+static void init_md_meta(struct md_info_t *m) -+{ -+ m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64); -+ m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64); -+ m->meta.unit_size = sizeof(u64); -+ m->meta.switches = MAX_SWITCHS; -+ m->meta.exceps = MAX_EXCEPS; -+ m->meta.global_dsqs = MAX_GLOBAL_DSQS; -+ m->meta.parse_dimens = PARSE_DIMENS; -+ m->meta.nr_cpus = num_possible_cpus(); -+ m->meta.real_cpus = nr_cpu_ids; -+ m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64); -+ m->meta.nr_meta_desc = 8; -+ m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64); -+ -+ init_desc_metas(m); -+} -+ -+#define MINIDUMP_DFL_SIZE (4 * 1024) -+ -+struct notifier_block hmbird_panic_blk; -+static int hmbird_panic_handler(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ if (!md_info) -+ return NOTIFY_DONE; -+ -+ get_hmbird_snapshot(&md_info->kern_dump.snap); -+ -+ return NOTIFY_DONE; -+} -+ -+static void panic_blk_init(void) -+{ -+ int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE); -+ -+ md_info = kzalloc(dump_size, GFP_KERNEL); -+ if (!md_info) -+ return; -+ init_md_meta(md_info); -+ -+ hmbird_panic_blk.notifier_call = hmbird_panic_handler; -+ /* make sure to execute before minidump. */ -+ hmbird_panic_blk.priority = INT_MAX; -+ atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk); -+ -+ hmbird_debug("register minidump.\n"); -+} -+ -+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size) -+{ -+ *vaddr = (unsigned long)md_info; -+ *size = sizeof(struct md_info_t); -+} -+// MTK minidump end -+ -+static inline void init_sched_prop_to_preempt_prio(void) -+{ -+ for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) { -+ switch (i) { -+ case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 5; -+ break; -+ -+ case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 4; -+ break; -+ -+ case HMBIRD_TASK_PROP_PIPELINE: -+ case HMBIRD_TASK_PROP_ISOLATE: -+ sched_prop_to_preempt_prio[i] = 3; -+ break; -+ -+ case HMBIRD_TASK_PROP_COMMON: -+ sched_prop_to_preempt_prio[i] = 1; -+ break; -+ -+ case HMBIRD_TASK_PROP_DEBUG_OR_LOG: -+ sched_prop_to_preempt_prio[i] = 0; -+ break; -+ -+ default: -+ sched_prop_to_preempt_prio[i] = 2; -+ break; -+ } -+ } -+} -+ -+void __init init_sched_hmbird_class(void) -+{ -+ int cpu; -+ u32 v; -+ struct hmbird_entity *init_hmbird; -+ -+ /* -+ * The following is to prevent the compiler from optimizing out the enum -+ * definitions so that BPF scheduler implementations can use them -+ * through the generated vmlinux.h. -+ */ -+ WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP | -+ HMBIRD_KICK_PREEMPT); -+ -+ init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL); -+ init_dsq_at_boot(); -+ init_isolate_cpus(); -+ hb_timer_init(); -+ init_sched_prop_to_preempt_prio(); -+#ifdef CONFIG_SMP -+ WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); -+#endif -+ -+ /* -+ * we can't static init init_task's hmbird struct, init here. -+ * init_task->hmbird would not use during boot. -+ */ -+ init_hmbird = kzalloc(sizeof(struct hmbird_entity), GFP_KERNEL); -+ init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird; -+ if (init_hmbird) { -+ INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo); -+ INIT_LIST_HEAD(&init_hmbird->watchdog_node); -+ init_hmbird->sticky_cpu = -1; -+ init_hmbird->holding_cpu = -1; -+ atomic64_set(&init_hmbird->ops_state, 0); -+ init_hmbird->runnable_at = jiffies; -+ init_hmbird->slice = HMBIRD_SLICE_DFL; -+ init_hmbird->task = &init_task; -+ hmbird_set_sched_prop(&init_task, 0); -+ } else { -+ hmbird_err(INIT_TASK_FAIL, ":alloc init_task.scx failed!!!\n"); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ /* -+ * exec during boot phase, no need to care about alloc failed. -+ * lifecycle same to rq, no need to free. -+ */ -+ rq->android_oem_data1[HMBIRD_RQ_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_rq), GFP_KERNEL); -+ if (get_hmbird_rq(rq)) { -+ get_hmbird_rq(rq)->rq = rq; -+ get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window; -+ } else { -+ hmbird_err(ALLOC_RQSCX_FAIL, ":alloc rq->scx failed!!!\n"); -+ } -+ -+ rq->android_oem_data1[HMBIRD_OPS_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL); -+ if (!get_hmbird_ops(rq)) -+ pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n"); -+ init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL); -+ INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list); -+ -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL)); -+ -+ hmbird_ops_init(get_hmbird_ops(rq)); -+ } -+ -+ INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn); -+ INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn); -+ hmbird_misc_init(); -+ -+ panic_blk_init(); -+} -+ -diff --git b/kernel/sched/hmbird/hmbird_misc.c a/kernel/sched/hmbird/hmbird_misc.c -new file mode 100755 -index 000000000000..895e8a29df33 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_misc.c -@@ -0,0 +1,148 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * @Description: -+ * @Version: -+ * @Author: wangrui8 -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include "hmbird_sched.h" -+#include -+#include -+#include "slim.h" -+ -+noinline int tracing_mark_write(const char *buf) -+{ -+ trace_printk(buf); -+ return 0; -+} -+ -+struct yield_opt_params yield_opt_params = { -+ .enable = 0, -+ .frame_per_sec = 120, -+ .frame_time_ns = NSEC_PER_SEC / 120, -+ .yield_headroom = 10, -+}; -+ -+DEFINE_PER_CPU(struct sched_yield_state, ystate); -+ -+static inline void hmbird_yield_state_update(struct sched_yield_state *ys) -+{ -+ if (!raw_spin_is_locked(&ys->lock)) -+ return; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ -+ if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1 -+ || ys->yield_cnt_after_sleep > yield_headroom) { -+ ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP); -+ } else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) { -+ ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP); -+ } -+ ys->yield_cnt = 0; -+ ys->sleep_times = 0; -+ ys->yield_cnt_after_sleep = 0; -+} -+ -+void hmbird_skip_yield(long *skip) -+{ -+ if (!get_hmbird_ops_enabled() || !yield_opt_params.enable) -+ return; -+ unsigned long flags, sleep_now = 0; -+ struct sched_yield_state *ys; -+ int cpu = raw_smp_processor_id(), cont_yield, new_frame; -+ int frame_time_ns = yield_opt_params.frame_time_ns; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ u64 wc; -+ -+ if (!(*skip)) { -+ wc = sched_clock(); -+ ys = &per_cpu(ystate, cpu); -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ -+ cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP; -+ new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1); -+ -+ if (!cont_yield && new_frame) { -+ hmbird_yield_state_update(ys); -+ ys->last_update_time = wc; -+ ys->sleep_end = ys->last_yield_time + frame_time_ns -+ - yield_headroom * YIELD_DURATION; -+ } -+ -+ if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) { -+ *skip = true; -+ -+ sleep_now = ys->sleep_times ? -+ max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep; -+ if (wc + sleep_now > ys->sleep_end) { -+ u64 delta = ys->sleep_end - wc; -+ -+ if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION) -+ sleep_now = delta; -+ else -+ sleep_now = 0; -+ } -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ if (sleep_now) { -+ sleep_now = div64_u64(sleep_now, 1000); -+ usleep_range_state(sleep_now, sleep_now, TASK_IDLE); -+ } -+ ys->sleep_times++; -+ ys->last_yield_time = sched_clock(); -+ return; -+ } -+ if (ys->sleep_times) -+ ys->yield_cnt_after_sleep++; -+ else -+ (ys->yield_cnt)++; -+ ys->last_yield_time = wc; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+} -+ -+struct boost_policy_params boost_policy_params = { -+ .enable = 0, -+ .bottom_freq = 1200000, -+ .boost_weight = 120, -+}; -+ -+int hmbird_get_boost_enable(void) -+{ -+ return boost_policy_params.enable; -+} -+ -+unsigned int hmbird_get_boost_bottom_freq(void) -+{ -+ return boost_policy_params.bottom_freq; -+} -+ -+int hmbird_get_boost_weight(void) -+{ -+ return boost_policy_params.boost_weight; -+} -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops) -+{ -+ hmbird_ops->scx_enable = get_hmbird_ops_enabled; -+ hmbird_ops->check_non_task = get_non_hmbird_task; -+ hmbird_ops->hmbird_get_md_info = hmbird_get_md_info; -+ hmbird_ops->hmbird_get_boost_enable = hmbird_get_boost_enable; -+ hmbird_ops->hmbird_get_boost_bottom_freq = hmbird_get_boost_bottom_freq; -+ hmbird_ops->hmbird_get_boost_weight = hmbird_get_boost_weight; -+} -+ -+void hmbird_misc_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_init(&ys->lock); -+ } -+ -+ set_hmbird_module_loaded(1); -+} -+ -diff --git b/kernel/sched/hmbird/hmbird_sched.h a/kernel/sched/hmbird/hmbird_sched.h -new file mode 100755 -index 000000000000..76acbe9685e4 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef __HMBIRD_SCHED__ -+#define __HMBIRD_SCHED__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include <../../kernel/time/tick-sched.h> -+#include <../../kernel/sched/sched.h> -+#include -+#include -+#include -+ -+#define REGISTER_TRACE_VH(vender_hook, handler) \ -+{ \ -+ ret = register_trace_##vender_hook(handler, NULL); \ -+ if (ret) { \ -+ pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \ -+ } \ -+} -+#define REGISTER_TRACE(vendor_hook, handler, data, err) \ -+do { \ -+ ret = register_trace_##vendor_hook(handler, data); \ -+ if (ret) { \ -+ pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret); \ -+ goto err; \ -+ } \ -+} while (0) -+ -+#define UNREGISTER_TRACE(vendor_hook, handler, data) \ -+ unregister_trace_##vendor_hook(handler, data) \ -+ -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+ -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int sched_ravg_window_frame_per_sec; -+extern int slim_gov_debug; -+extern int cpu7_tl; -+extern int scx_gov_ctrl; -+extern spinlock_t new_sched_ravg_window_lock; -+extern int cluster_separate; -+ -+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER BIT(31) -+#define MAX_YIELD_SLEEP (2000000ULL) -+#define MIN_YIELD_SLEEP (200000ULL) -+#define YIELD_DURATION (5000ULL) -+#define DEFAULT_YIELD_SLEEP_TH (10) -+ -+struct sched_yield_state { -+ raw_spinlock_t lock; -+ u64 last_yield_time; -+ u64 last_update_time; -+ u64 sleep_end; -+ unsigned long yield_cnt; -+ unsigned long yield_cnt_after_sleep; -+ unsigned long sleep; -+ int sleep_times; -+}; -+ -+DECLARE_PER_CPU(struct sched_yield_state, ystate); -+ -+void hmbird_window_rollover_run_once(struct rq *rq); -+void hmbird_yield_state_update_per_frame(void); -+void hmbird_misc_init(void); -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops); -+#endif /*__HMBIRD_SCHED__*/ -diff --git b/kernel/sched/hmbird/hmbird_sched_proc.c a/kernel/sched/hmbird/hmbird_sched_proc.c -new file mode 100755 -index 000000000000..295d45404608 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched_proc.c -@@ -0,0 +1,640 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "hmbird_sched_proc.h" -+#include -+ -+#include "hmbird_util_track.h" -+#include "slim.h" -+ -+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched" -+#define SLIM_FREQ_GOV_DIR "slim_freq_gov" -+#define LOAD_TRACK_DIR "slim_walt" -+#define HMBIRD_PROC_PERMISSION 0666 -+ -+int scx_enable; -+int partial_enable; -+int cpuctrl_high_ratio = 55; -+int cpuctrl_low_ratio = 40; -+int slim_stats; -+int hmbirdcore_debug; -+int slim_for_app; -+int misfit_ds = 90; -+unsigned int highres_tick_ctrl; -+unsigned int highres_tick_ctrl_dbg; -+int cpu7_tl = 70; -+int slim_walt_ctrl; -+int slim_walt_dump; -+int slim_walt_policy; -+int slim_gov_debug; -+int scx_gov_ctrl = 1; -+int sched_ravg_window_frame_per_sec = 125; -+int parctrl_high_ratio = 55; -+int parctrl_low_ratio = 40; -+int parctrl_high_ratio_l = 65; -+int parctrl_low_ratio_l = 50; -+int isoctrl_high_ratio = 75; -+int isoctrl_low_ratio = 60; -+int isolate_ctrl; -+int iso_free_rescue; -+int heartbeat; -+int heartbeat_enable = 1; -+int watchdog_enable; -+int save_gov; -+u64 cpu_cluster_masks; -+int hmbird_preempt_policy; -+int cluster_separate; -+ -+char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= 32) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtoint(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoint\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+static int set_proc_buf_val_u64(struct file *file, const char __user *buf, -+ size_t count, u64 *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= sizeof(kbuf)) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtou64(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoul\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/* common ops begin */ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ return count; -+} -+ -+static int hmbird_common_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%d\n", *(int *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_show, pde_data(inode)); -+} -+ -+static int hmbird_common_ul_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%lu\n", *(unsigned long *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_ul_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_ul_show, pde_data(inode)); -+} -+ -+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write); -+/* common ops end */ -+ -+/* scx_enable ops begin */ -+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC); -+ if (hmbird_ctrl(*pval)) -+ return -EFAULT; -+ -+ return count; -+} -+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write); -+/* scx_enable ops end */ -+ -+/* hmbird_stats ops begin */ -+#define MAX_STATS_BUF (4096) -+static int hmbird_stats_proc_show(struct seq_file *m, void *v) -+{ -+ char *buf; -+ -+ buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ stats_print(buf, MAX_STATS_BUF); -+ -+ seq_printf(m, "%s\n", buf); -+ -+ kfree(buf); -+ return 0; -+} -+ -+static int hmbird_stats_proc_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_stats_proc_show, inode); -+} -+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL); -+/* hmbird_stats ops end */ -+ -+/* sched_ravg_window_frame_per_sec ops begin */ -+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file, -+ const char __user *buf, size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ sched_ravg_window_change(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open, -+ sched_ravg_window_frame_per_sec_proc_write); -+/* sched_ravg_window_frame_per_sec ops end */ -+ -+static ssize_t save_gov_str(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ for_each_possible_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy || (cpu != policy->cpu)) -+ continue; -+ WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0); -+ hmbird_info_systrace(":save origin gov : %s\n", saved_gov[cpu]); -+ } -+ return count; -+} -+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str); -+ -+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ u64 *pval = (u64 *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val_u64(file, buf, count, pval)) -+ return -EFAULT; -+ -+ if (scx_enable == 0) -+ set_cpu_cluster(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write); -+ -+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ int tmp_val; -+ -+ if (set_proc_buf_val(file, buf, count, &tmp_val)) -+ return -EFAULT; -+ -+ slim_walt_enable(tmp_val); -+ *pval = tmp_val; -+ -+ return count; -+} -+ -+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write); -+ -+/* yield_opt ops begin */ -+static int yield_opt_show(struct seq_file *m, void *v) -+{ -+ struct yield_opt_params *data = m->private; -+ -+ seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n", -+ data->enable, data->frame_per_sec, data->yield_headroom); -+ return 0; -+} -+ -+static int yield_opt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, yield_opt_show, pde_data(inode)); -+} -+ -+static ssize_t yield_opt_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu; -+ unsigned long flags; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if (copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp -+ != 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) || -+ (yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp; -+ yield_opt_params.frame_per_sec = frame_per_sec_tmp; -+ yield_opt_params.yield_headroom = yield_headroom_tmp; -+ yield_opt_params.enable = enable_tmp; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ ys->last_yield_time = 0; -+ ys->last_update_time = 0; -+ ys->sleep_end = 0; -+ ys->yield_cnt = 0; -+ ys->yield_cnt_after_sleep = 0; -+ ys->sleep = 0; -+ ys->sleep_times = 0; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write); -+ -+/* boost_policy ops begin */ -+static int boost_policy_show(struct seq_file *m, void *v) -+{ -+ struct boost_policy_params *data = m->private; -+ -+ seq_printf(m, "boost_policy:{\"enable\":%d; \"bottom_freq\":%u; \"boost_weight\":%d}\n", -+ data->enable, data->bottom_freq, data->boost_weight); -+ return 0; -+} -+ -+static int boost_policy_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, boost_policy_show, pde_data(inode)); -+} -+ -+static ssize_t boost_policy_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, bottom_freq_tmp, boost_weight_tmp; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if(copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &bottom_freq_tmp, &boost_weight_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || -+ (boost_weight_tmp < 50 || boost_weight_tmp > 300) || -+ (bottom_freq_tmp < 400000 || bottom_freq_tmp > 2200000)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ boost_policy_params.bottom_freq = bottom_freq_tmp; -+ boost_policy_params.boost_weight = boost_weight_tmp; -+ boost_policy_params.enable = enable_tmp; -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(boost_policy, boost_policy_open, boost_policy_write); -+ -+/* tick_hit ops begin */ -+static int tick_hit_show(struct seq_file *m, void *v) -+{ -+ struct tick_hit_params *data = m->private; -+ -+ seq_printf(m, "tick_hit:{\"enable\":%d; \"hit_count_thres\":%d; \"jiffies_num\":%lu}\n", -+ data->enable, data->hit_count_thres, data->jiffies_num); -+ return 0; -+} -+ -+static int tick_hit_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, tick_hit_show, pde_data(inode)); -+} -+ -+static ssize_t tick_hit_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, hit_count_thres_tmp, jiffies_num_tmp; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if(copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &hit_count_thres_tmp, &jiffies_num_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ tick_hit_params.hit_count_thres = hit_count_thres_tmp; -+ tick_hit_params.jiffies_num = jiffies_num_tmp; -+ tick_hit_params.enable = enable_tmp; -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(tick_hit, tick_hit_open, tick_hit_write); -+ -+static int __init hmbird_proc_init(void) -+{ -+ struct proc_dir_entry *hmbird_dir; -+ struct proc_dir_entry *load_track_dir; -+ struct proc_dir_entry *freq_gov_dir; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ -+ /* mkdir /proc/hmbird_sched */ -+ hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL); -+ if (!hmbird_dir) { -+ pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &scx_enable_proc_ops, -+ &scx_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &partial_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_stats); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbirdcore_debug); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_for_app); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &misfit_ds); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl_dbg); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpu7_tl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &cpu_cluster_masks_proc_ops, -+ &cpu_cluster_masks); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &save_gov_proc_ops, -+ &save_gov); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &watchdog_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isolate_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &iso_free_rescue); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_stats_proc_ops); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &yield_opt_proc_ops, -+ &yield_opt_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("boost_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &boost_policy_proc_ops, -+ &boost_policy_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("tick_hit", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &tick_hit_proc_ops, -+ &tick_hit_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbird_preempt_policy); -+ /* /proc/hmbird_sched--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_walt */ -+ load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir); -+ if (!load_track_dir) { -+ pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_walt--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &slim_walt_ctrl_proc_ops, -+ &slim_walt_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_dump); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_policy); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &sched_ravg_window_frame_per_sec_proc_ops, -+ &sched_ravg_window_frame_per_sec); -+ /* /proc/hmbird_sched/slim_walt--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_freq_gov */ -+ freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir); -+ if (!freq_gov_dir) { -+ pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_freq_gov--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &slim_gov_debug); -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &scx_gov_ctrl); -+ /* /proc/hmbird_sched/slim_freq_gov--end */ -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cluster_separate); -+ -+ return 0; -+} -+ -+device_initcall(hmbird_proc_init); -diff --git b/kernel/sched/hmbird/hmbird_sched_proc.h a/kernel/sched/hmbird/hmbird_sched_proc.h -new file mode 100755 -index 000000000000..e22bc75f1dd7 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched_proc.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SCHED_PROC_H__ -+#define __HMBIRD_SCHED_PROC_H__ -+ -+#include -+#include -+ -+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \ -+ do { \ -+ if (!proc_create(name, mode, parent, proc_ops)) { \ -+ pr_err("Error creating proc entry %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \ -+ do { \ -+ if (!proc_create_data(name, mode, parent, proc_ops, data)) { \ -+ pr_err("Error creating proc entry with data %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_PROC_OPS(name, open_func, write_func) \ -+ static const struct proc_ops name##_proc_ops = { \ -+ .proc_open = open_func, \ -+ .proc_write = write_func, \ -+ .proc_read = seq_read, \ -+ .proc_lseek = seq_lseek, \ -+ .proc_release = single_release, \ -+ } -+ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos); -+static int hmbird_common_show(struct seq_file *m, void *v); -+static int hmbird_common_open(struct inode *inode, struct file *file); -+ -+#endif -diff --git b/kernel/sched/hmbird/hmbird_shadow_tick.c a/kernel/sched/hmbird/hmbird_shadow_tick.c -new file mode 100755 -index 000000000000..2744cd42bbfd ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_shadow_tick.c -@@ -0,0 +1,198 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include -+#include <../../kernel/time/tick-sched.h> -+#include -+ -+#include "hmbird_shadow_tick.h" -+#include "slim.h" -+ -+#define HIGHRES_WATCH_CPU 0 -+ -+#include -+static bool shadow_tick_enable(void) {return highres_tick_ctrl; } -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; } -+#endif -+static bool shadow_tick_timer_init_flag; -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define shadow_tick_printk(fmt, args...) \ -+do { \ -+ int cpu = smp_processor_id(); \ -+ if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU) \ -+ trace_printk("hmbird shadow tick :"fmt, args); \ -+} while (0) -+#else -+#define shadow_tick_printk(fmt, args...) -+#endif -+ -+DEFINE_PER_CPU(struct hrtimer, stt); -+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu))) -+#define STOP_IDLE_TRIGGER (1) -+#define PERIODIC_TICK_TRIGGER (2) -+#define TICK_INTVAL (1000000ULL) -+#define HMBIRD_TICK_HIT_BOOST BIT(29) -+/* -+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms, -+ * so we can't restart hrtimer in scheduler tick. -+ */ -+static DEFINE_PER_CPU(u8, trigger_event); -+ -+/* -+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick. -+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler. -+ */ -+static DEFINE_PER_CPU(u8, tick_phase); -+ -+static inline void highres_timer_ctrl(bool enable, int cpu) -+{ -+ if (enable && hmbird_enabled()) { -+ if (!hrtimer_active(shadow_tick_timer(cpu))) -+ hrtimer_start(shadow_tick_timer(cpu), -+ ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED); -+ } else { -+ if (!enable) -+ hrtimer_cancel(shadow_tick_timer(cpu)); -+ } -+} -+ -+static inline void high_res_clear_phase(int cpu) -+{ -+ per_cpu(tick_phase, cpu) = 0; -+} -+ -+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer) -+{ -+ per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3; -+ if (per_cpu(tick_phase, cpu)) { -+ hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) { -+ per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+struct tick_hit_params tick_hit_params = { -+ .enable = 0, -+ .jiffies_num = 2, -+ .hit_count_thres = 6, -+}; -+ -+static void tick_hit_critical_task(struct task_struct *curr, struct rq *rq) -+{ -+ struct hmbird_entity *see = get_hmbird_ts(curr); -+ if (!tick_hit_params.enable || !see) -+ return; -+ -+ if ((!task_is_top_task(curr) || curr->pid != curr->tgid) && (curr->pid != scx_systemui_pid)) -+ return; -+ -+ if (see->tick_hit_count == 0) { -+ if (see->start_jiffies == 0) { -+ see->start_jiffies = jiffies; -+ see->tick_hit_count = 1; -+ } else { -+ see->start_jiffies = 0; /* status reset */ -+ see->tick_hit_count = 0; -+ } -+ } else { -+ if (time_before_eq(jiffies, see->start_jiffies + tick_hit_params.jiffies_num)) { -+ see->tick_hit_count++; -+ if (see->tick_hit_count >= tick_hit_params.hit_count_thres) { -+ cpufreq_update_util(rq, HMBIRD_TICK_HIT_BOOST); -+ } -+ } else { -+ see->tick_hit_count = 1; -+ see->start_jiffies = jiffies; -+ } -+ } -+} -+ -+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ struct rq_flags rf; -+ -+ rq_lock(rq, &rf); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ tick_hit_critical_task(curr, rq); -+ rq_unlock(rq, &rf); -+ -+ return highres_next_phase(cpu, timer); -+} -+ -+void shadow_tick_timer_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance; -+ } -+} -+ -+void start_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable()) { -+ if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER) -+ highres_timer_ctrl(false, cpu); -+ per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static void stop_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ per_cpu(trigger_event, cpu) = 0; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(false, cpu); -+} -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ stop_shadow_tick_timer(); -+} -+ -+void scheduler_tick_handler(void *unused, struct rq *rq) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ start_shadow_tick_timer(); -+} -+ -+static int __init hmbird_shadow_tick_init(void) -+{ -+ int ret = 0; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ shadow_tick_timer_init(); -+ shadow_tick_timer_init_flag = true; -+ return ret; -+} -+ -+device_initcall(hmbird_shadow_tick_init); -diff --git b/kernel/sched/hmbird/hmbird_shadow_tick.h a/kernel/sched/hmbird/hmbird_shadow_tick.h -new file mode 100755 -index 000000000000..0c26fd984f0a ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_shadow_tick.h -@@ -0,0 +1,11 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SHADOW_TICK_H__ -+#define __HMBIRD_SHADOW_TICK_H__ -+ -+#include -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+void scheduler_tick_handler(void *unused, struct rq *rq); -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state); -+#endif -diff --git b/kernel/sched/hmbird/hmbird_trace.h a/kernel/sched/hmbird/hmbird_trace.h -new file mode 100755 -index 000000000000..8468b406f347 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_trace.h -@@ -0,0 +1,92 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hmbird -+ -+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HMBIRD_H -+ -+#include -+#include -+#include -+ -+#define MAX_FATAL_INFO (64) -+ -+TRACE_EVENT(hmbird_fatal_info, -+ -+ TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info), -+ -+ TP_ARGS(type, pe, lnr, bnr, info), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, type) -+ __field(int, pe) -+ __field(int, lnr) -+ __field(int, bnr) -+ __array(char, info, MAX_FATAL_INFO)), -+ -+ TP_fast_assign( -+ __entry->type = type; -+ __entry->pe = pe; -+ __entry->lnr = lnr; -+ __entry->bnr = bnr; -+ memcpy(__entry->info, info, MAX_FATAL_INFO);), -+ -+ TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s", -+ __entry->type, __entry->pe, __entry->lnr, __entry->bnr, -+ __entry->info) -+); -+ -+TRACE_EVENT(hmbird_update_history, -+ -+ TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq, -+ struct task_struct *p, u32 runtime, int samples, int event), -+ -+ TP_ARGS(hmbird, rq, p, runtime, samples, event), -+ -+ TP_STRUCT__entry( -+ __array(char, comm, TASK_COMM_LEN) -+ __field(pid_t, pid) -+ __field(unsigned int, runtime) -+ __field(int, samples) -+ __field(int, event) -+ __field(unsigned int, demand) -+ __array(u32, hist, RAVG_HIST_SIZE) -+ __field(u16, task_util) -+ __field(int, cpu)), -+ -+ TP_fast_assign( -+ memcpy(__entry->comm, p->comm, TASK_COMM_LEN); -+ __entry->pid = p->pid; -+ __entry->runtime = runtime; -+ __entry->samples = samples; -+ __entry->event = event; -+ __entry->demand = hmbird->sts.demand; -+ memcpy(__entry->hist, hmbird->sts.sum_history, -+ RAVG_HIST_SIZE * sizeof(u32)); -+ __entry->task_util = hmbird->sts.demand_scaled; -+ __entry->cpu = rq->cpu;), -+ -+ TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d", -+ __entry->comm, __entry->pid, -+ __entry->runtime, __entry->samples, -+ __entry->event, -+ __entry->demand, -+ __entry->hist[0], __entry->hist[1], -+ __entry->hist[2], __entry->hist[3], -+ __entry->hist[4], -+ __entry->task_util, -+ __entry->cpu) -+); -+ -+#endif /*_TRACE_HMBIRD_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird -+ -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE hmbird_trace -+/* This part must be outside protection */ -+#include -diff --git b/kernel/sched/hmbird/hmbird_util_track.c a/kernel/sched/hmbird/hmbird_util_track.c -new file mode 100755 -index 000000000000..d520a8403401 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_util_track.c -@@ -0,0 +1,626 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+ -+#define CREATE_TRACE_POINTS -+#include "hmbird_trace.h" -+#undef CREATE_TRACE_POINTS -+ -+#define HMBIRD_DEBUG_PANIC (1 << 3) -+static bool init_irq_work_inited; -+ -+extern noinline int tracing_mark_write(const char *buf); -+ -+int hmbird_sched_ravg_window = 8000000; -+int new_hmbird_sched_ravg_window = 8000000; -+DEFINE_SPINLOCK(new_sched_ravg_window_lock); -+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+ -+static struct irq_work hmbird_slim_walt_irq_work; -+ -+/*Sysctl related interface*/ -+#define WINDOW_STATS_RECENT 0 -+#define WINDOW_STATS_MAX 1 -+#define WINDOW_STATS_MAX_RECENT_AVG 2 -+#define WINDOW_STATS_AVG 3 -+#define WINDOW_STATS_INVALID_POLICY 4 -+ -+ -+#define SCX_SCHED_CAPACITY_SHIFT 10 -+#define SCHED_ACCOUNT_WAIT_TIME 0 -+ -+atomic64_t hmbird_irq_work_lastq_ws; -+static u64 tick_sched_clock; -+ -+static inline u64 scale_exec_time(u64 delta, struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT; -+} -+ -+static inline u64 scale_time_to_util(u64 d) -+{ -+ do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT); -+ return d; -+} -+ -+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ delta = scale_exec_time(delta, rq); -+ sts->sum += delta; -+ if (unlikely(sts->sum > hmbird_sched_ravg_window)) -+ sts->sum = hmbird_sched_ravg_window; -+ -+ return delta; -+} -+ -+ -+static int -+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) -+{ -+ /* -+ * No need to bother updating task demand for the idle task. -+ */ -+ if (is_idle_task(p)) -+ return 0; -+ -+ /* -+ * When a task is waking up it is completing a segment of non-busy -+ * time. Likewise, if wait time is not treated as busy time, then -+ * when a task begins to run or is migrated, it is not running and -+ * is completing a segment of non-busy time. -+ */ -+ if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && -+ (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) -+ return 0; -+ -+ /* -+ * The idle exit time is not accounted for the first task _picked_ up to -+ * run on the idle CPU. -+ */ -+ if (event == PICK_NEXT_TASK && rq->curr == rq->idle) -+ return 0; -+ -+ /* -+ * TASK_UPDATE can be called on sleeping task, when its moved between -+ * related groups -+ */ -+ if (event == TASK_UPDATE) { -+ if (rq->curr == p) -+ return 1; -+ -+ return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; -+ } -+ -+ return 1; -+} -+ -+static void rollover_cpu_window(struct rq *rq, bool full_window) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 curr_sum = srq->curr_runnable_sum; -+ -+ if (unlikely(full_window)) -+ curr_sum = 0; -+ -+ srq->prev_runnable_sum = curr_sum; -+ srq->curr_runnable_sum = 0; -+} -+ -+static u64 -+update_window_start(struct rq *rq, u64 wallclock, int event) -+{ -+ s64 delta; -+ int nr_windows; -+ bool full_window; -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start = srq->window_start; -+ -+ if (wallclock < srq->latest_clock) -+ wallclock = srq->latest_clock; -+ delta = wallclock - srq->window_start; -+ if (delta < 0) { -+ delta = 0; -+ wallclock = srq->window_start; -+ } -+ srq->latest_clock = wallclock; -+ if (delta < hmbird_sched_ravg_window) -+ return old_window_start; -+ -+ nr_windows = div64_u64(delta, hmbird_sched_ravg_window); -+ srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window; -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ full_window = nr_windows > 1; -+ rollover_cpu_window(rq, full_window); -+ -+ return old_window_start; -+} -+ -+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -+ -+static inline unsigned int get_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static inline unsigned int cpu_cur_freq(int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cur; -+} -+ -+static void -+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, -+ u64 wallclock) -+{ -+ int cpu = cpu_of(rq); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * -+ arch_scale_cpu_capacity(cpu), get_max_freq(cpu)); -+} -+ -+/* -+ * Called when new window is starting for a task, to record cpu usage over -+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 -+ * when, say, a real-time task runs without preemption for several windows at a -+ * stretch. -+ */ -+static void update_history(struct rq *rq, struct task_struct *p, -+ u32 runtime, int samples, int event) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u32 *hist = &sts->sum_history[0]; -+ int i; -+ u32 max = 0, avg, demand; -+ u64 sum = 0; -+ u16 demand_scaled; -+ -+ /* Ignore windows where task had no activity */ -+ if (!runtime || is_idle_task(p) || !samples) -+ goto done; -+ -+ /* Push new 'runtime' value onto stack */ -+ for (; samples > 0; samples--) { -+ hist[sts->cidx] = runtime; -+ sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE; -+ } -+ -+ for (i = 0; i < RAVG_HIST_SIZE; i++) { -+ sum += hist[i]; -+ if (hist[i] > max) -+ max = hist[i]; -+ } -+ -+ sts->sum = 0; -+ avg = div64_u64(sum, RAVG_HIST_SIZE); -+ -+ switch (slim_walt_policy) { -+ case WINDOW_STATS_RECENT: -+ demand = runtime; -+ break; -+ case WINDOW_STATS_MAX: -+ demand = max; -+ break; -+ case WINDOW_STATS_AVG: -+ demand = avg; -+ break; -+ default: -+ demand = max(avg, runtime); -+ } -+ -+ demand_scaled = scale_time_to_util(demand); -+ -+ sts->demand = demand; -+ sts->demand_scaled = demand_scaled; -+ -+done: -+ return; -+} -+ -+ -+static u64 update_task_demand(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ -+ u64 delta, window_start = srq->window_start; -+ int new_window, nr_full_windows; -+ u32 window_size = hmbird_sched_ravg_window; -+ u64 runtime; -+ -+ new_window = mark_start < window_start; -+ if (!account_busy_for_task_demand(rq, p, event)) { -+ if (new_window) -+ /* -+ * If the time accounted isn't being accounted as -+ * busy time, and a new window started, only the -+ * previous window need be closed out with the -+ * pre-existing demand. Multiple windows may have -+ * elapsed, but since empty windows are dropped, -+ * it is not necessary to account those. -+ */ -+ update_history(rq, p, sts->sum, 1, event); -+ return 0; -+ } -+ -+ if (!new_window) { -+ /* -+ * The simple case - busy time contained within the existing -+ * window. -+ */ -+ return add_to_task_demand(rq, p, wallclock - mark_start); -+ } -+ -+ /* -+ * Busy time spans at least two windows. Temporarily rewind -+ * window_start to first window boundary after mark_start. -+ */ -+ delta = window_start - mark_start; -+ nr_full_windows = div64_u64(delta, window_size); -+ window_start -= (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (window_start - mark_start) first */ -+ runtime = add_to_task_demand(rq, p, window_start - mark_start); -+ -+ /* Push new sample(s) into task's demand history */ -+ update_history(rq, p, sts->sum, 1, event); -+ if (nr_full_windows) { -+ u64 scaled_window = scale_exec_time(window_size, rq); -+ -+ update_history(rq, p, scaled_window, nr_full_windows, event); -+ runtime += nr_full_windows * scaled_window; -+ } -+ -+ /* -+ * Roll window_start back to current to process any remainder -+ * in current window. -+ */ -+ window_start += (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (wallclock - window_start) next */ -+ mark_start = window_start; -+ runtime += add_to_task_demand(rq, p, wallclock - mark_start); -+ -+ return runtime; -+} -+ -+u16 slim_walt_cpu_util(int cpu) -+{ -+ u64 prev_runnable_sum; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ prev_runnable_sum = srq->prev_runnable_sum; -+ do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT); -+ -+ return (u16)prev_runnable_sum; -+} -+ -+static DEFINE_PER_CPU(u16, prev_cpu_util); -+static inline void cpu_util_update_systrace_c(int cpu) -+{ -+ char buf[256]; -+ u16 cpu_util = slim_walt_cpu_util(cpu); -+ -+ if (cpu_util != per_cpu(prev_cpu_util, cpu)) { -+ snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n", -+ cpu, cpu_util); -+ tracing_mark_write(buf); -+ per_cpu(prev_cpu_util, cpu) = cpu_util; -+ } -+} -+ -+ -+static inline int account_busy_for_cpu_time(struct rq *rq, -+ struct task_struct *p, -+ int event) -+{ -+ return !is_idle_task(p) && (event == PUT_PREV_TASK -+ || event == TASK_UPDATE); -+} -+ -+ -+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ int new_window, full_window = 0; -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 window_start = srq->window_start; -+ u32 window_size = srq->prev_window_size; -+ u64 delta; -+ u64 *curr_runnable_sum = &srq->curr_runnable_sum; -+ u64 *prev_runnable_sum = &srq->prev_runnable_sum; -+ -+ new_window = mark_start < window_start; -+ if (new_window) -+ full_window = (window_start - mark_start) >= window_size; -+ -+ -+ if (!account_busy_for_cpu_time(rq, p, event)) -+ goto done; -+ -+ -+ if (!new_window) { -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. No rollover -+ * since we didn't start a new window. An example of this is -+ * when a task starts execution and then sleeps within the -+ * same window. -+ */ -+ delta = wallclock - mark_start; -+ -+ delta = scale_exec_time(delta, rq); -+ *curr_runnable_sum += delta; -+ -+ goto done; -+ } -+ -+ /* -+ * situations below this need window rollover, -+ * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done -+ * in update_window_start() -+ * -+ * For task counters curr/prev_window[_cpu] are rolled over in the early part of -+ * this function. If full_window(s) have expired and time since last update needs -+ * to be accounted as busy time, set the prev to a complete window size time, else -+ * add the prev window portion. -+ * -+ * For task curr counters a new window has begun, always assign -+ */ -+ -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. A new window -+ * must have been started in udpate_window_start() -+ * If any of these three above conditions are true -+ * then this busy time can't be accounted as irqtime. -+ * -+ * Busy time for the idle task need not be accounted. -+ * -+ * An example of this would be a task that starts execution -+ * and then sleeps once a new window has begun. -+ */ -+ -+ /* -+ * A full window hasn't elapsed, account partial -+ * contribution to previous completed window. -+ */ -+ -+ -+ delta = full_window ? scale_exec_time(window_size, rq) : -+ scale_exec_time(window_start - mark_start, rq); -+ -+ *prev_runnable_sum += delta; -+ -+ /* Account piece of busy time in the current window. */ -+ delta = scale_exec_time(wallclock - window_start, rq); -+ *curr_runnable_sum += delta; -+ -+done: -+ if (slim_walt_dump && new_window) -+ cpu_util_update_systrace_c(rq->cpu); -+} -+ -+ -+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq) -+{ -+ u64 result; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 new_window_start = srq->window_start; -+ -+ if (old_window_start == new_window_start) -+ return; -+ -+ result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws, -+ old_window_start, new_window_start); -+ if (result != old_window_start) -+ return; -+ -+ if (likely(cpu_online(raw_smp_processor_id()))) -+ irq_work_queue(&hmbird_slim_walt_irq_work); -+ else -+ irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask)); -+} -+ -+/* -+ * In the core scheduler, most of the load update points update the rq_clock after -+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of -+ * obtaining the time, but to prevent the subsequent migration of the load update -+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock -+ * update here. -+ */ -+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p, -+ struct rq *rq, int event) -+{ -+ if (!(rq->clock_update_flags & RQCF_UPDATED)) -+ update_rq_clock(rq); -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock)); -+} -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start; -+ -+ if (!slim_walt_ctrl) -+ return; -+ -+ if (!srq->window_start || sts->mark_start == wallclock) -+ return; -+ -+ old_window_start = update_window_start(rq, wallclock, event); -+ -+ if (!sts->window_start) -+ sts->window_start = srq->window_start; -+ -+ if (!sts->mark_start) -+ goto done; -+ -+ update_task_rq_cpu_cycles(p, rq, event, wallclock); -+ update_task_demand(p, rq, event, wallclock); -+ update_cpu_busy_time(p, rq, event, wallclock); -+ -+ sts->window_start = srq->window_start; -+ -+done: -+ sts->mark_start = wallclock; -+ slim_walt_window_rollover_run_once(old_window_start, rq); -+} -+ -+static void slim_walt_irq_work(struct irq_work *irq_work) -+{ -+ cpumask_t lock_cpus; -+ struct hmbird_sched_rq_stats *srq; -+ struct rq *rq; -+ int cpu; -+ int level = 0; -+ u64 wc; -+ unsigned long flags; -+ -+ cpumask_copy(&lock_cpus, cpu_possible_mask); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ if (level == 0) -+ raw_spin_lock(&cpu_rq(cpu)->__lock); -+ else -+ raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); -+ level++; -+ } -+ -+ wc = sched_clock(); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ rq = cpu_rq(cpu); -+ hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc); -+ } -+ -+ cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER); -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) { -+ srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id()); -+ if (wc < srq->window_start + new_hmbird_sched_ravg_window) -+ hmbird_sched_ravg_window = new_hmbird_sched_ravg_window; -+ } -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ raw_spin_unlock(&cpu_rq(cpu)->__lock); -+ } -+} -+static void hmbird_sched_init_rq(struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ srq->task_exec_scale = 1024; -+ srq->window_start = 0; -+} -+ -+void hmbird_sched_init_task(struct task_struct *p) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ memset(sts, 0, sizeof(struct hmbird_sched_task_stats)); -+} -+ -+static void hmbird_sched_stats_init(void) -+{ -+ unsigned long flags; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ raw_spin_lock_irqsave(&rq->__lock, flags); -+ hmbird_sched_init_rq(rq); -+ raw_spin_unlock_irqrestore(&rq->__lock, flags); -+ } -+ slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG; -+ -+ if (false == init_irq_work_inited) { -+ init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work); -+ init_irq_work_inited = true; -+ } -+} -+ -+void hmbird_scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (unlikely(!tick_sched_clock)) { -+ /* -+ * Let the window begin 20us prior to the tick, -+ * that way we are guaranteed a rollover when the tick occurs. -+ * Use rq->clock directly instead of rq_clock() since -+ * we do not have the rq lock and -+ * rq->clock was updated in the tick callpath. -+ */ -+ if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000)) -+ return; -+ for_each_possible_cpu(cpu) { -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ srq->window_start = tick_sched_clock; -+ } -+ atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock); -+ } -+} -+ -+void slim_walt_enable(int enable) -+{ -+ if (1 == !!enable) { -+ hmbird_sched_stats_init(); -+ WRITE_ONCE(tick_sched_clock, 0); -+ } else -+ slim_walt_ctrl = 0; -+} -+ -+void slim_get_cpu_util(int cpu, u64 *util) -+{ -+ if (cpu < 0) -+ return; -+ -+ *util = slim_walt_cpu_util(cpu); -+} -+ -+void slim_get_task_util(struct task_struct *p, u64 *util) -+{ -+ if (p == NULL) -+ return; -+ -+ *util = get_hmbird_ts(p)->sts.demand_scaled; -+} -+ -+void sched_ravg_window_change(int frame_per_sec) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec; -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+} -diff --git b/kernel/sched/hmbird/hmbird_util_track.h a/kernel/sched/hmbird/hmbird_util_track.h -new file mode 100644 -index 000000000000..17b85df7f83b ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_util_track.h -@@ -0,0 +1,33 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef __HMBIRD_UTIL_TRACK_H__ -+#define __HMBIRD_UTIL_TRACK_H__ -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock); -+void hmbird_sched_init_task(struct task_struct *p); -+void slim_walt_enable(int enable); -+void slim_get_cpu_util(int cpu, u64 *util); -+void slim_get_task_util(struct task_struct *p, u64 *util); -+ -+extern atomic64_t hmbird_irq_work_lastq_ws; -+ -+enum task_event { -+ PUT_PREV_TASK = 0, -+ PICK_NEXT_TASK = 1, -+ TASK_WAKE = 2, -+ TASK_MIGRATE = 3, -+ TASK_UPDATE = 4, -+ IRQ_UPDATE = 5, -+}; -+ -+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+#endif /* __HMBIRD_UTIL_TRACK_H__ */ -diff --git b/kernel/sched/hmbird/slim.h a/kernel/sched/hmbird/slim.h -new file mode 100755 -index 000000000000..dffe6506658e ---- /dev/null -+++ a/kernel/sched/hmbird/slim.h -@@ -0,0 +1,56 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+ -+#ifndef __SLIM_H -+#define __SLIM_H -+ -+extern atomic_t __hmbird_ops_enabled; -+extern atomic_t non_hmbird_task; -+extern int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+extern int heartbeat; -+extern int heartbeat_enable; -+extern int watchdog_enable; -+extern int isolate_ctrl; -+extern int parctrl_high_ratio; -+extern int parctrl_low_ratio; -+extern int isoctrl_high_ratio; -+extern int isoctrl_low_ratio; -+extern int iso_free_rescue; -+extern int yield_opt; -+ -+extern enum hmbird_switch_type sw_type; -+extern noinline int tracing_mark_write(const char *buf); -+int task_top_id(struct task_struct *p); -+void stats_print(char *buf, int len); -+void hmbird_skip_yield(long *skip); -+extern spinlock_t hmbird_tasks_lock; -+extern int scx_systemui_pid; -+ -+struct yield_opt_params { -+ int enable; -+ int frame_per_sec; -+ u64 frame_time_ns; -+ int yield_headroom; -+}; -+ -+extern struct yield_opt_params yield_opt_params; -+ -+struct tick_hit_params { -+ int enable; -+ unsigned long jiffies_num; -+ int hit_count_thres; -+}; -+ -+extern struct tick_hit_params tick_hit_params; -+ -+struct boost_policy_params { -+ int enable; -+ unsigned int bottom_freq; -+ int boost_weight; -+}; -+ -+extern struct boost_policy_params boost_policy_params; -+ -+#define MAX_GOV_LEN (16) -+extern char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+#endif -diff --git b/kernel/sched/idle.c a/kernel/sched/idle.c -index b33cefeb4188..487255ac6832 100755 ---- b/kernel/sched/idle.c -+++ a/kernel/sched/idle.c -@@ -408,13 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl - - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) - { -- scx_update_idle(rq, false); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, false); -+#endif - } - - static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) - { - update_idle_core(rq); -- scx_update_idle(rq, true); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, true); -+#endif - schedstat_inc(rq->sched_goidle); - } - -diff --git b/kernel/sched/sched.h a/kernel/sched/sched.h -index 509e8dc616ab..a7e9caea1efe 100755 ---- b/kernel/sched/sched.h -+++ a/kernel/sched/sched.h -@@ -188,18 +188,13 @@ static inline int idle_policy(int policy) - return policy == SCHED_IDLE; - } - --static inline int normal_policy(int policy) --{ --#ifdef CONFIG_SCHED_CLASS_EXT -- if (policy == SCHED_EXT) -- return true; --#endif -- return policy == SCHED_NORMAL; --} -- - static inline int fair_policy(int policy) - { -- return normal_policy(policy) || policy == SCHED_BATCH; -+#ifdef CONFIG_HMBIRD_SCHED -+ return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#else -+ return policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#endif - } - - static inline int rt_policy(int policy) -@@ -247,25 +242,7 @@ static inline void update_avg(u64 *avg, u64 sample) - #define shr_bound(val, shift) \ - (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) - --/* -- * cgroup weight knobs should use the common MIN, DFL and MAX values which are -- * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it -- * maps pretty well onto the shares value used by scheduler and the round-trip -- * conversions preserve the original value over the entire range. -- */ --static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) --{ -- return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); --} -- --static inline unsigned long sched_weight_to_cgroup(unsigned long weight) --{ -- return clamp_t(unsigned long, -- DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -- CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); --} -- --/* -+/* - * !! For sched_setattr_nocheck() (kernel) only !! - * - * This is actually gross. :( -@@ -532,10 +509,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) { } - #endif /* CONFIG_SMP */ - #else /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_HMBIRD_SCHED - static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) - { - return 0; - } -+#endif - #endif /* CONFIG_FAIR_GROUP_SCHED */ - - #else /* CONFIG_CGROUP_SCHED */ -@@ -700,29 +679,6 @@ struct cfs_rq { - #endif /* CONFIG_FAIR_GROUP_SCHED */ - }; - --#ifdef CONFIG_SCHED_CLASS_EXT --/* scx_rq->flags, protected by the rq lock */ --enum scx_rq_flags { -- SCX_RQ_CAN_STOP_TICK = 1 << 0, --}; -- --struct scx_rq { -- struct scx_dispatch_q local_dsq; -- struct list_head watchdog_list; -- u64 ops_qseq; -- u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -- u32 nr_running; -- u32 flags; -- bool cpu_released; -- cpumask_var_t cpus_to_kick; -- cpumask_var_t cpus_to_preempt; -- cpumask_var_t cpus_to_wait; -- u64 pnt_seq; -- struct irq_work kick_cpus_irq_work; -- struct rq *rq; --}; --#endif /* CONFIG_SCHED_CLASS_EXT */ -- - static inline int rt_bandwidth_enabled(void) - { - return sysctl_sched_rt_runtime >= 0; -@@ -1258,11 +1214,7 @@ struct rq { - - ANDROID_OEM_DATA_ARRAY(1, 16); - --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, struct scx_rq *scx); --#else - ANDROID_KABI_RESERVE(1); --#endif - ANDROID_KABI_RESERVE(2); - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); -@@ -2340,11 +2292,6 @@ struct affinity_context { - unsigned int flags; - }; - --enum rq_onoff_reason { -- RQ_ONOFF_HOTPLUG, /* CPU is going on/offline */ -- RQ_ONOFF_TOPOLOGY, /* sched domain topology update */ --}; -- - struct sched_class { - - #ifdef CONFIG_UCLAMP_TASK -@@ -2551,7 +2498,6 @@ extern void init_sched_rt_class(void); - extern void init_sched_fair_class(void); - - extern void reweight_task(struct task_struct *p, int prio); --extern void __setscheduler_prio(struct task_struct *p, int prio); - - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); -@@ -2632,53 +2578,6 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) - extern void activate_task(struct rq *rq, struct task_struct *p, int flags); - extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); - --struct sched_change_guard { -- struct task_struct *p; -- struct rq *rq; -- bool queued; -- bool running; -- bool done; --}; -- --extern struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -- --extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags); -- --/** -- * SCHED_CHANGE_BLOCK - Nested block for task attribute updates -- * @__rq: Runqueue the target task belongs to -- * @__p: Target task -- * @__flags: DEQUEUE/ENQUEUE_* flags -- * -- * A task may need to be dequeued and put_prev_task'd for attribute updates and -- * set_next_task'd and re-enqueued afterwards. This helper defines a nested -- * block which automatically handles these preparation and cleanup operations. -- * -- * SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- * update_attribute(p); -- * ... -- * } -- * -- * If @__flags is a variable, the variable may be updated in the block body and -- * the updated value will be used when re-enqueueing @p. -- * -- * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling -- * update_rq_clock() beforehand. Otherwise, the rq clock is automatically -- * updated iff the task needs to be dequeued and re-enqueued. Only the former -- * case guarantees that the rq clock is up-to-date inside and after the block. -- */ --#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -- for (struct sched_change_guard __cg = \ -- sched_change_guard_init(__rq, __p, __flags); \ -- !__cg.done; sched_change_guard_fini(&__cg, __flags)) -- --extern void check_class_changing(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class); --extern void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio); -- - extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); - - #ifdef CONFIG_PREEMPT_RT -@@ -3710,6 +3609,8 @@ static inline bool cpu_busy_with_softirqs(int cpu) - } - #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ - --#include "ext.h" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird.h" -+#endif - - #endif /* _KERNEL_SCHED_SCHED_H */ -diff --git b/kernel/time/tick-sched.c a/kernel/time/tick-sched.c -index 998c2eefe173..c13772ee823c 100755 ---- b/kernel/time/tick-sched.c -+++ a/kernel/time/tick-sched.c -@@ -34,6 +34,10 @@ - - #include - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "../sched/hmbird/hmbird_shadow_tick.h" -+#endif -+ - /* - * Per-CPU nohz control structure - */ -@@ -1124,6 +1128,9 @@ void tick_nohz_idle_stop_tick(void) - ktime_t expires; - - trace_android_vh_tick_nohz_idle_stop_tick(NULL); -+#ifdef CONFIG_HMBIRD_SCHED -+ android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL); -+#endif - /* - * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the - * tick timer expiration time is known already. -diff --git b/tools/sched_ext/.gitignore a/tools/sched_ext/.gitignore -deleted file mode 100644 -index a3240f9f7eba..000000000000 ---- b/tools/sched_ext/.gitignore -+++ /dev/null -@@ -1,9 +0,0 @@ --scx_example_simple --scx_example_qmap --scx_example_central --scx_example_pair --scx_example_flatcg --scx_example_userland --*.skel.h --*.subskel.h --/tools/ -diff --git b/tools/sched_ext/Makefile a/tools/sched_ext/Makefile -deleted file mode 100644 -index 73c43782837d..000000000000 ---- b/tools/sched_ext/Makefile -+++ /dev/null -@@ -1,216 +0,0 @@ --# SPDX-License-Identifier: GPL-2.0 --# Copyright (c) 2022 Meta Platforms, Inc. and affiliates. --include ../build/Build.include --include ../scripts/Makefile.arch --include ../scripts/Makefile.include -- --ifneq ($(LLVM),) --ifneq ($(filter %/,$(LLVM)),) --LLVM_PREFIX := $(LLVM) --else ifneq ($(filter -%,$(LLVM)),) --LLVM_SUFFIX := $(LLVM) --endif -- --CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi --CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu --CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl --CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu --CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu --CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu --CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu --CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu --CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu --CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) -- --ifeq ($(CROSS_COMPILE),) --ifeq ($(CLANG_TARGET_FLAGS),) --$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk --else --CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) --endif # CLANG_TARGET_FLAGS --else --CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) --endif # CROSS_COMPILE -- --CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as --else --CC := $(CROSS_COMPILE)gcc --endif # LLVM -- --CURDIR := $(abspath .) --TOOLSDIR := $(abspath ..) --LIBDIR := $(TOOLSDIR)/lib --BPFDIR := $(LIBDIR)/bpf --TOOLSINCDIR := $(TOOLSDIR)/include --BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool --APIDIR := $(TOOLSINCDIR)/uapi --GENDIR := $(abspath ../../include/generated) --GENHDR := $(GENDIR)/autoconf.h -- --SCRATCH_DIR := $(CURDIR)/tools --BUILD_DIR := $(SCRATCH_DIR)/build --INCLUDE_DIR := $(SCRATCH_DIR)/include --BPFOBJ_DIR := $(BUILD_DIR)/libbpf --BPFOBJ := $(BPFOBJ_DIR)/libbpf.a --ifneq ($(CROSS_COMPILE),) --HOST_BUILD_DIR := $(BUILD_DIR)/host --HOST_SCRATCH_DIR := host-tools --HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include --else --HOST_BUILD_DIR := $(BUILD_DIR) --HOST_SCRATCH_DIR := $(SCRATCH_DIR) --HOST_INCLUDE_DIR := $(INCLUDE_DIR) --endif --HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a --RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids --DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -- --VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ -- $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ -- ../../vmlinux \ -- /sys/kernel/btf/vmlinux \ -- /boot/vmlinux-$(shell uname -r) --VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) --ifeq ($(VMLINUX_BTF),) --$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") --endif -- --BPFTOOL ?= $(DEFAULT_BPFTOOL) -- --ifneq ($(wildcard $(GENHDR)),) -- GENFLAGS := -DHAVE_GENHDR --endif -- --CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ -- -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -- -I$(TOOLSINCDIR) -I$(APIDIR) -- --CARGOFLAGS := --release -- --# Silence some warnings when compiled with clang --ifneq ($(LLVM),) --CFLAGS += -Wno-unused-command-line-argument --endif -- --LDFLAGS = -lelf -lz -lpthread -- --IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - &1 \ -- | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ --$(shell $(1) -dM -E - $@ --else -- $(call msg,CP,,$@) -- $(Q)cp "$(VMLINUX_H)" $@ --endif -- --%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \ -- | $(BPFOBJ) -- $(call msg,CLNG-BPF,,$@) -- $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ -- --%.skel.h: %.bpf.o $(BPFTOOL) -- $(call msg,GEN-SKEL,,$@) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) -- $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) -- $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@ -- $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h) -- --scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_userland: scx_example_userland.c scx_example_userland.skel.h \ -- scx_example_userland_common.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR) --atropos: export ATROPOS_CLANG = $(CLANG) --atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS) --atropos: $(INCLUDE_DIR)/vmlinux.h -- cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS) -- --clean: -- cargo clean --manifest-path=atropos/Cargo.toml -- rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) -- rm -f *.o *.bpf.o *.skel.h *.subskel.h -- rm -f scx_example_simple scx_example_qmap scx_example_central \ -- scx_example_pair scx_example_flatcg scx_example_userland -- --.PHONY: all atropos clean -- --# delete failed targets --.DELETE_ON_ERROR: -- --# keep intermediate (.skel.h, .bpf.o, etc) targets --.SECONDARY: -diff --git b/tools/sched_ext/atropos/.gitignore a/tools/sched_ext/atropos/.gitignore -deleted file mode 100644 -index 186dba259ec2..000000000000 ---- b/tools/sched_ext/atropos/.gitignore -+++ /dev/null -@@ -1,3 +0,0 @@ --src/bpf/.output --Cargo.lock --target -diff --git b/tools/sched_ext/atropos/Cargo.toml a/tools/sched_ext/atropos/Cargo.toml -deleted file mode 100644 -index 7462a836d53d..000000000000 ---- b/tools/sched_ext/atropos/Cargo.toml -+++ /dev/null -@@ -1,28 +0,0 @@ --[package] --name = "scx_atropos" --version = "0.5.0" --authors = ["Dan Schatzberg ", "Meta"] --edition = "2021" --description = "Userspace scheduling with BPF" --license = "GPL-2.0-only" -- --[dependencies] --anyhow = "1.0.65" --bitvec = { version = "1.0", features = ["serde"] } --clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } --ctrlc = { version = "3.1", features = ["termination"] } --fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"} --hex = "0.4.3" --libbpf-rs = "0.19.1" --libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] } --libc = "0.2.137" --log = "0.4.17" --ordered-float = "3.4.0" --simplelog = "0.12.0" -- --[build-dependencies] --bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false } --libbpf-cargo = "0.13.0" -- --[features] --enable_backtrace = [] -diff --git b/tools/sched_ext/atropos/build.rs a/tools/sched_ext/atropos/build.rs -deleted file mode 100644 -index 26e792c5e17e..000000000000 ---- b/tools/sched_ext/atropos/build.rs -+++ /dev/null -@@ -1,70 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --extern crate bindgen; -- --use std::env; --use std::fs::create_dir_all; --use std::path::Path; --use std::path::PathBuf; -- --use libbpf_cargo::SkeletonBuilder; -- --const HEADER_PATH: &str = "src/bpf/atropos.h"; -- --fn bindgen_atropos() { -- // Tell cargo to invalidate the built crate whenever the wrapper changes -- println!("cargo:rerun-if-changed={}", HEADER_PATH); -- -- // The bindgen::Builder is the main entry point -- // to bindgen, and lets you build up options for -- // the resulting bindings. -- let bindings = bindgen::Builder::default() -- // The input header we would like to generate -- // bindings for. -- .header(HEADER_PATH) -- // Tell cargo to invalidate the built crate whenever any of the -- // included header files changed. -- .parse_callbacks(Box::new(bindgen::CargoCallbacks)) -- // Finish the builder and generate the bindings. -- .generate() -- // Unwrap the Result and panic on failure. -- .expect("Unable to generate bindings"); -- -- // Write the bindings to the $OUT_DIR/bindings.rs file. -- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); -- bindings -- .write_to_file(out_path.join("atropos-sys.rs")) -- .expect("Couldn't write bindings!"); --} -- --fn gen_bpf_sched(name: &str) { -- let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap(); -- let clang = env::var("ATROPOS_CLANG").unwrap(); -- eprintln!("{}", clang); -- let outpath = format!("./src/bpf/.output/{}.skel.rs", name); -- let skel = Path::new(&outpath); -- let src = format!("./src/bpf/{}.bpf.c", name); -- SkeletonBuilder::new() -- .source(src.clone()) -- .clang(clang) -- .clang_args(bpf_cflags) -- .build_and_generate(&skel) -- .unwrap(); -- println!("cargo:rerun-if-changed={}", src); --} -- --fn main() { -- bindgen_atropos(); -- // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton. -- // Reasons are because the generated skeleton contains compiler attributes -- // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]` -- // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside -- // the path attribute either (see https://github.com/rust-lang/rust/pull/83366). -- // -- // However, there is hope! When the above feature stabilizes we can clean this -- // all up. -- create_dir_all("./src/bpf/.output").unwrap(); -- gen_bpf_sched("atropos"); --} -diff --git b/tools/sched_ext/atropos/rustfmt.toml a/tools/sched_ext/atropos/rustfmt.toml -deleted file mode 100644 -index b7258ed0a8d8..000000000000 ---- b/tools/sched_ext/atropos/rustfmt.toml -+++ /dev/null -@@ -1,8 +0,0 @@ --# Get help on options with `rustfmt --help=config` --# Please keep these in alphabetical order. --edition = "2021" --group_imports = "StdExternalCrate" --imports_granularity = "Item" --merge_derives = false --use_field_init_shorthand = true --version = "Two" -diff --git b/tools/sched_ext/atropos/src/atropos_sys.rs a/tools/sched_ext/atropos/src/atropos_sys.rs -deleted file mode 100644 -index bbeaf856d40e..000000000000 ---- b/tools/sched_ext/atropos/src/atropos_sys.rs -+++ /dev/null -@@ -1,10 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#![allow(non_upper_case_globals)] --#![allow(non_camel_case_types)] --#![allow(non_snake_case)] --#![allow(dead_code)] -- --include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs")); -diff --git b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -deleted file mode 100644 -index 3905a403e940..000000000000 ---- b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -+++ /dev/null -@@ -1,743 +0,0 @@ --/* Copyright (c) Meta Platforms, Inc. and affiliates. */ --/* -- * This software may be used and distributed according to the terms of the -- * GNU General Public License version 2. -- * -- * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF -- * part does simple round robin in each domain and the userspace part -- * calculates the load factor of each domain and tells the BPF part how to load -- * balance the domains. -- * -- * Every task has an entry in the task_data map which lists which domain the -- * task belongs to. When a task first enters the system (atropos_prep_enable), -- * they are round-robined to a domain. -- * -- * atropos_select_cpu is the primary scheduling logic, invoked when a task -- * becomes runnable. The lb_data map is populated by userspace to inform the BPF -- * scheduler that a task should be migrated to a new domain. Otherwise, the task -- * is scheduled in priority order as follows: -- * * The current core if the task was woken up synchronously and there are idle -- * cpus in the system -- * * The previous core, if idle -- * * The pinned-to core if the task is pinned to a specific core -- * * Any idle cpu in the domain -- * -- * If none of the above conditions are met, then the task is enqueued to a -- * dispatch queue corresponding to the domain (atropos_enqueue). -- * -- * atropos_dispatch will attempt to consume a task from its domain's -- * corresponding dispatch queue (this occurs after scheduling any tasks directly -- * assigned to it due to the logic in atropos_select_cpu). If no task is found, -- * then greedy load stealing will attempt to find a task on another dispatch -- * queue to run. -- * -- * Load balancing is almost entirely handled by userspace. BPF populates the -- * task weight, dom mask and current dom in the task_data map and executes the -- * load balance based on userspace populating the lb_data map. -- */ --#include "../../../scx_common.bpf.h" --#include "atropos.h" -- --#include --#include --#include --#include --#include --#include -- --char _license[] SEC("license") = "GPL"; -- --/* -- * const volatiles are set during initialization and treated as consts by the -- * jit compiler. -- */ -- --/* -- * Domains and cpus -- */ --const volatile __u32 nr_doms = 32; /* !0 for veristat, set during init */ --const volatile __u32 nr_cpus = 64; /* !0 for veristat, set during init */ --const volatile __u32 cpu_dom_id_map[MAX_CPUS]; --const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; -- --const volatile bool kthreads_local; --const volatile bool fifo_sched; --const volatile bool switch_partial; --const volatile __u32 greedy_threshold; -- --/* base slice duration */ --const volatile __u64 slice_us = 20000; -- --/* -- * Exit info -- */ --int exit_type = SCX_EXIT_NONE; --char exit_msg[SCX_EXIT_MSG_LEN]; -- --struct pcpu_ctx { -- __u32 dom_rr_cur; /* used when scanning other doms */ -- -- /* libbpf-rs does not respect the alignment, so pad out the struct explicitly */ -- __u8 _padding[CACHELINE_SIZE - sizeof(u64)]; --} __attribute__((aligned(CACHELINE_SIZE))); -- --struct pcpu_ctx pcpu_ctx[MAX_CPUS]; -- --/* -- * Domain context -- */ --struct dom_ctx { -- struct bpf_cpumask __kptr *cpumask; -- u64 vtime_now; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __type(key, u32); -- __type(value, struct dom_ctx); -- __uint(max_entries, MAX_DOMS); -- __uint(map_flags, 0); --} dom_ctx SEC(".maps"); -- --/* -- * Statistics -- */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, ATROPOS_NR_STATS); --} stats SEC(".maps"); -- --static inline void stat_add(enum stat_idx idx, u64 addend) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p) += addend; --} -- --/* Map pid -> task_ctx */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, struct task_ctx); -- __uint(max_entries, 1000000); -- __uint(map_flags, 0); --} task_data SEC(".maps"); -- --/* -- * This is populated from userspace to indicate which pids should be reassigned -- * to new doms. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, u32); -- __uint(max_entries, 1000); -- __uint(map_flags, 0); --} lb_data SEC(".maps"); -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p, -- u32 new_dom_id) --{ -- struct dom_ctx *old_domc, *new_domc; -- struct bpf_cpumask *d_cpumask, *t_cpumask; -- u32 old_dom_id = task_ctx->dom_id; -- s64 vtime_delta; -- -- old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id); -- if (!old_domc) { -- scx_bpf_error("No dom%u", old_dom_id); -- return false; -- } -- -- vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now; -- -- new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id); -- if (!new_domc) { -- scx_bpf_error("No dom%u", new_dom_id); -- return false; -- } -- -- d_cpumask = new_domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to get domain %u cpumask kptr", -- new_dom_id); -- return false; -- } -- -- t_cpumask = task_ctx->cpumask; -- if (!t_cpumask) { -- scx_bpf_error("Failed to look up task cpumask"); -- return false; -- } -- -- /* -- * set_cpumask might have happened between userspace requesting LB and -- * here and @p might not be able to run in @dom_id anymore. Verify. -- */ -- if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- p->cpus_ptr)) { -- p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta; -- task_ctx->dom_id = new_dom_id; -- bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask, -- p->cpus_ptr); -- } -- -- return task_ctx->dom_id == new_dom_id; --} -- --s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu, -- u32 wake_flags) --{ -- s32 cpu; -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- struct bpf_cpumask *p_cpumask; -- -- if (!task_ctx) -- return -ENOENT; -- -- if (kthreads_local && -- (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- cpu = prev_cpu; -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local dsq of the waker. -- */ -- if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) { -- struct task_struct *current = (void *)bpf_get_current_task(); -- -- if (!(BPF_CORE_READ(current, flags) & PF_EXITING) && -- task_ctx->dom_id < MAX_DOMS) { -- struct dom_ctx *domc; -- struct bpf_cpumask *d_cpumask; -- const struct cpumask *idle_cpumask; -- bool has_idle; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id); -- if (!domc) { -- scx_bpf_error("Failed to find dom%u", -- task_ctx->dom_id); -- return prev_cpu; -- } -- d_cpumask = domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to acquire domain %u cpumask kptr", -- task_ctx->dom_id); -- return prev_cpu; -- } -- -- idle_cpumask = scx_bpf_get_idle_cpumask(); -- -- has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- idle_cpumask); -- -- scx_bpf_put_idle_cpumask(idle_cpumask); -- -- if (has_idle) { -- cpu = bpf_get_smp_processor_id(); -- if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- stat_add(ATROPOS_STAT_WAKE_SYNC, 1); -- goto local; -- } -- } -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- stat_add(ATROPOS_STAT_PREV_IDLE, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- /* If only one core is allowed, dispatch */ -- if (p->nr_cpus_allowed == 1) { -- stat_add(ATROPOS_STAT_PINNED, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) -- return -ENOENT; -- -- /* If there is an eligible idle CPU, dispatch directly */ -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- if (cpu >= 0) { -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * @prev_cpu may be in a different domain. Returning an out-of-domain -- * CPU can lead to stalls as all in-domain CPUs may be idle by the time -- * @p gets enqueued. -- */ -- if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask)) -- cpu = prev_cpu; -- else -- cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask); -- -- return cpu; -- --local: -- task_ctx->dispatch_local = true; -- return cpu; --} -- --void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- u32 *new_dom; -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- new_dom = bpf_map_lookup_elem(&lb_data, &pid); -- if (new_dom && *new_dom != task_ctx->dom_id && -- task_set_dsq(task_ctx, p, *new_dom)) { -- struct bpf_cpumask *p_cpumask; -- s32 cpu; -- -- stat_add(ATROPOS_STAT_LOAD_BALANCE, 1); -- -- /* -- * If dispatch_local is set, We own @p's idle state but we are -- * not gonna put the task in the associated local dsq which can -- * cause the CPU to stall. Kick it. -- */ -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0); -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) { -- scx_bpf_error("Failed to get task_ctx->cpumask"); -- return; -- } -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- } -- -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags); -- return; -- } -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000, -- enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- u32 dom_id = task_ctx->dom_id; -- struct dom_ctx *domc; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, domc->vtime_now - slice_us * 1000)) -- vtime = domc->vtime_now - slice_us * 1000; -- -- scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --static u32 cpu_to_dom_id(s32 cpu) --{ -- const volatile u32 *dom_idp; -- -- if (nr_doms <= 1) -- return 0; -- -- dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]); -- if (!dom_idp) -- return MAX_DOMS; -- -- return *dom_idp; --} -- --static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) --{ -- s32 cpu; -- -- if (dom_id >= MAX_DOMS) -- return false; -- -- bpf_for(cpu, 0, nr_cpus) { -- if (bpf_cpumask_test_cpu(cpu, cpumask) && -- (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64)))) -- return true; -- } -- return false; --} -- --static u32 dom_rr_next(s32 cpu) --{ -- struct pcpu_ctx *pcpuc; -- u32 dom_id; -- -- pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]); -- if (!pcpuc) -- return 0; -- -- dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms; -- -- if (dom_id == cpu_to_dom_id(cpu)) -- dom_id = (dom_id + 1) % nr_doms; -- -- pcpuc->dom_rr_cur = dom_id; -- return dom_id; --} -- --void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 dom = cpu_to_dom_id(cpu); -- -- if (scx_bpf_consume(dom)) { -- stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1); -- return; -- } -- -- if (!greedy_threshold) -- return; -- -- bpf_repeat(nr_doms - 1) { -- u32 dom_id = dom_rr_next(cpu); -- -- if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold && -- scx_bpf_consume(dom_id)) { -- stat_add(ATROPOS_STAT_GREEDY, 1); -- break; -- } -- } --} -- --void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_at = bpf_ktime_get_ns(); --} -- --void BPF_STRUCT_OPS(atropos_running, struct task_struct *p) --{ -- struct task_ctx *taskc; -- struct dom_ctx *domc; -- pid_t pid = p->pid; -- u32 dom_id; -- -- if (fifo_sched) -- return; -- -- taskc = bpf_map_lookup_elem(&task_data, &pid); -- if (!taskc) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- dom_id = taskc->dom_id; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(domc->vtime_now, p->scx.dsq_vtime)) -- domc->vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at; -- task_ctx->runnable_at = 0; --} -- --void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->weight = weight; --} -- --struct pick_task_domain_loop_ctx { -- struct task_struct *p; -- const struct cpumask *cpumask; -- u64 dom_mask; -- u32 dom_rr_base; -- u32 dom_id; --}; -- --static int pick_task_domain_loopfn(u32 idx, void *data) --{ -- struct pick_task_domain_loop_ctx *lctx = data; -- u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms; -- -- if (dom_id >= MAX_DOMS) -- return 1; -- -- if (cpumask_intersects_domain(lctx->cpumask, dom_id)) { -- lctx->dom_mask |= 1LLU << dom_id; -- if (lctx->dom_id == MAX_DOMS) -- lctx->dom_id = dom_id; -- } -- return 0; --} -- --static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- struct pick_task_domain_loop_ctx lctx = { -- .p = p, -- .cpumask = cpumask, -- .dom_id = MAX_DOMS, -- }; -- s32 cpu = bpf_get_smp_processor_id(); -- -- if (cpu < 0 || cpu >= MAX_CPUS) -- return MAX_DOMS; -- -- lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur); -- -- bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0); -- task_ctx->dom_mask = lctx.dom_mask; -- -- return lctx.dom_id; --} -- --static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- u32 dom_id = 0; -- -- if (nr_doms > 1) -- dom_id = pick_task_domain(task_ctx, p, cpumask); -- -- if (!task_set_dsq(task_ctx, p, dom_id)) -- scx_bpf_error("Failed to set domain %d for %s[%d]", -- dom_id, p->comm, p->pid); --} -- --void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_set_domain(task_ctx, p, cpumask); --} -- --s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct bpf_cpumask *cpumask; -- struct task_ctx task_ctx, *map_value; -- long ret; -- pid_t pid; -- -- memset(&task_ctx, 0, sizeof(task_ctx)); -- -- pid = p->pid; -- ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return ret; -- } -- -- /* -- * Read the entry from the map immediately so we can add the cpumask -- * with bpf_kptr_xchg(). -- */ -- map_value = bpf_map_lookup_elem(&task_data, &pid); -- if (!map_value) -- /* Should never happen -- it was just inserted above. */ -- return -EINVAL; -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- bpf_map_delete_elem(&task_data, &pid); -- return -ENOMEM; -- } -- -- cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask); -- if (cpumask) { -- /* Should never happen as we just inserted it above. */ -- bpf_cpumask_release(cpumask); -- bpf_map_delete_elem(&task_data, &pid); -- return -EINVAL; -- } -- -- task_set_domain(map_value, p, p->cpus_ptr); -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p) --{ -- pid_t pid = p->pid; -- long ret = bpf_map_delete_elem(&task_data, &pid); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return; -- } --} -- --static int create_dom_dsq(u32 idx, void *data) --{ -- struct dom_ctx domc_init = {}, *domc; -- struct bpf_cpumask *cpumask; -- u32 cpu, dom_id = idx; -- s32 ret; -- -- ret = scx_bpf_create_dsq(dom_id, -1); -- if (ret < 0) { -- scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret); -- return 1; -- } -- -- ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0); -- if (ret) { -- scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret); -- return 1; -- } -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- /* Should never happen, we just inserted it above. */ -- scx_bpf_error("No dom%u", dom_id); -- return 1; -- } -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id); -- return 1; -- } -- -- for (cpu = 0; cpu < MAX_CPUS; cpu++) { -- const volatile __u64 *dmask; -- -- dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]); -- if (!dmask) { -- scx_bpf_error("array index error"); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- if (*dmask & (1LLU << (cpu % 64))) -- bpf_cpumask_set_cpu(cpu, cpumask); -- } -- -- cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask); -- if (cpumask) { -- scx_bpf_error("Domain %u was already present", dom_id); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(atropos_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- bpf_loop(nr_doms, create_dom_dsq, NULL, 0); -- -- for (u32 i = 0; i < nr_cpus; i++) -- pcpu_ctx[i].dom_rr_cur = i; -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg); -- exit_type = ei->type; --} -- --SEC(".struct_ops") --struct sched_ext_ops atropos = { -- .select_cpu = (void *)atropos_select_cpu, -- .enqueue = (void *)atropos_enqueue, -- .dispatch = (void *)atropos_dispatch, -- .runnable = (void *)atropos_runnable, -- .running = (void *)atropos_running, -- .stopping = (void *)atropos_stopping, -- .quiescent = (void *)atropos_quiescent, -- .set_weight = (void *)atropos_set_weight, -- .set_cpumask = (void *)atropos_set_cpumask, -- .prep_enable = (void *)atropos_prep_enable, -- .disable = (void *)atropos_disable, -- .init = (void *)atropos_init, -- .exit = (void *)atropos_exit, -- .flags = 0, -- .name = "atropos", --}; -diff --git b/tools/sched_ext/atropos/src/bpf/atropos.h a/tools/sched_ext/atropos/src/bpf/atropos.h -deleted file mode 100644 -index addf29ca104a..000000000000 ---- b/tools/sched_ext/atropos/src/bpf/atropos.h -+++ /dev/null -@@ -1,44 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#ifndef __ATROPOS_H --#define __ATROPOS_H -- --#include --#ifndef __kptr --#ifdef __KERNEL__ --#error "__kptr_ref not defined in the kernel" --#endif --#define __kptr --#endif -- --#define MAX_CPUS 512 --#define MAX_DOMS 64 /* limited to avoid complex bitmask ops */ --#define CACHELINE_SIZE 64 -- --/* Statistics */ --enum stat_idx { -- ATROPOS_STAT_TASK_GET_ERR, -- ATROPOS_STAT_WAKE_SYNC, -- ATROPOS_STAT_PREV_IDLE, -- ATROPOS_STAT_PINNED, -- ATROPOS_STAT_DIRECT_DISPATCH, -- ATROPOS_STAT_DSQ_DISPATCH, -- ATROPOS_STAT_GREEDY, -- ATROPOS_STAT_LOAD_BALANCE, -- ATROPOS_STAT_LAST_TASK, -- ATROPOS_NR_STATS, --}; -- --struct task_ctx { -- unsigned long long dom_mask; /* the domains this task can run on */ -- struct bpf_cpumask __kptr *cpumask; -- unsigned int dom_id; -- unsigned int weight; -- unsigned long long runnable_at; -- unsigned long long runnable_for; -- bool dispatch_local; --}; -- --#endif /* __ATROPOS_H */ -diff --git b/tools/sched_ext/atropos/src/main.rs a/tools/sched_ext/atropos/src/main.rs -deleted file mode 100644 -index 0d313662f713..000000000000 ---- b/tools/sched_ext/atropos/src/main.rs -+++ /dev/null -@@ -1,942 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#[path = "bpf/.output/atropos.skel.rs"] --mod atropos; --pub use atropos::*; --pub mod atropos_sys; -- --use std::cell::Cell; --use std::collections::{BTreeMap, BTreeSet}; --use std::ffi::CStr; --use std::ops::Bound::{Included, Unbounded}; --use std::sync::atomic::{AtomicBool, Ordering}; --use std::sync::Arc; --use std::time::{Duration, SystemTime}; -- --use ::fb_procfs as procfs; --use anyhow::{anyhow, bail, Context, Result}; --use bitvec::prelude::*; --use clap::Parser; --use log::{info, trace, warn}; --use ordered_float::OrderedFloat; -- --/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF --/// part does simple round robin in each domain and the userspace part --/// calculates the load factor of each domain and tells the BPF part how to load --/// balance the domains. -- --/// This scheduler demonstrates dividing scheduling logic between BPF and --/// userspace and using rust to build the userspace part. An earlier variant of --/// this scheduler was used to balance across six domains, each representing a --/// chiplet in a six-chiplet AMD processor, and could match the performance of --/// production setup using CFS. --#[derive(Debug, Parser)] --struct Opts { -- /// Scheduling slice duration in microseconds. -- #[clap(short, long, default_value = "20000")] -- slice_us: u64, -- -- /// Monitoring and load balance interval in seconds. -- #[clap(short, long, default_value = "2.0")] -- interval: f64, -- -- /// Build domains according to how CPUs are grouped at this cache level -- /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id. -- #[clap(short = 'c', long, default_value = "3")] -- cache_level: u32, -- -- /// Instead of using cache locality, set the cpumask for each domain -- /// manually, provide multiple --cpumasks, one for each domain. E.g. -- /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with -- /// the corresponding CPUs belonging to each domain. Each CPU must -- /// belong to precisely one domain. -- #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")] -- cpumasks: Vec, -- -- /// When non-zero, enable greedy task stealing. When a domain is idle, a -- /// cpu will attempt to steal tasks from a domain with at least -- /// greedy_threshold tasks enqueued. These tasks aren't permanently -- /// stolen from the domain. -- #[clap(short, long, default_value = "4")] -- greedy_threshold: u32, -- -- /// The load decay factor. Every interval, the existing load is decayed -- /// by this factor and new load is added. Must be in the range [0.0, -- /// 0.99]. The smaller the value, the more sensitive load calculation -- /// is to recent changes. When 0.0, history is ignored and the load -- /// value from the latest period is used directly. -- #[clap(short, long, default_value = "0.5")] -- load_decay_factor: f64, -- -- /// Disable load balancing. Unless disabled, periodically userspace will -- /// calculate the load factor of each domain and instruct BPF which -- /// processes to move. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- no_load_balance: bool, -- -- /// Put per-cpu kthreads directly into local dsq's. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- kthreads_local: bool, -- -- /// Use FIFO scheduling instead of weighted vtime scheduling. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- fifo_sched: bool, -- -- /// If specified, only tasks which have their scheduling policy set to -- /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all -- /// tasks are switched. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- partial: bool, -- -- /// Enable verbose output including libbpf details. Specify multiple -- /// times to increase verbosity. -- #[clap(short, long, action = clap::ArgAction::Count)] -- verbose: u8, --} -- --fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result { -- Ok(reader -- .read_stat() -- .context("Failed to read procfs")? -- .total_cpu -- .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?) --} -- --fn now_monotonic() -> u64 { -- let mut time = libc::timespec { -- tv_sec: 0, -- tv_nsec: 0, -- }; -- let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) }; -- assert!(ret == 0); -- time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 --} -- --fn clear_map(map: &mut libbpf_rs::Map) { -- // XXX: libbpf_rs has some design flaw that make it impossible to -- // delete while iterating despite it being safe so we alias it here -- let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) }; -- for key in map.keys() { -- let _ = deleter.delete(&key); -- } --} -- --#[derive(Debug)] --struct TaskLoad { -- runnable_for: u64, -- load: f64, --} -- --#[derive(Debug)] --struct TaskInfo { -- pid: i32, -- dom_mask: u64, -- migrated: Cell, --} -- --struct LoadBalancer<'a, 'b, 'c> { -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- -- tasks_by_load: Vec, TaskInfo>>, -- load_avg: f64, -- dom_loads: Vec, -- -- imbal: Vec, -- doms_to_push: BTreeMap, u32>, -- doms_to_pull: BTreeMap, u32>, -- -- nr_lb_data_errors: &'c mut u64, --} -- --impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> { -- const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10; -- const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1; -- const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50; -- -- fn new( -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- nr_lb_data_errors: &'c mut u64, -- ) -> Self { -- Self { -- maps, -- task_loads, -- nr_doms, -- load_decay_factor, -- -- tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(), -- load_avg: 0f64, -- dom_loads: vec![0.0; nr_doms], -- -- imbal: vec![0.0; nr_doms], -- doms_to_pull: BTreeMap::new(), -- doms_to_push: BTreeMap::new(), -- -- nr_lb_data_errors, -- } -- } -- -- fn read_task_loads(&mut self, period: Duration) -> Result<()> { -- let now_mono = now_monotonic(); -- let task_data = self.maps.task_data(); -- let mut this_task_loads = BTreeMap::::new(); -- let mut load_sum = 0.0f64; -- self.dom_loads = vec![0f64; self.nr_doms]; -- -- for key in task_data.keys() { -- if let Some(task_ctx_vec) = task_data -- .lookup(&key, libbpf_rs::MapFlags::ANY) -- .context("Failed to lookup task_data")? -- { -- let task_ctx = -- unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) }; -- let pid = i32::from_ne_bytes( -- key.as_slice() -- .try_into() -- .context("Invalid key length in task_data map")?, -- ); -- -- let (this_at, this_for, weight) = unsafe { -- ( -- std::ptr::read_volatile(&task_ctx.runnable_at as *const u64), -- std::ptr::read_volatile(&task_ctx.runnable_for as *const u64), -- std::ptr::read_volatile(&task_ctx.weight as *const u32), -- ) -- }; -- -- let (mut delta, prev_load) = match self.task_loads.get(&pid) { -- Some(prev) => (this_for - prev.runnable_for, Some(prev.load)), -- None => (this_for, None), -- }; -- -- // Non-zero this_at indicates that the task is currently -- // runnable. Note that we read runnable_at and runnable_for -- // without any synchronization and there is a small window -- // where we end up misaccounting. While this can cause -- // temporary error, it's unlikely to cause any noticeable -- // misbehavior especially given the load value clamping. -- if this_at > 0 && this_at < now_mono { -- delta += now_mono - this_at; -- } -- -- delta = delta.min(period.as_nanos() as u64); -- let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64) -- .clamp(0.0, weight as f64); -- -- let this_load = match prev_load { -- Some(prev_load) => { -- prev_load * self.load_decay_factor -- + this_load * (1.0 - self.load_decay_factor) -- } -- None => this_load, -- }; -- -- this_task_loads.insert( -- pid, -- TaskLoad { -- runnable_for: this_for, -- load: this_load, -- }, -- ); -- -- load_sum += this_load; -- self.dom_loads[task_ctx.dom_id as usize] += this_load; -- // Only record pids that are eligible for load balancing -- if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) { -- continue; -- } -- self.tasks_by_load[task_ctx.dom_id as usize].insert( -- OrderedFloat(this_load), -- TaskInfo { -- pid, -- dom_mask: task_ctx.dom_mask, -- migrated: Cell::new(false), -- }, -- ); -- } -- } -- -- self.load_avg = load_sum / self.nr_doms as f64; -- *self.task_loads = this_task_loads; -- Ok(()) -- } -- -- // To balance dom loads we identify doms with lower and higher load than average -- fn calculate_dom_load_balance(&mut self) -> Result<()> { -- for (dom, dom_load) in self.dom_loads.iter().enumerate() { -- let imbal = dom_load - self.load_avg; -- if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO { -- if imbal > 0f64 { -- self.doms_to_push.insert(OrderedFloat(imbal), dom as u32); -- } else { -- self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32); -- } -- self.imbal[dom] = imbal; -- } -- } -- Ok(()) -- } -- -- // Find the first candidate pid which hasn't already been migrated and -- // can run in @pull_dom. -- fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)> -- where -- I: IntoIterator, &'d TaskInfo)>, -- { -- match tasks_by_load -- .into_iter() -- .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0) -- .next() -- { -- Some((OrderedFloat(load), task)) => Some((*load, task)), -- None => None, -- } -- } -- -- fn pick_victim( -- &self, -- (push_dom, to_push): (u32, f64), -- (pull_dom, to_pull): (u32, f64), -- ) -> Option<(&TaskInfo, f64)> { -- let to_xfer = to_pull.min(to_push); -- -- trace!( -- "considering dom {}@{:.2} -> {}@{:.2}", -- push_dom, -- to_push, -- pull_dom, -- to_pull -- ); -- -- let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs(); -- -- trace!( -- "to_xfer={:.2} tasks_by_load={:?}", -- to_xfer, -- &self.tasks_by_load[push_dom as usize] -- ); -- -- // We want to pick a task to transfer from push_dom to pull_dom to -- // maximize the reduction of load imbalance between the two. IOW, -- // pick a task which has the closest load value to $to_xfer that can -- // be migrated. Find such task by locating the first migratable task -- // while scanning left from $to_xfer and the counterpart while -- // scanning right and picking the better of the two. -- let (load, task, new_imbal) = match ( -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Unbounded, Included(&OrderedFloat(to_xfer)))) -- .rev(), -- pull_dom, -- ), -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Included(&OrderedFloat(to_xfer)), Unbounded)), -- pull_dom, -- ), -- ) { -- (None, None) => return None, -- (Some((load, task)), None) | (None, Some((load, task))) => { -- (load, task, calc_new_imbal(load)) -- } -- (Some((load0, task0)), Some((load1, task1))) => { -- let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1)); -- if new_imbal0 <= new_imbal1 { -- (load0, task0, new_imbal0) -- } else { -- (load1, task1, new_imbal1) -- } -- } -- }; -- -- // If the best candidate can't reduce the imbalance, there's nothing -- // to do for this pair. -- let old_imbal = to_push + to_pull; -- if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal { -- trace!( -- "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal -- ); -- return None; -- } -- -- trace!( -- "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal, -- ); -- -- Some((task, load)) -- } -- -- // Actually execute the load balancing. Concretely this writes pid -> dom -- // entries into the lb_data map for bpf side to consume. -- fn load_balance(&mut self) -> Result<()> { -- clear_map(self.maps.lb_data()); -- -- trace!("imbal={:?}", &self.imbal); -- trace!("doms_to_push={:?}", &self.doms_to_push); -- trace!("doms_to_pull={:?}", &self.doms_to_pull); -- -- // Push from the most imbalanced to least. -- while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() { -- let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO; -- let mut pushed = 0f64; -- -- // Transfer tasks from push_dom to reduce imbalance. -- loop { -- let last_pushed = pushed; -- -- // Pull from the most imbalaned to least. -- let mut doms_to_pull = BTreeMap::<_, _>::new(); -- std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull); -- let mut pull_doms = doms_to_pull.into_iter().rev().collect::>(); -- -- for (to_pull, pull_dom) in pull_doms.iter_mut() { -- if let Some((task, load)) = -- self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull))) -- { -- // Execute migration. -- task.migrated.set(true); -- to_push -= load; -- *to_pull -= load; -- pushed += load; -- -- // Ask BPF code to execute the migration. -- let pid = task.pid; -- let cpid = (pid as libc::pid_t).to_ne_bytes(); -- if let Err(e) = self.maps.lb_data().update( -- &cpid, -- &pull_dom.to_ne_bytes(), -- libbpf_rs::MapFlags::NO_EXIST, -- ) { -- warn!( -- "Failed to update lb_data map for pid={} error={:?}", -- pid, &e -- ); -- *self.nr_lb_data_errors += 1; -- } -- -- // Always break after a successful migration so that -- // the pulling domains are always considered in the -- // descending imbalance order. -- break; -- } -- } -- -- pull_doms -- .into_iter() -- .map(|(k, v)| self.doms_to_pull.insert(k, v)) -- .count(); -- -- // Stop repeating if nothing got transferred or pushed enough. -- if pushed == last_pushed || pushed >= push_max { -- break; -- } -- } -- } -- Ok(()) -- } --} -- --struct Scheduler<'a> { -- skel: AtroposSkel<'a>, -- struct_ops: Option, -- -- nr_cpus: usize, -- nr_doms: usize, -- load_decay_factor: f64, -- balance_load: bool, -- -- proc_reader: procfs::ProcReader, -- -- prev_at: SystemTime, -- prev_total_cpu: procfs::CpuStat, -- task_loads: BTreeMap, -- -- nr_lb_data_errors: u64, --} -- --impl<'a> Scheduler<'a> { -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn parse_cpusets( -- cpumasks: &[String], -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- if cpumasks.len() > atropos_sys::MAX_DOMS as usize { -- bail!( -- "Number of requested DSQs ({}) is greater than MAX_DOMS ({})", -- cpumasks.len(), -- atropos_sys::MAX_DOMS -- ); -- } -- let mut cpus = vec![-1i32; nr_cpus]; -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()]; -- for (dq, cpumask) in cpumasks.iter().enumerate() { -- let hex_str = { -- let mut tmp_str = cpumask -- .strip_prefix("0x") -- .unwrap_or(cpumask) -- .replace('_', ""); -- if tmp_str.len() % 2 != 0 { -- tmp_str = "0".to_string() + &tmp_str; -- } -- tmp_str -- }; -- let byte_vec = hex::decode(&hex_str) -- .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?; -- -- for (index, &val) in byte_vec.iter().rev().enumerate() { -- let mut v = val; -- while v != 0 { -- let lsb = v.trailing_zeros() as usize; -- v &= !(1 << lsb); -- let cpu = index * 8 + lsb; -- if cpu > nr_cpus { -- bail!( -- concat!( -- "Found cpu ({}) in cpumask ({}) which is larger", -- " than the number of cpus on the machine ({})" -- ), -- cpu, -- cpumask, -- nr_cpus -- ); -- } -- if cpus[cpu] != -1 { -- bail!( -- "Found cpu ({}) with dq ({}) but also in cpumask ({})", -- cpu, -- cpus[cpu], -- cpumask -- ); -- } -- cpus[cpu] = dq as i32; -- cpusets[dq].set(cpu, true); -- } -- } -- cpusets[dq].set_uninitialized(false); -- } -- -- for (cpu, &dq) in cpus.iter().enumerate() { -- if dq < 0 { -- bail!( -- "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.", -- cpu -- ); -- } -- } -- -- Ok((cpusets, cpus)) -- } -- -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn cpusets_from_cache( -- level: u32, -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- let mut cpu_to_cache = vec![]; // (cpu_id, cache_id) -- let mut cache_ids = BTreeSet::::new(); -- let mut nr_not_found = 0; -- -- // Build cpu -> cache ID mapping. -- for cpu in 0..nr_cpus { -- let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level); -- let id = match std::fs::read_to_string(&path) { -- Ok(val) => val -- .trim() -- .parse::() -- .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?, -- Err(e) if e.kind() == std::io::ErrorKind::NotFound => { -- nr_not_found += 1; -- 0 -- } -- Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)), -- }; -- -- cpu_to_cache.push(id); -- cache_ids.insert(id); -- } -- -- if nr_not_found > 1 { -- warn!( -- "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0", -- level, nr_not_found, nr_cpus -- ); -- } -- -- // Cache IDs may have holes. Assign consecutive domain IDs to -- // existing cache IDs. -- let mut cache_to_dom = BTreeMap::::new(); -- let mut nr_doms = 0; -- for cache_id in cache_ids.iter() { -- cache_to_dom.insert(*cache_id, nr_doms); -- nr_doms += 1; -- } -- -- if nr_doms > atropos_sys::MAX_DOMS { -- bail!( -- "Total number of doms {} is greater than MAX_DOMS ({})", -- nr_doms, -- atropos_sys::MAX_DOMS -- ); -- } -- -- // Build and return dom -> cpumask and cpu -> dom mappings. -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize]; -- let mut cpu_to_dom = vec![]; -- -- for cpu in 0..nr_cpus { -- let dom_id = cache_to_dom[&cpu_to_cache[cpu]]; -- cpusets[dom_id as usize].set(cpu, true); -- cpu_to_dom.push(dom_id as i32); -- } -- -- Ok((cpusets, cpu_to_dom)) -- } -- -- fn init(opts: &Opts) -> Result { -- // Open the BPF prog first for verification. -- let mut skel_builder = AtroposSkelBuilder::default(); -- skel_builder.obj_builder.debug(opts.verbose > 0); -- let mut skel = skel_builder.open().context("Failed to open BPF program")?; -- -- let nr_cpus = libbpf_rs::num_possible_cpus().unwrap(); -- if nr_cpus > atropos_sys::MAX_CPUS as usize { -- bail!( -- "nr_cpus ({}) is greater than MAX_CPUS ({})", -- nr_cpus, -- atropos_sys::MAX_CPUS -- ); -- } -- -- // Initialize skel according to @opts. -- let (cpusets, cpus) = if opts.cpumasks.len() > 0 { -- Self::parse_cpusets(&opts.cpumasks, nr_cpus)? -- } else { -- Self::cpusets_from_cache(opts.cache_level, nr_cpus)? -- }; -- let nr_doms = cpusets.len(); -- skel.rodata().nr_doms = nr_doms as u32; -- skel.rodata().nr_cpus = nr_cpus as u32; -- -- for (cpu, dom) in cpus.iter().enumerate() { -- skel.rodata().cpu_dom_id_map[cpu] = *dom as u32; -- } -- -- for (dom, cpuset) in cpusets.iter().enumerate() { -- let raw_cpuset_slice = cpuset.as_raw_slice(); -- let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom]; -- let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len()); -- left.clone_from_slice(cpuset.as_raw_slice()); -- let cpumask_str = dom_cpumask_slice -- .iter() -- .take((nr_cpus + 63) / 64) -- .rev() -- .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x)); -- info!( -- "DOM[{:02}] cpumask{} ({} cpus)", -- dom, -- &cpumask_str, -- cpuset.count_ones() -- ); -- } -- -- skel.rodata().slice_us = opts.slice_us; -- skel.rodata().kthreads_local = opts.kthreads_local; -- skel.rodata().fifo_sched = opts.fifo_sched; -- skel.rodata().switch_partial = opts.partial; -- skel.rodata().greedy_threshold = opts.greedy_threshold; -- -- // Attach. -- let mut skel = skel.load().context("Failed to load BPF program")?; -- skel.attach().context("Failed to attach BPF program")?; -- let struct_ops = Some( -- skel.maps_mut() -- .atropos() -- .attach_struct_ops() -- .context("Failed to attach atropos struct ops")?, -- ); -- info!("Atropos Scheduler Attached"); -- -- // Other stuff. -- let mut proc_reader = procfs::ProcReader::new(); -- let prev_total_cpu = read_total_cpu(&mut proc_reader)?; -- -- Ok(Self { -- skel, -- struct_ops, // should be held to keep it attached -- -- nr_cpus, -- nr_doms, -- load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99), -- balance_load: !opts.no_load_balance, -- -- proc_reader, -- -- prev_at: SystemTime::now(), -- prev_total_cpu, -- task_loads: BTreeMap::new(), -- -- nr_lb_data_errors: 0, -- }) -- } -- -- fn get_cpu_busy(&mut self) -> Result { -- let total_cpu = read_total_cpu(&mut self.proc_reader)?; -- let busy = match (&self.prev_total_cpu, &total_cpu) { -- ( -- procfs::CpuStat { -- user_usec: Some(prev_user), -- nice_usec: Some(prev_nice), -- system_usec: Some(prev_system), -- idle_usec: Some(prev_idle), -- iowait_usec: Some(prev_iowait), -- irq_usec: Some(prev_irq), -- softirq_usec: Some(prev_softirq), -- stolen_usec: Some(prev_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- procfs::CpuStat { -- user_usec: Some(curr_user), -- nice_usec: Some(curr_nice), -- system_usec: Some(curr_system), -- idle_usec: Some(curr_idle), -- iowait_usec: Some(curr_iowait), -- irq_usec: Some(curr_irq), -- softirq_usec: Some(curr_softirq), -- stolen_usec: Some(curr_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- ) => { -- let idle_usec = curr_idle - prev_idle; -- let iowait_usec = curr_iowait - prev_iowait; -- let user_usec = curr_user - prev_user; -- let system_usec = curr_system - prev_system; -- let nice_usec = curr_nice - prev_nice; -- let irq_usec = curr_irq - prev_irq; -- let softirq_usec = curr_softirq - prev_softirq; -- let stolen_usec = curr_stolen - prev_stolen; -- -- let busy_usec = -- user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec; -- let total_usec = idle_usec + busy_usec + iowait_usec; -- busy_usec as f64 / total_usec as f64 -- } -- _ => { -- bail!("Some procfs stats are not populated!"); -- } -- }; -- -- self.prev_total_cpu = total_cpu; -- Ok(busy) -- } -- -- fn read_bpf_stats(&mut self) -> Result> { -- let mut maps = self.skel.maps_mut(); -- let stats_map = maps.stats(); -- let mut stats: Vec = Vec::new(); -- let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus]; -- -- for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS { -- let cpu_stat_vec = stats_map -- .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY) -- .with_context(|| format!("Failed to lookup stat {}", stat))? -- .expect("per-cpu stat should exist"); -- let sum = cpu_stat_vec -- .iter() -- .map(|val| { -- u64::from_ne_bytes( -- val.as_slice() -- .try_into() -- .expect("Invalid value length in stat map"), -- ) -- }) -- .sum(); -- stats_map -- .update_percpu( -- &(stat as u32).to_ne_bytes(), -- &zero_vec, -- libbpf_rs::MapFlags::ANY, -- ) -- .context("Failed to zero stat")?; -- stats.push(sum); -- } -- Ok(stats) -- } -- -- fn report( -- &self, -- stats: &Vec, -- cpu_busy: f64, -- processing_dur: Duration, -- load_avg: f64, -- dom_loads: &Vec, -- imbal: &Vec, -- ) { -- let stat = |idx| stats[idx as usize]; -- let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK); -- -- info!( -- "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms", -- cpu_busy * 100.0, -- load_avg, -- stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize], -- stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize], -- self.nr_lb_data_errors, -- processing_dur.as_millis(), -- ); -- -- let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0; -- -- info!( -- "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}", -- total, -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY), -- ); -- -- for i in 0..self.nr_doms { -- info!( -- "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}", -- i, -- dom_loads[i], -- if imbal[i] < 0.0 { -imbal[i] } else { 0.0 }, -- if imbal[i] > 0.0 { imbal[i] } else { 0.0 }, -- ); -- } -- } -- -- fn step(&mut self) -> Result<()> { -- let started_at = std::time::SystemTime::now(); -- let bpf_stats = self.read_bpf_stats()?; -- let cpu_busy = self.get_cpu_busy()?; -- -- let mut lb = LoadBalancer::new( -- self.skel.maps_mut(), -- &mut self.task_loads, -- self.nr_doms, -- self.load_decay_factor, -- &mut self.nr_lb_data_errors, -- ); -- -- lb.read_task_loads(started_at.duration_since(self.prev_at)?)?; -- lb.calculate_dom_load_balance()?; -- -- if self.balance_load { -- lb.load_balance()?; -- } -- -- // Extract fields needed for reporting and drop lb to release -- // mutable borrows. -- let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal); -- -- self.report( -- &bpf_stats, -- cpu_busy, -- std::time::SystemTime::now().duration_since(started_at)?, -- load_avg, -- &dom_loads, -- &imbal, -- ); -- -- self.prev_at = started_at; -- Ok(()) -- } -- -- fn read_bpf_exit_type(&mut self) -> i32 { -- unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) } -- } -- -- fn report_bpf_exit_type(&mut self) -> Result<()> { -- // Report msg if EXT_OPS_EXIT_ERROR. -- match self.read_bpf_exit_type() { -- 0 => Ok(()), -- etype if etype == 2 => { -- let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) }; -- let msg = cstr -- .to_str() -- .context("Failed to convert exit msg to string") -- .unwrap(); -- bail!("BPF exit_type={} msg={}", etype, msg); -- } -- etype => { -- info!("BPF exit_type={}", etype); -- Ok(()) -- } -- } -- } --} -- --impl<'a> Drop for Scheduler<'a> { -- fn drop(&mut self) { -- if let Some(struct_ops) = self.struct_ops.take() { -- drop(struct_ops); -- } -- } --} -- --fn main() -> Result<()> { -- let opts = Opts::parse(); -- -- let llv = match opts.verbose { -- 0 => simplelog::LevelFilter::Info, -- 1 => simplelog::LevelFilter::Debug, -- _ => simplelog::LevelFilter::Trace, -- }; -- let mut lcfg = simplelog::ConfigBuilder::new(); -- lcfg.set_time_level(simplelog::LevelFilter::Error) -- .set_location_level(simplelog::LevelFilter::Off) -- .set_target_level(simplelog::LevelFilter::Off) -- .set_thread_level(simplelog::LevelFilter::Off); -- simplelog::TermLogger::init( -- llv, -- lcfg.build(), -- simplelog::TerminalMode::Stderr, -- simplelog::ColorChoice::Auto, -- )?; -- -- let shutdown = Arc::new(AtomicBool::new(false)); -- let shutdown_clone = shutdown.clone(); -- ctrlc::set_handler(move || { -- shutdown_clone.store(true, Ordering::Relaxed); -- }) -- .context("Error setting Ctrl-C handler")?; -- -- let mut sched = Scheduler::init(&opts)?; -- -- while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 { -- std::thread::sleep(Duration::from_secs_f64(opts.interval)); -- sched.step()?; -- } -- -- sched.report_bpf_exit_type() --} -diff --git b/tools/sched_ext/gnu/stubs.h a/tools/sched_ext/gnu/stubs.h -deleted file mode 100644 -index 719225b16626..000000000000 ---- b/tools/sched_ext/gnu/stubs.h -+++ /dev/null -@@ -1 +0,0 @@ --/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ -diff --git b/tools/sched_ext/scx_common.bpf.h a/tools/sched_ext/scx_common.bpf.h -deleted file mode 100644 -index e56de9dc86f2..000000000000 ---- b/tools/sched_ext/scx_common.bpf.h -+++ /dev/null -@@ -1,288 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __SCHED_EXT_COMMON_BPF_H --#define __SCHED_EXT_COMMON_BPF_H -- --#include "vmlinux.h" --#include --#include --#include --#include "user_exit_info.h" -- --#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ --#define PF_EXITING 0x00000004 --#define CLOCK_MONOTONIC 1 -- --/* -- * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can -- * lead to really confusing misbehaviors. Let's trigger a build failure. -- */ --static inline void ___vmlinux_h_sanity_check___(void) --{ -- _Static_assert(SCX_DSQ_FLAG_BUILTIN, -- "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); --} -- --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; -- --static inline __attribute__((format(printf, 1, 2))) --void ___scx_bpf_error_format_checker(const char *fmt, ...) {} -- --/* -- * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments -- * instead of an array of u64. Note that __param[] must have at least one -- * element to keep the verifier happy. -- */ --#define scx_bpf_error(fmt, args...) \ --({ \ -- static char ___fmt[] = fmt; \ -- unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ -- \ -- _Pragma("GCC diagnostic push") \ -- _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ -- ___bpf_fill(___param, args); \ -- _Pragma("GCC diagnostic pop") \ -- \ -- scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ -- \ -- ___scx_bpf_error_format_checker(fmt, ##args); \ --}) -- --void scx_bpf_switch_all(void) __ksym; --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; --bool scx_bpf_consume(u64 dsq_id) __ksym; --u32 scx_bpf_dispatch_nr_slots(void) __ksym; --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym; --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym; --void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; --s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym; --const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; --const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; --void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; --void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; --bool scx_bpf_task_running(const struct task_struct *p) __ksym; --s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym; --u32 scx_bpf_reenqueue_local(void) __ksym; -- --#define BPF_STRUCT_OPS(name, args...) \ --SEC("struct_ops/"#name) \ --BPF_PROG(name, ##args) -- --#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ --SEC("struct_ops.s/"#name) \ --BPF_PROG(name, ##args) -- --/** -- * MEMBER_VPTR - Obtain the verified pointer to a struct or array member -- * @base: struct or array to index -- * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...) -- * -- * The verifier often gets confused by the instruction sequence the compiler -- * generates for indexing struct fields or arrays. This macro forces the -- * compiler to generate a code sequence which first calculates the byte offset, -- * checks it against the struct or array size and add that byte offset to -- * generate the pointer to the member to help the verifier. -- * -- * Ideally, we want to abort if the calculated offset is out-of-bounds. However, -- * BPF currently doesn't support abort, so evaluate to NULL instead. The caller -- * must check for NULL and take appropriate action to appease the verifier. To -- * avoid confusing the verifier, it's best to check for NULL and dereference -- * immediately. -- * -- * vptr = MEMBER_VPTR(my_array, [i][j]); -- * if (!vptr) -- * return error; -- * *vptr = new_value; -- */ --#define MEMBER_VPTR(base, member) (typeof(base member) *)({ \ -- u64 __base = (u64)base; \ -- u64 __addr = (u64)&(base member) - __base; \ -- asm volatile ( \ -- "if %0 <= %[max] goto +2\n" \ -- "%0 = 0\n" \ -- "goto +1\n" \ -- "%0 += %1\n" \ -- : "+r"(__addr) \ -- : "r"(__base), \ -- [max]"i"(sizeof(base) - sizeof(base member))); \ -- __addr; \ --}) -- --/* -- * BPF core and other generic helpers -- */ -- --/* list and rbtree */ --#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) --#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) -- --void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; --void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; -- --#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) --#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) -- --void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; --struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; --struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, -- struct bpf_rb_node *node) __ksym; --void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, -- bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym; --struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; -- --/* task */ --struct task_struct *bpf_task_from_pid(s32 pid) __ksym; --struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; --void bpf_task_release(struct task_struct *p) __ksym; -- --/* cgroup */ --struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; --void bpf_cgroup_release(struct cgroup *cgrp) __ksym; --struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; -- --/* cpumask */ --struct bpf_cpumask *bpf_cpumask_create(void) __ksym; --struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; --u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; --u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym; -- --/* rcu */ --void bpf_rcu_read_lock(void) __ksym; --void bpf_rcu_read_unlock(void) __ksym; -- --/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */ --struct bpf_iter_num; -- --extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym; --extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym; --extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym; -- --#ifndef bpf_for_each --/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for -- * using BPF open-coded iterators without having to write mundane explicit -- * low-level loop logic. Instead, it provides for()-like generic construct -- * that can be used pretty naturally. E.g., for some hypothetical cgroup -- * iterator, you'd write: -- * -- * struct cgroup *cg, *parent_cg = <...>; -- * -- * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { -- * bpf_printk("Child cgroup id = %d", cg->cgroup_id); -- * if (cg->cgroup_id == 123) -- * break; -- * } -- * -- * I.e., it looks almost like high-level for each loop in other languages, -- * supports continue/break, and is verifiable by BPF verifier. -- * -- * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) -- * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to -- * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int -- * *`, not just `int`. So for integers bpf_for() is more convenient. -- * -- * Note: this macro relies on C99 feature of allowing to declare variables -- * inside for() loop, bound to for() loop lifetime. It also utilizes GCC -- * extension: __attribute__((cleanup())), supported by both GCC and -- * Clang. -- */ --#define bpf_for_each(type, cur, args...) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ -- cleanup(bpf_iter_##type##_destroy))), \ -- /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_##type##_new(&___it, ##args), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_##type##_destroy, (void *)0); \ -- /* iteration and termination check */ \ -- (((cur) = bpf_iter_##type##_next(&___it))); \ --) --#endif /* bpf_for_each */ -- --#ifndef bpf_for --/* bpf_for(i, start, end) implements a for()-like looping construct that sets -- * provided integer variable *i* to values starting from *start* through, -- * but not including, *end*. It also proves to BPF verifier that *i* belongs -- * to range [start, end), so this can be used for accessing arrays without -- * extra checks. -- * -- * Note: *start* and *end* are assumed to be expressions with no side effects -- * and whose values do not change throughout bpf_for() loop execution. They do -- * not have to be statically known or constant, though. -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_for(i, start, end) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, (start), (end)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- ({ \ -- /* iteration step */ \ -- int *___t = bpf_iter_num_next(&___it); \ -- /* termination and bounds check */ \ -- (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ -- }); \ --) --#endif /* bpf_for */ -- --#ifndef bpf_repeat --/* bpf_repeat(N) performs N iterations without exposing iteration number -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_repeat(N) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, 0, (N)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- bpf_iter_num_next(&___it); \ -- /* nothing here */ \ --) --#endif /* bpf_repeat */ -- --#endif /* __SCHED_EXT_COMMON_BPF_H */ -diff --git b/tools/sched_ext/scx_example_central.bpf.c a/tools/sched_ext/scx_example_central.bpf.c -deleted file mode 100644 -index 4cec04b4c2ed..000000000000 ---- b/tools/sched_ext/scx_example_central.bpf.c -+++ /dev/null -@@ -1,334 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A central FIFO sched_ext scheduler which demonstrates the followings: -- * -- * a. Making all scheduling decisions from one CPU: -- * -- * The central CPU is the only one making scheduling decisions. All other -- * CPUs kick the central CPU when they run out of tasks to run. -- * -- * There is one global BPF queue and the central CPU schedules all CPUs by -- * dispatching from the global queue to each CPU's local dsq from dispatch(). -- * This isn't the most straightforward. e.g. It'd be easier to bounce -- * through per-CPU BPF queues. The current design is chosen to maximally -- * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching. -- * -- * b. Tickless operation -- * -- * All tasks are dispatched with the infinite slice which allows stopping the -- * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full -- * parameter. The tickless operation can be observed through -- * /proc/interrupts. -- * -- * Periodic switching is enforced by a periodic timer checking all CPUs and -- * preempting them as necessary. Unfortunately, BPF timer currently doesn't -- * have a way to pin to a specific CPU, so the periodic timer isn't pinned to -- * the central CPU. -- * -- * c. Preemption -- * -- * Kthreads are unconditionally queued to the head of a matching local dsq -- * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always -- * prioritized over user threads, which is required for ensuring forward -- * progress as e.g. the periodic timer may run on a ksoftirqd and if the -- * ksoftirqd gets starved by a user thread, there may not be anything else to -- * vacate that user thread. -- * -- * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the -- * next tasks. -- * -- * This scheduler is designed to maximize usage of various SCX mechanisms. A -- * more practical implementation would likely put the scheduling loop outside -- * the central CPU's dispatch() path and add some form of priority mechanism. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --enum { -- FALLBACK_DSQ_ID = 0, -- MAX_CPUS = 4096, -- MS_TO_NS = 1000LLU * 1000, -- TIMER_INTERVAL_NS = 1 * MS_TO_NS, --}; -- --const volatile bool switch_partial; --const volatile s32 central_cpu; --const volatile u32 nr_cpu_ids = 64; /* !0 for veristat, set during init */ -- --u64 nr_total, nr_locals, nr_queued, nr_lost_pids; --u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries; --u64 nr_overflows; -- --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, s32); --} central_q SEC(".maps"); -- --/* can't use percpu map due to bad lookups */ --static bool cpu_gimme_task[MAX_CPUS]; --static u64 cpu_started_at[MAX_CPUS]; -- --struct central_timer { -- struct bpf_timer timer; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, 1); -- __type(key, u32); -- __type(value, struct central_timer); --} central_timer SEC(".maps"); -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- /* -- * Steer wakeups to the central CPU as much as possible to avoid -- * disturbing other CPUs. It's safe to blindly return the central cpu as -- * select_cpu() is a hint and if @p can't be on it, the kernel will -- * automatically pick a fallback CPU. -- */ -- return central_cpu; --} -- --void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags) --{ -- s32 pid = p->pid; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- /* -- * Push per-cpu kthreads at the head of local dsq's and preempt the -- * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked -- * behind other threads which is necessary for forward progress -- * guarantee as we depend on the BPF timer which may run from ksoftirqd. -- */ -- if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- __sync_fetch_and_add(&nr_locals, 1); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF, -- enq_flags | SCX_ENQ_PREEMPT); -- return; -- } -- -- if (bpf_map_push_elem(¢ral_q, &pid, 0)) { -- __sync_fetch_and_add(&nr_overflows, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_queued, 1); -- -- if (!scx_bpf_task_running(p)) -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); --} -- --static bool dispatch_to_cpu(s32 cpu) --{ -- struct task_struct *p; -- s32 pid; -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (bpf_map_pop_elem(¢ral_q, &pid)) -- break; -- -- __sync_fetch_and_sub(&nr_queued, 1); -- -- p = bpf_task_from_pid(pid); -- if (!p) { -- __sync_fetch_and_add(&nr_lost_pids, 1); -- continue; -- } -- -- /* -- * If we can't run the task at the top, do the dumb thing and -- * bounce it to the fallback dsq. -- */ -- if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- __sync_fetch_and_add(&nr_mismatches, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0); -- bpf_task_release(p); -- continue; -- } -- -- /* dispatch to local and mark that @cpu doesn't need more */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0); -- -- if (cpu != central_cpu) -- scx_bpf_kick_cpu(cpu, 0); -- -- bpf_task_release(p); -- return true; -- } -- -- return false; --} -- --void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (cpu == central_cpu) { -- /* dispatch for all other CPUs first */ -- __sync_fetch_and_add(&nr_dispatches, 1); -- -- bpf_for(cpu, 0, nr_cpu_ids) { -- bool *gimme; -- -- if (!scx_bpf_dispatch_nr_slots()) -- break; -- -- /* central's gimme is never set */ -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme && !*gimme) -- continue; -- -- if (dispatch_to_cpu(cpu)) -- *gimme = false; -- } -- -- /* -- * Retry if we ran out of dispatch buffer slots as we might have -- * skipped some CPUs and also need to dispatch for self. The ext -- * core automatically retries if the local dsq is empty but we -- * can't rely on that as we're dispatching for other CPUs too. -- * Kick self explicitly to retry. -- */ -- if (!scx_bpf_dispatch_nr_slots()) { -- __sync_fetch_and_add(&nr_retries, 1); -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- return; -- } -- -- /* look for a task to run on the central CPU */ -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- dispatch_to_cpu(central_cpu); -- } else { -- bool *gimme; -- -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme) -- *gimme = true; -- -- /* -- * Force dispatch on the scheduling CPU so that it finds a task -- * to run for us. -- */ -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- } --} -- --void BPF_STRUCT_OPS(central_running, struct task_struct *p) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ --} -- --void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = 0; --} -- --static int central_timerfn(void *map, int *key, struct bpf_timer *timer) --{ -- u64 now = bpf_ktime_get_ns(); -- u64 nr_to_kick = nr_queued; -- s32 i; -- -- bpf_for(i, 0, nr_cpu_ids) { -- s32 cpu = (nr_timers + i) % nr_cpu_ids; -- u64 *started_at; -- -- if (cpu == central_cpu) -- continue; -- -- /* kick iff the current one exhausted its slice */ -- started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at && *started_at && -- vtime_before(now, *started_at + SCX_SLICE_DFL)) -- continue; -- -- /* and there's something pending */ -- if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) || -- scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu)) -- ; -- else if (nr_to_kick) -- nr_to_kick--; -- else -- continue; -- -- scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); -- } -- -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- -- bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- __sync_fetch_and_add(&nr_timers, 1); -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(central_init) --{ -- u32 key = 0; -- struct bpf_timer *timer; -- int ret; -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1); -- if (ret) -- return ret; -- -- timer = bpf_map_lookup_elem(¢ral_timer, &key); -- if (!timer) -- return -ESRCH; -- -- bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); -- bpf_timer_set_callback(timer, central_timerfn); -- ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- return ret; --} -- --void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops central_ops = { -- /* -- * We are offloading all scheduling decisions to the central CPU and -- * thus being the last task on a given CPU doesn't mean anything -- * special. Enqueue the last tasks like any other tasks. -- */ -- .flags = SCX_OPS_ENQ_LAST, -- -- .select_cpu = (void *)central_select_cpu, -- .enqueue = (void *)central_enqueue, -- .dispatch = (void *)central_dispatch, -- .running = (void *)central_running, -- .stopping = (void *)central_stopping, -- .init = (void *)central_init, -- .exit = (void *)central_exit, -- .name = "central", --}; -diff --git b/tools/sched_ext/scx_example_central.c a/tools/sched_ext/scx_example_central.c -deleted file mode 100644 -index 7ad591cbdc65..000000000000 ---- b/tools/sched_ext/scx_example_central.c -+++ /dev/null -@@ -1,94 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_central.skel.h" -- --const char help_fmt[] = --"A central FIFO sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-c CPU] [-p]\n" --"\n" --" -c CPU Override the central CPU (default: 0)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_central *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_central__open(); -- assert(skel); -- -- skel->rodata->central_cpu = 0; -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "c:ph")) != -1) { -- switch (opt) { -- case 'c': -- skel->rodata->central_cpu = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_central__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.central_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf("total :%10lu local:%10lu queued:%10lu lost:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_locals, -- skel->bss->nr_queued, -- skel->bss->nr_lost_pids); -- printf("timer :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n", -- skel->bss->nr_timers, -- skel->bss->nr_dispatches, -- skel->bss->nr_mismatches, -- skel->bss->nr_retries); -- printf("overflow:%10lu\n", -- skel->bss->nr_overflows); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_central__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_flatcg.bpf.c a/tools/sched_ext/scx_example_flatcg.bpf.c -deleted file mode 100644 -index f6078b9a681f..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.bpf.c -+++ /dev/null -@@ -1,872 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext flattened cgroup hierarchy scheduler. It implements -- * hierarchical weight-based cgroup CPU control by flattening the cgroup -- * hierarchy into a single layer by compounding the active weight share at each -- * level. Consider the following hierarchy with weights in parentheses: -- * -- * R + A (100) + B (100) -- * | \ C (100) -- * \ D (200) -- * -- * Ignoring the root and threaded cgroups, only B, C and D can contain tasks. -- * Let's say all three have runnable tasks. The total share that each of these -- * three cgroups is entitled to can be calculated by compounding its share at -- * each level. -- * -- * For example, B is competing against C and in that competition its share is -- * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's -- * share in that competition is 200/(200+100) == 1/3. B's eventual share in the -- * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's -- * eventual shaer is the same at 1/6. D is only competing at the top level and -- * its share is 200/(100+200) == 2/3. -- * -- * So, instead of hierarchically scheduling level-by-level, we can consider it -- * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3 -- * and keep updating the eventual shares as the cgroups' runnable states change. -- * -- * This flattening of hierarchy can bring a substantial performance gain when -- * the cgroup hierarchy is nested multiple levels. in a simple benchmark using -- * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it -- * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two -- * apache instances competing with 2:1 weight ratio nested four level deep. -- * -- * However, the gain comes at the cost of not being able to properly handle -- * thundering herd of cgroups. For example, if many cgroups which are nested -- * behind a low priority parent cgroup wake up around the same time, they may be -- * able to consume more CPU cycles than they are entitled to. In many use cases, -- * this isn't a real concern especially given the performance gain. Also, there -- * are ways to mitigate the problem further by e.g. introducing an extra -- * scheduling layer on cgroup delegation boundaries. -- * -- * The scheduler first picks the cgroup to run and then schedule the tasks -- * within by using nested weighted vtime scheduling by default. The -- * cgroup-internal scheduling can be switched to FIFO with the -f option. -- */ --#include "scx_common.bpf.h" --#include "user_exit_info.h" --#include "scx_example_flatcg.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */ --const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL; --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --u64 cvtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, u64); -- __uint(max_entries, FCG_NR_STATS); --} stats SEC(".maps"); -- --static void stat_inc(enum fcg_stat_idx idx) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p)++; --} -- --struct fcg_cpu_ctx { -- u64 cur_cgid; -- u64 cur_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, struct fcg_cpu_ctx); -- __uint(max_entries, 1); --} cpu_ctx SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_cgrp_ctx); --} cgrp_ctx SEC(".maps"); -- --struct cgv_node { -- struct bpf_rb_node rb_node; -- __u64 cvtime; -- __u64 cgid; --}; -- --private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock; --private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node); -- --struct cgv_node_stash { -- struct cgv_node __kptr *node; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, 16384); -- __type(key, __u64); -- __type(value, struct cgv_node_stash); --} cgv_node_stash SEC(".maps"); -- --struct fcg_task_ctx { -- u64 bypassed_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_task_ctx); --} task_ctx SEC(".maps"); -- --/* gets inc'd on weight tree changes to expire the cached hweights */ --unsigned long hweight_gen = 1; -- --static u64 div_round_up(u64 dividend, u64 divisor) --{ -- return (dividend + divisor - 1) / divisor; --} -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) --{ -- struct cgv_node *cgc_a, *cgc_b; -- -- cgc_a = container_of(a, struct cgv_node, rb_node); -- cgc_b = container_of(b, struct cgv_node, rb_node); -- -- return cgc_a->cvtime < cgc_b->cvtime; --} -- --static struct fcg_cpu_ctx *find_cpu_ctx(void) --{ -- struct fcg_cpu_ctx *cpuc; -- u32 idx = 0; -- -- cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx); -- if (!cpuc) { -- scx_bpf_error("cpu_ctx lookup failed"); -- return NULL; -- } -- return cpuc; --} -- --static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id); -- return NULL; -- } -- return cgc; --} -- --static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgrp = bpf_cgroup_ancestor(cgrp, level); -- if (!cgrp) { -- scx_bpf_error("ancestor cgroup lookup failed"); -- return NULL; -- } -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- scx_bpf_error("ancestor cgrp_ctx lookup failed"); -- bpf_cgroup_release(cgrp); -- return cgc; --} -- --static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- int level; -- -- if (!cgc->nr_active) { -- stat_inc(FCG_STAT_HWT_SKIP); -- return; -- } -- -- if (cgc->hweight_gen == hweight_gen) { -- stat_inc(FCG_STAT_HWT_CACHE); -- return; -- } -- -- stat_inc(FCG_STAT_HWT_UPDATES); -- bpf_for(level, 0, cgrp->level + 1) { -- struct fcg_cgrp_ctx *cgc; -- bool is_active; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- -- if (!level) { -- cgc->hweight = FCG_HWEIGHT_ONE; -- cgc->hweight_gen = hweight_gen; -- } else { -- struct fcg_cgrp_ctx *pcgc; -- -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- -- /* -- * We can be oppotunistic here and not grab the -- * cgv_tree_lock and deal with the occasional races. -- * However, hweight updates are already cached and -- * relatively low-frequency. Let's just do the -- * straightforward thing. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- is_active = cgc->nr_active; -- if (is_active) { -- cgc->hweight_gen = pcgc->hweight_gen; -- cgc->hweight = -- div_round_up(pcgc->hweight * cgc->weight, -- pcgc->child_weight_sum); -- } -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!is_active) { -- stat_inc(FCG_STAT_HWT_RACE); -- break; -- } -- } -- } --} -- --static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) --{ -- u64 delta, cvtime, max_budget; -- -- /* -- * A node which is on the rbtree can't be pointed to from elsewhere yet -- * and thus can't be updated and repositioned. Instead, we collect the -- * vtime deltas separately and apply it asynchronously here. -- */ -- delta = cgc->cvtime_delta; -- __sync_fetch_and_sub(&cgc->cvtime_delta, delta); -- cvtime = cgv_node->cvtime + delta; -- -- /* -- * Allow a cgroup to carry the maximum budget proportional to its -- * hweight such that a full-hweight cgroup can immediately take up half -- * of the CPUs at the most while staying at the front of the rbtree. -- */ -- max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / -- (2 * FCG_HWEIGHT_ONE); -- if (vtime_before(cvtime, cvtime_now - max_budget)) -- cvtime = cvtime_now - max_budget; -- -- cgv_node->cvtime = cvtime; --} -- --static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- u64 cgid = cgrp->kn->id; -- -- /* paired with cmpxchg in try_pick_next_cgroup() */ -- if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { -- stat_inc(FCG_STAT_ENQ_SKIP); -- return; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid); -- return; -- } -- -- /* NULL if the node is already on the rbtree */ -- cgv_node = bpf_kptr_xchg(&stash->node, NULL); -- if (!cgv_node) { -- stat_inc(FCG_STAT_ENQ_RACE); -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); --} -- --void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * If select_cpu_dfl() is recommending local enqueue, the target CPU is -- * idle. Follow it and charge the cgroup later in fcg_stopping() after -- * the fact. Use the same mechanism to deal with tasks with custom -- * affinities so that we don't have to worry about per-cgroup dq's -- * containing tasks that can't be executed from some CPUs. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) { -- /* -- * Tell fcg_stopping() that this bypassed the regular scheduling -- * path and should be force charged to the cgroup. 0 is used to -- * indicate that the task isn't bypassing, so if the current -- * runtime is 0, go back by one nanosecond. -- */ -- taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1; -- -- /* -- * The global dq is deprioritized as we don't want to let tasks -- * to boost themselves by constraining its cpumask. The -- * deprioritization is rather severe, so let's not apply that to -- * per-cpu kernel threads. This is ham-fisted. We probably wanna -- * implement per-cgroup fallback dq's instead so that we have -- * more control over when tasks with custom cpumask get issued. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || -- (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) { -- stat_inc(FCG_STAT_LOCAL); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- } else { -- stat_inc(FCG_STAT_GLOBAL); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } -- return; -- } -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- goto out_release; -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 tvtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) -- tvtime = cgc->tvtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, -- tvtime, enq_flags); -- } -- -- cgrp_enqueued(cgrp, cgc); --out_release: -- bpf_cgroup_release(cgrp); --} -- --/* -- * Walk the cgroup tree to update the active weight sums as tasks wake up and -- * sleep. The weight sums are used as the base when calculating the proportion a -- * given cgroup or task is entitled to at each level. -- */ --static void update_active_weight_sums(struct cgroup *cgrp, bool runnable) --{ -- struct fcg_cgrp_ctx *cgc; -- bool updated = false; -- int idx; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- /* -- * In most cases, a hot cgroup would have multiple threads going to -- * sleep and waking up while the whole cgroup stays active. In leaf -- * cgroups, ->nr_runnable which is updated with __sync operations gates -- * ->nr_active updates, so that we don't have to grab the cgv_tree_lock -- * repeatedly for a busy cgroup which is staying active. -- */ -- if (runnable) { -- if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_ACT); -- } else { -- if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_DEACT); -- } -- -- /* -- * If @cgrp is becoming runnable, its hweight should be refreshed after -- * it's added to the weight tree so that enqueue has the up-to-date -- * value. If @cgrp is becoming quiescent, the hweight should be -- * refreshed before it's removed from the weight tree so that the usage -- * charging which happens afterwards has access to the latest value. -- */ -- if (!runnable) -- cgrp_refresh_hweight(cgrp, cgc); -- -- /* propagate upwards */ -- bpf_for(idx, 0, cgrp->level) { -- int level = cgrp->level - idx; -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- bool propagate = false; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- if (level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- } -- -- /* -- * We need the propagation protected by a lock to synchronize -- * against weight changes. There's no reason to drop the lock at -- * each level but bpf_spin_lock() doesn't want any function -- * calls while locked. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- -- if (runnable) { -- if (!cgc->nr_active++) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum += cgc->weight; -- } -- } -- } else { -- if (!--cgc->nr_active) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum -= cgc->weight; -- } -- } -- } -- -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!propagate) -- break; -- } -- -- if (updated) -- __sync_fetch_and_add(&hweight_gen, 1); -- -- if (runnable) -- cgrp_refresh_hweight(cgrp, cgc); --} -- --void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, true); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) --{ -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- if (fifo_sched) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- /* -- * @cgc->tvtime_now always progresses forward as tasks start -- * executing. The test and update can be performed concurrently -- * from multiple CPUs and thus racy. Any error should be -- * contained and temporary. Let's just live with it. -- */ -- if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) -- cgc->tvtime_now = p->scx.dsq_vtime; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- /* scale the execution time by the inverse of the weight and charge */ -- if (!fifo_sched) -- p->scx.dsq_vtime += -- (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- if (!taskc->bypassed_at) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- __sync_fetch_and_add(&cgc->cvtime_delta, -- p->se.sum_exec_runtime - taskc->bypassed_at); -- taskc->bypassed_at = 0; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, false); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) --{ -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- if (cgrp->level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1); -- if (!pcgc) -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- if (pcgc && cgc->nr_active) -- pcgc->child_weight_sum += (s64)weight - cgc->weight; -- cgc->weight = weight; -- bpf_spin_unlock(&cgv_tree_lock); --} -- --static bool try_pick_next_cgroup(u64 *cgidp) --{ -- struct bpf_rb_node *rb_node; -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 cgid; -- -- /* pop the front cgroup and wind cvtime_now accordingly */ -- bpf_spin_lock(&cgv_tree_lock); -- -- rb_node = bpf_rbtree_first(&cgv_tree); -- if (!rb_node) { -- bpf_spin_unlock(&cgv_tree_lock); -- stat_inc(FCG_STAT_PNC_NO_CGRP); -- *cgidp = 0; -- return true; -- } -- -- rb_node = bpf_rbtree_remove(&cgv_tree, rb_node); -- bpf_spin_unlock(&cgv_tree_lock); -- -- cgv_node = container_of(rb_node, struct cgv_node, rb_node); -- cgid = cgv_node->cgid; -- -- if (vtime_before(cvtime_now, cgv_node->cvtime)) -- cvtime_now = cgv_node->cvtime; -- -- /* -- * If lookup fails, the cgroup's gone. Free and move on. See -- * fcg_cgroup_exit(). -- */ -- cgrp = bpf_cgroup_from_id(cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- if (!scx_bpf_consume(cgid)) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_EMPTY); -- goto out_stash; -- } -- -- /* -- * Successfully consumed from the cgroup. This will be our current -- * cgroup for the new slice. Refresh its hweight. -- */ -- cgrp_refresh_hweight(cgrp, cgc); -- -- bpf_cgroup_release(cgrp); -- -- /* -- * As the cgroup may have more tasks, add it back to the rbtree. Note -- * that here we charge the full slice upfront and then exact later -- * according to the actual consumption. This prevents lowpri thundering -- * herd from saturating the machine. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- -- *cgidp = cgid; -- stat_inc(FCG_STAT_PNC_NEXT); -- return true; -- --out_stash: -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- /* -- * Paired with cmpxchg in cgrp_enqueued(). If they see the following -- * transition, they'll enqueue the cgroup. If they are earlier, we'll -- * see their task in the dq below and requeue the cgroup. -- */ -- __sync_val_compare_and_swap(&cgc->queued, 1, 0); -- -- if (scx_bpf_dsq_nr_queued(cgid)) { -- bpf_spin_lock(&cgv_tree_lock); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- goto out_free; -- } -- } -- -- return false; -- --out_free: -- bpf_obj_drop(cgv_node); -- return false; --} -- --void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) --{ -- struct fcg_cpu_ctx *cpuc; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 now = bpf_ktime_get_ns(); -- -- cpuc = find_cpu_ctx(); -- if (!cpuc) -- return; -- -- if (!cpuc->cur_cgid) -- goto pick_next_cgroup; -- -- if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { -- if (scx_bpf_consume(cpuc->cur_cgid)) { -- stat_inc(FCG_STAT_CNS_KEEP); -- return; -- } -- stat_inc(FCG_STAT_CNS_EMPTY); -- } else { -- stat_inc(FCG_STAT_CNS_EXPIRE); -- } -- -- /* -- * The current cgroup is expiring. It was already charged a full slice. -- * Calculate the actual usage and accumulate the delta. -- */ -- cgrp = bpf_cgroup_from_id(cpuc->cur_cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_CNS_GONE); -- goto pick_next_cgroup; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (cgc) { -- /* -- * We want to update the vtime delta and then look for the next -- * cgroup to execute but the latter needs to be done in a loop -- * and we can't keep the lock held. Oh well... -- */ -- bpf_spin_lock(&cgv_tree_lock); -- __sync_fetch_and_add(&cgc->cvtime_delta, -- (cpuc->cur_at + cgrp_slice_ns - now) * -- FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- stat_inc(FCG_STAT_CNS_GONE); -- } -- -- bpf_cgroup_release(cgrp); -- --pick_next_cgroup: -- cpuc->cur_at = now; -- -- if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { -- cpuc->cur_cgid = 0; -- return; -- } -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_pick_next_cgroup(&cpuc->cur_cgid)) -- break; -- } --} -- --s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct fcg_task_ctx *taskc; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- taskc = bpf_task_storage_get(&task_ctx, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!taskc) -- return -ENOMEM; -- -- taskc->bypassed_at = 0; -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp, -- struct scx_cgroup_init_args *args) --{ -- struct fcg_cgrp_ctx *cgc; -- struct cgv_node *cgv_node; -- struct cgv_node_stash empty_stash = {}, *stash; -- u64 cgid = cgrp->kn->id; -- int ret; -- -- /* -- * Technically incorrect as cgroup ID is full 64bit while dq ID is -- * 63bit. Should not be a problem in practice and easy to spot in the -- * unlikely case that it breaks. -- */ -- ret = scx_bpf_create_dsq(cgid, -1); -- if (ret) -- return ret; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!cgc) { -- ret = -ENOMEM; -- goto err_destroy_dsq; -- } -- -- cgc->weight = args->weight; -- cgc->hweight = FCG_HWEIGHT_ONE; -- -- ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash, -- BPF_NOEXIST); -- if (ret) { -- if (ret != -ENOMEM) -- scx_bpf_error("unexpected stash creation error (%d)", -- ret); -- goto err_destroy_dsq; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("unexpected cgv_node stash lookup failure"); -- ret = -ENOENT; -- goto err_destroy_dsq; -- } -- -- cgv_node = bpf_obj_new(struct cgv_node); -- if (!cgv_node) { -- ret = -ENOMEM; -- goto err_del_cgv_node; -- } -- -- cgv_node->cgid = cgid; -- cgv_node->cvtime = cvtime_now; -- -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- ret = -EBUSY; -- goto err_drop; -- } -- -- return 0; -- --err_drop: -- bpf_obj_drop(cgv_node); --err_del_cgv_node: -- bpf_map_delete_elem(&cgv_node_stash, &cgid); --err_destroy_dsq: -- scx_bpf_destroy_dsq(cgid); -- return ret; --} -- --void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- -- /* -- * For now, there's no way find and remove the cgv_node if it's on the -- * cgv_tree. Let's drain them in the dispatch path as they get popped -- * off the front of the tree. -- */ -- bpf_map_delete_elem(&cgv_node_stash, &cgid); -- scx_bpf_destroy_dsq(cgid); --} -- --s32 BPF_STRUCT_OPS(fcg_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops flatcg_ops = { -- .enqueue = (void *)fcg_enqueue, -- .dispatch = (void *)fcg_dispatch, -- .runnable = (void *)fcg_runnable, -- .running = (void *)fcg_running, -- .stopping = (void *)fcg_stopping, -- .quiescent = (void *)fcg_quiescent, -- .prep_enable = (void *)fcg_prep_enable, -- .cgroup_set_weight = (void *)fcg_cgroup_set_weight, -- .cgroup_init = (void *)fcg_cgroup_init, -- .cgroup_exit = (void *)fcg_cgroup_exit, -- .init = (void *)fcg_init, -- .exit = (void *)fcg_exit, -- .flags = SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING, -- .name = "flatcg", --}; -diff --git b/tools/sched_ext/scx_example_flatcg.c a/tools/sched_ext/scx_example_flatcg.c -deleted file mode 100644 -index f9c8a5b84a70..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.c -+++ /dev/null -@@ -1,232 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2023 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2023 Tejun Heo -- * Copyright (c) 2023 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_flatcg.h" --#include "scx_example_flatcg.skel.h" -- --#ifndef FILEID_KERNFS --#define FILEID_KERNFS 0xfe --#endif -- --const char help_fmt[] = --"A flattened cgroup hierarchy sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -i INTERVAL Report interval\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --static float read_cpu_util(__u64 *last_sum, __u64 *last_idle) --{ -- FILE *fp; -- char buf[4096]; -- char *line, *cur = NULL, *tok; -- __u64 sum = 0, idle = 0; -- __u64 delta_sum, delta_idle; -- int idx; -- -- fp = fopen("/proc/stat", "r"); -- if (!fp) { -- perror("fopen(\"/proc/stat\")"); -- return 0.0; -- } -- -- if (!fgets(buf, sizeof(buf), fp)) { -- perror("fgets(\"/proc/stat\")"); -- fclose(fp); -- return 0.0; -- } -- fclose(fp); -- -- line = buf; -- for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) { -- char *endp = NULL; -- __u64 v; -- -- if (idx == 0) { -- line = NULL; -- continue; -- } -- v = strtoull(tok, &endp, 0); -- if (!endp || *endp != '\0') { -- fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n", -- idx, tok); -- continue; -- } -- sum += v; -- if (idx == 4) -- idle = v; -- } -- -- delta_sum = sum - *last_sum; -- delta_idle = idle - *last_idle; -- *last_sum = sum; -- *last_idle = idle; -- -- return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0; --} -- --static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats) --{ -- __u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus]; -- __u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); -- -- for (idx = 0; idx < FCG_NR_STATS; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_flatcg *skel; -- struct bpf_link *link; -- struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 }; -- bool dump_cgrps = false; -- __u64 last_cpu_sum = 0, last_cpu_idle = 0; -- __u64 last_stats[FCG_NR_STATS] = {}; -- unsigned long seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_flatcg__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open: %s\n", strerror(errno)); -- return 1; -- } -- -- skel->rodata->nr_cpus = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) { -- double v; -- -- switch (opt) { -- case 's': -- v = strtod(optarg, NULL); -- skel->rodata->cgrp_slice_ns = v * 1000; -- break; -- case 'i': -- v = strtod(optarg, NULL); -- intv_ts.tv_sec = v; -- intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000; -- break; -- case 'd': -- dump_cgrps = true; -- break; -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- case 'h': -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d", -- (double)skel->rodata->cgrp_slice_ns / 1000000.0, -- (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0, -- dump_cgrps); -- -- if (scx_example_flatcg__load(skel)) { -- fprintf(stderr, "Failed to load: %s\n", strerror(errno)); -- return 1; -- } -- -- link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops); -- if (!link) { -- fprintf(stderr, "Failed to attach_struct_ops: %s\n", -- strerror(errno)); -- return 1; -- } -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- __u64 acc_stats[FCG_NR_STATS]; -- __u64 stats[FCG_NR_STATS]; -- float cpu_util; -- int i; -- -- cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle); -- -- fcg_read_stats(skel, acc_stats); -- for (i = 0; i < FCG_NR_STATS; i++) -- stats[i] = acc_stats[i] - last_stats[i]; -- -- memcpy(last_stats, acc_stats, sizeof(acc_stats)); -- -- printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n", -- seq++, cpu_util * 100.0, skel->data->hweight_gen); -- printf(" act:%6llu deact:%6llu local:%6llu global:%6llu\n", -- stats[FCG_STAT_ACT], -- stats[FCG_STAT_DEACT], -- stats[FCG_STAT_LOCAL], -- stats[FCG_STAT_GLOBAL]); -- printf("HWT skip:%6llu race:%6llu cache:%6llu update:%6llu\n", -- stats[FCG_STAT_HWT_SKIP], -- stats[FCG_STAT_HWT_RACE], -- stats[FCG_STAT_HWT_CACHE], -- stats[FCG_STAT_HWT_UPDATES]); -- printf("ENQ skip:%6llu race:%6llu\n", -- stats[FCG_STAT_ENQ_SKIP], -- stats[FCG_STAT_ENQ_RACE]); -- printf("CNS keep:%6llu expire:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_CNS_KEEP], -- stats[FCG_STAT_CNS_EXPIRE], -- stats[FCG_STAT_CNS_EMPTY], -- stats[FCG_STAT_CNS_GONE]); -- printf("PNC nocgrp:%6llu next:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_PNC_NO_CGRP], -- stats[FCG_STAT_PNC_NEXT], -- stats[FCG_STAT_PNC_EMPTY], -- stats[FCG_STAT_PNC_GONE]); -- printf("BAD remove:%6llu\n", -- acc_stats[FCG_STAT_BAD_REMOVAL]); -- -- nanosleep(&intv_ts, NULL); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_flatcg__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_flatcg.h a/tools/sched_ext/scx_example_flatcg.h -deleted file mode 100644 -index 490758ed41f0..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.h -+++ /dev/null -@@ -1,49 +0,0 @@ --#ifndef __SCX_EXAMPLE_FLATCG_H --#define __SCX_EXAMPLE_FLATCG_H -- --enum { -- FCG_HWEIGHT_ONE = 1LLU << 16, --}; -- --enum fcg_stat_idx { -- FCG_STAT_ACT, -- FCG_STAT_DEACT, -- FCG_STAT_LOCAL, -- FCG_STAT_GLOBAL, -- -- FCG_STAT_HWT_UPDATES, -- FCG_STAT_HWT_CACHE, -- FCG_STAT_HWT_SKIP, -- FCG_STAT_HWT_RACE, -- -- FCG_STAT_ENQ_SKIP, -- FCG_STAT_ENQ_RACE, -- -- FCG_STAT_CNS_KEEP, -- FCG_STAT_CNS_EXPIRE, -- FCG_STAT_CNS_EMPTY, -- FCG_STAT_CNS_GONE, -- -- FCG_STAT_PNC_NO_CGRP, -- FCG_STAT_PNC_NEXT, -- FCG_STAT_PNC_EMPTY, -- FCG_STAT_PNC_GONE, -- -- FCG_STAT_BAD_REMOVAL, -- -- FCG_NR_STATS, --}; -- --struct fcg_cgrp_ctx { -- u32 nr_active; -- u32 nr_runnable; -- u32 queued; -- u32 weight; -- u32 hweight; -- u64 child_weight_sum; -- u64 hweight_gen; -- s64 cvtime_delta; -- u64 tvtime_now; --}; -- --#endif /* __SCX_EXAMPLE_FLATCG_H */ -diff --git b/tools/sched_ext/scx_example_pair.bpf.c a/tools/sched_ext/scx_example_pair.bpf.c -deleted file mode 100644 -index 279efe58b777..000000000000 ---- b/tools/sched_ext/scx_example_pair.bpf.c -+++ /dev/null -@@ -1,627 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext core-scheduler which always makes every sibling CPU pair -- * execute from the same CPU cgroup. -- * -- * This scheduler is a minimal implementation and would need some form of -- * priority handling both inside each cgroup and across the cgroups to be -- * practically useful. -- * -- * Each CPU in the system is paired with exactly one other CPU, according to a -- * "stride" value that can be specified when the BPF scheduler program is first -- * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee -- * that they will only ever schedule tasks that belong to the same CPU cgroup. -- * -- * Scheduler Initialization -- * ------------------------ -- * -- * The scheduler BPF program is first initialized from user space, before it is -- * enabled. During this initialization process, each CPU on the system is -- * assigned several values that are constant throughout its runtime: -- * -- * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling -- * decisions. Paired CPUs always schedule tasks from the same -- * CPU cgroup, and synchronize with each other to guarantee -- * that this constraint is not violated. -- * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access -- * a struct pair_ctx object that is shared between the pair. -- * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the -- * pair. Each struct pair_ctx has an active_mask field, -- * which is a bitmap used to indicate whether each core -- * in the pair currently has an actively running task. -- * This index specifies which entry in the bitmap corresponds -- * to each CPU in the pair. -- * -- * During this initialization, the CPUs are paired according to a "stride" that -- * may be specified when invoking the user space program that initializes and -- * loads the scheduler. By default, the stride is 1/2 the total number of CPUs. -- * -- * Tasks and cgroups -- * ----------------- -- * -- * Every cgroup in the system is registered with the scheduler using the -- * pair_cgroup_init() callback, and every task in the system is associated with -- * exactly one cgroup. At a high level, the idea with the pair scheduler is to -- * always schedule tasks from the same cgroup within a given CPU pair. When a -- * task is enqueued (i.e. passed to the pair_enqueue() callback function), its -- * cgroup ID is read from its task struct, and then a corresponding queue map -- * is used to FIFO-enqueue the task for that cgroup. -- * -- * If you look through the implementation of the scheduler, you'll notice that -- * there is quite a bit of complexity involved with looking up the per-cgroup -- * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash -- * BPF hash map that is used to map a cgroup ID to a globally unique ID that's -- * allocated in the BPF program. This is done because we use separate maps to -- * store the FIFO queue of tasks, and the length of that map, per cgroup. This -- * complexity is only present because of current deficiencies in BPF that will -- * soon be addressed. The main point to keep in mind is that newly enqueued -- * tasks are added to their cgroup's FIFO queue. -- * -- * Dispatching tasks -- * ----------------- -- * -- * This section will describe how enqueued tasks are dispatched and scheduled. -- * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is -- * as follows: -- * -- * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is -- * the structure that's used to synchronize amongst the two pair CPUs in their -- * scheduling decisions. After any of the following events have occurred: -- * -- * - The cgroup's slice run has expired, or -- * - The cgroup becomes empty, or -- * - Either CPU in the pair is preempted by a higher priority scheduling class -- * -- * The cgroup transitions to the draining state and stops executing new tasks -- * from the cgroup. -- * -- * 2. If the pair is still executing a task, mark the pair_ctx as draining, and -- * wait for the pair CPU to be preempted. -- * -- * 3. Otherwise, if the pair CPU is not running a task, we can move onto -- * scheduling new tasks. Pop the next cgroup id from the top_q queue. -- * -- * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it. -- * -- * Note again that this scheduling behavior is simple, but the implementation -- * is complex mostly because this it hits several BPF shortcomings and has to -- * work around in often awkward ways. Most of the shortcomings are expected to -- * be resolved in the near future which should allow greatly simplifying this -- * scheduler. -- * -- * Dealing with preemption -- * ----------------------- -- * -- * SCX is the lowest priority sched_class, and could be preempted by them at -- * any time. To address this, the scheduler implements pair_cpu_release() and -- * pair_cpu_acquire() callbacks which are invoked by the core scheduler when -- * the scheduler loses and gains control of the CPU respectively. -- * -- * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and -- * then invoke: -- * -- * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- * -- * This preempts the pair CPU, and waits until it has re-entered the scheduler -- * before returning. This is necessary to ensure that the higher priority -- * sched_class that preempted our scheduler does not schedule a task -- * concurrently with our pair CPU. -- * -- * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption -- * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable -- * pair scheduling. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include "scx_example_pair.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; -- --/* !0 for veristat, set during init */ --const volatile u32 nr_cpu_ids = 64; -- --/* a pair of CPUs stay on a cgroup for this duration */ --const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL; -- --/* cpu ID -> pair cpu ID */ --const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 }; -- --/* cpu ID -> pair_id */ --const volatile u32 pair_id[MAX_CPUS]; -- --/* CPU ID -> CPU # in the pair (0 or 1) */ --const volatile u32 in_pair_idx[MAX_CPUS]; -- --struct pair_ctx { -- struct bpf_spin_lock lock; -- -- /* the cgroup the pair is currently executing */ -- u64 cgid; -- -- /* the pair started executing the current cgroup at */ -- u64 started_at; -- -- /* whether the current cgroup is draining */ -- bool draining; -- -- /* the CPUs that are currently active on the cgroup */ -- u32 active_mask; -- -- /* -- * the CPUs that are currently preempted and running tasks in a -- * different scheduler. -- */ -- u32 preempted_mask; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, MAX_CPUS / 2); -- __type(key, u32); -- __type(value, struct pair_ctx); --} pair_ctx SEC(".maps"); -- --/* queue of cgrp_q's possibly with tasks on them */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- /* -- * Because it's difficult to build strong synchronization encompassing -- * multiple non-trivial operations in BPF, this queue is managed in an -- * opportunistic way so that we guarantee that a cgroup w/ active tasks -- * is always on it but possibly multiple times. Once we have more robust -- * synchronization constructs and e.g. linked list, we should be able to -- * do this in a prettier way but for now just size it big enough. -- */ -- __uint(max_entries, 4 * MAX_CGRPS); -- __type(value, u64); --} top_q SEC(".maps"); -- --/* per-cgroup q which FIFOs the tasks from the cgroup */ --struct cgrp_q { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, MAX_QUEUED); -- __type(value, u32); --}; -- --/* -- * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local -- * storage; however, a cgroup local storage can only be accessed from the BPF -- * progs attached to the cgroup. For now, work around by allocating array of -- * cgrp_q's and then allocating per-cgroup indices. -- * -- * Another caveat: It's difficult to populate a large array of maps statically -- * or from BPF. Initialize it from userland. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, MAX_CGRPS); -- __type(key, s32); -- __array(values, struct cgrp_q); --} cgrp_q_arr SEC(".maps"); -- --static u64 cgrp_q_len[MAX_CGRPS]; -- --/* -- * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be -- * useful to have as a map type. -- */ --static u32 cgrp_q_idx_cursor; --static u64 cgrp_q_idx_busy[MAX_CGRPS]; -- --/* -- * All added up, the following is what we do: -- * -- * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking -- * for a free ID. If not found, fail cgroup creation with -EBUSY. -- * -- * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following -- * cgrp_q_idx_hash. -- * -- * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from -- * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr. -- * -- * This is sadly complicated for something pretty simple. Hopefully, we should -- * be able to simplify in the future. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, MAX_CGRPS); -- __uint(key_size, sizeof(u64)); /* cgrp ID */ -- __uint(value_size, sizeof(s32)); /* cgrp_q idx */ --} cgrp_q_idx_hash SEC(".maps"); -- --/* statistics */ --u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions; --u64 nr_exps, nr_exp_waits, nr_exp_empty; --u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty; -- --struct user_exit_info uei; -- --static bool time_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- struct cgrp_q *cgq; -- s32 pid = p->pid; -- u64 cgid; -- u32 *q_idx; -- u64 *cgq_len; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- cgrp = scx_bpf_task_cgroup(p); -- cgid = cgrp->kn->id; -- bpf_cgroup_release(cgrp); -- -- /* find the cgroup's q and push @p into it */ -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!q_idx) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return; -- } -- -- cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx); -- if (!cgq) { -- scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]", -- cgid, *q_idx); -- return; -- } -- -- if (bpf_map_push_elem(cgq, &pid, 0)) { -- scx_bpf_error("cgroup[%llu] queue overflow", cgid); -- return; -- } -- -- /* bump q len, if going 0 -> 1, queue cgroup into the top_q */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len) { -- scx_bpf_error("MEMBER_VTPR malfunction"); -- return; -- } -- -- if (!__sync_fetch_and_add(cgq_len, 1) && -- bpf_map_push_elem(&top_q, &cgid, 0)) { -- scx_bpf_error("top_q overflow"); -- return; -- } --} -- --static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) --{ -- u32 *vptr; -- -- vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *pairc = bpf_map_lookup_elem(&pair_ctx, vptr); -- if (!(*pairc)) -- return -EINVAL; -- -- vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *mask = 1U << *vptr; -- -- return 0; --} -- --static int try_dispatch(s32 cpu) --{ -- struct pair_ctx *pairc; -- struct bpf_map *cgq_map; -- struct task_struct *p; -- u64 now = bpf_ktime_get_ns(); -- bool kick_pair = false; -- bool expired, pair_preempted; -- u32 *vptr, in_pair_mask; -- s32 pid, q_idx; -- u64 cgid; -- int ret; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) { -- scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]", -- cpu); -- return -ENOENT; -- } -- -- bpf_spin_lock(&pairc->lock); -- pairc->active_mask &= ~in_pair_mask; -- -- expired = time_before(pairc->started_at + pair_batch_dur_ns, now); -- if (expired || pairc->draining) { -- u64 new_cgid = 0; -- -- __sync_fetch_and_add(&nr_exps, 1); -- -- /* -- * We're done with the current cgid. An obvious optimization -- * would be not draining if the next cgroup is the current one. -- * For now, be dumb and always expire. -- */ -- pairc->draining = true; -- -- pair_preempted = pairc->preempted_mask; -- if (pairc->active_mask || pair_preempted) { -- /* -- * The other CPU is still active, or is no longer under -- * our control due to e.g. being preempted by a higher -- * priority sched_class. We want to wait until this -- * cgroup expires, or until control of our pair CPU has -- * been returned to us. -- * -- * If the pair controls its CPU, and the time already -- * expired, kick. When the other CPU arrives at -- * dispatch and clears its active mask, it'll push the -- * pair to the next cgroup and kick this CPU. -- */ -- __sync_fetch_and_add(&nr_exp_waits, 1); -- bpf_spin_unlock(&pairc->lock); -- if (expired && !pair_preempted) -- kick_pair = true; -- goto out_maybe_kick; -- } -- -- bpf_spin_unlock(&pairc->lock); -- -- /* -- * Pick the next cgroup. It'd be easier / cleaner to not drop -- * pairc->lock and use stronger synchronization here especially -- * given that we'll be switching cgroups significantly less -- * frequently than tasks. Unfortunately, bpf_spin_lock can't -- * really protect anything non-trivial. Let's do opportunistic -- * operations instead. -- */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u32 *q_idx; -- u64 *cgq_len; -- -- if (bpf_map_pop_elem(&top_q, &new_cgid)) { -- /* no active cgroup, go idle */ -- __sync_fetch_and_add(&nr_exp_empty, 1); -- return 0; -- } -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid); -- if (!q_idx) -- continue; -- -- /* -- * This is the only place where empty cgroups are taken -- * off the top_q. -- */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len || !*cgq_len) -- continue; -- -- /* -- * If it has any tasks, requeue as we may race and not -- * execute it. -- */ -- bpf_map_push_elem(&top_q, &new_cgid, 0); -- break; -- } -- -- bpf_spin_lock(&pairc->lock); -- -- /* -- * The other CPU may already have started on a new cgroup while -- * we dropped the lock. Make sure that we're still draining and -- * start on the new cgroup. -- */ -- if (pairc->draining && !pairc->active_mask) { -- __sync_fetch_and_add(&nr_cgrp_next, 1); -- pairc->cgid = new_cgid; -- pairc->started_at = now; -- pairc->draining = false; -- kick_pair = true; -- } else { -- __sync_fetch_and_add(&nr_cgrp_coll, 1); -- } -- } -- -- cgid = pairc->cgid; -- pairc->active_mask |= in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- -- /* again, it'd be better to do all these with the lock held, oh well */ -- vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!vptr) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return -ENOENT; -- } -- q_idx = *vptr; -- -- /* claim one task from cgrp_q w/ q_idx */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u64 *cgq_len, len; -- -- cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]); -- if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) { -- /* the cgroup must be empty, expire and repeat */ -- __sync_fetch_and_add(&nr_cgrp_empty, 1); -- bpf_spin_lock(&pairc->lock); -- pairc->draining = true; -- pairc->active_mask &= ~in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- return -EAGAIN; -- } -- -- if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len) -- continue; -- -- break; -- } -- -- cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx); -- if (!cgq_map) { -- scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- if (bpf_map_pop_elem(cgq_map, &pid)) { -- scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- p = bpf_task_from_pid(pid); -- if (p) { -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } else { -- /* we don't handle dequeues, retry on lost tasks */ -- __sync_fetch_and_add(&nr_missing, 1); -- return -EAGAIN; -- } -- --out_maybe_kick: -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } -- return 0; --} -- --void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) --{ -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_dispatch(cpu) != -EAGAIN) -- break; -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask &= ~in_pair_mask; -- /* Kick the pair CPU, unless it was also preempted. */ -- kick_pair = !pairc->preempted_mask; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask |= in_pair_mask; -- pairc->active_mask &= ~in_pair_mask; -- /* Kick the pair CPU if it's still running. */ -- kick_pair = pairc->active_mask; -- pairc->draining = true; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- } -- } -- __sync_fetch_and_add(&nr_preemptions, 1); --} -- --s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 i, q_idx; -- -- bpf_for(i, 0, MAX_CGRPS) { -- q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS; -- if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1)) -- break; -- } -- if (i == MAX_CGRPS) -- return -EBUSY; -- -- if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]); -- if (busy) -- *busy = 0; -- return -EBUSY; -- } -- -- return 0; --} -- --void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 *q_idx; -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (q_idx) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]); -- if (busy) -- *busy = 0; -- bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid); -- } --} -- --s32 BPF_STRUCT_OPS(pair_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops pair_ops = { -- .enqueue = (void *)pair_enqueue, -- .dispatch = (void *)pair_dispatch, -- .cpu_acquire = (void *)pair_cpu_acquire, -- .cpu_release = (void *)pair_cpu_release, -- .cgroup_init = (void *)pair_cgroup_init, -- .cgroup_exit = (void *)pair_cgroup_exit, -- .init = (void *)pair_init, -- .exit = (void *)pair_exit, -- .name = "pair", --}; -diff --git b/tools/sched_ext/scx_example_pair.c a/tools/sched_ext/scx_example_pair.c -deleted file mode 100644 -index 18e032bbc173..000000000000 ---- b/tools/sched_ext/scx_example_pair.c -+++ /dev/null -@@ -1,143 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_pair.h" --#include "scx_example_pair.skel.h" -- --const char help_fmt[] = --"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n" --"execute from the same CPU cgroup.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-S STRIDE] [-p]\n" --"\n" --" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_pair *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 stride, i, opt, outer_fd; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_pair__open(); -- assert(skel); -- -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- /* pair up the earlier half to the latter by default, override with -s */ -- stride = skel->rodata->nr_cpu_ids / 2; -- -- while ((opt = getopt(argc, argv, "S:ph")) != -1) { -- switch (opt) { -- case 'S': -- stride = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- for (i = 0; i < skel->rodata->nr_cpu_ids; i++) { -- if (skel->rodata->pair_cpu[i] < 0) { -- skel->rodata->pair_cpu[i] = i + stride; -- skel->rodata->pair_cpu[i + stride] = i; -- skel->rodata->pair_id[i] = i; -- skel->rodata->pair_id[i + stride] = i; -- skel->rodata->in_pair_idx[i] = 0; -- skel->rodata->in_pair_idx[i + stride] = 1; -- } -- } -- -- assert(!scx_example_pair__load(skel)); -- -- /* -- * Populate the cgrp_q_arr map which is an array containing per-cgroup -- * queues. It'd probably be better to do this from BPF but there are too -- * many to initialize statically and there's no way to dynamically -- * populate from BPF. -- */ -- outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr); -- assert(outer_fd >= 0); -- -- printf("Initializing"); -- for (i = 0; i < MAX_CGRPS; i++) { -- s32 inner_fd; -- -- if (exit_req) -- break; -- -- inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, -- sizeof(u32), MAX_QUEUED, NULL); -- assert(inner_fd >= 0); -- assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY)); -- close(inner_fd); -- -- if (!(i % 10)) -- printf("."); -- fflush(stdout); -- } -- printf("\n"); -- -- /* -- * Fully initialized, attach and run. -- */ -- link = bpf_map__attach_struct_ops(skel->maps.pair_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf(" total:%10lu dispatch:%10lu missing:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_dispatched, -- skel->bss->nr_missing); -- printf(" kicks:%10lu preemptions:%7lu\n", -- skel->bss->nr_kicks, -- skel->bss->nr_preemptions); -- printf(" exp:%10lu exp_wait:%10lu exp_empty:%10lu\n", -- skel->bss->nr_exps, -- skel->bss->nr_exp_waits, -- skel->bss->nr_exp_empty); -- printf("cgnext:%10lu cgcoll:%10lu cgempty:%10lu\n", -- skel->bss->nr_cgrp_next, -- skel->bss->nr_cgrp_coll, -- skel->bss->nr_cgrp_empty); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_pair__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_pair.h a/tools/sched_ext/scx_example_pair.h -deleted file mode 100644 -index f60b824272f7..000000000000 ---- b/tools/sched_ext/scx_example_pair.h -+++ /dev/null -@@ -1,10 +0,0 @@ --#ifndef __SCX_EXAMPLE_PAIR_H --#define __SCX_EXAMPLE_PAIR_H -- --enum { -- MAX_CPUS = 4096, -- MAX_QUEUED = 4096, -- MAX_CGRPS = 4096, --}; -- --#endif /* __SCX_EXAMPLE_PAIR_H */ -diff --git b/tools/sched_ext/scx_example_qmap.bpf.c a/tools/sched_ext/scx_example_qmap.bpf.c -deleted file mode 100644 -index 579ab21ae403..000000000000 ---- b/tools/sched_ext/scx_example_qmap.bpf.c -+++ /dev/null -@@ -1,401 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple five-level FIFO queue scheduler. -- * -- * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets -- * assigned to one depending on its compound weight. Each CPU round robins -- * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from -- * queue0, 2 from queue1, 4 from queue2 and so on. -- * -- * This scheduler demonstrates: -- * -- * - BPF-side queueing using PIDs. -- * - Sleepable per-task storage allocation using ops.prep_enable(). -- * - Using ops.cpu_release() to handle a higher priority scheduling class taking -- * the CPU away. -- * - Core-sched support. -- * -- * This scheduler is primarily for demonstration and testing of sched_ext -- * features and unlikely to be useful for actual workloads. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include -- --char _license[] SEC("license") = "GPL"; -- --const volatile u64 slice_ns = SCX_SLICE_DFL; --const volatile bool switch_partial; --const volatile u32 stall_user_nth; --const volatile u32 stall_kernel_nth; --const volatile u32 dsp_inf_loop_after; --const volatile s32 disallow_tgid; -- --u32 test_error_cnt; -- --struct user_exit_info uei; -- --struct qmap { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, u32); --} queue0 SEC(".maps"), -- queue1 SEC(".maps"), -- queue2 SEC(".maps"), -- queue3 SEC(".maps"), -- queue4 SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, 5); -- __type(key, int); -- __array(values, struct qmap); --} queue_arr SEC(".maps") = { -- .values = { -- [0] = &queue0, -- [1] = &queue1, -- [2] = &queue2, -- [3] = &queue3, -- [4] = &queue4, -- }, --}; -- --/* -- * Per-queue sequence numbers to implement core-sched ordering. -- * -- * Tail seq is assigned to each queued task and incremented. Head seq tracks the -- * sequence number of the latest dispatched task. The distance between the a -- * task's seq and the associated queue's head seq is called the queue distance -- * and used when comparing two tasks for ordering. See qmap_core_sched_before(). -- */ --static u64 core_sched_head_seqs[5]; --static u64 core_sched_tail_seqs[5]; -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local_dsq */ -- u64 core_sched_seq; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --/* Per-cpu dispatch index and remaining count */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(max_entries, 2); -- __type(key, u32); -- __type(value, u64); --} dispatch_idx_cnt SEC(".maps"); -- --/* Statistics */ --unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued; --unsigned long nr_core_sched_execed; -- --s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- struct task_ctx *tctx; -- s32 cpu; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- return cpu; -- -- return prev_cpu; --} -- --static int weight_to_idx(u32 weight) --{ -- /* Coarsely map the compound weight to a FIFO. */ -- if (weight <= 25) -- return 0; -- else if (weight <= 50) -- return 1; -- else if (weight < 200) -- return 2; -- else if (weight < 400) -- return 3; -- else -- return 4; --} -- --void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) --{ -- static u32 user_cnt, kernel_cnt; -- struct task_ctx *tctx; -- u32 pid = p->pid; -- int idx = weight_to_idx(p->scx.weight); -- void *ring; -- -- if (p->flags & PF_KTHREAD) { -- if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth)) -- return; -- } else { -- if (stall_user_nth && !(++user_cnt % stall_user_nth)) -- return; -- } -- -- if (test_error_cnt && !--test_error_cnt) -- scx_bpf_error("test triggering error"); -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * All enqueued tasks must have their core_sched_seq updated for correct -- * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in -- * qmap_ops.flags. -- */ -- tctx->core_sched_seq = core_sched_tail_seqs[idx]++; -- -- /* -- * If qmap_select_cpu() is telling us to or this is the last runnable -- * task on the CPU, enqueue locally. -- */ -- if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) { -- tctx->force_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); -- return; -- } -- -- /* -- * If the task was re-enqueued due to the CPU being preempted by a -- * higher priority scheduling class, just re-enqueue the task directly -- * on the global DSQ. As we want another CPU to pick it up, find and -- * kick an idle CPU. -- */ -- if (enq_flags & SCX_ENQ_REENQ) { -- s32 cpu; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags); -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- return; -- } -- -- ring = bpf_map_lookup_elem(&queue_arr, &idx); -- if (!ring) { -- scx_bpf_error("failed to find ring %d", idx); -- return; -- } -- -- /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */ -- if (bpf_map_push_elem(ring, &pid, 0)) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_enqueued, 1); --} -- --/* -- * The BPF queue map doesn't support removal and sched_ext can handle spurious -- * dispatches. qmap_dequeue() is only used to collect statistics. -- */ --void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags) --{ -- __sync_fetch_and_add(&nr_dequeued, 1); -- if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) -- __sync_fetch_and_add(&nr_core_sched_execed, 1); --} -- --static void update_core_sched_head_seq(struct task_struct *p) --{ -- struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- int idx = weight_to_idx(p->scx.weight); -- -- if (tctx) -- core_sched_head_seqs[idx] = tctx->core_sched_seq; -- else -- scx_bpf_error("task_ctx lookup failed"); --} -- --void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 zero = 0, one = 1; -- u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero); -- u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one); -- void *fifo; -- s32 pid; -- int i; -- -- if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) { -- struct task_struct *p; -- -- /* -- * PID 2 should be kthreadd which should mostly be idle and off -- * the scheduler. Let's keep dispatching it to force the kernel -- * to call this function over and over again. -- */ -- p = bpf_task_from_pid(2); -- if (p) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- if (!idx || !cnt) { -- scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt); -- return; -- } -- -- for (i = 0; i < 5; i++) { -- /* Advance the dispatch cursor and pick the fifo. */ -- if (!*cnt) { -- *idx = (*idx + 1) % 5; -- *cnt = 1 << *idx; -- } -- (*cnt)--; -- -- fifo = bpf_map_lookup_elem(&queue_arr, idx); -- if (!fifo) { -- scx_bpf_error("failed to find ring %llu", *idx); -- return; -- } -- -- /* Dispatch or advance. */ -- if (!bpf_map_pop_elem(fifo, &pid)) { -- struct task_struct *p; -- -- p = bpf_task_from_pid(pid); -- if (p) { -- update_core_sched_head_seq(p); -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- *cnt = 0; -- } --} -- --/* -- * The distance from the head of the queue scaled by the weight of the queue. -- * The lower the number, the older the task and the higher the priority. -- */ --static s64 task_qdist(struct task_struct *p) --{ -- int idx = weight_to_idx(p->scx.weight); -- struct task_ctx *tctx; -- s64 qdist; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return 0; -- } -- -- qdist = tctx->core_sched_seq - core_sched_head_seqs[idx]; -- -- /* -- * As queue index increments, the priority doubles. The queue w/ index 3 -- * is dispatched twice more frequently than 2. Reflect the difference by -- * scaling qdists accordingly. Note that the shift amount needs to be -- * flipped depending on the sign to avoid flipping priority direction. -- */ -- if (qdist >= 0) -- return qdist << (4 - idx); -- else -- return qdist << idx; --} -- --/* -- * This is called to determine the task ordering when core-sched is picking -- * tasks to execute on SMT siblings and should encode about the same ordering as -- * the regular scheduling path. Use the priority-scaled distances from the head -- * of the queues to compare the two tasks which should be consistent with the -- * dispatch path behavior. -- */ --bool BPF_STRUCT_OPS(qmap_core_sched_before, -- struct task_struct *a, struct task_struct *b) --{ -- return task_qdist(a) > task_qdist(b); --} -- --void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- u32 cnt; -- -- /* -- * Called when @cpu is taken by a higher priority scheduling class. This -- * makes @cpu no longer available for executing sched_ext tasks. As we -- * don't want the tasks in @cpu's local dsq to sit there until @cpu -- * becomes available again, re-enqueue them into the global dsq. See -- * %SCX_ENQ_REENQ handling in qmap_enqueue(). -- */ -- cnt = scx_bpf_reenqueue_local(); -- if (cnt) -- __sync_fetch_and_add(&nr_reenqueued, cnt); --} -- --s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (p->tgid == disallow_tgid) -- p->scx.disallow = true; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(qmap_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops qmap_ops = { -- .select_cpu = (void *)qmap_select_cpu, -- .enqueue = (void *)qmap_enqueue, -- .dequeue = (void *)qmap_dequeue, -- .dispatch = (void *)qmap_dispatch, -- .core_sched_before = (void *)qmap_core_sched_before, -- .cpu_release = (void *)qmap_cpu_release, -- .prep_enable = (void *)qmap_prep_enable, -- .init = (void *)qmap_init, -- .exit = (void *)qmap_exit, -- .flags = SCX_OPS_ENQ_LAST, -- .timeout_ms = 5000U, -- .name = "qmap", --}; -diff --git b/tools/sched_ext/scx_example_qmap.c a/tools/sched_ext/scx_example_qmap.c -deleted file mode 100644 -index ccb4814ee61b..000000000000 ---- b/tools/sched_ext/scx_example_qmap.c -+++ /dev/null -@@ -1,107 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_qmap.skel.h" -- --const char help_fmt[] = --"A simple five-level FIFO queue sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" --" -t COUNT Stall every COUNT'th user thread\n" --" -T COUNT Stall every COUNT'th kernel thread\n" --" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n" --" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_qmap *skel; -- struct bpf_link *link; -- int opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_qmap__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) { -- switch (opt) { -- case 's': -- skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; -- break; -- case 'e': -- skel->bss->test_error_cnt = strtoul(optarg, NULL, 0); -- break; -- case 't': -- skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0); -- break; -- case 'T': -- skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0); -- break; -- case 'l': -- skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0); -- break; -- case 'd': -- skel->rodata->disallow_tgid = strtol(optarg, NULL, 0); -- if (skel->rodata->disallow_tgid < 0) -- skel->rodata->disallow_tgid = getpid(); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_qmap__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.qmap_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- long nr_enqueued = skel->bss->nr_enqueued; -- long nr_dispatched = skel->bss->nr_dispatched; -- -- printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n", -- nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, -- skel->bss->nr_reenqueued, skel->bss->nr_dequeued, -- skel->bss->nr_core_sched_execed); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_qmap__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_simple.bpf.c a/tools/sched_ext/scx_example_simple.bpf.c -deleted file mode 100644 -index 4bccca3e2047..000000000000 ---- b/tools/sched_ext/scx_example_simple.bpf.c -+++ /dev/null -@@ -1,128 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple scheduler. -- * -- * By default, it operates as a simple global weighted vtime scheduler and can -- * be switched to FIFO scheduling. It also demonstrates the following niceties. -- * -- * - Statistics tracking how many tasks are queued to local and global dsq's. -- * - Termination notification for userspace. -- * -- * While very simple, this scheduler should work reasonably well on CPUs with a -- * uniform L3 cache topology. While preemption is not implemented, the fact that -- * the scheduling queue is shared across all CPUs means that whatever is at the -- * front of the queue is likely to be executed fairly quickly given enough -- * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads -- * but comes with the usual problems with FIFO scheduling where saturating -- * threads can easily drown out interactive ones. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --static u64 vtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, 2); /* [local, global] */ --} stats SEC(".maps"); -- --static void stat_inc(u32 idx) --{ -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); -- if (cnt_p) -- (*cnt_p)++; --} -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) --{ -- /* -- * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that -- * running @p on its CPU directly shouldn't affect fairness. Just queue -- * it on the local FIFO. -- */ -- if (enq_flags & SCX_ENQ_LOCAL) { -- stat_inc(0); /* count local queueing */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- return; -- } -- -- stat_inc(1); /* count global queueing */ -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) -- vtime = vtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --void BPF_STRUCT_OPS(simple_running, struct task_struct *p) --{ -- if (fifo_sched) -- return; -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(vtime_now, p->scx.dsq_vtime)) -- vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --s32 BPF_STRUCT_OPS(simple_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .running = (void *)simple_running, -- .stopping = (void *)simple_stopping, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", --}; -diff --git b/tools/sched_ext/scx_example_simple.c a/tools/sched_ext/scx_example_simple.c -deleted file mode 100644 -index 486b401f7c95..000000000000 ---- b/tools/sched_ext/scx_example_simple.c -+++ /dev/null -@@ -1,101 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_simple.skel.h" -- --const char help_fmt[] = --"A simple sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-f] [-p]\n" --"\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int simple) --{ -- exit_req = 1; --} -- --static void read_stats(struct scx_example_simple *skel, u64 *stats) --{ -- int nr_cpus = libbpf_num_possible_cpus(); -- u64 cnts[2][nr_cpus]; -- u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * 2); -- -- for (idx = 0; idx < 2; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_simple *skel; -- struct bpf_link *link; -- u32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_simple__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "fph")) != -1) { -- switch (opt) { -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_simple__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.simple_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- u64 stats[2]; -- -- read_stats(skel, stats); -- printf("local=%lu global=%lu\n", stats[0], stats[1]); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_simple__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_userland.bpf.c a/tools/sched_ext/scx_example_userland.bpf.c -deleted file mode 100644 -index a089bc6bbe86..000000000000 ---- b/tools/sched_ext/scx_example_userland.bpf.c -+++ /dev/null -@@ -1,269 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A minimal userland scheduler. -- * -- * In terms of scheduling, this provides two different types of behaviors: -- * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity. -- * All such tasks are direct-dispatched from the kernel, and are never -- * enqueued in user space. -- * 2. A primitive vruntime scheduler that is implemented in user space, for all -- * other tasks. -- * -- * Some parts of this example user space scheduler could be implemented more -- * efficiently using more complex and sophisticated data structures. For -- * example, rather than using BPF_MAP_TYPE_QUEUE's, -- * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between -- * user space and kernel space. Similarly, we use a simple vruntime-sorted list -- * in user space, but an rbtree could be used instead. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include --#include "scx_common.bpf.h" --#include "scx_example_userland_common.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; --const volatile s32 usersched_pid; -- --/* !0 for veristat, set during init */ --const volatile u32 num_possible_cpus = 64; -- --/* Stats that are printed by user space. */ --u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues; -- --struct user_exit_info uei; -- --/* -- * Whether the user space scheduler needs to be scheduled due to a task being -- * enqueued in user space. -- */ --static bool usersched_needed; -- --/* -- * The map containing tasks that are enqueued in user space from the kernel. -- * -- * This map is drained by the user space scheduler. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, struct scx_userland_enqueued_task); --} enqueued SEC(".maps"); -- --/* -- * The map containing tasks that are dispatched to the kernel from user space. -- * -- * Drained by the kernel in userland_dispatch(). -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, s32); --} dispatched SEC(".maps"); -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local DSQ */ --}; -- --/* Map that contains task-local storage. */ --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --static bool is_usersched_task(const struct task_struct *p) --{ -- return p->pid == usersched_pid; --} -- --static bool keep_in_kernel(const struct task_struct *p) --{ -- return p->nr_cpus_allowed < num_possible_cpus; --} -- --static struct task_struct *usersched_task(void) --{ -- struct task_struct *p; -- -- p = bpf_task_from_pid(usersched_pid); -- /* -- * Should never happen -- the usersched task should always be managed -- * by sched_ext. -- */ -- if (!p) { -- scx_bpf_error("Failed to find usersched task %d", usersched_pid); -- /* -- * We should never hit this path, and we error out of the -- * scheduler above just in case, so the scheduler will soon be -- * be evicted regardless. So as to simplify the logic in the -- * caller to not have to check for NULL, return an acquired -- * reference to the current task here rather than NULL. -- */ -- return bpf_task_acquire(bpf_get_current_task_btf()); -- } -- -- return p; --} -- --s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- if (keep_in_kernel(p)) { -- s32 cpu; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to look up task-local storage for %s", p->comm); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- tctx->force_local = true; -- return cpu; -- } -- } -- -- return prev_cpu; --} -- --static void dispatch_user_scheduler(void) --{ -- struct task_struct *p; -- -- usersched_needed = false; -- p = usersched_task(); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); --} -- --static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags) --{ -- struct scx_userland_enqueued_task task; -- -- memset(&task, 0, sizeof(task)); -- task.pid = p->pid; -- task.sum_exec_runtime = p->se.sum_exec_runtime; -- task.weight = p->scx.weight; -- -- if (bpf_map_push_elem(&enqueued, &task, 0)) { -- /* -- * If we fail to enqueue the task in user space, put it -- * directly on the global DSQ. -- */ -- __sync_fetch_and_add(&nr_failed_enqueues, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- __sync_fetch_and_add(&nr_user_enqueues, 1); -- usersched_needed = true; -- } --} -- --void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags) --{ -- if (keep_in_kernel(p)) { -- u64 dsq_id = SCX_DSQ_GLOBAL; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to lookup task ctx for %s", p->comm); -- return; -- } -- -- if (tctx->force_local) -- dsq_id = SCX_DSQ_LOCAL; -- tctx->force_local = false; -- scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags); -- __sync_fetch_and_add(&nr_kernel_enqueues, 1); -- return; -- } else if (!is_usersched_task(p)) { -- enqueue_task_in_user_space(p, enq_flags); -- } --} -- --void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (usersched_needed) -- dispatch_user_scheduler(); -- -- bpf_repeat(4096) { -- s32 pid; -- struct task_struct *p; -- -- if (bpf_map_pop_elem(&dispatched, &pid)) -- break; -- -- /* -- * The task could have exited by the time we get around to -- * dispatching it. Treat this as a normal occurrence, and simply -- * move onto the next iteration. -- */ -- p = bpf_task_from_pid(pid); -- if (!p) -- continue; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } --} -- --s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(userland_init) --{ -- if (num_possible_cpus == 0) { -- scx_bpf_error("User scheduler # CPUs uninitialized (%d)", -- num_possible_cpus); -- return -EINVAL; -- } -- -- if (usersched_pid <= 0) { -- scx_bpf_error("User scheduler pid uninitialized (%d)", -- usersched_pid); -- return -EINVAL; -- } -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops userland_ops = { -- .select_cpu = (void *)userland_select_cpu, -- .enqueue = (void *)userland_enqueue, -- .dispatch = (void *)userland_dispatch, -- .prep_enable = (void *)userland_prep_enable, -- .init = (void *)userland_init, -- .exit = (void *)userland_exit, -- .timeout_ms = 3000, -- .name = "userland", --}; -diff --git b/tools/sched_ext/scx_example_userland.c a/tools/sched_ext/scx_example_userland.c -deleted file mode 100644 -index 4152b1e65fe1..000000000000 ---- b/tools/sched_ext/scx_example_userland.c -+++ /dev/null -@@ -1,402 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext user space scheduler which provides vruntime semantics -- * using a simple ordered-list implementation. -- * -- * Each CPU in the system resides in a single, global domain. This precludes -- * the need to do any load balancing between domains. The scheduler could -- * easily be extended to support multiple domains, with load balancing -- * happening in user space. -- * -- * Any task which has any CPU affinity is scheduled entirely in BPF. This -- * program only schedules tasks which may run on any CPU. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include "user_exit_info.h" --#include "scx_example_userland_common.h" --#include "scx_example_userland.skel.h" -- --const char help_fmt[] = --"A minimal userland sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-b BATCH] [-p]\n" --"\n" --" -b BATCH The number of tasks to batch when dispatching (default: 8)\n" --" -p Don't switch all, switch only tasks on SCHED_EXT policy\n" --" -h Display this help and exit\n"; -- --/* Defined in UAPI */ --#define SCHED_EXT 7 -- --/* Number of tasks to batch when dispatching to user space. */ --static __u32 batch_size = 8; -- --static volatile int exit_req; --static int enqueued_fd, dispatched_fd; -- --static struct scx_example_userland *skel; --static struct bpf_link *ops_link; -- --/* Stats collected in user space. */ --static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches; -- --/* The data structure containing tasks that are enqueued in user space. */ --struct enqueued_task { -- LIST_ENTRY(enqueued_task) entries; -- __u64 sum_exec_runtime; -- double vruntime; --}; -- --/* -- * Use a vruntime-sorted list to store tasks. This could easily be extended to -- * a more optimal data structure, such as an rbtree as is done in CFS. We -- * currently elect to use a sorted list to simplify the example for -- * illustrative purposes. -- */ --LIST_HEAD(listhead, enqueued_task); -- --/* -- * A vruntime-sorted list of tasks. The head of the list contains the task with -- * the lowest vruntime. That is, the task that has the "highest" claim to be -- * scheduled. -- */ --static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head); -- --/* -- * The statically allocated array of tasks. We use a statically allocated list -- * here to avoid having to allocate on the enqueue path, which could cause a -- * deadlock. A more substantive user space scheduler could e.g. provide a hook -- * for newly enabled tasks that are passed to the scheduler from the -- * .prep_enable() callback to allows the scheduler to allocate on safe paths. -- */ --struct enqueued_task tasks[USERLAND_MAX_TASKS]; -- --static double min_vruntime; -- --static void sigint_handler(int userland) --{ -- exit_req = 1; --} -- --static __u32 task_pid(const struct enqueued_task *task) --{ -- return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task); --} -- --static int dispatch_task(s32 pid) --{ -- int err; -- -- err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d\n", pid); -- exit_req = 1; -- } else { -- nr_vruntime_dispatches++; -- } -- -- return err; --} -- --static struct enqueued_task *get_enqueued_task(__s32 pid) --{ -- if (pid >= USERLAND_MAX_TASKS) -- return NULL; -- -- return &tasks[pid]; --} -- --static double calc_vruntime_delta(__u64 weight, __u64 delta) --{ -- double weight_f = (double)weight / 100.0; -- double delta_f = (double)delta; -- -- return delta_f / weight_f; --} -- --static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task) --{ -- __u64 delta; -- -- delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime; -- -- enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta); -- if (min_vruntime > enqueued->vruntime) -- enqueued->vruntime = min_vruntime; -- enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime; --} -- --static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task) --{ -- struct enqueued_task *curr, *enqueued, *prev; -- -- curr = get_enqueued_task(bpf_task->pid); -- if (!curr) -- return ENOENT; -- -- update_enqueued(curr, bpf_task); -- nr_vruntime_enqueues++; -- -- /* -- * Enqueue the task in a vruntime-sorted list. A more optimal data -- * structure such as an rbtree could easily be used as well. We elect -- * to use a list here simply because it's less code, and thus the -- * example is less convoluted and better serves to illustrate what a -- * user space scheduler could look like. -- */ -- -- if (LIST_EMPTY(&vruntime_head)) { -- LIST_INSERT_HEAD(&vruntime_head, curr, entries); -- return 0; -- } -- -- LIST_FOREACH(enqueued, &vruntime_head, entries) { -- if (curr->vruntime <= enqueued->vruntime) { -- LIST_INSERT_BEFORE(enqueued, curr, entries); -- return 0; -- } -- prev = enqueued; -- } -- -- LIST_INSERT_AFTER(prev, curr, entries); -- -- return 0; --} -- --static void drain_enqueued_map(void) --{ -- while (1) { -- struct scx_userland_enqueued_task task; -- int err; -- -- if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) -- return; -- -- err = vruntime_enqueue(&task); -- if (err) { -- fprintf(stderr, "Failed to enqueue task %d: %s\n", -- task.pid, strerror(err)); -- exit_req = 1; -- return; -- } -- } --} -- --static void dispatch_batch(void) --{ -- __u32 i; -- -- for (i = 0; i < batch_size; i++) { -- struct enqueued_task *task; -- int err; -- __s32 pid; -- -- task = LIST_FIRST(&vruntime_head); -- if (!task) -- return; -- -- min_vruntime = task->vruntime; -- pid = task_pid(task); -- LIST_REMOVE(task, entries); -- err = dispatch_task(pid); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d in %u\n", -- pid, i); -- return; -- } -- } --} -- --static void *run_stats_printer(void *arg) --{ -- while (!exit_req) { -- __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; -- -- nr_failed_enqueues = skel->bss->nr_failed_enqueues; -- nr_kernel_enqueues = skel->bss->nr_kernel_enqueues; -- nr_user_enqueues = skel->bss->nr_user_enqueues; -- total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues; -- -- printf("o-----------------------o\n"); -- printf("| BPF ENQUEUES |\n"); -- printf("|-----------------------|\n"); -- printf("| kern: %10llu |\n", nr_kernel_enqueues); -- printf("| user: %10llu |\n", nr_user_enqueues); -- printf("| failed: %10llu |\n", nr_failed_enqueues); -- printf("| -------------------- |\n"); -- printf("| total: %10llu |\n", total); -- printf("| |\n"); -- printf("|-----------------------|\n"); -- printf("| VRUNTIME / USER |\n"); -- printf("|-----------------------|\n"); -- printf("| enq: %10llu |\n", nr_vruntime_enqueues); -- printf("| disp: %10llu |\n", nr_vruntime_dispatches); -- printf("o-----------------------o\n"); -- printf("\n\n"); -- sleep(1); -- } -- -- return NULL; --} -- --static int spawn_stats_thread(void) --{ -- pthread_t stats_printer; -- -- return pthread_create(&stats_printer, NULL, run_stats_printer, NULL); --} -- --static int bootstrap(int argc, char **argv) --{ -- int err; -- __u32 opt; -- struct sched_param sched_param = { -- .sched_priority = sched_get_priority_max(SCHED_EXT), -- }; -- bool switch_partial = false; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- /* -- * Enforce that the user scheduler task is managed by sched_ext. The -- * task eagerly drains the list of enqueued tasks in its main work -- * loop, and then yields the CPU. The BPF scheduler only schedules the -- * user space scheduler task when at least one other task in the system -- * needs to be scheduled. -- */ -- err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param); -- if (err) { -- fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err)); -- return err; -- } -- -- while ((opt = getopt(argc, argv, "b:ph")) != -1) { -- switch (opt) { -- case 'b': -- batch_size = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- exit(opt != 'h'); -- } -- } -- -- /* -- * It's not always safe to allocate in a user space scheduler, as an -- * enqueued task could hold a lock that we require in order to be able -- * to allocate. -- */ -- err = mlockall(MCL_CURRENT | MCL_FUTURE); -- if (err) { -- fprintf(stderr, "Failed to prefault and lock address space: %s\n", -- strerror(err)); -- return err; -- } -- -- skel = scx_example_userland__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno)); -- return errno; -- } -- skel->rodata->num_possible_cpus = libbpf_num_possible_cpus(); -- assert(skel->rodata->num_possible_cpus > 0); -- skel->rodata->usersched_pid = getpid(); -- assert(skel->rodata->usersched_pid > 0); -- skel->rodata->switch_partial = switch_partial; -- -- err = scx_example_userland__load(skel); -- if (err) { -- fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- enqueued_fd = bpf_map__fd(skel->maps.enqueued); -- dispatched_fd = bpf_map__fd(skel->maps.dispatched); -- assert(enqueued_fd > 0); -- assert(dispatched_fd > 0); -- -- err = spawn_stats_thread(); -- if (err) { -- fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops); -- if (!ops_link) { -- fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno)); -- err = errno; -- goto destroy_skel; -- } -- -- return 0; -- --destroy_skel: -- scx_example_userland__destroy(skel); -- exit_req = 1; -- return err; --} -- --static void sched_main_loop(void) --{ -- while (!exit_req) { -- /* -- * Perform the following work in the main user space scheduler -- * loop: -- * -- * 1. Drain all tasks from the enqueued map, and enqueue them -- * to the vruntime sorted list. -- * -- * 2. Dispatch a batch of tasks from the vruntime sorted list -- * down to the kernel. -- * -- * 3. Yield the CPU back to the system. The BPF scheduler will -- * reschedule the user space scheduler once another task has -- * been enqueued to user space. -- */ -- drain_enqueued_map(); -- dispatch_batch(); -- sched_yield(); -- } --} -- --int main(int argc, char **argv) --{ -- int err; -- -- err = bootstrap(argc, argv); -- if (err) { -- fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err)); -- return err; -- } -- -- sched_main_loop(); -- -- exit_req = 1; -- bpf_link__destroy(ops_link); -- uei_print(&skel->bss->uei); -- scx_example_userland__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_userland_common.h a/tools/sched_ext/scx_example_userland_common.h -deleted file mode 100644 -index 639c6809c5ff..000000000000 ---- b/tools/sched_ext/scx_example_userland_common.h -+++ /dev/null -@@ -1,19 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* Copyright (c) 2022 Meta, Inc */ -- --#ifndef __SCX_USERLAND_COMMON_H --#define __SCX_USERLAND_COMMON_H -- --#define USERLAND_MAX_TASKS 8192 -- --/* -- * An instance of a task that has been enqueued by the kernel for consumption -- * by a user space global scheduler thread. -- */ --struct scx_userland_enqueued_task { -- __s32 pid; -- u64 sum_exec_runtime; -- u64 weight; --}; -- --#endif // __SCX_USERLAND_COMMON_H -diff --git b/tools/sched_ext/user_exit_info.h a/tools/sched_ext/user_exit_info.h -deleted file mode 100644 -index e701ef0e0b86..000000000000 ---- b/tools/sched_ext/user_exit_info.h -+++ /dev/null -@@ -1,50 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Define struct user_exit_info which is shared between BPF and userspace parts -- * to communicate exit status and other information. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __USER_EXIT_INFO_H --#define __USER_EXIT_INFO_H -- --struct user_exit_info { -- int type; -- char reason[128]; -- char msg[1024]; --}; -- --#ifdef __bpf__ -- --#include "vmlinux.h" --#include -- --static inline void uei_record(struct user_exit_info *uei, -- const struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason); -- bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg); -- /* use __sync to force memory barrier */ -- __sync_val_compare_and_swap(&uei->type, uei->type, ei->type); --} -- --#else /* !__bpf__ */ -- --static inline bool uei_exited(struct user_exit_info *uei) --{ -- /* use __sync to force memory barrier */ -- return __sync_val_compare_and_swap(&uei->type, -1, -1); --} -- --static inline void uei_print(const struct user_exit_info *uei) --{ -- fprintf(stderr, "EXIT: %s", uei->reason); -- if (uei->msg[0] != '\0') -- fprintf(stderr, " (%s)", uei->msg); -- fputs("\n", stderr); --} -- --#endif /* __bpf__ */ --#endif /* __USER_EXIT_INFO_H */ diff --git a/oneplus/hmbird/fengchi_OP13_A16.patch b/oneplus/hmbird/fengchi_OP13_A16.patch deleted file mode 100644 index 81dfa588..00000000 --- a/oneplus/hmbird/fengchi_OP13_A16.patch +++ /dev/null @@ -1,19895 +0,0 @@ -diff --git b/Documentation/scheduler/index.rst a/Documentation/scheduler/index.rst -index 0b650bb550e6..3170747226f6 100644 ---- b/Documentation/scheduler/index.rst -+++ a/Documentation/scheduler/index.rst -@@ -19,7 +19,6 @@ Scheduler - sched-nice-design - sched-rt-group - sched-stats -- sched-ext - sched-debug - - text_files -diff --git b/Documentation/scheduler/sched-ext.rst a/Documentation/scheduler/sched-ext.rst -deleted file mode 100644 -index 84c30b44f104..000000000000 ---- b/Documentation/scheduler/sched-ext.rst -+++ /dev/null -@@ -1,230 +0,0 @@ --========================== --Extensible Scheduler Class --========================== -- --sched_ext is a scheduler class whose behavior can be defined by a set of BPF --programs - the BPF scheduler. -- --* sched_ext exports a full scheduling interface so that any scheduling -- algorithm can be implemented on top. -- --* The BPF scheduler can group CPUs however it sees fit and schedule them -- together, as tasks aren't tied to specific CPUs at the time of wakeup. -- --* The BPF scheduler can be turned on and off dynamically anytime. -- --* The system integrity is maintained no matter what the BPF scheduler does. -- The default scheduling behavior is restored anytime an error is detected, -- a runnable task stalls, or on invoking the SysRq key sequence -- :kbd:`SysRq-S`. -- --Switching to and from sched_ext --=============================== -- --``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and --``tools/sched_ext`` contains the example schedulers. -- --sched_ext is used only when the BPF scheduler is loaded and running. -- --If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be --treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is --loaded. On load, such tasks will be switched to and scheduled by sched_ext. -- --The BPF scheduler can choose to schedule all normal and lower class tasks by --calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this --case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and --``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers, --this mode can be selected with the ``-a`` option. -- --Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or --detection of any internal error including stalled runnable tasks aborts the --BPF scheduler and reverts all tasks back to CFS. -- --.. code-block:: none -- -- # make -j16 -C tools/sched_ext -- # tools/sched_ext/scx_example_simple -- local=0 global=3 -- local=5 global=24 -- local=9 global=44 -- local=13 global=56 -- local=17 global=72 -- ^CEXIT: BPF scheduler unregistered -- --If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler --and whether a given task is on sched_ext can be determined as follows: -- --.. code-block:: none -- -- # cat /sys/kernel/debug/sched/ext -- ops : simple -- enabled : 1 -- switching_all : 1 -- switched_all : 1 -- enable_state : enabled -- -- # grep ext /proc/self/sched -- ext.enabled : 1 -- --The Basics --========== -- --Userspace can implement an arbitrary BPF scheduler by loading a set of BPF --programs that implement ``struct sched_ext_ops``. The only mandatory field --is ``ops.name`` which must be a valid BPF object name. All operations are --optional. The following modified excerpt is from --``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO --scheduler. -- --.. code-block:: c -- -- s32 BPF_STRUCT_OPS(simple_init) -- { -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; -- } -- -- void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) -- { -- if (enq_flags & SCX_ENQ_LOCAL) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags); -- } -- -- void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) -- { -- exit_type = ei->type; -- } -- -- SEC(".struct_ops") -- struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", -- }; -- --Dispatch Queues ----------------- -- --To match the impedance between the scheduler core and the BPF scheduler, --sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a --priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``), --and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage --an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and --``scx_bpf_destroy_dsq()``. -- --A CPU always executes a task from its local DSQ. A task is "dispatched" to a --DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's --local DSQ. -- --When a CPU is looking for the next task to run, if the local DSQ is not --empty, the first task is picked. Otherwise, the CPU tries to consume the --global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()`` --is invoked. -- --Scheduling Cycle ------------------ -- --The following briefly shows how a waking task is scheduled and executed. -- --1. When a task is waking up, ``ops.select_cpu()`` is the first operation -- invoked. This serves two purposes. First, CPU selection optimization -- hint. Second, waking up the selected CPU if idle. -- -- The CPU selected by ``ops.select_cpu()`` is an optimization hint and not -- binding. The actual decision is made at the last step of scheduling. -- However, there is a small performance gain if the CPU -- ``ops.select_cpu()`` returns matches the CPU the task eventually runs on. -- -- A side-effect of selecting a CPU is waking it up from idle. While a BPF -- scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper, -- using ``ops.select_cpu()`` judiciously can be simpler and more efficient. -- -- Note that the scheduler core will ignore an invalid CPU selection, for -- example, if it's outside the allowed cpumask of the task. -- --2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can -- make one of the following decisions: -- -- * Immediately dispatch the task to either the global or local DSQ by -- calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or -- ``SCX_DSQ_LOCAL``, respectively. -- -- * Immediately dispatch the task to a custom DSQ by calling -- ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63. -- -- * Queue the task on the BPF side. -- --3. When a CPU is ready to schedule, it first looks at its local DSQ. If -- empty, it then looks at the global DSQ. If there still isn't a task to -- run, ``ops.dispatch()`` is invoked which can use the following two -- functions to populate the local DSQ. -- -- * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can -- be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``, -- ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()`` -- currently can't be called with BPF locks held, this is being worked on -- and will be supported. ``scx_bpf_dispatch()`` schedules dispatching -- rather than performing them immediately. There can be up to -- ``ops.dispatch_max_batch`` pending tasks. -- -- * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ -- to the dispatching DSQ. This function cannot be called with any BPF -- locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks -- before trying to consume the specified DSQ. -- --4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ, -- the CPU runs the first one. If empty, the following steps are taken: -- -- * Try to consume the global DSQ. If successful, run the task. -- -- * If ``ops.dispatch()`` has dispatched any tasks, retry #3. -- -- * If the previous task is an SCX task and still runnable, keep executing -- it (see ``SCX_OPS_ENQ_LAST``). -- -- * Go idle. -- --Note that the BPF scheduler can always choose to dispatch tasks immediately --in ``ops.enqueue()`` as illustrated in the above simple example. If only the --built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as --a task is never queued on the BPF scheduler and both the local and global --DSQs are consumed automatically. -- --``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use --``scx_bpf_dispatch_vtime()`` for the priority queue. See the function --documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for --more information. -- --Where to Look --============= -- --* ``include/linux/sched/ext.h`` defines the core data structures, ops table -- and constants. -- --* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. -- The functions prefixed with ``scx_bpf_`` can be called from the BPF -- scheduler. -- --* ``tools/sched_ext/`` hosts example BPF scheduler implementations. -- -- * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example -- using a custom DSQ. -- -- * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting -- five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. -- --ABI Instability --=============== -- --The APIs provided by sched_ext to BPF schedulers programs have no stability --guarantees. This includes the ops table callbacks and constants defined in --``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in --``kernel/sched/ext.c``. -- --While we will attempt to provide a relatively stable API surface when --possible, they are subject to change without warning between kernel --versions. -diff --git b/MAINTAINERS a/MAINTAINERS -index 9fc6108503c3..2384b55682ee 100644 ---- b/MAINTAINERS -+++ a/MAINTAINERS -@@ -19132,8 +19132,6 @@ R: Ben Segall (CONFIG_CFS_BANDWIDTH) - R: Mel Gorman (CONFIG_NUMA_BALANCING) - R: Daniel Bristot de Oliveira (SCHED_DEADLINE) - R: Valentin Schneider (TOPOLOGY) --R: Tejun Heo (SCHED_EXT) --R: David Vernet (SCHED_EXT) - L: linux-kernel@vger.kernel.org - S: Maintained - T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core -@@ -19142,7 +19140,6 @@ F: include/linux/sched.h - F: include/linux/wait.h - F: include/uapi/linux/sched.h - F: kernel/sched/ --F: tools/sched_ext/ - - SCSI LIBSAS SUBSYSTEM - R: John Garry -diff --git b/arch/arm64/configs/defconfig a/arch/arm64/configs/defconfig -index f93980c09d7f..c2d530535980 100644 ---- b/arch/arm64/configs/defconfig -+++ a/arch/arm64/configs/defconfig -@@ -6,8 +6,6 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_BPF_SYSCALL=y - CONFIG_BPF_JIT=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_BSD_PROCESS_ACCT=y - CONFIG_BSD_PROCESS_ACCT_V3=y -@@ -1587,3 +1585,4 @@ CONFIG_CORESIGHT_STM=m - CONFIG_CORESIGHT_CPU_DEBUG=m - CONFIG_CORESIGHT_CTI=m - CONFIG_MEMTEST=y -+CONFIG_HMBIRD_SCHED=y -diff --git b/arch/arm64/configs/gki_defconfig a/arch/arm64/configs/gki_defconfig -index 4f756dca5e05..6c1bb94f875d 100644 ---- b/arch/arm64/configs/gki_defconfig -+++ a/arch/arm64/configs/gki_defconfig -@@ -10,8 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y - # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set - CONFIG_BPF_LSM=y - CONFIG_PREEMPT=y --CONFIG_SLIM_SCHED=y --CONFIG_SCHED_CLASS_EXT=y -+CONFIG_HMBIRD_SCHED=y - CONFIG_IRQ_TIME_ACCOUNTING=y - CONFIG_TASKSTATS=y - CONFIG_TASK_DELAY_ACCT=y -diff --git b/drivers/gpu/drm/msm/msm_drv.c a/drivers/gpu/drm/msm/msm_drv.c -index 4bd028fa7500..5825ce9d6d7b 100755 ---- b/drivers/gpu/drm/msm/msm_drv.c -+++ a/drivers/gpu/drm/msm/msm_drv.c -@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) - } - - sched_set_fifo(ev_thread->worker->task); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3); -+#endif - } - - ret = drm_vblank_init(ddev, priv->num_crtcs); -diff --git b/drivers/tty/sysrq.c a/drivers/tty/sysrq.c -index bd001445815e..dcf2e1ebf9c5 100644 ---- b/drivers/tty/sysrq.c -+++ a/drivers/tty/sysrq.c -@@ -524,7 +524,6 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { - NULL, /* P */ - NULL, /* Q */ - NULL, /* R */ -- /* S: May be registered by sched_ext for resetting */ - NULL, /* S */ - NULL, /* T */ - NULL, /* U */ -diff --git b/include/asm-generic/vmlinux.lds.h a/include/asm-generic/vmlinux.lds.h -index c45078a445c8..6c15659f874e 100755 ---- b/include/asm-generic/vmlinux.lds.h -+++ a/include/asm-generic/vmlinux.lds.h -@@ -131,7 +131,7 @@ - *(__dl_sched_class) \ - *(__rt_sched_class) \ - *(__fair_sched_class) \ -- *(__ext_sched_class) \ -+ *(__hmbird_sched_class) \ - *(__idle_sched_class) \ - __sched_class_lowest = .; - -diff --git b/include/linux/sched.h a/include/linux/sched.h -index ba49d7bd2b67..dd6b1d58af01 100755 ---- b/include/linux/sched.h -+++ a/include/linux/sched.h -@@ -73,7 +73,9 @@ struct task_delay_info; - struct task_group; - struct user_event_mm; - --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - - /* - * Task state bitmask. NOTE! These bits are also -@@ -298,11 +300,6 @@ enum { - - extern void scheduler_tick(void); - --#ifdef CONFIG_SLIM_SCHED --extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer); --extern void stop_shadow_tick_timer(void); --#endif -- - #define MAX_SCHEDULE_TIMEOUT LONG_MAX - - extern long schedule_timeout(long timeout); -@@ -1933,6 +1925,91 @@ static inline int task_nice(const struct task_struct *p) - return PRIO_TO_NICE((p)->static_prio); - } - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline bool task_is_top_task(struct task_struct *p) -+{ -+ return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) -+ ->top_task_prop & TOP_TASK_BITS_MASK); -+} -+ -+static inline int get_top_task_prop(struct task_struct *p) -+{ -+ return get_hmbird_ts(p)->top_task_prop; -+} -+ -+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear) -+{ -+ if (set) -+ get_hmbird_ts(p)->top_task_prop |= set; -+ if (clear) -+ get_hmbird_ts(p)->top_task_prop &= ~clear; -+ return 0; -+} -+ -+static inline void reset_top_task_prop(struct task_struct *p) -+{ -+ get_hmbird_ts(p)->top_task_prop = 0; -+} -+ -+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ entity->sched_prop = sp; -+ } -+ return 0; -+} -+ -+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->sched_prop; -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq) -+{ -+ unsigned long new_dsq; -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) { -+ new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq; -+ entity->sched_prop = new_dsq; -+ } -+} -+ -+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK); -+ else -+ return 0; -+} -+ -+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ entity->dsq_sync_ux = val; -+} -+ -+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p) -+{ -+ struct hmbird_entity *entity = get_hmbird_ts(p); -+ -+ if (entity) -+ return entity->dsq_sync_ux; -+ else -+ return 0; -+} -+#endif - - extern int can_nice(const struct task_struct *p, const int nice); - extern int task_curr(const struct task_struct *p); -diff --git b/include/linux/sched/ext.h a/include/linux/sched/ext.h -deleted file mode 100755 -index b737a00c1373..000000000000 ---- b/include/linux/sched/ext.h -+++ /dev/null -@@ -1,647 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef _LINUX_SCHED_EXT_H --#define _LINUX_SCHED_EXT_H -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --#include -- --enum scx_consts { -- SCX_OPS_NAME_LEN = 128, -- SCX_EXIT_REASON_LEN = 128, -- SCX_EXIT_BT_LEN = 64, -- SCX_EXIT_MSG_LEN = 1024, -- -- SCX_SLICE_DFL = 20 * NSEC_PER_MSEC, -- SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */ --}; -- --/* -- * DSQ (dispatch queue) IDs are 64bit of the format: -- * -- * Bits: [63] [62 .. 0] -- * [ B] [ ID ] -- * -- * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs -- * ID: 63 bit ID -- * -- * Built-in IDs: -- * -- * Bits: [63] [62] [61..32] [31 .. 0] -- * [ 1] [ L] [ R ] [ V ] -- * -- * 1: 1 for built-in DSQs. -- * L: 1 for LOCAL_ON DSQ IDs, 0 for others -- * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. -- */ --enum scx_dsq_id_flags { -- SCX_DSQ_FLAG_BUILTIN = 1LLU << 63, -- SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62, -- -- SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0, -- SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, -- SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, -- SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, -- SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, --}; -- --enum scx_exit_type { -- SCX_EXIT_NONE, -- SCX_EXIT_DONE, -- -- SCX_EXIT_UNREG = 64, /* BPF unregistration */ -- SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ -- -- SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ -- SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ -- SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ --}; -- --/* -- * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is -- * being disabled. -- */ --struct scx_exit_info { -- /* %SCX_EXIT_* - broad category of the exit reason */ -- enum scx_exit_type type; -- /* textual representation of the above */ -- char reason[SCX_EXIT_REASON_LEN]; -- /* number of entries in the backtrace */ -- u32 bt_len; -- /* backtrace if exiting due to an error */ -- unsigned long bt[SCX_EXIT_BT_LEN]; -- /* extra message */ -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --/* sched_ext_ops.flags */ --enum scx_ops_flags { -- /* -- * Keep built-in idle tracking even if ops.update_idle() is implemented. -- */ -- SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, -- -- /* -- * By default, if there are no other task to run on the CPU, ext core -- * keeps running the current task even after its slice expires. If this -- * flag is specified, such tasks are passed to ops.enqueue() with -- * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. -- */ -- SCX_OPS_ENQ_LAST = 1LLU << 1, -- -- /* -- * An exiting task may schedule after PF_EXITING is set. In such cases, -- * bpf_task_from_pid() may not be able to find the task and if the BPF -- * scheduler depends on pid lookup for dispatching, the task will be -- * lost leading to various issues including RCU grace period stalls. -- * -- * To mask this problem, by default, unhashed tasks are automatically -- * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't -- * depend on pid lookups and wants to handle these tasks directly, the -- * following flag can be used. -- */ -- SCX_OPS_ENQ_EXITING = 1LLU << 2, -- -- /* -- * CPU cgroup knob enable flags -- */ -- SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16, /* cpu.weight */ -- -- SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | -- SCX_OPS_ENQ_LAST | -- SCX_OPS_ENQ_EXITING | -- SCX_OPS_CGROUP_KNOB_WEIGHT, --}; -- --/* argument container for ops.enable() and friends */ --struct scx_enable_args { --}; -- --/* argument container for ops->cgroup_init() */ --struct scx_cgroup_init_args { -- /* the weight of the cgroup [1..10000] */ -- u32 weight; --}; -- --enum scx_cpu_preempt_reason { -- /* next task is being scheduled by &sched_class_rt */ -- SCX_CPU_PREEMPT_RT, -- /* next task is being scheduled by &sched_class_dl */ -- SCX_CPU_PREEMPT_DL, -- /* next task is being scheduled by &sched_class_stop */ -- SCX_CPU_PREEMPT_STOP, -- /* unknown reason for SCX being preempted */ -- SCX_CPU_PREEMPT_UNKNOWN, --}; -- --/* -- * Argument container for ops->cpu_acquire(). Currently empty, but may be -- * expanded in the future. -- */ --struct scx_cpu_acquire_args {}; -- --/* argument container for ops->cpu_release() */ --struct scx_cpu_release_args { -- /* the reason the CPU was preempted */ -- enum scx_cpu_preempt_reason reason; -- -- /* the task that's going to be scheduled on the CPU */ -- struct task_struct *task; --}; -- --/** -- * struct sched_ext_ops - Operation table for BPF scheduler implementation -- * -- * Userland can implement an arbitrary scheduling policy by implementing and -- * loading operations in this table. -- */ --struct sched_ext_ops { -- /** -- * select_cpu - Pick the target CPU for a task which is being woken up -- * @p: task being woken up -- * @prev_cpu: the cpu @p was on before sleeping -- * @wake_flags: SCX_WAKE_* -- * -- * Decision made here isn't final. @p may be moved to any CPU while it -- * is getting dispatched for execution later. However, as @p is not on -- * the rq at this point, getting the eventual execution CPU right here -- * saves a small bit of overhead down the line. -- * -- * If an idle CPU is returned, the CPU is kicked and will try to -- * dispatch. While an explicit custom mechanism can be added, -- * select_cpu() serves as the default way to wake up idle CPUs. -- */ -- s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); -- -- /** -- * enqueue - Enqueue a task on the BPF scheduler -- * @p: task being enqueued -- * @enq_flags: %SCX_ENQ_* -- * -- * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() -- * or enqueue on the BPF scheduler. If not directly dispatched, the bpf -- * scheduler owns @p and if it fails to dispatch @p, the task will -- * stall. -- */ -- void (*enqueue)(struct task_struct *p, u64 enq_flags); -- -- /** -- * dequeue - Remove a task from the BPF scheduler -- * @p: task being dequeued -- * @deq_flags: %SCX_DEQ_* -- * -- * Remove @p from the BPF scheduler. This is usually called to isolate -- * the task while updating its scheduling properties (e.g. priority). -- * -- * The ext core keeps track of whether the BPF side owns a given task or -- * not and can gracefully ignore spurious dispatches from BPF side, -- * which makes it safe to not implement this method. However, depending -- * on the scheduling logic, this can lead to confusing behaviors - e.g. -- * scheduling position not being updated across a priority change. -- */ -- void (*dequeue)(struct task_struct *p, u64 deq_flags); -- -- /** -- * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs -- * @cpu: CPU to dispatch tasks for -- * @prev: previous task being switched out -- * -- * Called when a CPU's local dsq is empty. The operation should dispatch -- * one or more tasks from the BPF scheduler into the DSQs using -- * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using -- * scx_bpf_consume(). -- * -- * The maximum number of times scx_bpf_dispatch() can be called without -- * an intervening scx_bpf_consume() is specified by -- * ops.dispatch_max_batch. See the comments on top of the two functions -- * for more details. -- * -- * When not %NULL, @prev is an SCX task with its slice depleted. If -- * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in -- * @prev->scx.flags, it is not enqueued yet and will be enqueued after -- * ops.dispatch() returns. To keep executing @prev, return without -- * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. -- */ -- void (*dispatch)(s32 cpu, struct task_struct *prev); -- -- /** -- * runnable - A task is becoming runnable on its associated CPU -- * @p: task becoming runnable -- * @enq_flags: %SCX_ENQ_* -- * -- * This and the following three functions can be used to track a task's -- * execution state transitions. A task becomes ->runnable() on a CPU, -- * and then goes through one or more ->running() and ->stopping() pairs -- * as it runs on the CPU, and eventually becomes ->quiescent() when it's -- * done running on the CPU. -- * -- * @p is becoming runnable on the CPU because it's -- * -- * - waking up (%SCX_ENQ_WAKEUP) -- * - being moved from another CPU -- * - being restored after temporarily taken off the queue for an -- * attribute change. -- * -- * This and ->enqueue() are related but not coupled. This operation -- * notifies @p's state transition and may not be followed by ->enqueue() -- * e.g. when @p is being dispatched to a remote CPU. Likewise, a task -- * may be ->enqueue()'d without being preceded by this operation e.g. -- * after exhausting its slice. -- */ -- void (*runnable)(struct task_struct *p, u64 enq_flags); -- -- /** -- * running - A task is starting to run on its associated CPU -- * @p: task starting to run -- * -- * See ->runnable() for explanation on the task state notifiers. -- */ -- void (*running)(struct task_struct *p); -- -- /** -- * stopping - A task is stopping execution -- * @p: task stopping to run -- * @runnable: is task @p still runnable? -- * -- * See ->runnable() for explanation on the task state notifiers. If -- * !@runnable, ->quiescent() will be invoked after this operation -- * returns. -- */ -- void (*stopping)(struct task_struct *p, bool runnable); -- -- /** -- * quiescent - A task is becoming not runnable on its associated CPU -- * @p: task becoming not runnable -- * @deq_flags: %SCX_DEQ_* -- * -- * See ->runnable() for explanation on the task state notifiers. -- * -- * @p is becoming quiescent on the CPU because it's -- * -- * - sleeping (%SCX_DEQ_SLEEP) -- * - being moved to another CPU -- * - being temporarily taken off the queue for an attribute change -- * (%SCX_DEQ_SAVE) -- * -- * This and ->dequeue() are related but not coupled. This operation -- * notifies @p's state transition and may not be preceded by ->dequeue() -- * e.g. when @p is being dispatched to a remote CPU. -- */ -- void (*quiescent)(struct task_struct *p, u64 deq_flags); -- -- /** -- * yield - Yield CPU -- * @from: yielding task -- * @to: optional yield target task -- * -- * If @to is NULL, @from is yielding the CPU to other runnable tasks. -- * The BPF scheduler should ensure that other available tasks are -- * dispatched before the yielding task. Return value is ignored in this -- * case. -- * -- * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf -- * scheduler can implement the request, return %true; otherwise, %false. -- */ -- bool (*yield)(struct task_struct *from, struct task_struct *to); -- -- /** -- * core_sched_before - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Used by core-sched to determine the ordering between two tasks. See -- * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on -- * core-sched. -- * -- * Both @a and @b are runnable and may or may not currently be queued on -- * the BPF scheduler. Should return %true if @a should run before @b. -- * %false if there's no required ordering or @b should run before @a. -- * -- * If not specified, the default is ordering them according to when they -- * became runnable. -- */ -- bool (*core_sched_before)(struct task_struct *a,struct task_struct *b); -- -- /** -- * set_weight - Set task weight -- * @p: task to set weight for -- * @weight: new eight [1..10000] -- * -- * Update @p's weight to @weight. -- */ -- void (*set_weight)(struct task_struct *p, u32 weight); -- -- /** -- * set_cpumask - Set CPU affinity -- * @p: task to set CPU affinity for -- * @cpumask: cpumask of cpus that @p can run on -- * -- * Update @p's CPU affinity to @cpumask. -- */ -- void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask); -- -- /** -- * update_idle - Update the idle state of a CPU -- * @cpu: CPU to udpate the idle state for -- * @idle: whether entering or exiting the idle state -- * -- * This operation is called when @rq's CPU goes or leaves the idle -- * state. By default, implementing this operation disables the built-in -- * idle CPU tracking and the following helpers become unavailable: -- * -- * - scx_bpf_select_cpu_dfl() -- * - scx_bpf_test_and_clear_cpu_idle() -- * - scx_bpf_pick_idle_cpu() -- * - scx_bpf_any_idle_cpu() -- * -- * The user also must implement ops.select_cpu() as the default -- * implementation relies on scx_bpf_select_cpu_dfl(). -- * -- * If you keep the built-in idle tracking, specify the -- * %SCX_OPS_KEEP_BUILTIN_IDLE flag. -- */ -- void (*update_idle)(s32 cpu, bool idle); -- -- /** -- * cpu_acquire - A CPU is becoming available to the BPF scheduler -- * @cpu: The CPU being acquired by the BPF scheduler. -- * @args: Acquire arguments, see the struct definition. -- * -- * A CPU that was previously released from the BPF scheduler is now once -- * again under its control. -- */ -- void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); -- -- /** -- * cpu_release - A CPU is taken away from the BPF scheduler -- * @cpu: The CPU being released by the BPF scheduler. -- * @args: Release arguments, see the struct definition. -- * -- * The specified CPU is no longer under the control of the BPF -- * scheduler. This could be because it was preempted by a higher -- * priority sched_class, though there may be other reasons as well. The -- * caller should consult @args->reason to determine the cause. -- */ -- void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -- -- /** -- * cpu_online - A CPU became online -- * @cpu: CPU which just came up -- * -- * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs beforehand. -- */ -- void (*cpu_online)(s32 cpu); -- -- /** -- * cpu_offline - A CPU is going offline -- * @cpu: CPU which is going offline -- * -- * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks -- * associated with other CPUs afterwards. -- */ -- void (*cpu_offline)(s32 cpu); -- -- /** -- * prep_enable - Prepare to enable BPF scheduling for a task -- * @p: task to prepare BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Either we're loading a BPF scheduler or a new task is being forked. -- * Prepare BPF scheduling for @p. This operation may block and can be -- * used for allocations. -- * -- * Return 0 for success, -errno for failure. An error return while -- * loading will abort loading of the BPF scheduler. During a fork, will -- * abort the specific fork. -- */ -- s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * enable - Enable BPF scheduling for a task -- * @p: task to enable BPF scheduling for -- * @args: enable arguments, see the struct definition -- * -- * Enable @p for BPF scheduling. @p is now in the cgroup specified for -- * the preceding prep_enable() and will start running soon. -- */ -- void (*enable)(struct task_struct *p, struct scx_enable_args *args); -- -- /** -- * cancel_enable - Cancel prep_enable() -- * @p: task being canceled -- * @args: enable arguments, see the struct definition -- * -- * @p was prep_enable()'d but failed before reaching enable(). Undo the -- * preparation. -- */ -- void (*cancel_enable)(struct task_struct *p, -- struct scx_enable_args *args); -- -- /** -- * disable - Disable BPF scheduling for a task -- * @p: task to disable BPF scheduling for -- * -- * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. -- * Disable BPF scheduling for @p. -- */ -- void (*disable)(struct task_struct *p); -- -- /* -- * All online ops must come before ops.init(). -- */ -- -- /** -- * init - Initialize the BPF scheduler -- */ -- s32 (*init)(void); -- -- /** -- * exit - Clean up after the BPF scheduler -- * @info: Exit info -- */ -- void (*exit)(struct scx_exit_info *info); -- -- /** -- * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch -- */ -- u32 dispatch_max_batch; -- -- /** -- * flags - %SCX_OPS_* flags -- */ -- u64 flags; -- -- /** -- * timeout_ms - The maximum amount of time, in milliseconds, that a -- * runnable task should be able to wait before being scheduled. The -- * maximum timeout may not exceed the default timeout of 30 seconds. -- * -- * Defaults to the maximum allowed timeout value of 30 seconds. -- */ -- u32 timeout_ms; -- -- /** -- * name - BPF scheduler's name -- * -- * Must be a non-zero valid BPF object name including only isalnum(), -- * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the -- * BPF scheduler is enabled. -- */ -- char name[SCX_OPS_NAME_LEN]; --}; -- --/* -- * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the -- * scheduler core and the BPF scheduler. See the documentation for more details. -- */ --struct scx_dispatch_q { -- raw_spinlock_t lock; -- struct list_head fifo; /* processed in dispatching order */ -- struct rb_root_cached priq; /* processed in p->scx.dsq_vtime order */ -- u32 nr; -- u64 id; -- struct llist_node free_node; -- struct rcu_head rcu; --}; -- --/* scx_entity.flags */ --enum scx_ent_flags { -- SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ -- SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ -- SCX_TASK_ENQ_LOCAL = 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */ -- -- SCX_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ -- SCX_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ -- -- SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ -- SCX_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ -- -- SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ --}; -- --/* scx_entity.dsq_flags */ --enum scx_ent_dsq_flags { -- SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ --}; -- --/* -- * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from -- * everywhere and the following bits track which kfunc sets are currently -- * allowed for %current. This simple per-task tracking works because SCX ops -- * nest in a limited way. BPF will likely implement a way to allow and disallow -- * kfuncs depending on the calling context which will replace this manual -- * mechanism. See scx_kf_allow(). -- */ --enum scx_kf_mask { -- SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ -- /* all non-sleepables may be nested inside INIT and SLEEPABLE */ -- SCX_KF_INIT = 1 << 0, /* running ops.init() */ -- SCX_KF_SLEEPABLE = 1 << 1, /* other sleepable init operations */ -- /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ -- SCX_KF_CPU_RELEASE = 1 << 2, /* ops.cpu_release() */ -- /* ops.dequeue (in REST) may be nested inside DISPATCH */ -- SCX_KF_DISPATCH = 1 << 3, /* ops.dispatch() */ -- SCX_KF_ENQUEUE = 1 << 4, /* ops.enqueue() */ -- SCX_KF_REST = 1 << 5, /* other rq-locked operations */ -- -- __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | -- SCX_KF_ENQUEUE | SCX_KF_REST, -- __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_REST, --}; -- --#define RAVG_HIST_SIZE 5 --struct scx_sched_task_stats { -- u64 mark_start; -- u64 window_start; -- u32 sum; -- u32 sum_history[RAVG_HIST_SIZE]; -- int cidx; -- -- u32 demand; -- u16 demand_scaled; -- void *sdsq; --}; -- -- -- --/* -- * The following is embedded in task_struct and contains all fields necessary -- * for a task to be scheduled by SCX. -- */ --struct sched_ext_entity { -- struct scx_dispatch_q *dsq; -- struct { -- struct list_head fifo; /* dispatch order */ -- struct rb_node priq; /* p->scx.dsq_vtime order */ -- } dsq_node; -- struct list_head watchdog_node; -- u32 flags; /* protected by rq lock */ -- u32 dsq_flags; /* protected by dsq lock */ -- u32 weight; -- s32 sticky_cpu; -- s32 holding_cpu; -- u32 kf_mask; /* see scx_kf_mask above */ -- struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ -- atomic64_t ops_state; -- unsigned long runnable_at; --#ifdef CONFIG_SCHED_CORE -- u64 core_sched_at; /* see scx_prio_less() */ --#endif -- -- /* BPF scheduler modifiable fields */ -- -- /* -- * Runtime budget in nsecs. This is usually set through -- * scx_bpf_dispatch() but can also be modified directly by the BPF -- * scheduler. Automatically decreased by SCX as the task executes. On -- * depletion, a scheduling event is triggered. -- * -- * This value is cleared to zero if the task is preempted by -- * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the -- * task ran. Use p->se.sum_exec_runtime instead. -- */ -- u64 slice; -- -- /* -- * Used to order tasks when dispatching to the vtime-ordered priority -- * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime() -- * but can also be modified directly by the BPF scheduler. Modifying it -- * while a task is queued on a dsq may mangle the ordering and is not -- * recommended. -- */ -- u64 dsq_vtime; -- -- /* -- * If set, reject future sched_setscheduler(2) calls updating the policy -- * to %SCHED_EXT with -%EACCES. -- * -- * If set from ops.prep_enable() and the task's policy is already -- * %SCHED_EXT, which can happen while the BPF scheduler is being loaded -- * or by inhering the parent's policy during fork, the task's policy is -- * rejected and forcefully reverted to %SCHED_NORMAL. The number of such -- * events are reported through /sys/kernel/debug/sched_ext::nr_rejected. -- */ -- bool disallow; /* reject switching into SCX */ -- -- /* cold fields */ -- struct list_head tasks_node; -- struct task_struct *task; -- struct scx_sched_task_stats sts; --}; -- --void sched_ext_free(struct task_struct *p); -- --#else /* !CONFIG_SCHED_CLASS_EXT */ -- --static inline void sched_ext_free(struct task_struct *p) {} -- --#endif /* CONFIG_SCHED_CLASS_EXT */ --#endif /* _LINUX_SCHED_EXT_H */ -diff --git b/include/linux/sched/hmbird_proc_val.h a/include/linux/sched/hmbird_proc_val.h -new file mode 100755 -index 000000000000..e59708b16ea3 ---- /dev/null -+++ a/include/linux/sched/hmbird_proc_val.h -@@ -0,0 +1,22 @@ -+#ifndef __HMBORD_PROC_VAL_H__ -+#define __HMBORD_PROC_VAL_H__ -+ -+extern int scx_enable; -+extern int partial_enable; -+extern int cpuctrl_high_ratio; -+extern int cpuctrl_low_ratio; -+extern int slim_stats; -+extern int hmbirdcore_debug; -+extern int slim_for_app; -+extern int misfit_ds; -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+extern int cpu7_tl; -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int slim_gov_debug; -+extern int scx_gov_ctrl; -+extern int sched_ravg_window_frame_per_sec; -+ -+#endif -\ No newline at end of file -diff --git b/include/linux/sched/hmbird_version.h a/include/linux/sched/hmbird_version.h -new file mode 100755 -index 000000000000..b2843881c26f ---- /dev/null -+++ a/include/linux/sched/hmbird_version.h -@@ -0,0 +1,52 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#ifndef _OPLUS_HMBIRD_VERSION_H_ -+#define _OPLUS_HMBIRD_VERSION_H_ -+#include -+#include -+#include -+ -+enum hmbird_version { -+ HMBIRD_UNINIT, -+ HMBIRD_GKI_VERSION, -+ HMBIRD_OGKI_VERSION, -+ HMBIRD_UNKNOW_VERSION, -+}; -+ -+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT; -+ -+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type" -+ -+static inline enum hmbird_version get_hmbird_version_type(void) -+{ -+ struct device_node *np = NULL; -+ const char *hmbird_version_str = NULL; -+ if (HMBIRD_UNINIT != hmbird_version_type) -+ return hmbird_version_type; -+ np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH); -+ if (np) { -+ of_property_read_string(np, "type", &hmbird_version_str); -+ if (NULL != hmbird_version_str) { -+ if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) { -+ hmbird_version_type = HMBIRD_OGKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi"); -+ } else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) { -+ hmbird_version_type = HMBIRD_GKI_VERSION; -+ pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi"); -+ } else { -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi"); -+ } -+ return hmbird_version_type; -+ } -+ } -+ -+ hmbird_version_type = HMBIRD_UNKNOW_VERSION; -+ pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION"); -+ return hmbird_version_type; -+} -+ -+#endif /*_OPLUS_HMBIRD_VERSION_H_ */ -diff --git b/include/linux/sched/task.h a/include/linux/sched/task.h -index 03d35e3eda3c..6a5f0c0d74bd 100644 ---- b/include/linux/sched/task.h -+++ a/include/linux/sched/task.h -@@ -61,8 +61,10 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); - extern void init_idle(struct task_struct *idle, int cpu); - - extern int sched_fork(unsigned long clone_flags, struct task_struct *p); --extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); --extern void sched_cancel_fork(struct task_struct *p); -+extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); -+#ifdef CONFIG_HMBIRD_SCHED -+extern void hmbird_cancel_fork(struct task_struct *p); -+#endif - extern void sched_post_fork(struct task_struct *p); - extern void sched_dead(struct task_struct *p); - -diff --git b/include/uapi/linux/sched.h a/include/uapi/linux/sched.h -index 359a14cc76a4..969716ab4320 100755 ---- b/include/uapi/linux/sched.h -+++ a/include/uapi/linux/sched.h -@@ -118,7 +118,7 @@ struct clone_args { - /* SCHED_ISO: reserved but not implemented yet */ - #define SCHED_IDLE 5 - #define SCHED_DEADLINE 6 --#define SCHED_EXT 7 -+#define SCHED_HMBIRD 7 - - /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ - #define SCHED_RESET_ON_FORK 0x40000000 -diff --git b/init/Kconfig a/init/Kconfig -index 337276cbc7f8..bf052ca532b4 100644 ---- b/init/Kconfig -+++ a/init/Kconfig -@@ -1030,10 +1030,6 @@ config RT_GROUP_SCHED - realtime bandwidth for them. - See Documentation/scheduler/sched-rt-group.rst for more information. - --config EXT_GROUP_SCHED -- bool -- depends on SCHED_CLASS_EXT && CGROUP_SCHED -- default y - endif #CGROUP_SCHED - - config SCHED_MM_CID -diff --git b/init/init_task.c a/init/init_task.c -index 69a046388995..31ceb0e469f7 100755 ---- b/init/init_task.c -+++ a/init/init_task.c -@@ -6,7 +6,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -102,9 +101,6 @@ struct task_struct init_task - #endif - #ifdef CONFIG_CGROUP_SCHED - .sched_task_group = &root_task_group, --#endif --#ifdef CONFIG_SLIM_SCHED -- .scx = NULL, - #endif - .ptraced = LIST_HEAD_INIT(init_task.ptraced), - .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), -diff --git b/kernel/Kconfig.preempt a/kernel/Kconfig.preempt -index cf22ef6107b8..ca8de6eb1e16 100755 ---- b/kernel/Kconfig.preempt -+++ a/kernel/Kconfig.preempt -@@ -133,12 +133,8 @@ config SCHED_CORE - which is the likely usage by Linux distributions, there should - be no measurable impact on performance. - --config SLIM_SCHED -- bool "slim sched" -- default n --config SCHED_CLASS_EXT -- bool "Extensible Scheduling Class" -- depends on BPF_SYSCALL && BPF_JIT -+config HMBIRD_SCHED -+ bool "hmbird Scheduling Class(base on ext scheduler)" - help - This option enables a new scheduler class sched_ext (SCX), which - allows scheduling policies to be implemented as BPF programs to -diff --git b/kernel/bpf/bpf_struct_ops_types.h a/kernel/bpf/bpf_struct_ops_types.h -index 3618769d853d..5678a9ddf817 100644 ---- b/kernel/bpf/bpf_struct_ops_types.h -+++ a/kernel/bpf/bpf_struct_ops_types.h -@@ -9,8 +9,4 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) - #include - BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) - #endif --#ifdef CONFIG_SCHED_CLASS_EXT --#include --BPF_STRUCT_OPS_TYPE(sched_ext_ops) --#endif - #endif -diff --git b/kernel/fork.c a/kernel/fork.c -index a43f97302332..7a91505b9378 100755 ---- b/kernel/fork.c -+++ a/kernel/fork.c -@@ -23,7 +23,9 @@ - #include - #include - #include --#include -+#ifdef CONFIG_HMBIRD_SCHED -+#include -+#endif - #include - #include - #include -@@ -999,8 +1001,9 @@ void __put_task_struct(struct task_struct *tsk) - WARN_ON(!tsk->exit_state); - WARN_ON(refcount_read(&tsk->usage)); - WARN_ON(tsk == current); -- -- sched_ext_free(tsk); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_free(tsk); -+#endif - io_uring_free(tsk); - cgroup_free(tsk); - task_numa_free(tsk, true); -@@ -2517,7 +2520,11 @@ __latent_entropy struct task_struct *copy_process( - - retval = perf_event_init_task(p, clone_flags); - if (retval) -- goto bad_fork_sched_cancel_fork; -+#ifdef CONFIG_HMBIRD_SCHED -+ goto bad_fork_hmbird_cancel_fork; -+#else -+ goto bad_fork_cleanup_policy; -+#endif - retval = audit_alloc(p); - if (retval) - goto bad_fork_cleanup_perf; -@@ -2649,9 +2656,7 @@ __latent_entropy struct task_struct *copy_process( - * cgroup specific, it unconditionally needs to place the task on a - * runqueue. - */ -- retval = sched_cgroup_fork(p, args); -- if (retval) -- goto bad_fork_cancel_cgroup; -+ sched_cgroup_fork(p, args); - - /* - * From this point on we must avoid any synchronous user-space -@@ -2697,13 +2702,13 @@ __latent_entropy struct task_struct *copy_process( - /* Don't start children in a dying pid namespace */ - if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { - retval = -ENOMEM; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* Let kill terminate clone/fork in the middle */ - if (fatal_signal_pending(current)) { - retval = -EINTR; -- goto bad_fork_core_free; -+ goto bad_fork_cancel_cgroup; - } - - /* No more failure paths after this point. */ -@@ -2780,11 +2785,10 @@ __latent_entropy struct task_struct *copy_process( - - return p; - --bad_fork_core_free: -+bad_fork_cancel_cgroup: - sched_core_free(p); - spin_unlock(¤t->sighand->siglock); - write_unlock_irq(&tasklist_lock); --bad_fork_cancel_cgroup: - cgroup_cancel_fork(p, args); - bad_fork_put_pidfd: - if (clone_flags & CLONE_PIDFD) { -@@ -2823,8 +2827,10 @@ __latent_entropy struct task_struct *copy_process( - audit_free(p); - bad_fork_cleanup_perf: - perf_event_free_task(p); --bad_fork_sched_cancel_fork: -- sched_cancel_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+bad_fork_hmbird_cancel_fork: -+ hmbird_cancel_fork(p); -+#endif - bad_fork_cleanup_policy: - lockdep_free_task(p); - #ifdef CONFIG_NUMA -diff --git b/kernel/sched/build_policy.c a/kernel/sched/build_policy.c -index 005025f55bea..c091539a0f60 100755 ---- b/kernel/sched/build_policy.c -+++ a/kernel/sched/build_policy.c -@@ -28,8 +28,10 @@ - #include - #include - #include -+#ifdef CONFIG_HMBIRD_SCHED - #include - #include -+#endif - - #include - -@@ -54,6 +56,10 @@ - #include "cputime.c" - #include "deadline.c" - --#ifdef CONFIG_SCHED_CLASS_EXT --# include "ext.c" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/hmbird_util_track.c" -+#include "hmbird/hmbird_sched_proc.c" -+#include "hmbird/hmbird_shadow_tick.c" -+#include "hmbird/hmbird.c" -+#include "hmbird/hmbird_misc.c" - #endif -diff --git b/kernel/sched/core.c a/kernel/sched/core.c -index 25d5703df992..b894417ce7df 100755 ---- b/kernel/sched/core.c -+++ a/kernel/sched/core.c -@@ -96,6 +96,12 @@ - #include "../../io_uring/io-wq.h" - #include "../smpboot.h" - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird/slim.h" -+#include "hmbird/hmbird_shadow_tick.h" -+#include "hmbird.h" -+#endif -+ - #include - #include - #include -@@ -170,7 +176,6 @@ __read_mostly int scheduler_running; - - DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); - -- - /* kernel prio, less is more */ - static inline int __task_prio(const struct task_struct *p) - { -@@ -183,11 +188,10 @@ static inline int __task_prio(const struct task_struct *p) - if (p->sched_class == &idle_sched_class) - return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - --#ifdef CONFIG_SCHED_CLASS_EXT -- if (p->sched_class == &ext_sched_class) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (p->sched_class == &hmbird_sched_class) - return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ - #endif -- - return MAX_RT_PRIO + MAX_NICE; /* 119, squash fair */ - } - -@@ -217,9 +221,9 @@ static inline bool prio_less(const struct task_struct *a, - if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ - return cfs_prio_less(a, b, in_fi); - --#ifdef CONFIG_SCHED_CLASS_EXT -+#ifdef CONFIG_HMBIRD_SCHED - if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ -- return scx_prio_less(a, b, in_fi); -+ return hmbird_prio_less(a, b, in_fi); - #endif - - return false; -@@ -1279,17 +1283,26 @@ bool sched_can_stop_tick(struct rq *rq) - if (fifo_nr_running) - return true; - -+#ifdef CONFIG_HMBIRD_SCHED - /* -- * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks -+ * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks - * left. For CFS, if there's more than one we need the tick for -- * involuntary preemption. For SCX, ask. -+ * involuntary preemption. For HMBIRD, ask. - */ -- if (!scx_switched_all() && rq->nr_running > 1) -+ if (!hmbird_enabled() && rq->nr_running > 1) - return false; - -- if (scx_enabled() && !scx_can_stop_tick(rq)) -+ if (hmbird_enabled() && !hmbird_can_stop_tick(rq)) - return false; -- -+#else -+ /* -+ * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; -+ * if there's more than one we need the tick for involuntary -+ * preemption. -+ */ -+ if (rq->nr_running > 1) -+ return false; -+#endif - /* - * If there is one task and it has CFS runtime bandwidth constraints - * and it's on the cpu now we don't want to stop the tick. -@@ -2222,10 +2235,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) - } - EXPORT_SYMBOL_GPL(deactivate_task); - --struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) -+#ifdef CONFIG_HMBIRD_SCHED -+struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - { -- struct sched_change_guard cg = { -+ struct hmbird_sched_change_guard cg = { - .rq = rq, - .p = p, - .queued = task_on_rq_queued(p), -@@ -2247,7 +2261,7 @@ sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) - return cg; - } - --void sched_change_guard_fini(struct sched_change_guard *cg, int flags) -+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags) - { - if (cg->queued) - enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); -@@ -2255,6 +2269,7 @@ void sched_change_guard_fini(struct sched_change_guard *cg, int flags) - set_next_task(cg->rq, cg->p); - cg->done = true; - } -+#endif - - static inline int __normal_prio(int policy, int rt_prio, int nice) - { -@@ -2320,9 +2335,9 @@ inline int task_curr(const struct task_struct *p) - * this means any call to check_class_changed() must be followed by a call to - * balance_callback(). - */ --void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio) -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) - { - if (prev_class != p->sched_class) { - if (prev_class->switched_from) -@@ -2885,6 +2900,7 @@ static void - __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - { - struct rq *rq = task_rq(p); -+ bool queued, running; - - /* - * This here violates the locking rules for affinity, since we're only -@@ -2903,9 +2919,26 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) - else - lockdep_assert_held(&p->pi_lock); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->sched_class->set_cpus_allowed(p, ctx); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ -+ if (queued) { -+ /* -+ * Because __kthread_bind() calls this on blocked tasks without -+ * holding rq->lock. -+ */ -+ lockdep_assert_rq_held(rq); -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); - } -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->sched_class->set_cpus_allowed(p, ctx); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - } - - /* -@@ -3772,7 +3805,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ -+#ifdef CONFIG_HMBIRD_SCHED -+ if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class)) -+#else - if (unlikely(!is_cpu_allowed(p, cpu))) -+#endif - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -@@ -4891,8 +4928,9 @@ late_initcall(sched_core_sysctl_init); - */ - int sched_fork(unsigned long clone_flags, struct task_struct *p) - { -- -+#ifdef CONFIG_HMBIRD_SCHED - int ret; -+#endif - - trace_android_rvh_sched_fork(p); - -@@ -4933,21 +4971,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - p->sched_reset_on_fork = 0; - } - -- scx_pre_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ ret = hmbird_pre_fork(p); -+ if (ret) -+ goto out_cancel; - - if (dl_prio(p->prio)) { - ret = -EAGAIN; - goto out_cancel; --#ifdef CONFIG_SCHED_CLASS_EXT -- } else if (task_on_scx(p)) { -- p->sched_class = &ext_sched_class; --#endif -+ } else if (task_on_hmbird(p)) { -+ p->sched_class = &hmbird_sched_class; - } else if (rt_prio(p->prio)) { - p->sched_class = &rt_sched_class; - } else { - p->sched_class = &fair_sched_class; - } -- -+#else -+ if (dl_prio(p->prio)) -+ return -EAGAIN; -+ else if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - init_entity_runnable_average(&p->se); - trace_android_rvh_finish_prio_fork(p); - -@@ -4966,12 +5012,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) - #endif - return 0; - -+#ifdef CONFIG_HMBIRD_SCHED - out_cancel: -- scx_cancel_fork(p); -+ hmbird_cancel_fork(p); - return ret; -+#endif - } - --int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) -+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - { - unsigned long flags; - -@@ -4998,20 +5046,14 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -- -- return scx_fork(p); --} -- --void sched_cancel_fork(struct task_struct *p) --{ -- scx_cancel_fork(p); - } - - void sched_post_fork(struct task_struct *p) - { - uclamp_post_fork(p); -- -- scx_post_fork(p); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_post_fork(p); -+#endif - } - - unsigned long to_ratio(u64 period, u64 runtime) -@@ -5875,21 +5917,28 @@ void scheduler_tick(void) - - if (sched_feat(LATENCY_WARN) && resched_latency) - resched_latency_warn(cpu, resched_latency); -- -- scx_notify_sched_tick(); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_notify_sched_tick(); -+#endif - perf_event_task_tick(); - - if (curr->flags & PF_WQ_WORKER) - wq_worker_tick(curr); - - #ifdef CONFIG_SMP -- if (!scx_switched_all()) { -+#ifdef CONFIG_HMBIRD_SCHED -+ if (!hmbird_enabled()) { -+#endif - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); -+#ifdef CONFIG_HMBIRD_SCHED - } -+#endif - #endif - trace_android_vh_scheduler_tick(rq); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ scheduler_tick_handler(NULL, NULL); -+#endif - } - - #ifdef CONFIG_NO_HZ_FULL -@@ -6191,7 +6240,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ -+#ifdef CONFIG_HMBIRD_SCHED - for_balance_class_range(class, prev->sched_class, &idle_sched_class) { -+#else -+ for_class_range(class, prev->sched_class, &idle_sched_class) { -+#endif - if (class->balance(rq, prev, rf)) - break; - } -@@ -6209,9 +6262,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - const struct sched_class *class; - struct task_struct *p; - -- if (scx_enabled()) -+#ifdef CONFIG_HMBIRD_SCHED -+ if (hmbird_enabled()) - goto restart; -- -+#endif - /* - * Optimization: we know that if all tasks are in the fair class we can - * call that function directly, but only if the @prev task wasn't of a -@@ -6237,13 +6291,21 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - restart: - put_prev_task_balance(rq, prev, rf); - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { - p = class->pick_next_task(rq); - if (p) { -- scx_notify_pick_next_task(rq, p, class); -+ hmbird_notify_pick_next_task(rq, p, class); - return p; - } - } -+#else -+ for_each_class(class) { -+ p = class->pick_next_task(rq); -+ if (p) -+ return p; -+ } -+#endif - - BUG(); /* The idle class should always have a runnable task. */ - } -@@ -6272,7 +6334,11 @@ static inline struct task_struct *pick_task(struct rq *rq) - const struct sched_class *class; - struct task_struct *p; - -+#ifdef CONFIG_HMBIRD_SCHED - for_each_active_class(class) { -+#else -+ for_each_class(class) { -+#endif - p = class->pick_task(rq); - if (p) - return p; -@@ -6916,7 +6982,9 @@ static void __sched notrace __schedule(unsigned int sched_mode) - psi_sched_switch(prev, next, !task_on_rq_queued(prev)); - - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); -+#endif - /* Also unlocks the rq: */ - rq = context_switch(rq, prev, next, &rf); - } else { -@@ -7244,24 +7312,31 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag - } - EXPORT_SYMBOL(default_wake_function); - --void __setscheduler_prio(struct task_struct *p, int prio) -+static void __setscheduler_prio(struct task_struct *p, int prio) - { -- /* -- * After switching all rt and fair class to ext, -- * stop class is switched to ext class too. Just -- * skip it. */ -+#ifdef CONFIG_HMBIRD_SCHED -+ bool on_hmbird = task_on_hmbird(p); -+ - if (p->sched_class == &stop_sched_class) -- ;/* do nothing */ -+ ; - else if (dl_prio(prio)) - p->sched_class = &dl_sched_class; --#ifdef CONFIG_SCHED_CLASS_EXT -- else if (task_on_scx(p)) -- p->sched_class = &ext_sched_class; --#endif -+ else if (rt_prio(prio) && on_hmbird) -+ p->sched_class = &hmbird_sched_class; - else if (rt_prio(prio)) - p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; - else - p->sched_class = &fair_sched_class; -+#else -+ if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+#endif - - p->prio = prio; - trace_android_rvh_setscheduler_prio(p); -@@ -7297,7 +7372,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - */ - void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - { -- int prio, oldprio, queue_flag = -+ int prio, oldprio, queued, running, queue_flag = - DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *prev_class; - struct rq_flags rf; -@@ -7360,42 +7435,52 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) - queue_flag &= ~DEQUEUE_MOVE; - - prev_class = p->sched_class; -- SCHED_CHANGE_BLOCK(rq, p, queue_flag) { -- /* -- * Boosting condition are: -- * 1. -rt task is running and holds mutex A -- * --> -dl task blocks on mutex A -- * -- * 2. -dl task is running and holds mutex A -- * --> -dl task blocks on mutex A and could preempt the -- * running task -- */ -- if (dl_prio(prio)) { -- if (!dl_prio(p->normal_prio) || -- (pi_task && dl_prio(pi_task->prio) && -- dl_entity_preempt(&pi_task->dl, &p->dl))) { -- p->dl.pi_se = pi_task->dl.pi_se; -- queue_flag |= ENQUEUE_REPLENISH; -- } else { -- p->dl.pi_se = &p->dl; -- } -- } else if (rt_prio(prio)) { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- if (oldprio < prio) -- queue_flag |= ENQUEUE_HEAD; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flag); -+ if (running) -+ put_prev_task(rq, p); -+ -+ /* -+ * Boosting condition are: -+ * 1. -rt task is running and holds mutex A -+ * --> -dl task blocks on mutex A -+ * -+ * 2. -dl task is running and holds mutex A -+ * --> -dl task blocks on mutex A and could preempt the -+ * running task -+ */ -+ if (dl_prio(prio)) { -+ if (!dl_prio(p->normal_prio) || -+ (pi_task && dl_prio(pi_task->prio) && -+ dl_entity_preempt(&pi_task->dl, &p->dl))) { -+ p->dl.pi_se = pi_task->dl.pi_se; -+ queue_flag |= ENQUEUE_REPLENISH; - } else { -- if (dl_prio(oldprio)) -- p->dl.pi_se = &p->dl; -- else if (rt_prio(oldprio)) -- p->rt.timeout = 0; -- else if (!task_has_idle_policy(p)) -- reweight_task(p, prio - MAX_RT_PRIO); -+ p->dl.pi_se = &p->dl; - } -- -- __setscheduler_prio(p, prio); -+ } else if (rt_prio(prio)) { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ if (oldprio < prio) -+ queue_flag |= ENQUEUE_HEAD; -+ } else { -+ if (dl_prio(oldprio)) -+ p->dl.pi_se = &p->dl; -+ else if (rt_prio(oldprio)) -+ p->rt.timeout = 0; -+ else if (!task_has_idle_policy(p)) -+ reweight_task(p, prio - MAX_RT_PRIO); - } - -+ __setscheduler_prio(p, prio); -+ -+ if (queued) -+ enqueue_task(rq, p, queue_flag); -+ if (running) -+ set_next_task(rq, p); -+ - check_class_changed(rq, p, prev_class, oldprio); - out_unlock: - /* Avoid rq from going away on us: */ -@@ -7416,7 +7501,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) - - void set_user_nice(struct task_struct *p, long nice) - { -- bool allowed = false; -+ bool queued, running, allowed = false; - int old_prio; - struct rq_flags rf; - struct rq *rq; -@@ -7445,13 +7530,22 @@ void set_user_nice(struct task_struct *p, long nice) - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); -+ if (running) -+ put_prev_task(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- p->static_prio = NICE_TO_PRIO(nice); -- set_load_weight(p, true); -- old_prio = p->prio; -- p->prio = effective_prio(p); -- } -+ p->static_prio = NICE_TO_PRIO(nice); -+ set_load_weight(p, true); -+ old_prio = p->prio; -+ p->prio = effective_prio(p); -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - - /* - * If the task increased its priority or is running and -@@ -7868,7 +7962,7 @@ static int __sched_setscheduler(struct task_struct *p, - bool user, bool pi) - { - int oldpolicy = -1, policy = attr->sched_policy; -- int retval, oldprio, newprio; -+ int retval, oldprio, newprio, queued, running; - const struct sched_class *prev_class; - struct balance_callback *head; - struct rq_flags rf; -@@ -7876,7 +7970,9 @@ static int __sched_setscheduler(struct task_struct *p, - int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - struct rq *rq; - bool cpuset_locked = false; -- -+#ifdef CONFIG_HMBIRD_SCHED -+ unsigned long flags; -+#endif - - /* The pi code expects interrupts enabled */ - BUG_ON(pi && in_interrupt()); -@@ -7942,7 +8038,9 @@ static int __sched_setscheduler(struct task_struct *p, - * To be able to change p->policy safely, the appropriate - * runqueue lock must be held. - */ -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+#endif - rq = task_rq_lock(p, &rf); - update_rq_clock(rq); - -@@ -7954,10 +8052,11 @@ static int __sched_setscheduler(struct task_struct *p, - goto unlock; - } - -- retval = scx_check_setscheduler(p, policy); -+#ifdef CONFIG_HMBIRD_SCHED -+ retval = hmbird_check_setscheduler(p, policy); - if (retval) - goto unlock; -- -+#endif - /* - * If not changing anything there's no need to proceed further, - * but store a possible modification of reset_on_fork. -@@ -8014,7 +8113,9 @@ static int __sched_setscheduler(struct task_struct *p, - if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { - policy = oldpolicy = -1; - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - goto recheck; -@@ -8047,16 +8148,23 @@ static int __sched_setscheduler(struct task_struct *p, - queue_flags &= ~DEQUEUE_MOVE; - } - -- SCHED_CHANGE_BLOCK(rq, p, queue_flags) { -- prev_class = p->sched_class; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, queue_flags); -+ if (running) -+ put_prev_task(rq, p); -+ -+ prev_class = p->sched_class; - -- if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -- __setscheduler_params(p, attr); -- __setscheduler_prio(p, newprio); -- trace_android_rvh_setscheduler(p); -- } -- __setscheduler_uclamp(p, attr); -+ if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -+ __setscheduler_params(p, attr); -+ __setscheduler_prio(p, newprio); -+ trace_android_rvh_setscheduler(p); -+ } -+ __setscheduler_uclamp(p, attr); - -+ if (queued) { - /* - * We enqueue to tail when the priority of a task is - * increased (user space view). -@@ -8064,7 +8172,10 @@ static int __sched_setscheduler(struct task_struct *p, - if (oldprio < p->prio) - queue_flags |= ENQUEUE_HEAD; - -+ enqueue_task(rq, p, queue_flags); - } -+ if (running) -+ set_next_task(rq, p); - - check_class_changed(rq, p, prev_class, oldprio); - -@@ -8072,7 +8183,9 @@ static int __sched_setscheduler(struct task_struct *p, - preempt_disable(); - head = splice_balance_callbacks(rq); - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (pi) { - if (cpuset_locked) - cpuset_unlock(); -@@ -8087,7 +8200,9 @@ static int __sched_setscheduler(struct task_struct *p, - - unlock: - task_rq_unlock(rq, p, &rf); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+#endif - if (cpuset_locked) - cpuset_unlock(); - return retval; -@@ -8785,6 +8900,9 @@ static void do_sched_yield(void) - long skip = 0; - - trace_android_rvh_before_do_sched_yield(&skip); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_skip_yield(&skip); -+#endif - if (skip) - return; - -@@ -9309,7 +9427,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - break; - } -@@ -9337,7 +9457,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) - case SCHED_NORMAL: - case SCHED_BATCH: - case SCHED_IDLE: -- case SCHED_EXT: -+#ifdef CONFIG_HMBIRD_SCHED -+ case SCHED_HMBIRD: -+#endif - ret = 0; - } - return ret; -@@ -9638,15 +9760,25 @@ int migrate_task_to(struct task_struct *p, int target_cpu) - */ - void sched_setnuma(struct task_struct *p, int nid) - { -+ bool queued, running; - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); - -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) { -- p->numa_preferred_nid = nid; -- } -+ if (queued) -+ dequeue_task(rq, p, DEQUEUE_SAVE); -+ if (running) -+ put_prev_task(rq, p); - -+ p->numa_preferred_nid = nid; -+ -+ if (queued) -+ enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); -+ if (running) -+ set_next_task(rq, p); - task_rq_unlock(rq, p, &rf); - } - #endif /* CONFIG_NUMA_BALANCING */ -@@ -10212,17 +10344,23 @@ void __init sched_init(void) - int i; - - /* Make sure the linker didn't screw up */ -+#ifdef CONFIG_HMBIRD_SCHED - #ifdef CONFIG_SMP -- BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class)); -+#endif -+ WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class)); -+ WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class)); -+#else -+ BUG_ON(&idle_sched_class != &fair_sched_class + 1 || -+ &fair_sched_class != &rt_sched_class + 1 || -+ &rt_sched_class != &dl_sched_class + 1); -+#ifdef CONFIG_SMP -+ BUG_ON(&dl_sched_class != &stop_sched_class + 1); - #endif -- BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); -- BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); -- BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); --#ifdef CONFIG_SCHED_CLASS_EXT -- BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); -- BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); - #endif -- - wait_bit_init(); - - #ifdef CONFIG_FAIR_GROUP_SCHED -@@ -10392,8 +10530,9 @@ void __init sched_init(void) - balance_push_set(smp_processor_id(), false); - #endif - init_sched_fair_class(); -- init_sched_ext_class(); -- -+#ifdef CONFIG_HMBIRD_SCHED -+ init_sched_hmbird_class(); -+#endif - psi_init(); - - init_uclamp(); -@@ -10863,6 +11002,11 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) - struct task_group *tg = css_tg(css); - struct task_group *parent = css_tg(css->parent); - -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_tg_online(tg); -+#endif -+ -+ - if (parent) - sched_online_group(tg, parent); - -@@ -10912,13 +11056,77 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) - } - #endif - -+#ifdef CONFIG_HMBIRD_SCHED -+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 dl) -+{ -+ int i; -+ -+ for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) { -+ if (dl < HMBIRD_BPF_DSQS_DEADLINE[i]) -+ break; -+ } -+ -+ i = max_t(int, i-1, MIN_CGROUP_DL_IDX); -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return 0; -+ update_cgroup_ids_table(css->cgroup->kn->id, i); -+ -+ return 0; -+} -+ -+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ u8 i; -+ -+ if (!css || !css->cgroup || !css->cgroup->kn) -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX]; -+ i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1); -+ if (i < 0) { -+ pr_err(" <%s> i is %d, less than 0, name is %s\n", -+ __func__, i, css->cgroup->kn->name); -+ i = DEFAULT_CGROUP_DL_IDX; -+ } -+ -+ return (u64) HMBIRD_BPF_DSQS_DEADLINE[i]; -+} -+ -+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css, -+ struct task_struct *p, int prio) -+{ -+ if (!css || !rt_prio(prio)) -+ return; -+ -+ if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) { -+ if (!strcmp(css->cgroup->kn->name, "display")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3); -+ else if (!strcmp(css->cgroup->kn->name, "touch")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2); -+ else if (!strcmp(css->cgroup->kn->name, "multimedia")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+ } -+} -+#endif -+ - static void cpu_cgroup_attach(struct cgroup_taskset *tset) - { - struct task_struct *task; - struct cgroup_subsys_state *css; - - cgroup_taskset_for_each(task, css, tset) { -- -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_change_rt_sched_prop(css, task, task->prio); -+#endif - sched_move_task(task); - } - -@@ -11600,7 +11808,13 @@ static struct cftype cpu_legacy_files[] = { - .write_u64 = cpu_uclamp_ls_write_u64, - }, - #endif -- -+#ifdef CONFIG_HMBIRD_SCHED -+ { -+ .name = "hmbird.deadline", -+ .read_u64 = cgroup_read_hmbird_deadline, -+ .write_u64 = cgroup_write_hmbird_deadline, -+ }, -+#endif - { } /* Terminate */ - }; - -@@ -11649,7 +11863,6 @@ static int cpu_local_stat_show(struct seq_file *sf, - } - - #ifdef CONFIG_FAIR_GROUP_SCHED -- - static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, - struct cftype *cft) - { -diff --git b/kernel/sched/debug.c a/kernel/sched/debug.c -index a6c99df9edf9..e09904f4d6e5 100755 ---- b/kernel/sched/debug.c -+++ a/kernel/sched/debug.c -@@ -375,9 +375,8 @@ static __init int sched_init_debug(void) - #endif - - debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); -- --#ifdef CONFIG_SCHED_CLASS_EXT -- debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops); -+#ifdef CONFIG_HMBIRD_SCHED -+ debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops); - #endif - return 0; - } -@@ -1006,9 +1005,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - SEQ_printf(m, - "---------------------------------------------------------" - "----------\n"); --#ifdef CONFIG_SLIM_SCHED -- SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop); --#endif - - #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) - #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) -@@ -1104,8 +1100,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - } else if (fair_policy(p->policy)) { - P(se.slice); - } --#ifdef CONFIG_SCHED_CLASS_EXT -- __PS("ext.enabled", p->sched_class == &ext_sched_class); -+#ifdef CONFIG_HMBIRD_SCHED -+ __PS("hmbird.enabled", p->sched_class == &hmbird_sched_class); -+ __PS("sched_prop", get_hmbird_ts(p)->sched_prop); -+ __PS("top_task_prop", get_hmbird_ts(p)->top_task_prop); - #endif - #undef PN_SCHEDSTAT - #undef P_SCHEDSTAT -diff --git b/kernel/sched/ext.c a/kernel/sched/ext.c -deleted file mode 100755 -index 4845d441df2b..000000000000 ---- b/kernel/sched/ext.c -+++ /dev/null -@@ -1,3978 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -- --enum scx_internal_consts { -- SCX_NR_ONLINE_OPS = SCX_OP_IDX(init), -- SCX_DSP_DFL_MAX_BATCH = 32, -- SCX_DSP_MAX_LOOPS = 32, -- SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, --}; -- --enum scx_ops_enable_state { -- SCX_OPS_PREPPING, -- SCX_OPS_ENABLING, -- SCX_OPS_ENABLED, -- SCX_OPS_DISABLING, -- SCX_OPS_DISABLED, --}; -- --static inline struct task_group *css_tg(struct cgroup_subsys_state *css) --{ -- return css ? container_of(css, struct task_group, css) : NULL; --} -- --/* -- * sched_ext_entity->ops_state -- * -- * Used to track the task ownership between the SCX core and the BPF scheduler. -- * State transitions look as follows: -- * -- * NONE -> QUEUEING -> QUEUED -> DISPATCHING -- * ^ | | -- * | v v -- * \-------------------------------/ -- * -- * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -- * sites for explanations on the conditions being waited upon and why they are -- * safe. Transitions out of them into NONE or QUEUED must store_release and the -- * waiters should load_acquire. -- * -- * Tracking scx_ops_state enables sched_ext core to reliably determine whether -- * any given task can be dispatched by the BPF scheduler at all times and thus -- * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -- * to try to dispatch any task anytime regardless of its state as the SCX core -- * can safely reject invalid dispatches. -- */ --enum scx_ops_state { -- SCX_OPSS_NONE, /* owned by the SCX core */ -- SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -- SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ -- SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ -- -- /* -- * QSEQ brands each QUEUED instance so that, when dispatch races -- * dequeue/requeue, the dispatcher can tell whether it still has a claim -- * on the task being dispatched. -- */ -- SCX_OPSS_QSEQ_SHIFT = 2, -- SCX_OPSS_STATE_MASK = (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1, -- SCX_OPSS_QSEQ_MASK = ~SCX_OPSS_STATE_MASK, --}; -- --/* -- * During exit, a task may schedule after losing its PIDs. When disabling the -- * BPF scheduler, we need to be able to iterate tasks in every state to -- * guarantee system safety. Maintain a dedicated task list which contains every -- * task between its fork and eventual free. -- */ --static DEFINE_SPINLOCK(scx_tasks_lock); --static LIST_HEAD(scx_tasks); -- --/* ops enable/disable */ --static struct kthread_worker *scx_ops_helper; --static DEFINE_MUTEX(scx_ops_enable_mutex); --DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); --DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); --static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); --static bool scx_switch_all_req; --static bool scx_switching_all; --DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -- --static struct sched_ext_ops scx_ops; --static bool scx_warned_zero_slice; -- --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); --static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); --DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); --static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); -- --struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] = -- { [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT }; -- --static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE); --static struct scx_exit_info scx_exit_info; -- --static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0); -- --/* -- * The maximum amount of time in jiffies that a task may be runnable without -- * being scheduled on a CPU. If this timeout is exceeded, it will trigger -- * scx_ops_error(). -- */ --unsigned long scx_watchdog_timeout; -- --/* -- * The last time the delayed work was run. This delayed work relies on -- * ksoftirqd being able to run to service timer interrupts, so it's possible -- * that this work itself could get wedged. To account for this, we check that -- * it's not stalled in the timer tick, and trigger an error if it is. -- */ --unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; -- --static struct delayed_work scx_watchdog_work; -- --/* idle tracking */ --#ifdef CONFIG_SMP --#ifdef CONFIG_CPUMASK_OFFSTACK --#define CL_ALIGNED_IF_ONSTACK --#else --#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp --#endif -- --static struct { -- cpumask_var_t cpu; -- cpumask_var_t smt; --} idle_masks CL_ALIGNED_IF_ONSTACK; -- --static bool __cacheline_aligned_in_smp scx_has_idle_cpus; --#endif /* CONFIG_SMP */ -- --/* for %SCX_KICK_WAIT */ --static u64 __percpu *scx_kick_cpus_pnt_seqs; -- --/* -- * Direct dispatch marker. -- * -- * Non-NULL values are used for direct dispatch from enqueue path. A valid -- * pointer points to the task currently being enqueued. An ERR_PTR value is used -- * to indicate that direct dispatch has already happened. -- */ --static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); -- --/* dispatch queues */ --static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; -- --static LLIST_HEAD(dsqs_to_free); -- --/* dispatch buf */ --struct scx_dsp_buf_ent { -- struct task_struct *task; -- u64 qseq; -- u64 dsq_id; -- u64 enq_flags; --}; -- --static u32 scx_dsp_max_batch; --static struct scx_dsp_buf_ent __percpu *scx_dsp_buf; -- --struct scx_dsp_ctx { -- struct rq *rq; -- struct rq_flags *rf; -- u32 buf_cursor; -- u32 nr_tasks; --}; -- --static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx); -- --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags); --void scx_bpf_kick_cpu(s32 cpu, u64 flags); -- --struct scx_task_iter { -- struct sched_ext_entity cursor; -- struct task_struct *locked; -- struct rq *rq; -- struct rq_flags rf; --}; -- --#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) -- --/* if the highest set bit is N, return a mask with bits [N+1, 31] set */ --static u32 higher_bits(u32 flags) --{ -- return ~((1 << fls(flags)) - 1); --} -- --/* return the mask with only the highest bit set */ --static u32 highest_bit(u32 flags) --{ -- int bit = fls(flags); -- return bit ? 1 << (bit - 1) : 0; --} -- --/* -- * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX -- * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate -- * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check -- * whether it's running from an allowed context. -- * -- * @mask is constant, always inline to cull the mask calculations. -- */ --static __always_inline void scx_kf_allow(u32 mask) --{ -- /* nesting is allowed only in increasing scx_kf_mask order */ -- WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask, -- "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n", -- current->scx->kf_mask, mask); -- current->scx->kf_mask |= mask; --} -- --static void scx_kf_disallow(u32 mask) --{ -- current->scx->kf_mask &= ~mask; --} -- --#define SCX_CALL_OP(mask, op, args...) \ --do { \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- scx_ops.op(args); \ -- } \ --} while (0) -- --#define SCX_CALL_OP_RET(mask, op, args...) \ --({ \ -- __typeof__(scx_ops.op(args)) __ret; \ -- if (mask) { \ -- scx_kf_allow(mask); \ -- __ret = scx_ops.op(args); \ -- scx_kf_disallow(mask); \ -- } else { \ -- __ret = scx_ops.op(args); \ -- } \ -- __ret; \ --}) -- --/* -- * Some kfuncs are allowed only on the tasks that are subjects of the -- * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such -- * restrictions, the following SCX_CALL_OP_*() variants should be used when -- * invoking scx_ops operations that take task arguments. These can only be used -- * for non-nesting operations due to the way the tasks are tracked. -- * -- * kfuncs which can only operate on such tasks can in turn use -- * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on -- * the specific task. -- */ --#define SCX_CALL_OP_TASK(mask, op, task, args...) \ --do { \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- SCX_CALL_OP(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ --} while (0) -- --#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ --({ \ -- __typeof__(scx_ops.op(task, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task; \ -- __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- __ret; \ --}) -- --#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ --({ \ -- __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ -- BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ -- current->scx->kf_tasks[0] = task0; \ -- current->scx->kf_tasks[1] = task1; \ -- __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ -- current->scx->kf_tasks[0] = NULL; \ -- current->scx->kf_tasks[1] = NULL; \ -- __ret; \ --}) -- --//#include "./slim_walt.c" -- --/* @mask is constant, always inline to cull unnecessary branches */ --static __always_inline bool scx_kf_allowed(u32 mask) --{ -- if (unlikely(!(current->scx->kf_mask & mask))) { -- scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", -- mask, current->scx->kf_mask); -- return false; -- } -- -- if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) && -- in_interrupt())) { -- scx_ops_error("sleepable kfunc called from non-sleepable context"); -- return false; -- } -- -- /* -- * Enforce nesting boundaries. e.g. A kfunc which can be called from -- * DISPATCH must not be called if we're running DEQUEUE which is nested -- * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE -- * boundary thanks to the above in_interrupt() check. -- */ -- if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && -- (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { -- scx_ops_error("cpu_release kfunc called from a nested operation"); -- return false; -- } -- -- if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && -- (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) { -- scx_ops_error("dispatch kfunc called from a nested operation"); -- return false; -- } -- -- return true; --} -- --/* see SCX_CALL_OP_TASK() */ --static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, -- struct task_struct *p) --{ -- if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED)) -- return false; -- -- if (unlikely((p != current->scx->kf_tasks[0] && -- p != current->scx->kf_tasks[1]))) { -- scx_ops_error("called on a task not being operated on"); -- return false; -- } -- -- return true; --} -- --/** -- * scx_task_iter_init - Initialize a task iterator -- * @iter: iterator to init -- * -- * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized, -- * @iter must eventually be exited with scx_task_iter_exit(). -- * -- * scx_tasks_lock may be released between this and the first next() call or -- * between any two next() calls. If scx_tasks_lock is released between two -- * next() calls, the caller is responsible for ensuring that the task being -- * iterated remains accessible either through RCU read lock or obtaining a -- * reference count. -- * -- * All tasks which existed when the iteration started are guaranteed to be -- * visited as long as they still exist. -- */ --static void scx_task_iter_init(struct scx_task_iter *iter) --{ -- lockdep_assert_held(&scx_tasks_lock); -- -- iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; -- list_add(&iter->cursor.tasks_node, &scx_tasks); -- iter->locked = NULL; --} -- --/** -- * scx_task_iter_exit - Exit a task iterator -- * @iter: iterator to exit -- * -- * Exit a previously initialized @iter. Must be called with scx_tasks_lock held. -- * If the iterator holds a task's rq lock, that rq lock is released. See -- * scx_task_iter_init() for details. -- */ --static void scx_task_iter_exit(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- if (list_empty(cursor)) -- return; -- -- list_del_init(cursor); --} -- --/** -- * scx_task_iter_next - Next task -- * @iter: iterator to walk -- * -- * Visit the next task. See scx_task_iter_init() for details. -- */ --static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) --{ -- struct list_head *cursor = &iter->cursor.tasks_node; -- struct sched_ext_entity *pos; -- -- lockdep_assert_held(&scx_tasks_lock); -- -- list_for_each_entry(pos, cursor, tasks_node) { -- if (&pos->tasks_node == &scx_tasks) -- return NULL; -- if (!(pos->flags & SCX_TASK_CURSOR)) { -- list_move(cursor, &pos->tasks_node); -- return pos->task; -- } -- } -- -- /* can't happen, should always terminate at scx_tasks above */ -- BUG(); --} -- --/** -- * scx_task_iter_next_filtered - Next non-idle task -- * @iter: iterator to walk -- * -- * Visit the next non-idle task. See scx_task_iter_init() for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- while ((p = scx_task_iter_next(iter))) { -- if (!is_idle_task(p)) -- return p; -- } -- return NULL; --} -- --/** -- * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked -- * @iter: iterator to walk -- * -- * Visit the next non-idle task with its rq lock held. See scx_task_iter_init() -- * for details. -- */ --static struct task_struct * --scx_task_iter_next_filtered_locked(struct scx_task_iter *iter) --{ -- struct task_struct *p; -- -- if (iter->locked) { -- task_rq_unlock(iter->rq, iter->locked, &iter->rf); -- iter->locked = NULL; -- } -- -- p = scx_task_iter_next_filtered(iter); -- if (!p) -- return NULL; -- -- iter->rq = task_rq_lock(p, &iter->rf); -- iter->locked = p; -- return p; --} -- --static enum scx_ops_enable_state scx_ops_enable_state(void) --{ -- return atomic_read(&scx_ops_enable_state_var); --} -- --static enum scx_ops_enable_state --scx_ops_set_enable_state(enum scx_ops_enable_state to) --{ -- return atomic_xchg(&scx_ops_enable_state_var, to); --} -- --static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, -- enum scx_ops_enable_state from) --{ -- int from_v = from; -- -- return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); --} -- --static bool scx_ops_disabling(void) --{ -- return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING); --} -- --/** -- * wait_ops_state - Busy-wait the specified ops state to end -- * @p: target task -- * @opss: state to wait the end of -- * -- * Busy-wait for @p to transition out of @opss. This can only be used when the -- * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also -- * has load_acquire semantics to ensure that the caller can see the updates made -- * in the enqueueing and dispatching paths. -- */ --static void wait_ops_state(struct task_struct *p, u64 opss) --{ -- do { -- cpu_relax(); -- } while (atomic64_read_acquire(&p->scx->ops_state) == opss); --} -- --/** -- * ops_cpu_valid - Verify a cpu number -- * @cpu: cpu number which came from a BPF ops -- * -- * @cpu is a cpu number which came from the BPF scheduler and can be any value. -- * Verify that it is in range and one of the possible cpus. -- */ --static bool ops_cpu_valid(s32 cpu) --{ -- return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); --} -- --/** -- * ops_sanitize_err - Sanitize a -errno value -- * @ops_name: operation to blame on failure -- * @err: -errno value to sanitize -- * -- * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return -- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can -- * cause misbehaviors. For an example, a large negative return from -- * ops.prep_enable() triggers an oops when passed up the call chain because the -- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is -- * handled as a pointer. -- */ --static int ops_sanitize_err(const char *ops_name, s32 err) --{ -- if (err < 0 && err >= -MAX_ERRNO) -- return err; -- -- scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); -- return -EPROTO; --} -- --/** -- * touch_core_sched - Update timestamp used for core-sched task ordering -- * @rq: rq to read clock from, must be locked -- * @p: task to update the timestamp for -- * -- * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to -- * implement global or local-DSQ FIFO ordering for core-sched. Should be called -- * when a task becomes runnable and its turn on the CPU ends (e.g. slice -- * exhaustion). -- */ --static void touch_core_sched(struct rq *rq, struct task_struct *p) --{ --#ifdef CONFIG_SCHED_CORE -- /* -- * It's okay to update the timestamp spuriously. Use -- * sched_core_disabled() which is cheaper than enabled(). -- */ -- if (!sched_core_disabled()) -- p->scx->core_sched_at = rq_clock_task(rq); --#endif --} -- --/** -- * touch_core_sched_dispatch - Update core-sched timestamp on dispatch -- * @rq: rq to read clock from, must be locked -- * @p: task being dispatched -- * -- * If the BPF scheduler implements custom core-sched ordering via -- * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO -- * ordering within each local DSQ. This function is called from dispatch paths -- * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect. -- */ --static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- assert_clock_updated(rq); -- --#ifdef CONFIG_SCHED_CORE -- if (SCX_HAS_OP(core_sched_before)) -- touch_core_sched(rq, p); --#endif --} -- --static void update_curr_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- u64 now = rq_clock_task(rq); -- u64 delta_exec; -- -- if (time_before_eq64(now, curr->se.exec_start)) -- return; -- -- delta_exec = now - curr->se.exec_start; -- curr->se.exec_start = now; -- curr->se.sum_exec_runtime += delta_exec; -- account_group_exec_runtime(curr, delta_exec); -- cgroup_account_cputime(curr, delta_exec); -- -- if (curr->scx->slice != SCX_SLICE_INF) { -- curr->scx->slice -= min(curr->scx->slice, delta_exec); -- if (!curr->scx->slice) -- touch_core_sched(rq, curr); -- } --} -- --static bool scx_dsq_priq_less(struct rb_node *node_a, -- const struct rb_node *node_b) --{ -- const struct sched_ext_entity *a = -- container_of(node_a, struct sched_ext_entity, dsq_node.priq); -- const struct sched_ext_entity *b = -- container_of(node_b, struct sched_ext_entity, dsq_node.priq); -- -- return time_before64(a->dsq_vtime, b->dsq_vtime); --} -- --static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, -- u64 enq_flags) --{ -- bool is_local = dsq->id == SCX_DSQ_LOCAL; -- -- WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo)); -- WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq)); -- -- if (!is_local) { -- raw_spin_lock(&dsq->lock); -- if (unlikely(dsq->id == SCX_DSQ_INVALID)) { -- scx_ops_error("attempting to dispatch to a destroyed dsq"); -- /* fall back to the global dsq */ -- raw_spin_unlock(&dsq->lock); -- dsq = &scx_dsq_global; -- raw_spin_lock(&dsq->lock); -- } -- } -- -- if (enq_flags & SCX_ENQ_DSQ_PRIQ) { -- p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; -- rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq, -- scx_dsq_priq_less); -- } else { -- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) -- list_add(&p->scx->dsq_node.fifo, &dsq->fifo); -- else -- list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo); -- } -- dsq->nr++; -- p->scx->dsq = dsq; -- -- /* -- * We're transitioning out of QUEUEING or DISPATCHING. store_release to -- * match waiters' load_acquire. -- */ -- if (enq_flags & SCX_ENQ_CLEAR_OPSS) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- if (is_local) { -- struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq); -- struct rq *rq = scx->rq; -- bool preempt = false; -- -- if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && -- rq->curr->sched_class == &ext_sched_class) { -- rq->curr->scx->slice = 0; -- preempt = true; -- } -- -- if (preempt || sched_class_above(&ext_sched_class, -- rq->curr->sched_class)) -- resched_curr(rq); -- } else { -- raw_spin_unlock(&dsq->lock); -- } --} -- --static void task_unlink_from_dsq(struct task_struct *p, -- struct scx_dispatch_q *dsq) --{ -- if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { -- rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; -- } else { -- list_del_init(&p->scx->dsq_node.fifo); -- } -- --} -- --static bool task_linked_on_dsq(struct task_struct *p) --{ -- return !list_empty(&p->scx->dsq_node.fifo) || -- !RB_EMPTY_NODE(&p->scx->dsq_node.priq); --} -- --static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p) --{ -- struct scx_dispatch_q *dsq = p->scx->dsq; -- bool is_local = dsq == &scx_rq->local_dsq; -- -- if (!dsq) { -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- /* -- * When dispatching directly from the BPF scheduler to a local -- * DSQ, the task isn't associated with any DSQ but -- * @p->scx->holding_cpu may be set under the protection of -- * %SCX_OPSS_DISPATCHING. -- */ -- if (p->scx->holding_cpu >= 0) -- p->scx->holding_cpu = -1; -- return; -- } -- -- if (!is_local) -- raw_spin_lock(&dsq->lock); -- -- /* -- * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node -- * can't change underneath us. -- */ -- if (p->scx->holding_cpu < 0) { -- /* @p must still be on @dsq, dequeue */ -- WARN_ON_ONCE(!task_linked_on_dsq(p)); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- } else { -- /* -- * We're racing against dispatch_to_local_dsq() which already -- * removed @p from @dsq and set @p->scx->holding_cpu. Clear the -- * holding_cpu which tells dispatch_to_local_dsq() that it lost -- * the race. -- */ -- WARN_ON_ONCE(task_linked_on_dsq(p)); -- p->scx->holding_cpu = -1; -- } -- p->scx->dsq = NULL; -- -- if (!is_local) -- raw_spin_unlock(&dsq->lock); --} -- --static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) --{ -- lockdep_assert(rcu_read_lock_any_held()); -- -- return &scx_dsq_global; --} -- --static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, -- struct task_struct *p) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id == SCX_DSQ_LOCAL) -- return &rq->scx->local_dsq; -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", -- dsq_id, p->comm, p->pid); -- return &scx_dsq_global; -- } -- -- return dsq; --} -- --static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- -- /* @p must match the task which is being enqueued */ -- if (unlikely(p != ddsp_task)) { -- if (IS_ERR(ddsp_task)) -- scx_ops_error("%s[%d] already direct-dispatched", -- p->comm, p->pid); -- else -- scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]", -- ddsp_task->comm, ddsp_task->pid, -- p->comm, p->pid); -- return; -- } -- -- /* -- * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because -- * dispatching to the local DSQ of a different CPU requires unlocking -- * the current rq which isn't allowed in the enqueue path. Use -- * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL. -- */ -- if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) { -- scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch"); -- return; -- } -- -- touch_core_sched_dispatch(task_rq(p), p); -- -- dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- -- /* -- * Mark that dispatch already happened by spoiling direct_dispatch_task -- * with a non-NULL value which can never match a valid task pointer. -- */ -- __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); --} -- --static bool test_rq_online(struct rq *rq) --{ --#ifdef CONFIG_SMP -- return rq->online; --#else -- return true; --#endif --} -- --static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- int sticky_cpu) --{ -- struct task_struct **ddsp_taskp; -- u64 qseq; -- -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- if (p->scx->flags & SCX_TASK_ENQ_LOCAL) { -- enq_flags |= SCX_ENQ_LOCAL; -- p->scx->flags &= ~SCX_TASK_ENQ_LOCAL; -- } -- -- /* rq migration */ -- if (sticky_cpu == cpu_of(rq)) -- goto local_norefill; -- -- /* -- * If !rq->online, we already told the BPF scheduler that the CPU is -- * offline. We're just trying to on/offline the CPU. Don't bother the -- * BPF scheduler. -- */ -- if (unlikely(!test_rq_online(rq))) -- goto local; -- -- /* see %SCX_OPS_ENQ_EXITING */ -- if (!static_branch_unlikely(&scx_ops_enq_exiting) && -- unlikely(p->flags & PF_EXITING)) -- goto local; -- -- /* see %SCX_OPS_ENQ_LAST */ -- if (!static_branch_unlikely(&scx_ops_enq_last) && -- (enq_flags & SCX_ENQ_LAST)) -- goto local; -- -- if (!SCX_HAS_OP(enqueue)) { -- if (enq_flags & SCX_ENQ_LOCAL) -- goto local; -- else -- goto global; -- } -- -- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ -- qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; -- -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq); -- -- ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); -- WARN_ON_ONCE(*ddsp_taskp); -- *ddsp_taskp = p; -- -- SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); -- -- /* -- * If not directly dispatched, QUEUEING isn't clear yet and dispatch or -- * dequeue may be waiting. The store_release matches their load_acquire. -- */ -- if (*ddsp_taskp == p) -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq); -- *ddsp_taskp = NULL; -- return; -- --local: -- /* -- * For task-ordering, slice refill must be treated as implying the end -- * of the current slice. Otherwise, the longer @p stays on the CPU, the -- * higher priority it becomes from scx_prio_less()'s POV. -- */ -- touch_core_sched(rq, p); -- p->scx->slice = SCX_SLICE_DFL; --local_norefill: -- dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags); -- return; -- --global: -- touch_core_sched(rq, p); /* see the comment in local: */ -- p->scx->slice = SCX_SLICE_DFL; -- dispatch_enqueue(&scx_dsq_global, p, enq_flags); --} -- --static bool watchdog_task_watched(const struct task_struct *p) --{ -- return !list_empty(&p->scx->watchdog_node); --} -- --static void watchdog_watch_task(struct rq *rq, struct task_struct *p) --{ -- lockdep_assert_rq_held(rq); -- if (p->scx->flags & SCX_TASK_WATCHDOG_RESET) -- p->scx->runnable_at = jiffies; -- p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET; -- list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list); --} -- --static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) --{ -- list_del_init(&p->scx->watchdog_node); -- if (reset_timeout) -- p->scx->flags |= SCX_TASK_WATCHDOG_RESET; --} -- --static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) --{ -- int sticky_cpu = p->scx->sticky_cpu; -- -- enq_flags |= rq->scx->extra_enq_flags; -- -- if (sticky_cpu >= 0) -- p->scx->sticky_cpu = -1; -- -- /* -- * Restoring a running task will be immediately followed by -- * set_next_task_scx() which expects the task to not be on the BPF -- * scheduler as tasks can only start running through local DSQs. Force -- * direct-dispatch into the local DSQ by setting the sticky_cpu. -- */ -- if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -- sticky_cpu = cpu_of(rq); -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- WARN_ON_ONCE(!watchdog_task_watched(p)); -- return; -- } -- -- watchdog_watch_task(rq, p); -- p->scx->flags |= SCX_TASK_QUEUED; -- rq->scx->nr_running++; -- add_nr_running(rq, 1); -- -- if (SCX_HAS_OP(runnable)) -- SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); -- -- if (enq_flags & SCX_ENQ_WAKEUP) -- touch_core_sched(rq, p); -- -- do_enqueue_task(rq, p, enq_flags, sticky_cpu); --} -- --static void ops_dequeue(struct task_struct *p, u64 deq_flags) --{ -- u64 opss; -- -- watchdog_unwatch_task(p, false); -- -- /* acquire ensures that we see the preceding updates on QUEUED */ -- opss = atomic64_read_acquire(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_NONE: -- break; -- case SCX_OPSS_QUEUEING: -- /* -- * QUEUEING is started and finished while holding @p's rq lock. -- * As we're holding the rq lock now, we shouldn't see QUEUEING. -- */ -- BUG(); -- case SCX_OPSS_QUEUED: -- if (SCX_HAS_OP(dequeue)) -- SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); -- -- if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_NONE)) -- break; -- fallthrough; -- case SCX_OPSS_DISPATCHING: -- /* -- * If @p is being dispatched from the BPF scheduler to a DSQ, -- * wait for the transfer to complete so that @p doesn't get -- * added to its DSQ after dequeueing is complete. -- * -- * As we're waiting on DISPATCHING with the rq locked, the -- * dispatching side shouldn't try to lock the rq while -- * DISPATCHING is set. See dispatch_to_local_dsq(). -- * -- * DISPATCHING shouldn't have qseq set and control can reach -- * here with NONE @opss from the above QUEUED case block. -- * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. -- */ -- wait_ops_state(p, SCX_OPSS_DISPATCHING); -- BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- break; -- } --} -- --static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) --{ -- struct scx_rq *scx_rq = rq->scx; -- -- if (!(p->scx->flags & SCX_TASK_QUEUED)) { -- WARN_ON_ONCE(watchdog_task_watched(p)); -- return; -- } -- -- ops_dequeue(p, deq_flags); -- -- /* -- * A currently running task which is going off @rq first gets dequeued -- * and then stops running. As we want running <-> stopping transitions -- * to be contained within runnable <-> quiescent transitions, trigger -- * ->stopping() early here instead of in put_prev_task_scx(). -- * -- * @p may go through multiple stopping <-> running transitions between -- * here and put_prev_task_scx() if task attribute changes occur while -- * balance_scx() leaves @rq unlocked. However, they don't contain any -- * information meaningful to the BPF scheduler and can be suppressed by -- * skipping the callbacks if the task is !QUEUED. -- */ -- if (SCX_HAS_OP(stopping) && task_current(rq, p)) { -- update_curr_scx(rq); -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); -- } -- -- //if(task_current(rq, p)) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- if (SCX_HAS_OP(quiescent)) -- SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); -- -- if (deq_flags & SCX_DEQ_SLEEP) -- p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP; -- else -- p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP; -- -- p->scx->flags &= ~SCX_TASK_QUEUED; -- BUG_ON(!scx_rq->nr_running); -- scx_rq->nr_running--; -- sub_nr_running(rq, 1); -- -- dispatch_dequeue(scx_rq, p); --} -- --static void yield_task_scx(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); -- else -- p->scx->slice = 0; --} -- --static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) --{ -- struct task_struct *from = rq->curr; -- -- if (SCX_HAS_OP(yield)) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); -- else -- return false; --} -- --#ifdef CONFIG_SMP --/** -- * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -- * @rq: rq to move the task into, currently locked -- * @p: task to move -- * @enq_flags: %SCX_ENQ_* -- * -- * Move @p which is currently on a different rq to @rq's local DSQ. The caller -- * must: -- * -- * 1. Start with exclusive access to @p either through its DSQ lock or -- * %SCX_OPSS_DISPATCHING flag. -- * -- * 2. Set @p->scx->holding_cpu to raw_smp_processor_id(). -- * -- * 3. Remember task_rq(@p). Release the exclusive access so that we don't -- * deadlock with dequeue. -- * -- * 4. Lock @rq and the task_rq from #3. -- * -- * 5. Call this function. -- * -- * Returns %true if @p was successfully moved. %false after racing dequeue and -- * losing. -- */ --static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -- u64 enq_flags) --{ -- struct rq *task_rq; -- -- lockdep_assert_rq_held(rq); -- -- /* -- * If dequeue got to @p while we were trying to lock both rq's, it'd -- * have cleared @p->scx->holding_cpu to -1. While other cpus may have -- * updated it to different values afterwards, as this operation can't be -- * preempted or recurse, @p->scx->holding_cpu can never become -- * raw_smp_processor_id() again before we're done. Thus, we can tell -- * whether we lost to dequeue by testing whether @p->scx->holding_cpu is -- * still raw_smp_processor_id(). -- * -- * See dispatch_dequeue() for the counterpart. -- */ -- if (unlikely(p->scx->holding_cpu != raw_smp_processor_id())) -- return false; -- -- /* @p->rq couldn't have changed if we're still the holding cpu */ -- task_rq = task_rq(p); -- lockdep_assert_rq_held(task_rq); -- -- WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)); -- deactivate_task(task_rq, p, 0); -- set_task_cpu(p, cpu_of(rq)); -- p->scx->sticky_cpu = cpu_of(rq); -- -- /* -- * We want to pass scx-specific enq_flags but activate_task() will -- * truncate the upper 32 bit. As we own @rq, we can pass them through -- * @rq->scx->extra_enq_flags instead. -- */ -- WARN_ON_ONCE(rq->scx->extra_enq_flags); -- rq->scx->extra_enq_flags = enq_flags; -- activate_task(rq, p, 0); -- rq->scx->extra_enq_flags = 0; -- -- return true; --} -- --/** -- * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * We're holding @rq lock and trying to dispatch a task from @src_rq to -- * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether -- * @rq stays locked isn't important as long as the state is restored after -- * dispatch_to_local_dsq_unlock(). -- */ --static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- rq_unpin_lock(rq, rf); -- -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(rq); -- raw_spin_rq_lock(dst_rq); -- } else if (rq == src_rq) { -- double_lock_balance(rq, dst_rq); -- rq_repin_lock(rq, rf); -- } else if (rq == dst_rq) { -- double_lock_balance(rq, src_rq); -- rq_repin_lock(rq, rf); -- } else { -- raw_spin_rq_unlock(rq); -- double_rq_lock(src_rq, dst_rq); -- } --} -- --/** -- * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock() -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @src_rq: rq to move task from -- * @dst_rq: rq to move task to -- * -- * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return. -- */ --static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf, -- struct rq *src_rq, struct rq *dst_rq) --{ -- if (src_rq == dst_rq) { -- raw_spin_rq_unlock(dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } else if (rq == src_rq) { -- double_unlock_balance(rq, dst_rq); -- } else if (rq == dst_rq) { -- double_unlock_balance(rq, src_rq); -- } else { -- double_rq_unlock(src_rq, dst_rq); -- raw_spin_rq_lock(rq); -- rq_repin_lock(rq, rf); -- } --} --#endif /* CONFIG_SMP */ -- -- --static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq) --{ -- return likely(test_rq_online(rq)) && !is_migration_disabled(p) && -- cpumask_test_cpu(cpu_of(rq), p->cpus_ptr); --} -- --static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -- struct scx_dispatch_q *dsq) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rb_node *rb_node; -- struct rq *task_rq; -- bool moved = false; --retry: -- if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -- return false; -- -- raw_spin_lock(&dsq->lock); -- -- list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- for (rb_node = rb_first_cached(&dsq->priq); rb_node; -- rb_node = rb_next(rb_node)) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- p = entity->task; -- task_rq = task_rq(p); -- if (rq == task_rq) -- goto this_rq; -- if (task_can_run_on_rq(p, rq)) -- goto remote_rq; -- } -- -- raw_spin_unlock(&dsq->lock); -- return false; -- --this_rq: -- /* @dsq is locked and @p is on this rq */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo); -- dsq->nr--; -- scx_rq->local_dsq.nr++; -- p->scx->dsq = &scx_rq->local_dsq; -- raw_spin_unlock(&dsq->lock); -- return true; -- --remote_rq: --#ifdef CONFIG_SMP -- /* -- * @dsq is locked and @p is on a remote rq. @p is currently protected by -- * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -- * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -- * rq lock or fail, do a little dancing from our side. See -- * move_task_to_local_dsq(). -- */ -- WARN_ON_ONCE(p->scx->holding_cpu >= 0); -- task_unlink_from_dsq(p, dsq); -- dsq->nr--; -- p->scx->holding_cpu = raw_smp_processor_id(); -- raw_spin_unlock(&dsq->lock); -- -- rq_unpin_lock(rq, rf); -- double_lock_balance(rq, task_rq); -- rq_repin_lock(rq, rf); -- -- moved = move_task_to_local_dsq(rq, p, 0); -- -- double_unlock_balance(rq, task_rq); --#endif /* CONFIG_SMP */ -- if (likely(moved)) -- return true; -- goto retry; --} -- --enum dispatch_to_local_dsq_ret { -- DTL_DISPATCHED, /* successfully dispatched */ -- DTL_LOST, /* lost race to dequeue */ -- DTL_NOT_LOCAL, /* destination is not a local DSQ */ -- DTL_INVALID, /* invalid local dsq_id */ --}; -- --/** -- * dispatch_to_local_dsq - Dispatch a task to a local dsq -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @dsq_id: destination dsq ID -- * @p: task to dispatch -- * @enq_flags: %SCX_ENQ_* -- * -- * We're holding @rq lock and want to dispatch @p to the local DSQ identified by -- * @dsq_id. This function performs all the synchronization dancing needed -- * because local DSQs are protected with rq locks. -- * -- * The caller must have exclusive ownership of @p (e.g. through -- * %SCX_OPSS_DISPATCHING). -- */ --static enum dispatch_to_local_dsq_ret --dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id, -- struct task_struct *p, u64 enq_flags) --{ -- struct rq *src_rq = task_rq(p); -- struct rq *dst_rq; -- -- /* -- * We're synchronized against dequeue through DISPATCHING. As @p can't -- * be dequeued, its task_rq and cpus_allowed are stable too. -- */ -- if (dsq_id == SCX_DSQ_LOCAL) { -- dst_rq = rq; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]", -- cpu, p->comm, p->pid); -- return DTL_INVALID; -- } -- dst_rq = cpu_rq(cpu); -- } else { -- return DTL_NOT_LOCAL; -- } -- -- /* if dispatching to @rq that @p is already on, no lock dancing needed */ -- if (rq == src_rq && rq == dst_rq) { -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags | SCX_ENQ_CLEAR_OPSS); -- return DTL_DISPATCHED; -- } -- --#ifdef CONFIG_SMP -- if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { -- struct rq *locked_dst_rq = dst_rq; -- bool dsp; -- -- /* -- * @p is on a possibly remote @src_rq which we need to lock to -- * move the task. If dequeue is in progress, it'd be locking -- * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq -- * lock while holding DISPATCHING. -- * -- * As DISPATCHING guarantees that @p is wholly ours, we can -- * pretend that we're moving from a DSQ and use the same -- * mechanism - mark the task under transfer with holding_cpu, -- * release DISPATCHING and then follow the same protocol. -- */ -- p->scx->holding_cpu = raw_smp_processor_id(); -- -- /* store_release ensures that dequeue sees the above */ -- atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); -- -- dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq); -- -- /* -- * We don't require the BPF scheduler to avoid dispatching to -- * offline CPUs mostly for convenience but also because CPUs can -- * go offline between scx_bpf_dispatch() calls and here. If @p -- * is destined to an offline CPU, queue it on its current CPU -- * instead, which should always be safe. As this is an allowed -- * behavior, don't trigger an ops error. -- */ -- if (unlikely(!test_rq_online(dst_rq))) -- dst_rq = src_rq; -- -- if (src_rq == dst_rq) { -- /* -- * As @p is staying on the same rq, there's no need to -- * go through the full deactivate/activate cycle. -- * Optimize by abbreviating the operations in -- * move_task_to_local_dsq(). -- */ -- dsp = p->scx->holding_cpu == raw_smp_processor_id(); -- if (likely(dsp)) { -- p->scx->holding_cpu = -1; -- dispatch_enqueue(&dst_rq->scx->local_dsq, p, -- enq_flags); -- } -- } else { -- dsp = move_task_to_local_dsq(dst_rq, p, enq_flags); -- } -- -- /* if the destination CPU is idle, wake it up */ -- if (dsp && p->sched_class > dst_rq->curr->sched_class) -- resched_curr(dst_rq); -- -- dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq); -- -- return dsp ? DTL_DISPATCHED : DTL_LOST; -- } --#endif /* CONFIG_SMP */ -- -- scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", -- cpu_of(dst_rq), p->comm, p->pid); -- return DTL_INVALID; --} -- --/** -- * finish_dispatch - Asynchronously finish dispatching a task -- * @rq: current rq which is locked -- * @rf: rq_flags to use when unlocking @rq -- * @p: task to finish dispatching -- * @qseq_at_dispatch: qseq when @p started getting dispatched -- * @dsq_id: destination DSQ ID -- * @enq_flags: %SCX_ENQ_* -- * -- * Dispatching to local DSQs may need to wait for queueing to complete or -- * require rq lock dancing. As we don't wanna do either while inside -- * ops.dispatch() to avoid locking order inversion, we split dispatching into -- * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the -- * task and its qseq. Once ops.dispatch() returns, this function is called to -- * finish up. -- * -- * There is no guarantee that @p is still valid for dispatching or even that it -- * was valid in the first place. Make sure that the task is still owned by the -- * BPF scheduler and claim the ownership before dispatching. -- */ --static void finish_dispatch(struct rq *rq, struct rq_flags *rf, -- struct task_struct *p, u64 qseq_at_dispatch, -- u64 dsq_id, u64 enq_flags) --{ -- struct scx_dispatch_q *dsq; -- u64 opss; -- -- touch_core_sched_dispatch(rq, p); --retry: -- /* -- * No need for _acquire here. @p is accessed only after a successful -- * try_cmpxchg to DISPATCHING. -- */ -- opss = atomic64_read(&p->scx->ops_state); -- -- switch (opss & SCX_OPSS_STATE_MASK) { -- case SCX_OPSS_DISPATCHING: -- case SCX_OPSS_NONE: -- /* someone else already got to it */ -- return; -- case SCX_OPSS_QUEUED: -- /* -- * If qseq doesn't match, @p has gone through at least one -- * dispatch/dequeue and re-enqueue cycle between -- * scx_bpf_dispatch() and here and we have no claim on it. -- */ -- if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) -- return; -- -- /* -- * While we know @p is accessible, we don't yet have a claim on -- * it - the BPF scheduler is allowed to dispatch tasks -- * spuriously and there can be a racing dequeue attempt. Let's -- * claim @p by atomically transitioning it from QUEUED to -- * DISPATCHING. -- */ -- if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss, -- SCX_OPSS_DISPATCHING))) -- break; -- goto retry; -- case SCX_OPSS_QUEUEING: -- /* -- * do_enqueue_task() is in the process of transferring the task -- * to the BPF scheduler while holding @p's rq lock. As we aren't -- * holding any kernel or BPF resource that the enqueue path may -- * depend upon, it's safe to wait. -- */ -- wait_ops_state(p, opss); -- goto retry; -- } -- -- BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED)); -- -- switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) { -- case DTL_DISPATCHED: -- break; -- case DTL_LOST: -- break; -- case DTL_INVALID: -- dsq_id = SCX_DSQ_GLOBAL; -- fallthrough; -- case DTL_NOT_LOCAL: -- dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()), -- dsq_id, p); -- dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -- break; -- } --} -- --static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- u32 u; -- -- for (u = 0; u < dspc->buf_cursor; u++) { -- struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u]; -- -- finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id, -- ent->enq_flags); -- } -- -- dspc->nr_tasks += dspc->buf_cursor; -- dspc->buf_cursor = 0; --} -- --static int balance_one(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf, bool local) --{ -- struct scx_rq *scx_rq = rq->scx; -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- bool prev_on_scx = prev->sched_class == &ext_sched_class; -- int nr_loops = SCX_DSP_MAX_LOOPS; -- -- lockdep_assert_rq_held(rq); -- -- if (static_branch_unlikely(&scx_ops_cpu_preempt) && -- unlikely(rq->scx->cpu_released)) { -- /* -- * If the previous sched_class for the current CPU was not SCX, -- * notify the BPF scheduler that it again has control of the -- * core. This callback complements ->cpu_release(), which is -- * emitted in scx_notify_pick_next_task(). -- */ -- if (SCX_HAS_OP(cpu_acquire)) -- SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq), -- NULL); -- rq->scx->cpu_released = false; -- } -- -- if (prev_on_scx) { -- WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP)); -- update_curr_scx(rq); -- -- /* -- * If @prev is runnable & has slice left, it has priority and -- * fetching more just increases latency for the fetched tasks. -- * Tell put_prev_task_scx() to put @prev on local_dsq. If the -- * BPF scheduler wants to handle this explicitly, it should -- * implement ->cpu_released(). -- * -- * See scx_ops_disable_workfn() for the explanation on the -- * disabling() test. -- * -- * When balancing a remote CPU for core-sched, there won't be a -- * following put_prev_task_scx() call and we don't own -- * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the -- * same conditions later and pick @rq->curr accordingly. -- */ -- if ((prev->scx->flags & SCX_TASK_QUEUED) && -- prev->scx->slice && !scx_ops_disabling()) { -- if (local) -- prev->scx->flags |= SCX_TASK_BAL_KEEP; -- return 1; -- } -- } -- -- /* if there already are tasks to run, nothing to do */ -- if (scx_rq->local_dsq.nr) -- return 1; -- -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- if (!SCX_HAS_OP(dispatch)) -- return 0; -- -- dspc->rq = rq; -- dspc->rf = rf; -- -- /* -- * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, -- * the local DSQ might still end up empty after a successful -- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() -- * produced some tasks, retry. The BPF scheduler may depend on this -- * looping behavior to simplify its implementation. -- */ -- do { -- dspc->nr_tasks = 0; -- -- SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), -- prev_on_scx ? prev : NULL); -- -- flush_dispatch_buf(rq, rf); -- -- if (scx_rq->local_dsq.nr) -- return 1; -- if (consume_dispatch_q(rq, rf, &scx_dsq_global)) -- return 1; -- -- /* -- * ops.dispatch() can trap us in this loop by repeatedly -- * dispatching ineligible tasks. Break out once in a while to -- * allow the watchdog to run. As IRQ can't be enabled in -- * balance(), we want to complete this scheduling cycle and then -- * start a new one. IOW, we want to call resched_curr() on the -- * next, most likely idle, task, not the current one. Use -- * scx_bpf_kick_cpu() for deferred kicking. -- */ -- if (unlikely(!--nr_loops)) { -- scx_bpf_kick_cpu(cpu_of(rq), 0); -- break; -- } -- } while (dspc->nr_tasks); -- -- return 0; --} -- --static int balance_scx(struct rq *rq, struct task_struct *prev, -- struct rq_flags *rf) --{ -- int ret; -- -- ret = balance_one(rq, prev, rf, true); --#ifdef CONFIG_SCHED_SMT -- /* -- * When core-sched is enabled, this ops.balance() call will be followed -- * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx() -- * on the SMT siblings. Balance the siblings too. -- */ -- if (sched_core_enabled(rq)) { -- const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); -- int scpu; -- -- for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { -- struct rq *srq = cpu_rq(scpu); -- struct rq_flags srf; -- struct task_struct *sprev = srq->curr; -- -- /* -- * While core-scheduling, rq lock is shared among -- * siblings but the debug annotations and rq clock -- * aren't. Do pinning dance to transfer the ownership. -- */ -- WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); -- rq_unpin_lock(rq, rf); -- rq_pin_lock(srq, &srf); -- -- update_rq_clock(srq); -- balance_one(srq, sprev, &srf, false); -- -- rq_unpin_lock(srq, &srf); -- rq_repin_lock(rq, rf); -- } -- } --#endif -- return ret; --} -- --static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) --{ -- if (p->scx->flags & SCX_TASK_QUEUED) { -- /* -- * Core-sched might decide to execute @p before it is -- * dispatched. Call ops_dequeue() to notify the BPF scheduler. -- */ -- ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); -- dispatch_dequeue(rq->scx, p); -- } -- -- p->se.exec_start = rq_clock_task(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, running, p); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock); -- -- watchdog_unwatch_task(p, true); -- -- /* -- * @p is getting newly scheduled or got kicked after someone updated its -- * slice. Refresh whether tick can be stopped. See can_stop_tick_scx(). -- */ -- if ((p->scx->slice == SCX_SLICE_INF) != -- (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) { -- if (p->scx->slice == SCX_SLICE_INF) -- rq->scx->flags |= SCX_RQ_CAN_STOP_TICK; -- else -- rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK; -- -- sched_update_tick_dependency(rq); -- } --} -- --static void put_prev_task_scx(struct rq *rq, struct task_struct *p) --{ --#ifndef CONFIG_SMP -- /* -- * UP workaround. -- * -- * Because SCX may transfer tasks across CPUs during dispatch, dispatch -- * is performed from its balance operation which isn't called in UP. -- * Let's work around by calling it from the operations which come right -- * after. -- * -- * 1. If the prev task is on SCX, pick_next_task() calls -- * .put_prev_task() right after. As .put_prev_task() is also called -- * from other places, we need to distinguish the calls which can be -- * done by looking at the previous task's state - if still queued or -- * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task(). -- * This case is handled here. -- * -- * 2. If the prev task is not on SCX, the first following call into SCX -- * will be .pick_next_task(), which is covered by calling -- * balance_scx() from pick_next_task_scx(). -- * -- * Note that we can't merge the first case into the second as -- * balance_scx() must be called before the previous SCX task goes -- * through put_prev_task_scx(). -- * -- * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf. -- * Pass in %NULL. -- */ -- if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP)) -- balance_scx(rq, p, NULL); --#endif -- -- update_curr_scx(rq); -- -- /* see dequeue_task_scx() on why we skip when !QUEUED */ -- if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED)) -- SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); -- -- //if(p->scx->flags & SCX_TASK_QUEUED) -- // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); -- -- -- /* -- * If we're being called from put_prev_task_balance(), balance_scx() may -- * have decided that @p should keep running. -- */ -- if (p->scx->flags & SCX_TASK_BAL_KEEP) { -- p->scx->flags &= ~SCX_TASK_BAL_KEEP; -- watchdog_watch_task(rq, p); -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- if (p->scx->flags & SCX_TASK_QUEUED) { -- watchdog_watch_task(rq, p); -- -- /* -- * If @p has slice left and balance_scx() didn't tag it for -- * keeping, @p is getting preempted by a higher priority -- * scheduler class or core-sched forcing a different task. Leave -- * it at the head of the local DSQ. -- */ -- if (p->scx->slice && !scx_ops_disabling()) { -- dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); -- return; -- } -- -- /* -- * If we're in the pick_next_task path, balance_scx() should -- * have already populated the local DSQ if there are any other -- * available tasks. If empty, tell ops.enqueue() that @p is the -- * only one available for this cpu. ops.enqueue() should put it -- * on the local DSQ so that the subsequent pick_next_task_scx() -- * can find the task unless it wants to trigger a separate -- * follow-up scheduling event. -- */ -- if (list_empty(&rq->scx->local_dsq.fifo)) -- do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1); -- else -- do_enqueue_task(rq, p, 0, -1); -- } --} -- --static struct task_struct *first_local_task(struct rq *rq) --{ -- struct rb_node *rb_node; -- struct sched_ext_entity *entity; -- -- if (!list_empty(&rq->scx->local_dsq.fifo)) { -- entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo); -- return entity->task; -- } -- -- rb_node = rb_first_cached(&rq->scx->local_dsq.priq); -- if (rb_node) { -- entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); -- return entity->task; -- } -- -- return NULL; --} -- --static struct task_struct *pick_next_task_scx(struct rq *rq) --{ -- struct task_struct *p; -- --#ifndef CONFIG_SMP -- /* UP workaround - see the comment at the head of put_prev_task_scx() */ -- if (unlikely(rq->curr->sched_class != &ext_sched_class)) -- balance_scx(rq, rq->curr, NULL); --#endif -- -- p = first_local_task(rq); -- if (!p) -- return NULL; -- -- if (unlikely(!p->scx->slice)) { -- if (!scx_ops_disabling() && !scx_warned_zero_slice) { -- printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n", -- p->comm, p->pid); -- scx_warned_zero_slice = true; -- } -- p->scx->slice = SCX_SLICE_DFL; -- } -- -- set_next_task_scx(rq, p, true); -- -- return p; --} -- --#ifdef CONFIG_SCHED_CORE --/** -- * scx_prio_less - Task ordering for core-sched -- * @a: task A -- * @b: task B -- * -- * Core-sched is implemented as an additional scheduling layer on top of the -- * usual sched_class'es and needs to find out the expected task ordering. For -- * SCX, core-sched calls this function to interrogate the task ordering. -- * -- * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used -- * to implement the default task ordering. The older the timestamp, the higher -- * prority the task - the global FIFO ordering matching the default scheduling -- * behavior. -- * -- * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to -- * implement FIFO ordering within each local DSQ. See pick_task_scx(). -- */ --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi) --{ -- /* -- * The const qualifiers are dropped from task_struct pointers when -- * calling ops.core_sched_before(). Accesses are controlled by the -- * verifier. -- */ -- if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling()) -- return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, -- (struct task_struct *)a, -- (struct task_struct *)b); -- else -- return time_after64(a->scx->core_sched_at, b->scx->core_sched_at); --} -- --/** -- * pick_task_scx - Pick a candidate task for core-sched -- * @rq: rq to pick the candidate task from -- * -- * Core-sched calls this function on each SMT sibling to determine the next -- * tasks to run on the SMT siblings. balance_one() has been called on all -- * siblings and put_prev_task_scx() has been called only for the current CPU. -- * -- * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look -- * at the first task in the local dsq. @rq->curr has to be considered explicitly -- * to mimic %SCX_TASK_BAL_KEEP. -- */ --static struct task_struct *pick_task_scx(struct rq *rq) --{ -- struct task_struct *curr = rq->curr; -- struct task_struct *first = first_local_task(rq); -- -- if (curr->scx->flags & SCX_TASK_QUEUED) { -- /* is curr the only runnable task? */ -- if (!first) -- return curr; -- -- /* -- * Does curr trump first? We can always go by core_sched_at for -- * this comparison as it represents global FIFO ordering when -- * the default core-sched ordering is used and local-DSQ FIFO -- * ordering otherwise. -- * -- * We can have a task with an earlier timestamp on the DSQ. For -- * example, when a current task is preempted by a sibling -- * picking a different cookie, the task would be requeued at the -- * head of the local DSQ with an earlier timestamp than the -- * core-sched picked next task. Besides, the BPF scheduler may -- * dispatch any tasks to the local DSQ anytime. -- */ -- if (curr->scx->slice && time_before64(curr->scx->core_sched_at, -- first->scx->core_sched_at)) -- return curr; -- } -- -- return first; /* this may be %NULL */ --} --#endif /* CONFIG_SCHED_CORE */ -- --static enum scx_cpu_preempt_reason --preempt_reason_from_class(const struct sched_class *class) --{ --#ifdef CONFIG_SMP -- if (class == &stop_sched_class) -- return SCX_CPU_PREEMPT_STOP; --#endif -- if (class == &dl_sched_class) -- return SCX_CPU_PREEMPT_DL; -- if (class == &rt_sched_class) -- return SCX_CPU_PREEMPT_RT; -- return SCX_CPU_PREEMPT_UNKNOWN; --} -- --void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task, -- const struct sched_class *active) --{ -- lockdep_assert_rq_held(rq); -- -- /* -- * The callback is conceptually meant to convey that the CPU is no -- * longer under the control of SCX. Therefore, don't invoke the -- * callback if the CPU is is staying on SCX, or going idle (in which -- * case the SCX scheduler has actively decided not to schedule any -- * tasks on the CPU). -- */ -- if (likely(active >= &ext_sched_class)) -- return; -- -- /* -- * At this point we know that SCX was preempted by a higher priority -- * sched_class, so invoke the ->cpu_release() callback if we have not -- * done so already. We only send the callback once between SCX being -- * preempted, and it regaining control of the CPU. -- * -- * ->cpu_release() complements ->cpu_acquire(), which is emitted the -- * next time that balance_scx() is invoked. -- */ -- if (!rq->scx->cpu_released) { -- if (SCX_HAS_OP(cpu_release)) { -- struct scx_cpu_release_args args = { -- .reason = preempt_reason_from_class(active), -- .task = task, -- }; -- -- SCX_CALL_OP(SCX_KF_CPU_RELEASE, -- cpu_release, cpu_of(rq), &args); -- } -- rq->scx->cpu_released = true; -- } --} -- --#ifdef CONFIG_SMP -- --static bool test_and_clear_cpu_idle(int cpu) --{ -- if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -- if (cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- return true; -- } else { -- return false; -- } --} -- --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- int cpu; -- -- do { -- cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); -- if (cpu < nr_cpu_ids) { -- const struct cpumask *sbm = topology_sibling_cpumask(cpu); -- -- /* -- * If offline, @cpu is not its own sibling and we can -- * get caught in an infinite loop as @cpu is never -- * cleared from idle_masks.smt. Clear @cpu directly in -- * such cases. -- */ -- if (likely(cpumask_test_cpu(cpu, sbm))) -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm); -- else -- cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu)); -- } else { -- cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -- if (cpu >= nr_cpu_ids) -- return -EBUSY; -- } -- } while (!test_and_clear_cpu_idle(cpu)); -- -- return cpu; --} -- --static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) --{ -- s32 cpu; -- -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return prev_cpu; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local DSQ of the waker. -- */ -- if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 && -- scx_has_idle_cpus && !(current->flags & PF_EXITING)) { -- cpu = smp_processor_id(); -- if (cpumask_test_cpu(cpu, p->cpus_ptr)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (test_and_clear_cpu_idle(prev_cpu)) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return prev_cpu; -- } -- -- if (p->nr_cpus_allowed == 1) -- return prev_cpu; -- -- cpu = scx_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- p->scx->flags |= SCX_TASK_ENQ_LOCAL; -- return cpu; -- } -- -- return prev_cpu; --} -- --static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) --{ -- if (SCX_HAS_OP(select_cpu)) { -- s32 cpu; -- -- cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu, -- wake_flags); -- if (ops_cpu_valid(cpu)) { -- return cpu; -- } else { -- scx_ops_error("select_cpu returned invalid cpu %d", cpu); -- return prev_cpu; -- } -- } else { -- return scx_select_cpu_dfl(p, prev_cpu, wake_flags); -- } --} -- --static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx) --{ -- set_cpus_allowed_common(p, ctx); -- -- /* -- * The effective cpumask is stored in @p->cpus_ptr which may temporarily -- * differ from the configured one in @p->cpus_mask. Always tell the bpf -- * scheduler the effective one. -- * -- * Fine-grained memory write control is enforced by BPF making the const -- * designation pointless. Cast it away when calling the operation. -- */ -- if (SCX_HAS_OP(set_cpumask)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, -- (struct cpumask *)p->cpus_ptr); --} -- --static void reset_idle_masks(void) --{ -- /* consider all cpus idle, should converge to the actual state quickly */ -- cpumask_setall(idle_masks.cpu); -- cpumask_setall(idle_masks.smt); -- scx_has_idle_cpus = true; --} -- --void __scx_update_idle(struct rq *rq, bool idle) --{ -- int cpu = cpu_of(rq); -- struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -- -- if (SCX_HAS_OP(update_idle)) { -- SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); -- if (!static_branch_unlikely(&scx_builtin_idle_enabled)) -- return; -- } -- -- if (idle) { -- cpumask_set_cpu(cpu, idle_masks.cpu); -- if (!scx_has_idle_cpus) -- scx_has_idle_cpus = true; -- -- /* -- * idle_masks.smt handling is racy but that's fine as it's only -- * for optimization and self-correcting. -- */ -- for_each_cpu(cpu, sib_mask) { -- if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -- return; -- } -- cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -- } else { -- cpumask_clear_cpu(cpu, idle_masks.cpu); -- if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -- scx_has_idle_cpus = false; -- -- cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -- } --} -- --#else /* !CONFIG_SMP */ -- --static bool test_and_clear_cpu_idle(int cpu) { return false; } --static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } --static void reset_idle_masks(void) {} -- --#endif /* CONFIG_SMP */ -- --static bool check_rq_for_timeouts(struct rq *rq) --{ -- struct sched_ext_entity *entity; -- struct task_struct *p; -- struct rq_flags rf; -- bool timed_out = false; -- -- rq_lock_irqsave(rq, &rf); -- list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) { -- unsigned long last_runnable; -- -- p = entity->task; -- last_runnable = p->scx->runnable_at; -- -- if (unlikely(time_after(jiffies, -- last_runnable + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "%s[%d] failed to run for %u.%03us", -- p->comm, p->pid, -- dur_ms / 1000, dur_ms % 1000); -- timed_out = true; -- break; -- } -- } -- rq_unlock_irqrestore(rq, &rf); -- -- return timed_out; --} -- --static void scx_watchdog_workfn(struct work_struct *work) --{ -- int cpu; -- -- scx_watchdog_timestamp = jiffies; -- -- for_each_online_cpu(cpu) { -- if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) -- break; -- -- cond_resched(); -- } -- queue_delayed_work(system_unbound_wq, to_delayed_work(work), -- scx_watchdog_timeout / 2); --} -- --static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) --{ -- update_curr_scx(rq); -- //scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock); -- /* -- * While disabling, always resched and refresh core-sched timestamp as -- * we can't trust the slice management or ops.core_sched_before(). -- */ -- if (scx_ops_disabling()) { -- curr->scx->slice = 0; -- touch_core_sched(rq, curr); -- } -- -- if (!curr->scx->slice) -- resched_curr(rq); --} -- --#define SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- --static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg) --{ -- int ret; -- -- WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED); -- -- p->scx->disallow = false; -- -- if (SCX_HAS_OP(prep_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(tg) -- }; -- -- ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args); -- if (unlikely(ret)) { -- ret = ops_sanitize_err("prep_enable", ret); -- return ret; -- } -- } -- //scx_sched_init_task(p); -- -- if (p->scx->disallow) { -- struct rq *rq; -- struct rq_flags rf; -- -- rq = task_rq_lock(p, &rf); -- -- /* -- * We're either in fork or load path and @p->policy will be -- * applied right after. Reverting @p->policy here and rejecting -- * %SCHED_EXT transitions from scx_check_setscheduler() -- * guarantees that if ops.prep_enable() sets @p->disallow, @p -- * can never be in SCX. -- */ -- if (p->policy == SCHED_EXT) { -- p->policy = SCHED_NORMAL; -- atomic64_inc(&scx_nr_rejected); -- } -- -- task_rq_unlock(rq, p, &rf); -- } -- -- p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET); -- return 0; --} -- --static void scx_ops_enable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED)); -- -- if (SCX_HAS_OP(enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- p->scx->flags |= SCX_TASK_OPS_ENABLED; --} -- --static void scx_ops_disable_task(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- if (p->scx->flags & SCX_TASK_OPS_PREPPED) { -- if (SCX_HAS_OP(cancel_enable)) { -- struct scx_enable_args args = { -- SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) -- }; -- SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args); -- } -- p->scx->flags &= ~SCX_TASK_OPS_PREPPED; -- } else if (p->scx->flags & SCX_TASK_OPS_ENABLED) { -- if (SCX_HAS_OP(disable)) -- SCX_CALL_OP(SCX_KF_REST, disable, p); -- p->scx->flags &= ~SCX_TASK_OPS_ENABLED; -- } --} -- --static void set_task_scx_weight(struct task_struct *p) --{ -- u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -- -- p->scx->weight = sched_weight_to_cgroup(weight); --} -- --/** -- * refresh_scx_weight - Refresh a task's ext weight -- * @p: task to refresh ext weight for -- * -- * @p->scx->weight carries the task's static priority in cgroup weight scale to -- * enable easy access from the BPF scheduler. To keep it synchronized with the -- * current task priority, this function should be called when a new task is -- * created, priority is changed for a task on sched_ext, and a task is switched -- * to sched_ext from other classes. -- */ --static void refresh_scx_weight(struct task_struct *p) --{ -- lockdep_assert_rq_held(task_rq(p)); -- set_task_scx_weight(p); -- if (SCX_HAS_OP(set_weight)) -- SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight); --} -- --void scx_pre_fork(struct task_struct *p) --{ -- p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL); -- if (!p->scx) { -- goto lock; -- } -- -- p->scx->dsq = NULL; -- INIT_LIST_HEAD(&p->scx->dsq_node.fifo); -- RB_CLEAR_NODE(&p->scx->dsq_node.priq); -- INIT_LIST_HEAD(&p->scx->watchdog_node); -- p->scx->flags = 0; -- p->scx->weight = 0; -- p->scx->sticky_cpu = -1; -- p->scx->holding_cpu = -1; -- p->scx->kf_mask = 0; -- atomic64_set(&p->scx->ops_state, 0); -- p->scx->runnable_at = INITIAL_JIFFIES; -- p->scx->slice = SCX_SLICE_DFL; -- p->scx->task = p; -- p->sched_prop = 0; -- -- /* -- * BPF scheduler enable/disable paths want to be able to iterate and -- * update all tasks which can become complex when racing forks. As -- * enable/disable are very cold paths, let's use a percpu_rwsem to -- * exclude forks. -- */ --lock: -- percpu_down_read(&scx_fork_rwsem); --} -- --int scx_fork(struct task_struct *p) --{ -- percpu_rwsem_assert_held(&scx_fork_rwsem); -- -- if (scx_enabled()) -- return scx_ops_prepare_task(p, task_group(p)); -- else -- return 0; --} -- --void scx_post_fork(struct task_struct *p) --{ -- if (scx_enabled()) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- /* -- * Set the weight manually before calling ops.enable() so that -- * the scheduler doesn't see a stale value if they inspect the -- * task struct. We'll invoke ops.set_weight() afterwards, as it -- * would be odd to receive a callback on the task before we -- * tell the scheduler that it's been fully enabled. -- */ -- set_task_scx_weight(p); -- scx_ops_enable_task(p); -- refresh_scx_weight(p); -- task_rq_unlock(rq, p, &rf); -- } -- -- spin_lock_irq(&scx_tasks_lock); -- list_add_tail(&p->scx->tasks_node, &scx_tasks); -- spin_unlock_irq(&scx_tasks_lock); -- -- percpu_up_read(&scx_fork_rwsem); --} -- --void scx_cancel_fork(struct task_struct *p) --{ -- if (scx_enabled()) -- scx_ops_disable_task(p); -- percpu_up_read(&scx_fork_rwsem); --} -- --void sched_ext_free(struct task_struct *p) --{ -- unsigned long flags; -- -- spin_lock_irqsave(&scx_tasks_lock, flags); -- list_del_init(&p->scx->tasks_node); -- spin_unlock_irqrestore(&scx_tasks_lock, flags); -- -- /* -- * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED -> -- * ENABLED transitions can't race us. Disable ops for @p. -- */ -- if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) { -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(p, &rf); -- scx_ops_disable_task(p); -- task_rq_unlock(rq, p, &rf); -- } --} -- --static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) --{ --} -- --static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} --static void switched_to_scx(struct rq *rq, struct task_struct *p) {} -- --int scx_check_setscheduler(struct task_struct *p, int policy) --{ -- lockdep_assert_rq_held(task_rq(p)); -- -- /* if disallow, reject transitioning into SCX */ -- if (scx_enabled() && READ_ONCE(p->scx->disallow) && -- p->policy != policy && policy == SCHED_EXT) -- return -EACCES; -- -- return 0; --} -- --#ifdef CONFIG_NO_HZ_FULL --bool scx_can_stop_tick(struct rq *rq) --{ -- struct task_struct *p = rq->curr; -- -- if (scx_ops_disabling()) -- return false; -- -- if (p->sched_class != &ext_sched_class) -- return true; -- -- /* -- * @rq can dispatch from different DSQs, so we can't tell whether it -- * needs the tick or not by looking at nr_running. Allow stopping ticks -- * iff the BPF scheduler indicated so. See set_next_task_scx(). -- */ -- return rq->scx->flags & SCX_RQ_CAN_STOP_TICK; --} --#endif -- --static inline void scx_cgroup_lock(void) {} --static inline void scx_cgroup_unlock(void) {} -- --/* -- * Omitted operations: -- * -- * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -- * task isn't tied to the CPU at that point. Preemption is implemented by -- * resetting the victim task's slice to 0 and triggering reschedule on the -- * target CPU. -- * -- * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -- * -- * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -- * their current sched_class. Call them directly from sched core instead. -- * -- * - task_woken, switched_from: Unnecessary. -- */ --DEFINE_SCHED_CLASS(ext) = { -- .enqueue_task = enqueue_task_scx, -- .dequeue_task = dequeue_task_scx, -- .yield_task = yield_task_scx, -- .yield_to_task = yield_to_task_scx, -- -- .check_preempt_curr = check_preempt_curr_scx, -- -- .pick_next_task = pick_next_task_scx, -- -- .put_prev_task = put_prev_task_scx, -- .set_next_task = set_next_task_scx, -- --#ifdef CONFIG_SMP -- .balance = balance_scx, -- .select_task_rq = select_task_rq_scx, -- .set_cpus_allowed = set_cpus_allowed_scx, --#endif -- --#ifdef CONFIG_SCHED_CORE -- .pick_task = pick_task_scx, --#endif -- -- .task_tick = task_tick_scx, -- -- .switched_to = switched_to_scx, -- .prio_changed = prio_changed_scx, -- -- .update_curr = update_curr_scx, -- --#ifdef CONFIG_UCLAMP_TASK -- .uclamp_enabled = 0, --#endif --}; -- --static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) --{ -- memset(dsq, 0, sizeof(*dsq)); -- -- raw_spin_lock_init(&dsq->lock); -- INIT_LIST_HEAD(&dsq->fifo); -- dsq->id = dsq_id; --} -- --static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) --{ -- struct scx_dispatch_q *dsq; -- -- if (dsq_id & SCX_DSQ_FLAG_BUILTIN) -- return ERR_PTR(-EINVAL); -- -- dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node); -- if (!dsq) -- return ERR_PTR(-ENOMEM); -- -- init_dsq(dsq, dsq_id); -- -- return dsq; --} -- --static void free_dsq_irq_workfn(struct irq_work *irq_work) --{ -- struct llist_node *to_free = llist_del_all(&dsqs_to_free); -- struct scx_dispatch_q *dsq, *tmp_dsq; -- -- llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) -- kfree_rcu(dsq, rcu); --} -- --static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); -- --static void destroy_dsq(u64 dsq_id) --{ --} -- --static void scx_cgroup_exit(void) {} --static int scx_cgroup_init(void) { return 0; } --static void scx_cgroup_config_knobs(void) {} -- --/* -- * Used by sched_fork() and __setscheduler_prio() to pick the matching -- * sched_class. dl/rt are already handled. -- */ --bool task_on_scx(struct task_struct *p) --{ -- if (!scx_enabled() || scx_ops_disabling()) -- return false; -- if (READ_ONCE(scx_switching_all)) -- return true; -- return p->policy == SCHED_EXT; --} -- --static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags) --{ -- if (enq_flags & SCX_ENQ_LAST) -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- else -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); --} -- --static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {} -- --static void scx_ops_disable_workfn(struct kthread_work *work) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- struct scx_task_iter sti; -- struct task_struct *p; -- const char *reason; -- int i, cpu, type; -- -- type = atomic_read(&scx_exit_type); -- while (true) { -- /* -- * NONE indicates that a new scx_ops has been registered since -- * disable was scheduled - don't kill the new ops. DONE -- * indicates that the ops has already been disabled. -- */ -- if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE) -- return; -- if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE)) -- break; -- } -- -- cancel_delayed_work_sync(&scx_watchdog_work); -- -- switch (type) { -- case SCX_EXIT_UNREG: -- reason = "BPF scheduler unregistered"; -- break; -- case SCX_EXIT_SYSRQ: -- reason = "disabled by sysrq-S"; -- break; -- case SCX_EXIT_ERROR: -- reason = "runtime error"; -- break; -- case SCX_EXIT_ERROR_BPF: -- reason = "scx_bpf_error"; -- break; -- case SCX_EXIT_ERROR_STALL: -- reason = "runnable task stall"; -- break; -- default: -- reason = ""; -- } -- -- ei->type = type; -- strlcpy(ei->reason, reason, sizeof(ei->reason)); -- -- switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { -- case SCX_OPS_DISABLED: -- pr_warn("sched_ext: ops error detected without ops (%s)\n", -- scx_exit_info.msg); -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- return; -- case SCX_OPS_PREPPING: -- goto forward_progress_guaranteed; -- case SCX_OPS_DISABLING: -- /* shouldn't happen but handle it like ENABLING if it does */ -- WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); -- fallthrough; -- case SCX_OPS_ENABLING: -- case SCX_OPS_ENABLED: -- break; -- } -- -- /* -- * DISABLING is set and ops was either ENABLING or ENABLED indicating -- * that the ops and static branches are set. -- * -- * We must guarantee that all runnable tasks make forward progress -- * without trusting the BPF scheduler. We can't grab any mutexes or -- * rwsems as they might be held by tasks that the BPF scheduler is -- * forgetting to run, which unfortunately also excludes toggling the -- * static branches. -- * -- * Let's work around by overriding a couple ops and modifying behaviors -- * based on the DISABLING state and then cycling the tasks through -- * dequeue/enqueue to force global FIFO scheduling. -- * -- * a. ops.enqueue() and .dispatch() are overridden for simple global -- * FIFO scheduling. -- * -- * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value -- * can't be trusted. Whenever a tick triggers, the running task is -- * rotated to the tail of the queue with core_sched_at touched. -- * -- * c. pick_next_task() suppresses zero slice warning. -- * -- * d. scx_prio_less() reverts to the default core_sched_at order. -- */ -- scx_ops.enqueue = scx_ops_fallback_enqueue; -- scx_ops.dispatch = scx_ops_fallback_dispatch; -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- SCHED_CHANGE_BLOCK(task_rq(p), p, -- DEQUEUE_SAVE | DEQUEUE_MOVE) { -- /* cycling deq/enq is enough, see above */ -- } -- } -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* kick all CPUs to restore ticks */ -- for_each_possible_cpu(cpu) -- resched_cpu(cpu); -- --forward_progress_guaranteed: -- /* -- * Here, every runnable task is guaranteed to make forward progress and -- * we can safely use blocking synchronization constructs. Actually -- * disable ops. -- */ -- mutex_lock(&scx_ops_enable_mutex); -- -- static_branch_disable(&__scx_switched_all); -- WRITE_ONCE(scx_switching_all, false); -- -- /* avoid racing against fork and cgroup changes */ -- cpus_read_lock(); -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- bool alive = READ_ONCE(p->__state) != TASK_DEAD; -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL); -- -- __setscheduler_prio(p, p->prio); -- } -- -- if (alive) -- check_class_changed(task_rq(p), p, old_class, p->prio); -- -- scx_ops_disable_task(p); -- } -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- -- /* no task is on scx, turn off all the switches and flush in-progress calls */ -- static_branch_disable_cpuslocked(&__scx_ops_enabled); -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- static_branch_disable_cpuslocked(&scx_has_op[i]); -- static_branch_disable_cpuslocked(&scx_ops_enq_last); -- static_branch_disable_cpuslocked(&scx_ops_enq_exiting); -- static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- synchronize_rcu(); -- -- scx_cgroup_exit(); -- -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- cpus_read_unlock(); -- -- if (ei->type >= SCX_EXIT_ERROR) { -- printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name); -- -- if (ei->msg[0] == '\0') -- printk(KERN_ERR "sched_ext: %s\n", ei->reason); -- else -- printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg); -- -- stack_trace_print(ei->bt, ei->bt_len, 2); -- } -- -- if (scx_ops.exit) -- SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); -- -- memset(&scx_ops, 0, sizeof(scx_ops)); -- -- free_percpu(scx_dsp_buf); -- scx_dsp_buf = NULL; -- scx_dsp_max_batch = 0; -- -- mutex_unlock(&scx_ops_enable_mutex); -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != -- SCX_OPS_DISABLING); -- -- scx_cgroup_config_knobs(); --} -- --static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); -- --static void schedule_scx_ops_disable_work(void) --{ -- struct kthread_worker *helper = READ_ONCE(scx_ops_helper); -- -- /* -- * We may be called spuriously before the first bpf_sched_ext_reg(). If -- * scx_ops_helper isn't set up yet, there's nothing to do. -- */ -- if (helper) -- kthread_queue_work(helper, &scx_ops_disable_work); --} -- --static void scx_ops_disable(enum scx_exit_type type) --{ -- int none = SCX_EXIT_NONE; -- -- if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)) -- type = SCX_EXIT_ERROR; -- -- atomic_try_cmpxchg(&scx_exit_type, &none, type); -- -- schedule_scx_ops_disable_work(); --} -- --static void scx_ops_error_irq_workfn(struct irq_work *irq_work) --{ -- schedule_scx_ops_disable_work(); --} -- --static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...) --{ -- struct scx_exit_info *ei = &scx_exit_info; -- int none = SCX_EXIT_NONE; -- va_list args; -- -- if (!atomic_try_cmpxchg(&scx_exit_type, &none, type)) -- return; -- -- ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1); -- -- va_start(args, fmt); -- vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args); -- va_end(args); -- -- irq_work_queue(&scx_ops_error_irq_work); --} -- --static struct kthread_worker *scx_create_rt_helper(const char *name) --{ -- struct kthread_worker *helper; -- -- helper = kthread_create_worker(0, name); -- if (helper) -- sched_set_fifo(helper->task); -- return helper; --} -- --static int scx_ops_enable(struct sched_ext_ops *ops) --{ -- struct scx_task_iter sti; -- struct task_struct *p; -- int i, ret; -- int tcnt = 0; -- unsigned long long start = 0; -- unsigned long long total_start = sched_clock(); -- -- mutex_lock(&scx_ops_enable_mutex); -- -- if (!scx_ops_helper) { -- WRITE_ONCE(scx_ops_helper, -- scx_create_rt_helper("sched_ext_ops_helper")); -- if (!scx_ops_helper) { -- ret = -ENOMEM; -- goto err_unlock; -- } -- } -- -- if (scx_ops_enable_state() != SCX_OPS_DISABLED) { -- ret = -EBUSY; -- goto err_unlock; -- } -- -- //slim_walt_enable(true); -- -- /* -- * Set scx_ops, transition to PREPPING and clear exit info to arm the -- * disable path. Failure triggers full disabling from here on. -- */ -- scx_ops = *ops; -- -- WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != -- SCX_OPS_DISABLED); -- -- memset(&scx_exit_info, 0, sizeof(scx_exit_info)); -- atomic_set(&scx_exit_type, SCX_EXIT_NONE); -- scx_warned_zero_slice = false; -- -- atomic64_set(&scx_nr_rejected, 0); -- -- /* -- * Keep CPUs stable during enable so that the BPF scheduler can track -- * online CPUs by watching ->on/offline_cpu() after ->init(). -- */ -- cpus_read_lock(); -- -- scx_switch_all_req = true; -- if (scx_ops.init) { -- ret = SCX_CALL_OP_RET(SCX_KF_INIT, init); -- if (ret) { -- ret = ops_sanitize_err("init", ret); -- goto err_disable; -- } -- -- /* -- * Exit early if ops.init() triggered scx_bpf_error(). Not -- * strictly necessary as we'll fail transitioning into ENABLING -- * later but that'd be after calling ops.prep_enable() on all -- * tasks and with -EBUSY which isn't very intuitive. Let's exit -- * early with success so that the condition is notified through -- * ops.exit() like other scx_bpf_error() invocations. -- */ -- if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE) -- goto err_disable; -- } -- -- WARN_ON_ONCE(scx_dsp_buf); -- scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; -- scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch, -- __alignof__(scx_dsp_buf[0])); -- if (!scx_dsp_buf) { -- ret = -ENOMEM; -- goto err_disable; -- } -- -- scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; -- if (ops->timeout_ms) -- scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); -- -- scx_watchdog_timestamp = jiffies; -- queue_delayed_work(system_unbound_wq, &scx_watchdog_work, -- scx_watchdog_timeout / 2); -- -- /* -- * Lock out forks, cgroup on/offlining and moves before opening the -- * floodgate so that they don't wander into the operations prematurely. -- */ -- percpu_down_write(&scx_fork_rwsem); -- scx_cgroup_lock(); -- -- for (i = 0; i < SCX_NR_ONLINE_OPS; i++) -- if (((void (**)(void))ops)[i]) -- static_branch_enable_cpuslocked(&scx_has_op[i]); -- -- if (ops->flags & SCX_OPS_ENQ_LAST) -- static_branch_enable_cpuslocked(&scx_ops_enq_last); -- -- if (ops->flags & SCX_OPS_ENQ_EXITING) -- static_branch_enable_cpuslocked(&scx_ops_enq_exiting); -- if (scx_ops.cpu_acquire || scx_ops.cpu_release) -- static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); -- -- if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { -- reset_idle_masks(); -- static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); -- } else { -- static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); -- } -- -- /* -- * All cgroups should be initialized before letting in tasks. cgroup -- * on/offlining and task migrations are already locked out. -- */ -- ret = scx_cgroup_init(); -- if (ret) -- goto err_disable_unlock; -- -- static_branch_enable_cpuslocked(&__scx_ops_enabled); -- -- /* -- * Enable ops for every task. Fork is excluded by scx_fork_rwsem -- * preventing new tasks from being added. No need to exclude tasks -- * leaving as sched_ext_free() can handle both prepped and enabled -- * tasks. Prep all tasks first and then enable them with preemption -- * disabled. -- */ -- spin_lock_irq(&scx_tasks_lock); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered(&sti))) { -- get_task_struct(p); -- spin_unlock_irq(&scx_tasks_lock); -- -- ret = scx_ops_prepare_task(p, task_group(p)); -- if (ret) { -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- scx_task_iter_exit(&sti); -- spin_unlock_irq(&scx_tasks_lock); -- pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n", -- ret, p->comm, p->pid); -- goto err_disable_unlock; -- } -- -- put_task_struct(p); -- spin_lock_irq(&scx_tasks_lock); -- } -- scx_task_iter_exit(&sti); -- -- /* -- * All tasks are prepped but are still ops-disabled. Ensure that -- * %current can't be scheduled out and switch everyone. -- * preempt_disable() is necessary because we can't guarantee that -- * %current won't be starved if scheduled out while switching. -- */ -- preempt_disable(); -- -- /* -- * From here on, the disable path must assume that tasks have ops -- * enabled and need to be recovered. -- */ -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { -- preempt_enable(); -- spin_unlock_irq(&scx_tasks_lock); -- ret = -EBUSY; -- goto err_disable_unlock; -- } -- -- /* -- * We're fully committed and can't fail. The PREPPED -> ENABLED -- * transitions here are synchronized against sched_ext_free() through -- * scx_tasks_lock. -- */ -- WRITE_ONCE(scx_switching_all, scx_switch_all_req); -- start = sched_clock(); -- -- scx_task_iter_init(&sti); -- while ((p = scx_task_iter_next_filtered_locked(&sti))) { -- tcnt++; -- if (READ_ONCE(p->__state) != TASK_DEAD) { -- const struct sched_class *old_class = p->sched_class; -- struct rq *rq = task_rq(p); -- -- update_rq_clock(rq); -- -- SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -- DEQUEUE_NOCLOCK) { -- scx_ops_enable_task(p); -- __setscheduler_prio(p, p->prio); -- } -- -- check_class_changed(task_rq(p), p, old_class, p->prio); -- } else { -- scx_ops_disable_task(p); -- } -- } -- printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id()); -- scx_task_iter_exit(&sti); -- -- spin_unlock_irq(&scx_tasks_lock); -- preempt_enable(); -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); -- -- if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { -- ret = -EBUSY; -- goto err_disable; -- } -- -- if (scx_switch_all_req) -- static_branch_enable_cpuslocked(&__scx_switched_all); -- -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- -- scx_cgroup_config_knobs(); -- printk("\n total duration = %llu , current cpu = %d\n", sched_clock() - total_start, smp_processor_id()); -- -- return 0; -- --err_unlock: -- mutex_unlock(&scx_ops_enable_mutex); -- return ret; -- --err_disable_unlock: -- scx_cgroup_unlock(); -- percpu_up_write(&scx_fork_rwsem); --err_disable: -- cpus_read_unlock(); -- mutex_unlock(&scx_ops_enable_mutex); -- /* must be fully disabled before returning */ -- scx_ops_disable(SCX_EXIT_ERROR); -- kthread_flush_work(&scx_ops_disable_work); -- return ret; --} -- --#ifdef CONFIG_SCHED_DEBUG --static const char *scx_ops_enable_state_str[] = { -- [SCX_OPS_PREPPING] = "prepping", -- [SCX_OPS_ENABLING] = "enabling", -- [SCX_OPS_ENABLED] = "enabled", -- [SCX_OPS_DISABLING] = "disabling", -- [SCX_OPS_DISABLED] = "disabled", --}; -- --static int scx_debug_show(struct seq_file *m, void *v) --{ -- mutex_lock(&scx_ops_enable_mutex); -- seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name); -- seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled()); -- seq_printf(m, "%-30s: %d\n", "switching_all", -- READ_ONCE(scx_switching_all)); -- seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all()); -- seq_printf(m, "%-30s: %s\n", "enable_state", -- scx_ops_enable_state_str[scx_ops_enable_state()]); -- seq_printf(m, "%-30s: %llu\n", "nr_rejected", -- atomic64_read(&scx_nr_rejected)); -- mutex_unlock(&scx_ops_enable_mutex); -- return 0; --} -- --static int scx_debug_open(struct inode *inode, struct file *file) --{ -- return single_open(file, scx_debug_show, NULL); --} -- --const struct file_operations sched_ext_fops = { -- .open = scx_debug_open, -- .read = seq_read, -- .llseek = seq_lseek, -- .release = single_release, --}; --#endif -- --/******************************************************************************** -- * bpf_struct_ops plumbing. -- */ --#include --#include --#include -- --extern struct btf *btf_vmlinux; --static const struct btf_type *task_struct_type; -- --static bool bpf_scx_is_valid_access(int off, int size, -- enum bpf_access_type type, -- const struct bpf_prog *prog, -- struct bpf_insn_access_aux *info) --{ -- if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) -- return false; -- if (type != BPF_READ) -- return false; -- if (off % size != 0) -- return false; -- -- return btf_ctx_access(off, size, type, prog, info); --} -- --static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, -- const struct bpf_reg_state *reg, -- int off, int size) --{ -- return 0; --} -- --static const struct bpf_func_proto * --bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) --{ -- switch (func_id) { -- case BPF_FUNC_task_storage_get: -- return &bpf_task_storage_get_proto; -- case BPF_FUNC_task_storage_delete: -- return &bpf_task_storage_delete_proto; -- default: -- return bpf_base_func_proto(func_id); -- } --} -- --const struct bpf_verifier_ops bpf_scx_verifier_ops = { -- .get_func_proto = bpf_scx_get_func_proto, -- .is_valid_access = bpf_scx_is_valid_access, -- .btf_struct_access = bpf_scx_btf_struct_access, --}; -- --static int bpf_scx_init_member(const struct btf_type *t, -- const struct btf_member *member, -- void *kdata, const void *udata) --{ -- const struct sched_ext_ops *uops = udata; -- struct sched_ext_ops *ops = kdata; -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- int ret; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, dispatch_max_batch): -- if (*(u32 *)(udata + moff) > INT_MAX) -- return -E2BIG; -- ops->dispatch_max_batch = *(u32 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, flags): -- if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) -- return -EINVAL; -- ops->flags = *(u64 *)(udata + moff); -- return 1; -- case offsetof(struct sched_ext_ops, name): -- ret = bpf_obj_name_cpy(ops->name, uops->name, -- sizeof(ops->name)); -- if (ret < 0) -- return ret; -- if (ret == 0) -- return -EINVAL; -- return 1; -- case offsetof(struct sched_ext_ops, timeout_ms): -- if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT) -- return -E2BIG; -- ops->timeout_ms = *(u32 *)(udata + moff); -- return 1; -- } -- -- return 0; --} -- --static int bpf_scx_check_member(const struct btf_type *t, -- const struct btf_member *member, -- const struct bpf_prog *prog) --{ -- u32 moff = __btf_member_bit_offset(t, member) / 8; -- -- switch (moff) { -- case offsetof(struct sched_ext_ops, prep_enable): -- case offsetof(struct sched_ext_ops, init): -- case offsetof(struct sched_ext_ops, exit): -- break; -- default: -- /* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */ -- /* if (prog->aux->sleepable) -- return -EINVAL; */ -- break; -- } -- -- return 0; --} -- --static int bpf_scx_reg(void *kdata) --{ -- return scx_ops_enable(kdata); --} -- --static void bpf_scx_unreg(void *kdata) --{ -- scx_ops_disable(SCX_EXIT_UNREG); -- kthread_flush_work(&scx_ops_disable_work); --} -- --static int bpf_scx_init(struct btf *btf) --{ -- u32 type_id; -- -- type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); -- if (type_id < 0) -- return -EINVAL; -- task_struct_type = btf_type_by_id(btf, type_id); -- -- return 0; --} -- --/* "extern" to avoid sparse warning, only used in this file */ --extern struct bpf_struct_ops bpf_sched_ext_ops; -- --struct bpf_struct_ops bpf_sched_ext_ops = { -- .verifier_ops = &bpf_scx_verifier_ops, -- .reg = bpf_scx_reg, -- .unreg = bpf_scx_unreg, -- .check_member = bpf_scx_check_member, -- .init_member = bpf_scx_init_member, -- .init = bpf_scx_init, -- .name = "sched_ext_ops", --}; -- --static void sysrq_handle_sched_ext_reset(u8 key) --{ -- if (scx_ops_helper) -- scx_ops_disable(SCX_EXIT_SYSRQ); -- else -- pr_info("sched_ext: BPF scheduler not yet used\n"); --} -- --static const struct sysrq_key_op sysrq_sched_ext_reset_op = { -- .handler = sysrq_handle_sched_ext_reset, -- .help_msg = "reset-sched-ext(S)", -- .action_msg = "Disable sched_ext and revert all tasks to CFS", -- .enable_mask = SYSRQ_ENABLE_RTNICE, --}; -- --static void kick_cpus_irq_workfn(struct irq_work *irq_work) --{ -- struct rq *this_rq = this_rq(); -- u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); -- int this_cpu = cpu_of(this_rq); -- int cpu; -- -- for_each_cpu(cpu, this_rq->scx->cpus_to_kick) { -- struct rq *rq = cpu_rq(cpu); -- unsigned long flags; -- -- raw_spin_rq_lock_irqsave(rq, flags); -- -- if (cpu_online(cpu) || cpu == this_cpu) { -- if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) && -- rq->curr->sched_class == &ext_sched_class) -- rq->curr->scx->slice = 0; -- pseqs[cpu] = rq->scx->pnt_seq; -- resched_curr(rq); -- } else { -- cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait); -- } -- -- raw_spin_rq_unlock_irqrestore(rq, flags); -- } -- -- for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait, -- cpumask_of(this_cpu)) { -- /* -- * Pairs with smp_store_release() issued by this CPU in -- * scx_notify_pick_next_task() on the resched path. -- * -- * We busy-wait here to guarantee that no other task can be -- * scheduled on our core before the target CPU has entered the -- * resched path. -- */ -- while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu]) -- cpu_relax(); -- } -- -- cpumask_clear(this_rq->scx->cpus_to_kick); -- cpumask_clear(this_rq->scx->cpus_to_preempt); -- cpumask_clear(this_rq->scx->cpus_to_wait); --} -- --void __init init_sched_ext_class(void) --{ -- int cpu; -- u32 v; -- -- /* -- * The following is to prevent the compiler from optimizing out the enum -- * definitions so that BPF scheduler implementations can use them -- * through the generated vmlinux.h. -- */ -- WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | -- SCX_TG_ONLINE | SCX_KICK_PREEMPT); -- -- init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); --#ifdef CONFIG_SMP -- BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -- BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); --#endif -- scx_kick_cpus_pnt_seqs = -- __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * -- num_possible_cpus(), -- __alignof__(scx_kick_cpus_pnt_seqs[0])); -- BUG_ON(!scx_kick_cpus_pnt_seqs); -- -- for_each_possible_cpu(cpu) { -- struct rq *rq = cpu_rq(cpu); -- -- rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL); -- if (rq->scx) -- rq->scx->rq = rq; -- else -- pr_err("fatal error : alloc rq->scx failed!!!\n"); -- -- init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL); -- INIT_LIST_HEAD(&rq->scx->watchdog_list); -- -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL)); -- BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL)); -- init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn); -- } -- -- register_sysrq_key('S', &sysrq_sched_ext_reset_op); -- INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); -- scx_cgroup_config_knobs(); --} -- -- --/******************************************************************************** -- * Helpers that can be called from the BPF scheduler. -- */ --#include -- --/* Disables missing prototype warnings for kfuncs */ --__diag_push(); --__diag_ignore_all("-Wmissing-prototypes", -- "Global functions as their definitions will be in vmlinux BTF"); -- --/** -- * scx_bpf_switch_all - Switch all tasks into SCX -- * @into_scx: switch direction -- * -- * If @into_scx is %true, all existing and future non-dl/rt tasks are switched -- * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on -- * SCX. The actual switching is asynchronous. Can be called from ops.init(). -- */ --void scx_bpf_switch_all(void) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return; -- -- scx_switch_all_req = true; --} -- --BTF_SET8_START(scx_kfunc_ids_init) --BTF_ID_FLAGS(func, scx_bpf_switch_all) --BTF_SET8_END(scx_kfunc_ids_init) -- --static const struct btf_kfunc_id_set scx_kfunc_set_init = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_init, --}; -- --/** -- * scx_bpf_create_dsq - Create a custom DSQ -- * @dsq_id: DSQ to create -- * @node: NUMA node to allocate from -- * -- * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(), -- * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move(). -- */ --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) --{ -- if (!scx_kf_allowed(SCX_KF_INIT)) -- return -EINVAL; -- -- if (unlikely(node >= (int)nr_node_ids || -- (node < 0 && node != NUMA_NO_NODE))) -- return -EINVAL; -- return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); --} -- --BTF_SET8_START(scx_kfunc_ids_sleepable) --BTF_ID_FLAGS(func, scx_bpf_create_dsq) --BTF_SET8_END(scx_kfunc_ids_sleepable) -- --static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_sleepable, --}; -- --static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) --{ -- if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) -- return false; -- -- lockdep_assert_irqs_disabled(); -- -- if (unlikely(!p)) { -- scx_ops_error("called with NULL task"); -- return false; -- } -- -- if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { -- scx_ops_error("invalid enq_flags 0x%llx", enq_flags); -- return false; -- } -- -- return true; --} -- --static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) --{ -- struct task_struct *ddsp_task; -- int idx; -- -- ddsp_task = __this_cpu_read(direct_dispatch_task); -- if (ddsp_task) { -- direct_dispatch(ddsp_task, p, dsq_id, enq_flags); -- return; -- } -- -- idx = __this_cpu_read(scx_dsp_ctx.buf_cursor); -- if (unlikely(idx >= scx_dsp_max_batch)) { -- scx_ops_error("dispatch buffer overflow"); -- return; -- } -- -- this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){ -- .task = p, -- .qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK, -- .dsq_id = dsq_id, -- .enq_flags = enq_flags, -- }; -- __this_cpu_inc(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe -- * to call this function spuriously. Can be called from ops.enqueue() and -- * ops.dispatch(). -- * -- * When called from ops.enqueue(), it's for direct dispatch and @p must match -- * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the -- * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be -- * on the target CPU in the first place. -- * -- * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id -- * and this function can be called upto ops.dispatch_max_batch times to dispatch -- * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the -- * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. -- * -- * This function doesn't have any locking restrictions and may be called under -- * BPF locks (in the future when BPF introduces more flexible locking). -- * -- * @p is allowed to run for @slice. The scheduling path is triggered on slice -- * exhaustion. If zero, the current residual slice is maintained. If -- * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with -- * scx_bpf_kick_cpu() to trigger scheduling. -- */ --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- scx_dispatch_commit(p, dsq_id, enq_flags); --} -- --/** -- * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ -- * @p: task_struct to dispatch -- * @dsq_id: DSQ to dispatch to -- * @slice: duration @p can run for in nsecs -- * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ -- * @enq_flags: SCX_ENQ_* -- * -- * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. -- * Tasks queued into the priority queue are ordered by @vtime and always -- * consumed after the tasks in the FIFO queue. All other aspects are identical -- * to scx_bpf_dispatch(). -- * -- * @vtime ordering is according to time_before64() which considers wrapping. A -- * numerically larger vtime may indicate an earlier position in the ordering and -- * vice-versa. -- */ --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, -- u64 vtime, u64 enq_flags) --{ -- if (!scx_dispatch_preamble(p, enq_flags)) -- return; -- -- if (slice) -- p->scx->slice = slice; -- else -- p->scx->slice = p->scx->slice ?: 1; -- -- p->scx->dsq_vtime = vtime; -- -- scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); --} -- --BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime) --BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_enqueue_dispatch, --}; -- --/** -- * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots -- * -- * Can only be called from ops.dispatch(). -- */ --u32 scx_bpf_dispatch_nr_slots(void) --{ -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return 0; -- -- return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor); --} -- --/** -- * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ -- * @dsq_id: DSQ to consume -- * -- * Consume a task from the non-local DSQ identified by @dsq_id and transfer it -- * to the current CPU's local DSQ for execution. Can only be called from -- * ops.dispatch(). -- * -- * This function flushes the in-flight dispatches from scx_bpf_dispatch() before -- * trying to consume the specified DSQ. It may also grab rq locks and thus can't -- * be called under any BPF locks. -- * -- * Returns %true if a task has been consumed, %false if there isn't any task to -- * consume. -- */ --bool scx_bpf_consume(u64 dsq_id) --{ -- struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); -- struct scx_dispatch_q *dsq; -- -- if (!scx_kf_allowed(SCX_KF_DISPATCH)) -- return false; -- -- flush_dispatch_buf(dspc->rq, dspc->rf); -- -- dsq = find_non_local_dsq(dsq_id); -- if (unlikely(!dsq)) { -- scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); -- return false; -- } -- -- if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) { -- /* -- * A successfully consumed task can be dequeued before it starts -- * running while the CPU is trying to migrate other dispatched -- * tasks. Bump nr_tasks to tell balance_scx() to retry on empty -- * local DSQ. -- */ -- dspc->nr_tasks++; -- return true; -- } else { -- return false; -- } --} -- --BTF_SET8_START(scx_kfunc_ids_dispatch) --BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) --BTF_ID_FLAGS(func, scx_bpf_consume) --BTF_SET8_END(scx_kfunc_ids_dispatch) -- --static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_dispatch, --}; -- --/** -- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ -- * -- * Iterate over all of the tasks currently enqueued on the local DSQ of the -- * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of -- * processed tasks. Can only be called from ops.cpu_release(). -- */ --u32 scx_bpf_reenqueue_local(void) --{ -- u32 nr_enqueued, i; -- struct rq *rq; -- struct scx_rq *scx_rq; -- -- if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) -- return 0; -- -- rq = cpu_rq(smp_processor_id()); -- lockdep_assert_rq_held(rq); -- scx_rq = rq->scx; -- -- /* -- * Get the number of tasks on the local DSQ before iterating over it to -- * pull off tasks. The enqueue callback below can signal that it wants -- * the task to stay on the local DSQ, and we want to prevent the BPF -- * scheduler from causing us to loop indefinitely. -- */ -- nr_enqueued = scx_rq->local_dsq.nr; -- for (i = 0; i < nr_enqueued; i++) { -- struct task_struct *p; -- -- p = first_local_task(rq); -- WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); -- WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); -- WARN_ON_ONCE(p->scx->holding_cpu != -1); -- dispatch_dequeue(scx_rq, p); -- do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); -- } -- -- return nr_enqueued; --} -- --BTF_SET8_START(scx_kfunc_ids_cpu_release) --BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) --BTF_SET8_END(scx_kfunc_ids_cpu_release) -- --static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_cpu_release, --}; -- --/** -- * scx_bpf_kick_cpu - Trigger reschedule on a CPU -- * @cpu: cpu to kick -- * @flags: SCX_KICK_* flags -- * -- * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or -- * trigger rescheduling on a busy CPU. This can be called from any online -- * scx_ops operation and the actual kicking is performed asynchronously through -- * an irq work. -- */ --void scx_bpf_kick_cpu(s32 cpu, u64 flags) --{ -- struct rq *rq; -- -- if (!ops_cpu_valid(cpu)) { -- scx_ops_error("invalid cpu %d", cpu); -- return; -- } -- -- preempt_disable(); -- rq = this_rq(); -- -- /* -- * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting -- * rq locks. We can probably be smarter and avoid bouncing if called -- * from ops which don't hold a rq lock. -- */ -- cpumask_set_cpu(cpu, rq->scx->cpus_to_kick); -- if (flags & SCX_KICK_PREEMPT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt); -- if (flags & SCX_KICK_WAIT) -- cpumask_set_cpu(cpu, rq->scx->cpus_to_wait); -- -- irq_work_queue(&rq->scx->kick_cpus_irq_work); -- preempt_enable(); --} -- --/** -- * scx_bpf_dsq_nr_queued - Return the number of queued tasks -- * @dsq_id: id of the DSQ -- * -- * Return the number of tasks in the DSQ matching @dsq_id. If not found, -- * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops -- * operations. -- */ --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) --{ -- struct scx_dispatch_q *dsq; -- -- lockdep_assert(rcu_read_lock_any_held()); -- -- if (dsq_id == SCX_DSQ_LOCAL) { -- return this_rq()->scx->local_dsq.nr; -- } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { -- s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; -- -- if (ops_cpu_valid(cpu)) -- return cpu_rq(cpu)->scx->local_dsq.nr; -- } else { -- dsq = find_non_local_dsq(dsq_id); -- if (dsq) -- return dsq->nr; -- } -- return -ENOENT; --} -- --/** -- * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state -- * @cpu: cpu to test and clear idle for -- * -- * Returns %true if @cpu was idle and its idle state was successfully cleared. -- * %false otherwise. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return false; -- } -- -- if (ops_cpu_valid(cpu)) -- return test_and_clear_cpu_idle(cpu); -- else -- return false; --} -- --/** -- * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu -- * @cpus_allowed: Allowed cpumask -- * -- * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked -- * idle cpu number on success. -%EBUSY if no matching cpu was found. -- * -- * Unavailable if ops.update_idle() is implemented and -- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. -- */ --s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return -EBUSY; -- } -- -- return scx_pick_idle_cpu(cpus_allowed); --} -- --/** -- * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking -- * per-CPU cpumask. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_cpumask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.cpu; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, -- * per-physical-core cpumask. Can be used to determine if an entire physical -- * core is free. -- * -- * Returns NULL if idle tracking is not enabled, or running on a UP kernel. -- */ --const struct cpumask *scx_bpf_get_idle_smtmask(void) --{ -- if (!static_branch_likely(&scx_builtin_idle_enabled)) { -- scx_ops_error("built-in idle tracking is disabled"); -- return cpu_none_mask; -- } -- --#ifdef CONFIG_SMP -- return idle_masks.smt; --#else -- return cpu_none_mask; --#endif --} -- --/** -- * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to -- * either the percpu, or SMT idle-tracking cpumask. -- */ --void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) --{ -- /* -- * Empty function body because we aren't actually acquiring or -- * releasing a reference to a global idle cpumask, which is read-only -- * in the caller and is never released. The acquire / release semantics -- * here are just used to make the cpumask is a trusted pointer in the -- * caller. -- */ --} -- --struct scx_bpf_error_bstr_bufs { -- u64 data[MAX_BPRINTF_VARARGS]; -- char msg[SCX_EXIT_MSG_LEN]; --}; -- --static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs); -- --/** -- * scx_bpf_error_bstr - Indicate fatal error -- * @fmt: error message format string -- * @data: format string parameters packaged using ___bpf_fill() macro -- * @data__sz: @data len, must end in '__sz' for the verifier -- * -- * Indicate that the BPF scheduler encountered a fatal error and initiate ops -- * disabling. -- */ --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) --{ -- struct scx_bpf_error_bstr_bufs *bufs; -- unsigned long flags; -- struct bpf_bprintf_data args = {}; -- int ret; -- -- local_irq_save(flags); -- bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs); -- -- if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || -- (data__sz && !data)) { -- scx_ops_error("invalid data=%p and data__sz=%u", -- (void *)data, data__sz); -- goto out_restore; -- } -- -- ret = copy_from_kernel_nofault(bufs->data, data, data__sz); -- if (ret) { -- scx_ops_error("failed to read data fields (%d)", ret); -- goto out_restore; -- } -- -- ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args); -- if (ret < 0) { -- scx_ops_error("failed to format prepration (%d)", ret); -- goto out_restore; -- } -- -- ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt, -- args.bin_args); -- bpf_bprintf_cleanup(&args); -- if (ret < 0) { -- scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format", -- fmt, data, data__sz); -- goto out_restore; -- } -- -- scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg); --out_restore: -- local_irq_restore(flags); --} -- --/** -- * scx_bpf_destroy_dsq - Destroy a custom DSQ -- * @dsq_id: DSQ to destroy -- * -- * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with -- * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is -- * empty and no further tasks are dispatched to it. Ignored if called on a DSQ -- * which doesn't exist. Can be called from any online scx_ops operations. -- */ --void scx_bpf_destroy_dsq(u64 dsq_id) --{ -- destroy_dsq(dsq_id); --} -- --/** -- * scx_bpf_task_running - Is task currently running? -- * @p: task of interest -- */ --bool scx_bpf_task_running(const struct task_struct *p) --{ -- return task_rq(p)->curr == p; --} -- --/** -- * scx_bpf_task_cpu - CPU a task is currently associated with -- * @p: task of interest -- */ --s32 scx_bpf_task_cpu(const struct task_struct *p) --{ -- return task_cpu(p); --} -- --/** -- * scx_bpf_task_cgroup - Return the sched cgroup of a task -- * @p: task of interest -- * -- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with -- * from the scheduler's POV. SCX operations should use this function to -- * determine @p's current cgroup as, unlike following @p->cgroups, -- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all -- * rq-locked operations. Can be called on the parameter tasks of rq-locked -- * operations. The restriction guarantees that @p's rq is locked by the caller. -- */ --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) --{ -- struct task_group *tg = p->sched_task_group; -- struct cgroup *cgrp = &cgrp_dfl_root.cgrp; -- -- if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) -- goto out; -- -- /* -- * A task_group may either be a cgroup or an autogroup. In the latter -- * case, @tg->css.cgroup is %NULL. A task_group can't become the other -- * kind once created. -- */ -- if (tg && tg->css.cgroup) -- cgrp = tg->css.cgroup; -- else -- cgrp = &cgrp_dfl_root.cgrp; --out: -- cgroup_get(cgrp); -- return cgrp; --} -- --BTF_SET8_START(scx_kfunc_ids_any) --BTF_ID_FLAGS(func, scx_bpf_kick_cpu) --BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) --BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) --BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu) --BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) --BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) --BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) --BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) --BTF_ID_FLAGS(func, scx_bpf_task_running) --BTF_ID_FLAGS(func, scx_bpf_task_cpu) --BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE) --BTF_SET8_END(scx_kfunc_ids_any) -- --static const struct btf_kfunc_id_set scx_kfunc_set_any = { -- .owner = THIS_MODULE, -- .set = &scx_kfunc_ids_any, --}; -- --__diag_pop(); -- --/* -- * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set() -- * needs most of the system to be up. -- */ --static int __init register_ext_kfuncs(void) --{ -- int ret; -- -- /* -- * Some kfuncs are context-sensitive and can only be called from -- * specific SCX ops. They are grouped into BTF sets accordingly. -- * Unfortunately, BPF currently doesn't have a way of enforcing such -- * restrictions. Eventually, the verifier should be able to enforce -- * them. For now, register them the same and make each kfunc explicitly -- * check using scx_kf_allowed(). -- */ -- if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_init)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_sleepable)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_enqueue_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_dispatch)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_cpu_release)) || -- (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, -- &scx_kfunc_set_any))) { -- pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret); -- return ret; -- } -- -- return 0; --} --__initcall(register_ext_kfuncs); -diff --git b/kernel/sched/ext.h a/kernel/sched/ext.h -deleted file mode 100755 -index fba8ed845f99..000000000000 ---- b/kernel/sched/ext.h -+++ /dev/null -@@ -1,257 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ -- -- --/* -- * Tag marking a kernel function as a kfunc. This is meant to minimize the -- * amount of copy-paste that kfunc authors have to include for correctness so -- * as to avoid issues such as the compiler inlining or eliding either a static -- * kfunc, or a global kfunc in an LTO build. -- */ --#define __bpf_kfunc __used noinline -- --enum scx_wake_flags { -- /* expose select WF_* flags as enums */ -- SCX_WAKE_EXEC = WF_EXEC, -- SCX_WAKE_FORK = WF_FORK, -- SCX_WAKE_TTWU = WF_TTWU, -- SCX_WAKE_SYNC = WF_SYNC, --}; -- --enum scx_enq_flags { -- /* expose select ENQUEUE_* flags as enums */ -- SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, -- SCX_ENQ_HEAD = ENQUEUE_HEAD, -- -- /* high 32bits are SCX specific */ -- -- /* -- * Set the following to trigger preemption when calling -- * scx_bpf_dispatch() with a local dsq as the target. The slice of the -- * current task is cleared to zero and the CPU is kicked into the -- * scheduling path. Implies %SCX_ENQ_HEAD. -- */ -- SCX_ENQ_PREEMPT = 1LLU << 32, -- -- /* -- * The task being enqueued was previously enqueued on the current CPU's -- * %SCX_DSQ_LOCAL, but was removed from it in a call to the -- * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was -- * invoked in a ->cpu_release() callback, and the task is again -- * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the -- * task will not be scheduled on the CPU until at least the next invocation -- * of the ->cpu_acquire() callback. -- */ -- SCX_ENQ_REENQ = 1LLU << 40, -- -- /* -- * The task being enqueued is the only task available for the cpu. By -- * default, ext core keeps executing such tasks but when -- * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with -- * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set. -- * -- * If the BPF scheduler wants to continue executing the task, -- * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately. -- * If the task gets queued on a different dsq or the BPF side, the BPF -- * scheduler is responsible for triggering a follow-up scheduling event. -- * Otherwise, Execution may stall. -- */ -- SCX_ENQ_LAST = 1LLU << 41, -- -- /* -- * A hint indicating that it's advisable to enqueue the task on the -- * local dsq of the currently selected CPU. Currently used by -- * select_cpu_dfl() and together with %SCX_ENQ_LAST. -- */ -- SCX_ENQ_LOCAL = 1LLU << 42, -- -- /* high 8 bits are internal */ -- __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, -- -- SCX_ENQ_CLEAR_OPSS = 1LLU << 56, -- SCX_ENQ_DSQ_PRIQ = 1LLU << 57, --}; -- --enum scx_deq_flags { -- /* expose select DEQUEUE_* flags as enums */ -- SCX_DEQ_SLEEP = DEQUEUE_SLEEP, -- -- /* high 32bits are SCX specific */ -- -- /* -- * The generic core-sched layer decided to execute the task even though -- * it hasn't been dispatched yet. Dequeue from the BPF side. -- */ -- SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, --}; -- --enum scx_tg_flags { -- SCX_TG_ONLINE = 1U << 0, -- SCX_TG_INITED = 1U << 1, --}; -- --enum scx_kick_flags { -- SCX_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -- SCX_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ --}; -- --#ifdef CONFIG_SCHED_CLASS_EXT -- --extern const struct sched_class ext_sched_class; --extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops; --extern const struct file_operations sched_ext_fops; --extern unsigned long scx_watchdog_timeout; --extern unsigned long scx_watchdog_timestamp; -- --DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); --DECLARE_STATIC_KEY_FALSE(__scx_switched_all); --#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) --#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) -- --DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); -- --bool task_on_scx(struct task_struct *p); --void scx_pre_fork(struct task_struct *p); --int scx_fork(struct task_struct *p); --void scx_post_fork(struct task_struct *p); --void scx_cancel_fork(struct task_struct *p); --int scx_check_setscheduler(struct task_struct *p, int policy); --bool scx_can_stop_tick(struct rq *rq); --void init_sched_ext_class(void); -- --__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, -- const char *fmt, ...); --#define scx_ops_error(fmt, args...) \ -- scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args) -- --void __scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active); -- --static inline void scx_notify_pick_next_task(struct rq *rq, -- struct task_struct *p, -- const struct sched_class *active) --{ -- if (!scx_enabled()) -- return; --#ifdef CONFIG_SMP -- /* -- * Pairs with the smp_load_acquire() issued by a CPU in -- * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -- * resched. -- */ -- smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1); --#endif -- if (!static_branch_unlikely(&scx_ops_cpu_preempt)) -- return; -- __scx_notify_pick_next_task(rq, p, active); --} -- --//extern void scx_scheduler_tick(void); --static inline void scx_notify_sched_tick(void) --{ -- unsigned long last_check; -- -- if (!scx_enabled()) -- return; -- //scx_scheduler_tick(); -- -- last_check = scx_watchdog_timestamp; -- if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) { -- u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -- -- scx_ops_error_type(SCX_EXIT_ERROR_STALL, -- "watchdog failed to check in for %u.%03us", -- dur_ms / 1000, dur_ms % 1000); -- } --} -- --static inline const struct sched_class *next_active_class(const struct sched_class *class) --{ -- class++; -- if (scx_switched_all() && class == &fair_sched_class) -- class++; -- if (!scx_enabled() && class == &ext_sched_class) -- class++; -- return class; --} -- --#define for_active_class_range(class, _from, _to) \ -- for (class = (_from); class != (_to); class = next_active_class(class)) -- --#define for_each_active_class(class) \ -- for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -- --/* -- * SCX requires a balance() call before every pick_next_task() call including -- * when waking up from idle. -- */ --#define for_balance_class_range(class, prev_class, end_class) \ -- for_active_class_range(class, (prev_class) > &ext_sched_class ? \ -- &ext_sched_class : (prev_class), (end_class)) -- --#ifdef CONFIG_SCHED_CORE --bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, -- bool in_fi); --#endif -- --#else /* CONFIG_SCHED_CLASS_EXT */ -- --#define scx_enabled() false --#define scx_switched_all() false -- --static inline void scx_pre_fork(struct task_struct *p) {} --static inline int scx_fork(struct task_struct *p) { return 0; } --static inline void scx_post_fork(struct task_struct *p) {} --static inline void scx_cancel_fork(struct task_struct *p) {} --static inline int scx_check_setscheduler(struct task_struct *p, -- int policy) { return 0; } --static inline bool scx_can_stop_tick(struct rq *rq) { return true; } --static inline void init_sched_ext_class(void) {} --static inline void scx_notify_pick_next_task(struct rq *rq, -- const struct task_struct *p, -- const struct sched_class *active) {} --static inline void scx_notify_sched_tick(void) {} -- --#define for_each_active_class for_each_class --#define for_balance_class_range for_class_range -- --#endif /* CONFIG_SCHED_CLASS_EXT */ -- --#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) --void __scx_update_idle(struct rq *rq, bool idle); -- --static inline void scx_update_idle(struct rq *rq, bool idle) --{ -- if (scx_enabled()) -- __scx_update_idle(rq, idle); --} --#else --static inline void scx_update_idle(struct rq *rq, bool idle) {} --#endif -- --#ifdef CONFIG_CGROUP_SCHED --#ifdef CONFIG_EXT_GROUP_SCHED --int scx_tg_online(struct task_group *tg); --void scx_tg_offline(struct task_group *tg); --int scx_cgroup_can_attach(struct cgroup_taskset *tset); --void scx_move_task(struct task_struct *p); --void scx_cgroup_finish_attach(void); --void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); --void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); --#else /* CONFIG_EXT_GROUP_SCHED */ --static inline int scx_tg_online(struct task_group *tg) { return 0; } --static inline void scx_tg_offline(struct task_group *tg) {} --static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } --static inline void scx_move_task(struct task_struct *p) {} --static inline void scx_cgroup_finish_attach(void) {} --static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} --static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} --#endif /* CONFIG_EXT_GROUP_SCHED */ --#endif /* CONFIG_CGROUP_SCHED */ -diff --git b/kernel/sched/hmbird.h a/kernel/sched/hmbird.h -new file mode 100755 -index 000000000000..fa76c7f09977 ---- /dev/null -+++ a/kernel/sched/hmbird.h -@@ -0,0 +1,342 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#ifndef _HMBIRD_H_ -+#define _HMBIRD_H_ -+ -+/* -+ * Tag marking a kernel function as a kfunc. This is meant to minimize the -+ * amount of copy-paste that kfunc authors have to include for correctness so -+ * as to avoid issues such as the compiler inlining or eliding either a static -+ * kfunc, or a global kfunc in an LTO build. -+ */ -+ -+#define DEBUG_INTERNAL (1 << 0) -+#define DEBUG_INFO_TRACE (1 << 1) -+#define DEBUG_INFO_SYSTRACE (1 << 2) -+ -+#define NUMS_CGROUP_KINDS (512) -+#define SLIM_FOR_SGAME (1 << 0) -+#define SLIM_FOR_GENSHIN (1 << 1) -+ -+#ifdef MAX_TASK_NR -+#define MAX_KEY_THREAD_RECORD ((MAX_TASK_NR + 1) >> 1) -+#else -+#define MAX_KEY_THREAD_RECORD (1 << 3) -+#endif /* MAX_TASK_NR */ -+ -+#define TOP_TASK_SHIFT (8) -+#define TOP_TASK_MAX (1 << TOP_TASK_SHIFT) -+#ifndef TOP_TASK_BITS_MASK -+#define TOP_TASK_BITS_MASK (TOP_TASK_MAX - 1) -+#endif /* TOP_TASK_BITS_MASK */ -+ -+extern struct md_info_t *md_info; -+ -+#define debug_enabled() \ -+ (unlikely(hmbirdcore_debug)) -+ -+#define hmbird_debug(fmt, ...) \ -+ pr_info(":"fmt, ##__VA_ARGS__) -+ -+#define hmbird_err(id, fmt, ...) \ -+do { \ -+ pr_err(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_deferred_err(id, fmt, ...) \ -+do { \ -+ printk_deferred(""fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+} while (0) -+ -+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \ -+do { \ -+ if (unlikely(cond)) { \ -+ hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__); \ -+ exceps_update(md_info, id, jiffies); \ -+ } \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_info_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_info_trace(fmt, ...) -+#endif -+ -+#define hmbird_info_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+ } \ -+} while (0) -+ -+#define hmbird_output_systrace(fmt, ...) \ -+do { \ -+ char buf[256]; \ -+ snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__); \ -+ tracing_mark_write(buf); \ -+} while (0) -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define hmbird_internal_trace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) \ -+ trace_printk(":"fmt, ##__VA_ARGS__); \ -+} while (0) -+#else -+#define hmbird_internal_trace(fmt, ...) -+#endif -+ -+#define hmbird_internal_systrace(fmt, ...) \ -+do { \ -+ if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) { \ -+ hmbird_output_systrace(fmt, ##__VA_ARGS__); \ -+ } \ -+} while (0) -+ -+enum hmbird_wake_flags { -+ /* expose select WF_* flags as enums */ -+ HMBIRD_WAKE_EXEC = WF_EXEC, -+ HMBIRD_WAKE_FORK = WF_FORK, -+ HMBIRD_WAKE_TTWU = WF_TTWU, -+ HMBIRD_WAKE_SYNC = WF_SYNC, -+}; -+ -+enum hmbird_enq_flags { -+ /* expose select ENQUEUE_* flags as enums */ -+ HMBIRD_ENQ_WAKEUP = ENQUEUE_WAKEUP, -+ HMBIRD_ENQ_HEAD = ENQUEUE_HEAD, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * Set the following to trigger preemption when calling -+ * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the -+ * current task is cleared to zero and the CPU is kicked into the -+ * scheduling path. Implies %HMBIRD_ENQ_HEAD. -+ */ -+ HMBIRD_ENQ_PREEMPT = 1LLU << 32, -+ HMBIRD_ENQ_REENQ = 1LLU << 40, -+ HMBIRD_ENQ_LAST = 1LLU << 41, -+ -+ /* -+ * A hint indicating that it's advisable to enqueue the task on the -+ * local dsq of the currently selected CPU. Currently used by -+ * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST. -+ */ -+ HMBIRD_ENQ_LOCAL = 1LLU << 42, -+ -+ /* high 8 bits are internal */ -+ __HMBIRD_ENQ_INTERNAL_MASK = 0xffLLU << 56, -+ -+ HMBIRD_ENQ_CLEAR_OPSS = 1LLU << 56, -+ HMBIRD_ENQ_DSQ_PRIQ = 1LLU << 57, -+}; -+ -+enum hmbird_deq_flags { -+ /* expose select DEQUEUE_* flags as enums */ -+ HMBIRD_DEQ_SLEEP = DEQUEUE_SLEEP, -+ -+ /* high 32bits are HMBIRD specific */ -+ -+ /* -+ * The generic core-sched layer decided to execute the task even though -+ * it hasn't been dispatched yet. Dequeue from the BPF side. -+ */ -+ HMBIRD_DEQ_CORE_SCHED_EXEC = 1LLU << 32, -+}; -+ -+enum hmbird_kick_flags { -+ HMBIRD_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ -+ HMBIRD_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ -+}; -+ -+enum hmbird_task_prop_type { -+ HMBIRD_TASK_PROP_COMMON, -+ HMBIRD_TASK_PROP_PIPELINE, -+ HMBIRD_TASK_PROP_DEBUG_OR_LOG, -+ HMBIRD_TASK_PROP_HIGH_LOAD, -+ HMBIRD_TASK_PROP_IO, -+ HMBIRD_TASK_PROP_NETWORK, -+ HMBIRD_TASK_PROP_PERIODIC, -+ HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL, -+ HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL, -+ HMBIRD_TASK_PROP_ISOLATE, -+ HMBIRD_TASK_PROP_MAX, -+}; -+ -+enum hmbird_preempt_policy_id { -+ HMBIRD_PREEMPT_POLICY_NONE, -+ HMBIRD_PREEMPT_POLICY_PRIO_BASED, -+}; -+ -+extern const struct sched_class hmbird_sched_class; -+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops; -+extern const struct file_operations sched_hmbird_fops; -+extern unsigned long hmbird_watchdog_timeout; -+extern unsigned long hmbird_watchdog_timestamp; -+ -+enum scx_rq_flags { -+ HMBIRD_RQ_CAN_STOP_TICK = 1 << 0, -+}; -+ -+struct hmbird_rq { -+ struct hmbird_dispatch_q local_dsq; -+ struct list_head watchdog_list; -+ u64 ops_qseq; -+ u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -+ u32 nr_running; -+ u32 flags; -+ bool cpu_released; -+ cpumask_var_t cpus_to_kick; -+ cpumask_var_t cpus_to_preempt; -+ cpumask_var_t cpus_to_wait; -+ u64 pnt_seq; -+ u64 *prev_runnable_sum_fixed; -+ u32 *prev_window_size; -+ struct irq_work kick_cpus_irq_work; -+ bool pipeline; -+ bool exclusive; -+ bool period_disallow; -+ bool nonperiod_disallow; -+ struct rq *rq; -+ struct hmbird_sched_rq_stats *srq; -+}; -+ -+struct hmbird_sched_change_guard { -+ struct task_struct *p; -+ struct rq *rq; -+ bool queued; -+ bool running; -+ bool done; -+}; -+ -+extern struct hmbird_sched_change_guard -+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -+ -+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags); -+ -+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -+ for (struct hmbird_sched_change_guard __cg = \ -+ hmbird_sched_change_guard_init(__rq, __p, __flags); \ -+ !__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags)) -+ -+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+bool task_on_hmbird(struct task_struct *p); -+int hmbird_pre_fork(struct task_struct *p); -+void hmbird_post_fork(struct task_struct *p); -+void hmbird_cancel_fork(struct task_struct *p); -+int hmbird_check_setscheduler(struct task_struct *p, int policy); -+bool hmbird_can_stop_tick(struct rq *rq); -+void init_sched_hmbird_class(void); -+ -+void hmbird_ops_exit(void); -+#define hmbird_ops_error(fmt, ...) \ -+do { \ -+ hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__); \ -+ hmbird_ops_exit(); \ -+} while (0) -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active); -+ -+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight) -+{ -+ return clamp_t(unsigned long, -+ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -+ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); -+} -+ -+static inline void hmbird_notify_pick_next_task(struct rq *rq, -+ struct task_struct *p, -+ const struct sched_class *active) -+{ -+ if (!hmbird_enabled()) -+ return; -+#ifdef CONFIG_SMP -+ /* -+ * Pairs with the smp_load_acquire() issued by a CPU in -+ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a -+ * resched. -+ */ -+ smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1); -+#endif -+ if (!static_branch_unlikely(&hmbird_ops_cpu_preempt)) -+ return; -+ __hmbird_notify_pick_next_task(rq, p, active); -+} -+ -+extern void hmbird_scheduler_tick(void); -+void scan_timeout(struct rq *rq); -+void hmbird_notify_sched_tick(void); -+ -+static inline const struct sched_class *next_active_class(const struct sched_class *class) -+{ -+ class++; -+ -+ if (hmbird_enabled() && class == &fair_sched_class) -+ class++; -+ if (!hmbird_enabled() && class == &hmbird_sched_class) -+ class++; -+ return class; -+} -+ -+#define for_active_class_range(class, _from, _to) \ -+ for (class = (_from); class != (_to); class = next_active_class(class)) -+ -+#define for_each_active_class(class) \ -+ for_active_class_range(class, __sched_class_highest, __sched_class_lowest) -+ -+/* -+ * HMBIRD requires a balance() call before every pick_next_task() call including -+ * when waking up from idle. -+ */ -+#define for_balance_class_range(class, prev_class, end_class) \ -+ for_active_class_range(class, (prev_class) > &hmbird_sched_class ? \ -+ &hmbird_sched_class : (prev_class), (end_class)) -+ -+#define MIN_CGROUP_DL_IDX (5) /* 8ms */ -+#define DEFAULT_CGROUP_DL_IDX (8) /* 64ms */ -+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS]; -+ -+ -+void __hmbird_update_idle(struct rq *rq, bool idle); -+ -+static inline void hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ if (hmbird_enabled()) -+ __hmbird_update_idle(rq, idle); -+} -+ -+int hmbird_ctrl(bool enable); -+ -+int hmbird_tg_online(struct task_group *tg); -+ -+ -+static inline u16 hmbird_task_util(struct task_struct *p) -+{ -+ return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0; -+} -+u16 hmbird_cpu_util(int cpu); -+ -+bool get_hmbird_ops_enabled(void); -+bool get_non_hmbird_task(void); -+void set_cpu_cluster(u64 cpu_cluster); -+#endif /*_EXT_H_*/ -diff --git b/kernel/sched/hmbird/hmbird.c a/kernel/sched/hmbird/hmbird.c -new file mode 100755 -index 000000000000..86f9fd092424 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird.c -@@ -0,0 +1,4354 @@ -+// SPDX-License-Identifier: GPL-2.0 -+ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+ -+#include -+#include -+ -+#include "slim.h" -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+#include -+ -+#define CLUSTER_SEPARATE -+ -+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0); -+atomic_t non_hmbird_task; -+atomic_t hmbird_module_loaded = ATOMIC_INIT(0); -+int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0}; -+ -+enum hmbird_internal_consts { -+ HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ, -+}; -+ -+enum hmbird_ops_enable_state { -+ HMBIRD_OPS_PREPPING, -+ HMBIRD_OPS_ENABLING, -+ HMBIRD_OPS_ENABLED, -+ HMBIRD_OPS_DISABLING, -+ HMBIRD_OPS_DISABLED, -+}; -+ -+static inline void put_hmbird_ts(struct task_struct *p) -+{ -+ kfree((void *)p->android_oem_data1[HMBIRD_TS_IDX]); -+ p->android_oem_data1[HMBIRD_TS_IDX] = 0; -+} -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+/* -+ * hmbird_entity->ops_state -+ * -+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler. -+ * State transitions look as follows: -+ * -+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING -+ * ^ | | -+ * | v v -+ * \-------------------------------/ -+ * -+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call -+ * sites for explanations on the conditions being waited upon and why they are -+ * safe. Transitions out of them into NONE or QUEUED must store_release and the -+ * waiters should load_acquire. -+ * -+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether -+ * any given task can be dispatched by the BPF scheduler at all times and thus -+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler -+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core -+ * can safely reject invalid dispatches. -+ */ -+enum hmbird_ops_state { -+ HMBIRD_OPSS_NONE, /* owned by the HMBIRD core */ -+ HMBIRD_OPSS_QUEUEING, /* in transit to the BPF scheduler */ -+ HMBIRD_OPSS_QUEUED, /* owned by the BPF scheduler */ -+ HMBIRD_OPSS_DISPATCHING, /* in transit back to the HMBIRD core */ -+ -+ /* -+ * QSEQ brands each QUEUED instance so that, when dispatch races -+ * dequeue/requeue, the dispatcher can tell whether it still has a claim -+ * on the task being dispatched. -+ */ -+ HMBIRD_OPSS_QSEQ_SHIFT = 2, -+ HMBIRD_OPSS_STATE_MASK = (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1, -+ HMBIRD_OPSS_QSEQ_MASK = ~HMBIRD_OPSS_STATE_MASK, -+}; -+ -+enum switch_stat { -+ HMBIRD_DISABLED, -+ HMBIRD_SWITCH_PREP, -+ HMBIRD_RQ_SWITCH_BEGIN, -+ HMBIRD_RQ_SWITCH_DONE, -+ HMBIRD_ENABLED, -+}; -+enum switch_stat curr_ss; -+ -+/* -+ * During exit, a task may schedule after losing its PIDs. When disabling the -+ * BPF scheduler, we need to be able to iterate tasks in every state to -+ * guarantee system safety. Maintain a dedicated task list which contains every -+ * task between its fork and eventual free. -+ */ -+DEFINE_SPINLOCK(hmbird_tasks_lock); -+static LIST_HEAD(hmbird_tasks); -+ -+/* ops enable/disable */ -+static struct kthread_worker *hmbird_ops_helper; -+static DEFINE_MUTEX(hmbird_ops_enable_mutex); -+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem); -+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED); -+ -+static bool hmbird_warned_zero_slice; -+enum hmbird_switch_type sw_type; -+static int skip_num[MAX_GLOBAL_DSQS]; -+static int big_distribute_mask_prev; -+static int little_distribute_mask_prev; -+ -+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt); -+ -+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0); -+ -+/* -+ * The maximum amount of time in jiffies that a task may be runnable without -+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger -+ * hmbird_ops_error(). -+ */ -+unsigned long hmbird_watchdog_timeout; -+ -+/* -+ * The last time the delayed work was run. This delayed work relies on -+ * ksoftirqd being able to run to service timer interrupts, so it's possible -+ * that this work itself could get wedged. To account for this, we check that -+ * it's not stalled in the timer tick, and trigger an error if it is. -+ */ -+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES; -+ -+static struct delayed_work hmbird_watchdog_work; -+static struct work_struct hmbird_err_exit_work; -+ -+/* idle tracking */ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_CPUMASK_OFFSTACK -+#define CL_ALIGNED_IF_ONSTACK -+#else -+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp -+#endif -+ -+static struct { -+ cpumask_var_t cpu; -+ cpumask_var_t smt; -+} idle_masks CL_ALIGNED_IF_ONSTACK; -+ -+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus; -+#endif /* CONFIG_SMP */ -+ -+/* dispatch queues */ -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global; -+ -+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128}; -+u32 pcp_dsq_deadline = 20; -+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS]; -+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq); -+ -+static u64 max_hmbird_dsq_internal_id; -+ -+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/ -+#define CLUSTER_SEPARATE_IDX (8) -+#define GDSQS_ID_BASE (3) -+#define UX_COMPATIBLE_IDX (4) -+#define NON_PERIOD_START (5) -+#define NON_PERIOD_END (MAX_GLOBAL_DSQS) -+#define CREATE_DSQ_LEVEL_WITHIN (1) -+ -+struct hmbird_sched_info { -+ spinlock_t lock; -+ int curr_idx[2]; -+ int rtime[MAX_GLOBAL_DSQS]; -+}; -+ -+struct pcp_sched_info { -+ s64 pcp_seq; -+ int rtime; -+ bool pcp_round; -+}; -+ -+/* -+ * pcp_info may rw by another cpu. -+ * protected by rq->lock. -+ */ -+atomic64_t pcp_dsq_round; -+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info); -+ -+struct md_info_t *md_info; -+ -+static int b_rescue_l, l_rescue_b; -+static struct hmbird_sched_info sinfo; -+ -+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC; -+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = { -+ 0, 0, 0, 0, 0, -+ 32 * NSEC_PER_MSEC, -+ 20 * NSEC_PER_MSEC, -+ 14 * NSEC_PER_MSEC, -+ 8 * NSEC_PER_MSEC, -+ 6 * NSEC_PER_MSEC -+}; -+ -+struct cluster_ctx { -+ /* cpu-dsq map must within [lower, upper) */ -+ int upper; -+ int lower; -+ int tidx; -+}; -+ -+enum stat_items { -+ GLOBAL_STAT, -+ CPU_ALLOW_FAIL, -+ RT_CNT, -+ KEY_TASK_CNT, -+ SWITCH_IDX, -+ TIMEOUT_CNT, -+ -+ TOTAL_DSP_CNT, -+ MOVE_RQ_CNT, -+ SELECT_CPU, -+ -+ DWORD_STAT_END = SELECT_CPU, -+ -+ GDSQ_CNT, -+ ERR_IDX, -+ PCP_TIMEOUT_CNT, -+ PCP_LDSQ_CNT, -+ PCP_ENQL_CNT, -+ -+ MAX_ITEMS, -+}; -+static DEFINE_SPINLOCK(stats_lock); -+static char *stats_str[MAX_ITEMS] = { -+ "global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt", -+ "switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt", -+ "select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt", -+ "pcp_ldsq_cnt", "pcp_enql_cnt" -+}; -+ -+ -+struct stats_struct { -+ u64 global_stat[2]; -+ u64 cpu_allow_fail[2]; -+ u64 rt_cnt[2]; -+ u64 key_task_cnt[2]; -+ u64 switch_idx[2]; -+ u64 timeout_cnt[2]; -+ -+ u64 total_dsp_cnt[2]; -+ u64 move_rq_cnt[2]; -+ u64 select_cpu[2]; -+ -+ u64 gdsq_count[MAX_GLOBAL_DSQS][2]; -+ u64 err_idx[5]; -+ u64 pcp_timeout_cnt[NR_CPUS]; -+ u64 pcp_ldsq_count[NR_CPUS][2]; -+ u64 pcp_enql_cnt[NR_CPUS]; -+} stats_data; -+ -+static struct { -+ cpumask_var_t ex_free; -+ cpumask_var_t exclusive; -+ cpumask_var_t partial; -+ cpumask_var_t big; -+ cpumask_var_t little; -+} iso_masks __read_mostly; -+ -+/* -+ * Need more synchronization for these two variables? -+ * I choose not to. -+ */ -+static int l_need_rescue, b_need_rescue; -+ -+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...) \ -+{ \ -+ char buf[MAX_FATAL_INFO]; \ -+ \ -+ scnprintf(buf, MAX_FATAL_INFO, fmt, ##args); \ -+ hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf); \ -+ trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable), \ -+ READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf); \ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); \ -+} \ -+ -+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2) -+{ -+ int c1, c2; -+ struct cpumask mask = {.bits = {0}}; -+ struct cpumask tmp = {.bits = {0}}; -+ -+ if (!slim_stats) -+ return false; -+ -+ if (!rq1 || !rq2) -+ return false; -+ -+ c1 = cpu_of(rq1); -+ c2 = cpu_of(rq2); -+ cpumask_set_cpu(c1, &mask); -+ cpumask_set_cpu(c2, &mask); -+ -+ if (cpumask_and(&tmp, iso_masks.little, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.big, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ if (cpumask_and(&tmp, iso_masks.partial, &mask)) -+ if (cpumask_equal(&tmp, &mask)) -+ return true; -+ -+ return false; -+} -+ -+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu) -+{ -+ unsigned long flags; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ if (!slim_stats) -+ return; -+ -+ switch (item) { -+ case GLOBAL_STAT: -+ fallthrough; -+ case CPU_ALLOW_FAIL: -+ fallthrough; -+ case RT_CNT: -+ fallthrough; -+ case KEY_TASK_CNT: -+ fallthrough; -+ case SWITCH_IDX: -+ fallthrough; -+ case TIMEOUT_CNT: -+ fallthrough; -+ case TOTAL_DSP_CNT: -+ fallthrough; -+ case MOVE_RQ_CNT: -+ fallthrough; -+ case SELECT_CPU: -+ pval = pbase + item * 2 + idx; -+ break; -+ case GDSQ_CNT: -+ pval = &stats_data.gdsq_count[dsq_id][idx]; -+ break; -+ case ERR_IDX: -+ pval = &stats_data.err_idx[idx]; -+ break; -+ case PCP_TIMEOUT_CNT: -+ pval = &stats_data.pcp_timeout_cnt[cpu]; -+ break; -+ case PCP_LDSQ_CNT: -+ pval = &stats_data.pcp_ldsq_count[cpu][idx]; -+ break; -+ case PCP_ENQL_CNT: -+ pval = &stats_data.pcp_enql_cnt[cpu]; -+ break; -+ default: -+ return; -+ } -+ -+ spin_lock_irqsave(&stats_lock, flags); -+ *pval += 1; -+ spin_unlock_irqrestore(&stats_lock, flags); -+} -+ -+static inline bool handle_ret(int ret, int *idx, int len) -+{ -+ if (ret < 0 || ret >= len - *idx) -+ return true; -+ *idx += ret; -+ return false; -+} -+ -+#define PRINT_INTV (5 * HZ) -+void stats_print(char *buf, int len) -+{ -+ int idx = 0, i, j, ret; -+ int item = 0; -+ u64 *pval; -+ u64 *pbase = (u64 *)&stats_data; -+ -+ ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ for (item = 0; item < MAX_ITEMS; item++) { -+ if (item <= DWORD_STAT_END) { -+ pval = pbase + item * 2; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n", -+ stats_str[item], pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == GDSQ_CNT) { -+ for (j = 0; j < MAX_GLOBAL_DSQS; j++) { -+ pval = (u64 *)&stats_data.gdsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == ERR_IDX) { -+ pval = (u64 *)&stats_data.err_idx; -+ ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n", -+ stats_str[item], pval[0], -+ pval[1], pval[2], pval[3], pval[4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } else if (item == PCP_TIMEOUT_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_timeout_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_LDSQ_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_ldsq_count[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n", -+ stats_str[item], j, pval[0], pval[1]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } else if (item == PCP_ENQL_CNT) { -+ for (j = 0; j < nr_cpu_ids; j++) { -+ pval = (u64 *)&stats_data.pcp_enql_cnt[j]; -+ ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n", -+ stats_str[item], j, *pval); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ } -+ } -+ -+ if (!md_info) { -+ buf[idx] = '\0'; -+ return; -+ } -+ -+ ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n"); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_SWITCHS; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "sw_rec[%d] = %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.sw_rec[i].switch_at, -+ md_info->kern_dump.sw_rec[i].is_success, -+ md_info->kern_dump.sw_rec[i].end_state, -+ md_info->kern_dump.sw_rec[i].switch_reason); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ for (i = 0; i < MAX_EXCEP_ID; i++) { -+ ret = snprintf(&buf[idx], len - idx, -+ "excep[%d] = %llu %llu %llu %llu %llu\n", i, -+ md_info->kern_dump.excep_rec[i][0], -+ md_info->kern_dump.excep_rec[i][1], -+ md_info->kern_dump.excep_rec[i][2], -+ md_info->kern_dump.excep_rec[i][3], -+ md_info->kern_dump.excep_rec[i][4]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ -+ ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n", -+ i, md_info->kern_dump.excep_idx[i]); -+ if (handle_ret(ret, &idx, len)) -+ return; -+ } -+ -+ buf[idx] = '\0'; -+} -+ -+enum cpu_type { -+ LITTLE, -+ BIG, -+ PARTIAL, -+ EXCLUSIVE, -+ EX_FREE, -+ INVALID -+}; -+ -+enum dsq_type { -+ GLOBAL_DSQ, -+ PCP_DSQ, -+ OTHER, -+ MAX_DSQ_TYPE, -+}; -+ -+static enum cpu_type cpu_cluster(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (get_hmbird_rq(rq)->exclusive) { -+ return EXCLUSIVE; -+ } else { -+ if (cpumask_test_cpu(cpu, iso_masks.little)) -+ return LITTLE; -+ else if (cpumask_test_cpu(cpu, iso_masks.big)) -+ return BIG; -+ else if (cpumask_test_cpu(cpu, iso_masks.partial)) -+ return PARTIAL; -+ else if (cpumask_test_cpu(cpu, iso_masks.exclusive)) -+ return EXCLUSIVE; -+ else if (cpumask_test_cpu(cpu, iso_masks.ex_free)) -+ return EX_FREE; -+ } -+ return INVALID; -+} -+ -+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq) -+{ -+ if (!dsq) -+ return OTHER; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= GDSQS_ID_BASE) && -+ ((dsq->id & 0xff) < MAX_GLOBAL_DSQS)) -+ return GLOBAL_DSQ; -+ -+ if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) && -+ ((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) && -+ ((dsq->id & 0xff) < max_hmbird_dsq_internal_id)) -+ return PCP_DSQ; -+ -+ return OTHER; -+} -+ -+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq) -+{ -+ enum dsq_type type; -+ -+ type = get_dsq_type(dsq); -+ switch (type) { -+ case GLOBAL_DSQ: -+ case PCP_DSQ: -+ return (dsq->id & 0xff) - GDSQS_ID_BASE; -+ default: -+ return -1; -+ } -+ return -1; -+} -+ -+static void update_cpus_idle(bool set, struct cpumask *mask) -+{ -+ int cpu; -+ struct rq *rq; -+ -+ if (set) { -+ for_each_cpu(cpu, mask) { -+ rq = cpu_rq(cpu); -+ if (is_idle_task(rq->curr)) -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ } -+ } else -+ cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask); -+} -+ -+static void set_partial_status(bool enable, bool little, bool big) -+{ -+ WRITE_ONCE(partial_enable, enable); -+ WRITE_ONCE(l_need_rescue, little); -+ WRITE_ONCE(b_need_rescue, big); -+} -+ -+static bool is_little_need_rescue(void) -+{ -+ return READ_ONCE(l_need_rescue); -+} -+ -+static bool is_big_need_rescue(void) -+{ -+ return READ_ONCE(b_need_rescue); -+} -+ -+static bool is_partial_enabled(void) -+{ -+ return READ_ONCE(partial_enable); -+} -+ -+static bool is_partial_cpu(int cpu) -+{ -+ return cpumask_test_cpu(cpu, iso_masks.partial); -+} -+ -+static void set_iso_par_free(bool enable) -+{ -+ WRITE_ONCE(isolate_ctrl, enable); -+} -+ -+static bool is_iso_par_free(void) -+{ -+ return READ_ONCE(isolate_ctrl); -+} -+ -+static bool skip_update_idle(void) -+{ -+ int cpu = smp_processor_id(); -+ enum cpu_type type = cpu_cluster(cpu); -+ -+ if ((type == EXCLUSIVE && !is_iso_par_free()) || -+ /* partial enable may changed during idle, it doesn't matter. */ -+ (!is_partial_enabled() && type == PARTIAL)) -+ return true; -+ -+ return false; -+} -+ -+static void init_isolate_cpus(void) -+{ -+ WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL)); -+ cpumask_set_cpu(0, iso_masks.big); -+ cpumask_set_cpu(1, iso_masks.big); -+ cpumask_set_cpu(2, iso_masks.big); -+ cpumask_set_cpu(3, iso_masks.big); -+ cpumask_set_cpu(4, iso_masks.big); -+ cpumask_set_cpu(5, iso_masks.ex_free); -+ cpumask_set_cpu(6, iso_masks.partial); -+ cpumask_set_cpu(7, iso_masks.ex_free); -+} -+ -+extern spinlock_t css_set_lock; -+ -+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp) -+{ -+ int i; -+ struct cgroup *anc; -+ -+ spin_lock_irq(&css_set_lock); -+ for (i = 0; i < cgrp->level; i++) { -+ anc = cgrp->ancestors[i]; -+ if (anc->level != CREATE_DSQ_LEVEL_WITHIN) -+ continue; -+ cgroup_get(anc); -+ spin_unlock_irq(&css_set_lock); -+ return anc; -+ } -+ spin_unlock_irq(&css_set_lock); -+ hmbird_err(NO_CGROUP_L1, ":error cgroup = %s\n", cgrp->kn->name); -+ return NULL; -+} -+ -+#define PCP_IDX_BIT (1 << 31) -+ -+static bool is_pcp_rt(struct task_struct *p) -+{ -+ return rt_prio(p->prio) && (p->nr_cpus_allowed == 1); -+} -+ -+static bool is_pcp_idx(int idx) -+{ -+ return idx >= MAX_GLOBAL_DSQS; -+} -+ -+static bool is_critical_system_task(struct task_struct *p) -+{ -+ int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK; -+ -+ return (p->prio < (MAX_RT_PRIO >> 1) && -+ (sp_dl < SCHED_PROP_DEADLINE_LEVEL3)); -+} -+ -+#define ISOLATE_TASK_TOP_BIT (1 << 17) -+#define PIPELINE_TASK_TOP_BIT (1 << 9) -+static bool is_pipeline_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT); -+} -+ -+static bool is_isolate_task(struct task_struct *p) -+{ -+ return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT); -+} -+ -+static bool is_critical_app_task_without_isolate(struct task_struct *p) -+{ -+ return task_is_top_task(p) && !is_isolate_task(p); -+} -+ -+static int find_idx_from_task(struct task_struct *p) -+{ -+ int idx, cpu; -+ int sp_dl; -+ struct task_group *tg = p->sched_task_group; -+ -+ if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) { -+ cpu = cpumask_any(p->cpus_ptr); -+ idx = cpu + MAX_GLOBAL_DSQS; -+ return idx; -+ } -+ -+ if (is_critical_system_task(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL0; -+ goto done; -+ } -+ -+ if (is_critical_app_task_without_isolate(p)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL1; -+ goto done; -+ } -+ -+ if (p->pid == scx_systemui_pid) { -+ idx = SCHED_PROP_DEADLINE_LEVEL4; -+ goto done; -+ } -+ -+ sp_dl = hmbird_get_dsq_id(p); -+ if (sp_dl) { -+ idx = sp_dl; -+ goto done; -+ } -+ -+ if (rt_prio(p->prio)) { -+ idx = SCHED_PROP_DEADLINE_LEVEL3; -+ goto done; -+ } -+ -+ if (tg && tg->css.cgroup && tg->css.cgroup->kn) { -+ if (likely(tg->css.cgroup->kn->id >= 0 && -+ tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS)) -+ idx = cgroup_ids_table[tg->css.cgroup->kn->id]; -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } else -+ idx = DEFAULT_CGROUP_DL_IDX; -+ -+done: -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) { -+ hmbird_err(DSQ_ID_ERR, " : idx error, idx = %d-----\n", idx); -+ idx = DEFAULT_CGROUP_DL_IDX; -+ } -+ return idx; -+} -+ -+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p) -+{ -+ int idx; -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq; -+ -+ if (!p) -+ return NULL; -+ -+ idx = find_idx_from_task(p); -+ if (is_pcp_idx(idx)) { -+ idx -= MAX_GLOBAL_DSQS; -+ dsq = &per_cpu(pcp_ldsq, idx); -+ get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq); -+ slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx); -+ } else { -+ dsq = &gdsqs[idx]; -+ get_hmbird_ts(p)->gdsq_idx = idx; -+ slim_stats_record(GDSQ_CNT, 0, idx, 0); -+ } -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ dsq->last_consume_at = jiffies; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ return dsq; -+} -+ -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq); -+ -+static void set_partial_rescue(bool p_state, bool l_over, bool b_over) -+{ -+ set_partial_status(p_state, l_over, b_over); -+ update_cpus_idle(p_state, iso_masks.partial); -+ hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled()); -+ hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue()); -+ hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue()); -+} -+ -+static void free_isocpu(bool enable) -+{ -+ set_iso_par_free(enable); -+ update_cpus_idle(enable, iso_masks.ex_free); -+ hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free()); -+} -+ -+inline u64 get_hmbird_cpu_util(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed) -+ return 0; -+ u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed); -+ u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size); -+ -+ do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT); -+ -+ return prev_runnable_sum_fixed; -+} -+ -+static inline unsigned int get_scaling_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->max; -+} -+ -+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static u64 get_cpus_max_util(struct cpumask *mask) -+{ -+ int cpu; -+ u64 max = 0; -+ u64 util, ratio; -+ unsigned long effective_cap = 0; -+ -+ for_each_cpu(cpu, mask) { -+ if (slim_walt_ctrl) -+ slim_get_cpu_util(cpu, &util); -+ else -+ util = get_hmbird_cpu_util(cpu); -+ -+ /* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/ -+ if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu))) -+ effective_cap = arch_scale_cpu_capacity(cpu); -+ else -+ effective_cap = arch_scale_cpu_capacity(cpu) * -+ get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu); -+ -+ ratio = util * 100 / effective_cap; -+ hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util); -+ hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n", -+ cpu, (u64)arch_scale_cpu_capacity(cpu)); -+ hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n", -+ cpu, (u64)effective_cap); -+ -+ if (ratio > 100) -+ ratio = 100; -+ -+ if (ratio > max) -+ max = ratio; -+ } -+ return max; -+} -+ -+static bool cluster_need_rescue(struct cpumask *mask, int hres) -+{ -+ return get_cpus_max_util(mask) > hres; -+} -+ -+void partial_dynamic_ctrl(void) -+{ -+ u64 lmax = 0, bmax = 0; -+ bool l_over, l_under, b_over, b_under; -+ static bool last_l_over, last_b_over; -+ static unsigned long last_check; -+ -+ /* Check partial every jiffies. need lock? */ -+ if (time_before_eq(jiffies, READ_ONCE(last_check))) -+ return; -+ -+ WRITE_ONCE(last_check, jiffies); -+ -+ lmax = get_cpus_max_util(iso_masks.little); -+ l_over = lmax > parctrl_high_ratio_l; -+ l_under = lmax < parctrl_low_ratio_l; -+ -+ bmax = get_cpus_max_util(iso_masks.big); -+ b_over = bmax > parctrl_high_ratio; -+ b_under = bmax < parctrl_low_ratio; -+ -+ if (is_partial_enabled() && (l_over || b_over)) { -+ if (last_l_over != l_over || last_b_over != b_over) -+ set_partial_rescue(true, l_over, b_over); -+ } else if (!is_partial_enabled() && (l_over || b_over)) { -+ set_partial_rescue(true, l_over, b_over); -+ } else if (is_partial_enabled() && l_under && b_under) { -+ set_partial_rescue(false, false, false); -+ } -+ last_l_over = l_over; -+ last_b_over = b_over; -+ -+ if (is_partial_enabled()) { -+ if (cpumask_empty(iso_masks.partial)) { -+ bmax = bmax > lmax ? bmax : lmax; -+ } else { -+ bmax = get_cpus_max_util(iso_masks.partial); -+ } -+ if (!is_iso_par_free() && bmax > isoctrl_high_ratio) { -+ hmbird_info_trace("partial max = %llu\n", bmax); -+ free_isocpu(true); -+ } else if (is_iso_par_free() && bmax < isoctrl_low_ratio) -+ free_isocpu(false); -+ } else if (is_iso_par_free()) { -+ free_isocpu(false); -+ } -+} -+ -+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx) -+{ -+ hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx); -+} -+ -+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx) -+{ -+ if (idx < 0 || idx >= MAX_GLOBAL_DSQS) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[idx])) { -+ slim_stats_record(GDSQ_CNT, 1, idx, 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ int i; -+ -+ for (i = 0; i < UX_COMPATIBLE_IDX; i++) { -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ slim_stats_record(GDSQ_CNT, 1, i, 0); -+ return 1; -+ } -+ } -+ return 0; -+} -+ -+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) { -+ slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (list_empty(&dsq->fifo)) -+ goto clear_timeout; -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) -+ goto clear_timeout; -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ hmbird_info_trace( -+ "dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n", -+ dsq_id_to_internal(dsq), entity->task->comm, -+ jiffies, msecs_to_jiffies(deadline), entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1); -+ return; -+ -+clear_timeout: -+ hmbird_info_trace("dsq[%d] clear timeout\n", -+ dsq_id_to_internal(dsq)); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0); -+ dsq->is_timeout = false; -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq)); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime) -+{ -+ hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime); -+} -+ -+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any) -+{ -+ bool is_timeout; -+ int cpu = cpu_of(rq); -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu); -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ is_timeout = dsq->is_timeout; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ /* -+ * dsq->is_timeout may change here, let it be. -+ * it won't cause serious problems. -+ * the same for consume_dispatch_q later. -+ */ -+ if (!is_timeout && !any) -+ return 0; -+ -+ if (consume_dispatch_q(rq, rf, dsq)) { -+ if (is_timeout) { -+ hmbird_info_trace("dsq[%d] consume a pcp timeout task\n", -+ dsq_id_to_internal(dsq)); -+ update_timeout_stats(rq, dsq, pcp_dsq_deadline); -+ slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu); -+ } -+ slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu); -+ return 1; -+ } -+ /* -+ * No pcp task, clear quota. -+ */ -+ if (any) { -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+ } -+ return 0; -+} -+ -+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf) -+{ -+ if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ } -+ return 0; -+} -+ -+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf, -+ int tmp, int cidx, int tidx) -+{ -+ unsigned long flags; -+ -+ if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) { -+ if (tmp != cidx) { -+ spin_lock(&sinfo.lock); -+ sinfo.curr_idx[tidx] = tmp; -+ spin_unlock(&sinfo.lock); -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]); -+ slim_stats_record(SWITCH_IDX, 0, 0, 0); -+ } -+ slim_stats_record(GDSQ_CNT, 1, tmp, 0); -+ -+ raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags); -+ gdsqs[tmp].last_consume_at = jiffies; -+ raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags); -+ return 1; -+ } -+ return 0; -+ -+} -+ -+static int get_cidx(struct cluster_ctx *ctx) -+{ -+ int cidx; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ if (cidx < ctx->lower || cidx >= ctx->upper) { -+ sinfo.curr_idx[ctx->tidx] = ctx->lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]); -+ slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0); -+ cidx = sinfo.curr_idx[ctx->tidx]; -+ } -+ spin_unlock(&sinfo.lock); -+ -+ return cidx; -+} -+ -+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = CLUSTER_SEPARATE_IDX; -+ if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) { -+ pr_debug(" %s iso_masks.little first is %d\n", -+ __func__, cpumask_first(iso_masks.little)); -+ ctx->upper = NON_PERIOD_END; -+ } -+ ctx->tidx = 0; -+ break; -+ case LITTLE: -+ ctx->lower = CLUSTER_SEPARATE_IDX; -+ ctx->upper = NON_PERIOD_END; -+ if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) { -+ pr_debug(" %s iso_masks.big first is %d", -+ __func__, cpumask_first(iso_masks.big)); -+ ctx->lower = NON_PERIOD_START; -+ } -+ ctx->tidx = 1; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ switch (type) { -+ case PARTIAL: -+ if (!is_partial_enabled()) -+ return -1; -+ fallthrough; -+ case BIG: -+ case LITTLE: -+ ctx->lower = NON_PERIOD_START; -+ ctx->upper = NON_PERIOD_END; -+ ctx->tidx = 0; -+ break; -+ default: -+ hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n"); -+ return -1; -+ } -+ return 0; -+} -+ -+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type) -+{ -+ if (cluster_separate) { -+ return gen_cluster_ctx_separate(ctx, type); -+ } else { -+ return gen_cluster_ctx_common(ctx, type); -+ } -+} -+ -+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ int i; -+ bool is_timeout; -+ unsigned long flags; -+ struct cluster_ctx ctx; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ /* Third param means only consume timeout pcp dsq. */ -+ if (consume_pcp_dsq(rq, rf, false)) -+ return 1; -+ -+ for (i = ctx.lower; i < ctx.upper; i++) { -+ raw_spin_lock_irqsave(&gdsqs[i].lock, flags); -+ is_timeout = gdsqs[i].is_timeout; -+ raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags); -+ /* gdsqs[i].is_timeout may change here, let it be... */ -+ if (is_timeout) { -+ /* -+ * consume_dispatch_q will acquire dsq-lock, -+ * So cannot keep lock here, annoy enough. -+ * may rewrite a consume_dispatch_q_locked, TODO. -+ */ -+ if (consume_dispatch_q(rq, rf, &gdsqs[i])) { -+ hmbird_info_trace("dsq[%d] consume a timeout task\n", i); -+ slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0); -+ update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]); -+ return 1; -+ } -+ } -+ } -+ return 0; -+} -+ -+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type) -+{ -+ struct cluster_ctx ctx; -+ int cidx; -+ int tmp; -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return 0; -+ -+ cidx = get_cidx(&ctx); -+ tmp = cidx; -+ do { -+ if (check_pcp_dsq_round(rq, rf)) -+ return 1; -+ if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx)) -+ return 1; -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[tmp] = 0; -+ systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]); -+ tmp++; -+ if (tmp >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ tmp = ctx.lower; -+ } -+ spin_unlock(&sinfo.lock); -+ } while (tmp != cidx); -+ -+ return consume_pcp_dsq(rq, rf, true); -+} -+ -+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf) -+{ -+ enum cpu_type type = cpu_cluster(cpu_of(rq)); -+ bool period_allowed = !get_hmbird_rq(rq)->period_disallow; -+ bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow; -+ -+ switch (type) { -+ case EX_FREE: -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ break; -+ case EXCLUSIVE: -+ if (!is_iso_par_free()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ } -+ /* Only non-period task can run on isolate cpus */ -+ if (!READ_ONCE(iso_free_rescue)) { -+ if (is_big_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ } else { -+ /* Free run, can run on any task. */ -+ if (consume_period_dsq(rq, rf)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ case PARTIAL: -+ if (!is_partial_enabled()) { -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ return 0; -+ } -+ if (is_big_need_rescue()) { -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG)) -+ return 1; -+ } -+ if (is_little_need_rescue()) { -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE)) -+ return 1; -+ } -+ if (consume_pcp_dsq(rq, rf, true)) -+ return 1; -+ break; -+ -+ case BIG: -+ if (period_allowed && consume_period_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (period_allowed && consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ if (is_iso_par_free() || (is_little_need_rescue() && -+ !cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) { -+ if (consume_timeout_dsq(rq, rf, LITTLE)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, LITTLE)) { -+ hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++); -+ return 1; -+ } -+ } -+ break; -+ -+ case LITTLE: -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2)) -+ return 1; -+ if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3)) -+ return 1; -+ if (non_period_allowed && consume_timeout_dsq(rq, rf, type)) -+ return 1; -+ if (consume_ux_dsq(rq, rf)) -+ return 1; -+ if (non_period_allowed && consume_non_period_dsq(rq, rf, type)) -+ return 1; -+ -+ if (is_iso_par_free() || (is_big_need_rescue() && -+ !cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) { -+ if (consume_timeout_dsq(rq, rf, BIG)) -+ return 1; -+ if (consume_non_period_dsq(rq, rf, BIG)) { -+ hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++); -+ return 1; -+ } -+ } -+ break; -+ -+ default: -+ break; -+ } -+ return 0; -+} -+ -+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf) -+{ -+ return consume_hmbird_global_dsq(rq, rf); -+} -+ -+ -+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time) -+{ -+ int idx; -+ -+ /* which dsq belongs to while task enqueue, task will consume its running time. */ -+ idx = get_hmbird_ts(p)->gdsq_idx; -+ /* Only non-period dsq share running time between each other. */ -+ if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id) -+ return; -+ -+ if (idx >= MAX_GLOBAL_DSQS) { -+ per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } else { -+ spin_lock(&sinfo.lock); -+ sinfo.rtime[idx] += exec_time; -+ spin_unlock(&sinfo.lock); -+ systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]); -+ } -+} -+ -+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type) -+{ -+ int cidx; -+ struct cluster_ctx ctx; -+ int cpu = cpu_of(rq); -+ -+ if (gen_cluster_ctx(&ctx, type)) -+ return; -+ -+ spin_lock(&sinfo.lock); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ if (cidx < ctx.lower || cidx >= ctx.upper) { -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ slim_stats_record(ERR_IDX, ctx.tidx, 0, 0); -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ } -+ -+ while (1) { -+ if (per_cpu(pcp_info, cpu).pcp_round) { -+ if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) { -+ hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n", -+ cpu, per_cpu(pcp_info, cpu).rtime); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ per_cpu(pcp_info, cpu).pcp_round = false; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false); -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu_of(rq)).rtime); -+ } -+ } -+ if (sinfo.rtime[cidx] < dsq_quota[cidx]) -+ break; -+ -+ /* clear current dsq rtime */ -+ sinfo.rtime[cidx] = 0; -+ systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]); -+ -+ sinfo.curr_idx[ctx.tidx]++; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) { -+ atomic64_inc(&pcp_dsq_round); -+ hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n", -+ atomic64_read(&pcp_dsq_round)); -+ sinfo.curr_idx[ctx.tidx] = ctx.lower; -+ hmbird_info_systrace("C|9999|cidx_%d|%d\n", -+ ctx.tidx, sinfo.curr_idx[ctx.tidx]); -+ } -+ cidx = sinfo.curr_idx[ctx.tidx]; -+ slim_stats_record(SWITCH_IDX, 1, 0, 0); -+ } -+ spin_unlock(&sinfo.lock); -+} -+ -+ -+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p) -+{ -+ enum cpu_type type; -+ -+ if (!rq || !p) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ switch (type) { -+ case PARTIAL: -+ return; -+ case EXCLUSIVE: -+ return; -+ case EX_FREE: -+ return; -+ default: -+ break; -+ } -+ update_dsq_idx(rq, p, type); -+} -+ -+ -+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline) -+{ -+ struct hmbird_entity *entity; -+ int dsq_id; -+ -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo) || dsq->is_timeout) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ hmbird_deferred_err(SCAN_ENTITY_NULL, -+ " : entity is NULL, dsq->id = %llu\n", dsq->id); -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) { -+ raw_spin_unlock(&dsq->lock); -+ return false; -+ } -+ -+ dsq->is_timeout = true; -+ dsq_id = dsq_id_to_internal(dsq); -+ hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n", -+ dsq_id, entity->task->comm, jiffies, entity->runnable_at); -+ hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1); -+ raw_spin_unlock(&dsq->lock); -+ -+ return true; -+} -+ -+void scan_timeout(struct rq *rq) -+{ -+ int i; -+ int cpu = cpu_of(rq); -+ struct hmbird_dispatch_q *dsq; -+ static u64 last_scan_at; -+ static DEFINE_PER_CPU(u64, pcp_last_scan_at); -+ -+ if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu))) -+ return; -+ per_cpu(pcp_last_scan_at, cpu) = jiffies; -+ -+ dsq = &per_cpu(pcp_ldsq, cpu); -+ scan_dsq_timeout(rq, dsq, pcp_dsq_deadline); -+ -+ if (time_before_eq(jiffies, (unsigned long)last_scan_at)) -+ return; -+ last_scan_at = jiffies; -+ -+ for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) { -+ dsq = &gdsqs[i]; -+ scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]); -+ } -+} -+ -+/*******************************Initialize***********************************/ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id); -+static void init_dsq_at_boot(void) -+{ -+ int i, cpu; -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i)); -+ } -+ for_each_possible_cpu(cpu) -+ init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN | -+ (GDSQS_ID_BASE + i + cpu)); -+ -+ max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu; -+ spin_lock_init(&sinfo.lock); -+} -+ -+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx) -+{ -+ if (ids < 0 || ids >= NUMS_CGROUP_KINDS) { -+ pr_err("update_cgroup_ids_tab idx err!\n"); -+ return; -+ } -+ cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx; -+} -+ -+static int cgrp_name_to_idx(struct cgroup *cgrp) -+{ -+ int idx; -+ -+ if (!cgrp) -+ return -1; -+ -+ if (!strcmp(cgrp->kn->name, "display") -+ || !strcmp(cgrp->kn->name, "multimedia")) -+ idx = 5; /* 8ms */ -+ else if (!strcmp(cgrp->kn->name, "top-app") -+ || !strcmp(cgrp->kn->name, "ss-top")) -+ idx = 6; /* 16ms */ -+ else if (!strcmp(cgrp->kn->name, "ssfg") -+ || !strcmp(cgrp->kn->name, "foreground")) -+ idx = 7; /* 32ms */ -+ else if (!strcmp(cgrp->kn->name, "bg") -+ || !strcmp(cgrp->kn->name, "log") -+ || !strcmp(cgrp->kn->name, "dex2oat") -+ || !strcmp(cgrp->kn->name, "background")) -+ idx = 9; /* 128ms */ -+ else -+ idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */ -+ -+ return idx; -+} -+ -+static void init_root_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX); -+} -+ -+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) { -+ pr_err("%s idx err!\n", __func__); -+ return; -+ } -+ -+ if (cgroup_ids_table[cgrp->kn->id] == -1) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp)); -+} -+ -+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg) -+{ -+ struct cgroup *l1cgrp; -+ -+ if (!cgrp || !tg || !(cgrp->kn)) -+ return; -+ -+ l1cgrp = cgroup_ancestor_l1(cgrp); -+ if (l1cgrp) -+ update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp)); -+ cgroup_put(l1cgrp); -+} -+ -+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg) -+{ -+ switch (cgrp->level) { -+ case 0: -+ init_root_tg(cgrp, tg); -+ break; -+ case 1: -+ init_level1_tg(cgrp, tg); -+ break; -+ default: -+ init_child_tg(cgrp, tg); -+ break; -+ } -+} -+ -+/**************************************************************************/ -+ -+struct hmbird_task_iter { -+ struct hmbird_entity cursor; -+ struct task_struct *locked; -+ struct rq *rq; -+ struct rq_flags rf; -+}; -+ -+/** -+ * hmbird_task_iter_init - Initialize a task iterator -+ * @iter: iterator to init -+ * -+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized, -+ * @iter must eventually be exited with hmbird_task_iter_exit(). -+ * -+ * hmbird_tasks_lock may be released between this and the first next() call or -+ * between any two next() calls. If hmbird_tasks_lock is released between two -+ * next() calls, the caller is responsible for ensuring that the task being -+ * iterated remains accessible either through RCU read lock or obtaining a -+ * reference count. -+ * -+ * All tasks which existed when the iteration started are guaranteed to be -+ * visited as long as they still exist. -+ */ -+static void hmbird_task_iter_init(struct hmbird_task_iter *iter) -+{ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR }; -+ list_add(&iter->cursor.tasks_node, &hmbird_tasks); -+ iter->locked = NULL; -+} -+ -+/** -+ * hmbird_task_iter_exit - Exit a task iterator -+ * @iter: iterator to exit -+ * -+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held. -+ * If the iterator holds a task's rq lock, that rq lock is released. See -+ * hmbird_task_iter_init() for details. -+ */ -+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ if (list_empty(cursor)) -+ return; -+ -+ list_del_init(cursor); -+} -+ -+/** -+ * hmbird_task_iter_next - Next task -+ * @iter: iterator to walk -+ * -+ * Visit the next task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter) -+{ -+ struct list_head *cursor = &iter->cursor.tasks_node; -+ struct hmbird_entity *pos; -+ -+ lockdep_assert_held(&hmbird_tasks_lock); -+ -+ list_for_each_entry(pos, cursor, tasks_node) { -+ if (&pos->tasks_node == &hmbird_tasks) -+ return NULL; -+ if (!(pos->flags & HMBIRD_TASK_CURSOR)) { -+ list_move(cursor, &pos->tasks_node); -+ return pos->task; -+ } -+ } -+ -+ /* can't happen, should always terminate at hmbird_tasks above */ -+ hmbird_deferred_err(ITER_RET_NULL, " : unreachable path in scx_task_iter_next\n"); -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered - Next non-idle task -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task. See hmbird_task_iter_init() for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ while ((p = hmbird_task_iter_next(iter))) { -+ if (!is_idle_task(p)) -+ return p; -+ } -+ return NULL; -+} -+ -+/** -+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked -+ * @iter: iterator to walk -+ * -+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init() -+ * for details. -+ */ -+static struct task_struct * -+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter) -+{ -+ struct task_struct *p; -+ -+ if (iter->locked) { -+ task_rq_unlock(iter->rq, iter->locked, &iter->rf); -+ iter->locked = NULL; -+ } -+ -+ p = hmbird_task_iter_next_filtered(iter); -+ if (!p) -+ return NULL; -+ -+ iter->rq = task_rq_lock(p, &iter->rf); -+ iter->locked = p; -+ return p; -+} -+ -+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void) -+{ -+ return atomic_read(&hmbird_ops_enable_state_var); -+} -+ -+static enum hmbird_ops_enable_state -+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to) -+{ -+ return atomic_xchg(&hmbird_ops_enable_state_var, to); -+} -+ -+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to, -+ enum hmbird_ops_enable_state from) -+{ -+ int from_v = from; -+ -+ return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to); -+} -+ -+static bool hmbird_ops_disabling(void) -+{ -+ return false; -+} -+ -+/** -+ * wait_ops_state - Busy-wait the specified ops state to end -+ * @p: target task -+ * @opss: state to wait the end of -+ * -+ * Busy-wait for @p to transition out of @opss. This can only be used when the -+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also -+ * has load_acquire semantics to ensure that the caller can see the updates made -+ * in the enqueueing and dispatching paths. -+ */ -+static void wait_ops_state(struct task_struct *p, u64 opss) -+{ -+ do { -+ cpu_relax(); -+ } while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss); -+} -+ -+ -+static void update_curr_hmbird(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ u64 now = rq_clock_task(rq); -+ u64 delta_exec; -+ -+ if (time_before_eq64(now, curr->se.exec_start)) -+ return; -+ -+ delta_exec = now - curr->se.exec_start; -+ curr->se.exec_start = now; -+ update_runningtime(rq, curr, delta_exec); -+ curr->se.sum_exec_runtime += delta_exec; -+ account_group_exec_runtime(curr, delta_exec); -+ cgroup_account_cputime(curr, delta_exec); -+ -+ if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF) -+ get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec); -+ -+ trace_sched_stat_runtime(curr, delta_exec, 0); -+} -+ -+static bool hmbird_dsq_priq_less(struct rb_node *node_a, -+ const struct rb_node *node_b) -+{ -+ const struct hmbird_entity *a = -+ container_of(node_a, struct hmbird_entity, dsq_node.priq); -+ const struct hmbird_entity *b = -+ container_of(node_b, struct hmbird_entity, dsq_node.priq); -+ -+ return time_before64(a->dsq_vtime, b->dsq_vtime); -+} -+ -+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ bool is_local = dsq->id == HMBIRD_DSQ_LOCAL; -+ unsigned long flags; -+ -+ hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq || -+ !list_empty(&get_hmbird_ts(p)->dsq_node.fifo), -+ "task = %s, dsq->id = %llu\n", p->comm, dsq->id); -+ hmbird_cond_deferred_err(ENQ_EXIST2, -+ (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq), -+ "task = %s\n", p->comm); -+ -+ if (!is_local) { -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ); -+ hmbird_ops_error(": %s\n", -+ "attempting to dispatch to a destroyed dsq"); -+ /* fall back to the global dsq */ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ dsq = &hmbird_dsq_global; -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ } -+ } -+ -+ if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) { -+ get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ; -+ rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq, -+ hmbird_dsq_priq_less); -+ } else { -+ if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT)) -+ list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ else -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo); -+ } -+ dsq->nr++; -+ get_hmbird_ts(p)->dsq = dsq; -+ -+ /* -+ * We're transitioning out of QUEUEING or DISPATCHING. store_release to -+ * match waiters' load_acquire. -+ */ -+ if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS) -+ atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE); -+ -+ if (is_local) { -+ struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq); -+ struct rq *rq = hmbird->rq; -+ bool preempt = false; -+ -+ if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr && -+ rq->curr->sched_class == &hmbird_sched_class) { -+ get_hmbird_ts(rq->curr)->slice = 0; -+ preempt = true; -+ } -+ -+ if (preempt || sched_class_above(&hmbird_sched_class, -+ rq->curr->sched_class)) -+ resched_curr(rq); -+ } else { -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ } -+} -+ -+static void task_unlink_from_dsq(struct task_struct *p, -+ struct hmbird_dispatch_q *dsq) -+{ -+ if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) { -+ rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ; -+ } else { -+ list_del_init(&get_hmbird_ts(p)->dsq_node.fifo); -+ } -+} -+ -+static bool task_linked_on_dsq(struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) || -+ !RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+} -+ -+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p) -+{ -+ unsigned long flags; -+ struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq; -+ bool is_local = dsq == &hmbird_rq->local_dsq; -+ -+ if (!dsq) { -+ hmbird_cond_deferred_err(TASK_LINKED1, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ /* -+ * When dispatching directly from the BPF scheduler to a local -+ * DSQ, the task isn't associated with any DSQ but -+ * @get_hmbird_ts(p)->holding_cpu may be set under the protection of -+ * %HMBIRD_OPSS_DISPATCHING. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu >= 0) -+ get_hmbird_ts(p)->holding_cpu = -1; -+ return; -+ } -+ -+ if (!is_local) -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ /* -+ * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node -+ * can't change underneath us. -+ */ -+ if (get_hmbird_ts(p)->holding_cpu < 0) { -+ /* @p must still be on @dsq, dequeue */ -+ hmbird_cond_deferred_err(TASK_UNLINKED, -+ !task_linked_on_dsq(p), "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ } else { -+ /* -+ * We're racing against dispatch_to_local_dsq() which already -+ * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the -+ * holding_cpu which tells dispatch_to_local_dsq() that it lost -+ * the race. -+ */ -+ hmbird_cond_deferred_err(TASK_LINKED2, -+ task_linked_on_dsq(p), "task = %s\n", p->comm); -+ get_hmbird_ts(p)->holding_cpu = -1; -+ } -+ get_hmbird_ts(p)->dsq = NULL; -+ -+ if (!is_local) -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+} -+ -+ -+static bool test_rq_online(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ return rq->online; -+#else -+ return true; -+#endif -+} -+ -+static void refill_task_slice(struct task_struct *p) -+{ -+ if (is_isolate_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO; -+ else if (is_pipeline_task(p)) -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2; -+ else -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+} -+ -+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -+ int sticky_cpu) -+{ -+ struct hmbird_dispatch_q *d; -+ s32 cpu = -1; -+ -+ hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), -+ "task = %s\n", p->comm); -+ -+ if (is_pcp_rt(p)) { -+ /* Enqueue percpu rt task to local directly. */ -+ /* Or cause a bug when disable dispatch. */ -+ if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ } -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (cpu >= 0) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ -+ if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ enq_flags |= HMBIRD_ENQ_LOCAL; -+ clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ } -+ /* rq migration */ -+ if (sticky_cpu == cpu_of(rq)) -+ goto local_norefill; -+ /* -+ * If !rq->online, we already told the BPF scheduler that the CPU is -+ * offline. We're just trying to on/offline the CPU. Don't bother the -+ * BPF scheduler. -+ */ -+ if (unlikely(!test_rq_online(rq))) -+ goto local; -+ -+ /* see %HMBIRD_OPS_ENQ_LAST */ -+ if (enq_flags & HMBIRD_ENQ_LAST) -+ goto local; -+ -+ if (enq_flags & HMBIRD_ENQ_LOCAL) -+ goto local; -+ else -+ goto global; -+local: -+ /* -+ * For task-ordering, slice refill must be treated as implying the end -+ * of the current slice. Otherwise, the longer @p stays on the CPU, the -+ * higher priority it becomes from hmbird_prio_less()'s POV. -+ */ -+ refill_task_slice(p); -+local_norefill: -+ dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags); -+ slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq)); -+ return; -+ -+global: -+ d = find_dsq_from_task(p); -+ if (d) { -+ refill_task_slice(p); -+ dispatch_enqueue(d, p, enq_flags); -+ return; -+ } -+ slim_stats_record(GLOBAL_STAT, 0, 0, 0); -+ refill_task_slice(p); -+ dispatch_enqueue(&hmbird_dsq_global, p, enq_flags); -+} -+ -+static bool watchdog_task_watched(const struct task_struct *p) -+{ -+ return !list_empty(&get_hmbird_ts(p)->watchdog_node); -+} -+ -+static void watchdog_watch_task(struct rq *rq, struct task_struct *p) -+{ -+ lockdep_assert_rq_held(rq); -+ if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ get_hmbird_ts(p)->runnable_at = jiffies; -+ clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list); -+} -+ -+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) -+{ -+ list_del_init(&get_hmbird_ts(p)->watchdog_node); -+ if (reset_timeout) -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags) -+{ -+ int sticky_cpu = get_hmbird_ts(p)->sticky_cpu; -+ -+ enq_flags |= get_hmbird_rq(rq)->extra_enq_flags; -+ -+ if (sticky_cpu >= 0) -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ -+ /* -+ * Restoring a running task will be immediately followed by -+ * set_next_task_hmbird() which expects the task to not be on the BPF -+ * scheduler as tasks can only start running through local DSQs. Force -+ * direct-dispatch into the local DSQ by setting the sticky_cpu. -+ */ -+ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) -+ sticky_cpu = cpu_of(rq); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_UNWATCHED, -+ !watchdog_task_watched(p), "task = %s\n", p->comm); -+ return; -+ } -+ -+ watchdog_watch_task(rq, p); -+ set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ get_hmbird_rq(rq)->nr_running++; -+ add_nr_running(rq, 1); -+ -+ do_enqueue_task(rq, p, enq_flags, sticky_cpu); -+} -+ -+static void ops_dequeue(struct task_struct *p, u64 deq_flags) -+{ -+ u64 opss; -+ -+ watchdog_unwatch_task(p, false); -+ -+ /* acquire ensures that we see the preceding updates on QUEUED */ -+ opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state); -+ -+ switch (opss & HMBIRD_OPSS_STATE_MASK) { -+ case HMBIRD_OPSS_NONE: -+ break; -+ case HMBIRD_OPSS_QUEUEING: -+ /* -+ * QUEUEING is started and finished while holding @p's rq lock. -+ * As we're holding the rq lock now, we shouldn't see QUEUEING. -+ */ -+ hmbird_deferred_err(DEQ_DEQING, " : unreachable path in %s\n", __func__); -+ break; -+ case HMBIRD_OPSS_QUEUED: -+ if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss, -+ HMBIRD_OPSS_NONE)) -+ break; -+ fallthrough; -+ case HMBIRD_OPSS_DISPATCHING: -+ /* -+ * If @p is being dispatched from the BPF scheduler to a DSQ, -+ * wait for the transfer to complete so that @p doesn't get -+ * added to its DSQ after dequeueing is complete. -+ * -+ * As we're waiting on DISPATCHING with the rq locked, the -+ * dispatching side shouldn't try to lock the rq while -+ * DISPATCHING is set. See dispatch_to_local_dsq(). -+ * -+ * DISPATCHING shouldn't have qseq set and control can reach -+ * here with NONE @opss from the above QUEUED case block. -+ * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss. -+ */ -+ wait_ops_state(p, HMBIRD_OPSS_DISPATCHING); -+ hmbird_cond_deferred_err(HMBIRD_OPN, -+ atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE, -+ "task = %s\n", p->comm); -+ break; -+ } -+} -+ -+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ -+ if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p), -+ "task = %s\n", p->comm); -+ return; -+ } -+ -+ ops_dequeue(p, deq_flags); -+ -+ if (slim_walt_ctrl) { -+ if (task_current(rq, p)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ if (deq_flags & HMBIRD_DEQ_SLEEP) -+ set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else -+ clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), -+ (unsigned long *)&get_hmbird_ts(p)->flags); -+ -+ clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm); -+ hmbird_rq->nr_running--; -+ sub_nr_running(rq, 1); -+ dispatch_dequeue(hmbird_rq, p); -+} -+ -+static void yield_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ get_hmbird_ts(p)->slice = 0; -+} -+ -+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to) -+{ -+ return false; -+} -+ -+#ifdef CONFIG_SMP -+/** -+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ -+ * @rq: rq to move the task into, currently locked -+ * @p: task to move -+ * @enq_flags: %HMBIRD_ENQ_* -+ * -+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller -+ * must: -+ * -+ * 1. Start with exclusive access to @p either through its DSQ lock or -+ * %HMBIRD_OPSS_DISPATCHING flag. -+ * -+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id(). -+ * -+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't -+ * deadlock with dequeue. -+ * -+ * 4. Lock @rq and the task_rq from #3. -+ * -+ * 5. Call this function. -+ * -+ * Returns %true if @p was successfully moved. %false after racing dequeue and -+ * losing. -+ */ -+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, -+ u64 enq_flags) -+{ -+ struct rq *task_rq; -+ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * If dequeue got to @p while we were trying to lock both rq's, it'd -+ * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have -+ * updated it to different values afterwards, as this operation can't be -+ * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become -+ * raw_smp_processor_id() again before we're done. Thus, we can tell -+ * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is -+ * still raw_smp_processor_id(). -+ * -+ * See dispatch_dequeue() for the counterpart. -+ */ -+ if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id())) -+ return false; -+ -+ /* @p->rq couldn't have changed if we're still the holding cpu */ -+ task_rq = task_rq(p); -+ lockdep_assert_rq_held(task_rq); -+ deactivate_task(task_rq, p, 0); -+ set_task_cpu(p, cpu_of(rq)); -+ get_hmbird_ts(p)->sticky_cpu = cpu_of(rq); -+ -+ /* -+ * We want to pass hmbird-specific enq_flags but activate_task() will -+ * truncate the upper 32 bit. As we own @rq, we can pass them through -+ * @get_hmbird_rq(rq)->extra_enq_flags instead. -+ */ -+ hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags, -+ "task = %s\n", p->comm); -+ get_hmbird_rq(rq)->extra_enq_flags = enq_flags; -+ activate_task(rq, p, 0); -+ get_hmbird_rq(rq)->extra_enq_flags = 0; -+ -+ return true; -+} -+ -+#endif /* CONFIG_SMP */ -+ -+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu) -+{ -+ int fitable = 1; -+ -+ return fitable; -+} -+ -+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq, -+ struct hmbird_dispatch_q *dsq) -+{ -+ bool dsq_misfit; -+ int cpu = cpu_of(rq); -+ u64 task_util = 0; -+ struct cluster_ctx ctx; -+ int dsq_int = dsq_id_to_internal(dsq); -+ -+ if (!cpumask_test_cpu(cpu, iso_masks.little)) -+ return false; -+ if (p->pid == scx_systemui_pid) -+ return true; -+ -+ gen_cluster_ctx(&ctx, BIG); -+ dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 && -+ dsq_int <= SCHED_PROP_DEADLINE_LEVEL4); -+#ifdef CLUSTER_SEPARATE -+ /* In rescue mode, little will consume big cluster's dsq.*/ -+ dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper); -+#endif -+ if (!dsq_misfit) -+ return false; -+ -+ if (p) { -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &task_util); -+ else -+ task_util = get_hmbird_ts(p)->demand_scaled; -+ } -+ -+ if (task_util <= misfit_ds) -+ return false; -+ -+ hmbird_info_trace(":task %s can't run on cpu%d, util = %llu\n", -+ p->comm, cpu, task_util); -+ return true; -+} -+ -+static int check_misfit_task_on_fake_big(struct task_struct *p, struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (likely(p->pid != scx_systemui_pid)) -+ return false; -+ -+ if (topology_cluster_id(num_possible_cpus() - 1) > 1 && -+ arch_scale_cpu_capacity(cpu) == arch_scale_cpu_capacity(0) && -+ (parctrl_high_ratio <= 0 || parctrl_high_ratio_l <= 0)) -+ return true; -+ -+ return false; -+} -+ -+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq) -+{ -+ if (!cpumask_test_cpu(cpu_of(rq), task_cpu_possible_mask(p))) -+ return false; -+ -+ if (!task_fits_cpu_hmbird(p, cpu_of(rq))) -+ return false; -+ -+ if (check_misfit_task_on_little(p, rq, dsq)) -+ return false; -+ -+ if (check_misfit_task_on_fake_big(p, rq)) -+ return false; -+ -+ return likely(test_rq_online(rq)); -+} -+ -+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return; -+ -+ if (add) -+ skipn[idx]++; -+ else -+ skipn[idx] = 0; -+} -+ -+static bool skip_too_much(struct hmbird_dispatch_q *dsq) -+{ -+ int idx = dsq_id_to_internal(dsq); -+ int type = get_dsq_type(dsq); -+ -+ if (type != GLOBAL_DSQ) -+ return false; -+ -+ if (skip_num[idx] > 3) { -+ skip_num[idx] = 0; -+ return true; -+ } -+ -+ return false; -+} -+ -+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, -+ struct hmbird_dispatch_q *dsq) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rb_node *rb_node; -+ struct rq *task_rq; -+ unsigned long flags; -+ bool moved = false; -+ struct task_struct *may_fit = NULL; -+ int skip = 0; -+ -+retry: -+ if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) -+ return false; -+ -+ raw_spin_lock_irqsave(&dsq->lock, flags); -+ -+ list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) { -+ set_skip_num(dsq, skip_num, (bool)may_fit); -+ goto this_rq; -+ } -+ if (skip_too_much(dsq)) -+ goto remote_rq; -+ if (!may_fit) -+ may_fit = p; -+ if (++skip <= 3) -+ continue; -+ /* -+ * If the recent 3 tasks not fit, use the first one. -+ * and clear the skip, because the first one is consumed. -+ */ -+ set_skip_num(dsq, skip_num, false); -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ /* No more task, use the first may fit task.*/ -+ if (may_fit) { -+ p = may_fit; -+ task_rq = task_rq(p); -+ goto remote_rq; -+ } -+ -+ for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ p = entity->task; -+ task_rq = task_rq(p); -+ if (!task_can_run_on_rq(p, rq, dsq)) -+ continue; -+ if (rq == task_rq) -+ goto this_rq; -+ goto remote_rq; -+ } -+ -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ return false; -+ -+this_rq: -+ /* @dsq is locked and @p is on this rq */ -+ hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo); -+ dsq->nr--; -+ hmbird_rq->local_dsq.nr++; -+ get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq; -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ -+remote_rq: -+#ifdef CONFIG_SMP -+ if (cpu_same_cluster_stat(p, rq, task_rq)) -+ slim_stats_record(MOVE_RQ_CNT, 0, 0, 0); -+ else -+ slim_stats_record(MOVE_RQ_CNT, 1, 0, 0); -+ /* -+ * @dsq is locked and @p is on a remote rq. @p is currently protected by -+ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab -+ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the -+ * rq lock or fail, do a little dancing from our side. See -+ * move_task_to_local_dsq(). -+ */ -+ hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0, -+ "task = %s\n", p->comm); -+ task_unlink_from_dsq(p, dsq); -+ dsq->nr--; -+ get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id(); -+ raw_spin_unlock_irqrestore(&dsq->lock, flags); -+ -+ rq_unpin_lock(rq, rf); -+ double_lock_balance(rq, task_rq); -+ rq_repin_lock(rq, rf); -+ -+ moved = move_task_to_local_dsq(rq, p, 0); -+ -+ double_unlock_balance(rq, task_rq); -+#endif /* CONFIG_SMP */ -+ if (likely(moved)) { -+ slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0); -+ return true; -+ } -+ may_fit = NULL; -+ goto retry; -+} -+ -+ -+static int balance_one(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf, bool local) -+{ -+ struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq); -+ bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class; -+ -+ if (!hmbird_rq) -+ return 1; -+ -+ lockdep_assert_rq_held(rq); -+ -+ if (static_branch_unlikely(&hmbird_ops_cpu_preempt) && -+ unlikely(get_hmbird_rq(rq)->cpu_released)) { -+ /* -+ * If the previous sched_class for the current CPU was not HMBIRD, -+ * notify the BPF scheduler that it again has control of the -+ * core. This callback complements ->cpu_release(), which is -+ * emitted in hmbird_notify_pick_next_task(). -+ */ -+ get_hmbird_rq(rq)->cpu_released = false; -+ } -+ -+ if (prev_on_hmbird) -+ update_curr_hmbird(rq); -+ /* if there already are tasks to run, nothing to do */ -+ if (hmbird_rq->local_dsq.nr) -+ return 1; -+ -+ if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) { -+ slim_stats_record(GLOBAL_STAT, 1, 0, 0); -+ return 1; -+ } -+ -+ if (consume_dispatch_global(rq, rf)) -+ return 1; -+ -+ return 0; -+} -+ -+static int balance_hmbird(struct rq *rq, struct task_struct *prev, -+ struct rq_flags *rf) -+{ -+ return balance_one(rq, prev, rf, true); -+} -+ -+/* -+ * output task util to systrace, only for debug mode. -+ * we can not output too many logs to systrace buffer even in debug mode -+ * only output debug-info while it exceed misfit_ds. -+ */ -+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p) -+{ -+ static DEFINE_PER_CPU(int, is_last_exceed); -+ int cpu = cpu_of(rq); -+ u64 util = 0; -+ -+ if (likely(!debug_enabled())) -+ return; -+ -+ if (!p) -+ return; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ util = uclamp_rq_util_with(rq, util, p); -+ -+ if (util >= misfit_ds) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util); -+ per_cpu(is_last_exceed, cpu) = true; -+ } else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) { -+ hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0); -+ per_cpu(is_last_exceed, cpu) = false; -+ } else { -+ } -+} -+ -+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first) -+{ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ /* -+ * Core-sched might decide to execute @p before it is -+ * dispatched. Call ops_dequeue() to notify the BPF scheduler. -+ */ -+ ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC); -+ dispatch_dequeue(get_hmbird_rq(rq), p); -+ } -+ -+ p->se.exec_start = rq_clock_task(rq); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK); -+ } -+ -+ watchdog_unwatch_task(p, true); -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx); -+ systrace_output_cpu_ds(rq, p); -+ /* -+ * @p is getting newly scheduled or got kicked after someone updated its -+ * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird(). -+ */ -+ if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) != -+ (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) { -+ if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) -+ get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK; -+ else -+ get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK; -+ -+ sched_update_tick_dependency(rq); -+ } -+ -+ p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime; -+} -+ -+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p) -+{ -+ update_curr_hmbird(rq); -+ -+ update_dispatch_dsq_info(rq, p); -+ -+ if (slim_walt_ctrl) { -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK); -+ } -+ -+ slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0); -+ -+ if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ watchdog_watch_task(rq, p); -+ -+ if (is_pipeline_task(p)) { -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1); -+ return; -+ } -+ -+ /* -+ * If we're in the pick_next_task path, balance_hmbird() should -+ * have already populated the local DSQ if there are any other -+ * available tasks. If empty, tell ops.enqueue() that @p is the -+ * only one available for this cpu. ops.enqueue() should put it -+ * on the local DSQ so that the subsequent pick_next_task_hmbird() -+ * can find the task unless it wants to trigger a separate -+ * follow-up scheduling event. -+ */ -+ if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) -+ do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1); -+ else -+ do_enqueue_task(rq, p, 0, -1); -+ } -+} -+ -+static struct task_struct *first_local_task(struct rq *rq) -+{ -+ struct rb_node *rb_node; -+ struct hmbird_entity *entity; -+ -+ if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) { -+ entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo, -+ struct hmbird_entity, dsq_node.fifo); -+ return entity->task; -+ } -+ -+ rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq); -+ if (rb_node) { -+ entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq); -+ return entity->task; -+ } -+ return NULL; -+} -+ -+static struct task_struct *pick_next_task_hmbird(struct rq *rq) -+{ -+ struct task_struct *p; -+ -+ p = first_local_task(rq); -+ if (!p) -+ return NULL; -+ -+ if (unlikely(!get_hmbird_ts(p)->slice)) { -+ if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice) -+ hmbird_warned_zero_slice = true; -+ -+ refill_task_slice(p); -+ } -+ -+ set_next_task_hmbird(rq, p, true); -+ -+ return p; -+} -+ -+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task, -+ const struct sched_class *active) -+{ -+ lockdep_assert_rq_held(rq); -+ -+ /* -+ * The callback is conceptually meant to convey that the CPU is no -+ * longer under the control of HMBIRD. Therefore, don't invoke the -+ * callback if the CPU is staying on HMBIRD, or going idle (in which -+ * case the HMBIRD scheduler has actively decided not to schedule any -+ * tasks on the CPU). -+ */ -+ if (likely(active >= &hmbird_sched_class)) -+ return; -+ -+ /* -+ * At this point we know that HMBIRD was preempted by a higher priority -+ * sched_class, so invoke the ->cpu_release() callback if we have not -+ * done so already. We only send the callback once between HMBIRD being -+ * preempted, and it regaining control of the CPU. -+ * -+ * ->cpu_release() complements ->cpu_acquire(), which is emitted the -+ * next time that balance_hmbird() is invoked. -+ */ -+ if (!get_hmbird_rq(rq)->cpu_released) -+ get_hmbird_rq(rq)->cpu_released = true; -+} -+ -+#ifdef CONFIG_SMP -+ -+static bool test_and_clear_cpu_idle(int cpu) -+{ -+ if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { -+ if (cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ return true; -+ } else { -+ return false; -+ } -+} -+ -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) -+{ -+ int cpu; -+ -+ do { -+ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); -+ if (cpu >= nr_cpu_ids) -+ return -EBUSY; -+ } while (!test_and_clear_cpu_idle(cpu)); -+ -+ return cpu; -+} -+ -+ -+static bool prev_cpu_misfit(int prev) -+{ -+ if (!is_partial_enabled() && is_partial_cpu(prev)) -+ return true; -+ -+ return false; -+} -+ -+static int heavy_rt_placement(struct task_struct *p, int prev) -+{ -+ struct cpumask tmp = {.bits = {0}}; -+ int cpu; -+ u64 util = 0; -+ -+ if (!rt_prio(p->prio)) -+ return -EFAULT; -+ -+ if (slim_walt_ctrl) -+ slim_get_task_util(p, &util); -+ else -+ util = get_hmbird_ts(p)->demand_scaled; -+ -+ if (util < misfit_ds) -+ return -EFAULT; -+ -+ if (is_partial_enabled()) -+ cpumask_or(&tmp, iso_masks.big, iso_masks.partial); -+ else -+ cpumask_copy(&tmp, iso_masks.big); -+ -+ cpu = hmbird_pick_idle_cpu(&tmp); -+ if (cpu >= 0) -+ return cpu; -+ -+ if (cpumask_test_cpu(prev, iso_masks.big) || -+ (is_partial_enabled() && is_partial_cpu(prev))) -+ return prev; -+ -+ return cpumask_first(&tmp); -+} -+ -+static int spec_task_before_pick_idle(struct task_struct *p, int prev) -+{ -+ int cpu; -+ -+ cpu = heavy_rt_placement(p, prev); -+ if (cpu >= 0) -+ return cpu; -+ return -EFAULT; -+} -+ -+static int cpumask_distribute_next(struct cpumask *mask, int *prev) -+{ -+ int p = *prev, n; -+ -+ n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1); -+ if (n < nr_cpu_ids) -+ WRITE_ONCE(*prev, n); -+ return n; -+} -+ -+static int repick_fallback_cpu(void) -+{ -+ /* -+ * partial cpu follow big cluster's Scheduling policy, -+ * simply return first bit cpu. -+ */ -+ return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev); -+} -+ -+/* -+ * Must return a valid cpu num, as this task's cpu. -+ */ -+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu, -+ struct cpumask *mask, int *prev_mask) -+{ -+ int cpu; -+ -+ cpu = hmbird_pick_idle_cpu(mask); -+ if (cpu >= 0) -+ return cpu; -+ return cpumask_distribute_next(mask, prev_mask); -+} -+ -+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type) -+{ -+ int idx; -+ struct cluster_ctx ctx; -+ -+ idx = find_idx_from_task(p); -+ if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS) -+ return false; -+ -+ gen_cluster_ctx(&ctx, type); -+ if (idx >= ctx.lower && idx < ctx.upper) -+ return true; -+ return false; -+} -+ -+static bool is_valid_cpu(int cpu) -+{ -+ return (cpu >= 0) && (cpu < nr_cpu_ids); -+} -+ -+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) -+{ -+ s32 cpu = -1; -+ struct cpumask mask = {.bits = {0}}; -+ -+ cpu = get_hmbird_ts(p)->critical_affinity_cpu; -+ if (is_valid_cpu(cpu)) { -+ set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ partial_dynamic_ctrl(); -+ -+ if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ iso_masks.big, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ if (p->nr_cpus_allowed == 1) { -+ cpu = cpumask_any(p->cpus_ptr); -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* For non-period global dsq, not contain pcp task. */ -+ if (task_only_blongs_to_cluster(p, LITTLE)) { -+ cpumask_copy(&mask, iso_masks.little); -+ if (unlikely(l_need_rescue)) { -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ cpumask_or(&mask, iso_masks.ex_free, &mask); -+ } -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &little_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ if (task_only_blongs_to_cluster(p, BIG)) { -+ cpumask_copy(&mask, iso_masks.big); -+ if (unlikely(b_need_rescue)) { -+ cpumask_or(&mask, iso_masks.partial, &mask); -+ cpumask_or(&mask, iso_masks.ex_free, &mask); -+ } -+ if (!cpumask_empty(&mask)) { -+ cpu = select_cpu_from_cluster(p, prev_cpu, -+ &mask, &big_distribute_mask_prev); -+ slim_stats_record(SELECT_CPU, 1, 0, 0); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ } -+ -+ cpu = spec_task_before_pick_idle(p, prev_cpu); -+ if (is_valid_cpu(cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ /* if the previous CPU is idle, dispatch directly to it */ -+ if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+ } -+ -+ cpu = hmbird_pick_idle_cpu(cpu_possible_mask); -+ if (is_valid_cpu(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return cpu; -+ } -+ -+ if (prev_cpu_misfit(prev_cpu)) { -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ cpu = repick_fallback_cpu(); -+ if (is_valid_cpu(cpu)) -+ return cpu; -+ } -+ -+ slim_stats_record(SELECT_CPU, 0, 0, 0); -+ return prev_cpu; -+} -+ -+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags) -+{ -+ return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags); -+} -+ -+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx) -+{ -+ set_cpus_allowed_common(p, ctx); -+} -+ -+static void reset_idle_masks(void) -+{ -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little); -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big); -+ if (is_partial_enabled()) -+ cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial); -+ hmbird_has_idle_cpus = true; -+} -+ -+void __hmbird_update_idle(struct rq *rq, bool idle) -+{ -+ int cpu = cpu_of(rq); -+ struct cpumask *sib_mask = topology_sibling_cpumask(cpu); -+ -+ if (skip_update_idle()) -+ return; -+ -+ if (idle) { -+ cpumask_set_cpu(cpu, idle_masks.cpu); -+ if (!hmbird_has_idle_cpus) -+ hmbird_has_idle_cpus = true; -+ -+ /* -+ * idle_masks.smt handling is racy but that's fine as it's only -+ * for optimization and self-correcting. -+ */ -+ for_each_cpu(cpu, sib_mask) { -+ if (!cpumask_test_cpu(cpu, idle_masks.cpu)) -+ return; -+ } -+ cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); -+ } else { -+ cpumask_clear_cpu(cpu, idle_masks.cpu); -+ if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu)) -+ hmbird_has_idle_cpus = false; -+ -+ cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); -+ } -+} -+ -+#else /* !CONFIG_SMP */ -+ -+static bool test_and_clear_cpu_idle(int cpu) { return false; } -+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } -+static void reset_idle_masks(void) {} -+ -+#endif /* CONFIG_SMP */ -+ -+static bool check_rq_for_timeouts(struct rq *rq) -+{ -+ struct hmbird_entity *entity; -+ struct task_struct *p; -+ struct rq_flags rf; -+ -+ rq_lock_irqsave(rq, &rf); -+ list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) { -+ unsigned long last_runnable; -+ -+ p = entity->task; -+ last_runnable = get_hmbird_ts(p)->runnable_at; -+ -+ if (unlikely(time_after(jiffies, -+ last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); -+ -+ rq_unlock_irqrestore(rq, &rf); -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb", -+ p->comm, p->pid, -+ dur_ms / 1000, dur_ms % 1000, -+ get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0, -+ cpumask_pr_args(&p->cpus_mask)); -+ return 1; -+ } -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ return 0; -+} -+ -+static void hmbird_watchdog_workfn(struct work_struct *work) -+{ -+ int cpu; -+ bool timeout; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ -+ for_each_online_cpu(cpu) { -+ timeout = check_rq_for_timeouts(cpu_rq(cpu)); -+ if (unlikely(timeout)) -+ break; -+ -+ cond_resched(); -+ } -+ if (!timeout) -+ queue_delayed_work(system_unbound_wq, to_delayed_work(work), -+ hmbird_watchdog_timeout / 2); -+} -+ -+static void set_pcp_round(struct rq *rq) -+{ -+ int cpu = cpu_of(rq); -+ -+ if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) { -+ per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round); -+ per_cpu(pcp_info, cpu).pcp_round = true; -+ hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true); -+ per_cpu(pcp_info, cpu).rtime = 0; -+ systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu), -+ per_cpu(pcp_info, cpu).rtime); -+ } -+} -+ -+ -+/* -+ * Just for debug: output hmbird on/off state per 10s. -+ */ -+#define OUTPUT_INTVAL (msecs_to_jiffies(10 * 1000)) -+static void inform_hmbird_onoff_from_systrace(void) -+{ -+ static unsigned long __read_mostly next_print; -+ -+ if (time_before(jiffies, READ_ONCE(next_print))) -+ return; -+ -+ WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL); -+ hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l); -+ hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio); -+ hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio); -+} -+ -+void hmbird_notify_sched_tick(void) -+{ -+ unsigned long last_check; -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ hmbird_scheduler_tick(); -+ -+ if (!hmbird_enabled()) -+ return; -+ -+ last_check = hmbird_watchdog_timestamp; -+ if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) { -+ u32 dur_ms = jiffies_to_msecs(jiffies - last_check); -+ -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL, -+ "watchdog failed to check in for %u.%03us", -+ dur_ms / 1000, dur_ms % 1000); -+ } -+ scan_timeout(rq); -+} -+ -+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued) -+{ -+ update_curr_hmbird(rq); -+ -+ set_pcp_round(rq); -+ -+ if (slim_walt_ctrl) -+ hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE); -+ /* -+ * While disabling, always resched and refresh core-sched timestamp as -+ * we can't trust the slice management or ops.core_sched_before(). -+ */ -+ if (hmbird_ops_disabling()) -+ get_hmbird_ts(curr)->slice = 0; -+ -+ if (!get_hmbird_ts(curr)->slice) -+ resched_curr(rq); -+ -+ inform_hmbird_onoff_from_systrace(); -+} -+ -+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg) -+{ -+ hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ get_hmbird_ts(p)->disallow = false; -+ -+ hmbird_sched_init_task(p); -+ -+ set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags); -+ return 0; -+} -+ -+static void hmbird_ops_enable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), -+ (unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm); -+ -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void hmbird_ops_disable_task(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags); -+ else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) -+ clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags); -+} -+ -+static void set_task_hmbird_weight(struct task_struct *p) -+{ -+ u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; -+ -+ get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight); -+} -+ -+/** -+ * refresh_hmbird_weight - Refresh a task's hmbird weight -+ * @p: task to refresh hmbird weight for -+ * -+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to -+ * enable easy access from the BPF scheduler. To keep it synchronized with the -+ * current task priority, this function should be called when a new task is -+ * created, priority is changed for a task on hmbird, and a task is switched -+ * to hmbird from other classes. -+ */ -+static void refresh_hmbird_weight(struct task_struct *p) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ set_task_hmbird_weight(p); -+} -+ -+int hmbird_pre_fork(struct task_struct *p) -+{ -+ int ret = 0; -+ -+ p->android_oem_data1[HMBIRD_TS_IDX] = -+ (u64)(kzalloc(sizeof(struct hmbird_entity), GFP_KERNEL)); -+ if (!get_hmbird_ts(p)) -+ return -1; -+ -+ get_hmbird_ts(p)->dsq = NULL; -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo); -+ RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq); -+ INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node); -+ get_hmbird_ts(p)->flags = 0; -+ get_hmbird_ts(p)->weight = 0; -+ get_hmbird_ts(p)->sticky_cpu = -1; -+ get_hmbird_ts(p)->holding_cpu = -1; -+ get_hmbird_ts(p)->kf_mask = 0; -+ atomic64_set(&get_hmbird_ts(p)->ops_state, 0); -+ get_hmbird_ts(p)->runnable_at = INITIAL_JIFFIES; -+ get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL; -+ get_hmbird_ts(p)->task = p; -+ hmbird_set_sched_prop(p, 0); -+ -+ get_hmbird_ts(p)->critical_affinity_cpu = -1; -+ get_hmbird_ts(p)->sched_class = &hmbird_sched_class; -+ get_hmbird_ts(p)->tick_hit_count = 0; -+ get_hmbird_ts(p)->start_jiffies = 0; -+ -+ /* -+ * BPF scheduler enable/disable paths want to be able to iterate and -+ * update all tasks which can become complex when racing forks. As -+ * enable/disable are very cold paths, let's use a percpu_rwsem to -+ * exclude forks. -+ */ -+ -+ return ret; -+} -+ -+void hmbird_post_fork(struct task_struct *p) -+{ -+ percpu_down_read(&hmbird_fork_rwsem); -+ -+ if (hmbird_enabled()) { -+ struct rq_flags rf; -+ struct rq *rq; -+ p->sched_class = &hmbird_sched_class; -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ rq = task_rq_lock(p, &rf); -+ /* -+ * Set the weight manually before calling ops.enable() so that -+ * the scheduler doesn't see a stale value if they inspect the -+ * task struct. We'll invoke ops.set_weight() afterwards, as it -+ * would be odd to receive a callback on the task before we -+ * tell the scheduler that it's been fully enabled. -+ */ -+ set_task_hmbird_weight(p); -+ hmbird_ops_enable_task(p); -+ refresh_hmbird_weight(p); -+ task_rq_unlock(rq, p, &rf); -+ } else { -+ if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ } -+ -+ spin_lock_irq(&hmbird_tasks_lock); -+ list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ -+ percpu_up_read(&hmbird_fork_rwsem); -+} -+ -+void hmbird_cancel_fork(struct task_struct *p) -+{ -+ if (hmbird_enabled()) -+ hmbird_ops_disable_task(p); -+ put_hmbird_ts(p); -+} -+ -+void hmbird_free(struct task_struct *p) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&hmbird_tasks_lock, flags); -+ list_del_init(&get_hmbird_ts(p)->tasks_node); -+ spin_unlock_irqrestore(&hmbird_tasks_lock, flags); -+ -+ /* -+ * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED -> -+ * ENABLED transitions can't race us. Disable ops for @p. -+ */ -+ if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) || -+ test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) { -+ struct rq_flags rf; -+ struct rq *rq; -+ -+ rq = task_rq_lock(p, &rf); -+ hmbird_ops_disable_task(p); -+ task_rq_unlock(rq, p, &rf); -+ } -+ put_hmbird_ts(p); -+} -+ -+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio) -+{ -+} -+ -+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type) -+{ -+ return (prop >> TOP_TASK_SHIFT) & (1 << type); -+} -+ -+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p) -+{ -+ uint32_t prop = get_top_task_prop(p); -+ -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL)) -+ return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE)) -+ return HMBIRD_TASK_PROP_PIPELINE; -+ if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) || -+ !task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) { -+ return HMBIRD_TASK_PROP_COMMON; -+ } -+ return HMBIRD_TASK_PROP_DEBUG_OR_LOG; -+} -+ -+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b) -+{ -+ int type_a = hmbird_get_task_type(a); -+ int type_b = hmbird_get_task_type(b); -+ -+ return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b]; -+} -+ -+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ enum cpu_type type; -+ int sp_dl; -+ struct task_struct *curr = NULL; -+ -+ switch (hmbird_preempt_policy) { -+ case HMBIRD_PREEMPT_POLICY_PRIO_BASED: -+ curr = rq->curr; -+ if (curr && hmbird_prio_higher(p, curr)) -+ goto preempt; -+ break; -+ default: -+ break; -+ } -+ if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) || -+ (is_critical_system_task(p) && !is_critical_system_task(rq->curr))) -+ goto preempt; -+ -+ sp_dl = find_idx_from_task(p); -+ if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1) -+ return; -+ -+ type = cpu_cluster(cpu_of(rq)); -+ if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled())) -+ return; -+ -+ if (rq->curr->prio > p->prio) -+ goto preempt; -+ -+ return; -+ -+preempt: -+ resched_curr(rq); -+} -+ -+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {} -+ -+int hmbird_check_setscheduler(struct task_struct *p, int policy) -+{ -+ lockdep_assert_rq_held(task_rq(p)); -+ -+ /* if disallow, reject transitioning into HMBIRD */ -+ if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) && -+ p->policy != policy && policy == SCHED_HMBIRD) -+ return -EACCES; -+ -+ return 0; -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool hmbird_can_stop_tick(struct rq *rq) -+{ -+ struct task_struct *p = rq->curr; -+ -+ if (hmbird_ops_disabling()) -+ return false; -+ -+ if (p->sched_class != &hmbird_sched_class) -+ return true; -+ -+ return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK; -+} -+#endif -+ -+int hmbird_tg_online(struct task_group *tg) -+{ -+ struct cgroup *cgrp; -+ -+ if (!tg) -+ return 0; -+ cgrp = tg->css.cgroup; -+ if (!cgrp || !(cgrp->kn)) -+ return 0; -+ update_cgroup_ids_table(cgrp->kn->id, -1); -+ return 0; -+} -+ -+/* -+ * Omitted operations: -+ * -+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the -+ * task isn't tied to the CPU at that point. Preemption is implemented by -+ * resetting the victim task's slice to 0 and triggering reschedule on the -+ * target CPU. -+ * -+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient. -+ * -+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of -+ * their current sched_class. Call them directly from sched core instead. -+ * -+ * - task_woken, switched_from: Unnecessary. -+ */ -+DEFINE_SCHED_CLASS(hmbird) = { -+ .enqueue_task = enqueue_task_hmbird, -+ .dequeue_task = dequeue_task_hmbird, -+ .yield_task = yield_task_hmbird, -+ .yield_to_task = yield_to_task_hmbird, -+ -+ .check_preempt_curr = check_preempt_curr_hmbird, -+ -+ .pick_next_task = pick_next_task_hmbird, -+ -+ .put_prev_task = put_prev_task_hmbird, -+ .set_next_task = set_next_task_hmbird, -+ -+#ifdef CONFIG_SMP -+ .balance = balance_hmbird, -+ .select_task_rq = select_task_rq_hmbird, -+ .set_cpus_allowed = set_cpus_allowed_hmbird, -+#endif -+ .task_tick = task_tick_hmbird, -+ -+ .switched_to = switched_to_hmbird, -+ .prio_changed = prio_changed_hmbird, -+ -+ .update_curr = update_curr_hmbird, -+ -+#ifdef CONFIG_UCLAMP_TASK -+ .uclamp_enabled = 0, -+#endif -+}; -+ -+/* -+ * Must with rq lock held. -+ */ -+bool task_is_hmbird(struct task_struct *p) -+{ -+ return p->sched_class == &hmbird_sched_class; -+} -+ -+ -+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id) -+{ -+ memset(dsq, 0, sizeof(*dsq)); -+ -+ raw_spin_lock_init(&dsq->lock); -+ INIT_LIST_HEAD(&dsq->fifo); -+ dsq->id = dsq_id; -+} -+ -+static int hmbird_cgroup_init(void) -+{ -+ struct cgroup_subsys_state *css; -+ -+ css_for_each_descendant_pre(css, &root_task_group.css) { -+ struct task_group *tg = css_tg(css); -+ -+ cgrp_dsq_idx_init(css->cgroup, tg); -+ } -+ return 0; -+} -+ -+/* -+ * Used by sched_fork() and __setscheduler_prio() to pick the matching -+ * sched_class. dl/rt are already handled. -+ */ -+bool task_on_hmbird(struct task_struct *p) -+{ -+ return hmbird_enabled(); -+} -+ -+static void __setscheduler_prio(struct task_struct *p, int prio) -+{ -+ bool on_hmbird = task_on_hmbird(p); -+ -+ if (p->sched_class == &stop_sched_class) { -+ p->prio = prio; -+ return; -+ } else if (dl_prio(prio)) -+ p->sched_class = &dl_sched_class; -+ else if (on_hmbird && rt_prio(prio)) -+ p->sched_class = &hmbird_sched_class; -+ else if (rt_prio(prio)) -+ p->sched_class = &rt_sched_class; -+ else if (on_hmbird) -+ p->sched_class = &hmbird_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+ -+ p->prio = prio; -+} -+ -+/* -+ * Heartbeat, avoid humbird keep running while APP already exit. -+ * Check whether APP send alive-signal periodly. -+ */ -+#define HEARTBEAT_TIMEOUT (msecs_to_jiffies(2500)) -+#define HEARTBEAT_CHECK_INTERVAL (msecs_to_jiffies(1000)) -+static struct timer_list hb_timer; -+static unsigned long next_hb; -+ -+void hb_timer_handler(struct timer_list *timer) -+{ -+ if (!heartbeat_enable) -+ goto refill; -+ -+ pr_info(": enter timer.\n"); -+ if (heartbeat) { -+ heartbeat = 0; -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ } -+ -+ /* can't detect heartbeat, disable ext. */ -+ if (time_after(jiffies, READ_ONCE(next_hb))) { -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB); -+ HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT, -+ "can't detect heartbeat, disable ext\n"); -+ } -+refill: -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_start(void) -+{ -+ mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL); -+} -+ -+static void hb_timer_init(void) -+{ -+ timer_setup(&hb_timer, hb_timer_handler, 0); -+} -+ -+static void hb_timer_exit(void) -+{ -+ del_timer(&hb_timer); -+} -+ -+static bool check_and_disable_cpuhp(void) -+{ -+ struct rq *rq; -+ struct rq_flags rf; -+ int cpu; -+ -+ cpu_hotplug_disable(); -+ cpus_read_lock(); -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ rq_lock_irqsave(rq, &rf); -+ if (!rq->online) { -+ rq_unlock_irqrestore(rq, &rf); -+ goto err_offline; -+ } -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ cpus_read_unlock(); -+ return true; -+ -+err_offline: -+ cpus_read_unlock(); -+ cpu_hotplug_enable(); -+ return false; -+} -+ -+static void reenable_cpuhp(void) -+{ -+ cpu_hotplug_enable(); -+} -+ -+static void scheduler_switch_done(bool final_state) -+{ -+ /* set scx_enable again, switch may fail. */ -+ scx_enable = final_state; -+ /* reset heartbeat*/ -+ WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT); -+ -+ if (final_state) -+ hb_timer_start(); -+ else -+ hb_timer_exit(); -+} -+ -+/** -+ * ss : curr switch state -+ * finish : success or fail -+ * enable : curr operation is diabling or enabling? -+ * fail_reason : string output when failed, empty string when success. -+ */ -+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason) -+{ -+ char *s1 = finish ? "finished" : "failed"; -+ char *s2 = enable ? "enabled" : "disabled"; -+ -+ hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss); -+ if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) { -+ sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type)); -+ hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n", -+ s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason); -+ } -+ curr_ss = ss; -+} -+ -+bool get_hmbird_ops_enabled(void) -+{ -+ return atomic_read(&__hmbird_ops_enabled); -+} -+ -+bool get_non_hmbird_task(void) -+{ -+ return atomic_read(&non_hmbird_task); -+} -+ -+static void hmbird_ops_disable_workfn(struct kthread_work *work) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int cpu; -+ -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, ""); -+ cancel_delayed_work_sync(&hmbird_watchdog_work); -+ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) { -+ case HMBIRD_OPS_DISABLED: -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled"); -+ scheduler_switch_done(false); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return; -+ case HMBIRD_OPS_PREPPING: -+ fallthrough; -+ case HMBIRD_OPS_DISABLING: -+ /* shouldn't happen but handle it like ENABLING if it does */ -+ WARN_ONCE(true, "hmbird: duplicate disabling instance?"); -+ fallthrough; -+ case HMBIRD_OPS_ENABLING: -+ case HMBIRD_OPS_ENABLED: -+ break; -+ } -+ -+ /* kick all CPUs to restore ticks */ -+ for_each_possible_cpu(cpu) -+ resched_cpu(cpu); -+ -+ /* avoid racing against fork and cgroup changes */ -+ cpus_read_lock(); -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, ""); -+ spin_lock_irq(&hmbird_tasks_lock); -+ atomic_set(&__hmbird_ops_enabled, false); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ bool alive = READ_ONCE(p->__state) != TASK_DEAD; -+ -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ get_hmbird_ts(p)->slice = min_t(u64, -+ get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL); -+ -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ if (alive) -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ -+ hmbird_ops_disable_task(p); -+ } -+ hmbird_task_iter_exit(&sti); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, ""); -+ -+ atomic_set(&non_hmbird_task, true); -+ /* no task is on hmbird, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt); -+ synchronize_rcu(); -+ -+ percpu_up_write(&hmbird_fork_rwsem); -+ cpus_read_unlock(); -+ -+ if (slim_walt_ctrl) -+ slim_walt_enable(false); -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) != -+ HMBIRD_OPS_DISABLING); -+ -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ -+ hmbird_switch_log(HMBIRD_DISABLED, 1, 0, ""); -+ scheduler_switch_done(false); -+ reenable_cpuhp(); -+} -+ -+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn); -+ -+static void schedule_hmbird_ops_disable_work(void) -+{ -+ struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper); -+ -+ /* -+ * We may be called spuriously before the first bpf_hmbird_reg(). If -+ * hmbird_ops_helper isn't set up yet, there's nothing to do. -+ */ -+ if (helper) -+ kthread_queue_work(helper, &hmbird_ops_disable_work); -+} -+ -+static void hmbird_ops_disable(void) -+{ -+ schedule_hmbird_ops_disable_work(); -+} -+ -+static void hmbird_err_exit_workfn(struct work_struct *work) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ hmbird_ctrl(false); -+ -+ for_each_present_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy) -+ continue; -+ if (cpu != policy->cpu) -+ goto put; -+ down_write(&policy->rwsem); -+ WARN_ON(store_scaling_governor(policy, -+ saved_gov[cpu], strlen(saved_gov[cpu])) <= 0); -+ up_write(&policy->rwsem); -+ hmbird_info_trace(":restore origin gov : %s\n", saved_gov[cpu]); -+put: -+ cpufreq_cpu_put(policy); -+ } -+ memset((char *)saved_gov, 0, sizeof(saved_gov)); -+} -+ -+void hmbird_ops_exit(void) -+{ -+ queue_work(system_unbound_wq, &hmbird_err_exit_work); -+} -+ -+static struct kthread_worker *hmbird_create_rt_helper(const char *name) -+{ -+ struct kthread_worker *helper; -+ -+ helper = kthread_create_worker(KTW_FREEZABLE, name); -+ if (helper) -+ sched_set_fifo(helper->task); -+ return helper; -+} -+ -+static inline void set_audio_thread_sched_prop(struct task_struct *p) -+{ -+ struct cgroup_subsys_state *css; -+ -+ if (likely(p->prio >= MAX_RT_PRIO)) -+ return; -+ -+ rcu_read_lock(); -+ css = task_css(p, cpuset_cgrp_id); -+ if (!css) { -+ rcu_read_unlock(); -+ return; -+ } -+ rcu_read_unlock(); -+ -+ if (!strcmp(css->cgroup->kn->name, "audio-app")) -+ hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1); -+} -+ -+int scx_systemui_pid = -1; -+void set_systemui_thread_pid(struct task_struct *p) -+{ -+ if ((strcmp(p->comm, "ndroid.systemui") == 0) && (p->pid == p->tgid)) -+ scx_systemui_pid = p->pid; -+} -+ -+static int hmbird_ops_enable(void *unused) -+{ -+ struct hmbird_task_iter sti; -+ struct task_struct *p; -+ int ret; -+ int tcnt = 0; -+ unsigned long long start = 0; -+ -+ if (!check_and_disable_cpuhp()) { -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline"); -+ return -EBUSY; -+ } -+ hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, ""); -+ mutex_lock(&hmbird_ops_enable_mutex); -+ -+ if (!hmbird_ops_helper) { -+ WRITE_ONCE(hmbird_ops_helper, -+ hmbird_create_rt_helper("hmbird_ops_helper")); -+ if (!hmbird_ops_helper) { -+ ret = -ENOMEM; -+ goto err_unlock; -+ } -+ } -+ -+ if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) { -+ ret = -EBUSY; -+ goto err_unlock; -+ } -+ -+ WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) != -+ HMBIRD_OPS_DISABLED); -+ -+ hmbird_warned_zero_slice = false; -+ -+ atomic64_set(&hmbird_nr_rejected, 0); -+ -+ /* -+ * Keep CPUs stable during enable so that the BPF scheduler can track -+ * online CPUs by watching ->on/offline_cpu() after ->init(). -+ */ -+ cpus_read_lock(); -+ -+ hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT; -+ -+ hmbird_watchdog_timestamp = jiffies; -+ queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work, -+ hmbird_watchdog_timeout / 2); -+ -+ /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ */ -+ percpu_down_write(&hmbird_fork_rwsem); -+ -+ reset_idle_masks(); -+ -+ /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. -+ */ -+ ret = hmbird_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock; -+ -+ /* -+ * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem -+ * preventing new tasks from being added. No need to exclude tasks -+ * leaving as hmbird_free() can handle both prepped and enabled -+ * tasks. Prep all tasks first and then enable them with preemption -+ * disabled. -+ */ -+ spin_lock_irq(&hmbird_tasks_lock); -+ -+ atomic_set(&non_hmbird_task, false); -+ atomic_set(&__hmbird_ops_enabled, true); -+ -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered(&sti))) -+ hmbird_ops_prepare_task(p, task_group(p)); -+ -+ hmbird_task_iter_exit(&sti); -+ -+ /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. -+ */ -+ preempt_disable(); -+ -+ /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ */ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) { -+ atomic_set(&non_hmbird_task, true); -+ atomic_set(&__hmbird_ops_enabled, false); -+ preempt_enable(); -+ spin_unlock_irq(&hmbird_tasks_lock); -+ ret = -EBUSY; -+ goto err_disable_unlock; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED -+ * transitions here are synchronized against hmbird_free() through -+ * hmbird_tasks_lock. -+ */ -+ start = sched_clock(); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, ""); -+ hmbird_task_iter_init(&sti); -+ while ((p = hmbird_task_iter_next_filtered_locked(&sti))) { -+ tcnt++; -+ if (READ_ONCE(p->__state) != TASK_DEAD) { -+ const struct sched_class *old_class = p->sched_class; -+ struct rq *rq = task_rq(p); -+ -+ set_audio_thread_sched_prop(p); -+ set_systemui_thread_pid(p); -+ update_rq_clock(rq); -+ -+ SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | -+ DEQUEUE_NOCLOCK) { -+ hmbird_ops_enable_task(p); -+ __setscheduler_prio(p, p->prio); -+ } -+ -+ check_class_changed(task_rq(p), p, old_class, p->prio); -+ } else { -+ hmbird_ops_disable_task(p); -+ } -+ } -+ hmbird_task_iter_exit(&sti); -+ -+ spin_unlock_irq(&hmbird_tasks_lock); -+ hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, ""); -+ preempt_enable(); -+ percpu_up_write(&hmbird_fork_rwsem); -+ -+ if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) { -+ ret = -EBUSY; -+ goto err_disable; -+ } -+ -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_ENABLED, 1, 1, ""); -+ scheduler_switch_done(true); -+ -+ return 0; -+ -+err_unlock: -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock"); -+ scheduler_switch_done(false); -+ return ret; -+ -+err_disable_unlock: -+ percpu_up_write(&hmbird_fork_rwsem); -+err_disable: -+ cpus_read_unlock(); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ /* must be fully disabled before returning */ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable"); -+ scheduler_switch_done(false); -+ return ret; -+} -+ -+#ifdef CONFIG_SCHED_DEBUG -+static const char *hmbird_ops_enable_state_str[] = { -+ [HMBIRD_OPS_PREPPING] = "prepping", -+ [HMBIRD_OPS_ENABLING] = "enabling", -+ [HMBIRD_OPS_ENABLED] = "enabled", -+ [HMBIRD_OPS_DISABLING] = "disabling", -+ [HMBIRD_OPS_DISABLED] = "disabled", -+}; -+ -+static int hmbird_debug_show(struct seq_file *m, void *v) -+{ -+ mutex_lock(&hmbird_ops_enable_mutex); -+ seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled()); -+ seq_printf(m, "%-30s: %s\n", "enable_state", -+ hmbird_ops_enable_state_str[hmbird_ops_enable_state()]); -+ seq_printf(m, "%-30s: %llu\n", "nr_rejected", -+ atomic64_read(&hmbird_nr_rejected)); -+ mutex_unlock(&hmbird_ops_enable_mutex); -+ return 0; -+} -+ -+static int hmbird_debug_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_debug_show, NULL); -+} -+ -+const struct file_operations sched_hmbird_fops = { -+ .open = hmbird_debug_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif -+ -+ -+static int bpf_hmbird_reg(void *kdata) -+{ -+ return hmbird_ops_enable(kdata); -+} -+ -+static int bpf_hmbird_unreg(void *kdata) -+{ -+ hmbird_ops_disable(); -+ kthread_flush_work(&hmbird_ops_disable_work); -+ return 0; -+} -+ -+void set_hmbird_module_loaded(int is_loaded) -+{ -+ atomic_set(&hmbird_module_loaded, is_loaded); -+} -+ -+/* -+ * MUST load hmbird module before enable hmbird scheduler -+ * load track & hmbird gover implemented in hmbird module -+ */ -+int hmbird_ctrl(bool enable) -+{ -+ if (!atomic_read(&hmbird_module_loaded)) { -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "ext module unloaded\n"); -+ return -EINVAL; -+ } -+ -+ if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED -+ || hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING -+ || hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already enabled(ing)\n"); -+ hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING || -+ hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) { -+ /* Executing or completed, no need to repeat. */ -+ hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED, -+ 0, enable, "already disabled(ing)\n"); -+ hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n"); -+ return -EBUSY; -+ } -+ if (enable) -+ return bpf_hmbird_reg(NULL); -+ else -+ return bpf_hmbird_unreg(NULL); -+} -+ -+void set_cpu_isomask(int cpu, cpumask_var_t *mask) -+{ -+ cpumask_clear_cpu(cpu, iso_masks.ex_free); -+ cpumask_clear_cpu(cpu, iso_masks.exclusive); -+ cpumask_clear_cpu(cpu, iso_masks.partial); -+ cpumask_clear_cpu(cpu, iso_masks.big); -+ cpumask_clear_cpu(cpu, iso_masks.little); -+ cpumask_set_cpu(cpu, *mask); -+} -+ -+void set_cpu_cluster(u64 cpu_cluster) -+{ -+ int cpu; -+ -+ for_each_present_cpu(cpu) { -+ u64 pos = 1 << cpu; -+ -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.ex_free)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.exclusive)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.partial)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) { -+ set_cpu_isomask(cpu, &(iso_masks.big)); -+ continue; -+ } else -+ pos = pos << 8; -+ if ((pos & cpu_cluster) != 0) -+ set_cpu_isomask(cpu, &(iso_masks.little)); -+ } -+} -+ -+static void get_hmbird_snapshot(struct panic_snapshot_t *p) -+{ -+ struct hmbird_dispatch_q *dsq; -+ struct rq *rq; -+ int cpu, i; -+ -+ for_each_possible_cpu(cpu) { -+ rq = cpu_rq(cpu); -+ p->rq_nr[cpu] = rq->nr_running; -+ p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running; -+ } -+ -+ for (i = 0; i < MAX_GLOBAL_DSQS; i++) { -+ struct hmbird_entity *entity; -+ -+ dsq = &gdsqs[i]; -+ raw_spin_lock(&dsq->lock); -+ if (list_empty(&dsq->fifo)) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ -+ entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo); -+ if (!entity) { -+ raw_spin_unlock(&dsq->lock); -+ continue; -+ } -+ p->runnable_at[i] = entity->runnable_at; -+ raw_spin_unlock(&dsq->lock); -+ -+ } -+ -+ p->snap_misc.hmbird_enabled = hmbird_enabled(); -+ p->snap_misc.curr_ss = curr_ss; -+ p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state(); -+ p->snap_misc.parctrl_high_ratio = parctrl_high_ratio; -+ p->snap_misc.parctrl_low_ratio = parctrl_low_ratio; -+ p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l; -+ p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l; -+ p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio; -+ p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio; -+ p->snap_misc.misfit_ds = misfit_ds; -+ p->snap_misc.partial_enable = partial_enable; -+ p->snap_misc.iso_free_rescue = iso_free_rescue; -+ p->snap_misc.isolate_ctrl = isolate_ctrl; -+ p->snap_misc.snap_jiffies = jiffies; -+ p->snap_misc.snap_time = local_clock(); -+} -+ -+// MTK minidump begin -+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3) -+{ -+ strscpy(m->desc_str, str, DESC_STR_LEN); -+ m->len = d1 * d2 * d3; -+ m->parse[0] = d1; -+ m->parse[1] = d2; -+ m->parse[2] = d3; -+} -+ -+static void init_desc_metas(struct md_info_t *m) -+{ -+ init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :", -+ 1, MAX_SWITCHS, SWITCH_ITEMS); -+ -+ init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1); -+ -+ init_desc_meta(&m->kern_dump.excep_rec_meta, -+ "excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS); -+ -+ init_desc_meta(&m->kern_dump.excep_idx_meta, -+ "excep idx :", 1, 1, MAX_EXCEP_ID); -+ -+ init_desc_meta(&m->kern_dump.snap.runnable_at_meta, -+ "each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS); -+ -+ init_desc_meta(&m->kern_dump.snap.rq_nr_meta, -+ "rq runnable task nr:", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta, -+ "scxrq runnable task nr :", 1, 1, num_possible_cpus()); -+ -+ init_desc_meta(&m->kern_dump.snap.snap_misc_meta, -+ "misc snap params :", 1, 1, SNAP_ITEMS); -+} -+ -+static void init_md_meta(struct md_info_t *m) -+{ -+ m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64); -+ m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64); -+ m->meta.unit_size = sizeof(u64); -+ m->meta.switches = MAX_SWITCHS; -+ m->meta.exceps = MAX_EXCEPS; -+ m->meta.global_dsqs = MAX_GLOBAL_DSQS; -+ m->meta.parse_dimens = PARSE_DIMENS; -+ m->meta.nr_cpus = num_possible_cpus(); -+ m->meta.real_cpus = nr_cpu_ids; -+ m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64); -+ m->meta.nr_meta_desc = 8; -+ m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64); -+ -+ init_desc_metas(m); -+} -+ -+#define MINIDUMP_DFL_SIZE (4 * 1024) -+ -+struct notifier_block hmbird_panic_blk; -+static int hmbird_panic_handler(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ if (!md_info) -+ return NOTIFY_DONE; -+ -+ get_hmbird_snapshot(&md_info->kern_dump.snap); -+ -+ return NOTIFY_DONE; -+} -+ -+static void panic_blk_init(void) -+{ -+ int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE); -+ -+ md_info = kzalloc(dump_size, GFP_KERNEL); -+ if (!md_info) -+ return; -+ init_md_meta(md_info); -+ -+ hmbird_panic_blk.notifier_call = hmbird_panic_handler; -+ /* make sure to execute before minidump. */ -+ hmbird_panic_blk.priority = INT_MAX; -+ atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk); -+ -+ hmbird_debug("register minidump.\n"); -+} -+ -+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size) -+{ -+ *vaddr = (unsigned long)md_info; -+ *size = sizeof(struct md_info_t); -+} -+// MTK minidump end -+ -+static inline void init_sched_prop_to_preempt_prio(void) -+{ -+ for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) { -+ switch (i) { -+ case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 5; -+ break; -+ -+ case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL: -+ sched_prop_to_preempt_prio[i] = 4; -+ break; -+ -+ case HMBIRD_TASK_PROP_PIPELINE: -+ case HMBIRD_TASK_PROP_ISOLATE: -+ sched_prop_to_preempt_prio[i] = 3; -+ break; -+ -+ case HMBIRD_TASK_PROP_COMMON: -+ sched_prop_to_preempt_prio[i] = 1; -+ break; -+ -+ case HMBIRD_TASK_PROP_DEBUG_OR_LOG: -+ sched_prop_to_preempt_prio[i] = 0; -+ break; -+ -+ default: -+ sched_prop_to_preempt_prio[i] = 2; -+ break; -+ } -+ } -+} -+ -+void __init init_sched_hmbird_class(void) -+{ -+ int cpu; -+ u32 v; -+ struct hmbird_entity *init_hmbird; -+ -+ /* -+ * The following is to prevent the compiler from optimizing out the enum -+ * definitions so that BPF scheduler implementations can use them -+ * through the generated vmlinux.h. -+ */ -+ WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP | -+ HMBIRD_KICK_PREEMPT); -+ -+ init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL); -+ init_dsq_at_boot(); -+ init_isolate_cpus(); -+ hb_timer_init(); -+ init_sched_prop_to_preempt_prio(); -+#ifdef CONFIG_SMP -+ WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); -+ WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); -+#endif -+ -+ /* -+ * we can't static init init_task's hmbird struct, init here. -+ * init_task->hmbird would not use during boot. -+ */ -+ init_hmbird = kzalloc(sizeof(struct hmbird_entity), GFP_KERNEL); -+ init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird; -+ if (init_hmbird) { -+ INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo); -+ INIT_LIST_HEAD(&init_hmbird->watchdog_node); -+ init_hmbird->sticky_cpu = -1; -+ init_hmbird->holding_cpu = -1; -+ atomic64_set(&init_hmbird->ops_state, 0); -+ init_hmbird->runnable_at = jiffies; -+ init_hmbird->slice = HMBIRD_SLICE_DFL; -+ init_hmbird->task = &init_task; -+ hmbird_set_sched_prop(&init_task, 0); -+ } else { -+ hmbird_err(INIT_TASK_FAIL, ":alloc init_task.scx failed!!!\n"); -+ } -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ /* -+ * exec during boot phase, no need to care about alloc failed. -+ * lifecycle same to rq, no need to free. -+ */ -+ rq->android_oem_data1[HMBIRD_RQ_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_rq), GFP_KERNEL); -+ if (get_hmbird_rq(rq)) { -+ get_hmbird_rq(rq)->rq = rq; -+ get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window; -+ } else { -+ hmbird_err(ALLOC_RQSCX_FAIL, ":alloc rq->scx failed!!!\n"); -+ } -+ -+ rq->android_oem_data1[HMBIRD_OPS_IDX] = -+ (u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL); -+ if (!get_hmbird_ops(rq)) -+ pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n"); -+ init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL); -+ INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list); -+ -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL)); -+ WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL)); -+ -+ hmbird_ops_init(get_hmbird_ops(rq)); -+ } -+ -+ INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn); -+ INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn); -+ hmbird_misc_init(); -+ -+ panic_blk_init(); -+} -+ -diff --git b/kernel/sched/hmbird/hmbird_misc.c a/kernel/sched/hmbird/hmbird_misc.c -new file mode 100755 -index 000000000000..895e8a29df33 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_misc.c -@@ -0,0 +1,148 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * @Description: -+ * @Version: -+ * @Author: wangrui8 -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include "hmbird_sched.h" -+#include -+#include -+#include "slim.h" -+ -+noinline int tracing_mark_write(const char *buf) -+{ -+ trace_printk(buf); -+ return 0; -+} -+ -+struct yield_opt_params yield_opt_params = { -+ .enable = 0, -+ .frame_per_sec = 120, -+ .frame_time_ns = NSEC_PER_SEC / 120, -+ .yield_headroom = 10, -+}; -+ -+DEFINE_PER_CPU(struct sched_yield_state, ystate); -+ -+static inline void hmbird_yield_state_update(struct sched_yield_state *ys) -+{ -+ if (!raw_spin_is_locked(&ys->lock)) -+ return; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ -+ if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1 -+ || ys->yield_cnt_after_sleep > yield_headroom) { -+ ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP); -+ } else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) { -+ ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP); -+ } -+ ys->yield_cnt = 0; -+ ys->sleep_times = 0; -+ ys->yield_cnt_after_sleep = 0; -+} -+ -+void hmbird_skip_yield(long *skip) -+{ -+ if (!get_hmbird_ops_enabled() || !yield_opt_params.enable) -+ return; -+ unsigned long flags, sleep_now = 0; -+ struct sched_yield_state *ys; -+ int cpu = raw_smp_processor_id(), cont_yield, new_frame; -+ int frame_time_ns = yield_opt_params.frame_time_ns; -+ int yield_headroom = yield_opt_params.yield_headroom; -+ u64 wc; -+ -+ if (!(*skip)) { -+ wc = sched_clock(); -+ ys = &per_cpu(ystate, cpu); -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ -+ cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP; -+ new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1); -+ -+ if (!cont_yield && new_frame) { -+ hmbird_yield_state_update(ys); -+ ys->last_update_time = wc; -+ ys->sleep_end = ys->last_yield_time + frame_time_ns -+ - yield_headroom * YIELD_DURATION; -+ } -+ -+ if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) { -+ *skip = true; -+ -+ sleep_now = ys->sleep_times ? -+ max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep; -+ if (wc + sleep_now > ys->sleep_end) { -+ u64 delta = ys->sleep_end - wc; -+ -+ if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION) -+ sleep_now = delta; -+ else -+ sleep_now = 0; -+ } -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ if (sleep_now) { -+ sleep_now = div64_u64(sleep_now, 1000); -+ usleep_range_state(sleep_now, sleep_now, TASK_IDLE); -+ } -+ ys->sleep_times++; -+ ys->last_yield_time = sched_clock(); -+ return; -+ } -+ if (ys->sleep_times) -+ ys->yield_cnt_after_sleep++; -+ else -+ (ys->yield_cnt)++; -+ ys->last_yield_time = wc; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+} -+ -+struct boost_policy_params boost_policy_params = { -+ .enable = 0, -+ .bottom_freq = 1200000, -+ .boost_weight = 120, -+}; -+ -+int hmbird_get_boost_enable(void) -+{ -+ return boost_policy_params.enable; -+} -+ -+unsigned int hmbird_get_boost_bottom_freq(void) -+{ -+ return boost_policy_params.bottom_freq; -+} -+ -+int hmbird_get_boost_weight(void) -+{ -+ return boost_policy_params.boost_weight; -+} -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops) -+{ -+ hmbird_ops->scx_enable = get_hmbird_ops_enabled; -+ hmbird_ops->check_non_task = get_non_hmbird_task; -+ hmbird_ops->hmbird_get_md_info = hmbird_get_md_info; -+ hmbird_ops->hmbird_get_boost_enable = hmbird_get_boost_enable; -+ hmbird_ops->hmbird_get_boost_bottom_freq = hmbird_get_boost_bottom_freq; -+ hmbird_ops->hmbird_get_boost_weight = hmbird_get_boost_weight; -+} -+ -+void hmbird_misc_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_init(&ys->lock); -+ } -+ -+ set_hmbird_module_loaded(1); -+} -+ -diff --git b/kernel/sched/hmbird/hmbird_sched.h a/kernel/sched/hmbird/hmbird_sched.h -new file mode 100755 -index 000000000000..76acbe9685e4 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0-only */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#ifndef __HMBIRD_SCHED__ -+#define __HMBIRD_SCHED__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include <../../kernel/time/tick-sched.h> -+#include <../../kernel/sched/sched.h> -+#include -+#include -+#include -+ -+#define REGISTER_TRACE_VH(vender_hook, handler) \ -+{ \ -+ ret = register_trace_##vender_hook(handler, NULL); \ -+ if (ret) { \ -+ pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \ -+ } \ -+} -+#define REGISTER_TRACE(vendor_hook, handler, data, err) \ -+do { \ -+ ret = register_trace_##vendor_hook(handler, data); \ -+ if (ret) { \ -+ pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret); \ -+ goto err; \ -+ } \ -+} while (0) -+ -+#define UNREGISTER_TRACE(vendor_hook, handler, data) \ -+ unregister_trace_##vendor_hook(handler, data) \ -+ -+extern unsigned int highres_tick_ctrl; -+extern unsigned int highres_tick_ctrl_dbg; -+ -+extern int slim_walt_ctrl; -+extern int slim_walt_dump; -+extern int slim_walt_policy; -+extern int sched_ravg_window_frame_per_sec; -+extern int slim_gov_debug; -+extern int cpu7_tl; -+extern int scx_gov_ctrl; -+extern spinlock_t new_sched_ravg_window_lock; -+extern int cluster_separate; -+ -+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER BIT(31) -+#define MAX_YIELD_SLEEP (2000000ULL) -+#define MIN_YIELD_SLEEP (200000ULL) -+#define YIELD_DURATION (5000ULL) -+#define DEFAULT_YIELD_SLEEP_TH (10) -+ -+struct sched_yield_state { -+ raw_spinlock_t lock; -+ u64 last_yield_time; -+ u64 last_update_time; -+ u64 sleep_end; -+ unsigned long yield_cnt; -+ unsigned long yield_cnt_after_sleep; -+ unsigned long sleep; -+ int sleep_times; -+}; -+ -+DECLARE_PER_CPU(struct sched_yield_state, ystate); -+ -+void hmbird_window_rollover_run_once(struct rq *rq); -+void hmbird_yield_state_update_per_frame(void); -+void hmbird_misc_init(void); -+ -+void hmbird_ops_init(struct hmbird_ops *hmbird_ops); -+#endif /*__HMBIRD_SCHED__*/ -diff --git b/kernel/sched/hmbird/hmbird_sched_proc.c a/kernel/sched/hmbird/hmbird_sched_proc.c -new file mode 100755 -index 000000000000..295d45404608 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched_proc.c -@@ -0,0 +1,640 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "hmbird_sched_proc.h" -+#include -+ -+#include "hmbird_util_track.h" -+#include "slim.h" -+ -+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched" -+#define SLIM_FREQ_GOV_DIR "slim_freq_gov" -+#define LOAD_TRACK_DIR "slim_walt" -+#define HMBIRD_PROC_PERMISSION 0666 -+ -+int scx_enable; -+int partial_enable; -+int cpuctrl_high_ratio = 55; -+int cpuctrl_low_ratio = 40; -+int slim_stats; -+int hmbirdcore_debug; -+int slim_for_app; -+int misfit_ds = 90; -+unsigned int highres_tick_ctrl; -+unsigned int highres_tick_ctrl_dbg; -+int cpu7_tl = 70; -+int slim_walt_ctrl; -+int slim_walt_dump; -+int slim_walt_policy; -+int slim_gov_debug; -+int scx_gov_ctrl = 1; -+int sched_ravg_window_frame_per_sec = 125; -+int parctrl_high_ratio = 55; -+int parctrl_low_ratio = 40; -+int parctrl_high_ratio_l = 65; -+int parctrl_low_ratio_l = 50; -+int isoctrl_high_ratio = 75; -+int isoctrl_low_ratio = 60; -+int isolate_ctrl; -+int iso_free_rescue; -+int heartbeat; -+int heartbeat_enable = 1; -+int watchdog_enable; -+int save_gov; -+u64 cpu_cluster_masks; -+int hmbird_preempt_policy; -+int cluster_separate; -+ -+char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= 32) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtoint(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoint\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+static int set_proc_buf_val_u64(struct file *file, const char __user *buf, -+ size_t count, u64 *val) -+{ -+ char kbuf[32] = {0}; -+ int err; -+ -+ if (count >= sizeof(kbuf)) -+ return -EFAULT; -+ -+ if (copy_from_user(kbuf, buf, count)) { -+ pr_err("hmbird_sched : Failed to copy_from_user\n"); -+ return -EFAULT; -+ } -+ -+ err = kstrtou64(strstrip(kbuf), 0, val); -+ if (err < 0) { -+ pr_err("hmbird_sched: Failed to exec kstrtoul\n"); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/* common ops begin */ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ return count; -+} -+ -+static int hmbird_common_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%d\n", *(int *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_show, pde_data(inode)); -+} -+ -+static int hmbird_common_ul_show(struct seq_file *m, void *v) -+{ -+ seq_printf(m, "%lu\n", *(unsigned long *) m->private); -+ return 0; -+} -+ -+static int hmbird_common_ul_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_common_ul_show, pde_data(inode)); -+} -+ -+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write); -+/* common ops end */ -+ -+/* scx_enable ops begin */ -+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC); -+ if (hmbird_ctrl(*pval)) -+ return -EFAULT; -+ -+ return count; -+} -+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write); -+/* scx_enable ops end */ -+ -+/* hmbird_stats ops begin */ -+#define MAX_STATS_BUF (4096) -+static int hmbird_stats_proc_show(struct seq_file *m, void *v) -+{ -+ char *buf; -+ -+ buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ stats_print(buf, MAX_STATS_BUF); -+ -+ seq_printf(m, "%s\n", buf); -+ -+ kfree(buf); -+ return 0; -+} -+ -+static int hmbird_stats_proc_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, hmbird_stats_proc_show, inode); -+} -+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL); -+/* hmbird_stats ops end */ -+ -+/* sched_ravg_window_frame_per_sec ops begin */ -+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file, -+ const char __user *buf, size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val(file, buf, count, pval)) -+ return -EFAULT; -+ -+ sched_ravg_window_change(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open, -+ sched_ravg_window_frame_per_sec_proc_write); -+/* sched_ravg_window_frame_per_sec ops end */ -+ -+static ssize_t save_gov_str(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int cpu; -+ struct cpufreq_policy *policy; -+ -+ for_each_possible_cpu(cpu) { -+ policy = cpufreq_cpu_get(cpu); -+ if (!policy || (cpu != policy->cpu)) -+ continue; -+ WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0); -+ hmbird_info_systrace(":save origin gov : %s\n", saved_gov[cpu]); -+ } -+ return count; -+} -+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str); -+ -+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ u64 *pval = (u64 *)pde_data(file_inode(file)); -+ -+ if (set_proc_buf_val_u64(file, buf, count, pval)) -+ return -EFAULT; -+ -+ if (scx_enable == 0) -+ set_cpu_cluster(*pval); -+ -+ return count; -+} -+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write); -+ -+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ int *pval = (int *)pde_data(file_inode(file)); -+ int tmp_val; -+ -+ if (set_proc_buf_val(file, buf, count, &tmp_val)) -+ return -EFAULT; -+ -+ slim_walt_enable(tmp_val); -+ *pval = tmp_val; -+ -+ return count; -+} -+ -+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write); -+ -+/* yield_opt ops begin */ -+static int yield_opt_show(struct seq_file *m, void *v) -+{ -+ struct yield_opt_params *data = m->private; -+ -+ seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n", -+ data->enable, data->frame_per_sec, data->yield_headroom); -+ return 0; -+} -+ -+static int yield_opt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, yield_opt_show, pde_data(inode)); -+} -+ -+static ssize_t yield_opt_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu; -+ unsigned long flags; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if (copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp -+ != 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) || -+ (yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp; -+ yield_opt_params.frame_per_sec = frame_per_sec_tmp; -+ yield_opt_params.yield_headroom = yield_headroom_tmp; -+ yield_opt_params.enable = enable_tmp; -+ -+ for_each_possible_cpu(cpu) { -+ struct sched_yield_state *ys = &per_cpu(ystate, cpu); -+ -+ raw_spin_lock_irqsave(&ys->lock, flags); -+ ys->last_yield_time = 0; -+ ys->last_update_time = 0; -+ ys->sleep_end = 0; -+ ys->yield_cnt = 0; -+ ys->yield_cnt_after_sleep = 0; -+ ys->sleep = 0; -+ ys->sleep_times = 0; -+ raw_spin_unlock_irqrestore(&ys->lock, flags); -+ } -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write); -+ -+/* boost_policy ops begin */ -+static int boost_policy_show(struct seq_file *m, void *v) -+{ -+ struct boost_policy_params *data = m->private; -+ -+ seq_printf(m, "boost_policy:{\"enable\":%d; \"bottom_freq\":%u; \"boost_weight\":%d}\n", -+ data->enable, data->bottom_freq, data->boost_weight); -+ return 0; -+} -+ -+static int boost_policy_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, boost_policy_show, pde_data(inode)); -+} -+ -+static ssize_t boost_policy_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, bottom_freq_tmp, boost_weight_tmp; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if(copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &bottom_freq_tmp, &boost_weight_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1) || -+ (boost_weight_tmp < 50 || boost_weight_tmp > 300) || -+ (bottom_freq_tmp < 400000 || bottom_freq_tmp > 2200000)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ boost_policy_params.bottom_freq = bottom_freq_tmp; -+ boost_policy_params.boost_weight = boost_weight_tmp; -+ boost_policy_params.enable = enable_tmp; -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(boost_policy, boost_policy_open, boost_policy_write); -+ -+/* tick_hit ops begin */ -+static int tick_hit_show(struct seq_file *m, void *v) -+{ -+ struct tick_hit_params *data = m->private; -+ -+ seq_printf(m, "tick_hit:{\"enable\":%d; \"hit_count_thres\":%d; \"jiffies_num\":%lu}\n", -+ data->enable, data->hit_count_thres, data->jiffies_num); -+ return 0; -+} -+ -+static int tick_hit_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, tick_hit_show, pde_data(inode)); -+} -+ -+static ssize_t tick_hit_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) -+{ -+ char *data; -+ int enable_tmp, hit_count_thres_tmp, jiffies_num_tmp; -+ -+ data = kmalloc(count + 1, GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ if(copy_from_user(data, buf, count)) { -+ kfree(data); -+ return -EFAULT; -+ } -+ -+ data[count] = '\0'; -+ -+ if (sscanf(data, "%d %d %d", &enable_tmp, &hit_count_thres_tmp, &jiffies_num_tmp) != 3) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ if ((enable_tmp != 0 && enable_tmp != 1)) { -+ kfree(data); -+ return -EINVAL; -+ } -+ -+ tick_hit_params.hit_count_thres = hit_count_thres_tmp; -+ tick_hit_params.jiffies_num = jiffies_num_tmp; -+ tick_hit_params.enable = enable_tmp; -+ -+ kfree(data); -+ return count; -+} -+ -+HMBIRD_PROC_OPS(tick_hit, tick_hit_open, tick_hit_write); -+ -+static int __init hmbird_proc_init(void) -+{ -+ struct proc_dir_entry *hmbird_dir; -+ struct proc_dir_entry *load_track_dir; -+ struct proc_dir_entry *freq_gov_dir; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ -+ /* mkdir /proc/hmbird_sched */ -+ hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL); -+ if (!hmbird_dir) { -+ pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &scx_enable_proc_ops, -+ &scx_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &partial_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpuctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_stats); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbirdcore_debug); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &slim_for_app); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &misfit_ds); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &highres_tick_ctrl_dbg); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cpu7_tl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &cpu_cluster_masks_proc_ops, -+ &cpu_cluster_masks); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &save_gov_proc_ops, -+ &save_gov); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &heartbeat_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &watchdog_enable); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isolate_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_high_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &isoctrl_low_ratio); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &iso_free_rescue); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_high_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &parctrl_low_ratio_l); -+ -+ HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_stats_proc_ops); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &yield_opt_proc_ops, -+ &yield_opt_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("boost_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &boost_policy_proc_ops, -+ &boost_policy_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("tick_hit", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &tick_hit_proc_ops, -+ &tick_hit_params); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &hmbird_preempt_policy); -+ /* /proc/hmbird_sched--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_walt */ -+ load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir); -+ if (!load_track_dir) { -+ pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_walt--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &slim_walt_ctrl_proc_ops, -+ &slim_walt_ctrl); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_dump); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &hmbird_common_proc_ops, -+ &slim_walt_policy); -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION, -+ load_track_dir, -+ &sched_ravg_window_frame_per_sec_proc_ops, -+ &sched_ravg_window_frame_per_sec); -+ /* /proc/hmbird_sched/slim_walt--end */ -+ -+ /* mkdir /proc/hmbird_sched/slim_freq_gov */ -+ freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir); -+ if (!freq_gov_dir) { -+ pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR); -+ return -ENOMEM; -+ } -+ -+ /* /proc/hmbird_sched/slim_freq_gov--begin */ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &slim_gov_debug); -+ HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION, -+ freq_gov_dir, -+ &hmbird_common_proc_ops, -+ &scx_gov_ctrl); -+ /* /proc/hmbird_sched/slim_freq_gov--end */ -+ -+ HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION, -+ hmbird_dir, -+ &hmbird_common_proc_ops, -+ &cluster_separate); -+ -+ return 0; -+} -+ -+device_initcall(hmbird_proc_init); -diff --git b/kernel/sched/hmbird/hmbird_sched_proc.h a/kernel/sched/hmbird/hmbird_sched_proc.h -new file mode 100755 -index 000000000000..e22bc75f1dd7 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_sched_proc.h -@@ -0,0 +1,39 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SCHED_PROC_H__ -+#define __HMBIRD_SCHED_PROC_H__ -+ -+#include -+#include -+ -+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \ -+ do { \ -+ if (!proc_create(name, mode, parent, proc_ops)) { \ -+ pr_err("Error creating proc entry %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \ -+ do { \ -+ if (!proc_create_data(name, mode, parent, proc_ops, data)) { \ -+ pr_err("Error creating proc entry with data %s\n", name); \ -+ return -ENOMEM; \ -+ } \ -+ } while (0) -+ -+#define HMBIRD_PROC_OPS(name, open_func, write_func) \ -+ static const struct proc_ops name##_proc_ops = { \ -+ .proc_open = open_func, \ -+ .proc_write = write_func, \ -+ .proc_read = seq_read, \ -+ .proc_lseek = seq_lseek, \ -+ .proc_release = single_release, \ -+ } -+ -+static ssize_t hmbird_common_write(struct file *file, -+ const char __user *buf, -+ size_t count, loff_t *ppos); -+static int hmbird_common_show(struct seq_file *m, void *v); -+static int hmbird_common_open(struct inode *inode, struct file *file); -+ -+#endif -diff --git b/kernel/sched/hmbird/hmbird_shadow_tick.c a/kernel/sched/hmbird/hmbird_shadow_tick.c -new file mode 100755 -index 000000000000..2744cd42bbfd ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_shadow_tick.c -@@ -0,0 +1,198 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include -+#include <../../kernel/time/tick-sched.h> -+#include -+ -+#include "hmbird_shadow_tick.h" -+#include "slim.h" -+ -+#define HIGHRES_WATCH_CPU 0 -+ -+#include -+static bool shadow_tick_enable(void) {return highres_tick_ctrl; } -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; } -+#endif -+static bool shadow_tick_timer_init_flag; -+ -+#ifdef CONFIG_HMBIRD_DEBUG_MODE -+#define shadow_tick_printk(fmt, args...) \ -+do { \ -+ int cpu = smp_processor_id(); \ -+ if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU) \ -+ trace_printk("hmbird shadow tick :"fmt, args); \ -+} while (0) -+#else -+#define shadow_tick_printk(fmt, args...) -+#endif -+ -+DEFINE_PER_CPU(struct hrtimer, stt); -+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu))) -+#define STOP_IDLE_TRIGGER (1) -+#define PERIODIC_TICK_TRIGGER (2) -+#define TICK_INTVAL (1000000ULL) -+#define HMBIRD_TICK_HIT_BOOST BIT(29) -+/* -+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms, -+ * so we can't restart hrtimer in scheduler tick. -+ */ -+static DEFINE_PER_CPU(u8, trigger_event); -+ -+/* -+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick. -+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler. -+ */ -+static DEFINE_PER_CPU(u8, tick_phase); -+ -+static inline void highres_timer_ctrl(bool enable, int cpu) -+{ -+ if (enable && hmbird_enabled()) { -+ if (!hrtimer_active(shadow_tick_timer(cpu))) -+ hrtimer_start(shadow_tick_timer(cpu), -+ ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED); -+ } else { -+ if (!enable) -+ hrtimer_cancel(shadow_tick_timer(cpu)); -+ } -+} -+ -+static inline void high_res_clear_phase(int cpu) -+{ -+ per_cpu(tick_phase, cpu) = 0; -+} -+ -+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer) -+{ -+ per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3; -+ if (per_cpu(tick_phase, cpu)) { -+ hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) { -+ per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+struct tick_hit_params tick_hit_params = { -+ .enable = 0, -+ .jiffies_num = 2, -+ .hit_count_thres = 6, -+}; -+ -+static void tick_hit_critical_task(struct task_struct *curr, struct rq *rq) -+{ -+ struct hmbird_entity *see = get_hmbird_ts(curr); -+ if (!tick_hit_params.enable || !see) -+ return; -+ -+ if ((!task_is_top_task(curr) || curr->pid != curr->tgid) && (curr->pid != scx_systemui_pid)) -+ return; -+ -+ if (see->tick_hit_count == 0) { -+ if (see->start_jiffies == 0) { -+ see->start_jiffies = jiffies; -+ see->tick_hit_count = 1; -+ } else { -+ see->start_jiffies = 0; /* status reset */ -+ see->tick_hit_count = 0; -+ } -+ } else { -+ if (time_before_eq(jiffies, see->start_jiffies + tick_hit_params.jiffies_num)) { -+ see->tick_hit_count++; -+ if (see->tick_hit_count >= tick_hit_params.hit_count_thres) { -+ cpufreq_update_util(rq, HMBIRD_TICK_HIT_BOOST); -+ } -+ } else { -+ see->tick_hit_count = 1; -+ see->start_jiffies = jiffies; -+ } -+ } -+} -+ -+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ struct rq_flags rf; -+ -+ rq_lock(rq, &rf); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ tick_hit_critical_task(curr, rq); -+ rq_unlock(rq, &rf); -+ -+ return highres_next_phase(cpu, timer); -+} -+ -+void shadow_tick_timer_init(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance; -+ } -+} -+ -+void start_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (shadow_tick_enable()) { -+ if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER) -+ highres_timer_ctrl(false, cpu); -+ per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(true, cpu); -+ } -+} -+ -+static void stop_shadow_tick_timer(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ per_cpu(trigger_event, cpu) = 0; -+ high_res_clear_phase(cpu); -+ highres_timer_ctrl(false, cpu); -+} -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ stop_shadow_tick_timer(); -+} -+ -+void scheduler_tick_handler(void *unused, struct rq *rq) -+{ -+ if (!shadow_tick_timer_init_flag) -+ return; -+ start_shadow_tick_timer(); -+} -+ -+static int __init hmbird_shadow_tick_init(void) -+{ -+ int ret = 0; -+ -+ if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION) -+ return 0; -+ shadow_tick_timer_init(); -+ shadow_tick_timer_init_flag = true; -+ return ret; -+} -+ -+device_initcall(hmbird_shadow_tick_init); -diff --git b/kernel/sched/hmbird/hmbird_shadow_tick.h a/kernel/sched/hmbird/hmbird_shadow_tick.h -new file mode 100755 -index 000000000000..0c26fd984f0a ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_shadow_tick.h -@@ -0,0 +1,11 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef __HMBIRD_SHADOW_TICK_H__ -+#define __HMBIRD_SHADOW_TICK_H__ -+ -+#include -+ -+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data); -+void scheduler_tick_handler(void *unused, struct rq *rq); -+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev, -+ struct task_struct *next, unsigned int prev_state); -+#endif -diff --git b/kernel/sched/hmbird/hmbird_trace.h a/kernel/sched/hmbird/hmbird_trace.h -new file mode 100755 -index 000000000000..8468b406f347 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_trace.h -@@ -0,0 +1,92 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hmbird -+ -+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HMBIRD_H -+ -+#include -+#include -+#include -+ -+#define MAX_FATAL_INFO (64) -+ -+TRACE_EVENT(hmbird_fatal_info, -+ -+ TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info), -+ -+ TP_ARGS(type, pe, lnr, bnr, info), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, type) -+ __field(int, pe) -+ __field(int, lnr) -+ __field(int, bnr) -+ __array(char, info, MAX_FATAL_INFO)), -+ -+ TP_fast_assign( -+ __entry->type = type; -+ __entry->pe = pe; -+ __entry->lnr = lnr; -+ __entry->bnr = bnr; -+ memcpy(__entry->info, info, MAX_FATAL_INFO);), -+ -+ TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s", -+ __entry->type, __entry->pe, __entry->lnr, __entry->bnr, -+ __entry->info) -+); -+ -+TRACE_EVENT(hmbird_update_history, -+ -+ TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq, -+ struct task_struct *p, u32 runtime, int samples, int event), -+ -+ TP_ARGS(hmbird, rq, p, runtime, samples, event), -+ -+ TP_STRUCT__entry( -+ __array(char, comm, TASK_COMM_LEN) -+ __field(pid_t, pid) -+ __field(unsigned int, runtime) -+ __field(int, samples) -+ __field(int, event) -+ __field(unsigned int, demand) -+ __array(u32, hist, RAVG_HIST_SIZE) -+ __field(u16, task_util) -+ __field(int, cpu)), -+ -+ TP_fast_assign( -+ memcpy(__entry->comm, p->comm, TASK_COMM_LEN); -+ __entry->pid = p->pid; -+ __entry->runtime = runtime; -+ __entry->samples = samples; -+ __entry->event = event; -+ __entry->demand = hmbird->sts.demand; -+ memcpy(__entry->hist, hmbird->sts.sum_history, -+ RAVG_HIST_SIZE * sizeof(u32)); -+ __entry->task_util = hmbird->sts.demand_scaled; -+ __entry->cpu = rq->cpu;), -+ -+ TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d", -+ __entry->comm, __entry->pid, -+ __entry->runtime, __entry->samples, -+ __entry->event, -+ __entry->demand, -+ __entry->hist[0], __entry->hist[1], -+ __entry->hist[2], __entry->hist[3], -+ __entry->hist[4], -+ __entry->task_util, -+ __entry->cpu) -+); -+ -+#endif /*_TRACE_HMBIRD_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird -+ -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE hmbird_trace -+/* This part must be outside protection */ -+#include -diff --git b/kernel/sched/hmbird/hmbird_util_track.c a/kernel/sched/hmbird/hmbird_util_track.c -new file mode 100755 -index 000000000000..d520a8403401 ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_util_track.c -@@ -0,0 +1,626 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * Copyright (C) 2024 Oplus. All rights reserved. -+ */ -+#include "hmbird_sched.h" -+#include "hmbird_util_track.h" -+ -+#define CREATE_TRACE_POINTS -+#include "hmbird_trace.h" -+#undef CREATE_TRACE_POINTS -+ -+#define HMBIRD_DEBUG_PANIC (1 << 3) -+static bool init_irq_work_inited; -+ -+extern noinline int tracing_mark_write(const char *buf); -+ -+int hmbird_sched_ravg_window = 8000000; -+int new_hmbird_sched_ravg_window = 8000000; -+DEFINE_SPINLOCK(new_sched_ravg_window_lock); -+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+ -+static struct irq_work hmbird_slim_walt_irq_work; -+ -+/*Sysctl related interface*/ -+#define WINDOW_STATS_RECENT 0 -+#define WINDOW_STATS_MAX 1 -+#define WINDOW_STATS_MAX_RECENT_AVG 2 -+#define WINDOW_STATS_AVG 3 -+#define WINDOW_STATS_INVALID_POLICY 4 -+ -+ -+#define SCX_SCHED_CAPACITY_SHIFT 10 -+#define SCHED_ACCOUNT_WAIT_TIME 0 -+ -+atomic64_t hmbird_irq_work_lastq_ws; -+static u64 tick_sched_clock; -+ -+static inline u64 scale_exec_time(u64 delta, struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT; -+} -+ -+static inline u64 scale_time_to_util(u64 d) -+{ -+ do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT); -+ return d; -+} -+ -+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ delta = scale_exec_time(delta, rq); -+ sts->sum += delta; -+ if (unlikely(sts->sum > hmbird_sched_ravg_window)) -+ sts->sum = hmbird_sched_ravg_window; -+ -+ return delta; -+} -+ -+ -+static int -+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) -+{ -+ /* -+ * No need to bother updating task demand for the idle task. -+ */ -+ if (is_idle_task(p)) -+ return 0; -+ -+ /* -+ * When a task is waking up it is completing a segment of non-busy -+ * time. Likewise, if wait time is not treated as busy time, then -+ * when a task begins to run or is migrated, it is not running and -+ * is completing a segment of non-busy time. -+ */ -+ if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && -+ (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) -+ return 0; -+ -+ /* -+ * The idle exit time is not accounted for the first task _picked_ up to -+ * run on the idle CPU. -+ */ -+ if (event == PICK_NEXT_TASK && rq->curr == rq->idle) -+ return 0; -+ -+ /* -+ * TASK_UPDATE can be called on sleeping task, when its moved between -+ * related groups -+ */ -+ if (event == TASK_UPDATE) { -+ if (rq->curr == p) -+ return 1; -+ -+ return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; -+ } -+ -+ return 1; -+} -+ -+static void rollover_cpu_window(struct rq *rq, bool full_window) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 curr_sum = srq->curr_runnable_sum; -+ -+ if (unlikely(full_window)) -+ curr_sum = 0; -+ -+ srq->prev_runnable_sum = curr_sum; -+ srq->curr_runnable_sum = 0; -+} -+ -+static u64 -+update_window_start(struct rq *rq, u64 wallclock, int event) -+{ -+ s64 delta; -+ int nr_windows; -+ bool full_window; -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start = srq->window_start; -+ -+ if (wallclock < srq->latest_clock) -+ wallclock = srq->latest_clock; -+ delta = wallclock - srq->window_start; -+ if (delta < 0) { -+ delta = 0; -+ wallclock = srq->window_start; -+ } -+ srq->latest_clock = wallclock; -+ if (delta < hmbird_sched_ravg_window) -+ return old_window_start; -+ -+ nr_windows = div64_u64(delta, hmbird_sched_ravg_window); -+ srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window; -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ full_window = nr_windows > 1; -+ rollover_cpu_window(rq, full_window); -+ -+ return old_window_start; -+} -+ -+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -+ -+static inline unsigned int get_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cpuinfo.max_freq; -+} -+ -+static inline unsigned int cpu_cur_freq(int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); -+ -+ return (policy == NULL) ? 0 : policy->cur; -+} -+ -+static void -+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, -+ u64 wallclock) -+{ -+ int cpu = cpu_of(rq); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * -+ arch_scale_cpu_capacity(cpu), get_max_freq(cpu)); -+} -+ -+/* -+ * Called when new window is starting for a task, to record cpu usage over -+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 -+ * when, say, a real-time task runs without preemption for several windows at a -+ * stretch. -+ */ -+static void update_history(struct rq *rq, struct task_struct *p, -+ u32 runtime, int samples, int event) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u32 *hist = &sts->sum_history[0]; -+ int i; -+ u32 max = 0, avg, demand; -+ u64 sum = 0; -+ u16 demand_scaled; -+ -+ /* Ignore windows where task had no activity */ -+ if (!runtime || is_idle_task(p) || !samples) -+ goto done; -+ -+ /* Push new 'runtime' value onto stack */ -+ for (; samples > 0; samples--) { -+ hist[sts->cidx] = runtime; -+ sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE; -+ } -+ -+ for (i = 0; i < RAVG_HIST_SIZE; i++) { -+ sum += hist[i]; -+ if (hist[i] > max) -+ max = hist[i]; -+ } -+ -+ sts->sum = 0; -+ avg = div64_u64(sum, RAVG_HIST_SIZE); -+ -+ switch (slim_walt_policy) { -+ case WINDOW_STATS_RECENT: -+ demand = runtime; -+ break; -+ case WINDOW_STATS_MAX: -+ demand = max; -+ break; -+ case WINDOW_STATS_AVG: -+ demand = avg; -+ break; -+ default: -+ demand = max(avg, runtime); -+ } -+ -+ demand_scaled = scale_time_to_util(demand); -+ -+ sts->demand = demand; -+ sts->demand_scaled = demand_scaled; -+ -+done: -+ return; -+} -+ -+ -+static u64 update_task_demand(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ -+ u64 delta, window_start = srq->window_start; -+ int new_window, nr_full_windows; -+ u32 window_size = hmbird_sched_ravg_window; -+ u64 runtime; -+ -+ new_window = mark_start < window_start; -+ if (!account_busy_for_task_demand(rq, p, event)) { -+ if (new_window) -+ /* -+ * If the time accounted isn't being accounted as -+ * busy time, and a new window started, only the -+ * previous window need be closed out with the -+ * pre-existing demand. Multiple windows may have -+ * elapsed, but since empty windows are dropped, -+ * it is not necessary to account those. -+ */ -+ update_history(rq, p, sts->sum, 1, event); -+ return 0; -+ } -+ -+ if (!new_window) { -+ /* -+ * The simple case - busy time contained within the existing -+ * window. -+ */ -+ return add_to_task_demand(rq, p, wallclock - mark_start); -+ } -+ -+ /* -+ * Busy time spans at least two windows. Temporarily rewind -+ * window_start to first window boundary after mark_start. -+ */ -+ delta = window_start - mark_start; -+ nr_full_windows = div64_u64(delta, window_size); -+ window_start -= (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (window_start - mark_start) first */ -+ runtime = add_to_task_demand(rq, p, window_start - mark_start); -+ -+ /* Push new sample(s) into task's demand history */ -+ update_history(rq, p, sts->sum, 1, event); -+ if (nr_full_windows) { -+ u64 scaled_window = scale_exec_time(window_size, rq); -+ -+ update_history(rq, p, scaled_window, nr_full_windows, event); -+ runtime += nr_full_windows * scaled_window; -+ } -+ -+ /* -+ * Roll window_start back to current to process any remainder -+ * in current window. -+ */ -+ window_start += (u64)nr_full_windows * (u64)window_size; -+ -+ /* Process (wallclock - window_start) next */ -+ mark_start = window_start; -+ runtime += add_to_task_demand(rq, p, wallclock - mark_start); -+ -+ return runtime; -+} -+ -+u16 slim_walt_cpu_util(int cpu) -+{ -+ u64 prev_runnable_sum; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ prev_runnable_sum = srq->prev_runnable_sum; -+ do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT); -+ -+ return (u16)prev_runnable_sum; -+} -+ -+static DEFINE_PER_CPU(u16, prev_cpu_util); -+static inline void cpu_util_update_systrace_c(int cpu) -+{ -+ char buf[256]; -+ u16 cpu_util = slim_walt_cpu_util(cpu); -+ -+ if (cpu_util != per_cpu(prev_cpu_util, cpu)) { -+ snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n", -+ cpu, cpu_util); -+ tracing_mark_write(buf); -+ per_cpu(prev_cpu_util, cpu) = cpu_util; -+ } -+} -+ -+ -+static inline int account_busy_for_cpu_time(struct rq *rq, -+ struct task_struct *p, -+ int event) -+{ -+ return !is_idle_task(p) && (event == PUT_PREV_TASK -+ || event == TASK_UPDATE); -+} -+ -+ -+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, -+ int event, u64 wallclock) -+{ -+ int new_window, full_window = 0; -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ u64 mark_start = sts->mark_start; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 window_start = srq->window_start; -+ u32 window_size = srq->prev_window_size; -+ u64 delta; -+ u64 *curr_runnable_sum = &srq->curr_runnable_sum; -+ u64 *prev_runnable_sum = &srq->prev_runnable_sum; -+ -+ new_window = mark_start < window_start; -+ if (new_window) -+ full_window = (window_start - mark_start) >= window_size; -+ -+ -+ if (!account_busy_for_cpu_time(rq, p, event)) -+ goto done; -+ -+ -+ if (!new_window) { -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. No rollover -+ * since we didn't start a new window. An example of this is -+ * when a task starts execution and then sleeps within the -+ * same window. -+ */ -+ delta = wallclock - mark_start; -+ -+ delta = scale_exec_time(delta, rq); -+ *curr_runnable_sum += delta; -+ -+ goto done; -+ } -+ -+ /* -+ * situations below this need window rollover, -+ * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done -+ * in update_window_start() -+ * -+ * For task counters curr/prev_window[_cpu] are rolled over in the early part of -+ * this function. If full_window(s) have expired and time since last update needs -+ * to be accounted as busy time, set the prev to a complete window size time, else -+ * add the prev window portion. -+ * -+ * For task curr counters a new window has begun, always assign -+ */ -+ -+ /* -+ * account_busy_for_cpu_time() = 1 so busy time needs -+ * to be accounted to the current window. A new window -+ * must have been started in udpate_window_start() -+ * If any of these three above conditions are true -+ * then this busy time can't be accounted as irqtime. -+ * -+ * Busy time for the idle task need not be accounted. -+ * -+ * An example of this would be a task that starts execution -+ * and then sleeps once a new window has begun. -+ */ -+ -+ /* -+ * A full window hasn't elapsed, account partial -+ * contribution to previous completed window. -+ */ -+ -+ -+ delta = full_window ? scale_exec_time(window_size, rq) : -+ scale_exec_time(window_start - mark_start, rq); -+ -+ *prev_runnable_sum += delta; -+ -+ /* Account piece of busy time in the current window. */ -+ delta = scale_exec_time(wallclock - window_start, rq); -+ *curr_runnable_sum += delta; -+ -+done: -+ if (slim_walt_dump && new_window) -+ cpu_util_update_systrace_c(rq->cpu); -+} -+ -+ -+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq) -+{ -+ u64 result; -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 new_window_start = srq->window_start; -+ -+ if (old_window_start == new_window_start) -+ return; -+ -+ result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws, -+ old_window_start, new_window_start); -+ if (result != old_window_start) -+ return; -+ -+ if (likely(cpu_online(raw_smp_processor_id()))) -+ irq_work_queue(&hmbird_slim_walt_irq_work); -+ else -+ irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask)); -+} -+ -+/* -+ * In the core scheduler, most of the load update points update the rq_clock after -+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of -+ * obtaining the time, but to prevent the subsequent migration of the load update -+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock -+ * update here. -+ */ -+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p, -+ struct rq *rq, int event) -+{ -+ if (!(rq->clock_update_flags & RQCF_UPDATED)) -+ update_rq_clock(rq); -+ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock)); -+} -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ u64 old_window_start; -+ -+ if (!slim_walt_ctrl) -+ return; -+ -+ if (!srq->window_start || sts->mark_start == wallclock) -+ return; -+ -+ old_window_start = update_window_start(rq, wallclock, event); -+ -+ if (!sts->window_start) -+ sts->window_start = srq->window_start; -+ -+ if (!sts->mark_start) -+ goto done; -+ -+ update_task_rq_cpu_cycles(p, rq, event, wallclock); -+ update_task_demand(p, rq, event, wallclock); -+ update_cpu_busy_time(p, rq, event, wallclock); -+ -+ sts->window_start = srq->window_start; -+ -+done: -+ sts->mark_start = wallclock; -+ slim_walt_window_rollover_run_once(old_window_start, rq); -+} -+ -+static void slim_walt_irq_work(struct irq_work *irq_work) -+{ -+ cpumask_t lock_cpus; -+ struct hmbird_sched_rq_stats *srq; -+ struct rq *rq; -+ int cpu; -+ int level = 0; -+ u64 wc; -+ unsigned long flags; -+ -+ cpumask_copy(&lock_cpus, cpu_possible_mask); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ if (level == 0) -+ raw_spin_lock(&cpu_rq(cpu)->__lock); -+ else -+ raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); -+ level++; -+ } -+ -+ wc = sched_clock(); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ rq = cpu_rq(cpu); -+ hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc); -+ } -+ -+ cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER); -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) { -+ srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id()); -+ if (wc < srq->window_start + new_hmbird_sched_ravg_window) -+ hmbird_sched_ravg_window = new_hmbird_sched_ravg_window; -+ } -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+ -+ for_each_cpu(cpu, &lock_cpus) { -+ raw_spin_unlock(&cpu_rq(cpu)->__lock); -+ } -+} -+static void hmbird_sched_init_rq(struct rq *rq) -+{ -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq)); -+ -+ srq->prev_window_size = hmbird_sched_ravg_window; -+ srq->task_exec_scale = 1024; -+ srq->window_start = 0; -+} -+ -+void hmbird_sched_init_task(struct task_struct *p) -+{ -+ struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts); -+ -+ memset(sts, 0, sizeof(struct hmbird_sched_task_stats)); -+} -+ -+static void hmbird_sched_stats_init(void) -+{ -+ unsigned long flags; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct rq *rq = cpu_rq(cpu); -+ -+ raw_spin_lock_irqsave(&rq->__lock, flags); -+ hmbird_sched_init_rq(rq); -+ raw_spin_unlock_irqrestore(&rq->__lock, flags); -+ } -+ slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG; -+ -+ if (false == init_irq_work_inited) { -+ init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work); -+ init_irq_work_inited = true; -+ } -+} -+ -+void hmbird_scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (unlikely(!tick_sched_clock)) { -+ /* -+ * Let the window begin 20us prior to the tick, -+ * that way we are guaranteed a rollover when the tick occurs. -+ * Use rq->clock directly instead of rq_clock() since -+ * we do not have the rq lock and -+ * rq->clock was updated in the tick callpath. -+ */ -+ if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000)) -+ return; -+ for_each_possible_cpu(cpu) { -+ struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu); -+ -+ srq->window_start = tick_sched_clock; -+ } -+ atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock); -+ } -+} -+ -+void slim_walt_enable(int enable) -+{ -+ if (1 == !!enable) { -+ hmbird_sched_stats_init(); -+ WRITE_ONCE(tick_sched_clock, 0); -+ } else -+ slim_walt_ctrl = 0; -+} -+ -+void slim_get_cpu_util(int cpu, u64 *util) -+{ -+ if (cpu < 0) -+ return; -+ -+ *util = slim_walt_cpu_util(cpu); -+} -+ -+void slim_get_task_util(struct task_struct *p, u64 *util) -+{ -+ if (p == NULL) -+ return; -+ -+ *util = get_hmbird_ts(p)->sts.demand_scaled; -+} -+ -+void sched_ravg_window_change(int frame_per_sec) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&new_sched_ravg_window_lock, flags); -+ new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec; -+ spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags); -+} -diff --git b/kernel/sched/hmbird/hmbird_util_track.h a/kernel/sched/hmbird/hmbird_util_track.h -new file mode 100644 -index 000000000000..17b85df7f83b ---- /dev/null -+++ a/kernel/sched/hmbird/hmbird_util_track.h -@@ -0,0 +1,33 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * HMBIRD scheduler class -+ * -+ * Copyright (c) 2024 OPlus. -+ * Copyright (c) 2024 Dao Huang -+ * Copyright (c) 2024 Yuxing Wang -+ * Copyright (c) 2024 Taiyu Li -+ */ -+#ifndef __HMBIRD_UTIL_TRACK_H__ -+#define __HMBIRD_UTIL_TRACK_H__ -+ -+void hmbird_update_task_ravg(struct task_struct *p, -+ struct rq *rq, int event, u64 wallclock); -+void hmbird_sched_init_task(struct task_struct *p); -+void slim_walt_enable(int enable); -+void slim_get_cpu_util(int cpu, u64 *util); -+void slim_get_task_util(struct task_struct *p, u64 *util); -+ -+extern atomic64_t hmbird_irq_work_lastq_ws; -+ -+enum task_event { -+ PUT_PREV_TASK = 0, -+ PICK_NEXT_TASK = 1, -+ TASK_WAKE = 2, -+ TASK_MIGRATE = 3, -+ TASK_UPDATE = 4, -+ IRQ_UPDATE = 5, -+}; -+ -+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats); -+ -+#endif /* __HMBIRD_UTIL_TRACK_H__ */ -diff --git b/kernel/sched/hmbird/slim.h a/kernel/sched/hmbird/slim.h -new file mode 100755 -index 000000000000..dffe6506658e ---- /dev/null -+++ a/kernel/sched/hmbird/slim.h -@@ -0,0 +1,56 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+ -+#ifndef __SLIM_H -+#define __SLIM_H -+ -+extern atomic_t __hmbird_ops_enabled; -+extern atomic_t non_hmbird_task; -+extern int cgroup_ids_table[NUMS_CGROUP_KINDS]; -+extern int heartbeat; -+extern int heartbeat_enable; -+extern int watchdog_enable; -+extern int isolate_ctrl; -+extern int parctrl_high_ratio; -+extern int parctrl_low_ratio; -+extern int isoctrl_high_ratio; -+extern int isoctrl_low_ratio; -+extern int iso_free_rescue; -+extern int yield_opt; -+ -+extern enum hmbird_switch_type sw_type; -+extern noinline int tracing_mark_write(const char *buf); -+int task_top_id(struct task_struct *p); -+void stats_print(char *buf, int len); -+void hmbird_skip_yield(long *skip); -+extern spinlock_t hmbird_tasks_lock; -+extern int scx_systemui_pid; -+ -+struct yield_opt_params { -+ int enable; -+ int frame_per_sec; -+ u64 frame_time_ns; -+ int yield_headroom; -+}; -+ -+extern struct yield_opt_params yield_opt_params; -+ -+struct tick_hit_params { -+ int enable; -+ unsigned long jiffies_num; -+ int hit_count_thres; -+}; -+ -+extern struct tick_hit_params tick_hit_params; -+ -+struct boost_policy_params { -+ int enable; -+ unsigned int bottom_freq; -+ int boost_weight; -+}; -+ -+extern struct boost_policy_params boost_policy_params; -+ -+#define MAX_GOV_LEN (16) -+extern char saved_gov[NR_CPUS][MAX_GOV_LEN]; -+ -+#endif -diff --git b/kernel/sched/idle.c a/kernel/sched/idle.c -index b33cefeb4188..487255ac6832 100755 ---- b/kernel/sched/idle.c -+++ a/kernel/sched/idle.c -@@ -408,13 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl - - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) - { -- scx_update_idle(rq, false); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, false); -+#endif - } - - static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) - { - update_idle_core(rq); -- scx_update_idle(rq, true); -+#ifdef CONFIG_HMBIRD_SCHED -+ hmbird_update_idle(rq, true); -+#endif - schedstat_inc(rq->sched_goidle); - } - -diff --git b/kernel/sched/sched.h a/kernel/sched/sched.h -index 509e8dc616ab..a7e9caea1efe 100755 ---- b/kernel/sched/sched.h -+++ a/kernel/sched/sched.h -@@ -188,18 +188,13 @@ static inline int idle_policy(int policy) - return policy == SCHED_IDLE; - } - --static inline int normal_policy(int policy) --{ --#ifdef CONFIG_SCHED_CLASS_EXT -- if (policy == SCHED_EXT) -- return true; --#endif -- return policy == SCHED_NORMAL; --} -- - static inline int fair_policy(int policy) - { -- return normal_policy(policy) || policy == SCHED_BATCH; -+#ifdef CONFIG_HMBIRD_SCHED -+ return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#else -+ return policy == SCHED_NORMAL || policy == SCHED_BATCH; -+#endif - } - - static inline int rt_policy(int policy) -@@ -247,25 +242,7 @@ static inline void update_avg(u64 *avg, u64 sample) - #define shr_bound(val, shift) \ - (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) - --/* -- * cgroup weight knobs should use the common MIN, DFL and MAX values which are -- * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it -- * maps pretty well onto the shares value used by scheduler and the round-trip -- * conversions preserve the original value over the entire range. -- */ --static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) --{ -- return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); --} -- --static inline unsigned long sched_weight_to_cgroup(unsigned long weight) --{ -- return clamp_t(unsigned long, -- DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), -- CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); --} -- --/* -+/* - * !! For sched_setattr_nocheck() (kernel) only !! - * - * This is actually gross. :( -@@ -532,10 +509,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, - struct cfs_rq *prev, struct cfs_rq *next) { } - #endif /* CONFIG_SMP */ - #else /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_HMBIRD_SCHED - static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) - { - return 0; - } -+#endif - #endif /* CONFIG_FAIR_GROUP_SCHED */ - - #else /* CONFIG_CGROUP_SCHED */ -@@ -700,29 +679,6 @@ struct cfs_rq { - #endif /* CONFIG_FAIR_GROUP_SCHED */ - }; - --#ifdef CONFIG_SCHED_CLASS_EXT --/* scx_rq->flags, protected by the rq lock */ --enum scx_rq_flags { -- SCX_RQ_CAN_STOP_TICK = 1 << 0, --}; -- --struct scx_rq { -- struct scx_dispatch_q local_dsq; -- struct list_head watchdog_list; -- u64 ops_qseq; -- u64 extra_enq_flags; /* see move_task_to_local_dsq() */ -- u32 nr_running; -- u32 flags; -- bool cpu_released; -- cpumask_var_t cpus_to_kick; -- cpumask_var_t cpus_to_preempt; -- cpumask_var_t cpus_to_wait; -- u64 pnt_seq; -- struct irq_work kick_cpus_irq_work; -- struct rq *rq; --}; --#endif /* CONFIG_SCHED_CLASS_EXT */ -- - static inline int rt_bandwidth_enabled(void) - { - return sysctl_sched_rt_runtime >= 0; -@@ -1258,11 +1214,7 @@ struct rq { - - ANDROID_OEM_DATA_ARRAY(1, 16); - --#ifdef CONFIG_SLIM_SCHED -- ANDROID_KABI_USE(1, struct scx_rq *scx); --#else - ANDROID_KABI_RESERVE(1); --#endif - ANDROID_KABI_RESERVE(2); - ANDROID_KABI_RESERVE(3); - ANDROID_KABI_RESERVE(4); -@@ -2340,11 +2292,6 @@ struct affinity_context { - unsigned int flags; - }; - --enum rq_onoff_reason { -- RQ_ONOFF_HOTPLUG, /* CPU is going on/offline */ -- RQ_ONOFF_TOPOLOGY, /* sched domain topology update */ --}; -- - struct sched_class { - - #ifdef CONFIG_UCLAMP_TASK -@@ -2551,7 +2498,6 @@ extern void init_sched_rt_class(void); - extern void init_sched_fair_class(void); - - extern void reweight_task(struct task_struct *p, int prio); --extern void __setscheduler_prio(struct task_struct *p, int prio); - - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); -@@ -2632,53 +2578,6 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) - extern void activate_task(struct rq *rq, struct task_struct *p, int flags); - extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); - --struct sched_change_guard { -- struct task_struct *p; -- struct rq *rq; -- bool queued; -- bool running; -- bool done; --}; -- --extern struct sched_change_guard --sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); -- --extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags); -- --/** -- * SCHED_CHANGE_BLOCK - Nested block for task attribute updates -- * @__rq: Runqueue the target task belongs to -- * @__p: Target task -- * @__flags: DEQUEUE/ENQUEUE_* flags -- * -- * A task may need to be dequeued and put_prev_task'd for attribute updates and -- * set_next_task'd and re-enqueued afterwards. This helper defines a nested -- * block which automatically handles these preparation and cleanup operations. -- * -- * SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { -- * update_attribute(p); -- * ... -- * } -- * -- * If @__flags is a variable, the variable may be updated in the block body and -- * the updated value will be used when re-enqueueing @p. -- * -- * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling -- * update_rq_clock() beforehand. Otherwise, the rq clock is automatically -- * updated iff the task needs to be dequeued and re-enqueued. Only the former -- * case guarantees that the rq clock is up-to-date inside and after the block. -- */ --#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ -- for (struct sched_change_guard __cg = \ -- sched_change_guard_init(__rq, __p, __flags); \ -- !__cg.done; sched_change_guard_fini(&__cg, __flags)) -- --extern void check_class_changing(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class); --extern void check_class_changed(struct rq *rq, struct task_struct *p, -- const struct sched_class *prev_class, -- int oldprio); -- - extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); - - #ifdef CONFIG_PREEMPT_RT -@@ -3710,6 +3609,8 @@ static inline bool cpu_busy_with_softirqs(int cpu) - } - #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ - --#include "ext.h" -+#ifdef CONFIG_HMBIRD_SCHED -+#include "hmbird.h" -+#endif - - #endif /* _KERNEL_SCHED_SCHED_H */ -diff --git b/kernel/time/tick-sched.c a/kernel/time/tick-sched.c -index 998c2eefe173..c13772ee823c 100755 ---- b/kernel/time/tick-sched.c -+++ a/kernel/time/tick-sched.c -@@ -34,6 +34,10 @@ - - #include - -+#ifdef CONFIG_HMBIRD_SCHED -+#include "../sched/hmbird/hmbird_shadow_tick.h" -+#endif -+ - /* - * Per-CPU nohz control structure - */ -@@ -1124,6 +1128,9 @@ void tick_nohz_idle_stop_tick(void) - ktime_t expires; - - trace_android_vh_tick_nohz_idle_stop_tick(NULL); -+#ifdef CONFIG_HMBIRD_SCHED -+ android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL); -+#endif - /* - * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the - * tick timer expiration time is known already. -diff --git b/tools/sched_ext/.gitignore a/tools/sched_ext/.gitignore -deleted file mode 100644 -index a3240f9f7eba..000000000000 ---- b/tools/sched_ext/.gitignore -+++ /dev/null -@@ -1,9 +0,0 @@ --scx_example_simple --scx_example_qmap --scx_example_central --scx_example_pair --scx_example_flatcg --scx_example_userland --*.skel.h --*.subskel.h --/tools/ -diff --git b/tools/sched_ext/Makefile a/tools/sched_ext/Makefile -deleted file mode 100644 -index 73c43782837d..000000000000 ---- b/tools/sched_ext/Makefile -+++ /dev/null -@@ -1,216 +0,0 @@ --# SPDX-License-Identifier: GPL-2.0 --# Copyright (c) 2022 Meta Platforms, Inc. and affiliates. --include ../build/Build.include --include ../scripts/Makefile.arch --include ../scripts/Makefile.include -- --ifneq ($(LLVM),) --ifneq ($(filter %/,$(LLVM)),) --LLVM_PREFIX := $(LLVM) --else ifneq ($(filter -%,$(LLVM)),) --LLVM_SUFFIX := $(LLVM) --endif -- --CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi --CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu --CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl --CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu --CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu --CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu --CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu --CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu --CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu --CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) -- --ifeq ($(CROSS_COMPILE),) --ifeq ($(CLANG_TARGET_FLAGS),) --$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk --else --CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) --endif # CLANG_TARGET_FLAGS --else --CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) --endif # CROSS_COMPILE -- --CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as --else --CC := $(CROSS_COMPILE)gcc --endif # LLVM -- --CURDIR := $(abspath .) --TOOLSDIR := $(abspath ..) --LIBDIR := $(TOOLSDIR)/lib --BPFDIR := $(LIBDIR)/bpf --TOOLSINCDIR := $(TOOLSDIR)/include --BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool --APIDIR := $(TOOLSINCDIR)/uapi --GENDIR := $(abspath ../../include/generated) --GENHDR := $(GENDIR)/autoconf.h -- --SCRATCH_DIR := $(CURDIR)/tools --BUILD_DIR := $(SCRATCH_DIR)/build --INCLUDE_DIR := $(SCRATCH_DIR)/include --BPFOBJ_DIR := $(BUILD_DIR)/libbpf --BPFOBJ := $(BPFOBJ_DIR)/libbpf.a --ifneq ($(CROSS_COMPILE),) --HOST_BUILD_DIR := $(BUILD_DIR)/host --HOST_SCRATCH_DIR := host-tools --HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include --else --HOST_BUILD_DIR := $(BUILD_DIR) --HOST_SCRATCH_DIR := $(SCRATCH_DIR) --HOST_INCLUDE_DIR := $(INCLUDE_DIR) --endif --HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a --RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids --DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -- --VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ -- $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ -- ../../vmlinux \ -- /sys/kernel/btf/vmlinux \ -- /boot/vmlinux-$(shell uname -r) --VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) --ifeq ($(VMLINUX_BTF),) --$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") --endif -- --BPFTOOL ?= $(DEFAULT_BPFTOOL) -- --ifneq ($(wildcard $(GENHDR)),) -- GENFLAGS := -DHAVE_GENHDR --endif -- --CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ -- -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -- -I$(TOOLSINCDIR) -I$(APIDIR) -- --CARGOFLAGS := --release -- --# Silence some warnings when compiled with clang --ifneq ($(LLVM),) --CFLAGS += -Wno-unused-command-line-argument --endif -- --LDFLAGS = -lelf -lz -lpthread -- --IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - &1 \ -- | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ --$(shell $(1) -dM -E - $@ --else -- $(call msg,CP,,$@) -- $(Q)cp "$(VMLINUX_H)" $@ --endif -- --%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \ -- | $(BPFOBJ) -- $(call msg,CLNG-BPF,,$@) -- $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ -- --%.skel.h: %.bpf.o $(BPFTOOL) -- $(call msg,GEN-SKEL,,$@) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) -- $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) -- $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) -- $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@ -- $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h) -- --scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --scx_example_userland: scx_example_userland.c scx_example_userland.skel.h \ -- scx_example_userland_common.h user_exit_info.h -- $(CC) $(CFLAGS) -c $< -o $@.o -- $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) -- --atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR) --atropos: export ATROPOS_CLANG = $(CLANG) --atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS) --atropos: $(INCLUDE_DIR)/vmlinux.h -- cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS) -- --clean: -- cargo clean --manifest-path=atropos/Cargo.toml -- rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) -- rm -f *.o *.bpf.o *.skel.h *.subskel.h -- rm -f scx_example_simple scx_example_qmap scx_example_central \ -- scx_example_pair scx_example_flatcg scx_example_userland -- --.PHONY: all atropos clean -- --# delete failed targets --.DELETE_ON_ERROR: -- --# keep intermediate (.skel.h, .bpf.o, etc) targets --.SECONDARY: -diff --git b/tools/sched_ext/atropos/.gitignore a/tools/sched_ext/atropos/.gitignore -deleted file mode 100644 -index 186dba259ec2..000000000000 ---- b/tools/sched_ext/atropos/.gitignore -+++ /dev/null -@@ -1,3 +0,0 @@ --src/bpf/.output --Cargo.lock --target -diff --git b/tools/sched_ext/atropos/Cargo.toml a/tools/sched_ext/atropos/Cargo.toml -deleted file mode 100644 -index 7462a836d53d..000000000000 ---- b/tools/sched_ext/atropos/Cargo.toml -+++ /dev/null -@@ -1,28 +0,0 @@ --[package] --name = "scx_atropos" --version = "0.5.0" --authors = ["Dan Schatzberg ", "Meta"] --edition = "2021" --description = "Userspace scheduling with BPF" --license = "GPL-2.0-only" -- --[dependencies] --anyhow = "1.0.65" --bitvec = { version = "1.0", features = ["serde"] } --clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } --ctrlc = { version = "3.1", features = ["termination"] } --fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"} --hex = "0.4.3" --libbpf-rs = "0.19.1" --libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] } --libc = "0.2.137" --log = "0.4.17" --ordered-float = "3.4.0" --simplelog = "0.12.0" -- --[build-dependencies] --bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false } --libbpf-cargo = "0.13.0" -- --[features] --enable_backtrace = [] -diff --git b/tools/sched_ext/atropos/build.rs a/tools/sched_ext/atropos/build.rs -deleted file mode 100644 -index 26e792c5e17e..000000000000 ---- b/tools/sched_ext/atropos/build.rs -+++ /dev/null -@@ -1,70 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --extern crate bindgen; -- --use std::env; --use std::fs::create_dir_all; --use std::path::Path; --use std::path::PathBuf; -- --use libbpf_cargo::SkeletonBuilder; -- --const HEADER_PATH: &str = "src/bpf/atropos.h"; -- --fn bindgen_atropos() { -- // Tell cargo to invalidate the built crate whenever the wrapper changes -- println!("cargo:rerun-if-changed={}", HEADER_PATH); -- -- // The bindgen::Builder is the main entry point -- // to bindgen, and lets you build up options for -- // the resulting bindings. -- let bindings = bindgen::Builder::default() -- // The input header we would like to generate -- // bindings for. -- .header(HEADER_PATH) -- // Tell cargo to invalidate the built crate whenever any of the -- // included header files changed. -- .parse_callbacks(Box::new(bindgen::CargoCallbacks)) -- // Finish the builder and generate the bindings. -- .generate() -- // Unwrap the Result and panic on failure. -- .expect("Unable to generate bindings"); -- -- // Write the bindings to the $OUT_DIR/bindings.rs file. -- let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); -- bindings -- .write_to_file(out_path.join("atropos-sys.rs")) -- .expect("Couldn't write bindings!"); --} -- --fn gen_bpf_sched(name: &str) { -- let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap(); -- let clang = env::var("ATROPOS_CLANG").unwrap(); -- eprintln!("{}", clang); -- let outpath = format!("./src/bpf/.output/{}.skel.rs", name); -- let skel = Path::new(&outpath); -- let src = format!("./src/bpf/{}.bpf.c", name); -- SkeletonBuilder::new() -- .source(src.clone()) -- .clang(clang) -- .clang_args(bpf_cflags) -- .build_and_generate(&skel) -- .unwrap(); -- println!("cargo:rerun-if-changed={}", src); --} -- --fn main() { -- bindgen_atropos(); -- // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton. -- // Reasons are because the generated skeleton contains compiler attributes -- // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]` -- // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside -- // the path attribute either (see https://github.com/rust-lang/rust/pull/83366). -- // -- // However, there is hope! When the above feature stabilizes we can clean this -- // all up. -- create_dir_all("./src/bpf/.output").unwrap(); -- gen_bpf_sched("atropos"); --} -diff --git b/tools/sched_ext/atropos/rustfmt.toml a/tools/sched_ext/atropos/rustfmt.toml -deleted file mode 100644 -index b7258ed0a8d8..000000000000 ---- b/tools/sched_ext/atropos/rustfmt.toml -+++ /dev/null -@@ -1,8 +0,0 @@ --# Get help on options with `rustfmt --help=config` --# Please keep these in alphabetical order. --edition = "2021" --group_imports = "StdExternalCrate" --imports_granularity = "Item" --merge_derives = false --use_field_init_shorthand = true --version = "Two" -diff --git b/tools/sched_ext/atropos/src/atropos_sys.rs a/tools/sched_ext/atropos/src/atropos_sys.rs -deleted file mode 100644 -index bbeaf856d40e..000000000000 ---- b/tools/sched_ext/atropos/src/atropos_sys.rs -+++ /dev/null -@@ -1,10 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#![allow(non_upper_case_globals)] --#![allow(non_camel_case_types)] --#![allow(non_snake_case)] --#![allow(dead_code)] -- --include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs")); -diff --git b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -deleted file mode 100644 -index 3905a403e940..000000000000 ---- b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c -+++ /dev/null -@@ -1,743 +0,0 @@ --/* Copyright (c) Meta Platforms, Inc. and affiliates. */ --/* -- * This software may be used and distributed according to the terms of the -- * GNU General Public License version 2. -- * -- * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF -- * part does simple round robin in each domain and the userspace part -- * calculates the load factor of each domain and tells the BPF part how to load -- * balance the domains. -- * -- * Every task has an entry in the task_data map which lists which domain the -- * task belongs to. When a task first enters the system (atropos_prep_enable), -- * they are round-robined to a domain. -- * -- * atropos_select_cpu is the primary scheduling logic, invoked when a task -- * becomes runnable. The lb_data map is populated by userspace to inform the BPF -- * scheduler that a task should be migrated to a new domain. Otherwise, the task -- * is scheduled in priority order as follows: -- * * The current core if the task was woken up synchronously and there are idle -- * cpus in the system -- * * The previous core, if idle -- * * The pinned-to core if the task is pinned to a specific core -- * * Any idle cpu in the domain -- * -- * If none of the above conditions are met, then the task is enqueued to a -- * dispatch queue corresponding to the domain (atropos_enqueue). -- * -- * atropos_dispatch will attempt to consume a task from its domain's -- * corresponding dispatch queue (this occurs after scheduling any tasks directly -- * assigned to it due to the logic in atropos_select_cpu). If no task is found, -- * then greedy load stealing will attempt to find a task on another dispatch -- * queue to run. -- * -- * Load balancing is almost entirely handled by userspace. BPF populates the -- * task weight, dom mask and current dom in the task_data map and executes the -- * load balance based on userspace populating the lb_data map. -- */ --#include "../../../scx_common.bpf.h" --#include "atropos.h" -- --#include --#include --#include --#include --#include --#include -- --char _license[] SEC("license") = "GPL"; -- --/* -- * const volatiles are set during initialization and treated as consts by the -- * jit compiler. -- */ -- --/* -- * Domains and cpus -- */ --const volatile __u32 nr_doms = 32; /* !0 for veristat, set during init */ --const volatile __u32 nr_cpus = 64; /* !0 for veristat, set during init */ --const volatile __u32 cpu_dom_id_map[MAX_CPUS]; --const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; -- --const volatile bool kthreads_local; --const volatile bool fifo_sched; --const volatile bool switch_partial; --const volatile __u32 greedy_threshold; -- --/* base slice duration */ --const volatile __u64 slice_us = 20000; -- --/* -- * Exit info -- */ --int exit_type = SCX_EXIT_NONE; --char exit_msg[SCX_EXIT_MSG_LEN]; -- --struct pcpu_ctx { -- __u32 dom_rr_cur; /* used when scanning other doms */ -- -- /* libbpf-rs does not respect the alignment, so pad out the struct explicitly */ -- __u8 _padding[CACHELINE_SIZE - sizeof(u64)]; --} __attribute__((aligned(CACHELINE_SIZE))); -- --struct pcpu_ctx pcpu_ctx[MAX_CPUS]; -- --/* -- * Domain context -- */ --struct dom_ctx { -- struct bpf_cpumask __kptr *cpumask; -- u64 vtime_now; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __type(key, u32); -- __type(value, struct dom_ctx); -- __uint(max_entries, MAX_DOMS); -- __uint(map_flags, 0); --} dom_ctx SEC(".maps"); -- --/* -- * Statistics -- */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, ATROPOS_NR_STATS); --} stats SEC(".maps"); -- --static inline void stat_add(enum stat_idx idx, u64 addend) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p) += addend; --} -- --/* Map pid -> task_ctx */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, struct task_ctx); -- __uint(max_entries, 1000000); -- __uint(map_flags, 0); --} task_data SEC(".maps"); -- --/* -- * This is populated from userspace to indicate which pids should be reassigned -- * to new doms. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __type(key, pid_t); -- __type(value, u32); -- __uint(max_entries, 1000); -- __uint(map_flags, 0); --} lb_data SEC(".maps"); -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p, -- u32 new_dom_id) --{ -- struct dom_ctx *old_domc, *new_domc; -- struct bpf_cpumask *d_cpumask, *t_cpumask; -- u32 old_dom_id = task_ctx->dom_id; -- s64 vtime_delta; -- -- old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id); -- if (!old_domc) { -- scx_bpf_error("No dom%u", old_dom_id); -- return false; -- } -- -- vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now; -- -- new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id); -- if (!new_domc) { -- scx_bpf_error("No dom%u", new_dom_id); -- return false; -- } -- -- d_cpumask = new_domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to get domain %u cpumask kptr", -- new_dom_id); -- return false; -- } -- -- t_cpumask = task_ctx->cpumask; -- if (!t_cpumask) { -- scx_bpf_error("Failed to look up task cpumask"); -- return false; -- } -- -- /* -- * set_cpumask might have happened between userspace requesting LB and -- * here and @p might not be able to run in @dom_id anymore. Verify. -- */ -- if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- p->cpus_ptr)) { -- p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta; -- task_ctx->dom_id = new_dom_id; -- bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask, -- p->cpus_ptr); -- } -- -- return task_ctx->dom_id == new_dom_id; --} -- --s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu, -- u32 wake_flags) --{ -- s32 cpu; -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- struct bpf_cpumask *p_cpumask; -- -- if (!task_ctx) -- return -ENOENT; -- -- if (kthreads_local && -- (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- cpu = prev_cpu; -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the -- * local dsq of the waker. -- */ -- if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) { -- struct task_struct *current = (void *)bpf_get_current_task(); -- -- if (!(BPF_CORE_READ(current, flags) & PF_EXITING) && -- task_ctx->dom_id < MAX_DOMS) { -- struct dom_ctx *domc; -- struct bpf_cpumask *d_cpumask; -- const struct cpumask *idle_cpumask; -- bool has_idle; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id); -- if (!domc) { -- scx_bpf_error("Failed to find dom%u", -- task_ctx->dom_id); -- return prev_cpu; -- } -- d_cpumask = domc->cpumask; -- if (!d_cpumask) { -- scx_bpf_error("Failed to acquire domain %u cpumask kptr", -- task_ctx->dom_id); -- return prev_cpu; -- } -- -- idle_cpumask = scx_bpf_get_idle_cpumask(); -- -- has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask, -- idle_cpumask); -- -- scx_bpf_put_idle_cpumask(idle_cpumask); -- -- if (has_idle) { -- cpu = bpf_get_smp_processor_id(); -- if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- stat_add(ATROPOS_STAT_WAKE_SYNC, 1); -- goto local; -- } -- } -- } -- } -- -- /* if the previous CPU is idle, dispatch directly to it */ -- if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- stat_add(ATROPOS_STAT_PREV_IDLE, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- /* If only one core is allowed, dispatch */ -- if (p->nr_cpus_allowed == 1) { -- stat_add(ATROPOS_STAT_PINNED, 1); -- cpu = prev_cpu; -- goto local; -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) -- return -ENOENT; -- -- /* If there is an eligible idle CPU, dispatch directly */ -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- if (cpu >= 0) { -- stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); -- goto local; -- } -- -- /* -- * @prev_cpu may be in a different domain. Returning an out-of-domain -- * CPU can lead to stalls as all in-domain CPUs may be idle by the time -- * @p gets enqueued. -- */ -- if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask)) -- cpu = prev_cpu; -- else -- cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask); -- -- return cpu; -- --local: -- task_ctx->dispatch_local = true; -- return cpu; --} -- --void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- u32 *new_dom; -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- new_dom = bpf_map_lookup_elem(&lb_data, &pid); -- if (new_dom && *new_dom != task_ctx->dom_id && -- task_set_dsq(task_ctx, p, *new_dom)) { -- struct bpf_cpumask *p_cpumask; -- s32 cpu; -- -- stat_add(ATROPOS_STAT_LOAD_BALANCE, 1); -- -- /* -- * If dispatch_local is set, We own @p's idle state but we are -- * not gonna put the task in the associated local dsq which can -- * cause the CPU to stall. Kick it. -- */ -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0); -- } -- -- p_cpumask = task_ctx->cpumask; -- if (!p_cpumask) { -- scx_bpf_error("Failed to get task_ctx->cpumask"); -- return; -- } -- cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); -- -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- } -- -- if (task_ctx->dispatch_local) { -- task_ctx->dispatch_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags); -- return; -- } -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000, -- enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- u32 dom_id = task_ctx->dom_id; -- struct dom_ctx *domc; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, domc->vtime_now - slice_us * 1000)) -- vtime = domc->vtime_now - slice_us * 1000; -- -- scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --static u32 cpu_to_dom_id(s32 cpu) --{ -- const volatile u32 *dom_idp; -- -- if (nr_doms <= 1) -- return 0; -- -- dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]); -- if (!dom_idp) -- return MAX_DOMS; -- -- return *dom_idp; --} -- --static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) --{ -- s32 cpu; -- -- if (dom_id >= MAX_DOMS) -- return false; -- -- bpf_for(cpu, 0, nr_cpus) { -- if (bpf_cpumask_test_cpu(cpu, cpumask) && -- (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64)))) -- return true; -- } -- return false; --} -- --static u32 dom_rr_next(s32 cpu) --{ -- struct pcpu_ctx *pcpuc; -- u32 dom_id; -- -- pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]); -- if (!pcpuc) -- return 0; -- -- dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms; -- -- if (dom_id == cpu_to_dom_id(cpu)) -- dom_id = (dom_id + 1) % nr_doms; -- -- pcpuc->dom_rr_cur = dom_id; -- return dom_id; --} -- --void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 dom = cpu_to_dom_id(cpu); -- -- if (scx_bpf_consume(dom)) { -- stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1); -- return; -- } -- -- if (!greedy_threshold) -- return; -- -- bpf_repeat(nr_doms - 1) { -- u32 dom_id = dom_rr_next(cpu); -- -- if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold && -- scx_bpf_consume(dom_id)) { -- stat_add(ATROPOS_STAT_GREEDY, 1); -- break; -- } -- } --} -- --void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_at = bpf_ktime_get_ns(); --} -- --void BPF_STRUCT_OPS(atropos_running, struct task_struct *p) --{ -- struct task_ctx *taskc; -- struct dom_ctx *domc; -- pid_t pid = p->pid; -- u32 dom_id; -- -- if (fifo_sched) -- return; -- -- taskc = bpf_map_lookup_elem(&task_data, &pid); -- if (!taskc) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- dom_id = taskc->dom_id; -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- scx_bpf_error("No dom[%u]", dom_id); -- return; -- } -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(domc->vtime_now, p->scx.dsq_vtime)) -- domc->vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at; -- task_ctx->runnable_at = 0; --} -- --void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_ctx->weight = weight; --} -- --struct pick_task_domain_loop_ctx { -- struct task_struct *p; -- const struct cpumask *cpumask; -- u64 dom_mask; -- u32 dom_rr_base; -- u32 dom_id; --}; -- --static int pick_task_domain_loopfn(u32 idx, void *data) --{ -- struct pick_task_domain_loop_ctx *lctx = data; -- u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms; -- -- if (dom_id >= MAX_DOMS) -- return 1; -- -- if (cpumask_intersects_domain(lctx->cpumask, dom_id)) { -- lctx->dom_mask |= 1LLU << dom_id; -- if (lctx->dom_id == MAX_DOMS) -- lctx->dom_id = dom_id; -- } -- return 0; --} -- --static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- struct pick_task_domain_loop_ctx lctx = { -- .p = p, -- .cpumask = cpumask, -- .dom_id = MAX_DOMS, -- }; -- s32 cpu = bpf_get_smp_processor_id(); -- -- if (cpu < 0 || cpu >= MAX_CPUS) -- return MAX_DOMS; -- -- lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur); -- -- bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0); -- task_ctx->dom_mask = lctx.dom_mask; -- -- return lctx.dom_id; --} -- --static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- u32 dom_id = 0; -- -- if (nr_doms > 1) -- dom_id = pick_task_domain(task_ctx, p, cpumask); -- -- if (!task_set_dsq(task_ctx, p, dom_id)) -- scx_bpf_error("Failed to set domain %d for %s[%d]", -- dom_id, p->comm, p->pid); --} -- --void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p, -- const struct cpumask *cpumask) --{ -- pid_t pid = p->pid; -- struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); -- if (!task_ctx) { -- scx_bpf_error("No task_ctx[%d]", pid); -- return; -- } -- -- task_set_domain(task_ctx, p, cpumask); --} -- --s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct bpf_cpumask *cpumask; -- struct task_ctx task_ctx, *map_value; -- long ret; -- pid_t pid; -- -- memset(&task_ctx, 0, sizeof(task_ctx)); -- -- pid = p->pid; -- ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return ret; -- } -- -- /* -- * Read the entry from the map immediately so we can add the cpumask -- * with bpf_kptr_xchg(). -- */ -- map_value = bpf_map_lookup_elem(&task_data, &pid); -- if (!map_value) -- /* Should never happen -- it was just inserted above. */ -- return -EINVAL; -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- bpf_map_delete_elem(&task_data, &pid); -- return -ENOMEM; -- } -- -- cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask); -- if (cpumask) { -- /* Should never happen as we just inserted it above. */ -- bpf_cpumask_release(cpumask); -- bpf_map_delete_elem(&task_data, &pid); -- return -EINVAL; -- } -- -- task_set_domain(map_value, p, p->cpus_ptr); -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p) --{ -- pid_t pid = p->pid; -- long ret = bpf_map_delete_elem(&task_data, &pid); -- if (ret) { -- stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); -- return; -- } --} -- --static int create_dom_dsq(u32 idx, void *data) --{ -- struct dom_ctx domc_init = {}, *domc; -- struct bpf_cpumask *cpumask; -- u32 cpu, dom_id = idx; -- s32 ret; -- -- ret = scx_bpf_create_dsq(dom_id, -1); -- if (ret < 0) { -- scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret); -- return 1; -- } -- -- ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0); -- if (ret) { -- scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret); -- return 1; -- } -- -- domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); -- if (!domc) { -- /* Should never happen, we just inserted it above. */ -- scx_bpf_error("No dom%u", dom_id); -- return 1; -- } -- -- cpumask = bpf_cpumask_create(); -- if (!cpumask) { -- scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id); -- return 1; -- } -- -- for (cpu = 0; cpu < MAX_CPUS; cpu++) { -- const volatile __u64 *dmask; -- -- dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]); -- if (!dmask) { -- scx_bpf_error("array index error"); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- if (*dmask & (1LLU << (cpu % 64))) -- bpf_cpumask_set_cpu(cpu, cpumask); -- } -- -- cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask); -- if (cpumask) { -- scx_bpf_error("Domain %u was already present", dom_id); -- bpf_cpumask_release(cpumask); -- return 1; -- } -- -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(atropos_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- bpf_loop(nr_doms, create_dom_dsq, NULL, 0); -- -- for (u32 i = 0; i < nr_cpus; i++) -- pcpu_ctx[i].dom_rr_cur = i; -- -- return 0; --} -- --void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg); -- exit_type = ei->type; --} -- --SEC(".struct_ops") --struct sched_ext_ops atropos = { -- .select_cpu = (void *)atropos_select_cpu, -- .enqueue = (void *)atropos_enqueue, -- .dispatch = (void *)atropos_dispatch, -- .runnable = (void *)atropos_runnable, -- .running = (void *)atropos_running, -- .stopping = (void *)atropos_stopping, -- .quiescent = (void *)atropos_quiescent, -- .set_weight = (void *)atropos_set_weight, -- .set_cpumask = (void *)atropos_set_cpumask, -- .prep_enable = (void *)atropos_prep_enable, -- .disable = (void *)atropos_disable, -- .init = (void *)atropos_init, -- .exit = (void *)atropos_exit, -- .flags = 0, -- .name = "atropos", --}; -diff --git b/tools/sched_ext/atropos/src/bpf/atropos.h a/tools/sched_ext/atropos/src/bpf/atropos.h -deleted file mode 100644 -index addf29ca104a..000000000000 ---- b/tools/sched_ext/atropos/src/bpf/atropos.h -+++ /dev/null -@@ -1,44 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#ifndef __ATROPOS_H --#define __ATROPOS_H -- --#include --#ifndef __kptr --#ifdef __KERNEL__ --#error "__kptr_ref not defined in the kernel" --#endif --#define __kptr --#endif -- --#define MAX_CPUS 512 --#define MAX_DOMS 64 /* limited to avoid complex bitmask ops */ --#define CACHELINE_SIZE 64 -- --/* Statistics */ --enum stat_idx { -- ATROPOS_STAT_TASK_GET_ERR, -- ATROPOS_STAT_WAKE_SYNC, -- ATROPOS_STAT_PREV_IDLE, -- ATROPOS_STAT_PINNED, -- ATROPOS_STAT_DIRECT_DISPATCH, -- ATROPOS_STAT_DSQ_DISPATCH, -- ATROPOS_STAT_GREEDY, -- ATROPOS_STAT_LOAD_BALANCE, -- ATROPOS_STAT_LAST_TASK, -- ATROPOS_NR_STATS, --}; -- --struct task_ctx { -- unsigned long long dom_mask; /* the domains this task can run on */ -- struct bpf_cpumask __kptr *cpumask; -- unsigned int dom_id; -- unsigned int weight; -- unsigned long long runnable_at; -- unsigned long long runnable_for; -- bool dispatch_local; --}; -- --#endif /* __ATROPOS_H */ -diff --git b/tools/sched_ext/atropos/src/main.rs a/tools/sched_ext/atropos/src/main.rs -deleted file mode 100644 -index 0d313662f713..000000000000 ---- b/tools/sched_ext/atropos/src/main.rs -+++ /dev/null -@@ -1,942 +0,0 @@ --// Copyright (c) Meta Platforms, Inc. and affiliates. -- --// This software may be used and distributed according to the terms of the --// GNU General Public License version 2. --#[path = "bpf/.output/atropos.skel.rs"] --mod atropos; --pub use atropos::*; --pub mod atropos_sys; -- --use std::cell::Cell; --use std::collections::{BTreeMap, BTreeSet}; --use std::ffi::CStr; --use std::ops::Bound::{Included, Unbounded}; --use std::sync::atomic::{AtomicBool, Ordering}; --use std::sync::Arc; --use std::time::{Duration, SystemTime}; -- --use ::fb_procfs as procfs; --use anyhow::{anyhow, bail, Context, Result}; --use bitvec::prelude::*; --use clap::Parser; --use log::{info, trace, warn}; --use ordered_float::OrderedFloat; -- --/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF --/// part does simple round robin in each domain and the userspace part --/// calculates the load factor of each domain and tells the BPF part how to load --/// balance the domains. -- --/// This scheduler demonstrates dividing scheduling logic between BPF and --/// userspace and using rust to build the userspace part. An earlier variant of --/// this scheduler was used to balance across six domains, each representing a --/// chiplet in a six-chiplet AMD processor, and could match the performance of --/// production setup using CFS. --#[derive(Debug, Parser)] --struct Opts { -- /// Scheduling slice duration in microseconds. -- #[clap(short, long, default_value = "20000")] -- slice_us: u64, -- -- /// Monitoring and load balance interval in seconds. -- #[clap(short, long, default_value = "2.0")] -- interval: f64, -- -- /// Build domains according to how CPUs are grouped at this cache level -- /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id. -- #[clap(short = 'c', long, default_value = "3")] -- cache_level: u32, -- -- /// Instead of using cache locality, set the cpumask for each domain -- /// manually, provide multiple --cpumasks, one for each domain. E.g. -- /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with -- /// the corresponding CPUs belonging to each domain. Each CPU must -- /// belong to precisely one domain. -- #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")] -- cpumasks: Vec, -- -- /// When non-zero, enable greedy task stealing. When a domain is idle, a -- /// cpu will attempt to steal tasks from a domain with at least -- /// greedy_threshold tasks enqueued. These tasks aren't permanently -- /// stolen from the domain. -- #[clap(short, long, default_value = "4")] -- greedy_threshold: u32, -- -- /// The load decay factor. Every interval, the existing load is decayed -- /// by this factor and new load is added. Must be in the range [0.0, -- /// 0.99]. The smaller the value, the more sensitive load calculation -- /// is to recent changes. When 0.0, history is ignored and the load -- /// value from the latest period is used directly. -- #[clap(short, long, default_value = "0.5")] -- load_decay_factor: f64, -- -- /// Disable load balancing. Unless disabled, periodically userspace will -- /// calculate the load factor of each domain and instruct BPF which -- /// processes to move. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- no_load_balance: bool, -- -- /// Put per-cpu kthreads directly into local dsq's. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- kthreads_local: bool, -- -- /// Use FIFO scheduling instead of weighted vtime scheduling. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- fifo_sched: bool, -- -- /// If specified, only tasks which have their scheduling policy set to -- /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all -- /// tasks are switched. -- #[clap(short, long, action = clap::ArgAction::SetTrue)] -- partial: bool, -- -- /// Enable verbose output including libbpf details. Specify multiple -- /// times to increase verbosity. -- #[clap(short, long, action = clap::ArgAction::Count)] -- verbose: u8, --} -- --fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result { -- Ok(reader -- .read_stat() -- .context("Failed to read procfs")? -- .total_cpu -- .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?) --} -- --fn now_monotonic() -> u64 { -- let mut time = libc::timespec { -- tv_sec: 0, -- tv_nsec: 0, -- }; -- let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) }; -- assert!(ret == 0); -- time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 --} -- --fn clear_map(map: &mut libbpf_rs::Map) { -- // XXX: libbpf_rs has some design flaw that make it impossible to -- // delete while iterating despite it being safe so we alias it here -- let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) }; -- for key in map.keys() { -- let _ = deleter.delete(&key); -- } --} -- --#[derive(Debug)] --struct TaskLoad { -- runnable_for: u64, -- load: f64, --} -- --#[derive(Debug)] --struct TaskInfo { -- pid: i32, -- dom_mask: u64, -- migrated: Cell, --} -- --struct LoadBalancer<'a, 'b, 'c> { -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- -- tasks_by_load: Vec, TaskInfo>>, -- load_avg: f64, -- dom_loads: Vec, -- -- imbal: Vec, -- doms_to_push: BTreeMap, u32>, -- doms_to_pull: BTreeMap, u32>, -- -- nr_lb_data_errors: &'c mut u64, --} -- --impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> { -- const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10; -- const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1; -- const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50; -- -- fn new( -- maps: AtroposMapsMut<'a>, -- task_loads: &'b mut BTreeMap, -- nr_doms: usize, -- load_decay_factor: f64, -- nr_lb_data_errors: &'c mut u64, -- ) -> Self { -- Self { -- maps, -- task_loads, -- nr_doms, -- load_decay_factor, -- -- tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(), -- load_avg: 0f64, -- dom_loads: vec![0.0; nr_doms], -- -- imbal: vec![0.0; nr_doms], -- doms_to_pull: BTreeMap::new(), -- doms_to_push: BTreeMap::new(), -- -- nr_lb_data_errors, -- } -- } -- -- fn read_task_loads(&mut self, period: Duration) -> Result<()> { -- let now_mono = now_monotonic(); -- let task_data = self.maps.task_data(); -- let mut this_task_loads = BTreeMap::::new(); -- let mut load_sum = 0.0f64; -- self.dom_loads = vec![0f64; self.nr_doms]; -- -- for key in task_data.keys() { -- if let Some(task_ctx_vec) = task_data -- .lookup(&key, libbpf_rs::MapFlags::ANY) -- .context("Failed to lookup task_data")? -- { -- let task_ctx = -- unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) }; -- let pid = i32::from_ne_bytes( -- key.as_slice() -- .try_into() -- .context("Invalid key length in task_data map")?, -- ); -- -- let (this_at, this_for, weight) = unsafe { -- ( -- std::ptr::read_volatile(&task_ctx.runnable_at as *const u64), -- std::ptr::read_volatile(&task_ctx.runnable_for as *const u64), -- std::ptr::read_volatile(&task_ctx.weight as *const u32), -- ) -- }; -- -- let (mut delta, prev_load) = match self.task_loads.get(&pid) { -- Some(prev) => (this_for - prev.runnable_for, Some(prev.load)), -- None => (this_for, None), -- }; -- -- // Non-zero this_at indicates that the task is currently -- // runnable. Note that we read runnable_at and runnable_for -- // without any synchronization and there is a small window -- // where we end up misaccounting. While this can cause -- // temporary error, it's unlikely to cause any noticeable -- // misbehavior especially given the load value clamping. -- if this_at > 0 && this_at < now_mono { -- delta += now_mono - this_at; -- } -- -- delta = delta.min(period.as_nanos() as u64); -- let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64) -- .clamp(0.0, weight as f64); -- -- let this_load = match prev_load { -- Some(prev_load) => { -- prev_load * self.load_decay_factor -- + this_load * (1.0 - self.load_decay_factor) -- } -- None => this_load, -- }; -- -- this_task_loads.insert( -- pid, -- TaskLoad { -- runnable_for: this_for, -- load: this_load, -- }, -- ); -- -- load_sum += this_load; -- self.dom_loads[task_ctx.dom_id as usize] += this_load; -- // Only record pids that are eligible for load balancing -- if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) { -- continue; -- } -- self.tasks_by_load[task_ctx.dom_id as usize].insert( -- OrderedFloat(this_load), -- TaskInfo { -- pid, -- dom_mask: task_ctx.dom_mask, -- migrated: Cell::new(false), -- }, -- ); -- } -- } -- -- self.load_avg = load_sum / self.nr_doms as f64; -- *self.task_loads = this_task_loads; -- Ok(()) -- } -- -- // To balance dom loads we identify doms with lower and higher load than average -- fn calculate_dom_load_balance(&mut self) -> Result<()> { -- for (dom, dom_load) in self.dom_loads.iter().enumerate() { -- let imbal = dom_load - self.load_avg; -- if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO { -- if imbal > 0f64 { -- self.doms_to_push.insert(OrderedFloat(imbal), dom as u32); -- } else { -- self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32); -- } -- self.imbal[dom] = imbal; -- } -- } -- Ok(()) -- } -- -- // Find the first candidate pid which hasn't already been migrated and -- // can run in @pull_dom. -- fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)> -- where -- I: IntoIterator, &'d TaskInfo)>, -- { -- match tasks_by_load -- .into_iter() -- .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0) -- .next() -- { -- Some((OrderedFloat(load), task)) => Some((*load, task)), -- None => None, -- } -- } -- -- fn pick_victim( -- &self, -- (push_dom, to_push): (u32, f64), -- (pull_dom, to_pull): (u32, f64), -- ) -> Option<(&TaskInfo, f64)> { -- let to_xfer = to_pull.min(to_push); -- -- trace!( -- "considering dom {}@{:.2} -> {}@{:.2}", -- push_dom, -- to_push, -- pull_dom, -- to_pull -- ); -- -- let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs(); -- -- trace!( -- "to_xfer={:.2} tasks_by_load={:?}", -- to_xfer, -- &self.tasks_by_load[push_dom as usize] -- ); -- -- // We want to pick a task to transfer from push_dom to pull_dom to -- // maximize the reduction of load imbalance between the two. IOW, -- // pick a task which has the closest load value to $to_xfer that can -- // be migrated. Find such task by locating the first migratable task -- // while scanning left from $to_xfer and the counterpart while -- // scanning right and picking the better of the two. -- let (load, task, new_imbal) = match ( -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Unbounded, Included(&OrderedFloat(to_xfer)))) -- .rev(), -- pull_dom, -- ), -- Self::find_first_candidate( -- self.tasks_by_load[push_dom as usize] -- .range((Included(&OrderedFloat(to_xfer)), Unbounded)), -- pull_dom, -- ), -- ) { -- (None, None) => return None, -- (Some((load, task)), None) | (None, Some((load, task))) => { -- (load, task, calc_new_imbal(load)) -- } -- (Some((load0, task0)), Some((load1, task1))) => { -- let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1)); -- if new_imbal0 <= new_imbal1 { -- (load0, task0, new_imbal0) -- } else { -- (load1, task1, new_imbal1) -- } -- } -- }; -- -- // If the best candidate can't reduce the imbalance, there's nothing -- // to do for this pair. -- let old_imbal = to_push + to_pull; -- if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal { -- trace!( -- "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal -- ); -- return None; -- } -- -- trace!( -- "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}", -- task.pid, -- push_dom, -- pull_dom, -- old_imbal, -- new_imbal, -- ); -- -- Some((task, load)) -- } -- -- // Actually execute the load balancing. Concretely this writes pid -> dom -- // entries into the lb_data map for bpf side to consume. -- fn load_balance(&mut self) -> Result<()> { -- clear_map(self.maps.lb_data()); -- -- trace!("imbal={:?}", &self.imbal); -- trace!("doms_to_push={:?}", &self.doms_to_push); -- trace!("doms_to_pull={:?}", &self.doms_to_pull); -- -- // Push from the most imbalanced to least. -- while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() { -- let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO; -- let mut pushed = 0f64; -- -- // Transfer tasks from push_dom to reduce imbalance. -- loop { -- let last_pushed = pushed; -- -- // Pull from the most imbalaned to least. -- let mut doms_to_pull = BTreeMap::<_, _>::new(); -- std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull); -- let mut pull_doms = doms_to_pull.into_iter().rev().collect::>(); -- -- for (to_pull, pull_dom) in pull_doms.iter_mut() { -- if let Some((task, load)) = -- self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull))) -- { -- // Execute migration. -- task.migrated.set(true); -- to_push -= load; -- *to_pull -= load; -- pushed += load; -- -- // Ask BPF code to execute the migration. -- let pid = task.pid; -- let cpid = (pid as libc::pid_t).to_ne_bytes(); -- if let Err(e) = self.maps.lb_data().update( -- &cpid, -- &pull_dom.to_ne_bytes(), -- libbpf_rs::MapFlags::NO_EXIST, -- ) { -- warn!( -- "Failed to update lb_data map for pid={} error={:?}", -- pid, &e -- ); -- *self.nr_lb_data_errors += 1; -- } -- -- // Always break after a successful migration so that -- // the pulling domains are always considered in the -- // descending imbalance order. -- break; -- } -- } -- -- pull_doms -- .into_iter() -- .map(|(k, v)| self.doms_to_pull.insert(k, v)) -- .count(); -- -- // Stop repeating if nothing got transferred or pushed enough. -- if pushed == last_pushed || pushed >= push_max { -- break; -- } -- } -- } -- Ok(()) -- } --} -- --struct Scheduler<'a> { -- skel: AtroposSkel<'a>, -- struct_ops: Option, -- -- nr_cpus: usize, -- nr_doms: usize, -- load_decay_factor: f64, -- balance_load: bool, -- -- proc_reader: procfs::ProcReader, -- -- prev_at: SystemTime, -- prev_total_cpu: procfs::CpuStat, -- task_loads: BTreeMap, -- -- nr_lb_data_errors: u64, --} -- --impl<'a> Scheduler<'a> { -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn parse_cpusets( -- cpumasks: &[String], -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- if cpumasks.len() > atropos_sys::MAX_DOMS as usize { -- bail!( -- "Number of requested DSQs ({}) is greater than MAX_DOMS ({})", -- cpumasks.len(), -- atropos_sys::MAX_DOMS -- ); -- } -- let mut cpus = vec![-1i32; nr_cpus]; -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()]; -- for (dq, cpumask) in cpumasks.iter().enumerate() { -- let hex_str = { -- let mut tmp_str = cpumask -- .strip_prefix("0x") -- .unwrap_or(cpumask) -- .replace('_', ""); -- if tmp_str.len() % 2 != 0 { -- tmp_str = "0".to_string() + &tmp_str; -- } -- tmp_str -- }; -- let byte_vec = hex::decode(&hex_str) -- .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?; -- -- for (index, &val) in byte_vec.iter().rev().enumerate() { -- let mut v = val; -- while v != 0 { -- let lsb = v.trailing_zeros() as usize; -- v &= !(1 << lsb); -- let cpu = index * 8 + lsb; -- if cpu > nr_cpus { -- bail!( -- concat!( -- "Found cpu ({}) in cpumask ({}) which is larger", -- " than the number of cpus on the machine ({})" -- ), -- cpu, -- cpumask, -- nr_cpus -- ); -- } -- if cpus[cpu] != -1 { -- bail!( -- "Found cpu ({}) with dq ({}) but also in cpumask ({})", -- cpu, -- cpus[cpu], -- cpumask -- ); -- } -- cpus[cpu] = dq as i32; -- cpusets[dq].set(cpu, true); -- } -- } -- cpusets[dq].set_uninitialized(false); -- } -- -- for (cpu, &dq) in cpus.iter().enumerate() { -- if dq < 0 { -- bail!( -- "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.", -- cpu -- ); -- } -- } -- -- Ok((cpusets, cpus)) -- } -- -- // Returns Vec of cpuset for each dq and a vec of dq for each cpu -- fn cpusets_from_cache( -- level: u32, -- nr_cpus: usize, -- ) -> Result<(Vec>, Vec)> { -- let mut cpu_to_cache = vec![]; // (cpu_id, cache_id) -- let mut cache_ids = BTreeSet::::new(); -- let mut nr_not_found = 0; -- -- // Build cpu -> cache ID mapping. -- for cpu in 0..nr_cpus { -- let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level); -- let id = match std::fs::read_to_string(&path) { -- Ok(val) => val -- .trim() -- .parse::() -- .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?, -- Err(e) if e.kind() == std::io::ErrorKind::NotFound => { -- nr_not_found += 1; -- 0 -- } -- Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)), -- }; -- -- cpu_to_cache.push(id); -- cache_ids.insert(id); -- } -- -- if nr_not_found > 1 { -- warn!( -- "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0", -- level, nr_not_found, nr_cpus -- ); -- } -- -- // Cache IDs may have holes. Assign consecutive domain IDs to -- // existing cache IDs. -- let mut cache_to_dom = BTreeMap::::new(); -- let mut nr_doms = 0; -- for cache_id in cache_ids.iter() { -- cache_to_dom.insert(*cache_id, nr_doms); -- nr_doms += 1; -- } -- -- if nr_doms > atropos_sys::MAX_DOMS { -- bail!( -- "Total number of doms {} is greater than MAX_DOMS ({})", -- nr_doms, -- atropos_sys::MAX_DOMS -- ); -- } -- -- // Build and return dom -> cpumask and cpu -> dom mappings. -- let mut cpusets = -- vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize]; -- let mut cpu_to_dom = vec![]; -- -- for cpu in 0..nr_cpus { -- let dom_id = cache_to_dom[&cpu_to_cache[cpu]]; -- cpusets[dom_id as usize].set(cpu, true); -- cpu_to_dom.push(dom_id as i32); -- } -- -- Ok((cpusets, cpu_to_dom)) -- } -- -- fn init(opts: &Opts) -> Result { -- // Open the BPF prog first for verification. -- let mut skel_builder = AtroposSkelBuilder::default(); -- skel_builder.obj_builder.debug(opts.verbose > 0); -- let mut skel = skel_builder.open().context("Failed to open BPF program")?; -- -- let nr_cpus = libbpf_rs::num_possible_cpus().unwrap(); -- if nr_cpus > atropos_sys::MAX_CPUS as usize { -- bail!( -- "nr_cpus ({}) is greater than MAX_CPUS ({})", -- nr_cpus, -- atropos_sys::MAX_CPUS -- ); -- } -- -- // Initialize skel according to @opts. -- let (cpusets, cpus) = if opts.cpumasks.len() > 0 { -- Self::parse_cpusets(&opts.cpumasks, nr_cpus)? -- } else { -- Self::cpusets_from_cache(opts.cache_level, nr_cpus)? -- }; -- let nr_doms = cpusets.len(); -- skel.rodata().nr_doms = nr_doms as u32; -- skel.rodata().nr_cpus = nr_cpus as u32; -- -- for (cpu, dom) in cpus.iter().enumerate() { -- skel.rodata().cpu_dom_id_map[cpu] = *dom as u32; -- } -- -- for (dom, cpuset) in cpusets.iter().enumerate() { -- let raw_cpuset_slice = cpuset.as_raw_slice(); -- let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom]; -- let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len()); -- left.clone_from_slice(cpuset.as_raw_slice()); -- let cpumask_str = dom_cpumask_slice -- .iter() -- .take((nr_cpus + 63) / 64) -- .rev() -- .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x)); -- info!( -- "DOM[{:02}] cpumask{} ({} cpus)", -- dom, -- &cpumask_str, -- cpuset.count_ones() -- ); -- } -- -- skel.rodata().slice_us = opts.slice_us; -- skel.rodata().kthreads_local = opts.kthreads_local; -- skel.rodata().fifo_sched = opts.fifo_sched; -- skel.rodata().switch_partial = opts.partial; -- skel.rodata().greedy_threshold = opts.greedy_threshold; -- -- // Attach. -- let mut skel = skel.load().context("Failed to load BPF program")?; -- skel.attach().context("Failed to attach BPF program")?; -- let struct_ops = Some( -- skel.maps_mut() -- .atropos() -- .attach_struct_ops() -- .context("Failed to attach atropos struct ops")?, -- ); -- info!("Atropos Scheduler Attached"); -- -- // Other stuff. -- let mut proc_reader = procfs::ProcReader::new(); -- let prev_total_cpu = read_total_cpu(&mut proc_reader)?; -- -- Ok(Self { -- skel, -- struct_ops, // should be held to keep it attached -- -- nr_cpus, -- nr_doms, -- load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99), -- balance_load: !opts.no_load_balance, -- -- proc_reader, -- -- prev_at: SystemTime::now(), -- prev_total_cpu, -- task_loads: BTreeMap::new(), -- -- nr_lb_data_errors: 0, -- }) -- } -- -- fn get_cpu_busy(&mut self) -> Result { -- let total_cpu = read_total_cpu(&mut self.proc_reader)?; -- let busy = match (&self.prev_total_cpu, &total_cpu) { -- ( -- procfs::CpuStat { -- user_usec: Some(prev_user), -- nice_usec: Some(prev_nice), -- system_usec: Some(prev_system), -- idle_usec: Some(prev_idle), -- iowait_usec: Some(prev_iowait), -- irq_usec: Some(prev_irq), -- softirq_usec: Some(prev_softirq), -- stolen_usec: Some(prev_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- procfs::CpuStat { -- user_usec: Some(curr_user), -- nice_usec: Some(curr_nice), -- system_usec: Some(curr_system), -- idle_usec: Some(curr_idle), -- iowait_usec: Some(curr_iowait), -- irq_usec: Some(curr_irq), -- softirq_usec: Some(curr_softirq), -- stolen_usec: Some(curr_stolen), -- guest_usec: _, -- guest_nice_usec: _, -- }, -- ) => { -- let idle_usec = curr_idle - prev_idle; -- let iowait_usec = curr_iowait - prev_iowait; -- let user_usec = curr_user - prev_user; -- let system_usec = curr_system - prev_system; -- let nice_usec = curr_nice - prev_nice; -- let irq_usec = curr_irq - prev_irq; -- let softirq_usec = curr_softirq - prev_softirq; -- let stolen_usec = curr_stolen - prev_stolen; -- -- let busy_usec = -- user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec; -- let total_usec = idle_usec + busy_usec + iowait_usec; -- busy_usec as f64 / total_usec as f64 -- } -- _ => { -- bail!("Some procfs stats are not populated!"); -- } -- }; -- -- self.prev_total_cpu = total_cpu; -- Ok(busy) -- } -- -- fn read_bpf_stats(&mut self) -> Result> { -- let mut maps = self.skel.maps_mut(); -- let stats_map = maps.stats(); -- let mut stats: Vec = Vec::new(); -- let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus]; -- -- for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS { -- let cpu_stat_vec = stats_map -- .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY) -- .with_context(|| format!("Failed to lookup stat {}", stat))? -- .expect("per-cpu stat should exist"); -- let sum = cpu_stat_vec -- .iter() -- .map(|val| { -- u64::from_ne_bytes( -- val.as_slice() -- .try_into() -- .expect("Invalid value length in stat map"), -- ) -- }) -- .sum(); -- stats_map -- .update_percpu( -- &(stat as u32).to_ne_bytes(), -- &zero_vec, -- libbpf_rs::MapFlags::ANY, -- ) -- .context("Failed to zero stat")?; -- stats.push(sum); -- } -- Ok(stats) -- } -- -- fn report( -- &self, -- stats: &Vec, -- cpu_busy: f64, -- processing_dur: Duration, -- load_avg: f64, -- dom_loads: &Vec, -- imbal: &Vec, -- ) { -- let stat = |idx| stats[idx as usize]; -- let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY) -- + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK); -- -- info!( -- "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms", -- cpu_busy * 100.0, -- load_avg, -- stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize], -- stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize], -- self.nr_lb_data_errors, -- processing_dur.as_millis(), -- ); -- -- let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0; -- -- info!( -- "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}", -- total, -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH), -- stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY), -- ); -- -- for i in 0..self.nr_doms { -- info!( -- "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}", -- i, -- dom_loads[i], -- if imbal[i] < 0.0 { -imbal[i] } else { 0.0 }, -- if imbal[i] > 0.0 { imbal[i] } else { 0.0 }, -- ); -- } -- } -- -- fn step(&mut self) -> Result<()> { -- let started_at = std::time::SystemTime::now(); -- let bpf_stats = self.read_bpf_stats()?; -- let cpu_busy = self.get_cpu_busy()?; -- -- let mut lb = LoadBalancer::new( -- self.skel.maps_mut(), -- &mut self.task_loads, -- self.nr_doms, -- self.load_decay_factor, -- &mut self.nr_lb_data_errors, -- ); -- -- lb.read_task_loads(started_at.duration_since(self.prev_at)?)?; -- lb.calculate_dom_load_balance()?; -- -- if self.balance_load { -- lb.load_balance()?; -- } -- -- // Extract fields needed for reporting and drop lb to release -- // mutable borrows. -- let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal); -- -- self.report( -- &bpf_stats, -- cpu_busy, -- std::time::SystemTime::now().duration_since(started_at)?, -- load_avg, -- &dom_loads, -- &imbal, -- ); -- -- self.prev_at = started_at; -- Ok(()) -- } -- -- fn read_bpf_exit_type(&mut self) -> i32 { -- unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) } -- } -- -- fn report_bpf_exit_type(&mut self) -> Result<()> { -- // Report msg if EXT_OPS_EXIT_ERROR. -- match self.read_bpf_exit_type() { -- 0 => Ok(()), -- etype if etype == 2 => { -- let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) }; -- let msg = cstr -- .to_str() -- .context("Failed to convert exit msg to string") -- .unwrap(); -- bail!("BPF exit_type={} msg={}", etype, msg); -- } -- etype => { -- info!("BPF exit_type={}", etype); -- Ok(()) -- } -- } -- } --} -- --impl<'a> Drop for Scheduler<'a> { -- fn drop(&mut self) { -- if let Some(struct_ops) = self.struct_ops.take() { -- drop(struct_ops); -- } -- } --} -- --fn main() -> Result<()> { -- let opts = Opts::parse(); -- -- let llv = match opts.verbose { -- 0 => simplelog::LevelFilter::Info, -- 1 => simplelog::LevelFilter::Debug, -- _ => simplelog::LevelFilter::Trace, -- }; -- let mut lcfg = simplelog::ConfigBuilder::new(); -- lcfg.set_time_level(simplelog::LevelFilter::Error) -- .set_location_level(simplelog::LevelFilter::Off) -- .set_target_level(simplelog::LevelFilter::Off) -- .set_thread_level(simplelog::LevelFilter::Off); -- simplelog::TermLogger::init( -- llv, -- lcfg.build(), -- simplelog::TerminalMode::Stderr, -- simplelog::ColorChoice::Auto, -- )?; -- -- let shutdown = Arc::new(AtomicBool::new(false)); -- let shutdown_clone = shutdown.clone(); -- ctrlc::set_handler(move || { -- shutdown_clone.store(true, Ordering::Relaxed); -- }) -- .context("Error setting Ctrl-C handler")?; -- -- let mut sched = Scheduler::init(&opts)?; -- -- while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 { -- std::thread::sleep(Duration::from_secs_f64(opts.interval)); -- sched.step()?; -- } -- -- sched.report_bpf_exit_type() --} -diff --git b/tools/sched_ext/gnu/stubs.h a/tools/sched_ext/gnu/stubs.h -deleted file mode 100644 -index 719225b16626..000000000000 ---- b/tools/sched_ext/gnu/stubs.h -+++ /dev/null -@@ -1 +0,0 @@ --/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ -diff --git b/tools/sched_ext/scx_common.bpf.h a/tools/sched_ext/scx_common.bpf.h -deleted file mode 100644 -index e56de9dc86f2..000000000000 ---- b/tools/sched_ext/scx_common.bpf.h -+++ /dev/null -@@ -1,288 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __SCHED_EXT_COMMON_BPF_H --#define __SCHED_EXT_COMMON_BPF_H -- --#include "vmlinux.h" --#include --#include --#include --#include "user_exit_info.h" -- --#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ --#define PF_EXITING 0x00000004 --#define CLOCK_MONOTONIC 1 -- --/* -- * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can -- * lead to really confusing misbehaviors. Let's trigger a build failure. -- */ --static inline void ___vmlinux_h_sanity_check___(void) --{ -- _Static_assert(SCX_DSQ_FLAG_BUILTIN, -- "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); --} -- --void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; -- --static inline __attribute__((format(printf, 1, 2))) --void ___scx_bpf_error_format_checker(const char *fmt, ...) {} -- --/* -- * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments -- * instead of an array of u64. Note that __param[] must have at least one -- * element to keep the verifier happy. -- */ --#define scx_bpf_error(fmt, args...) \ --({ \ -- static char ___fmt[] = fmt; \ -- unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ -- \ -- _Pragma("GCC diagnostic push") \ -- _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ -- ___bpf_fill(___param, args); \ -- _Pragma("GCC diagnostic pop") \ -- \ -- scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ -- \ -- ___scx_bpf_error_format_checker(fmt, ##args); \ --}) -- --void scx_bpf_switch_all(void) __ksym; --s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; --bool scx_bpf_consume(u64 dsq_id) __ksym; --u32 scx_bpf_dispatch_nr_slots(void) __ksym; --void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym; --void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym; --void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; --s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; --bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; --s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym; --const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; --const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; --void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; --void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; --bool scx_bpf_task_running(const struct task_struct *p) __ksym; --s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; --struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym; --u32 scx_bpf_reenqueue_local(void) __ksym; -- --#define BPF_STRUCT_OPS(name, args...) \ --SEC("struct_ops/"#name) \ --BPF_PROG(name, ##args) -- --#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ --SEC("struct_ops.s/"#name) \ --BPF_PROG(name, ##args) -- --/** -- * MEMBER_VPTR - Obtain the verified pointer to a struct or array member -- * @base: struct or array to index -- * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...) -- * -- * The verifier often gets confused by the instruction sequence the compiler -- * generates for indexing struct fields or arrays. This macro forces the -- * compiler to generate a code sequence which first calculates the byte offset, -- * checks it against the struct or array size and add that byte offset to -- * generate the pointer to the member to help the verifier. -- * -- * Ideally, we want to abort if the calculated offset is out-of-bounds. However, -- * BPF currently doesn't support abort, so evaluate to NULL instead. The caller -- * must check for NULL and take appropriate action to appease the verifier. To -- * avoid confusing the verifier, it's best to check for NULL and dereference -- * immediately. -- * -- * vptr = MEMBER_VPTR(my_array, [i][j]); -- * if (!vptr) -- * return error; -- * *vptr = new_value; -- */ --#define MEMBER_VPTR(base, member) (typeof(base member) *)({ \ -- u64 __base = (u64)base; \ -- u64 __addr = (u64)&(base member) - __base; \ -- asm volatile ( \ -- "if %0 <= %[max] goto +2\n" \ -- "%0 = 0\n" \ -- "goto +1\n" \ -- "%0 += %1\n" \ -- : "+r"(__addr) \ -- : "r"(__base), \ -- [max]"i"(sizeof(base) - sizeof(base member))); \ -- __addr; \ --}) -- --/* -- * BPF core and other generic helpers -- */ -- --/* list and rbtree */ --#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) --#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) -- --void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; --void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; -- --#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) --#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) -- --void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; --struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; --struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; --struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, -- struct bpf_rb_node *node) __ksym; --void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, -- bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym; --struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; -- --/* task */ --struct task_struct *bpf_task_from_pid(s32 pid) __ksym; --struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; --void bpf_task_release(struct task_struct *p) __ksym; -- --/* cgroup */ --struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; --void bpf_cgroup_release(struct cgroup *cgrp) __ksym; --struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; -- --/* cpumask */ --struct bpf_cpumask *bpf_cpumask_create(void) __ksym; --struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; --u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; --void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; --bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, -- const struct cpumask *src2) __ksym; --bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; --bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; --bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; --void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; --u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym; --u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym; -- --/* rcu */ --void bpf_rcu_read_lock(void) __ksym; --void bpf_rcu_read_unlock(void) __ksym; -- --/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */ --struct bpf_iter_num; -- --extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym; --extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym; --extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym; -- --#ifndef bpf_for_each --/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for -- * using BPF open-coded iterators without having to write mundane explicit -- * low-level loop logic. Instead, it provides for()-like generic construct -- * that can be used pretty naturally. E.g., for some hypothetical cgroup -- * iterator, you'd write: -- * -- * struct cgroup *cg, *parent_cg = <...>; -- * -- * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { -- * bpf_printk("Child cgroup id = %d", cg->cgroup_id); -- * if (cg->cgroup_id == 123) -- * break; -- * } -- * -- * I.e., it looks almost like high-level for each loop in other languages, -- * supports continue/break, and is verifiable by BPF verifier. -- * -- * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) -- * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to -- * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int -- * *`, not just `int`. So for integers bpf_for() is more convenient. -- * -- * Note: this macro relies on C99 feature of allowing to declare variables -- * inside for() loop, bound to for() loop lifetime. It also utilizes GCC -- * extension: __attribute__((cleanup())), supported by both GCC and -- * Clang. -- */ --#define bpf_for_each(type, cur, args...) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ -- cleanup(bpf_iter_##type##_destroy))), \ -- /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_##type##_new(&___it, ##args), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_##type##_destroy, (void *)0); \ -- /* iteration and termination check */ \ -- (((cur) = bpf_iter_##type##_next(&___it))); \ --) --#endif /* bpf_for_each */ -- --#ifndef bpf_for --/* bpf_for(i, start, end) implements a for()-like looping construct that sets -- * provided integer variable *i* to values starting from *start* through, -- * but not including, *end*. It also proves to BPF verifier that *i* belongs -- * to range [start, end), so this can be used for accessing arrays without -- * extra checks. -- * -- * Note: *start* and *end* are assumed to be expressions with no side effects -- * and whose values do not change throughout bpf_for() loop execution. They do -- * not have to be statically known or constant, though. -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_for(i, start, end) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, (start), (end)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- ({ \ -- /* iteration step */ \ -- int *___t = bpf_iter_num_next(&___it); \ -- /* termination and bounds check */ \ -- (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ -- }); \ --) --#endif /* bpf_for */ -- --#ifndef bpf_repeat --/* bpf_repeat(N) performs N iterations without exposing iteration number -- * -- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() -- * loop bound variables and cleanup attribute, supported by GCC and Clang. -- */ --#define bpf_repeat(N) for ( \ -- /* initialize and define destructor */ \ -- struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ -- cleanup(bpf_iter_num_destroy))), \ -- /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ -- *___p __attribute__((unused)) = ( \ -- bpf_iter_num_new(&___it, 0, (N)), \ -- /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ -- /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ -- (void)bpf_iter_num_destroy, (void *)0); \ -- bpf_iter_num_next(&___it); \ -- /* nothing here */ \ --) --#endif /* bpf_repeat */ -- --#endif /* __SCHED_EXT_COMMON_BPF_H */ -diff --git b/tools/sched_ext/scx_example_central.bpf.c a/tools/sched_ext/scx_example_central.bpf.c -deleted file mode 100644 -index 4cec04b4c2ed..000000000000 ---- b/tools/sched_ext/scx_example_central.bpf.c -+++ /dev/null -@@ -1,334 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A central FIFO sched_ext scheduler which demonstrates the followings: -- * -- * a. Making all scheduling decisions from one CPU: -- * -- * The central CPU is the only one making scheduling decisions. All other -- * CPUs kick the central CPU when they run out of tasks to run. -- * -- * There is one global BPF queue and the central CPU schedules all CPUs by -- * dispatching from the global queue to each CPU's local dsq from dispatch(). -- * This isn't the most straightforward. e.g. It'd be easier to bounce -- * through per-CPU BPF queues. The current design is chosen to maximally -- * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching. -- * -- * b. Tickless operation -- * -- * All tasks are dispatched with the infinite slice which allows stopping the -- * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full -- * parameter. The tickless operation can be observed through -- * /proc/interrupts. -- * -- * Periodic switching is enforced by a periodic timer checking all CPUs and -- * preempting them as necessary. Unfortunately, BPF timer currently doesn't -- * have a way to pin to a specific CPU, so the periodic timer isn't pinned to -- * the central CPU. -- * -- * c. Preemption -- * -- * Kthreads are unconditionally queued to the head of a matching local dsq -- * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always -- * prioritized over user threads, which is required for ensuring forward -- * progress as e.g. the periodic timer may run on a ksoftirqd and if the -- * ksoftirqd gets starved by a user thread, there may not be anything else to -- * vacate that user thread. -- * -- * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the -- * next tasks. -- * -- * This scheduler is designed to maximize usage of various SCX mechanisms. A -- * more practical implementation would likely put the scheduling loop outside -- * the central CPU's dispatch() path and add some form of priority mechanism. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --enum { -- FALLBACK_DSQ_ID = 0, -- MAX_CPUS = 4096, -- MS_TO_NS = 1000LLU * 1000, -- TIMER_INTERVAL_NS = 1 * MS_TO_NS, --}; -- --const volatile bool switch_partial; --const volatile s32 central_cpu; --const volatile u32 nr_cpu_ids = 64; /* !0 for veristat, set during init */ -- --u64 nr_total, nr_locals, nr_queued, nr_lost_pids; --u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries; --u64 nr_overflows; -- --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, s32); --} central_q SEC(".maps"); -- --/* can't use percpu map due to bad lookups */ --static bool cpu_gimme_task[MAX_CPUS]; --static u64 cpu_started_at[MAX_CPUS]; -- --struct central_timer { -- struct bpf_timer timer; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, 1); -- __type(key, u32); -- __type(value, struct central_timer); --} central_timer SEC(".maps"); -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- /* -- * Steer wakeups to the central CPU as much as possible to avoid -- * disturbing other CPUs. It's safe to blindly return the central cpu as -- * select_cpu() is a hint and if @p can't be on it, the kernel will -- * automatically pick a fallback CPU. -- */ -- return central_cpu; --} -- --void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags) --{ -- s32 pid = p->pid; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- /* -- * Push per-cpu kthreads at the head of local dsq's and preempt the -- * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked -- * behind other threads which is necessary for forward progress -- * guarantee as we depend on the BPF timer which may run from ksoftirqd. -- */ -- if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { -- __sync_fetch_and_add(&nr_locals, 1); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF, -- enq_flags | SCX_ENQ_PREEMPT); -- return; -- } -- -- if (bpf_map_push_elem(¢ral_q, &pid, 0)) { -- __sync_fetch_and_add(&nr_overflows, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_queued, 1); -- -- if (!scx_bpf_task_running(p)) -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); --} -- --static bool dispatch_to_cpu(s32 cpu) --{ -- struct task_struct *p; -- s32 pid; -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (bpf_map_pop_elem(¢ral_q, &pid)) -- break; -- -- __sync_fetch_and_sub(&nr_queued, 1); -- -- p = bpf_task_from_pid(pid); -- if (!p) { -- __sync_fetch_and_add(&nr_lost_pids, 1); -- continue; -- } -- -- /* -- * If we can't run the task at the top, do the dumb thing and -- * bounce it to the fallback dsq. -- */ -- if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { -- __sync_fetch_and_add(&nr_mismatches, 1); -- scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0); -- bpf_task_release(p); -- continue; -- } -- -- /* dispatch to local and mark that @cpu doesn't need more */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0); -- -- if (cpu != central_cpu) -- scx_bpf_kick_cpu(cpu, 0); -- -- bpf_task_release(p); -- return true; -- } -- -- return false; --} -- --void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (cpu == central_cpu) { -- /* dispatch for all other CPUs first */ -- __sync_fetch_and_add(&nr_dispatches, 1); -- -- bpf_for(cpu, 0, nr_cpu_ids) { -- bool *gimme; -- -- if (!scx_bpf_dispatch_nr_slots()) -- break; -- -- /* central's gimme is never set */ -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme && !*gimme) -- continue; -- -- if (dispatch_to_cpu(cpu)) -- *gimme = false; -- } -- -- /* -- * Retry if we ran out of dispatch buffer slots as we might have -- * skipped some CPUs and also need to dispatch for self. The ext -- * core automatically retries if the local dsq is empty but we -- * can't rely on that as we're dispatching for other CPUs too. -- * Kick self explicitly to retry. -- */ -- if (!scx_bpf_dispatch_nr_slots()) { -- __sync_fetch_and_add(&nr_retries, 1); -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- return; -- } -- -- /* look for a task to run on the central CPU */ -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- dispatch_to_cpu(central_cpu); -- } else { -- bool *gimme; -- -- if (scx_bpf_consume(FALLBACK_DSQ_ID)) -- return; -- -- gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); -- if (gimme) -- *gimme = true; -- -- /* -- * Force dispatch on the scheduling CPU so that it finds a task -- * to run for us. -- */ -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- } --} -- --void BPF_STRUCT_OPS(central_running, struct task_struct *p) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ --} -- --void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) --{ -- s32 cpu = scx_bpf_task_cpu(p); -- u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at) -- *started_at = 0; --} -- --static int central_timerfn(void *map, int *key, struct bpf_timer *timer) --{ -- u64 now = bpf_ktime_get_ns(); -- u64 nr_to_kick = nr_queued; -- s32 i; -- -- bpf_for(i, 0, nr_cpu_ids) { -- s32 cpu = (nr_timers + i) % nr_cpu_ids; -- u64 *started_at; -- -- if (cpu == central_cpu) -- continue; -- -- /* kick iff the current one exhausted its slice */ -- started_at = MEMBER_VPTR(cpu_started_at, [cpu]); -- if (started_at && *started_at && -- vtime_before(now, *started_at + SCX_SLICE_DFL)) -- continue; -- -- /* and there's something pending */ -- if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) || -- scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu)) -- ; -- else if (nr_to_kick) -- nr_to_kick--; -- else -- continue; -- -- scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); -- } -- -- scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); -- -- bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- __sync_fetch_and_add(&nr_timers, 1); -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(central_init) --{ -- u32 key = 0; -- struct bpf_timer *timer; -- int ret; -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- -- ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1); -- if (ret) -- return ret; -- -- timer = bpf_map_lookup_elem(¢ral_timer, &key); -- if (!timer) -- return -ESRCH; -- -- bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); -- bpf_timer_set_callback(timer, central_timerfn); -- ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); -- return ret; --} -- --void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops central_ops = { -- /* -- * We are offloading all scheduling decisions to the central CPU and -- * thus being the last task on a given CPU doesn't mean anything -- * special. Enqueue the last tasks like any other tasks. -- */ -- .flags = SCX_OPS_ENQ_LAST, -- -- .select_cpu = (void *)central_select_cpu, -- .enqueue = (void *)central_enqueue, -- .dispatch = (void *)central_dispatch, -- .running = (void *)central_running, -- .stopping = (void *)central_stopping, -- .init = (void *)central_init, -- .exit = (void *)central_exit, -- .name = "central", --}; -diff --git b/tools/sched_ext/scx_example_central.c a/tools/sched_ext/scx_example_central.c -deleted file mode 100644 -index 7ad591cbdc65..000000000000 ---- b/tools/sched_ext/scx_example_central.c -+++ /dev/null -@@ -1,94 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_central.skel.h" -- --const char help_fmt[] = --"A central FIFO sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-c CPU] [-p]\n" --"\n" --" -c CPU Override the central CPU (default: 0)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_central *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_central__open(); -- assert(skel); -- -- skel->rodata->central_cpu = 0; -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "c:ph")) != -1) { -- switch (opt) { -- case 'c': -- skel->rodata->central_cpu = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_central__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.central_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf("total :%10lu local:%10lu queued:%10lu lost:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_locals, -- skel->bss->nr_queued, -- skel->bss->nr_lost_pids); -- printf("timer :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n", -- skel->bss->nr_timers, -- skel->bss->nr_dispatches, -- skel->bss->nr_mismatches, -- skel->bss->nr_retries); -- printf("overflow:%10lu\n", -- skel->bss->nr_overflows); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_central__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_flatcg.bpf.c a/tools/sched_ext/scx_example_flatcg.bpf.c -deleted file mode 100644 -index f6078b9a681f..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.bpf.c -+++ /dev/null -@@ -1,872 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext flattened cgroup hierarchy scheduler. It implements -- * hierarchical weight-based cgroup CPU control by flattening the cgroup -- * hierarchy into a single layer by compounding the active weight share at each -- * level. Consider the following hierarchy with weights in parentheses: -- * -- * R + A (100) + B (100) -- * | \ C (100) -- * \ D (200) -- * -- * Ignoring the root and threaded cgroups, only B, C and D can contain tasks. -- * Let's say all three have runnable tasks. The total share that each of these -- * three cgroups is entitled to can be calculated by compounding its share at -- * each level. -- * -- * For example, B is competing against C and in that competition its share is -- * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's -- * share in that competition is 200/(200+100) == 1/3. B's eventual share in the -- * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's -- * eventual shaer is the same at 1/6. D is only competing at the top level and -- * its share is 200/(100+200) == 2/3. -- * -- * So, instead of hierarchically scheduling level-by-level, we can consider it -- * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3 -- * and keep updating the eventual shares as the cgroups' runnable states change. -- * -- * This flattening of hierarchy can bring a substantial performance gain when -- * the cgroup hierarchy is nested multiple levels. in a simple benchmark using -- * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it -- * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two -- * apache instances competing with 2:1 weight ratio nested four level deep. -- * -- * However, the gain comes at the cost of not being able to properly handle -- * thundering herd of cgroups. For example, if many cgroups which are nested -- * behind a low priority parent cgroup wake up around the same time, they may be -- * able to consume more CPU cycles than they are entitled to. In many use cases, -- * this isn't a real concern especially given the performance gain. Also, there -- * are ways to mitigate the problem further by e.g. introducing an extra -- * scheduling layer on cgroup delegation boundaries. -- * -- * The scheduler first picks the cgroup to run and then schedule the tasks -- * within by using nested weighted vtime scheduling by default. The -- * cgroup-internal scheduling can be switched to FIFO with the -f option. -- */ --#include "scx_common.bpf.h" --#include "user_exit_info.h" --#include "scx_example_flatcg.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */ --const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL; --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --u64 cvtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, u64); -- __uint(max_entries, FCG_NR_STATS); --} stats SEC(".maps"); -- --static void stat_inc(enum fcg_stat_idx idx) --{ -- u32 idx_v = idx; -- -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); -- if (cnt_p) -- (*cnt_p)++; --} -- --struct fcg_cpu_ctx { -- u64 cur_cgid; -- u64 cur_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __type(key, u32); -- __type(value, struct fcg_cpu_ctx); -- __uint(max_entries, 1); --} cpu_ctx SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_cgrp_ctx); --} cgrp_ctx SEC(".maps"); -- --struct cgv_node { -- struct bpf_rb_node rb_node; -- __u64 cvtime; -- __u64 cgid; --}; -- --private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock; --private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node); -- --struct cgv_node_stash { -- struct cgv_node __kptr *node; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, 16384); -- __type(key, __u64); -- __type(value, struct cgv_node_stash); --} cgv_node_stash SEC(".maps"); -- --struct fcg_task_ctx { -- u64 bypassed_at; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct fcg_task_ctx); --} task_ctx SEC(".maps"); -- --/* gets inc'd on weight tree changes to expire the cached hweights */ --unsigned long hweight_gen = 1; -- --static u64 div_round_up(u64 dividend, u64 divisor) --{ -- return (dividend + divisor - 1) / divisor; --} -- --static bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) --{ -- struct cgv_node *cgc_a, *cgc_b; -- -- cgc_a = container_of(a, struct cgv_node, rb_node); -- cgc_b = container_of(b, struct cgv_node, rb_node); -- -- return cgc_a->cvtime < cgc_b->cvtime; --} -- --static struct fcg_cpu_ctx *find_cpu_ctx(void) --{ -- struct fcg_cpu_ctx *cpuc; -- u32 idx = 0; -- -- cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx); -- if (!cpuc) { -- scx_bpf_error("cpu_ctx lookup failed"); -- return NULL; -- } -- return cpuc; --} -- --static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id); -- return NULL; -- } -- return cgc; --} -- --static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level) --{ -- struct fcg_cgrp_ctx *cgc; -- -- cgrp = bpf_cgroup_ancestor(cgrp, level); -- if (!cgrp) { -- scx_bpf_error("ancestor cgroup lookup failed"); -- return NULL; -- } -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- scx_bpf_error("ancestor cgrp_ctx lookup failed"); -- bpf_cgroup_release(cgrp); -- return cgc; --} -- --static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- int level; -- -- if (!cgc->nr_active) { -- stat_inc(FCG_STAT_HWT_SKIP); -- return; -- } -- -- if (cgc->hweight_gen == hweight_gen) { -- stat_inc(FCG_STAT_HWT_CACHE); -- return; -- } -- -- stat_inc(FCG_STAT_HWT_UPDATES); -- bpf_for(level, 0, cgrp->level + 1) { -- struct fcg_cgrp_ctx *cgc; -- bool is_active; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- -- if (!level) { -- cgc->hweight = FCG_HWEIGHT_ONE; -- cgc->hweight_gen = hweight_gen; -- } else { -- struct fcg_cgrp_ctx *pcgc; -- -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- -- /* -- * We can be oppotunistic here and not grab the -- * cgv_tree_lock and deal with the occasional races. -- * However, hweight updates are already cached and -- * relatively low-frequency. Let's just do the -- * straightforward thing. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- is_active = cgc->nr_active; -- if (is_active) { -- cgc->hweight_gen = pcgc->hweight_gen; -- cgc->hweight = -- div_round_up(pcgc->hweight * cgc->weight, -- pcgc->child_weight_sum); -- } -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!is_active) { -- stat_inc(FCG_STAT_HWT_RACE); -- break; -- } -- } -- } --} -- --static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) --{ -- u64 delta, cvtime, max_budget; -- -- /* -- * A node which is on the rbtree can't be pointed to from elsewhere yet -- * and thus can't be updated and repositioned. Instead, we collect the -- * vtime deltas separately and apply it asynchronously here. -- */ -- delta = cgc->cvtime_delta; -- __sync_fetch_and_sub(&cgc->cvtime_delta, delta); -- cvtime = cgv_node->cvtime + delta; -- -- /* -- * Allow a cgroup to carry the maximum budget proportional to its -- * hweight such that a full-hweight cgroup can immediately take up half -- * of the CPUs at the most while staying at the front of the rbtree. -- */ -- max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / -- (2 * FCG_HWEIGHT_ONE); -- if (vtime_before(cvtime, cvtime_now - max_budget)) -- cvtime = cvtime_now - max_budget; -- -- cgv_node->cvtime = cvtime; --} -- --static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) --{ -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- u64 cgid = cgrp->kn->id; -- -- /* paired with cmpxchg in try_pick_next_cgroup() */ -- if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { -- stat_inc(FCG_STAT_ENQ_SKIP); -- return; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid); -- return; -- } -- -- /* NULL if the node is already on the rbtree */ -- cgv_node = bpf_kptr_xchg(&stash->node, NULL); -- if (!cgv_node) { -- stat_inc(FCG_STAT_ENQ_RACE); -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); --} -- --void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * If select_cpu_dfl() is recommending local enqueue, the target CPU is -- * idle. Follow it and charge the cgroup later in fcg_stopping() after -- * the fact. Use the same mechanism to deal with tasks with custom -- * affinities so that we don't have to worry about per-cgroup dq's -- * containing tasks that can't be executed from some CPUs. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) { -- /* -- * Tell fcg_stopping() that this bypassed the regular scheduling -- * path and should be force charged to the cgroup. 0 is used to -- * indicate that the task isn't bypassing, so if the current -- * runtime is 0, go back by one nanosecond. -- */ -- taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1; -- -- /* -- * The global dq is deprioritized as we don't want to let tasks -- * to boost themselves by constraining its cpumask. The -- * deprioritization is rather severe, so let's not apply that to -- * per-cpu kernel threads. This is ham-fisted. We probably wanna -- * implement per-cgroup fallback dq's instead so that we have -- * more control over when tasks with custom cpumask get issued. -- */ -- if ((enq_flags & SCX_ENQ_LOCAL) || -- (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) { -- stat_inc(FCG_STAT_LOCAL); -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- } else { -- stat_inc(FCG_STAT_GLOBAL); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } -- return; -- } -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- goto out_release; -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 tvtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) -- tvtime = cgc->tvtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, -- tvtime, enq_flags); -- } -- -- cgrp_enqueued(cgrp, cgc); --out_release: -- bpf_cgroup_release(cgrp); --} -- --/* -- * Walk the cgroup tree to update the active weight sums as tasks wake up and -- * sleep. The weight sums are used as the base when calculating the proportion a -- * given cgroup or task is entitled to at each level. -- */ --static void update_active_weight_sums(struct cgroup *cgrp, bool runnable) --{ -- struct fcg_cgrp_ctx *cgc; -- bool updated = false; -- int idx; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- /* -- * In most cases, a hot cgroup would have multiple threads going to -- * sleep and waking up while the whole cgroup stays active. In leaf -- * cgroups, ->nr_runnable which is updated with __sync operations gates -- * ->nr_active updates, so that we don't have to grab the cgv_tree_lock -- * repeatedly for a busy cgroup which is staying active. -- */ -- if (runnable) { -- if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_ACT); -- } else { -- if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) -- return; -- stat_inc(FCG_STAT_DEACT); -- } -- -- /* -- * If @cgrp is becoming runnable, its hweight should be refreshed after -- * it's added to the weight tree so that enqueue has the up-to-date -- * value. If @cgrp is becoming quiescent, the hweight should be -- * refreshed before it's removed from the weight tree so that the usage -- * charging which happens afterwards has access to the latest value. -- */ -- if (!runnable) -- cgrp_refresh_hweight(cgrp, cgc); -- -- /* propagate upwards */ -- bpf_for(idx, 0, cgrp->level) { -- int level = cgrp->level - idx; -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- bool propagate = false; -- -- cgc = find_ancestor_cgrp_ctx(cgrp, level); -- if (!cgc) -- break; -- if (level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); -- if (!pcgc) -- break; -- } -- -- /* -- * We need the propagation protected by a lock to synchronize -- * against weight changes. There's no reason to drop the lock at -- * each level but bpf_spin_lock() doesn't want any function -- * calls while locked. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- -- if (runnable) { -- if (!cgc->nr_active++) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum += cgc->weight; -- } -- } -- } else { -- if (!--cgc->nr_active) { -- updated = true; -- if (pcgc) { -- propagate = true; -- pcgc->child_weight_sum -= cgc->weight; -- } -- } -- } -- -- bpf_spin_unlock(&cgv_tree_lock); -- -- if (!propagate) -- break; -- } -- -- if (updated) -- __sync_fetch_and_add(&hweight_gen, 1); -- -- if (runnable) -- cgrp_refresh_hweight(cgrp, cgc); --} -- --void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, true); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) --{ -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- if (fifo_sched) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- /* -- * @cgc->tvtime_now always progresses forward as tasks start -- * executing. The test and update can be performed concurrently -- * from multiple CPUs and thus racy. Any error should be -- * contained and temporary. Let's just live with it. -- */ -- if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) -- cgc->tvtime_now = p->scx.dsq_vtime; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) --{ -- struct fcg_task_ctx *taskc; -- struct cgroup *cgrp; -- struct fcg_cgrp_ctx *cgc; -- -- /* scale the execution time by the inverse of the weight and charge */ -- if (!fifo_sched) -- p->scx.dsq_vtime += -- (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; -- -- taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); -- if (!taskc) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- if (!taskc->bypassed_at) -- return; -- -- cgrp = scx_bpf_task_cgroup(p); -- cgc = find_cgrp_ctx(cgrp); -- if (cgc) { -- __sync_fetch_and_add(&cgc->cvtime_delta, -- p->se.sum_exec_runtime - taskc->bypassed_at); -- taskc->bypassed_at = 0; -- } -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) --{ -- struct cgroup *cgrp; -- -- cgrp = scx_bpf_task_cgroup(p); -- update_active_weight_sums(cgrp, false); -- bpf_cgroup_release(cgrp); --} -- --void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) --{ -- struct fcg_cgrp_ctx *cgc, *pcgc = NULL; -- -- cgc = find_cgrp_ctx(cgrp); -- if (!cgc) -- return; -- -- if (cgrp->level) { -- pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1); -- if (!pcgc) -- return; -- } -- -- bpf_spin_lock(&cgv_tree_lock); -- if (pcgc && cgc->nr_active) -- pcgc->child_weight_sum += (s64)weight - cgc->weight; -- cgc->weight = weight; -- bpf_spin_unlock(&cgv_tree_lock); --} -- --static bool try_pick_next_cgroup(u64 *cgidp) --{ -- struct bpf_rb_node *rb_node; -- struct cgv_node_stash *stash; -- struct cgv_node *cgv_node; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 cgid; -- -- /* pop the front cgroup and wind cvtime_now accordingly */ -- bpf_spin_lock(&cgv_tree_lock); -- -- rb_node = bpf_rbtree_first(&cgv_tree); -- if (!rb_node) { -- bpf_spin_unlock(&cgv_tree_lock); -- stat_inc(FCG_STAT_PNC_NO_CGRP); -- *cgidp = 0; -- return true; -- } -- -- rb_node = bpf_rbtree_remove(&cgv_tree, rb_node); -- bpf_spin_unlock(&cgv_tree_lock); -- -- cgv_node = container_of(rb_node, struct cgv_node, rb_node); -- cgid = cgv_node->cgid; -- -- if (vtime_before(cvtime_now, cgv_node->cvtime)) -- cvtime_now = cgv_node->cvtime; -- -- /* -- * If lookup fails, the cgroup's gone. Free and move on. See -- * fcg_cgroup_exit(). -- */ -- cgrp = bpf_cgroup_from_id(cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (!cgc) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- if (!scx_bpf_consume(cgid)) { -- bpf_cgroup_release(cgrp); -- stat_inc(FCG_STAT_PNC_EMPTY); -- goto out_stash; -- } -- -- /* -- * Successfully consumed from the cgroup. This will be our current -- * cgroup for the new slice. Refresh its hweight. -- */ -- cgrp_refresh_hweight(cgrp, cgc); -- -- bpf_cgroup_release(cgrp); -- -- /* -- * As the cgroup may have more tasks, add it back to the rbtree. Note -- * that here we charge the full slice upfront and then exact later -- * according to the actual consumption. This prevents lowpri thundering -- * herd from saturating the machine. -- */ -- bpf_spin_lock(&cgv_tree_lock); -- cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); -- cgrp_cap_budget(cgv_node, cgc); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- -- *cgidp = cgid; -- stat_inc(FCG_STAT_PNC_NEXT); -- return true; -- --out_stash: -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- stat_inc(FCG_STAT_PNC_GONE); -- goto out_free; -- } -- -- /* -- * Paired with cmpxchg in cgrp_enqueued(). If they see the following -- * transition, they'll enqueue the cgroup. If they are earlier, we'll -- * see their task in the dq below and requeue the cgroup. -- */ -- __sync_val_compare_and_swap(&cgc->queued, 1, 0); -- -- if (scx_bpf_dsq_nr_queued(cgid)) { -- bpf_spin_lock(&cgv_tree_lock); -- bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- goto out_free; -- } -- } -- -- return false; -- --out_free: -- bpf_obj_drop(cgv_node); -- return false; --} -- --void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) --{ -- struct fcg_cpu_ctx *cpuc; -- struct fcg_cgrp_ctx *cgc; -- struct cgroup *cgrp; -- u64 now = bpf_ktime_get_ns(); -- -- cpuc = find_cpu_ctx(); -- if (!cpuc) -- return; -- -- if (!cpuc->cur_cgid) -- goto pick_next_cgroup; -- -- if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { -- if (scx_bpf_consume(cpuc->cur_cgid)) { -- stat_inc(FCG_STAT_CNS_KEEP); -- return; -- } -- stat_inc(FCG_STAT_CNS_EMPTY); -- } else { -- stat_inc(FCG_STAT_CNS_EXPIRE); -- } -- -- /* -- * The current cgroup is expiring. It was already charged a full slice. -- * Calculate the actual usage and accumulate the delta. -- */ -- cgrp = bpf_cgroup_from_id(cpuc->cur_cgid); -- if (!cgrp) { -- stat_inc(FCG_STAT_CNS_GONE); -- goto pick_next_cgroup; -- } -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); -- if (cgc) { -- /* -- * We want to update the vtime delta and then look for the next -- * cgroup to execute but the latter needs to be done in a loop -- * and we can't keep the lock held. Oh well... -- */ -- bpf_spin_lock(&cgv_tree_lock); -- __sync_fetch_and_add(&cgc->cvtime_delta, -- (cpuc->cur_at + cgrp_slice_ns - now) * -- FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); -- bpf_spin_unlock(&cgv_tree_lock); -- } else { -- stat_inc(FCG_STAT_CNS_GONE); -- } -- -- bpf_cgroup_release(cgrp); -- --pick_next_cgroup: -- cpuc->cur_at = now; -- -- if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { -- cpuc->cur_cgid = 0; -- return; -- } -- -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_pick_next_cgroup(&cpuc->cur_cgid)) -- break; -- } --} -- --s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- struct fcg_task_ctx *taskc; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- taskc = bpf_task_storage_get(&task_ctx, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!taskc) -- return -ENOMEM; -- -- taskc->bypassed_at = 0; -- return 0; --} -- --int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp, -- struct scx_cgroup_init_args *args) --{ -- struct fcg_cgrp_ctx *cgc; -- struct cgv_node *cgv_node; -- struct cgv_node_stash empty_stash = {}, *stash; -- u64 cgid = cgrp->kn->id; -- int ret; -- -- /* -- * Technically incorrect as cgroup ID is full 64bit while dq ID is -- * 63bit. Should not be a problem in practice and easy to spot in the -- * unlikely case that it breaks. -- */ -- ret = scx_bpf_create_dsq(cgid, -1); -- if (ret) -- return ret; -- -- cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE); -- if (!cgc) { -- ret = -ENOMEM; -- goto err_destroy_dsq; -- } -- -- cgc->weight = args->weight; -- cgc->hweight = FCG_HWEIGHT_ONE; -- -- ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash, -- BPF_NOEXIST); -- if (ret) { -- if (ret != -ENOMEM) -- scx_bpf_error("unexpected stash creation error (%d)", -- ret); -- goto err_destroy_dsq; -- } -- -- stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); -- if (!stash) { -- scx_bpf_error("unexpected cgv_node stash lookup failure"); -- ret = -ENOENT; -- goto err_destroy_dsq; -- } -- -- cgv_node = bpf_obj_new(struct cgv_node); -- if (!cgv_node) { -- ret = -ENOMEM; -- goto err_del_cgv_node; -- } -- -- cgv_node->cgid = cgid; -- cgv_node->cvtime = cvtime_now; -- -- cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); -- if (cgv_node) { -- scx_bpf_error("unexpected !NULL cgv_node stash"); -- ret = -EBUSY; -- goto err_drop; -- } -- -- return 0; -- --err_drop: -- bpf_obj_drop(cgv_node); --err_del_cgv_node: -- bpf_map_delete_elem(&cgv_node_stash, &cgid); --err_destroy_dsq: -- scx_bpf_destroy_dsq(cgid); -- return ret; --} -- --void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- -- /* -- * For now, there's no way find and remove the cgv_node if it's on the -- * cgv_tree. Let's drain them in the dispatch path as they get popped -- * off the front of the tree. -- */ -- bpf_map_delete_elem(&cgv_node_stash, &cgid); -- scx_bpf_destroy_dsq(cgid); --} -- --s32 BPF_STRUCT_OPS(fcg_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops flatcg_ops = { -- .enqueue = (void *)fcg_enqueue, -- .dispatch = (void *)fcg_dispatch, -- .runnable = (void *)fcg_runnable, -- .running = (void *)fcg_running, -- .stopping = (void *)fcg_stopping, -- .quiescent = (void *)fcg_quiescent, -- .prep_enable = (void *)fcg_prep_enable, -- .cgroup_set_weight = (void *)fcg_cgroup_set_weight, -- .cgroup_init = (void *)fcg_cgroup_init, -- .cgroup_exit = (void *)fcg_cgroup_exit, -- .init = (void *)fcg_init, -- .exit = (void *)fcg_exit, -- .flags = SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING, -- .name = "flatcg", --}; -diff --git b/tools/sched_ext/scx_example_flatcg.c a/tools/sched_ext/scx_example_flatcg.c -deleted file mode 100644 -index f9c8a5b84a70..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.c -+++ /dev/null -@@ -1,232 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2023 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2023 Tejun Heo -- * Copyright (c) 2023 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_flatcg.h" --#include "scx_example_flatcg.skel.h" -- --#ifndef FILEID_KERNFS --#define FILEID_KERNFS 0xfe --#endif -- --const char help_fmt[] = --"A flattened cgroup hierarchy sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -i INTERVAL Report interval\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --static float read_cpu_util(__u64 *last_sum, __u64 *last_idle) --{ -- FILE *fp; -- char buf[4096]; -- char *line, *cur = NULL, *tok; -- __u64 sum = 0, idle = 0; -- __u64 delta_sum, delta_idle; -- int idx; -- -- fp = fopen("/proc/stat", "r"); -- if (!fp) { -- perror("fopen(\"/proc/stat\")"); -- return 0.0; -- } -- -- if (!fgets(buf, sizeof(buf), fp)) { -- perror("fgets(\"/proc/stat\")"); -- fclose(fp); -- return 0.0; -- } -- fclose(fp); -- -- line = buf; -- for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) { -- char *endp = NULL; -- __u64 v; -- -- if (idx == 0) { -- line = NULL; -- continue; -- } -- v = strtoull(tok, &endp, 0); -- if (!endp || *endp != '\0') { -- fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n", -- idx, tok); -- continue; -- } -- sum += v; -- if (idx == 4) -- idle = v; -- } -- -- delta_sum = sum - *last_sum; -- delta_idle = idle - *last_idle; -- *last_sum = sum; -- *last_idle = idle; -- -- return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0; --} -- --static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats) --{ -- __u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus]; -- __u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); -- -- for (idx = 0; idx < FCG_NR_STATS; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_flatcg *skel; -- struct bpf_link *link; -- struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 }; -- bool dump_cgrps = false; -- __u64 last_cpu_sum = 0, last_cpu_idle = 0; -- __u64 last_stats[FCG_NR_STATS] = {}; -- unsigned long seq = 0; -- s32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_flatcg__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open: %s\n", strerror(errno)); -- return 1; -- } -- -- skel->rodata->nr_cpus = libbpf_num_possible_cpus(); -- -- while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) { -- double v; -- -- switch (opt) { -- case 's': -- v = strtod(optarg, NULL); -- skel->rodata->cgrp_slice_ns = v * 1000; -- break; -- case 'i': -- v = strtod(optarg, NULL); -- intv_ts.tv_sec = v; -- intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000; -- break; -- case 'd': -- dump_cgrps = true; -- break; -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- case 'h': -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d", -- (double)skel->rodata->cgrp_slice_ns / 1000000.0, -- (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0, -- dump_cgrps); -- -- if (scx_example_flatcg__load(skel)) { -- fprintf(stderr, "Failed to load: %s\n", strerror(errno)); -- return 1; -- } -- -- link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops); -- if (!link) { -- fprintf(stderr, "Failed to attach_struct_ops: %s\n", -- strerror(errno)); -- return 1; -- } -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- __u64 acc_stats[FCG_NR_STATS]; -- __u64 stats[FCG_NR_STATS]; -- float cpu_util; -- int i; -- -- cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle); -- -- fcg_read_stats(skel, acc_stats); -- for (i = 0; i < FCG_NR_STATS; i++) -- stats[i] = acc_stats[i] - last_stats[i]; -- -- memcpy(last_stats, acc_stats, sizeof(acc_stats)); -- -- printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n", -- seq++, cpu_util * 100.0, skel->data->hweight_gen); -- printf(" act:%6llu deact:%6llu local:%6llu global:%6llu\n", -- stats[FCG_STAT_ACT], -- stats[FCG_STAT_DEACT], -- stats[FCG_STAT_LOCAL], -- stats[FCG_STAT_GLOBAL]); -- printf("HWT skip:%6llu race:%6llu cache:%6llu update:%6llu\n", -- stats[FCG_STAT_HWT_SKIP], -- stats[FCG_STAT_HWT_RACE], -- stats[FCG_STAT_HWT_CACHE], -- stats[FCG_STAT_HWT_UPDATES]); -- printf("ENQ skip:%6llu race:%6llu\n", -- stats[FCG_STAT_ENQ_SKIP], -- stats[FCG_STAT_ENQ_RACE]); -- printf("CNS keep:%6llu expire:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_CNS_KEEP], -- stats[FCG_STAT_CNS_EXPIRE], -- stats[FCG_STAT_CNS_EMPTY], -- stats[FCG_STAT_CNS_GONE]); -- printf("PNC nocgrp:%6llu next:%6llu empty:%6llu gone:%6llu\n", -- stats[FCG_STAT_PNC_NO_CGRP], -- stats[FCG_STAT_PNC_NEXT], -- stats[FCG_STAT_PNC_EMPTY], -- stats[FCG_STAT_PNC_GONE]); -- printf("BAD remove:%6llu\n", -- acc_stats[FCG_STAT_BAD_REMOVAL]); -- -- nanosleep(&intv_ts, NULL); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_flatcg__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_flatcg.h a/tools/sched_ext/scx_example_flatcg.h -deleted file mode 100644 -index 490758ed41f0..000000000000 ---- b/tools/sched_ext/scx_example_flatcg.h -+++ /dev/null -@@ -1,49 +0,0 @@ --#ifndef __SCX_EXAMPLE_FLATCG_H --#define __SCX_EXAMPLE_FLATCG_H -- --enum { -- FCG_HWEIGHT_ONE = 1LLU << 16, --}; -- --enum fcg_stat_idx { -- FCG_STAT_ACT, -- FCG_STAT_DEACT, -- FCG_STAT_LOCAL, -- FCG_STAT_GLOBAL, -- -- FCG_STAT_HWT_UPDATES, -- FCG_STAT_HWT_CACHE, -- FCG_STAT_HWT_SKIP, -- FCG_STAT_HWT_RACE, -- -- FCG_STAT_ENQ_SKIP, -- FCG_STAT_ENQ_RACE, -- -- FCG_STAT_CNS_KEEP, -- FCG_STAT_CNS_EXPIRE, -- FCG_STAT_CNS_EMPTY, -- FCG_STAT_CNS_GONE, -- -- FCG_STAT_PNC_NO_CGRP, -- FCG_STAT_PNC_NEXT, -- FCG_STAT_PNC_EMPTY, -- FCG_STAT_PNC_GONE, -- -- FCG_STAT_BAD_REMOVAL, -- -- FCG_NR_STATS, --}; -- --struct fcg_cgrp_ctx { -- u32 nr_active; -- u32 nr_runnable; -- u32 queued; -- u32 weight; -- u32 hweight; -- u64 child_weight_sum; -- u64 hweight_gen; -- s64 cvtime_delta; -- u64 tvtime_now; --}; -- --#endif /* __SCX_EXAMPLE_FLATCG_H */ -diff --git b/tools/sched_ext/scx_example_pair.bpf.c a/tools/sched_ext/scx_example_pair.bpf.c -deleted file mode 100644 -index 279efe58b777..000000000000 ---- b/tools/sched_ext/scx_example_pair.bpf.c -+++ /dev/null -@@ -1,627 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext core-scheduler which always makes every sibling CPU pair -- * execute from the same CPU cgroup. -- * -- * This scheduler is a minimal implementation and would need some form of -- * priority handling both inside each cgroup and across the cgroups to be -- * practically useful. -- * -- * Each CPU in the system is paired with exactly one other CPU, according to a -- * "stride" value that can be specified when the BPF scheduler program is first -- * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee -- * that they will only ever schedule tasks that belong to the same CPU cgroup. -- * -- * Scheduler Initialization -- * ------------------------ -- * -- * The scheduler BPF program is first initialized from user space, before it is -- * enabled. During this initialization process, each CPU on the system is -- * assigned several values that are constant throughout its runtime: -- * -- * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling -- * decisions. Paired CPUs always schedule tasks from the same -- * CPU cgroup, and synchronize with each other to guarantee -- * that this constraint is not violated. -- * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access -- * a struct pair_ctx object that is shared between the pair. -- * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the -- * pair. Each struct pair_ctx has an active_mask field, -- * which is a bitmap used to indicate whether each core -- * in the pair currently has an actively running task. -- * This index specifies which entry in the bitmap corresponds -- * to each CPU in the pair. -- * -- * During this initialization, the CPUs are paired according to a "stride" that -- * may be specified when invoking the user space program that initializes and -- * loads the scheduler. By default, the stride is 1/2 the total number of CPUs. -- * -- * Tasks and cgroups -- * ----------------- -- * -- * Every cgroup in the system is registered with the scheduler using the -- * pair_cgroup_init() callback, and every task in the system is associated with -- * exactly one cgroup. At a high level, the idea with the pair scheduler is to -- * always schedule tasks from the same cgroup within a given CPU pair. When a -- * task is enqueued (i.e. passed to the pair_enqueue() callback function), its -- * cgroup ID is read from its task struct, and then a corresponding queue map -- * is used to FIFO-enqueue the task for that cgroup. -- * -- * If you look through the implementation of the scheduler, you'll notice that -- * there is quite a bit of complexity involved with looking up the per-cgroup -- * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash -- * BPF hash map that is used to map a cgroup ID to a globally unique ID that's -- * allocated in the BPF program. This is done because we use separate maps to -- * store the FIFO queue of tasks, and the length of that map, per cgroup. This -- * complexity is only present because of current deficiencies in BPF that will -- * soon be addressed. The main point to keep in mind is that newly enqueued -- * tasks are added to their cgroup's FIFO queue. -- * -- * Dispatching tasks -- * ----------------- -- * -- * This section will describe how enqueued tasks are dispatched and scheduled. -- * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is -- * as follows: -- * -- * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is -- * the structure that's used to synchronize amongst the two pair CPUs in their -- * scheduling decisions. After any of the following events have occurred: -- * -- * - The cgroup's slice run has expired, or -- * - The cgroup becomes empty, or -- * - Either CPU in the pair is preempted by a higher priority scheduling class -- * -- * The cgroup transitions to the draining state and stops executing new tasks -- * from the cgroup. -- * -- * 2. If the pair is still executing a task, mark the pair_ctx as draining, and -- * wait for the pair CPU to be preempted. -- * -- * 3. Otherwise, if the pair CPU is not running a task, we can move onto -- * scheduling new tasks. Pop the next cgroup id from the top_q queue. -- * -- * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it. -- * -- * Note again that this scheduling behavior is simple, but the implementation -- * is complex mostly because this it hits several BPF shortcomings and has to -- * work around in often awkward ways. Most of the shortcomings are expected to -- * be resolved in the near future which should allow greatly simplifying this -- * scheduler. -- * -- * Dealing with preemption -- * ----------------------- -- * -- * SCX is the lowest priority sched_class, and could be preempted by them at -- * any time. To address this, the scheduler implements pair_cpu_release() and -- * pair_cpu_acquire() callbacks which are invoked by the core scheduler when -- * the scheduler loses and gains control of the CPU respectively. -- * -- * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and -- * then invoke: -- * -- * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- * -- * This preempts the pair CPU, and waits until it has re-entered the scheduler -- * before returning. This is necessary to ensure that the higher priority -- * sched_class that preempted our scheduler does not schedule a task -- * concurrently with our pair CPU. -- * -- * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption -- * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable -- * pair scheduling. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include "scx_example_pair.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; -- --/* !0 for veristat, set during init */ --const volatile u32 nr_cpu_ids = 64; -- --/* a pair of CPUs stay on a cgroup for this duration */ --const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL; -- --/* cpu ID -> pair cpu ID */ --const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 }; -- --/* cpu ID -> pair_id */ --const volatile u32 pair_id[MAX_CPUS]; -- --/* CPU ID -> CPU # in the pair (0 or 1) */ --const volatile u32 in_pair_idx[MAX_CPUS]; -- --struct pair_ctx { -- struct bpf_spin_lock lock; -- -- /* the cgroup the pair is currently executing */ -- u64 cgid; -- -- /* the pair started executing the current cgroup at */ -- u64 started_at; -- -- /* whether the current cgroup is draining */ -- bool draining; -- -- /* the CPUs that are currently active on the cgroup */ -- u32 active_mask; -- -- /* -- * the CPUs that are currently preempted and running tasks in a -- * different scheduler. -- */ -- u32 preempted_mask; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY); -- __uint(max_entries, MAX_CPUS / 2); -- __type(key, u32); -- __type(value, struct pair_ctx); --} pair_ctx SEC(".maps"); -- --/* queue of cgrp_q's possibly with tasks on them */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- /* -- * Because it's difficult to build strong synchronization encompassing -- * multiple non-trivial operations in BPF, this queue is managed in an -- * opportunistic way so that we guarantee that a cgroup w/ active tasks -- * is always on it but possibly multiple times. Once we have more robust -- * synchronization constructs and e.g. linked list, we should be able to -- * do this in a prettier way but for now just size it big enough. -- */ -- __uint(max_entries, 4 * MAX_CGRPS); -- __type(value, u64); --} top_q SEC(".maps"); -- --/* per-cgroup q which FIFOs the tasks from the cgroup */ --struct cgrp_q { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, MAX_QUEUED); -- __type(value, u32); --}; -- --/* -- * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local -- * storage; however, a cgroup local storage can only be accessed from the BPF -- * progs attached to the cgroup. For now, work around by allocating array of -- * cgrp_q's and then allocating per-cgroup indices. -- * -- * Another caveat: It's difficult to populate a large array of maps statically -- * or from BPF. Initialize it from userland. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, MAX_CGRPS); -- __type(key, s32); -- __array(values, struct cgrp_q); --} cgrp_q_arr SEC(".maps"); -- --static u64 cgrp_q_len[MAX_CGRPS]; -- --/* -- * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be -- * useful to have as a map type. -- */ --static u32 cgrp_q_idx_cursor; --static u64 cgrp_q_idx_busy[MAX_CGRPS]; -- --/* -- * All added up, the following is what we do: -- * -- * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking -- * for a free ID. If not found, fail cgroup creation with -EBUSY. -- * -- * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following -- * cgrp_q_idx_hash. -- * -- * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from -- * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr. -- * -- * This is sadly complicated for something pretty simple. Hopefully, we should -- * be able to simplify in the future. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_HASH); -- __uint(max_entries, MAX_CGRPS); -- __uint(key_size, sizeof(u64)); /* cgrp ID */ -- __uint(value_size, sizeof(s32)); /* cgrp_q idx */ --} cgrp_q_idx_hash SEC(".maps"); -- --/* statistics */ --u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions; --u64 nr_exps, nr_exp_waits, nr_exp_empty; --u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty; -- --struct user_exit_info uei; -- --static bool time_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags) --{ -- struct cgroup *cgrp; -- struct cgrp_q *cgq; -- s32 pid = p->pid; -- u64 cgid; -- u32 *q_idx; -- u64 *cgq_len; -- -- __sync_fetch_and_add(&nr_total, 1); -- -- cgrp = scx_bpf_task_cgroup(p); -- cgid = cgrp->kn->id; -- bpf_cgroup_release(cgrp); -- -- /* find the cgroup's q and push @p into it */ -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!q_idx) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return; -- } -- -- cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx); -- if (!cgq) { -- scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]", -- cgid, *q_idx); -- return; -- } -- -- if (bpf_map_push_elem(cgq, &pid, 0)) { -- scx_bpf_error("cgroup[%llu] queue overflow", cgid); -- return; -- } -- -- /* bump q len, if going 0 -> 1, queue cgroup into the top_q */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len) { -- scx_bpf_error("MEMBER_VTPR malfunction"); -- return; -- } -- -- if (!__sync_fetch_and_add(cgq_len, 1) && -- bpf_map_push_elem(&top_q, &cgid, 0)) { -- scx_bpf_error("top_q overflow"); -- return; -- } --} -- --static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) --{ -- u32 *vptr; -- -- vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *pairc = bpf_map_lookup_elem(&pair_ctx, vptr); -- if (!(*pairc)) -- return -EINVAL; -- -- vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]); -- if (!vptr) -- return -EINVAL; -- -- *mask = 1U << *vptr; -- -- return 0; --} -- --static int try_dispatch(s32 cpu) --{ -- struct pair_ctx *pairc; -- struct bpf_map *cgq_map; -- struct task_struct *p; -- u64 now = bpf_ktime_get_ns(); -- bool kick_pair = false; -- bool expired, pair_preempted; -- u32 *vptr, in_pair_mask; -- s32 pid, q_idx; -- u64 cgid; -- int ret; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) { -- scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]", -- cpu); -- return -ENOENT; -- } -- -- bpf_spin_lock(&pairc->lock); -- pairc->active_mask &= ~in_pair_mask; -- -- expired = time_before(pairc->started_at + pair_batch_dur_ns, now); -- if (expired || pairc->draining) { -- u64 new_cgid = 0; -- -- __sync_fetch_and_add(&nr_exps, 1); -- -- /* -- * We're done with the current cgid. An obvious optimization -- * would be not draining if the next cgroup is the current one. -- * For now, be dumb and always expire. -- */ -- pairc->draining = true; -- -- pair_preempted = pairc->preempted_mask; -- if (pairc->active_mask || pair_preempted) { -- /* -- * The other CPU is still active, or is no longer under -- * our control due to e.g. being preempted by a higher -- * priority sched_class. We want to wait until this -- * cgroup expires, or until control of our pair CPU has -- * been returned to us. -- * -- * If the pair controls its CPU, and the time already -- * expired, kick. When the other CPU arrives at -- * dispatch and clears its active mask, it'll push the -- * pair to the next cgroup and kick this CPU. -- */ -- __sync_fetch_and_add(&nr_exp_waits, 1); -- bpf_spin_unlock(&pairc->lock); -- if (expired && !pair_preempted) -- kick_pair = true; -- goto out_maybe_kick; -- } -- -- bpf_spin_unlock(&pairc->lock); -- -- /* -- * Pick the next cgroup. It'd be easier / cleaner to not drop -- * pairc->lock and use stronger synchronization here especially -- * given that we'll be switching cgroups significantly less -- * frequently than tasks. Unfortunately, bpf_spin_lock can't -- * really protect anything non-trivial. Let's do opportunistic -- * operations instead. -- */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u32 *q_idx; -- u64 *cgq_len; -- -- if (bpf_map_pop_elem(&top_q, &new_cgid)) { -- /* no active cgroup, go idle */ -- __sync_fetch_and_add(&nr_exp_empty, 1); -- return 0; -- } -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid); -- if (!q_idx) -- continue; -- -- /* -- * This is the only place where empty cgroups are taken -- * off the top_q. -- */ -- cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); -- if (!cgq_len || !*cgq_len) -- continue; -- -- /* -- * If it has any tasks, requeue as we may race and not -- * execute it. -- */ -- bpf_map_push_elem(&top_q, &new_cgid, 0); -- break; -- } -- -- bpf_spin_lock(&pairc->lock); -- -- /* -- * The other CPU may already have started on a new cgroup while -- * we dropped the lock. Make sure that we're still draining and -- * start on the new cgroup. -- */ -- if (pairc->draining && !pairc->active_mask) { -- __sync_fetch_and_add(&nr_cgrp_next, 1); -- pairc->cgid = new_cgid; -- pairc->started_at = now; -- pairc->draining = false; -- kick_pair = true; -- } else { -- __sync_fetch_and_add(&nr_cgrp_coll, 1); -- } -- } -- -- cgid = pairc->cgid; -- pairc->active_mask |= in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- -- /* again, it'd be better to do all these with the lock held, oh well */ -- vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (!vptr) { -- scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); -- return -ENOENT; -- } -- q_idx = *vptr; -- -- /* claim one task from cgrp_q w/ q_idx */ -- bpf_repeat(BPF_MAX_LOOPS) { -- u64 *cgq_len, len; -- -- cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]); -- if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) { -- /* the cgroup must be empty, expire and repeat */ -- __sync_fetch_and_add(&nr_cgrp_empty, 1); -- bpf_spin_lock(&pairc->lock); -- pairc->draining = true; -- pairc->active_mask &= ~in_pair_mask; -- bpf_spin_unlock(&pairc->lock); -- return -EAGAIN; -- } -- -- if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len) -- continue; -- -- break; -- } -- -- cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx); -- if (!cgq_map) { -- scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- if (bpf_map_pop_elem(cgq_map, &pid)) { -- scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]", -- cgid, q_idx); -- return -ENOENT; -- } -- -- p = bpf_task_from_pid(pid); -- if (p) { -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } else { -- /* we don't handle dequeues, retry on lost tasks */ -- __sync_fetch_and_add(&nr_missing, 1); -- return -EAGAIN; -- } -- --out_maybe_kick: -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } -- return 0; --} -- --void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) --{ -- bpf_repeat(BPF_MAX_LOOPS) { -- if (try_dispatch(cpu) != -EAGAIN) -- break; -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask &= ~in_pair_mask; -- /* Kick the pair CPU, unless it was also preempted. */ -- kick_pair = !pairc->preempted_mask; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); -- } -- } --} -- --void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- int ret; -- u32 in_pair_mask; -- struct pair_ctx *pairc; -- bool kick_pair; -- -- ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); -- if (ret) -- return; -- -- bpf_spin_lock(&pairc->lock); -- pairc->preempted_mask |= in_pair_mask; -- pairc->active_mask &= ~in_pair_mask; -- /* Kick the pair CPU if it's still running. */ -- kick_pair = pairc->active_mask; -- pairc->draining = true; -- bpf_spin_unlock(&pairc->lock); -- -- if (kick_pair) { -- s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); -- -- if (pair) { -- __sync_fetch_and_add(&nr_kicks, 1); -- scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); -- } -- } -- __sync_fetch_and_add(&nr_preemptions, 1); --} -- --s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 i, q_idx; -- -- bpf_for(i, 0, MAX_CGRPS) { -- q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS; -- if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1)) -- break; -- } -- if (i == MAX_CGRPS) -- return -EBUSY; -- -- if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]); -- if (busy) -- *busy = 0; -- return -EBUSY; -- } -- -- return 0; --} -- --void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp) --{ -- u64 cgid = cgrp->kn->id; -- s32 *q_idx; -- -- q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); -- if (q_idx) { -- u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]); -- if (busy) -- *busy = 0; -- bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid); -- } --} -- --s32 BPF_STRUCT_OPS(pair_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops pair_ops = { -- .enqueue = (void *)pair_enqueue, -- .dispatch = (void *)pair_dispatch, -- .cpu_acquire = (void *)pair_cpu_acquire, -- .cpu_release = (void *)pair_cpu_release, -- .cgroup_init = (void *)pair_cgroup_init, -- .cgroup_exit = (void *)pair_cgroup_exit, -- .init = (void *)pair_init, -- .exit = (void *)pair_exit, -- .name = "pair", --}; -diff --git b/tools/sched_ext/scx_example_pair.c a/tools/sched_ext/scx_example_pair.c -deleted file mode 100644 -index 18e032bbc173..000000000000 ---- b/tools/sched_ext/scx_example_pair.c -+++ /dev/null -@@ -1,143 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_pair.h" --#include "scx_example_pair.skel.h" -- --const char help_fmt[] = --"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n" --"execute from the same CPU cgroup.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-S STRIDE] [-p]\n" --"\n" --" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_pair *skel; -- struct bpf_link *link; -- u64 seq = 0; -- s32 stride, i, opt, outer_fd; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_pair__open(); -- assert(skel); -- -- skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); -- -- /* pair up the earlier half to the latter by default, override with -s */ -- stride = skel->rodata->nr_cpu_ids / 2; -- -- while ((opt = getopt(argc, argv, "S:ph")) != -1) { -- switch (opt) { -- case 'S': -- stride = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- for (i = 0; i < skel->rodata->nr_cpu_ids; i++) { -- if (skel->rodata->pair_cpu[i] < 0) { -- skel->rodata->pair_cpu[i] = i + stride; -- skel->rodata->pair_cpu[i + stride] = i; -- skel->rodata->pair_id[i] = i; -- skel->rodata->pair_id[i + stride] = i; -- skel->rodata->in_pair_idx[i] = 0; -- skel->rodata->in_pair_idx[i + stride] = 1; -- } -- } -- -- assert(!scx_example_pair__load(skel)); -- -- /* -- * Populate the cgrp_q_arr map which is an array containing per-cgroup -- * queues. It'd probably be better to do this from BPF but there are too -- * many to initialize statically and there's no way to dynamically -- * populate from BPF. -- */ -- outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr); -- assert(outer_fd >= 0); -- -- printf("Initializing"); -- for (i = 0; i < MAX_CGRPS; i++) { -- s32 inner_fd; -- -- if (exit_req) -- break; -- -- inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, -- sizeof(u32), MAX_QUEUED, NULL); -- assert(inner_fd >= 0); -- assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY)); -- close(inner_fd); -- -- if (!(i % 10)) -- printf("."); -- fflush(stdout); -- } -- printf("\n"); -- -- /* -- * Fully initialized, attach and run. -- */ -- link = bpf_map__attach_struct_ops(skel->maps.pair_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- printf("[SEQ %lu]\n", seq++); -- printf(" total:%10lu dispatch:%10lu missing:%10lu\n", -- skel->bss->nr_total, -- skel->bss->nr_dispatched, -- skel->bss->nr_missing); -- printf(" kicks:%10lu preemptions:%7lu\n", -- skel->bss->nr_kicks, -- skel->bss->nr_preemptions); -- printf(" exp:%10lu exp_wait:%10lu exp_empty:%10lu\n", -- skel->bss->nr_exps, -- skel->bss->nr_exp_waits, -- skel->bss->nr_exp_empty); -- printf("cgnext:%10lu cgcoll:%10lu cgempty:%10lu\n", -- skel->bss->nr_cgrp_next, -- skel->bss->nr_cgrp_coll, -- skel->bss->nr_cgrp_empty); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_pair__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_pair.h a/tools/sched_ext/scx_example_pair.h -deleted file mode 100644 -index f60b824272f7..000000000000 ---- b/tools/sched_ext/scx_example_pair.h -+++ /dev/null -@@ -1,10 +0,0 @@ --#ifndef __SCX_EXAMPLE_PAIR_H --#define __SCX_EXAMPLE_PAIR_H -- --enum { -- MAX_CPUS = 4096, -- MAX_QUEUED = 4096, -- MAX_CGRPS = 4096, --}; -- --#endif /* __SCX_EXAMPLE_PAIR_H */ -diff --git b/tools/sched_ext/scx_example_qmap.bpf.c a/tools/sched_ext/scx_example_qmap.bpf.c -deleted file mode 100644 -index 579ab21ae403..000000000000 ---- b/tools/sched_ext/scx_example_qmap.bpf.c -+++ /dev/null -@@ -1,401 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple five-level FIFO queue scheduler. -- * -- * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets -- * assigned to one depending on its compound weight. Each CPU round robins -- * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from -- * queue0, 2 from queue1, 4 from queue2 and so on. -- * -- * This scheduler demonstrates: -- * -- * - BPF-side queueing using PIDs. -- * - Sleepable per-task storage allocation using ops.prep_enable(). -- * - Using ops.cpu_release() to handle a higher priority scheduling class taking -- * the CPU away. -- * - Core-sched support. -- * -- * This scheduler is primarily for demonstration and testing of sched_ext -- * features and unlikely to be useful for actual workloads. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" --#include -- --char _license[] SEC("license") = "GPL"; -- --const volatile u64 slice_ns = SCX_SLICE_DFL; --const volatile bool switch_partial; --const volatile u32 stall_user_nth; --const volatile u32 stall_kernel_nth; --const volatile u32 dsp_inf_loop_after; --const volatile s32 disallow_tgid; -- --u32 test_error_cnt; -- --struct user_exit_info uei; -- --struct qmap { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, 4096); -- __type(value, u32); --} queue0 SEC(".maps"), -- queue1 SEC(".maps"), -- queue2 SEC(".maps"), -- queue3 SEC(".maps"), -- queue4 SEC(".maps"); -- --struct { -- __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); -- __uint(max_entries, 5); -- __type(key, int); -- __array(values, struct qmap); --} queue_arr SEC(".maps") = { -- .values = { -- [0] = &queue0, -- [1] = &queue1, -- [2] = &queue2, -- [3] = &queue3, -- [4] = &queue4, -- }, --}; -- --/* -- * Per-queue sequence numbers to implement core-sched ordering. -- * -- * Tail seq is assigned to each queued task and incremented. Head seq tracks the -- * sequence number of the latest dispatched task. The distance between the a -- * task's seq and the associated queue's head seq is called the queue distance -- * and used when comparing two tasks for ordering. See qmap_core_sched_before(). -- */ --static u64 core_sched_head_seqs[5]; --static u64 core_sched_tail_seqs[5]; -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local_dsq */ -- u64 core_sched_seq; --}; -- --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --/* Per-cpu dispatch index and remaining count */ --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(max_entries, 2); -- __type(key, u32); -- __type(value, u64); --} dispatch_idx_cnt SEC(".maps"); -- --/* Statistics */ --unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued; --unsigned long nr_core_sched_execed; -- --s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- struct task_ctx *tctx; -- s32 cpu; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- return cpu; -- -- return prev_cpu; --} -- --static int weight_to_idx(u32 weight) --{ -- /* Coarsely map the compound weight to a FIFO. */ -- if (weight <= 25) -- return 0; -- else if (weight <= 50) -- return 1; -- else if (weight < 200) -- return 2; -- else if (weight < 400) -- return 3; -- else -- return 4; --} -- --void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) --{ -- static u32 user_cnt, kernel_cnt; -- struct task_ctx *tctx; -- u32 pid = p->pid; -- int idx = weight_to_idx(p->scx.weight); -- void *ring; -- -- if (p->flags & PF_KTHREAD) { -- if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth)) -- return; -- } else { -- if (stall_user_nth && !(++user_cnt % stall_user_nth)) -- return; -- } -- -- if (test_error_cnt && !--test_error_cnt) -- scx_bpf_error("test triggering error"); -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return; -- } -- -- /* -- * All enqueued tasks must have their core_sched_seq updated for correct -- * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in -- * qmap_ops.flags. -- */ -- tctx->core_sched_seq = core_sched_tail_seqs[idx]++; -- -- /* -- * If qmap_select_cpu() is telling us to or this is the last runnable -- * task on the CPU, enqueue locally. -- */ -- if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) { -- tctx->force_local = false; -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); -- return; -- } -- -- /* -- * If the task was re-enqueued due to the CPU being preempted by a -- * higher priority scheduling class, just re-enqueue the task directly -- * on the global DSQ. As we want another CPU to pick it up, find and -- * kick an idle CPU. -- */ -- if (enq_flags & SCX_ENQ_REENQ) { -- s32 cpu; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags); -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) -- scx_bpf_kick_cpu(cpu, 0); -- return; -- } -- -- ring = bpf_map_lookup_elem(&queue_arr, &idx); -- if (!ring) { -- scx_bpf_error("failed to find ring %d", idx); -- return; -- } -- -- /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */ -- if (bpf_map_push_elem(ring, &pid, 0)) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags); -- return; -- } -- -- __sync_fetch_and_add(&nr_enqueued, 1); --} -- --/* -- * The BPF queue map doesn't support removal and sched_ext can handle spurious -- * dispatches. qmap_dequeue() is only used to collect statistics. -- */ --void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags) --{ -- __sync_fetch_and_add(&nr_dequeued, 1); -- if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) -- __sync_fetch_and_add(&nr_core_sched_execed, 1); --} -- --static void update_core_sched_head_seq(struct task_struct *p) --{ -- struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- int idx = weight_to_idx(p->scx.weight); -- -- if (tctx) -- core_sched_head_seqs[idx] = tctx->core_sched_seq; -- else -- scx_bpf_error("task_ctx lookup failed"); --} -- --void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) --{ -- u32 zero = 0, one = 1; -- u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero); -- u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one); -- void *fifo; -- s32 pid; -- int i; -- -- if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) { -- struct task_struct *p; -- -- /* -- * PID 2 should be kthreadd which should mostly be idle and off -- * the scheduler. Let's keep dispatching it to force the kernel -- * to call this function over and over again. -- */ -- p = bpf_task_from_pid(2); -- if (p) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- if (!idx || !cnt) { -- scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt); -- return; -- } -- -- for (i = 0; i < 5; i++) { -- /* Advance the dispatch cursor and pick the fifo. */ -- if (!*cnt) { -- *idx = (*idx + 1) % 5; -- *cnt = 1 << *idx; -- } -- (*cnt)--; -- -- fifo = bpf_map_lookup_elem(&queue_arr, idx); -- if (!fifo) { -- scx_bpf_error("failed to find ring %llu", *idx); -- return; -- } -- -- /* Dispatch or advance. */ -- if (!bpf_map_pop_elem(fifo, &pid)) { -- struct task_struct *p; -- -- p = bpf_task_from_pid(pid); -- if (p) { -- update_core_sched_head_seq(p); -- __sync_fetch_and_add(&nr_dispatched, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); -- bpf_task_release(p); -- return; -- } -- } -- -- *cnt = 0; -- } --} -- --/* -- * The distance from the head of the queue scaled by the weight of the queue. -- * The lower the number, the older the task and the higher the priority. -- */ --static s64 task_qdist(struct task_struct *p) --{ -- int idx = weight_to_idx(p->scx.weight); -- struct task_ctx *tctx; -- s64 qdist; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("task_ctx lookup failed"); -- return 0; -- } -- -- qdist = tctx->core_sched_seq - core_sched_head_seqs[idx]; -- -- /* -- * As queue index increments, the priority doubles. The queue w/ index 3 -- * is dispatched twice more frequently than 2. Reflect the difference by -- * scaling qdists accordingly. Note that the shift amount needs to be -- * flipped depending on the sign to avoid flipping priority direction. -- */ -- if (qdist >= 0) -- return qdist << (4 - idx); -- else -- return qdist << idx; --} -- --/* -- * This is called to determine the task ordering when core-sched is picking -- * tasks to execute on SMT siblings and should encode about the same ordering as -- * the regular scheduling path. Use the priority-scaled distances from the head -- * of the queues to compare the two tasks which should be consistent with the -- * dispatch path behavior. -- */ --bool BPF_STRUCT_OPS(qmap_core_sched_before, -- struct task_struct *a, struct task_struct *b) --{ -- return task_qdist(a) > task_qdist(b); --} -- --void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) --{ -- u32 cnt; -- -- /* -- * Called when @cpu is taken by a higher priority scheduling class. This -- * makes @cpu no longer available for executing sched_ext tasks. As we -- * don't want the tasks in @cpu's local dsq to sit there until @cpu -- * becomes available again, re-enqueue them into the global dsq. See -- * %SCX_ENQ_REENQ handling in qmap_enqueue(). -- */ -- cnt = scx_bpf_reenqueue_local(); -- if (cnt) -- __sync_fetch_and_add(&nr_reenqueued, cnt); --} -- --s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (p->tgid == disallow_tgid) -- p->scx.disallow = true; -- -- /* -- * @p is new. Let's ensure that its task_ctx is available. We can sleep -- * in this function and the following will automatically use GFP_KERNEL. -- */ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(qmap_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops qmap_ops = { -- .select_cpu = (void *)qmap_select_cpu, -- .enqueue = (void *)qmap_enqueue, -- .dequeue = (void *)qmap_dequeue, -- .dispatch = (void *)qmap_dispatch, -- .core_sched_before = (void *)qmap_core_sched_before, -- .cpu_release = (void *)qmap_cpu_release, -- .prep_enable = (void *)qmap_prep_enable, -- .init = (void *)qmap_init, -- .exit = (void *)qmap_exit, -- .flags = SCX_OPS_ENQ_LAST, -- .timeout_ms = 5000U, -- .name = "qmap", --}; -diff --git b/tools/sched_ext/scx_example_qmap.c a/tools/sched_ext/scx_example_qmap.c -deleted file mode 100644 -index ccb4814ee61b..000000000000 ---- b/tools/sched_ext/scx_example_qmap.c -+++ /dev/null -@@ -1,107 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_qmap.skel.h" -- --const char help_fmt[] = --"A simple five-level FIFO queue sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n" --"\n" --" -s SLICE_US Override slice duration\n" --" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" --" -t COUNT Stall every COUNT'th user thread\n" --" -T COUNT Stall every COUNT'th kernel thread\n" --" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n" --" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int dummy) --{ -- exit_req = 1; --} -- --int main(int argc, char **argv) --{ -- struct scx_example_qmap *skel; -- struct bpf_link *link; -- int opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_qmap__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) { -- switch (opt) { -- case 's': -- skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; -- break; -- case 'e': -- skel->bss->test_error_cnt = strtoul(optarg, NULL, 0); -- break; -- case 't': -- skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0); -- break; -- case 'T': -- skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0); -- break; -- case 'l': -- skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0); -- break; -- case 'd': -- skel->rodata->disallow_tgid = strtol(optarg, NULL, 0); -- if (skel->rodata->disallow_tgid < 0) -- skel->rodata->disallow_tgid = getpid(); -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_qmap__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.qmap_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- long nr_enqueued = skel->bss->nr_enqueued; -- long nr_dispatched = skel->bss->nr_dispatched; -- -- printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n", -- nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, -- skel->bss->nr_reenqueued, skel->bss->nr_dequeued, -- skel->bss->nr_core_sched_execed); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_qmap__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_simple.bpf.c a/tools/sched_ext/scx_example_simple.bpf.c -deleted file mode 100644 -index 4bccca3e2047..000000000000 ---- b/tools/sched_ext/scx_example_simple.bpf.c -+++ /dev/null -@@ -1,128 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A simple scheduler. -- * -- * By default, it operates as a simple global weighted vtime scheduler and can -- * be switched to FIFO scheduling. It also demonstrates the following niceties. -- * -- * - Statistics tracking how many tasks are queued to local and global dsq's. -- * - Termination notification for userspace. -- * -- * While very simple, this scheduler should work reasonably well on CPUs with a -- * uniform L3 cache topology. While preemption is not implemented, the fact that -- * the scheduling queue is shared across all CPUs means that whatever is at the -- * front of the queue is likely to be executed fairly quickly given enough -- * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads -- * but comes with the usual problems with FIFO scheduling where saturating -- * threads can easily drown out interactive ones. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include "scx_common.bpf.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool fifo_sched; --const volatile bool switch_partial; -- --static u64 vtime_now; --struct user_exit_info uei; -- --struct { -- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); -- __uint(key_size, sizeof(u32)); -- __uint(value_size, sizeof(u64)); -- __uint(max_entries, 2); /* [local, global] */ --} stats SEC(".maps"); -- --static void stat_inc(u32 idx) --{ -- u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); -- if (cnt_p) -- (*cnt_p)++; --} -- --static inline bool vtime_before(u64 a, u64 b) --{ -- return (s64)(a - b) < 0; --} -- --void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) --{ -- /* -- * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that -- * running @p on its CPU directly shouldn't affect fairness. Just queue -- * it on the local FIFO. -- */ -- if (enq_flags & SCX_ENQ_LOCAL) { -- stat_inc(0); /* count local queueing */ -- scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); -- return; -- } -- -- stat_inc(1); /* count global queueing */ -- -- if (fifo_sched) { -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- u64 vtime = p->scx.dsq_vtime; -- -- /* -- * Limit the amount of budget that an idling task can accumulate -- * to one slice. -- */ -- if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) -- vtime = vtime_now - SCX_SLICE_DFL; -- -- scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime, -- enq_flags); -- } --} -- --void BPF_STRUCT_OPS(simple_running, struct task_struct *p) --{ -- if (fifo_sched) -- return; -- -- /* -- * Global vtime always progresses forward as tasks start executing. The -- * test and update can be performed concurrently from multiple CPUs and -- * thus racy. Any error should be contained and temporary. Let's just -- * live with it. -- */ -- if (vtime_before(vtime_now, p->scx.dsq_vtime)) -- vtime_now = p->scx.dsq_vtime; --} -- --void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) --{ -- if (fifo_sched) -- return; -- -- /* scale the execution time by the inverse of the weight and charge */ -- p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; --} -- --s32 BPF_STRUCT_OPS(simple_init) --{ -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops simple_ops = { -- .enqueue = (void *)simple_enqueue, -- .running = (void *)simple_running, -- .stopping = (void *)simple_stopping, -- .init = (void *)simple_init, -- .exit = (void *)simple_exit, -- .name = "simple", --}; -diff --git b/tools/sched_ext/scx_example_simple.c a/tools/sched_ext/scx_example_simple.c -deleted file mode 100644 -index 486b401f7c95..000000000000 ---- b/tools/sched_ext/scx_example_simple.c -+++ /dev/null -@@ -1,101 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include "user_exit_info.h" --#include "scx_example_simple.skel.h" -- --const char help_fmt[] = --"A simple sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-f] [-p]\n" --"\n" --" -f Use FIFO scheduling instead of weighted vtime scheduling\n" --" -p Switch only tasks on SCHED_EXT policy intead of all\n" --" -h Display this help and exit\n"; -- --static volatile int exit_req; -- --static void sigint_handler(int simple) --{ -- exit_req = 1; --} -- --static void read_stats(struct scx_example_simple *skel, u64 *stats) --{ -- int nr_cpus = libbpf_num_possible_cpus(); -- u64 cnts[2][nr_cpus]; -- u32 idx; -- -- memset(stats, 0, sizeof(stats[0]) * 2); -- -- for (idx = 0; idx < 2; idx++) { -- int ret, cpu; -- -- ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), -- &idx, cnts[idx]); -- if (ret < 0) -- continue; -- for (cpu = 0; cpu < nr_cpus; cpu++) -- stats[idx] += cnts[idx][cpu]; -- } --} -- --int main(int argc, char **argv) --{ -- struct scx_example_simple *skel; -- struct bpf_link *link; -- u32 opt; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- skel = scx_example_simple__open(); -- assert(skel); -- -- while ((opt = getopt(argc, argv, "fph")) != -1) { -- switch (opt) { -- case 'f': -- skel->rodata->fifo_sched = true; -- break; -- case 'p': -- skel->rodata->switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- return opt != 'h'; -- } -- } -- -- assert(!scx_example_simple__load(skel)); -- -- link = bpf_map__attach_struct_ops(skel->maps.simple_ops); -- assert(link); -- -- while (!exit_req && !uei_exited(&skel->bss->uei)) { -- u64 stats[2]; -- -- read_stats(skel, stats); -- printf("local=%lu global=%lu\n", stats[0], stats[1]); -- fflush(stdout); -- sleep(1); -- } -- -- bpf_link__destroy(link); -- uei_print(&skel->bss->uei); -- scx_example_simple__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_userland.bpf.c a/tools/sched_ext/scx_example_userland.bpf.c -deleted file mode 100644 -index a089bc6bbe86..000000000000 ---- b/tools/sched_ext/scx_example_userland.bpf.c -+++ /dev/null -@@ -1,269 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A minimal userland scheduler. -- * -- * In terms of scheduling, this provides two different types of behaviors: -- * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity. -- * All such tasks are direct-dispatched from the kernel, and are never -- * enqueued in user space. -- * 2. A primitive vruntime scheduler that is implemented in user space, for all -- * other tasks. -- * -- * Some parts of this example user space scheduler could be implemented more -- * efficiently using more complex and sophisticated data structures. For -- * example, rather than using BPF_MAP_TYPE_QUEUE's, -- * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between -- * user space and kernel space. Similarly, we use a simple vruntime-sorted list -- * in user space, but an rbtree could be used instead. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#include --#include "scx_common.bpf.h" --#include "scx_example_userland_common.h" -- --char _license[] SEC("license") = "GPL"; -- --const volatile bool switch_partial; --const volatile s32 usersched_pid; -- --/* !0 for veristat, set during init */ --const volatile u32 num_possible_cpus = 64; -- --/* Stats that are printed by user space. */ --u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues; -- --struct user_exit_info uei; -- --/* -- * Whether the user space scheduler needs to be scheduled due to a task being -- * enqueued in user space. -- */ --static bool usersched_needed; -- --/* -- * The map containing tasks that are enqueued in user space from the kernel. -- * -- * This map is drained by the user space scheduler. -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, struct scx_userland_enqueued_task); --} enqueued SEC(".maps"); -- --/* -- * The map containing tasks that are dispatched to the kernel from user space. -- * -- * Drained by the kernel in userland_dispatch(). -- */ --struct { -- __uint(type, BPF_MAP_TYPE_QUEUE); -- __uint(max_entries, USERLAND_MAX_TASKS); -- __type(value, s32); --} dispatched SEC(".maps"); -- --/* Per-task scheduling context */ --struct task_ctx { -- bool force_local; /* Dispatch directly to local DSQ */ --}; -- --/* Map that contains task-local storage. */ --struct { -- __uint(type, BPF_MAP_TYPE_TASK_STORAGE); -- __uint(map_flags, BPF_F_NO_PREALLOC); -- __type(key, int); -- __type(value, struct task_ctx); --} task_ctx_stor SEC(".maps"); -- --static bool is_usersched_task(const struct task_struct *p) --{ -- return p->pid == usersched_pid; --} -- --static bool keep_in_kernel(const struct task_struct *p) --{ -- return p->nr_cpus_allowed < num_possible_cpus; --} -- --static struct task_struct *usersched_task(void) --{ -- struct task_struct *p; -- -- p = bpf_task_from_pid(usersched_pid); -- /* -- * Should never happen -- the usersched task should always be managed -- * by sched_ext. -- */ -- if (!p) { -- scx_bpf_error("Failed to find usersched task %d", usersched_pid); -- /* -- * We should never hit this path, and we error out of the -- * scheduler above just in case, so the scheduler will soon be -- * be evicted regardless. So as to simplify the logic in the -- * caller to not have to check for NULL, return an acquired -- * reference to the current task here rather than NULL. -- */ -- return bpf_task_acquire(bpf_get_current_task_btf()); -- } -- -- return p; --} -- --s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p, -- s32 prev_cpu, u64 wake_flags) --{ -- if (keep_in_kernel(p)) { -- s32 cpu; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to look up task-local storage for %s", p->comm); -- return -ESRCH; -- } -- -- if (p->nr_cpus_allowed == 1 || -- scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { -- tctx->force_local = true; -- return prev_cpu; -- } -- -- cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); -- if (cpu >= 0) { -- tctx->force_local = true; -- return cpu; -- } -- } -- -- return prev_cpu; --} -- --static void dispatch_user_scheduler(void) --{ -- struct task_struct *p; -- -- usersched_needed = false; -- p = usersched_task(); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); --} -- --static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags) --{ -- struct scx_userland_enqueued_task task; -- -- memset(&task, 0, sizeof(task)); -- task.pid = p->pid; -- task.sum_exec_runtime = p->se.sum_exec_runtime; -- task.weight = p->scx.weight; -- -- if (bpf_map_push_elem(&enqueued, &task, 0)) { -- /* -- * If we fail to enqueue the task in user space, put it -- * directly on the global DSQ. -- */ -- __sync_fetch_and_add(&nr_failed_enqueues, 1); -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); -- } else { -- __sync_fetch_and_add(&nr_user_enqueues, 1); -- usersched_needed = true; -- } --} -- --void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags) --{ -- if (keep_in_kernel(p)) { -- u64 dsq_id = SCX_DSQ_GLOBAL; -- struct task_ctx *tctx; -- -- tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); -- if (!tctx) { -- scx_bpf_error("Failed to lookup task ctx for %s", p->comm); -- return; -- } -- -- if (tctx->force_local) -- dsq_id = SCX_DSQ_LOCAL; -- tctx->force_local = false; -- scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags); -- __sync_fetch_and_add(&nr_kernel_enqueues, 1); -- return; -- } else if (!is_usersched_task(p)) { -- enqueue_task_in_user_space(p, enq_flags); -- } --} -- --void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev) --{ -- if (usersched_needed) -- dispatch_user_scheduler(); -- -- bpf_repeat(4096) { -- s32 pid; -- struct task_struct *p; -- -- if (bpf_map_pop_elem(&dispatched, &pid)) -- break; -- -- /* -- * The task could have exited by the time we get around to -- * dispatching it. Treat this as a normal occurrence, and simply -- * move onto the next iteration. -- */ -- p = bpf_task_from_pid(pid); -- if (!p) -- continue; -- -- scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); -- bpf_task_release(p); -- } --} -- --s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p, -- struct scx_enable_args *args) --{ -- if (bpf_task_storage_get(&task_ctx_stor, p, 0, -- BPF_LOCAL_STORAGE_GET_F_CREATE)) -- return 0; -- else -- return -ENOMEM; --} -- --s32 BPF_STRUCT_OPS(userland_init) --{ -- if (num_possible_cpus == 0) { -- scx_bpf_error("User scheduler # CPUs uninitialized (%d)", -- num_possible_cpus); -- return -EINVAL; -- } -- -- if (usersched_pid <= 0) { -- scx_bpf_error("User scheduler pid uninitialized (%d)", -- usersched_pid); -- return -EINVAL; -- } -- -- if (!switch_partial) -- scx_bpf_switch_all(); -- return 0; --} -- --void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei) --{ -- uei_record(&uei, ei); --} -- --SEC(".struct_ops") --struct sched_ext_ops userland_ops = { -- .select_cpu = (void *)userland_select_cpu, -- .enqueue = (void *)userland_enqueue, -- .dispatch = (void *)userland_dispatch, -- .prep_enable = (void *)userland_prep_enable, -- .init = (void *)userland_init, -- .exit = (void *)userland_exit, -- .timeout_ms = 3000, -- .name = "userland", --}; -diff --git b/tools/sched_ext/scx_example_userland.c a/tools/sched_ext/scx_example_userland.c -deleted file mode 100644 -index 4152b1e65fe1..000000000000 ---- b/tools/sched_ext/scx_example_userland.c -+++ /dev/null -@@ -1,402 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * A demo sched_ext user space scheduler which provides vruntime semantics -- * using a simple ordered-list implementation. -- * -- * Each CPU in the system resides in a single, global domain. This precludes -- * the need to do any load balancing between domains. The scheduler could -- * easily be extended to support multiple domains, with load balancing -- * happening in user space. -- * -- * Any task which has any CPU affinity is scheduled entirely in BPF. This -- * program only schedules tasks which may run on any CPU. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#define _GNU_SOURCE --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include "user_exit_info.h" --#include "scx_example_userland_common.h" --#include "scx_example_userland.skel.h" -- --const char help_fmt[] = --"A minimal userland sched_ext scheduler.\n" --"\n" --"See the top-level comment in .bpf.c for more details.\n" --"\n" --"Usage: %s [-b BATCH] [-p]\n" --"\n" --" -b BATCH The number of tasks to batch when dispatching (default: 8)\n" --" -p Don't switch all, switch only tasks on SCHED_EXT policy\n" --" -h Display this help and exit\n"; -- --/* Defined in UAPI */ --#define SCHED_EXT 7 -- --/* Number of tasks to batch when dispatching to user space. */ --static __u32 batch_size = 8; -- --static volatile int exit_req; --static int enqueued_fd, dispatched_fd; -- --static struct scx_example_userland *skel; --static struct bpf_link *ops_link; -- --/* Stats collected in user space. */ --static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches; -- --/* The data structure containing tasks that are enqueued in user space. */ --struct enqueued_task { -- LIST_ENTRY(enqueued_task) entries; -- __u64 sum_exec_runtime; -- double vruntime; --}; -- --/* -- * Use a vruntime-sorted list to store tasks. This could easily be extended to -- * a more optimal data structure, such as an rbtree as is done in CFS. We -- * currently elect to use a sorted list to simplify the example for -- * illustrative purposes. -- */ --LIST_HEAD(listhead, enqueued_task); -- --/* -- * A vruntime-sorted list of tasks. The head of the list contains the task with -- * the lowest vruntime. That is, the task that has the "highest" claim to be -- * scheduled. -- */ --static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head); -- --/* -- * The statically allocated array of tasks. We use a statically allocated list -- * here to avoid having to allocate on the enqueue path, which could cause a -- * deadlock. A more substantive user space scheduler could e.g. provide a hook -- * for newly enabled tasks that are passed to the scheduler from the -- * .prep_enable() callback to allows the scheduler to allocate on safe paths. -- */ --struct enqueued_task tasks[USERLAND_MAX_TASKS]; -- --static double min_vruntime; -- --static void sigint_handler(int userland) --{ -- exit_req = 1; --} -- --static __u32 task_pid(const struct enqueued_task *task) --{ -- return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task); --} -- --static int dispatch_task(s32 pid) --{ -- int err; -- -- err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d\n", pid); -- exit_req = 1; -- } else { -- nr_vruntime_dispatches++; -- } -- -- return err; --} -- --static struct enqueued_task *get_enqueued_task(__s32 pid) --{ -- if (pid >= USERLAND_MAX_TASKS) -- return NULL; -- -- return &tasks[pid]; --} -- --static double calc_vruntime_delta(__u64 weight, __u64 delta) --{ -- double weight_f = (double)weight / 100.0; -- double delta_f = (double)delta; -- -- return delta_f / weight_f; --} -- --static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task) --{ -- __u64 delta; -- -- delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime; -- -- enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta); -- if (min_vruntime > enqueued->vruntime) -- enqueued->vruntime = min_vruntime; -- enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime; --} -- --static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task) --{ -- struct enqueued_task *curr, *enqueued, *prev; -- -- curr = get_enqueued_task(bpf_task->pid); -- if (!curr) -- return ENOENT; -- -- update_enqueued(curr, bpf_task); -- nr_vruntime_enqueues++; -- -- /* -- * Enqueue the task in a vruntime-sorted list. A more optimal data -- * structure such as an rbtree could easily be used as well. We elect -- * to use a list here simply because it's less code, and thus the -- * example is less convoluted and better serves to illustrate what a -- * user space scheduler could look like. -- */ -- -- if (LIST_EMPTY(&vruntime_head)) { -- LIST_INSERT_HEAD(&vruntime_head, curr, entries); -- return 0; -- } -- -- LIST_FOREACH(enqueued, &vruntime_head, entries) { -- if (curr->vruntime <= enqueued->vruntime) { -- LIST_INSERT_BEFORE(enqueued, curr, entries); -- return 0; -- } -- prev = enqueued; -- } -- -- LIST_INSERT_AFTER(prev, curr, entries); -- -- return 0; --} -- --static void drain_enqueued_map(void) --{ -- while (1) { -- struct scx_userland_enqueued_task task; -- int err; -- -- if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) -- return; -- -- err = vruntime_enqueue(&task); -- if (err) { -- fprintf(stderr, "Failed to enqueue task %d: %s\n", -- task.pid, strerror(err)); -- exit_req = 1; -- return; -- } -- } --} -- --static void dispatch_batch(void) --{ -- __u32 i; -- -- for (i = 0; i < batch_size; i++) { -- struct enqueued_task *task; -- int err; -- __s32 pid; -- -- task = LIST_FIRST(&vruntime_head); -- if (!task) -- return; -- -- min_vruntime = task->vruntime; -- pid = task_pid(task); -- LIST_REMOVE(task, entries); -- err = dispatch_task(pid); -- if (err) { -- fprintf(stderr, "Failed to dispatch task %d in %u\n", -- pid, i); -- return; -- } -- } --} -- --static void *run_stats_printer(void *arg) --{ -- while (!exit_req) { -- __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; -- -- nr_failed_enqueues = skel->bss->nr_failed_enqueues; -- nr_kernel_enqueues = skel->bss->nr_kernel_enqueues; -- nr_user_enqueues = skel->bss->nr_user_enqueues; -- total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues; -- -- printf("o-----------------------o\n"); -- printf("| BPF ENQUEUES |\n"); -- printf("|-----------------------|\n"); -- printf("| kern: %10llu |\n", nr_kernel_enqueues); -- printf("| user: %10llu |\n", nr_user_enqueues); -- printf("| failed: %10llu |\n", nr_failed_enqueues); -- printf("| -------------------- |\n"); -- printf("| total: %10llu |\n", total); -- printf("| |\n"); -- printf("|-----------------------|\n"); -- printf("| VRUNTIME / USER |\n"); -- printf("|-----------------------|\n"); -- printf("| enq: %10llu |\n", nr_vruntime_enqueues); -- printf("| disp: %10llu |\n", nr_vruntime_dispatches); -- printf("o-----------------------o\n"); -- printf("\n\n"); -- sleep(1); -- } -- -- return NULL; --} -- --static int spawn_stats_thread(void) --{ -- pthread_t stats_printer; -- -- return pthread_create(&stats_printer, NULL, run_stats_printer, NULL); --} -- --static int bootstrap(int argc, char **argv) --{ -- int err; -- __u32 opt; -- struct sched_param sched_param = { -- .sched_priority = sched_get_priority_max(SCHED_EXT), -- }; -- bool switch_partial = false; -- -- signal(SIGINT, sigint_handler); -- signal(SIGTERM, sigint_handler); -- libbpf_set_strict_mode(LIBBPF_STRICT_ALL); -- -- /* -- * Enforce that the user scheduler task is managed by sched_ext. The -- * task eagerly drains the list of enqueued tasks in its main work -- * loop, and then yields the CPU. The BPF scheduler only schedules the -- * user space scheduler task when at least one other task in the system -- * needs to be scheduled. -- */ -- err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param); -- if (err) { -- fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err)); -- return err; -- } -- -- while ((opt = getopt(argc, argv, "b:ph")) != -1) { -- switch (opt) { -- case 'b': -- batch_size = strtoul(optarg, NULL, 0); -- break; -- case 'p': -- switch_partial = true; -- break; -- default: -- fprintf(stderr, help_fmt, basename(argv[0])); -- exit(opt != 'h'); -- } -- } -- -- /* -- * It's not always safe to allocate in a user space scheduler, as an -- * enqueued task could hold a lock that we require in order to be able -- * to allocate. -- */ -- err = mlockall(MCL_CURRENT | MCL_FUTURE); -- if (err) { -- fprintf(stderr, "Failed to prefault and lock address space: %s\n", -- strerror(err)); -- return err; -- } -- -- skel = scx_example_userland__open(); -- if (!skel) { -- fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno)); -- return errno; -- } -- skel->rodata->num_possible_cpus = libbpf_num_possible_cpus(); -- assert(skel->rodata->num_possible_cpus > 0); -- skel->rodata->usersched_pid = getpid(); -- assert(skel->rodata->usersched_pid > 0); -- skel->rodata->switch_partial = switch_partial; -- -- err = scx_example_userland__load(skel); -- if (err) { -- fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- enqueued_fd = bpf_map__fd(skel->maps.enqueued); -- dispatched_fd = bpf_map__fd(skel->maps.dispatched); -- assert(enqueued_fd > 0); -- assert(dispatched_fd > 0); -- -- err = spawn_stats_thread(); -- if (err) { -- fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err)); -- goto destroy_skel; -- } -- -- ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops); -- if (!ops_link) { -- fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno)); -- err = errno; -- goto destroy_skel; -- } -- -- return 0; -- --destroy_skel: -- scx_example_userland__destroy(skel); -- exit_req = 1; -- return err; --} -- --static void sched_main_loop(void) --{ -- while (!exit_req) { -- /* -- * Perform the following work in the main user space scheduler -- * loop: -- * -- * 1. Drain all tasks from the enqueued map, and enqueue them -- * to the vruntime sorted list. -- * -- * 2. Dispatch a batch of tasks from the vruntime sorted list -- * down to the kernel. -- * -- * 3. Yield the CPU back to the system. The BPF scheduler will -- * reschedule the user space scheduler once another task has -- * been enqueued to user space. -- */ -- drain_enqueued_map(); -- dispatch_batch(); -- sched_yield(); -- } --} -- --int main(int argc, char **argv) --{ -- int err; -- -- err = bootstrap(argc, argv); -- if (err) { -- fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err)); -- return err; -- } -- -- sched_main_loop(); -- -- exit_req = 1; -- bpf_link__destroy(ops_link); -- uei_print(&skel->bss->uei); -- scx_example_userland__destroy(skel); -- return 0; --} -diff --git b/tools/sched_ext/scx_example_userland_common.h a/tools/sched_ext/scx_example_userland_common.h -deleted file mode 100644 -index 639c6809c5ff..000000000000 ---- b/tools/sched_ext/scx_example_userland_common.h -+++ /dev/null -@@ -1,19 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* Copyright (c) 2022 Meta, Inc */ -- --#ifndef __SCX_USERLAND_COMMON_H --#define __SCX_USERLAND_COMMON_H -- --#define USERLAND_MAX_TASKS 8192 -- --/* -- * An instance of a task that has been enqueued by the kernel for consumption -- * by a user space global scheduler thread. -- */ --struct scx_userland_enqueued_task { -- __s32 pid; -- u64 sum_exec_runtime; -- u64 weight; --}; -- --#endif // __SCX_USERLAND_COMMON_H -diff --git b/tools/sched_ext/user_exit_info.h a/tools/sched_ext/user_exit_info.h -deleted file mode 100644 -index e701ef0e0b86..000000000000 ---- b/tools/sched_ext/user_exit_info.h -+++ /dev/null -@@ -1,50 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --/* -- * Define struct user_exit_info which is shared between BPF and userspace parts -- * to communicate exit status and other information. -- * -- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. -- * Copyright (c) 2022 Tejun Heo -- * Copyright (c) 2022 David Vernet -- */ --#ifndef __USER_EXIT_INFO_H --#define __USER_EXIT_INFO_H -- --struct user_exit_info { -- int type; -- char reason[128]; -- char msg[1024]; --}; -- --#ifdef __bpf__ -- --#include "vmlinux.h" --#include -- --static inline void uei_record(struct user_exit_info *uei, -- const struct scx_exit_info *ei) --{ -- bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason); -- bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg); -- /* use __sync to force memory barrier */ -- __sync_val_compare_and_swap(&uei->type, uei->type, ei->type); --} -- --#else /* !__bpf__ */ -- --static inline bool uei_exited(struct user_exit_info *uei) --{ -- /* use __sync to force memory barrier */ -- return __sync_val_compare_and_swap(&uei->type, -1, -1); --} -- --static inline void uei_print(const struct user_exit_info *uei) --{ -- fprintf(stderr, "EXIT: %s", uei->reason); -- if (uei->msg[0] != '\0') -- fprintf(stderr, " (%s)", uei->msg); -- fputs("\n", stderr); --} -- --#endif /* __bpf__ */ --#endif /* __USER_EXIT_INFO_H */